339 lines
14 KiB
JavaScript
339 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { spawn } from 'node:child_process';
|
|
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { arch, platform, release, tmpdir } from 'node:os';
|
|
import { dirname, join, resolve } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
|
|
import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs';
|
|
import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs';
|
|
import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs';
|
|
import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs';
|
|
import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
|
|
|
function runCommand(executable, args, options = {}) {
|
|
return new Promise((resolvePromise, reject) => {
|
|
const child = spawn(executable, args, {
|
|
cwd: options.cwd,
|
|
env: options.env ?? process.env,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
windowsHide: true,
|
|
});
|
|
let stdout = '';
|
|
let stderr = '';
|
|
child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
|
|
child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
|
|
child.once('error', reject);
|
|
child.once('exit', (code, signal) => {
|
|
if (code === 0) resolvePromise({ stdout, stderr });
|
|
else reject(new Error(
|
|
`${executable} ${args.join(' ')} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}:\n${stderr || stdout}`,
|
|
));
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readJson(path) {
|
|
return JSON.parse(await readFile(path, 'utf8'));
|
|
}
|
|
|
|
async function readJsonLines(path) {
|
|
return (await readFile(path, 'utf8'))
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
}
|
|
|
|
function everyBudgetPass(budgets) {
|
|
return Object.values(budgets).every((value) => value === true);
|
|
}
|
|
|
|
async function gitEvidence(projectRoot) {
|
|
const [{ stdout: commit }, { stdout: status }] = await Promise.all([
|
|
runCommand('git', ['rev-parse', 'HEAD'], { cwd: projectRoot }),
|
|
runCommand('git', ['status', '--porcelain'], { cwd: projectRoot }),
|
|
]);
|
|
return { commit: commit.trim(), dirty: Boolean(status.trim()) };
|
|
}
|
|
|
|
async function runLocalPerformanceFragments(projectRoot, scratchRoot, samples) {
|
|
const vitestCli = resolve(projectRoot, 'node_modules', 'vitest', 'vitest.mjs');
|
|
const metadataPath = join(scratchRoot, 'metadata.json');
|
|
const pressurePath = join(scratchRoot, 'pressure.json');
|
|
const focusedFiles = [
|
|
'tests/unit/pi-release-performance-fragments.test.ts',
|
|
'tests/unit/coding-chat-pressure.test.tsx',
|
|
];
|
|
const focused = await runCommand(process.execPath, [
|
|
vitestCli,
|
|
'run',
|
|
...focusedFiles,
|
|
'--maxWorkers=1',
|
|
], {
|
|
cwd: projectRoot,
|
|
env: {
|
|
...process.env,
|
|
MAKELORE_PI_PERF_SAMPLES: String(samples),
|
|
MAKELORE_PI_PERF_METADATA_FRAGMENT: metadataPath,
|
|
MAKELORE_PI_PERF_PRESSURE_FRAGMENT: pressurePath,
|
|
},
|
|
});
|
|
|
|
const scenarioFiles = [
|
|
'tests/unit/pi-worker-pool.test.ts',
|
|
'tests/unit/pi-subagent.test.ts',
|
|
'tests/unit/pi-write-lease.test.ts',
|
|
'tests/unit/pi-session-projector.test.ts',
|
|
'tests/unit/pi-event-projector.test.ts',
|
|
'tests/unit/coding-attachments-routes.test.ts',
|
|
];
|
|
const scenarios = await runCommand(process.execPath, [
|
|
vitestCli,
|
|
'run',
|
|
...scenarioFiles,
|
|
'--maxWorkers=1',
|
|
], { cwd: projectRoot });
|
|
return {
|
|
metadata: await readJson(metadataPath),
|
|
pressure: await readJson(pressurePath),
|
|
commands: {
|
|
focused: { files: focusedFiles, result: 'pass', outputBytes: focused.stdout.length },
|
|
scenarios: { files: scenarioFiles, result: 'pass', outputBytes: scenarios.stdout.length },
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runComposerSamples(projectRoot, scratchRoot, samples) {
|
|
const reportPath = join(scratchRoot, 'composer.jsonl');
|
|
const playwrightCli = resolve(projectRoot, 'node_modules', '@playwright', 'test', 'cli.js');
|
|
await runCommand(process.execPath, [
|
|
playwrightCli,
|
|
'test',
|
|
'tests/e2e/pi-coding-first-chat.spec.ts',
|
|
'--grep=first PI Conversation',
|
|
`--repeat-each=${samples}`,
|
|
'--workers=1',
|
|
], {
|
|
cwd: projectRoot,
|
|
env: { ...process.env, MAKELORE_PI_PERF_COMPOSER_FRAGMENT: reportPath },
|
|
});
|
|
const values = (await readJsonLines(reportPath)).map(({ editableMs }) => editableMs);
|
|
if (values.length !== samples) {
|
|
throw new Error(`Expected ${samples} Composer samples, got ${values.length}`);
|
|
}
|
|
return summarizeMeasurements(values);
|
|
}
|
|
|
|
async function runProductProofSamples(artifact, projectRoot, samples) {
|
|
const reports = [];
|
|
for (let index = 0; index < samples; index += 1) {
|
|
reports.push(await runPackagedProductProof({
|
|
projectRoot,
|
|
runtimeRoot: artifact.artifact.runtimeRoot,
|
|
electronExecutable: artifact.artifact.executable,
|
|
reportPath: undefined,
|
|
}));
|
|
}
|
|
return reports;
|
|
}
|
|
|
|
function summarizeManagedMilestone(reports, cold, milestone) {
|
|
return summarizeMeasurements(reports.flatMap(({ extension }) => (
|
|
extension.managedTurns
|
|
.filter((turn) => turn.cold === cold)
|
|
.flatMap((turn) => turn.milestones)
|
|
.filter((event) => event.milestone === milestone)
|
|
.map(({ durationMs }) => durationMs)
|
|
)));
|
|
}
|
|
|
|
export async function runPiReleasePerformance(options) {
|
|
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-'));
|
|
try {
|
|
const artifact = await verifyPiProductArtifact(options);
|
|
const common = {
|
|
samples: options.samples,
|
|
timeoutMs: options.timeoutMs,
|
|
stage: false,
|
|
keepStage: false,
|
|
reportPath: undefined,
|
|
providerFixturePath: undefined,
|
|
imagePath: undefined,
|
|
electronExecutablePath: artifact.artifact.executable,
|
|
cliPath: artifact.artifact.cliPath,
|
|
artifactLabel: 'final-makelore-product-artifact',
|
|
};
|
|
const runtime = await runProbe(common, options.projectRoot);
|
|
if (runtime.result === 'fail') throw new Error('Packaged runtime performance probe failed');
|
|
const providerContracts = await runLocalProviderContracts(common, options.projectRoot);
|
|
const fragments = await runLocalPerformanceFragments(
|
|
options.projectRoot,
|
|
scratchRoot,
|
|
options.samples,
|
|
);
|
|
const composerInteractiveMs = await runComposerSamples(
|
|
options.projectRoot,
|
|
scratchRoot,
|
|
options.samples,
|
|
);
|
|
const productProofs = await runProductProofSamples(
|
|
artifact,
|
|
options.projectRoot,
|
|
options.samples,
|
|
);
|
|
const managedMilestones = {
|
|
definition: 'Each sample is one Main-owned correlated final-ASAR turn against the final pi-runtime; cold creates a session and invokes a real packaged subagent, warm reopens the same session. No milestone is joined from an external probe.',
|
|
cold: {
|
|
workerQueueWaitMs: summarizeManagedMilestone(productProofs, true, 'worker.queue_wait'),
|
|
resourcesReadyMs: summarizeManagedMilestone(productProofs, true, 'resources.ready'),
|
|
workerSpawnMs: summarizeManagedMilestone(productProofs, true, 'worker.spawn'),
|
|
rpcReadyMs: summarizeManagedMilestone(productProofs, true, 'rpc.ready'),
|
|
sessionOpenMs: summarizeManagedMilestone(productProofs, true, 'session.open'),
|
|
promptAcceptedMs: summarizeManagedMilestone(productProofs, true, 'prompt.accepted'),
|
|
agentStartMs: summarizeManagedMilestone(productProofs, true, 'agent.start'),
|
|
providerFirstEventMs: summarizeManagedMilestone(productProofs, true, 'provider.first_event'),
|
|
agentSettledMs: summarizeManagedMilestone(productProofs, true, 'agent.settled'),
|
|
},
|
|
warm: {
|
|
workerQueueWaitMs: summarizeManagedMilestone(productProofs, false, 'worker.queue_wait'),
|
|
resourcesReadyMs: summarizeManagedMilestone(productProofs, false, 'resources.ready'),
|
|
workerSpawnMs: summarizeManagedMilestone(productProofs, false, 'worker.spawn'),
|
|
rpcReadyMs: summarizeManagedMilestone(productProofs, false, 'rpc.ready'),
|
|
sessionOpenMs: summarizeManagedMilestone(productProofs, false, 'session.open'),
|
|
promptAcceptedMs: summarizeManagedMilestone(productProofs, false, 'prompt.accepted'),
|
|
agentStartMs: summarizeManagedMilestone(productProofs, false, 'agent.start'),
|
|
providerFirstEventMs: summarizeManagedMilestone(productProofs, false, 'provider.first_event'),
|
|
agentSettledMs: summarizeManagedMilestone(productProofs, false, 'agent.settled'),
|
|
},
|
|
};
|
|
const pressureUiInteractiveMs = summarizeMeasurements(
|
|
productProofs.map(({ pressure }) => pressure.ui.durationMs),
|
|
);
|
|
const rendererFirstCommitMs = fragments.pressure.mainToReactMs;
|
|
const budgets = {
|
|
projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000,
|
|
agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500,
|
|
conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500,
|
|
composerInteractive: composerInteractiveMs.p95 <= 500,
|
|
warmRpcReady: managedMilestones.warm.rpcReadyMs.p95 <= 1_500,
|
|
coldRpcReady: managedMilestones.cold.rpcReadyMs.p95 <= 3_000,
|
|
warmPromptAccepted: managedMilestones.warm.promptAcceptedMs.p95 <= 250,
|
|
coldPromptAccepted: managedMilestones.cold.promptAcceptedMs.p95 <= 3_000,
|
|
rendererFirstCommit: rendererFirstCommitMs.p95 <= 50,
|
|
gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000,
|
|
};
|
|
if (!everyBudgetPass(budgets)) {
|
|
throw new Error(`PI-150 performance budget failed: ${JSON.stringify(budgets)}`);
|
|
}
|
|
const git = await gitEvidence(options.projectRoot);
|
|
const scenarios = [
|
|
{ id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' },
|
|
{ id: 2, name: 'first prompt milestone split', evidence: ['single correlated final-ASAR worker.queue_wait/resources.ready/worker.spawn/rpc.ready/session.open/prompt.accepted/agent.start/provider.first_event/agent.settled timeline per turn', 'Renderer commit'], result: 'pass' },
|
|
{ id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' },
|
|
{ id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' },
|
|
{ id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' },
|
|
{ id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
|
{
|
|
id: 7,
|
|
name: '4 parent + 4 child final-product pressure, interactive UI, and cleanup',
|
|
evidence: {
|
|
samples: productProofs.length,
|
|
packagedMain: productProofs.every(({ packagedMain }) => packagedMain.appPathUsesAsar),
|
|
active: productProofs[0]?.pressure.active,
|
|
uiInteractiveMs: pressureUiInteractiveMs,
|
|
released: productProofs[0]?.pressure.released,
|
|
},
|
|
result: 'pass',
|
|
},
|
|
{ id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' },
|
|
{ id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' },
|
|
{ id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' },
|
|
];
|
|
const report = {
|
|
schemaVersion: 1,
|
|
generatedAt: new Date().toISOString(),
|
|
git,
|
|
platform: { platform: platform(), arch: arch(), release: release(), packaged: true },
|
|
samples: options.samples,
|
|
localOverhead: {
|
|
...fragments.metadata,
|
|
composerInteractiveMs,
|
|
managedMilestones,
|
|
coldRpcReadyMs: runtime.performance.coldReadyMs,
|
|
warmRpcReadyMs: runtime.performance.warmReadyMs,
|
|
pressureUiInteractiveMs,
|
|
rendererFirstCommitMs,
|
|
exitMs: runtime.performance.exitMs,
|
|
},
|
|
controlledProviderShaped: {
|
|
firstEventMs: {
|
|
cold: managedMilestones.cold.providerFirstEventMs,
|
|
warm: managedMilestones.warm.providerFirstEventMs,
|
|
},
|
|
protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({
|
|
protocol,
|
|
overlapMs: qualification.overlapMs,
|
|
abortIsolation: qualification.abortIsolation.passed,
|
|
imageRequests: qualification.imageInput,
|
|
})),
|
|
realTurnVerified: false,
|
|
realProviderDecision: 'explicitly-waived-accepted-risk',
|
|
},
|
|
rssKb: runtime.performance.rssKb,
|
|
ipc: {
|
|
runtimePatchItems: fragments.pressure.runtimePatchItems,
|
|
patchBatches: fragments.pressure.patchBatches,
|
|
sseFrames: fragments.pressure.sseFrames,
|
|
wireBytes: fragments.pressure.wireBytes,
|
|
rendererTransactions: fragments.pressure.rendererTransactions,
|
|
reactCommits: fragments.pressure.reactCommits,
|
|
},
|
|
scenarios,
|
|
commands: {
|
|
...fragments.commands,
|
|
finalPackagedProductProof: {
|
|
samples: productProofs.length,
|
|
result: 'pass',
|
|
executedFromFinalAsarMain: true,
|
|
},
|
|
},
|
|
budgets,
|
|
result: 'pass',
|
|
crossPlatformReleaseReady: false,
|
|
releaseBlockers: platform() === 'linux' && !release().toLowerCase().includes('microsoft')
|
|
? []
|
|
: ['Native non-WSL Linux desktop/compositor PI-150 evidence is not established by this report'],
|
|
deferredToPi160: [
|
|
'macOS x64 and macOS arm64 final release validation was skipped by explicit user direction and remains Not Pass',
|
|
],
|
|
};
|
|
if (options.reportPath) {
|
|
await mkdir(dirname(options.reportPath), { recursive: true });
|
|
await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
return report;
|
|
} finally {
|
|
await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const options = parsePiArtifactVerifierArgs(process.argv.slice(2));
|
|
if (options.help) {
|
|
process.stdout.write('Usage: node scripts/run-pi-release-performance.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n');
|
|
return;
|
|
}
|
|
const report = await runPiReleasePerformance(options);
|
|
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
const isMain = process.argv[1]
|
|
&& pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
|
|
if (isMain) {
|
|
main().catch((error) => {
|
|
process.stderr.write(`${error.stack ?? error.message}\n`);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|