#!/usr/bin/env node import { spawn } from 'node:child_process'; import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { arch, platform, release, tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs'; import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs'; import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs'; import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs'; function runCommand(executable, args, options = {}) { return new Promise((resolvePromise, reject) => { const child = spawn(executable, args, { cwd: options.cwd, env: options.env ?? process.env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, }); let stdout = ''; let stderr = ''; child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); child.once('error', reject); child.once('exit', (code, signal) => { if (code === 0) resolvePromise({ stdout, stderr }); else reject(new Error( `${executable} ${args.join(' ')} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}:\n${stderr || stdout}`, )); }); }); } async function readJson(path) { return JSON.parse(await readFile(path, 'utf8')); } async function readJsonLines(path) { return (await readFile(path, 'utf8')) .split(/\r?\n/) .filter(Boolean) .map((line) => JSON.parse(line)); } function everyBudgetPass(budgets) { return Object.values(budgets).every((value) => value === true); } async function gitEvidence(projectRoot) { const [{ stdout: commit }, { stdout: status }] = await Promise.all([ runCommand('git', ['rev-parse', 'HEAD'], { cwd: projectRoot }), runCommand('git', ['status', '--porcelain'], { cwd: projectRoot }), ]); return { commit: commit.trim(), dirty: Boolean(status.trim()) }; } async function runLocalPerformanceFragments(projectRoot, scratchRoot, samples) { const vitestCli = resolve(projectRoot, 'node_modules', 'vitest', 'vitest.mjs'); const metadataPath = join(scratchRoot, 'metadata.json'); const pressurePath = join(scratchRoot, 'pressure.json'); const focusedFiles = [ 'tests/unit/pi-release-performance-fragments.test.ts', 'tests/unit/coding-chat-pressure.test.tsx', ]; const focused = await runCommand(process.execPath, [ vitestCli, 'run', ...focusedFiles, '--maxWorkers=1', ], { cwd: projectRoot, env: { ...process.env, MAKELORE_PI_PERF_SAMPLES: String(samples), MAKELORE_PI_PERF_METADATA_FRAGMENT: metadataPath, MAKELORE_PI_PERF_PRESSURE_FRAGMENT: pressurePath, }, }); const scenarioFiles = [ 'tests/unit/pi-worker-pool.test.ts', 'tests/unit/pi-subagent.test.ts', 'tests/unit/pi-write-lease.test.ts', 'tests/unit/pi-session-projector.test.ts', 'tests/unit/pi-event-projector.test.ts', 'tests/unit/coding-attachments-routes.test.ts', ]; const scenarios = await runCommand(process.execPath, [ vitestCli, 'run', ...scenarioFiles, '--maxWorkers=1', ], { cwd: projectRoot }); return { metadata: await readJson(metadataPath), pressure: await readJson(pressurePath), commands: { focused: { files: focusedFiles, result: 'pass', outputBytes: focused.stdout.length }, scenarios: { files: scenarioFiles, result: 'pass', outputBytes: scenarios.stdout.length }, }, }; } async function runComposerSamples(projectRoot, scratchRoot, samples) { const reportPath = join(scratchRoot, 'composer.jsonl'); const playwrightCli = resolve(projectRoot, 'node_modules', '@playwright', 'test', 'cli.js'); await runCommand(process.execPath, [ playwrightCli, 'test', 'tests/e2e/pi-coding-first-chat.spec.ts', '--grep=first PI Conversation', `--repeat-each=${samples}`, '--workers=1', ], { cwd: projectRoot, env: { ...process.env, MAKELORE_PI_PERF_COMPOSER_FRAGMENT: reportPath }, }); const values = (await readJsonLines(reportPath)).map(({ editableMs }) => editableMs); if (values.length !== samples) { throw new Error(`Expected ${samples} Composer samples, got ${values.length}`); } return summarizeMeasurements(values); } export async function runPiReleasePerformance(options) { const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-')); try { const artifact = await verifyPiProductArtifact(options); const common = { samples: options.samples, timeoutMs: options.timeoutMs, stage: false, keepStage: false, reportPath: undefined, providerFixturePath: undefined, imagePath: undefined, electronExecutablePath: artifact.artifact.executable, cliPath: artifact.artifact.cliPath, artifactLabel: 'final-makelore-product-artifact', }; const runtime = await runProbe(common, options.projectRoot); if (runtime.result === 'fail') throw new Error('Packaged runtime performance probe failed'); const providerContracts = await runLocalProviderContracts(common, options.projectRoot); const fragments = await runLocalPerformanceFragments( options.projectRoot, scratchRoot, options.samples, ); const composerInteractiveMs = await runComposerSamples( options.projectRoot, scratchRoot, options.samples, ); const promptAcceptedSamples = providerContracts.protocols.flatMap( ({ qualification }) => qualification.promptAcceptedSamplesMs, ); const agentStartSamples = providerContracts.protocols.flatMap( ({ qualification }) => qualification.agentStartSamplesMs, ); const providerFirstEventSamples = providerContracts.protocols.flatMap( ({ qualification }) => qualification.providerFirstEventSamplesMs, ); const promptAcceptedMs = summarizeMeasurements(promptAcceptedSamples); const agentStartMs = summarizeMeasurements(agentStartSamples); const providerFirstEventMs = summarizeMeasurements(providerFirstEventSamples); const rendererFirstCommitMs = fragments.pressure.mainToReactMs; const budgets = { projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000, agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500, conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500, composerInteractive: composerInteractiveMs.p95 <= 500, warmRpcReady: runtime.performance.warmReadyMs.p95 <= 1_500, coldRpcReady: runtime.performance.coldReadyMs.p95 <= 3_000, warmPromptAccepted: promptAcceptedMs.p95 <= 250, coldPromptAccepted: promptAcceptedMs.p95 <= 3_000, rendererFirstCommit: rendererFirstCommitMs.p95 <= 50, gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000, }; if (!everyBudgetPass(budgets)) { throw new Error(`PI-150 performance budget failed: ${JSON.stringify(budgets)}`); } const git = await gitEvidence(options.projectRoot); const scenarios = [ { id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' }, { id: 2, name: 'first prompt milestone split', evidence: ['packaged rpc.ready', 'provider-shaped accepted/agent-start/first-event', 'Renderer commit'], result: 'pass' }, { id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' }, { id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' }, { id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' }, { id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' }, { id: 7, name: 'four-child subagent pressure and global cap', evidence: ['pi-subagent.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' }, { id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' }, { id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' }, { id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' }, ]; const report = { schemaVersion: 1, generatedAt: new Date().toISOString(), git, platform: { platform: platform(), arch: arch(), release: release(), packaged: true }, samples: options.samples, localOverhead: { ...fragments.metadata, composerInteractiveMs, coldRpcReadyMs: runtime.performance.coldReadyMs, warmRpcReadyMs: runtime.performance.warmReadyMs, promptAcceptedMs, agentStartMs, rendererFirstCommitMs, exitMs: runtime.performance.exitMs, }, controlledProviderShaped: { firstEventMs: providerFirstEventMs, protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({ protocol, overlapMs: qualification.overlapMs, abortIsolation: qualification.abortIsolation.passed, imageRequests: qualification.imageInput, })), realTurnVerified: false, realProviderDecision: 'explicitly-waived-accepted-risk', }, rssKb: runtime.performance.rssKb, ipc: { runtimePatchItems: fragments.pressure.runtimePatchItems, patchBatches: fragments.pressure.patchBatches, sseFrames: fragments.pressure.sseFrames, wireBytes: fragments.pressure.wireBytes, rendererTransactions: fragments.pressure.rendererTransactions, reactCommits: fragments.pressure.reactCommits, }, scenarios, commands: fragments.commands, budgets, result: 'pass', crossPlatformReleaseReady: false, releaseBlockers: [ 'macOS x64 and macOS arm64 PI-150 performance/final-artifact validation was skipped by explicit user direction', ], }; if (options.reportPath) { await mkdir(dirname(options.reportPath), { recursive: true }); await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`); } return report; } finally { await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); } } async function main() { const options = parsePiArtifactVerifierArgs(process.argv.slice(2)); if (options.help) { process.stdout.write('Usage: node scripts/run-pi-release-performance.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n'); return; } const report = await runPiReleasePerformance(options); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); } const isMain = process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url; if (isMain) { main().catch((error) => { process.stderr.write(`${error.stack ?? error.message}\n`); process.exitCode = 1; }); }