#!/usr/bin/env node import { spawn } from 'node:child_process'; import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { arch, platform, release, tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs'; import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs'; import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs'; import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs'; import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs'; function runCommand(executable, args, options = {}) { return new Promise((resolvePromise, reject) => { const child = spawn(executable, args, { cwd: options.cwd, env: options.env ?? process.env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, }); let stdout = ''; let stderr = ''; child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); child.once('error', reject); child.once('exit', (code, signal) => { if (code === 0) resolvePromise({ stdout, stderr }); else reject(new Error( `${executable} ${args.join(' ')} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}:\n${stderr || stdout}`, )); }); }); } async function readJson(path) { return JSON.parse(await readFile(path, 'utf8')); } async function readJsonLines(path) { return (await readFile(path, 'utf8')) .split(/\r?\n/) .filter(Boolean) .map((line) => JSON.parse(line)); } function everyBudgetPass(budgets) { return Object.values(budgets).every((value) => value === true); } async function gitEvidence(projectRoot) { const [{ stdout: commit }, { stdout: status }] = await Promise.all([ runCommand('git', ['rev-parse', 'HEAD'], { cwd: projectRoot }), runCommand('git', ['status', '--porcelain'], { cwd: projectRoot }), ]); return { commit: commit.trim(), dirty: Boolean(status.trim()) }; } async function runLocalPerformanceFragments(projectRoot, scratchRoot, samples) { const vitestCli = resolve(projectRoot, 'node_modules', 'vitest', 'vitest.mjs'); const metadataPath = join(scratchRoot, 'metadata.json'); const pressurePath = join(scratchRoot, 'pressure.json'); const focusedFiles = [ 'tests/unit/pi-release-performance-fragments.test.ts', 'tests/unit/coding-chat-pressure.test.tsx', ]; const focused = await runCommand(process.execPath, [ vitestCli, 'run', ...focusedFiles, '--maxWorkers=1', ], { cwd: projectRoot, env: { ...process.env, MAKELORE_PI_PERF_SAMPLES: String(samples), MAKELORE_PI_PERF_METADATA_FRAGMENT: metadataPath, MAKELORE_PI_PERF_PRESSURE_FRAGMENT: pressurePath, }, }); const scenarioFiles = [ 'tests/unit/pi-worker-pool.test.ts', 'tests/unit/pi-subagent.test.ts', 'tests/unit/pi-write-lease.test.ts', 'tests/unit/pi-session-projector.test.ts', 'tests/unit/pi-event-projector.test.ts', 'tests/unit/coding-attachments-routes.test.ts', ]; const scenarios = await runCommand(process.execPath, [ vitestCli, 'run', ...scenarioFiles, '--maxWorkers=1', ], { cwd: projectRoot }); return { metadata: await readJson(metadataPath), pressure: await readJson(pressurePath), commands: { focused: { files: focusedFiles, result: 'pass', outputBytes: focused.stdout.length }, scenarios: { files: scenarioFiles, result: 'pass', outputBytes: scenarios.stdout.length }, }, }; } async function runComposerSamples(projectRoot, scratchRoot, samples) { const reportPath = join(scratchRoot, 'composer.jsonl'); const playwrightCli = resolve(projectRoot, 'node_modules', '@playwright', 'test', 'cli.js'); await runCommand(process.execPath, [ playwrightCli, 'test', 'tests/e2e/pi-coding-first-chat.spec.ts', '--grep=first PI Conversation', `--repeat-each=${samples}`, '--workers=1', ], { cwd: projectRoot, env: { ...process.env, MAKELORE_PI_PERF_COMPOSER_FRAGMENT: reportPath }, }); const values = (await readJsonLines(reportPath)).map(({ editableMs }) => editableMs); if (values.length !== samples) { throw new Error(`Expected ${samples} Composer samples, got ${values.length}`); } return summarizeMeasurements(values); } async function runProductProofSamples(artifact, projectRoot, samples) { const reports = []; for (let index = 0; index < samples; index += 1) { reports.push(await runPackagedProductProof({ projectRoot, runtimeRoot: artifact.artifact.runtimeRoot, electronExecutable: artifact.artifact.executable, reportPath: undefined, verifyCleanupFailure: false, })); } return reports; } function summarizeManagedMilestone(reports, cold, milestone) { return summarizeMeasurements(reports.flatMap(({ extension }) => ( extension.managedTurns .filter((turn) => turn.cold === cold) .flatMap((turn) => turn.milestones) .filter((event) => event.milestone === milestone) .map(({ durationMs }) => durationMs) ))); } export async function runPiReleasePerformance(options) { const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-')); try { const artifact = await verifyPiProductArtifact(options); const common = { samples: options.samples, timeoutMs: options.timeoutMs, stage: false, keepStage: false, reportPath: undefined, providerFixturePath: undefined, imagePath: undefined, electronExecutablePath: artifact.artifact.executable, cliPath: artifact.artifact.cliPath, artifactLabel: 'final-makelore-product-artifact', }; const runtime = await runProbe(common, options.projectRoot); if (runtime.result === 'fail') throw new Error('Packaged runtime performance probe failed'); const providerContracts = await runLocalProviderContracts(common, options.projectRoot); const fragments = await runLocalPerformanceFragments( options.projectRoot, scratchRoot, options.samples, ); const composerInteractiveMs = await runComposerSamples( options.projectRoot, scratchRoot, options.samples, ); const productProofs = await runProductProofSamples( artifact, options.projectRoot, options.samples, ); const managedMilestones = { definition: 'Each sample is one Main-owned correlated final-ASAR turn against the final pi-runtime; cold creates a session and invokes a real packaged subagent, warm reopens the same session. No milestone is joined from an external probe.', cold: { workerQueueWaitMs: summarizeManagedMilestone(productProofs, true, 'worker.queue_wait'), resourcesReadyMs: summarizeManagedMilestone(productProofs, true, 'resources.ready'), workerSpawnMs: summarizeManagedMilestone(productProofs, true, 'worker.spawn'), rpcReadyMs: summarizeManagedMilestone(productProofs, true, 'rpc.ready'), sessionOpenMs: summarizeManagedMilestone(productProofs, true, 'session.open'), promptAcceptedMs: summarizeManagedMilestone(productProofs, true, 'prompt.accepted'), agentStartMs: summarizeManagedMilestone(productProofs, true, 'agent.start'), providerFirstEventMs: summarizeManagedMilestone(productProofs, true, 'provider.first_event'), agentSettledMs: summarizeManagedMilestone(productProofs, true, 'agent.settled'), }, warm: { workerQueueWaitMs: summarizeManagedMilestone(productProofs, false, 'worker.queue_wait'), resourcesReadyMs: summarizeManagedMilestone(productProofs, false, 'resources.ready'), workerSpawnMs: summarizeManagedMilestone(productProofs, false, 'worker.spawn'), rpcReadyMs: summarizeManagedMilestone(productProofs, false, 'rpc.ready'), sessionOpenMs: summarizeManagedMilestone(productProofs, false, 'session.open'), promptAcceptedMs: summarizeManagedMilestone(productProofs, false, 'prompt.accepted'), agentStartMs: summarizeManagedMilestone(productProofs, false, 'agent.start'), providerFirstEventMs: summarizeManagedMilestone(productProofs, false, 'provider.first_event'), agentSettledMs: summarizeManagedMilestone(productProofs, false, 'agent.settled'), }, }; const pressureUiInteractiveMs = summarizeMeasurements( productProofs.map(({ pressure }) => pressure.ui.durationMs), ); const rendererFirstCommitMs = fragments.pressure.mainToReactMs; const budgets = { projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000, agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500, conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500, composerInteractive: composerInteractiveMs.p95 <= 500, warmRpcReady: managedMilestones.warm.rpcReadyMs.p95 <= 1_500, coldRpcReady: managedMilestones.cold.rpcReadyMs.p95 <= 3_000, warmPromptAccepted: managedMilestones.warm.promptAcceptedMs.p95 <= 250, coldPromptAccepted: managedMilestones.cold.promptAcceptedMs.p95 <= 3_000, rendererFirstCommit: rendererFirstCommitMs.p95 <= 50, gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000, }; if (!everyBudgetPass(budgets)) { throw new Error(`PI-150 performance budget failed: ${JSON.stringify(budgets)}`); } const git = await gitEvidence(options.projectRoot); const scenarios = [ { id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' }, { id: 2, name: 'first prompt milestone split', evidence: ['single correlated final-ASAR worker.queue_wait/resources.ready/worker.spawn/rpc.ready/session.open/prompt.accepted/agent.start/provider.first_event/agent.settled timeline per turn', 'Renderer commit'], result: 'pass' }, { id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' }, { id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' }, { id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' }, { id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' }, { id: 7, name: '4 parent threads in 1 Agent Server + 4 child process pressure, interactive UI, and cleanup', evidence: { samples: productProofs.length, packagedMain: productProofs.every(({ packagedMain }) => packagedMain.appPathUsesAsar), active: productProofs[0]?.pressure.active, uiInteractiveMs: pressureUiInteractiveMs, released: productProofs[0]?.pressure.released, }, result: 'pass', }, { id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' }, { id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' }, { id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' }, ]; const report = { schemaVersion: 1, generatedAt: new Date().toISOString(), git, platform: { platform: platform(), arch: arch(), release: release(), packaged: true }, samples: options.samples, localOverhead: { ...fragments.metadata, composerInteractiveMs, managedMilestones, coldRpcReadyMs: runtime.performance.coldReadyMs, warmRpcReadyMs: runtime.performance.warmReadyMs, pressureUiInteractiveMs, rendererFirstCommitMs, exitMs: runtime.performance.exitMs, }, controlledProviderShaped: { firstEventMs: { cold: managedMilestones.cold.providerFirstEventMs, warm: managedMilestones.warm.providerFirstEventMs, }, protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({ protocol, overlapMs: qualification.overlapMs, abortIsolation: qualification.abortIsolation.passed, imageRequests: qualification.imageInput, })), realTurnVerified: false, realProviderDecision: 'explicitly-waived-accepted-risk', }, rssKb: runtime.performance.rssKb, ipc: { runtimePatchItems: fragments.pressure.runtimePatchItems, patchBatches: fragments.pressure.patchBatches, sseFrames: fragments.pressure.sseFrames, wireBytes: fragments.pressure.wireBytes, rendererTransactions: fragments.pressure.rendererTransactions, reactCommits: fragments.pressure.reactCommits, }, scenarios, commands: { ...fragments.commands, finalPackagedProductProof: { samples: productProofs.length, result: 'pass', executedFromFinalAsarMain: true, }, }, budgets, result: 'pass', crossPlatformReleaseReady: false, releaseBlockers: platform() === 'linux' && !release().toLowerCase().includes('microsoft') ? [] : ['Native non-WSL Linux desktop/compositor PI-150 evidence is not established by this report'], deferredToPi160: [ 'macOS x64 and macOS arm64 final release validation was skipped by explicit user direction and remains Not Pass', ], }; if (options.reportPath) { await mkdir(dirname(options.reportPath), { recursive: true }); await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`); } return report; } finally { await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); } } async function main() { const options = parsePiArtifactVerifierArgs(process.argv.slice(2)); if (options.help) { process.stdout.write('Usage: node scripts/run-pi-release-performance.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n'); return; } const report = await runPiReleasePerformance(options); process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); } const isMain = process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url; if (isMain) { main().catch((error) => { process.stderr.write(`${error.stack ?? error.message}\n`); process.exitCode = 1; }); }