test: add PI release proof harness
This commit is contained in:
269
scripts/run-pi-release-performance.mjs
Normal file
269
scripts/run-pi-release-performance.mjs
Normal file
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { arch, platform, release, tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs';
|
||||
import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs';
|
||||
import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs';
|
||||
import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
||||
|
||||
function runCommand(executable, args, options = {}) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(executable, args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env ?? process.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
|
||||
child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) resolvePromise({ stdout, stderr });
|
||||
else reject(new Error(
|
||||
`${executable} ${args.join(' ')} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}:\n${stderr || stdout}`,
|
||||
));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function readJson(path) {
|
||||
return JSON.parse(await readFile(path, 'utf8'));
|
||||
}
|
||||
|
||||
async function readJsonLines(path) {
|
||||
return (await readFile(path, 'utf8'))
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line));
|
||||
}
|
||||
|
||||
function everyBudgetPass(budgets) {
|
||||
return Object.values(budgets).every((value) => value === true);
|
||||
}
|
||||
|
||||
async function gitEvidence(projectRoot) {
|
||||
const [{ stdout: commit }, { stdout: status }] = await Promise.all([
|
||||
runCommand('git', ['rev-parse', 'HEAD'], { cwd: projectRoot }),
|
||||
runCommand('git', ['status', '--porcelain'], { cwd: projectRoot }),
|
||||
]);
|
||||
return { commit: commit.trim(), dirty: Boolean(status.trim()) };
|
||||
}
|
||||
|
||||
async function runLocalPerformanceFragments(projectRoot, scratchRoot, samples) {
|
||||
const vitestCli = resolve(projectRoot, 'node_modules', 'vitest', 'vitest.mjs');
|
||||
const metadataPath = join(scratchRoot, 'metadata.json');
|
||||
const pressurePath = join(scratchRoot, 'pressure.json');
|
||||
const focusedFiles = [
|
||||
'tests/unit/pi-release-performance-fragments.test.ts',
|
||||
'tests/unit/coding-chat-pressure.test.tsx',
|
||||
];
|
||||
const focused = await runCommand(process.execPath, [
|
||||
vitestCli,
|
||||
'run',
|
||||
...focusedFiles,
|
||||
'--maxWorkers=1',
|
||||
], {
|
||||
cwd: projectRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
MAKELORE_PI_PERF_SAMPLES: String(samples),
|
||||
MAKELORE_PI_PERF_METADATA_FRAGMENT: metadataPath,
|
||||
MAKELORE_PI_PERF_PRESSURE_FRAGMENT: pressurePath,
|
||||
},
|
||||
});
|
||||
|
||||
const scenarioFiles = [
|
||||
'tests/unit/pi-worker-pool.test.ts',
|
||||
'tests/unit/pi-subagent.test.ts',
|
||||
'tests/unit/pi-write-lease.test.ts',
|
||||
'tests/unit/pi-session-projector.test.ts',
|
||||
'tests/unit/pi-event-projector.test.ts',
|
||||
'tests/unit/coding-attachments-routes.test.ts',
|
||||
];
|
||||
const scenarios = await runCommand(process.execPath, [
|
||||
vitestCli,
|
||||
'run',
|
||||
...scenarioFiles,
|
||||
'--maxWorkers=1',
|
||||
], { cwd: projectRoot });
|
||||
return {
|
||||
metadata: await readJson(metadataPath),
|
||||
pressure: await readJson(pressurePath),
|
||||
commands: {
|
||||
focused: { files: focusedFiles, result: 'pass', outputBytes: focused.stdout.length },
|
||||
scenarios: { files: scenarioFiles, result: 'pass', outputBytes: scenarios.stdout.length },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runComposerSamples(projectRoot, scratchRoot, samples) {
|
||||
const reportPath = join(scratchRoot, 'composer.jsonl');
|
||||
const playwrightCli = resolve(projectRoot, 'node_modules', '@playwright', 'test', 'cli.js');
|
||||
await runCommand(process.execPath, [
|
||||
playwrightCli,
|
||||
'test',
|
||||
'tests/e2e/pi-coding-first-chat.spec.ts',
|
||||
'--grep=first PI Conversation',
|
||||
`--repeat-each=${samples}`,
|
||||
'--workers=1',
|
||||
], {
|
||||
cwd: projectRoot,
|
||||
env: { ...process.env, MAKELORE_PI_PERF_COMPOSER_FRAGMENT: reportPath },
|
||||
});
|
||||
const values = (await readJsonLines(reportPath)).map(({ editableMs }) => editableMs);
|
||||
if (values.length !== samples) {
|
||||
throw new Error(`Expected ${samples} Composer samples, got ${values.length}`);
|
||||
}
|
||||
return summarizeMeasurements(values);
|
||||
}
|
||||
|
||||
export async function runPiReleasePerformance(options) {
|
||||
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-'));
|
||||
try {
|
||||
const artifact = await verifyPiProductArtifact(options);
|
||||
const common = {
|
||||
samples: options.samples,
|
||||
timeoutMs: options.timeoutMs,
|
||||
stage: false,
|
||||
keepStage: false,
|
||||
reportPath: undefined,
|
||||
providerFixturePath: undefined,
|
||||
imagePath: undefined,
|
||||
electronExecutablePath: artifact.artifact.executable,
|
||||
cliPath: artifact.artifact.cliPath,
|
||||
artifactLabel: 'final-makelore-product-artifact',
|
||||
};
|
||||
const runtime = await runProbe(common, options.projectRoot);
|
||||
if (runtime.result === 'fail') throw new Error('Packaged runtime performance probe failed');
|
||||
const providerContracts = await runLocalProviderContracts(common, options.projectRoot);
|
||||
const fragments = await runLocalPerformanceFragments(
|
||||
options.projectRoot,
|
||||
scratchRoot,
|
||||
options.samples,
|
||||
);
|
||||
const composerInteractiveMs = await runComposerSamples(
|
||||
options.projectRoot,
|
||||
scratchRoot,
|
||||
options.samples,
|
||||
);
|
||||
const promptAcceptedSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.promptAcceptedSamplesMs,
|
||||
);
|
||||
const agentStartSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.agentStartSamplesMs,
|
||||
);
|
||||
const providerFirstEventSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.providerFirstEventSamplesMs,
|
||||
);
|
||||
const promptAcceptedMs = summarizeMeasurements(promptAcceptedSamples);
|
||||
const agentStartMs = summarizeMeasurements(agentStartSamples);
|
||||
const providerFirstEventMs = summarizeMeasurements(providerFirstEventSamples);
|
||||
const rendererFirstCommitMs = fragments.pressure.mainToReactMs;
|
||||
const budgets = {
|
||||
projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000,
|
||||
agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500,
|
||||
conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500,
|
||||
composerInteractive: composerInteractiveMs.p95 <= 500,
|
||||
warmRpcReady: runtime.performance.warmReadyMs.p95 <= 1_500,
|
||||
coldRpcReady: runtime.performance.coldReadyMs.p95 <= 3_000,
|
||||
warmPromptAccepted: promptAcceptedMs.p95 <= 250,
|
||||
coldPromptAccepted: promptAcceptedMs.p95 <= 3_000,
|
||||
rendererFirstCommit: rendererFirstCommitMs.p95 <= 50,
|
||||
gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000,
|
||||
};
|
||||
if (!everyBudgetPass(budgets)) {
|
||||
throw new Error(`PI-150 performance budget failed: ${JSON.stringify(budgets)}`);
|
||||
}
|
||||
const git = await gitEvidence(options.projectRoot);
|
||||
const scenarios = [
|
||||
{ id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['packaged rpc.ready', 'provider-shaped accepted/agent-start/first-event', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' },
|
||||
{ id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' },
|
||||
{ id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' },
|
||||
{ id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{ id: 7, name: 'four-child subagent pressure and global cap', evidence: ['pi-subagent.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{ id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' },
|
||||
{ id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' },
|
||||
{ id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' },
|
||||
];
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
git,
|
||||
platform: { platform: platform(), arch: arch(), release: release(), packaged: true },
|
||||
samples: options.samples,
|
||||
localOverhead: {
|
||||
...fragments.metadata,
|
||||
composerInteractiveMs,
|
||||
coldRpcReadyMs: runtime.performance.coldReadyMs,
|
||||
warmRpcReadyMs: runtime.performance.warmReadyMs,
|
||||
promptAcceptedMs,
|
||||
agentStartMs,
|
||||
rendererFirstCommitMs,
|
||||
exitMs: runtime.performance.exitMs,
|
||||
},
|
||||
controlledProviderShaped: {
|
||||
firstEventMs: providerFirstEventMs,
|
||||
protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({
|
||||
protocol,
|
||||
overlapMs: qualification.overlapMs,
|
||||
abortIsolation: qualification.abortIsolation.passed,
|
||||
imageRequests: qualification.imageInput,
|
||||
})),
|
||||
realTurnVerified: false,
|
||||
realProviderDecision: 'explicitly-waived-accepted-risk',
|
||||
},
|
||||
rssKb: runtime.performance.rssKb,
|
||||
ipc: {
|
||||
runtimePatchItems: fragments.pressure.runtimePatchItems,
|
||||
patchBatches: fragments.pressure.patchBatches,
|
||||
sseFrames: fragments.pressure.sseFrames,
|
||||
wireBytes: fragments.pressure.wireBytes,
|
||||
rendererTransactions: fragments.pressure.rendererTransactions,
|
||||
reactCommits: fragments.pressure.reactCommits,
|
||||
},
|
||||
scenarios,
|
||||
commands: fragments.commands,
|
||||
budgets,
|
||||
result: 'pass',
|
||||
crossPlatformReleaseReady: false,
|
||||
releaseBlockers: [
|
||||
'macOS x64 and macOS arm64 PI-150 performance/final-artifact validation was skipped by explicit user direction',
|
||||
],
|
||||
};
|
||||
if (options.reportPath) {
|
||||
await mkdir(dirname(options.reportPath), { recursive: true });
|
||||
await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
return report;
|
||||
} finally {
|
||||
await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parsePiArtifactVerifierArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
process.stdout.write('Usage: node scripts/run-pi-release-performance.mjs [--app-exe path] [--samples count] [--timeout-ms ms] [--report path]\n');
|
||||
return;
|
||||
}
|
||||
const report = await runPiReleasePerformance(options);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
const isMain = process.argv[1]
|
||||
&& pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
|
||||
if (isMain) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user