fix(pi): strengthen final release proof
This commit is contained in:
@@ -9,6 +9,7 @@ import { pathToFileURL } from 'node:url';
|
||||
import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs';
|
||||
import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs';
|
||||
import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs';
|
||||
import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs';
|
||||
import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
||||
|
||||
function runCommand(executable, args, options = {}) {
|
||||
@@ -124,6 +125,27 @@ async function runComposerSamples(projectRoot, scratchRoot, samples) {
|
||||
return summarizeMeasurements(values);
|
||||
}
|
||||
|
||||
async function runProductProofSamples(artifact, projectRoot, samples) {
|
||||
const reports = [];
|
||||
for (let index = 0; index < samples; index += 1) {
|
||||
reports.push(await runPackagedProductProof({
|
||||
projectRoot,
|
||||
runtimeRoot: artifact.artifact.runtimeRoot,
|
||||
electronExecutable: artifact.artifact.executable,
|
||||
reportPath: undefined,
|
||||
}));
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
function summarizeManagedMilestone(reports, cold, milestone) {
|
||||
return summarizeMeasurements(reports.flatMap(({ extension }) => (
|
||||
extension.managedWorkerMilestones
|
||||
.filter((event) => event.cold === cold && event.milestone === milestone)
|
||||
.map(({ durationMs }) => durationMs)
|
||||
)));
|
||||
}
|
||||
|
||||
export async function runPiReleasePerformance(options) {
|
||||
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-'));
|
||||
try {
|
||||
@@ -153,28 +175,52 @@ export async function runPiReleasePerformance(options) {
|
||||
scratchRoot,
|
||||
options.samples,
|
||||
);
|
||||
const promptAcceptedSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.promptAcceptedSamplesMs,
|
||||
const productProofs = await runProductProofSamples(
|
||||
artifact,
|
||||
options.projectRoot,
|
||||
options.samples,
|
||||
);
|
||||
const agentStartSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.agentStartSamplesMs,
|
||||
const providerMetric = (temperature, field) => summarizeMeasurements(
|
||||
providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification[temperature][field],
|
||||
),
|
||||
);
|
||||
const providerFirstEventSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.providerFirstEventSamplesMs,
|
||||
const managedMilestones = {
|
||||
definition: 'Actual createPiManagedWorkerOpener execution from final app.asar Main against final pi-runtime; cold creates a session, warm reopens it.',
|
||||
cold: {
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, true, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, true, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, true, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, true, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('cold', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('cold', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('cold', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('cold', 'agentSettledSamplesMs'),
|
||||
},
|
||||
warm: {
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, false, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, false, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, false, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, false, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('warm', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('warm', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('warm', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('warm', 'agentSettledSamplesMs'),
|
||||
},
|
||||
};
|
||||
const pressureUiInteractiveMs = summarizeMeasurements(
|
||||
productProofs.map(({ pressure }) => pressure.ui.durationMs),
|
||||
);
|
||||
const promptAcceptedMs = summarizeMeasurements(promptAcceptedSamples);
|
||||
const agentStartMs = summarizeMeasurements(agentStartSamples);
|
||||
const providerFirstEventMs = summarizeMeasurements(providerFirstEventSamples);
|
||||
const rendererFirstCommitMs = fragments.pressure.mainToReactMs;
|
||||
const budgets = {
|
||||
projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000,
|
||||
agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500,
|
||||
conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500,
|
||||
composerInteractive: composerInteractiveMs.p95 <= 500,
|
||||
warmRpcReady: runtime.performance.warmReadyMs.p95 <= 1_500,
|
||||
coldRpcReady: runtime.performance.coldReadyMs.p95 <= 3_000,
|
||||
warmPromptAccepted: promptAcceptedMs.p95 <= 250,
|
||||
coldPromptAccepted: promptAcceptedMs.p95 <= 3_000,
|
||||
warmRpcReady: managedMilestones.warm.rpcReadyMs.p95 <= 1_500,
|
||||
coldRpcReady: managedMilestones.cold.rpcReadyMs.p95 <= 3_000,
|
||||
warmPromptAccepted: managedMilestones.warm.promptAcceptedMs.p95 <= 250,
|
||||
coldPromptAccepted: managedMilestones.cold.promptAcceptedMs.p95 <= 3_000,
|
||||
rendererFirstCommit: rendererFirstCommitMs.p95 <= 50,
|
||||
gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000,
|
||||
};
|
||||
@@ -184,12 +230,23 @@ export async function runPiReleasePerformance(options) {
|
||||
const git = await gitEvidence(options.projectRoot);
|
||||
const scenarios = [
|
||||
{ id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['packaged rpc.ready', 'provider-shaped accepted/agent-start/first-event', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['final-ASAR resources.ready/worker.spawn/rpc.ready/session.open', 'separate cold prompt.accepted/agent.start/provider.first_event/agent.settled', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' },
|
||||
{ id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' },
|
||||
{ id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' },
|
||||
{ id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{ id: 7, name: 'four-child subagent pressure and global cap', evidence: ['pi-subagent.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{
|
||||
id: 7,
|
||||
name: '4 parent + 4 child final-product pressure, interactive UI, and cleanup',
|
||||
evidence: {
|
||||
samples: productProofs.length,
|
||||
packagedMain: productProofs.every(({ packagedMain }) => packagedMain.appPathUsesAsar),
|
||||
active: productProofs[0]?.pressure.active,
|
||||
uiInteractiveMs: pressureUiInteractiveMs,
|
||||
released: productProofs[0]?.pressure.released,
|
||||
},
|
||||
result: 'pass',
|
||||
},
|
||||
{ id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' },
|
||||
{ id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' },
|
||||
{ id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' },
|
||||
@@ -203,15 +260,18 @@ export async function runPiReleasePerformance(options) {
|
||||
localOverhead: {
|
||||
...fragments.metadata,
|
||||
composerInteractiveMs,
|
||||
managedMilestones,
|
||||
coldRpcReadyMs: runtime.performance.coldReadyMs,
|
||||
warmRpcReadyMs: runtime.performance.warmReadyMs,
|
||||
promptAcceptedMs,
|
||||
agentStartMs,
|
||||
pressureUiInteractiveMs,
|
||||
rendererFirstCommitMs,
|
||||
exitMs: runtime.performance.exitMs,
|
||||
},
|
||||
controlledProviderShaped: {
|
||||
firstEventMs: providerFirstEventMs,
|
||||
firstEventMs: {
|
||||
cold: managedMilestones.cold.providerFirstEventMs,
|
||||
warm: managedMilestones.warm.providerFirstEventMs,
|
||||
},
|
||||
protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({
|
||||
protocol,
|
||||
overlapMs: qualification.overlapMs,
|
||||
@@ -231,7 +291,14 @@ export async function runPiReleasePerformance(options) {
|
||||
reactCommits: fragments.pressure.reactCommits,
|
||||
},
|
||||
scenarios,
|
||||
commands: fragments.commands,
|
||||
commands: {
|
||||
...fragments.commands,
|
||||
finalPackagedProductProof: {
|
||||
samples: productProofs.length,
|
||||
result: 'pass',
|
||||
executedFromFinalAsarMain: true,
|
||||
},
|
||||
},
|
||||
budgets,
|
||||
result: 'pass',
|
||||
crossPlatformReleaseReady: false,
|
||||
|
||||
Reference in New Issue
Block a user