fix(pi): prove packaged workers end to end
This commit is contained in:
@@ -140,8 +140,10 @@ async function runProductProofSamples(artifact, projectRoot, samples) {
|
||||
|
||||
function summarizeManagedMilestone(reports, cold, milestone) {
|
||||
return summarizeMeasurements(reports.flatMap(({ extension }) => (
|
||||
extension.managedWorkerMilestones
|
||||
.filter((event) => event.cold === cold && event.milestone === milestone)
|
||||
extension.managedTurns
|
||||
.filter((turn) => turn.cold === cold)
|
||||
.flatMap((turn) => turn.milestones)
|
||||
.filter((event) => event.milestone === milestone)
|
||||
.map(({ durationMs }) => durationMs)
|
||||
)));
|
||||
}
|
||||
@@ -180,32 +182,29 @@ export async function runPiReleasePerformance(options) {
|
||||
options.projectRoot,
|
||||
options.samples,
|
||||
);
|
||||
const providerMetric = (temperature, field) => summarizeMeasurements(
|
||||
providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification[temperature][field],
|
||||
),
|
||||
);
|
||||
const managedMilestones = {
|
||||
definition: 'Actual createPiManagedWorkerOpener execution from final app.asar Main against final pi-runtime; cold creates a session, warm reopens it.',
|
||||
definition: 'Each sample is one Main-owned correlated final-ASAR turn against the final pi-runtime; cold creates a session and invokes a real packaged subagent, warm reopens the same session. No milestone is joined from an external probe.',
|
||||
cold: {
|
||||
workerQueueWaitMs: summarizeManagedMilestone(productProofs, true, 'worker.queue_wait'),
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, true, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, true, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, true, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, true, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('cold', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('cold', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('cold', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('cold', 'agentSettledSamplesMs'),
|
||||
promptAcceptedMs: summarizeManagedMilestone(productProofs, true, 'prompt.accepted'),
|
||||
agentStartMs: summarizeManagedMilestone(productProofs, true, 'agent.start'),
|
||||
providerFirstEventMs: summarizeManagedMilestone(productProofs, true, 'provider.first_event'),
|
||||
agentSettledMs: summarizeManagedMilestone(productProofs, true, 'agent.settled'),
|
||||
},
|
||||
warm: {
|
||||
workerQueueWaitMs: summarizeManagedMilestone(productProofs, false, 'worker.queue_wait'),
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, false, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, false, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, false, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, false, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('warm', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('warm', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('warm', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('warm', 'agentSettledSamplesMs'),
|
||||
promptAcceptedMs: summarizeManagedMilestone(productProofs, false, 'prompt.accepted'),
|
||||
agentStartMs: summarizeManagedMilestone(productProofs, false, 'agent.start'),
|
||||
providerFirstEventMs: summarizeManagedMilestone(productProofs, false, 'provider.first_event'),
|
||||
agentSettledMs: summarizeManagedMilestone(productProofs, false, 'agent.settled'),
|
||||
},
|
||||
};
|
||||
const pressureUiInteractiveMs = summarizeMeasurements(
|
||||
@@ -230,7 +229,7 @@ export async function runPiReleasePerformance(options) {
|
||||
const git = await gitEvidence(options.projectRoot);
|
||||
const scenarios = [
|
||||
{ id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['final-ASAR resources.ready/worker.spawn/rpc.ready/session.open', 'separate cold prompt.accepted/agent.start/provider.first_event/agent.settled', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['single correlated final-ASAR worker.queue_wait/resources.ready/worker.spawn/rpc.ready/session.open/prompt.accepted/agent.start/provider.first_event/agent.settled timeline per turn', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' },
|
||||
{ id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' },
|
||||
{ id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' },
|
||||
@@ -302,8 +301,11 @@ export async function runPiReleasePerformance(options) {
|
||||
budgets,
|
||||
result: 'pass',
|
||||
crossPlatformReleaseReady: false,
|
||||
releaseBlockers: [
|
||||
'macOS x64 and macOS arm64 PI-150 performance/final-artifact validation was skipped by explicit user direction',
|
||||
releaseBlockers: platform() === 'linux' && !release().toLowerCase().includes('microsoft')
|
||||
? []
|
||||
: ['Native non-WSL Linux desktop/compositor PI-150 evidence is not established by this report'],
|
||||
deferredToPi160: [
|
||||
'macOS x64 and macOS arm64 final release validation was skipped by explicit user direction and remains Not Pass',
|
||||
],
|
||||
};
|
||||
if (options.reportPath) {
|
||||
|
||||
@@ -56,7 +56,11 @@ function assertActivePressure(pressure) {
|
||||
const expected = {
|
||||
parentWorkers: 4,
|
||||
childWorkers: 4,
|
||||
parentProcesses: 4,
|
||||
childProcesses: 4,
|
||||
liveProcesses: 8,
|
||||
parentProviderRequests: 4,
|
||||
childProviderRequests: 4,
|
||||
processBudget: 8,
|
||||
childPermits: 4,
|
||||
dispatches: 4,
|
||||
@@ -65,7 +69,11 @@ function assertActivePressure(pressure) {
|
||||
const actual = {
|
||||
parentWorkers: pressure?.parentWorkers,
|
||||
childWorkers: pressure?.childWorkers,
|
||||
parentProcesses: pressure?.parentProcessIds?.length,
|
||||
childProcesses: pressure?.childProcessIds?.length,
|
||||
liveProcesses: pressure?.liveProcessIds?.length,
|
||||
parentProviderRequests: pressure?.providerRequests?.parent,
|
||||
childProviderRequests: pressure?.providerRequests?.child,
|
||||
processBudget: pressure?.processBudget?.active,
|
||||
childPermits: pressure?.childPermits?.active,
|
||||
dispatches: pressure?.dispatches?.active,
|
||||
@@ -80,7 +88,11 @@ function assertReleasedPressure(pressure) {
|
||||
const counts = [
|
||||
pressure?.parentWorkers,
|
||||
pressure?.childWorkers,
|
||||
pressure?.parentProcessIds?.length,
|
||||
pressure?.childProcessIds?.length,
|
||||
pressure?.liveProcessIds?.length,
|
||||
pressure?.providerRequests?.parent,
|
||||
pressure?.providerRequests?.child,
|
||||
pressure?.processBudget?.active,
|
||||
pressure?.processBudget?.waiting,
|
||||
pressure?.childPermits?.active,
|
||||
@@ -147,10 +159,16 @@ export async function runPackagedProductProof(options) {
|
||||
const extension = await evaluateProof(electronApplication, 'extension');
|
||||
assertPackagedMain(extension);
|
||||
if (extension.extension?.subagentStatus !== 'complete'
|
||||
|| extension.extension?.childToolNames?.length !== 0
|
||||
|| extension.extension?.childToolNames?.join(',') !== 'find,grep,ls,read'
|
||||
|| !extension.extension?.parentToolNames?.includes('subagent')
|
||||
|| extension.extension?.managedWorkerMilestones?.filter(({ cold }) => cold).length !== 4
|
||||
|| extension.extension?.managedWorkerMilestones?.filter(({ cold }) => !cold).length !== 4) {
|
||||
|| extension.extension?.parentProcessIds?.length < 2
|
||||
|| extension.extension?.childProcessIds?.length !== 1
|
||||
|| extension.extension?.providerRequests?.child !== 1
|
||||
|| extension.extension?.managedTurns?.length !== 2
|
||||
|| extension.extension.managedTurns.some(({ milestones }) => (
|
||||
milestones.map(({ milestone }) => milestone).join(',')
|
||||
!== 'worker.queue_wait,resources.ready,worker.spawn,rpc.ready,session.open,prompt.accepted,agent.start,provider.first_event,agent.settled'
|
||||
))) {
|
||||
throw new Error(`Final ASAR extension/subagent proof failed: ${JSON.stringify(extension.extension)}`);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { arch, platform } from 'node:os';
|
||||
import { arch, platform, release } from 'node:os';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
@@ -11,6 +12,23 @@ import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
||||
import { runProbe } from './probe-pi-runtime.mjs';
|
||||
import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs';
|
||||
|
||||
function runGit(projectRoot, args) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
execFile('git', args, { cwd: projectRoot, windowsHide: true }, (error, stdout) => {
|
||||
if (error) reject(error);
|
||||
else resolvePromise(stdout.trim());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function gitEvidence(projectRoot) {
|
||||
const [commit, status] = await Promise.all([
|
||||
runGit(projectRoot, ['rev-parse', 'HEAD']),
|
||||
runGit(projectRoot, ['status', '--porcelain']),
|
||||
]);
|
||||
return { commit, dirty: Boolean(status) };
|
||||
}
|
||||
|
||||
async function runExtensionSmoke(projectRoot, artifact) {
|
||||
return await runPackagedProductProof({
|
||||
projectRoot,
|
||||
@@ -21,6 +39,7 @@ async function runExtensionSmoke(projectRoot, artifact) {
|
||||
}
|
||||
|
||||
export async function runRealPiSmoke(options) {
|
||||
const git = await gitEvidence(options.projectRoot);
|
||||
const artifact = await verifyPiProductArtifact(options);
|
||||
const common = {
|
||||
samples: options.samples,
|
||||
@@ -41,7 +60,8 @@ export async function runRealPiSmoke(options) {
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
commit: process.env.MAKELORE_BUILD_COMMIT ?? null,
|
||||
commit: git.commit,
|
||||
git,
|
||||
platform: { platform: platform(), arch: arch() },
|
||||
scope: {
|
||||
finalProductArtifact: true,
|
||||
@@ -76,8 +96,11 @@ export async function runRealPiSmoke(options) {
|
||||
},
|
||||
result: 'pass',
|
||||
crossPlatformReleaseReady: false,
|
||||
releaseBlockers: [
|
||||
'macOS x64 and macOS arm64 PI-150 final-artifact/runtime/resource/performance validation was skipped by explicit user direction',
|
||||
releaseBlockers: platform() === 'linux' && !release().toLowerCase().includes('microsoft')
|
||||
? []
|
||||
: ['Native non-WSL Linux desktop/compositor PI-150 evidence is not established by this report'],
|
||||
deferredToPi160: [
|
||||
'macOS x64 and macOS arm64 final release validation was skipped by explicit user direction and remains Not Pass',
|
||||
],
|
||||
};
|
||||
if (Object.values(report.coverage).some((value) => value !== true)) {
|
||||
|
||||
Reference in New Issue
Block a user