fix(pi): strengthen final release proof
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { readFile, readdir, stat } from 'node:fs/promises';
|
||||
import { createRequire } from 'node:module';
|
||||
import { arch as hostArch, platform as hostPlatform } from 'node:os';
|
||||
import {
|
||||
dirname,
|
||||
@@ -22,6 +23,8 @@ import {
|
||||
packagedResourcesDirectory,
|
||||
} from '../probe-pi-packaged-runtime.mjs';
|
||||
|
||||
const { listPackage } = createRequire(import.meta.url)('@electron/asar');
|
||||
|
||||
const PRODUCT_NAME = 'Makelore';
|
||||
const LINUX_EXECUTABLE_NAME = 'niancode';
|
||||
const EXTENSION_CONTRACT_MARKERS = Object.freeze([
|
||||
@@ -31,6 +34,7 @@ const EXTENSION_CONTRACT_MARKERS = Object.freeze([
|
||||
'MAKELORE_PI_BRIDGE_URL',
|
||||
]);
|
||||
const PI_AI_PROVIDER_PREFIX = 'pi-runtime/node_modules/@earendil-works/pi-ai/dist/providers/';
|
||||
const PI_AI_PROVIDER_ASAR_PREFIX = 'app.asar/node_modules/@earendil-works/pi-ai/dist/providers/';
|
||||
|
||||
async function pathExists(path) {
|
||||
try {
|
||||
@@ -179,11 +183,23 @@ export async function collectForbiddenResourcePaths(root, pattern = /opencode/i)
|
||||
}
|
||||
|
||||
export function classifyOpenCodeResourcePaths(paths) {
|
||||
const upstreamPiProvider = paths.filter((path) => path.startsWith(PI_AI_PROVIDER_PREFIX));
|
||||
const productOwned = paths.filter((path) => !path.startsWith(PI_AI_PROVIDER_PREFIX));
|
||||
const isUpstreamPiProvider = (path) => path.startsWith(PI_AI_PROVIDER_PREFIX)
|
||||
|| path.startsWith(PI_AI_PROVIDER_ASAR_PREFIX);
|
||||
const upstreamPiProvider = paths.filter(isUpstreamPiProvider);
|
||||
const productOwned = paths.filter((path) => !isUpstreamPiProvider(path));
|
||||
return { productOwned, upstreamPiProvider };
|
||||
}
|
||||
|
||||
export function collectForbiddenAsarPaths(appAsar, pattern = /opencode/i) {
|
||||
const paths = listPackage(appAsar, { isPack: false }).map((entry) => (
|
||||
`app.asar/${entry.replace(/^[\\/]+/, '').replaceAll('\\', '/')}`
|
||||
));
|
||||
return {
|
||||
entryCount: paths.length,
|
||||
matches: paths.filter((entry) => pattern.test(entry)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
async function filesContainingNeedles(root, needles) {
|
||||
const matches = [];
|
||||
const visit = async (path) => {
|
||||
@@ -283,9 +299,12 @@ export async function verifyPiProductArtifact({ projectRoot, executable }) {
|
||||
if (absoluteManifestValues.length > 0) {
|
||||
throw new Error(`Pi runtime manifest contains absolute paths: ${JSON.stringify(absoluteManifestValues)}`);
|
||||
}
|
||||
const openCodeResourcePaths = classifyOpenCodeResourcePaths(
|
||||
await collectForbiddenResourcePaths(resourcesDirectory),
|
||||
);
|
||||
const physicalOpenCodePaths = await collectForbiddenResourcePaths(resourcesDirectory);
|
||||
const asarOpenCodePaths = collectForbiddenAsarPaths(appAsar);
|
||||
const openCodeResourcePaths = classifyOpenCodeResourcePaths([
|
||||
...physicalOpenCodePaths,
|
||||
...asarOpenCodePaths.matches,
|
||||
]);
|
||||
if (openCodeResourcePaths.productOwned.length > 0) {
|
||||
throw new Error(
|
||||
`Product-owned resources contain OpenCode paths: ${openCodeResourcePaths.productOwned.join(', ')}`,
|
||||
@@ -355,6 +374,8 @@ export async function verifyPiProductArtifact({ projectRoot, executable }) {
|
||||
skills: actualSkills,
|
||||
openCodeResourcePaths: {
|
||||
...openCodeResourcePaths,
|
||||
physicalMatches: physicalOpenCodePaths,
|
||||
asar: asarOpenCodePaths,
|
||||
upstreamDecision: openCodeResourcePaths.upstreamPiProvider.length > 0
|
||||
? 'retained-required-files-from-exact-pinned-pi-production-package'
|
||||
: 'none',
|
||||
|
||||
@@ -205,7 +205,7 @@ function validateCapturedRequests(protocol, modelId, requests) {
|
||||
const authHeader = protocol === 'anthropic-messages' ? 'x-api-key' : 'authorization';
|
||||
const expectedAuth = protocol === 'anthropic-messages' ? LOCAL_API_KEY : `Bearer ${LOCAL_API_KEY}`;
|
||||
const problems = [];
|
||||
if (matching.length !== 4) problems.push(`expected 4 requests, got ${matching.length}`);
|
||||
if (matching.length !== 6) problems.push(`expected 6 requests, got ${matching.length}`);
|
||||
if (matching.some((request) => request.method !== 'POST')) problems.push('non-POST request');
|
||||
if (matching.some((request) => request.path !== expectedPath(protocol))) problems.push('unexpected endpoint path');
|
||||
if (matching.some((request) => request.headers[authHeader] !== expectedAuth)) problems.push('credential header mismatch');
|
||||
|
||||
@@ -82,6 +82,7 @@ export function parseProbeArgs(argv) {
|
||||
return value;
|
||||
};
|
||||
|
||||
if (argument === '--') continue;
|
||||
if (argument === '--samples') options.samples = parsePositiveInteger(next(), argument);
|
||||
else if (argument === '--timeout-ms') options.timeoutMs = parsePositiveInteger(next(), argument);
|
||||
else if (argument === '--stage') options.stage = true;
|
||||
@@ -422,7 +423,10 @@ export class PiRpcWorker {
|
||||
});
|
||||
|
||||
await new Promise((resolveSpawn, rejectSpawn) => {
|
||||
this.child.once('spawn', resolveSpawn);
|
||||
this.child.once('spawn', () => {
|
||||
this.spawnedAt = performance.now();
|
||||
resolveSpawn();
|
||||
});
|
||||
this.child.once('error', rejectSpawn);
|
||||
});
|
||||
return this;
|
||||
@@ -583,9 +587,11 @@ async function runReadySample(runtime, paths, timeoutMs) {
|
||||
await worker.start();
|
||||
const response = await worker.request({ type: 'get_state' });
|
||||
const readyMs = response.receivedAt - worker.startedAt;
|
||||
const workerSpawnMs = worker.spawnedAt - worker.startedAt;
|
||||
const rpcReadyMs = response.receivedAt - worker.spawnedAt;
|
||||
const rssKb = await getProcessRssKb(worker.child.pid);
|
||||
const stop = await worker.stop();
|
||||
return { readyMs, rssKb, stop, sessionId: response.data.sessionId };
|
||||
return { readyMs, workerSpawnMs, rpcReadyMs, rssKb, stop, sessionId: response.data.sessionId };
|
||||
} finally {
|
||||
await worker.stop().catch(() => undefined);
|
||||
}
|
||||
@@ -611,9 +617,15 @@ async function runPerformanceSamples(runtime, scratchRoot, sampleCount, timeoutM
|
||||
definition: {
|
||||
cold: 'fresh Pi config, session, and project directories per process',
|
||||
warm: 'shared primed Pi config directory with fresh session and project directories per process',
|
||||
workerSpawn: 'final product executable spawn event minus spawn request',
|
||||
rpcReady: 'first successful get_state response minus child spawn event',
|
||||
},
|
||||
coldReadyMs: summarizeMeasurements(cold.map((sample) => sample.readyMs)),
|
||||
warmReadyMs: summarizeMeasurements(warm.map((sample) => sample.readyMs)),
|
||||
coldWorkerSpawnMs: summarizeMeasurements(cold.map((sample) => sample.workerSpawnMs)),
|
||||
warmWorkerSpawnMs: summarizeMeasurements(warm.map((sample) => sample.workerSpawnMs)),
|
||||
coldRpcReadyMs: summarizeMeasurements(cold.map((sample) => sample.rpcReadyMs)),
|
||||
warmRpcReadyMs: summarizeMeasurements(warm.map((sample) => sample.rpcReadyMs)),
|
||||
rssKb: summarizeMeasurements([...cold, ...warm].flatMap((sample) => sample.rssKb == null ? [] : [sample.rssKb])),
|
||||
exitMs: summarizeMeasurements([...cold, ...warm].map((sample) => sample.stop.exitMs)),
|
||||
exitModes: [...new Set([...cold, ...warm].map((sample) => sample.stop.mode))],
|
||||
@@ -867,6 +879,7 @@ async function promptAndSettle(worker, message, images, timeoutMs) {
|
||||
providerFirstEventMs: firstProviderEvent
|
||||
? firstProviderEvent.receivedAt - agentStart.receivedAt
|
||||
: settled.receivedAt - agentStart.receivedAt,
|
||||
agentSettledMs: settled.receivedAt - accepted.receivedAt,
|
||||
startedAt: agentStart.receivedAt,
|
||||
settledAt: settled.receivedAt,
|
||||
stopReason: lastAssistantStopReason(messagesResponse),
|
||||
@@ -908,6 +921,20 @@ export async function runProviderQualification(runtime, scratchRoot, fixturePath
|
||||
throw new Error('Provider workers shared a session id');
|
||||
}
|
||||
|
||||
const [leftWarmTurn, rightWarmTurn] = await Promise.all([
|
||||
promptAndSettle(left, 'Reply with exactly PI_PROVIDER_LEFT_WARM_OK. Do not use tools.', undefined, timeoutMs),
|
||||
promptAndSettle(right, 'Reply with exactly PI_PROVIDER_RIGHT_WARM_OK. Do not use tools.', undefined, timeoutMs),
|
||||
]);
|
||||
if (!successfulReasons.has(leftWarmTurn.stopReason)
|
||||
|| !successfulReasons.has(rightWarmTurn.stopReason)) {
|
||||
throw new Error(
|
||||
`Warm provider turns did not succeed: left=${leftWarmTurn.stopReason}, right=${rightWarmTurn.stopReason}`,
|
||||
);
|
||||
}
|
||||
const warmOverlapMs = Math.min(leftWarmTurn.settledAt, rightWarmTurn.settledAt)
|
||||
- Math.max(leftWarmTurn.startedAt, rightWarmTurn.startedAt);
|
||||
if (warmOverlapMs <= 0) throw new Error('Warm provider turns did not overlap');
|
||||
|
||||
const abortStartIndex = left.events.length;
|
||||
const abortedPrompt = left.request({
|
||||
type: 'prompt',
|
||||
@@ -950,20 +977,49 @@ export async function runProviderQualification(runtime, scratchRoot, fixturePath
|
||||
apiKeyEnv: fixture.apiKeyEnv,
|
||||
imageInput: Boolean(imagePath),
|
||||
distinctSessionIds: true,
|
||||
cold: {
|
||||
promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round),
|
||||
promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]),
|
||||
agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round),
|
||||
agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]),
|
||||
providerFirstEventSamplesMs: [
|
||||
leftTurn.providerFirstEventMs,
|
||||
rightTurn.providerFirstEventMs,
|
||||
].map(Math.round),
|
||||
providerFirstEventMs: summarizeMeasurements([
|
||||
leftTurn.providerFirstEventMs,
|
||||
rightTurn.providerFirstEventMs,
|
||||
]),
|
||||
agentSettledSamplesMs: [leftTurn.agentSettledMs, rightTurn.agentSettledMs].map(Math.round),
|
||||
agentSettledMs: summarizeMeasurements([leftTurn.agentSettledMs, rightTurn.agentSettledMs]),
|
||||
},
|
||||
warm: {
|
||||
promptAcceptedSamplesMs: [leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs].map(Math.round),
|
||||
promptAcceptedMs: summarizeMeasurements([leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs]),
|
||||
agentStartSamplesMs: [leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs].map(Math.round),
|
||||
agentStartMs: summarizeMeasurements([leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs]),
|
||||
providerFirstEventSamplesMs: [
|
||||
leftWarmTurn.providerFirstEventMs,
|
||||
rightWarmTurn.providerFirstEventMs,
|
||||
].map(Math.round),
|
||||
providerFirstEventMs: summarizeMeasurements([
|
||||
leftWarmTurn.providerFirstEventMs,
|
||||
rightWarmTurn.providerFirstEventMs,
|
||||
]),
|
||||
agentSettledSamplesMs: [leftWarmTurn.agentSettledMs, rightWarmTurn.agentSettledMs].map(Math.round),
|
||||
agentSettledMs: summarizeMeasurements([
|
||||
leftWarmTurn.agentSettledMs,
|
||||
rightWarmTurn.agentSettledMs,
|
||||
]),
|
||||
},
|
||||
overlapMs: Math.round(overlapMs),
|
||||
promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round),
|
||||
promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]),
|
||||
agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round),
|
||||
agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]),
|
||||
providerFirstEventSamplesMs: [
|
||||
leftTurn.providerFirstEventMs,
|
||||
rightTurn.providerFirstEventMs,
|
||||
].map(Math.round),
|
||||
providerFirstEventMs: summarizeMeasurements([
|
||||
leftTurn.providerFirstEventMs,
|
||||
rightTurn.providerFirstEventMs,
|
||||
]),
|
||||
stopReasons: [leftTurn.stopReason, rightTurn.stopReason],
|
||||
warmOverlapMs: Math.round(warmOverlapMs),
|
||||
stopReasons: [
|
||||
leftTurn.stopReason,
|
||||
rightTurn.stopReason,
|
||||
leftWarmTurn.stopReason,
|
||||
rightWarmTurn.stopReason,
|
||||
],
|
||||
abortIsolation: {
|
||||
abortedStopReason,
|
||||
unaffectedStopReason: unaffected.stopReason,
|
||||
|
||||
@@ -9,6 +9,7 @@ import { pathToFileURL } from 'node:url';
|
||||
import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs';
|
||||
import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs';
|
||||
import { runProbe, summarizeMeasurements } from './probe-pi-runtime.mjs';
|
||||
import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs';
|
||||
import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
||||
|
||||
function runCommand(executable, args, options = {}) {
|
||||
@@ -124,6 +125,27 @@ async function runComposerSamples(projectRoot, scratchRoot, samples) {
|
||||
return summarizeMeasurements(values);
|
||||
}
|
||||
|
||||
async function runProductProofSamples(artifact, projectRoot, samples) {
|
||||
const reports = [];
|
||||
for (let index = 0; index < samples; index += 1) {
|
||||
reports.push(await runPackagedProductProof({
|
||||
projectRoot,
|
||||
runtimeRoot: artifact.artifact.runtimeRoot,
|
||||
electronExecutable: artifact.artifact.executable,
|
||||
reportPath: undefined,
|
||||
}));
|
||||
}
|
||||
return reports;
|
||||
}
|
||||
|
||||
function summarizeManagedMilestone(reports, cold, milestone) {
|
||||
return summarizeMeasurements(reports.flatMap(({ extension }) => (
|
||||
extension.managedWorkerMilestones
|
||||
.filter((event) => event.cold === cold && event.milestone === milestone)
|
||||
.map(({ durationMs }) => durationMs)
|
||||
)));
|
||||
}
|
||||
|
||||
export async function runPiReleasePerformance(options) {
|
||||
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-release-performance-'));
|
||||
try {
|
||||
@@ -153,28 +175,52 @@ export async function runPiReleasePerformance(options) {
|
||||
scratchRoot,
|
||||
options.samples,
|
||||
);
|
||||
const promptAcceptedSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.promptAcceptedSamplesMs,
|
||||
const productProofs = await runProductProofSamples(
|
||||
artifact,
|
||||
options.projectRoot,
|
||||
options.samples,
|
||||
);
|
||||
const agentStartSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.agentStartSamplesMs,
|
||||
const providerMetric = (temperature, field) => summarizeMeasurements(
|
||||
providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification[temperature][field],
|
||||
),
|
||||
);
|
||||
const providerFirstEventSamples = providerContracts.protocols.flatMap(
|
||||
({ qualification }) => qualification.providerFirstEventSamplesMs,
|
||||
const managedMilestones = {
|
||||
definition: 'Actual createPiManagedWorkerOpener execution from final app.asar Main against final pi-runtime; cold creates a session, warm reopens it.',
|
||||
cold: {
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, true, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, true, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, true, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, true, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('cold', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('cold', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('cold', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('cold', 'agentSettledSamplesMs'),
|
||||
},
|
||||
warm: {
|
||||
resourcesReadyMs: summarizeManagedMilestone(productProofs, false, 'resources.ready'),
|
||||
workerSpawnMs: summarizeManagedMilestone(productProofs, false, 'worker.spawn'),
|
||||
rpcReadyMs: summarizeManagedMilestone(productProofs, false, 'rpc.ready'),
|
||||
sessionOpenMs: summarizeManagedMilestone(productProofs, false, 'session.open'),
|
||||
promptAcceptedMs: providerMetric('warm', 'promptAcceptedSamplesMs'),
|
||||
agentStartMs: providerMetric('warm', 'agentStartSamplesMs'),
|
||||
providerFirstEventMs: providerMetric('warm', 'providerFirstEventSamplesMs'),
|
||||
agentSettledMs: providerMetric('warm', 'agentSettledSamplesMs'),
|
||||
},
|
||||
};
|
||||
const pressureUiInteractiveMs = summarizeMeasurements(
|
||||
productProofs.map(({ pressure }) => pressure.ui.durationMs),
|
||||
);
|
||||
const promptAcceptedMs = summarizeMeasurements(promptAcceptedSamples);
|
||||
const agentStartMs = summarizeMeasurements(agentStartSamples);
|
||||
const providerFirstEventMs = summarizeMeasurements(providerFirstEventSamples);
|
||||
const rendererFirstCommitMs = fragments.pressure.mainToReactMs;
|
||||
const budgets = {
|
||||
projectMetadata: fragments.metadata.projectMetadataMs.p95 <= 1_000,
|
||||
agentMetadata: fragments.metadata.agentMetadataMs.p95 <= 500,
|
||||
conversationMetadata: fragments.metadata.conversationMetadataMs.p95 <= 500,
|
||||
composerInteractive: composerInteractiveMs.p95 <= 500,
|
||||
warmRpcReady: runtime.performance.warmReadyMs.p95 <= 1_500,
|
||||
coldRpcReady: runtime.performance.coldReadyMs.p95 <= 3_000,
|
||||
warmPromptAccepted: promptAcceptedMs.p95 <= 250,
|
||||
coldPromptAccepted: promptAcceptedMs.p95 <= 3_000,
|
||||
warmRpcReady: managedMilestones.warm.rpcReadyMs.p95 <= 1_500,
|
||||
coldRpcReady: managedMilestones.cold.rpcReadyMs.p95 <= 3_000,
|
||||
warmPromptAccepted: managedMilestones.warm.promptAcceptedMs.p95 <= 250,
|
||||
coldPromptAccepted: managedMilestones.cold.promptAcceptedMs.p95 <= 3_000,
|
||||
rendererFirstCommit: rendererFirstCommitMs.p95 <= 50,
|
||||
gracefulShutdown: runtime.performance.exitMs.p95 <= 3_000,
|
||||
};
|
||||
@@ -184,12 +230,23 @@ export async function runPiReleasePerformance(options) {
|
||||
const git = await gitEvidence(options.projectRoot);
|
||||
const scenarios = [
|
||||
{ id: 1, name: 'fresh userData metadata and Composer', evidence: ['metadata fragments', 'Electron Composer samples'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['packaged rpc.ready', 'provider-shaped accepted/agent-start/first-event', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 2, name: 'first prompt milestone split', evidence: ['final-ASAR resources.ready/worker.spawn/rpc.ready/session.open', 'separate cold prompt.accepted/agent.start/provider.first_event/agent.settled', 'Renderer commit'], result: 'pass' },
|
||||
{ id: 3, name: 'warm Conversation restore', evidence: ['packaged session stop/reopen/get_entries', 'warm rpc.ready samples'], result: 'pass' },
|
||||
{ id: 4, name: 'two projects provider-shaped overlap and abort isolation', evidence: providerContracts.protocols.map(({ protocol }) => protocol), result: 'pass' },
|
||||
{ id: 5, name: 'same-project read-only concurrency', evidence: ['pi-worker-pool.test.ts', 'pi-subagent.test.ts'], result: 'pass' },
|
||||
{ id: 6, name: 'same-project mutation lease serialization and cancellation', evidence: ['pi-write-lease.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{ id: 7, name: 'four-child subagent pressure and global cap', evidence: ['pi-subagent.test.ts', 'pi-worker-pool.test.ts'], result: 'pass' },
|
||||
{
|
||||
id: 7,
|
||||
name: '4 parent + 4 child final-product pressure, interactive UI, and cleanup',
|
||||
evidence: {
|
||||
samples: productProofs.length,
|
||||
packagedMain: productProofs.every(({ packagedMain }) => packagedMain.appPathUsesAsar),
|
||||
active: productProofs[0]?.pressure.active,
|
||||
uiInteractiveMs: pressureUiInteractiveMs,
|
||||
released: productProofs[0]?.pressure.released,
|
||||
},
|
||||
result: 'pass',
|
||||
},
|
||||
{ id: 8, name: '100 KB mixed blocks and batch recovery', evidence: ['coding-chat-pressure.test.tsx', 'pi-session-projector.test.ts', 'pi-event-projector.test.ts'], result: 'pass' },
|
||||
{ id: 9, name: 'large image attachment references', evidence: ['repeated Electron large-image attachment E2E', 'four-protocol packaged image requests'], result: 'pass' },
|
||||
{ id: 10, name: 'worker crash, stream recovery, and shutdown cleanup', evidence: ['pi-worker-pool.test.ts', 'pi-session-projector.test.ts', 'packaged exit samples'], result: 'pass' },
|
||||
@@ -203,15 +260,18 @@ export async function runPiReleasePerformance(options) {
|
||||
localOverhead: {
|
||||
...fragments.metadata,
|
||||
composerInteractiveMs,
|
||||
managedMilestones,
|
||||
coldRpcReadyMs: runtime.performance.coldReadyMs,
|
||||
warmRpcReadyMs: runtime.performance.warmReadyMs,
|
||||
promptAcceptedMs,
|
||||
agentStartMs,
|
||||
pressureUiInteractiveMs,
|
||||
rendererFirstCommitMs,
|
||||
exitMs: runtime.performance.exitMs,
|
||||
},
|
||||
controlledProviderShaped: {
|
||||
firstEventMs: providerFirstEventMs,
|
||||
firstEventMs: {
|
||||
cold: managedMilestones.cold.providerFirstEventMs,
|
||||
warm: managedMilestones.warm.providerFirstEventMs,
|
||||
},
|
||||
protocols: providerContracts.protocols.map(({ protocol, qualification }) => ({
|
||||
protocol,
|
||||
overlapMs: qualification.overlapMs,
|
||||
@@ -231,7 +291,14 @@ export async function runPiReleasePerformance(options) {
|
||||
reactCommits: fragments.pressure.reactCommits,
|
||||
},
|
||||
scenarios,
|
||||
commands: fragments.commands,
|
||||
commands: {
|
||||
...fragments.commands,
|
||||
finalPackagedProductProof: {
|
||||
samples: productProofs.length,
|
||||
result: 'pass',
|
||||
executedFromFinalAsarMain: true,
|
||||
},
|
||||
},
|
||||
budgets,
|
||||
result: 'pass',
|
||||
crossPlatformReleaseReady: false,
|
||||
|
||||
@@ -1,103 +1,218 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { _electron as electron } from '@playwright/test';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { createServer } from 'node:net';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
import { bundlePiRuntime } from './bundle-pi-runtime.mjs';
|
||||
import { defaultPiBundleTarget } from './lib/pi-runtime-bundle.mjs';
|
||||
import { defaultProductExecutable } from './lib/pi-product-artifact.mjs';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { runtimeRoot: undefined, electronExecutable: undefined };
|
||||
function parseArgs(argv, projectRoot = process.cwd()) {
|
||||
const options = {
|
||||
projectRoot: resolve(projectRoot),
|
||||
runtimeRoot: undefined,
|
||||
electronExecutable: undefined,
|
||||
reportPath: undefined,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`);
|
||||
if (argument === '--runtime-root') options.runtimeRoot = resolve(value);
|
||||
else if (argument === '--electron-executable') options.electronExecutable = resolve(value);
|
||||
else if (argument === '--report') options.reportPath = resolve(value);
|
||||
else throw new Error(`Unknown argument: ${argument}`);
|
||||
index += 1;
|
||||
}
|
||||
if (Boolean(options.runtimeRoot) !== Boolean(options.electronExecutable)) {
|
||||
throw new Error('--runtime-root and --electron-executable must be provided together');
|
||||
}
|
||||
options.electronExecutable ??= defaultProductExecutable(options.projectRoot);
|
||||
return options;
|
||||
}
|
||||
|
||||
function runVitest(runtimeRoot, electronExecutable) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(process.execPath, [
|
||||
resolve('node_modules/vitest/vitest.mjs'),
|
||||
'run',
|
||||
'tests/unit/pi-worker-process-real.test.ts',
|
||||
], {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
MAKELORE_PI_STAGED_RUNTIME_ROOT: runtimeRoot,
|
||||
...(electronExecutable ? { MAKELORE_PI_ELECTRON_EXECUTABLE: electronExecutable } : {}),
|
||||
},
|
||||
stdio: 'inherit',
|
||||
windowsHide: true,
|
||||
});
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) resolvePromise();
|
||||
else {
|
||||
reject(new Error(
|
||||
`Packaged subagent smoke failed with code ${code ?? 'null'} signal ${signal ?? 'none'}`,
|
||||
));
|
||||
async function allocatePort() {
|
||||
return await new Promise((resolvePort, reject) => {
|
||||
const server = createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
server.close(() => reject(new Error('Failed to allocate PI proof Host API port')));
|
||||
return;
|
||||
}
|
||||
server.close((error) => error ? reject(error) : resolvePort(address.port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runElectronProductTools(electronExecutable) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(process.execPath, [
|
||||
resolve('scripts/run-electron-vitest.mjs'),
|
||||
'tests/unit/pi-extension-bundle.test.ts',
|
||||
], {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
...(electronExecutable ? {
|
||||
MAKELORE_ELECTRON_EXECUTABLE: electronExecutable,
|
||||
MAKELORE_PI_ELECTRON_EXECUTABLE: electronExecutable,
|
||||
} : {}),
|
||||
},
|
||||
stdio: 'inherit',
|
||||
windowsHide: true,
|
||||
});
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) resolvePromise();
|
||||
else {
|
||||
reject(new Error(
|
||||
`Packaged product-tools Electron smoke failed with code ${code ?? 'null'} signal ${signal ?? 'none'}`,
|
||||
));
|
||||
}
|
||||
});
|
||||
});
|
||||
function assertPackagedMain(result) {
|
||||
if (result?.packagedMain?.isPackaged !== true || result?.packagedMain?.appPathUsesAsar !== true) {
|
||||
throw new Error(`PI proof did not execute from packaged app.asar Main: ${JSON.stringify(result?.packagedMain)}`);
|
||||
}
|
||||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const outputRoot = options.runtimeRoot
|
||||
? null
|
||||
: await mkdtemp(join(tmpdir(), 'makelore-pi-subagent-package-'));
|
||||
try {
|
||||
let runtimeRoot = options.runtimeRoot;
|
||||
if (!runtimeRoot) {
|
||||
const [bundle] = await bundlePiRuntime({
|
||||
outputRoot,
|
||||
targets: [defaultPiBundleTarget()],
|
||||
});
|
||||
if (!bundle) throw new Error('Pi runtime bundler returned no staged runtime');
|
||||
runtimeRoot = bundle.destination;
|
||||
}
|
||||
await runVitest(runtimeRoot, options.electronExecutable);
|
||||
await runElectronProductTools(options.electronExecutable);
|
||||
} finally {
|
||||
if (outputRoot) {
|
||||
await rm(outputRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
||||
function assertActivePressure(pressure) {
|
||||
const expected = {
|
||||
parentWorkers: 4,
|
||||
childWorkers: 4,
|
||||
liveProcesses: 8,
|
||||
processBudget: 8,
|
||||
childPermits: 4,
|
||||
dispatches: 4,
|
||||
writeLeases: 4,
|
||||
};
|
||||
const actual = {
|
||||
parentWorkers: pressure?.parentWorkers,
|
||||
childWorkers: pressure?.childWorkers,
|
||||
liveProcesses: pressure?.liveProcessIds?.length,
|
||||
processBudget: pressure?.processBudget?.active,
|
||||
childPermits: pressure?.childPermits?.active,
|
||||
dispatches: pressure?.dispatches?.active,
|
||||
writeLeases: pressure?.writeLeases?.active,
|
||||
};
|
||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||
throw new Error(`PI pressure did not reach 4 parent + 4 child state: ${JSON.stringify(actual)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertReleasedPressure(pressure) {
|
||||
const counts = [
|
||||
pressure?.parentWorkers,
|
||||
pressure?.childWorkers,
|
||||
pressure?.liveProcessIds?.length,
|
||||
pressure?.processBudget?.active,
|
||||
pressure?.processBudget?.waiting,
|
||||
pressure?.childPermits?.active,
|
||||
pressure?.childPermits?.waiting,
|
||||
pressure?.dispatches?.active,
|
||||
pressure?.dispatches?.parents,
|
||||
pressure?.writeLeases?.active,
|
||||
pressure?.writeLeases?.waiting,
|
||||
];
|
||||
if (counts.some((value) => value !== 0)) {
|
||||
throw new Error(`PI pressure resources were not fully released: ${JSON.stringify(pressure)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function evaluateProof(electronApplication, action) {
|
||||
return await electronApplication.evaluate(async (_electron, requestedAction) => {
|
||||
const proof = globalThis.__niancodeRunPiReleaseProofE2E;
|
||||
if (typeof proof !== 'function') throw new Error('Packaged Main has no PI release proof entry');
|
||||
return await proof(requestedAction);
|
||||
}, action);
|
||||
}
|
||||
|
||||
async function closeApplication(electronApplication) {
|
||||
await Promise.race([
|
||||
electronApplication.close().catch(() => undefined),
|
||||
new Promise((resolveTimeout) => setTimeout(resolveTimeout, 5_000)),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function runPackagedProductProof(options) {
|
||||
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-final-product-proof-'));
|
||||
const homeDir = join(scratchRoot, 'home');
|
||||
const userDataDir = join(scratchRoot, 'user-data');
|
||||
await Promise.all([
|
||||
mkdir(join(homeDir, '.config'), { recursive: true }),
|
||||
mkdir(join(homeDir, 'AppData', 'Local'), { recursive: true }),
|
||||
mkdir(join(homeDir, 'AppData', 'Roaming'), { recursive: true }),
|
||||
mkdir(userDataDir, { recursive: true }),
|
||||
]);
|
||||
const hostApiPort = await allocatePort();
|
||||
let electronApplication;
|
||||
let pressureActive = false;
|
||||
try {
|
||||
electronApplication = await electron.launch({
|
||||
executablePath: options.electronExecutable,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
USERPROFILE: homeDir,
|
||||
APPDATA: join(homeDir, 'AppData', 'Roaming'),
|
||||
LOCALAPPDATA: join(homeDir, 'AppData', 'Local'),
|
||||
XDG_CONFIG_HOME: join(homeDir, '.config'),
|
||||
NIANCODE_E2E: '1',
|
||||
NIANCODE_E2E_SKIP_SETUP: '1',
|
||||
NIANCODE_USER_DATA_DIR: userDataDir,
|
||||
NIANCODE_PORT_NIANCODE_HOST_API: String(hostApiPort),
|
||||
...(process.platform === 'linux' ? { ELECTRON_DISABLE_SANDBOX: '1' } : {}),
|
||||
},
|
||||
timeout: 90_000,
|
||||
});
|
||||
const page = await electronApplication.firstWindow();
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
|
||||
const extension = await evaluateProof(electronApplication, 'extension');
|
||||
assertPackagedMain(extension);
|
||||
if (extension.extension?.subagentStatus !== 'complete'
|
||||
|| extension.extension?.childToolNames?.length !== 0
|
||||
|| !extension.extension?.parentToolNames?.includes('subagent')
|
||||
|| extension.extension?.managedWorkerMilestones?.filter(({ cold }) => cold).length !== 4
|
||||
|| extension.extension?.managedWorkerMilestones?.filter(({ cold }) => !cold).length !== 4) {
|
||||
throw new Error(`Final ASAR extension/subagent proof failed: ${JSON.stringify(extension.extension)}`);
|
||||
}
|
||||
|
||||
const pressureStart = await evaluateProof(electronApplication, 'pressure.start');
|
||||
pressureActive = true;
|
||||
assertPackagedMain(pressureStart);
|
||||
assertActivePressure(pressureStart.pressure);
|
||||
|
||||
const uiStartedAt = performance.now();
|
||||
await page.getByTestId('ai-module-option-programming').click();
|
||||
await page.getByTestId('main-layout').waitFor({ state: 'visible', timeout: 10_000 });
|
||||
const uiInteractiveMs = Math.round(performance.now() - uiStartedAt);
|
||||
|
||||
const pressureFinish = await evaluateProof(electronApplication, 'pressure.finish');
|
||||
pressureActive = false;
|
||||
assertPackagedMain(pressureFinish);
|
||||
assertReleasedPressure(pressureFinish.pressure);
|
||||
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
executable: options.electronExecutable,
|
||||
runtimeRoot: options.runtimeRoot ?? null,
|
||||
packagedMain: extension.packagedMain,
|
||||
extension: extension.extension,
|
||||
pressure: {
|
||||
processKind: 'final-product-executable-with-ELECTRON_RUN_AS_NODE',
|
||||
active: pressureStart.pressure,
|
||||
ui: {
|
||||
action: 'select Makelore Code and render main layout',
|
||||
interactive: true,
|
||||
durationMs: uiInteractiveMs,
|
||||
},
|
||||
released: pressureFinish.pressure,
|
||||
},
|
||||
result: 'pass',
|
||||
};
|
||||
if (options.reportPath) {
|
||||
await mkdir(dirname(options.reportPath), { recursive: true });
|
||||
await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
return report;
|
||||
} finally {
|
||||
if (electronApplication && pressureActive) {
|
||||
await evaluateProof(electronApplication, 'pressure.finish').catch(() => undefined);
|
||||
}
|
||||
if (electronApplication) await closeApplication(electronApplication);
|
||||
await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const report = await runPackagedProductProof(options);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
const isMain = process.argv[1]
|
||||
&& pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
|
||||
if (isMain) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { arch, platform } from 'node:os';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
@@ -10,25 +9,14 @@ import { verifyPiProductArtifact } from './lib/pi-product-artifact.mjs';
|
||||
import { runLocalProviderContracts } from './probe-pi-provider-contracts.mjs';
|
||||
import { parsePiArtifactVerifierArgs } from './verify-pi-product-artifact.mjs';
|
||||
import { runProbe } from './probe-pi-runtime.mjs';
|
||||
import { runPackagedProductProof } from './run-pi-subagent-packaged-smoke.mjs';
|
||||
|
||||
function runExtensionSmoke(projectRoot, artifact) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(process.execPath, [
|
||||
resolve(projectRoot, 'scripts', 'run-pi-subagent-packaged-smoke.mjs'),
|
||||
'--runtime-root', artifact.artifact.runtimeRoot,
|
||||
'--electron-executable', artifact.artifact.executable,
|
||||
], {
|
||||
cwd: projectRoot,
|
||||
stdio: 'inherit',
|
||||
windowsHide: true,
|
||||
});
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) resolvePromise({ result: 'pass' });
|
||||
else reject(new Error(
|
||||
`Final product extension/subagent smoke failed with code ${code ?? 'null'} signal ${signal ?? 'none'}`,
|
||||
));
|
||||
});
|
||||
async function runExtensionSmoke(projectRoot, artifact) {
|
||||
return await runPackagedProductProof({
|
||||
projectRoot,
|
||||
runtimeRoot: artifact.artifact.runtimeRoot,
|
||||
electronExecutable: artifact.artifact.executable,
|
||||
reportPath: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user