import { spawn } from 'node:child_process'; import { StringDecoder } from 'node:string_decoder'; import { cp, mkdtemp, mkdir, readFile, readdir, rm, stat, writeFile, } from 'node:fs/promises'; import { realpathSync } from 'node:fs'; import { createRequire } from 'node:module'; import { arch, platform, release, tmpdir } from 'node:os'; import { dirname, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { performance } from 'node:perf_hooks'; export const PI_RUNTIME_IDENTITY = Object.freeze({ packageName: '@earendil-works/pi-coding-agent', version: '0.84.2', tag: 'v0.84.2', gitHead: '914cf1472e715297caa30db4b9535d534a9eb718', nodeEngine: '>=22.19.0', }); export const MAKELore_PROVIDER_PROTOCOLS = Object.freeze([ 'openai-completions', 'openai-responses', 'anthropic-messages', 'openrouter', ]); export function buildMissingEvidence() { return [ 'Phase-0 artifact aggregation: Windows x64 and Linux x64; macOS x64/arm64 deferred to PI-150 by user decision on 2026-08-22', ]; } export function buildEvidenceWaivers() { return [ 'macOS x64/arm64 Phase-0 execution deferred to PI-150 by user decision on 2026-08-22; no macOS Pass is claimed', 'QG-004/QG-005 real Provider Account compatibility and real-provider concurrency/isolation explicitly waived by user decision on 2026-08-22; real authentication, endpoint/proxy/rate-limit/provider variation, protocol compatibility, concurrency, abort/event/session/model/credential isolation, and image behavior remain unverified', ]; } const DEFAULT_TIMEOUT_MS = 10_000; const DEFAULT_SAMPLE_COUNT = 5; const MAX_STDERR_LENGTH = 16_000; function parsePositiveInteger(value, flag) { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { throw new Error(`${flag} must be a positive integer`); } return parsed; } export function parseProbeArgs(argv) { const options = { samples: DEFAULT_SAMPLE_COUNT, timeoutMs: DEFAULT_TIMEOUT_MS, stage: false, keepStage: false, reportPath: undefined, providerFixturePath: undefined, imagePath: undefined, electronExecutablePath: undefined, cliPath: undefined, artifactLabel: undefined, }; for (let index = 0; index < argv.length; index += 1) { const argument = argv[index]; const next = () => { const value = argv[index + 1]; if (!value || value.startsWith('--')) { throw new Error(`${argument} requires a value`); } index += 1; return value; }; if (argument === '--') continue; if (argument === '--samples') options.samples = parsePositiveInteger(next(), argument); else if (argument === '--timeout-ms') options.timeoutMs = parsePositiveInteger(next(), argument); else if (argument === '--stage') options.stage = true; else if (argument === '--keep-stage') options.keepStage = true; else if (argument === '--report') options.reportPath = resolve(next()); else if (argument === '--provider-fixture') options.providerFixturePath = resolve(next()); else if (argument === '--image') options.imagePath = resolve(next()); else if (argument === '--electron-executable') options.electronExecutablePath = resolve(next()); else if (argument === '--cli-path') options.cliPath = resolve(next()); else if (argument === '--artifact-label') options.artifactLabel = next(); else if (argument === '--help') options.help = true; else throw new Error(`Unknown argument: ${argument}`); } if (options.keepStage && !options.stage) { throw new Error('--keep-stage requires --stage'); } if (options.imagePath && !options.providerFixturePath) { throw new Error('--image requires --provider-fixture'); } if (Boolean(options.electronExecutablePath) !== Boolean(options.cliPath)) { throw new Error('--electron-executable and --cli-path must be provided together'); } if (options.stage && options.electronExecutablePath) { throw new Error('--stage cannot be combined with a packaged runtime override'); } return options; } export function createStrictJsonlParser(onRecord) { const decoder = new StringDecoder('utf8'); let buffer = ''; const emitLines = () => { while (true) { const newlineIndex = buffer.indexOf('\n'); if (newlineIndex === -1) return; let line = buffer.slice(0, newlineIndex); buffer = buffer.slice(newlineIndex + 1); if (line.endsWith('\r')) line = line.slice(0, -1); if (line.length > 0) onRecord(JSON.parse(line)); } }; return { push(chunk) { buffer += typeof chunk === 'string' ? chunk : decoder.write(chunk); emitLines(); }, finish() { buffer += decoder.end(); if (buffer.length > 0) { const line = buffer.endsWith('\r') ? buffer.slice(0, -1) : buffer; buffer = ''; if (line.length > 0) onRecord(JSON.parse(line)); } }, }; } export function percentile(values, requestedPercentile) { if (values.length === 0) return null; const sorted = [...values].sort((left, right) => left - right); const rank = Math.ceil((requestedPercentile / 100) * sorted.length) - 1; return sorted[Math.max(0, Math.min(sorted.length - 1, rank))]; } export function summarizeMeasurements(values) { if (values.length === 0) { return { samples: 0, p50: null, p95: null, max: null }; } return { samples: values.length, p50: Math.round(percentile(values, 50)), p95: Math.round(percentile(values, 95)), max: Math.round(Math.max(...values)), }; } function readJson(path) { return readFile(path, 'utf8').then((source) => JSON.parse(source)); } async function pathExists(path) { try { await stat(path); return true; } catch { return false; } } export async function resolvePiRuntime(projectRoot = process.cwd()) { const resolvedProjectRoot = resolve(projectRoot); const requireFromProject = createRequire(join(resolvedProjectRoot, 'package.json')); const electronExecutable = requireFromProject('electron'); const packageRoot = realpathSync(join( resolvedProjectRoot, 'node_modules', ...PI_RUNTIME_IDENTITY.packageName.split('/'), )); return { projectRoot: resolvedProjectRoot, electronExecutable, packageRoot, cliPath: join(packageRoot, 'dist', 'cli.js'), }; } export async function validatePiIdentity(runtime) { const rootPackage = await readJson(join(runtime.projectRoot, 'package.json')); const piPackage = await readJson(join(runtime.packageRoot, 'package.json')); const pinnedVersion = rootPackage.dependencies?.[PI_RUNTIME_IDENTITY.packageName]; const problems = []; if (pinnedVersion !== PI_RUNTIME_IDENTITY.version) { problems.push(`root dependency must be exactly ${PI_RUNTIME_IDENTITY.version}, got ${pinnedVersion ?? 'missing'}`); } if (piPackage.version !== PI_RUNTIME_IDENTITY.version) { problems.push(`installed package must be ${PI_RUNTIME_IDENTITY.version}, got ${piPackage.version ?? 'missing'}`); } if (piPackage.engines?.node !== PI_RUNTIME_IDENTITY.nodeEngine) { problems.push(`Pi Node engine must be ${PI_RUNTIME_IDENTITY.nodeEngine}, got ${piPackage.engines?.node ?? 'missing'}`); } if (piPackage.bin?.pi !== 'dist/cli.js') { problems.push(`Pi CLI entry must be dist/cli.js, got ${piPackage.bin?.pi ?? 'missing'}`); } if (!await pathExists(runtime.cliPath)) problems.push(`Pi CLI does not exist at ${runtime.cliPath}`); if (!await pathExists(runtime.electronExecutable)) { problems.push(`Electron executable does not exist at ${runtime.electronExecutable}`); } if (problems.length > 0) throw new Error(problems.join('; ')); return { ...PI_RUNTIME_IDENTITY, installedPackageRoot: runtime.packageRoot, cliEntry: relative(runtime.packageRoot, runtime.cliPath), }; } function runCommand(executable, args, options = {}) { return new Promise((resolvePromise, reject) => { const child = spawn(executable, args, { cwd: options.cwd, env: options.env ?? process.env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, }); let stdout = ''; let stderr = ''; child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); child.once('error', reject); child.once('exit', (code, signal) => { if (code === 0) resolvePromise({ stdout, stderr }); else reject(new Error(`${executable} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}: ${stderr || stdout}`)); }); }); } async function inspectElectronNodeRuntime(executable) { const { stdout } = await runCommand( executable, ['-p', 'JSON.stringify(process.versions)'], { env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' } }, ); const versions = JSON.parse(stdout.trim()); const [major, minor] = versions.node.split('.').map((value) => Number.parseInt(value, 10)); if (major < 22 || (major === 22 && minor < 19)) { throw new Error(`Electron Node ${versions.node} does not satisfy Pi ${PI_RUNTIME_IDENTITY.nodeEngine}`); } return { electron: versions.electron, node: versions.node, modules: versions.modules, napi: versions.napi, }; } export async function stagePiRuntime(runtime, destination) { await mkdir(destination, { recursive: true }); await cp(runtime.packageRoot, destination, { recursive: true, filter(source) { return relative(runtime.packageRoot, source).split(sep)[0] !== 'node_modules'; }, }); // The published shrinkwrap intentionally contains the production graph only, // while the published package.json still lists development dependencies. npm // validates those dev entries before applying --omit=dev, so make the copied // qualification root match the production-only shrinkwrap before npm ci. const stagedPackagePath = join(destination, 'package.json'); const stagedPackage = await readJson(stagedPackagePath); const omittedDevDependencies = Object.keys(stagedPackage.devDependencies ?? {}).length; delete stagedPackage.devDependencies; await writeFile(stagedPackagePath, `${JSON.stringify(stagedPackage, null, 2)}\n`); const npmCli = join( realpathSync(join(runtime.projectRoot, 'node_modules', 'npm')), 'bin', 'npm-cli.js', ); const install = await runCommand(process.execPath, [ npmCli, 'ci', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', ], { cwd: destination, env: { ...process.env, npm_config_update_notifier: 'false', }, }); return { destination, installStderr: install.stderr.trim(), omittedDevDependencies }; } function currentLibc() { if (platform() !== 'linux') return undefined; return process.report?.getReport?.().header?.glibcVersionRuntime ? 'glibc' : 'musl'; } function constraintMatches(constraints, current) { if (!constraints || constraints.length === 0 || !current) return true; const positive = constraints.filter((value) => !value.startsWith('!')); const negative = constraints.filter((value) => value.startsWith('!')).map((value) => value.slice(1)); return !negative.includes(current) && (positive.length === 0 || positive.includes(current)); } export function lockEntrySupportsCurrentPlatform(entry) { return constraintMatches(entry.os, platform()) && constraintMatches(entry.cpu, arch()) && constraintMatches(entry.libc, currentLibc()); } async function collectRuntimeAssets(root) { const assets = []; const visit = async (directory) => { for (const entry of await readdir(directory, { withFileTypes: true })) { const path = join(directory, entry.name); if (entry.isDirectory()) await visit(path); else if (entry.name.endsWith('.wasm') || entry.name.endsWith('.node')) { assets.push(relative(root, path).split(sep).join('/')); } } }; for (const runtimeDirectory of ['dist', 'node_modules']) { const path = join(root, runtimeDirectory); if (await pathExists(path)) await visit(path); } return assets.sort(); } export async function validateStagedClosure(stageRoot) { const shrinkwrap = await readJson(join(stageRoot, 'npm-shrinkwrap.json')); const missing = []; let expectedPackages = 0; let platformSkippedPackages = 0; for (const [packagePath, entry] of Object.entries(shrinkwrap.packages ?? {})) { if (!packagePath || entry.dev) continue; if (!lockEntrySupportsCurrentPlatform(entry)) { platformSkippedPackages += 1; continue; } expectedPackages += 1; if (!await pathExists(join(stageRoot, ...packagePath.split('/')))) missing.push(packagePath); } const requiredWasm = 'node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm'; if (!await pathExists(join(stageRoot, ...requiredWasm.split('/')))) missing.push(requiredWasm); if (missing.length > 0) { throw new Error(`Staged Pi production closure is incomplete: ${missing.join(', ')}`); } return { lockfileVersion: shrinkwrap.lockfileVersion, lockedPackages: Math.max(0, Object.keys(shrinkwrap.packages ?? {}).length - 1), expectedPackages, platformSkippedPackages, assets: await collectRuntimeAssets(stageRoot), }; } function timeoutAfter(milliseconds, message) { let timer; const promise = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(message)), milliseconds); }); return { promise, cancel: () => clearTimeout(timer) }; } export class PiRpcWorker { constructor({ executable, cliPath, cwd, env, args, timeoutMs = DEFAULT_TIMEOUT_MS }) { this.executable = executable; this.cliPath = cliPath; this.cwd = cwd; this.env = env; this.args = args; this.timeoutMs = timeoutMs; this.pending = new Map(); this.waiters = []; this.events = []; this.stderr = ''; this.sequence = 0; } async start() { if (this.child) throw new Error('Pi RPC worker already started'); this.startedAt = performance.now(); this.child = spawn(this.executable, [this.cliPath, ...this.args], { cwd: this.cwd, env: { ...process.env, ...this.env, ELECTRON_RUN_AS_NODE: '1' }, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, }); this.exitResult = new Promise((resolveExit, rejectExit) => { this.child.once('error', rejectExit); this.child.once('exit', (code, signal) => { this.exitedAt = performance.now(); const result = { code, signal }; resolveExit(result); for (const pending of this.pending.values()) { pending.reject(new Error(`Pi RPC worker exited before response (code=${code}, signal=${signal})`)); } this.pending.clear(); }); }); const parser = createStrictJsonlParser((record) => this.handleRecord(record)); this.child.stdout.on('data', (chunk) => parser.push(chunk)); this.child.stdout.on('end', () => parser.finish()); this.child.stderr.on('data', (chunk) => { this.stderr = `${this.stderr}${chunk.toString()}`.slice(-MAX_STDERR_LENGTH); }); await new Promise((resolveSpawn, rejectSpawn) => { this.child.once('spawn', () => { this.spawnedAt = performance.now(); resolveSpawn(); }); this.child.once('error', rejectSpawn); }); return this; } handleRecord(record) { record.receivedAt = performance.now(); if (record.type === 'response' && record.id && this.pending.has(record.id)) { const pending = this.pending.get(record.id); this.pending.delete(record.id); pending.resolve(record); return; } this.events.push(record); for (const waiter of [...this.waiters]) { if (!waiter.predicate(record)) continue; this.waiters.splice(this.waiters.indexOf(waiter), 1); waiter.cancelTimeout(); waiter.resolve(record); } } async request(command, timeoutMs = this.timeoutMs) { const id = `probe-${++this.sequence}`; const timeout = timeoutAfter(timeoutMs, `Pi RPC command ${command.type} timed out after ${timeoutMs}ms`); const response = new Promise((resolveResponse, reject) => { this.pending.set(id, { resolve: resolveResponse, reject }); }); this.child.stdin.write(`${JSON.stringify({ ...command, id })}\n`); try { const record = await Promise.race([response, timeout.promise]); if (!record.success) throw new Error(`Pi RPC ${record.command} failed: ${record.error}`); return record; } finally { timeout.cancel(); this.pending.delete(id); } } waitForEvent(predicate, timeoutMs = this.timeoutMs) { const existing = this.events.find(predicate); if (existing) return Promise.resolve(existing); return new Promise((resolveEvent, reject) => { const timeout = timeoutAfter(timeoutMs, `Pi RPC event timed out after ${timeoutMs}ms`); const waiter = { predicate, resolve: resolveEvent, cancelTimeout: timeout.cancel, }; this.waiters.push(waiter); timeout.promise.catch((error) => { const index = this.waiters.indexOf(waiter); if (index !== -1) this.waiters.splice(index, 1); reject(error); }); }); } stop(graceMs = 2_000) { if (!this.stopPromise) this.stopPromise = this.performStop(graceMs); return this.stopPromise; } async performStop(graceMs) { if (!this.child) return { mode: 'not-started', code: null, signal: null, exitMs: 0 }; const stopStartedAt = performance.now(); this.child.stdin.end(); const gracefulTimeout = timeoutAfter(graceMs, 'Pi RPC stdin close did not exit'); try { const result = await Promise.race([this.exitResult, gracefulTimeout.promise]); return { mode: 'stdin-close', ...result, exitMs: Math.round(performance.now() - stopStartedAt) }; } catch { this.child.kill('SIGTERM'); const terminateTimeout = timeoutAfter(graceMs, 'Pi RPC SIGTERM did not exit'); try { const result = await Promise.race([this.exitResult, terminateTimeout.promise]); return { mode: 'sigterm', ...result, exitMs: Math.round(performance.now() - stopStartedAt) }; } catch { if (platform() === 'win32') { await runCommand('taskkill.exe', ['/pid', String(this.child.pid), '/t', '/f']).catch(() => undefined); } else { this.child.kill('SIGKILL'); } const result = await this.exitResult; return { mode: platform() === 'win32' ? 'taskkill' : 'sigkill', ...result, exitMs: Math.round(performance.now() - stopStartedAt), }; } finally { terminateTimeout.cancel(); } } finally { gracefulTimeout.cancel(); } } } async function getProcessRssKb(pid) { if (!pid) return null; try { if (platform() === 'win32') { const { stdout } = await runCommand('tasklist.exe', ['/fi', `PID eq ${pid}`, '/fo', 'csv', '/nh']); const columns = stdout.trim().replace(/^"|"$/g, '').split('","'); const digits = columns.at(-1)?.replace(/[^0-9]/g, ''); return digits ? Number.parseInt(digits, 10) : null; } const { stdout } = await runCommand('ps', ['-o', 'rss=', '-p', String(pid)]); return Number.parseInt(stdout.trim(), 10) || null; } catch { return null; } } function baseRpcArgs(sessionDir, extraArgs = []) { return [ '--mode', 'rpc', '--offline', '--session-dir', sessionDir, '--no-extensions', '--no-skills', '--no-prompt-templates', '--no-themes', '--no-context-files', '--no-approve', '--no-tools', ...extraArgs, ]; } async function makeWorkerPaths(root, prefix) { const base = await mkdtemp(join(root, `${prefix}-`)); const configDir = join(base, 'config'); const sessionDir = join(base, 'sessions'); const cwd = join(base, 'project'); await Promise.all([mkdir(configDir), mkdir(sessionDir), mkdir(cwd)]); return { base, configDir, sessionDir, cwd }; } function workerEnv(configDir) { return { PI_CODING_AGENT_DIR: configDir, PI_OFFLINE: '1', PI_TELEMETRY: '0', }; } async function runReadySample(runtime, paths, timeoutMs) { const worker = new PiRpcWorker({ executable: runtime.electronExecutable, cliPath: runtime.cliPath, cwd: paths.cwd, env: workerEnv(paths.configDir), args: baseRpcArgs(paths.sessionDir, ['--session-id', `probe-${Date.now()}-${Math.random()}`]), timeoutMs, }); try { await worker.start(); const response = await worker.request({ type: 'get_state' }); const readyMs = response.receivedAt - worker.startedAt; const workerSpawnMs = worker.spawnedAt - worker.startedAt; const rpcReadyMs = response.receivedAt - worker.spawnedAt; const rssKb = await getProcessRssKb(worker.child.pid); const stop = await worker.stop(); return { readyMs, workerSpawnMs, rpcReadyMs, rssKb, stop, sessionId: response.data.sessionId }; } finally { await worker.stop().catch(() => undefined); } } async function runPerformanceSamples(runtime, scratchRoot, sampleCount, timeoutMs) { const cold = []; for (let index = 0; index < sampleCount; index += 1) { const paths = await makeWorkerPaths(scratchRoot, `cold-${index}`); cold.push(await runReadySample(runtime, paths, timeoutMs)); } const warmConfig = join(scratchRoot, 'warm-config'); await mkdir(warmConfig); const warm = []; for (let index = 0; index < sampleCount; index += 1) { const paths = await makeWorkerPaths(scratchRoot, `warm-${index}`); paths.configDir = warmConfig; warm.push(await runReadySample(runtime, paths, timeoutMs)); } return { definition: { cold: 'fresh Pi config, session, and project directories per process', warm: 'shared primed Pi config directory with fresh session and project directories per process', workerSpawn: 'final product executable spawn event minus spawn request', rpcReady: 'first successful get_state response minus child spawn event', }, coldReadyMs: summarizeMeasurements(cold.map((sample) => sample.readyMs)), warmReadyMs: summarizeMeasurements(warm.map((sample) => sample.readyMs)), coldWorkerSpawnMs: summarizeMeasurements(cold.map((sample) => sample.workerSpawnMs)), warmWorkerSpawnMs: summarizeMeasurements(warm.map((sample) => sample.workerSpawnMs)), coldRpcReadyMs: summarizeMeasurements(cold.map((sample) => sample.rpcReadyMs)), warmRpcReadyMs: summarizeMeasurements(warm.map((sample) => sample.rpcReadyMs)), rssKb: summarizeMeasurements([...cold, ...warm].flatMap((sample) => sample.rssKb == null ? [] : [sample.rssKb])), exitMs: summarizeMeasurements([...cold, ...warm].map((sample) => sample.stop.exitMs)), exitModes: [...new Set([...cold, ...warm].map((sample) => sample.stop.mode))], }; } async function runSessionLifecycle(runtime, scratchRoot, timeoutMs) { const paths = await makeWorkerPaths(scratchRoot, 'session'); const failureFixture = { id: 'makelore-deterministic-failure', apiProtocol: 'openai-completions', baseUrl: 'http://127.0.0.1:1/v1', apiKeyEnv: 'PI_PROBE_FAILURE_KEY', model: { id: 'pi-probe-failure-model' }, }; await writeFile( join(paths.configDir, 'models.json'), `${JSON.stringify(buildPiProviderConfig(failureFixture), null, 2)}\n`, ); const common = { executable: runtime.electronExecutable, cliPath: runtime.cliPath, cwd: paths.cwd, env: { ...workerEnv(paths.configDir), PI_PROBE_FAILURE_KEY: 'qualification-only' }, timeoutMs, }; const first = new PiRpcWorker({ ...common, args: baseRpcArgs(paths.sessionDir, [ '--session-id', `probe-session-${Date.now()}`, '--provider', failureFixture.id, '--model', failureFixture.model.id, ]), }); let reopened; try { await first.start(); const initial = await first.request({ type: 'get_state' }); await first.request({ type: 'set_session_name', name: 'PI runtime qualification' }); const shell = await first.request({ type: 'bash', command: 'echo PI_RPC_SHELL_OK' }); await first.request({ type: 'abort' }); await first.request({ type: 'set_auto_retry', enabled: false }); const failureStartedAt = performance.now(); const failedTurn = await promptAndSettle( first, 'This request must fail against the closed local probe endpoint.', undefined, timeoutMs, ); const deterministicFailureMs = performance.now() - failureStartedAt; const firstStop = await first.stop(); if (!initial.data.sessionFile || !await pathExists(initial.data.sessionFile)) { throw new Error('Pi RPC session file was not persisted'); } reopened = new PiRpcWorker({ ...common, args: baseRpcArgs(paths.sessionDir, [ '--session', initial.data.sessionFile, '--provider', failureFixture.id, '--model', failureFixture.model.id, ]), }); await reopened.start(); const reopenedState = await reopened.request({ type: 'get_state' }); const entries = await reopened.request({ type: 'get_entries' }); const reopenedStop = await reopened.stop(); if (reopenedState.data.sessionId !== initial.data.sessionId) { throw new Error('Pi RPC reopened a different session id'); } return { sessionIdPreserved: true, sessionFilePersisted: true, entryCount: entries.data.entries.length, shellOutputMatched: shell.data.output.includes('PI_RPC_SHELL_OK'), idleAbortAccepted: true, promptAccepted: true, agentSettled: true, deterministicFailureStopReason: failedTurn.stopReason, deterministicFailureMs: Math.round(deterministicFailureMs), exitModes: [firstStop.mode, reopenedStop.mode], }; } finally { await Promise.all([ first.stop().catch(() => undefined), reopened?.stop().catch(() => undefined), ]); } } function longShellCommand(label, seconds) { if (platform() === 'win32') return `ping 127.0.0.1 -n ${seconds + 1} > nul && echo ${label}`; return `sleep ${seconds}; echo ${label}`; } async function runLocalWorkerIsolation(runtime, scratchRoot, timeoutMs) { const [leftPaths, rightPaths] = await Promise.all([ makeWorkerPaths(scratchRoot, 'worker-left'), makeWorkerPaths(scratchRoot, 'worker-right'), ]); const create = (paths) => new PiRpcWorker({ executable: runtime.electronExecutable, cliPath: runtime.cliPath, cwd: paths.cwd, env: workerEnv(paths.configDir), args: baseRpcArgs(paths.sessionDir), timeoutMs, }); const left = create(leftPaths); const right = create(rightPaths); try { await Promise.all([left.start(), right.start()]); const [leftState, rightState] = await Promise.all([ left.request({ type: 'get_state' }), right.request({ type: 'get_state' }), ]); const startedAt = performance.now(); const leftBash = left.request({ type: 'bash', command: longShellCommand('PI_LEFT_ABORTED', 4) }); const rightBash = right.request({ type: 'bash', command: longShellCommand('PI_RIGHT_OK', 2) }); await new Promise((resolveWait) => setTimeout(resolveWait, 500)); await left.request({ type: 'abort_bash' }); const [leftResult, rightResult] = await Promise.all([leftBash, rightBash]); const elapsedMs = performance.now() - startedAt; const stops = await Promise.all([left.stop(), right.stop()]); if (leftState.data.sessionId === rightState.data.sessionId) { throw new Error('Independent Pi workers shared a session id'); } if (!leftResult.data.cancelled) throw new Error('Aborted Pi bash command was not marked cancelled'); if (rightResult.data.cancelled || !rightResult.data.output.includes('PI_RIGHT_OK')) { throw new Error('Aborting one Pi worker affected the other worker'); } return { distinctSessionIds: true, abortIsolated: true, overlappingLocalWork: elapsedMs < 3_500, elapsedMs: Math.round(elapsedMs), exitModes: stops.map((stop) => stop.mode), limitation: 'This proves local worker and shell isolation, not overlapping real-provider turns.', }; } finally { await Promise.all([ left.stop().catch(() => undefined), right.stop().catch(() => undefined), ]); } } export function mapMakeloreProtocolToPi(protocol) { if (!MAKELore_PROVIDER_PROTOCOLS.includes(protocol)) { throw new Error(`Unsupported Makelore provider protocol: ${protocol}`); } if (protocol === 'openrouter') { return { api: 'openai-completions', compat: { thinkingFormat: 'openrouter', sessionAffinityFormat: 'openrouter' }, }; } return { api: protocol, compat: undefined }; } export function buildPiProviderConfig(fixture) { const mapping = mapMakeloreProtocolToPi(fixture.apiProtocol); const model = fixture.model ?? {}; return { providers: { [fixture.id]: { baseUrl: fixture.baseUrl, apiKey: `$${fixture.apiKeyEnv}`, api: mapping.api, ...(fixture.headers ? { headers: fixture.headers } : {}), models: [{ id: model.id, name: model.name ?? model.id, reasoning: model.reasoning ?? false, input: model.input ?? ['text'], cost: model.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: model.contextWindow ?? 128_000, maxTokens: model.maxTokens ?? 8_192, ...(mapping.compat ? { compat: mapping.compat } : {}), }], }, }, }; } async function loadProviderFixture(path) { const fixture = await readJson(path); for (const field of ['id', 'apiProtocol', 'baseUrl', 'apiKeyEnv']) { if (!fixture[field]) throw new Error(`Provider fixture is missing ${field}`); } if (!fixture.model?.id) throw new Error('Provider fixture is missing model.id'); mapMakeloreProtocolToPi(fixture.apiProtocol); if (!process.env[fixture.apiKeyEnv]) { throw new Error(`Provider credential environment variable is not set: ${fixture.apiKeyEnv}`); } return fixture; } async function imageContentFromPath(path) { const data = await readFile(path); const extension = path.toLowerCase().split('.').at(-1); const mimeType = extension === 'png' ? 'image/png' : extension === 'webp' ? 'image/webp' : 'image/jpeg'; return { type: 'image', data: data.toString('base64'), mimeType }; } function lastAssistantStopReason(messagesResponse) { const messages = messagesResponse.data.messages; return [...messages].reverse().find((message) => message.role === 'assistant')?.stopReason ?? null; } async function runProviderWorker(runtime, paths, fixture, timeoutMs) { await writeFile( join(paths.configDir, 'models.json'), `${JSON.stringify(buildPiProviderConfig(fixture), null, 2)}\n`, ); const worker = new PiRpcWorker({ executable: runtime.electronExecutable, cliPath: runtime.cliPath, cwd: paths.cwd, env: workerEnv(paths.configDir), args: baseRpcArgs(paths.sessionDir, ['--provider', fixture.id, '--model', fixture.model.id]), timeoutMs, }); await worker.start(); return worker; } async function promptAndSettle(worker, message, images, timeoutMs) { const eventStartIndex = worker.events.length; const sentAt = performance.now(); const accepted = await worker.request({ type: 'prompt', message, ...(images ? { images } : {}) }); const agentStart = await worker.waitForEvent( (event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_start', timeoutMs, ); const settled = await worker.waitForEvent( (event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_settled', timeoutMs, ); const firstProviderEvent = worker.events .slice(eventStartIndex) .find((event) => event !== agentStart && event.receivedAt >= agentStart.receivedAt); const messagesResponse = await worker.request({ type: 'get_messages' }); return { acceptedMs: accepted.receivedAt - sentAt, agentStartMs: agentStart.receivedAt - accepted.receivedAt, providerFirstEventMs: firstProviderEvent ? firstProviderEvent.receivedAt - agentStart.receivedAt : settled.receivedAt - agentStart.receivedAt, agentSettledMs: settled.receivedAt - accepted.receivedAt, startedAt: agentStart.receivedAt, settledAt: settled.receivedAt, stopReason: lastAssistantStopReason(messagesResponse), }; } export async function runProviderQualification(runtime, scratchRoot, fixturePath, imagePath, timeoutMs) { const fixture = await loadProviderFixture(fixturePath); const [leftPaths, rightPaths] = await Promise.all([ makeWorkerPaths(scratchRoot, 'provider-left'), makeWorkerPaths(scratchRoot, 'provider-right'), ]); const [left, right] = await Promise.all([ runProviderWorker(runtime, leftPaths, fixture, timeoutMs), runProviderWorker(runtime, rightPaths, fixture, timeoutMs), ]); try { const state = await Promise.all([ left.request({ type: 'get_state' }), right.request({ type: 'get_state' }), ]); await Promise.all([ left.request({ type: 'set_auto_retry', enabled: false }), right.request({ type: 'set_auto_retry', enabled: false }), ]); const images = imagePath ? [await imageContentFromPath(imagePath)] : undefined; const [leftTurn, rightTurn] = await Promise.all([ promptAndSettle(left, 'Reply with exactly PI_PROVIDER_LEFT_OK. Do not use tools.', images, timeoutMs), promptAndSettle(right, 'Reply with exactly PI_PROVIDER_RIGHT_OK. Do not use tools.', images, timeoutMs), ]); const overlapMs = Math.min(leftTurn.settledAt, rightTurn.settledAt) - Math.max(leftTurn.startedAt, rightTurn.startedAt); const successfulReasons = new Set(['stop', 'length', 'toolUse']); if (!successfulReasons.has(leftTurn.stopReason) || !successfulReasons.has(rightTurn.stopReason)) { throw new Error(`Provider turns did not succeed: left=${leftTurn.stopReason}, right=${rightTurn.stopReason}`); } if (overlapMs <= 0) throw new Error('Provider turns did not overlap'); if (state[0].data.sessionId === state[1].data.sessionId) { throw new Error('Provider workers shared a session id'); } const [leftWarmTurn, rightWarmTurn] = await Promise.all([ promptAndSettle(left, 'Reply with exactly PI_PROVIDER_LEFT_WARM_OK. Do not use tools.', undefined, timeoutMs), promptAndSettle(right, 'Reply with exactly PI_PROVIDER_RIGHT_WARM_OK. Do not use tools.', undefined, timeoutMs), ]); if (!successfulReasons.has(leftWarmTurn.stopReason) || !successfulReasons.has(rightWarmTurn.stopReason)) { throw new Error( `Warm provider turns did not succeed: left=${leftWarmTurn.stopReason}, right=${rightWarmTurn.stopReason}`, ); } const warmOverlapMs = Math.min(leftWarmTurn.settledAt, rightWarmTurn.settledAt) - Math.max(leftWarmTurn.startedAt, rightWarmTurn.startedAt); if (warmOverlapMs <= 0) throw new Error('Warm provider turns did not overlap'); const abortStartIndex = left.events.length; const abortedPrompt = left.request({ type: 'prompt', message: 'Start a normal response. This turn will be aborted by the qualification probe.', }); const unaffectedTurn = promptAndSettle( right, 'Reply with exactly PI_PROVIDER_ABORT_ISOLATION_OK. Do not use tools.', undefined, timeoutMs, ); await abortedPrompt; await left.waitForEvent( (event) => left.events.indexOf(event) >= abortStartIndex && event.type === 'agent_start', timeoutMs, ); await left.request({ type: 'abort' }); await left.waitForEvent( (event) => left.events.indexOf(event) >= abortStartIndex && event.type === 'agent_settled', timeoutMs, ); const [abortedMessages, unaffected] = await Promise.all([ left.request({ type: 'get_messages' }), unaffectedTurn, ]); const abortedStopReason = lastAssistantStopReason(abortedMessages); if (abortedStopReason !== 'aborted') { throw new Error(`Aborted provider turn ended with ${abortedStopReason ?? 'no assistant message'}`); } if (!successfulReasons.has(unaffected.stopReason)) { throw new Error(`Aborting one provider worker affected the other: ${unaffected.stopReason}`); } const stops = await Promise.all([left.stop(), right.stop()]); return { protocol: fixture.apiProtocol, providerId: fixture.id, modelId: fixture.model.id, headerNames: Object.keys(fixture.headers ?? {}).sort(), apiKeyEnv: fixture.apiKeyEnv, imageInput: Boolean(imagePath), distinctSessionIds: true, cold: { promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round), promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]), agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round), agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]), providerFirstEventSamplesMs: [ leftTurn.providerFirstEventMs, rightTurn.providerFirstEventMs, ].map(Math.round), providerFirstEventMs: summarizeMeasurements([ leftTurn.providerFirstEventMs, rightTurn.providerFirstEventMs, ]), agentSettledSamplesMs: [leftTurn.agentSettledMs, rightTurn.agentSettledMs].map(Math.round), agentSettledMs: summarizeMeasurements([leftTurn.agentSettledMs, rightTurn.agentSettledMs]), }, warm: { promptAcceptedSamplesMs: [leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs].map(Math.round), promptAcceptedMs: summarizeMeasurements([leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs]), agentStartSamplesMs: [leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs].map(Math.round), agentStartMs: summarizeMeasurements([leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs]), providerFirstEventSamplesMs: [ leftWarmTurn.providerFirstEventMs, rightWarmTurn.providerFirstEventMs, ].map(Math.round), providerFirstEventMs: summarizeMeasurements([ leftWarmTurn.providerFirstEventMs, rightWarmTurn.providerFirstEventMs, ]), agentSettledSamplesMs: [leftWarmTurn.agentSettledMs, rightWarmTurn.agentSettledMs].map(Math.round), agentSettledMs: summarizeMeasurements([ leftWarmTurn.agentSettledMs, rightWarmTurn.agentSettledMs, ]), }, overlapMs: Math.round(overlapMs), warmOverlapMs: Math.round(warmOverlapMs), stopReasons: [ leftTurn.stopReason, rightTurn.stopReason, leftWarmTurn.stopReason, rightWarmTurn.stopReason, ], abortIsolation: { abortedStopReason, unaffectedStopReason: unaffected.stopReason, passed: true, }, exitModes: stops.map((stop) => stop.mode), }; } finally { await Promise.all([ left.stop().catch(() => undefined), right.stop().catch(() => undefined), ]); } } function printHelp() { process.stdout.write(`Usage: node scripts/probe-pi-runtime.mjs [options]\n\n`); process.stdout.write(' --stage Install the package shrinkwrap into a temporary staged runtime\n'); process.stdout.write(' --keep-stage Preserve the temporary staged runtime for inspection\n'); process.stdout.write(' --samples Cold and warm startup sample count (default: 5)\n'); process.stdout.write(' --timeout-ms RPC/provider timeout (default: 10000)\n'); process.stdout.write(' --report Write the JSON report to this path\n'); process.stdout.write(' --provider-fixture Opt in to a real-provider overlap qualification run\n'); process.stdout.write(' --image Add an image to provider prompts (requires fixture)\n'); process.stdout.write(' --electron-executable

Use a packaged Electron executable (requires --cli-path)\n'); process.stdout.write(' --cli-path Pi CLI path visible to packaged Electron\n'); process.stdout.write(' --artifact-label