1177 lines
44 KiB
JavaScript
1177 lines
44 KiB
JavaScript
import { spawn } from 'node:child_process';
|
|
import { StringDecoder } from 'node:string_decoder';
|
|
import {
|
|
cp,
|
|
mkdtemp,
|
|
mkdir,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
stat,
|
|
writeFile,
|
|
} from 'node:fs/promises';
|
|
import { realpathSync } from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import { arch, platform, release, tmpdir } from 'node:os';
|
|
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
import { performance } from 'node:perf_hooks';
|
|
|
|
export const PI_RUNTIME_IDENTITY = Object.freeze({
|
|
packageName: '@earendil-works/pi-coding-agent',
|
|
version: '0.84.2',
|
|
tag: 'v0.84.2',
|
|
gitHead: '914cf1472e715297caa30db4b9535d534a9eb718',
|
|
nodeEngine: '>=22.19.0',
|
|
});
|
|
|
|
export const MAKELore_PROVIDER_PROTOCOLS = Object.freeze([
|
|
'openai-completions',
|
|
'openai-responses',
|
|
'anthropic-messages',
|
|
'openrouter',
|
|
]);
|
|
|
|
export function buildMissingEvidence() {
|
|
return [
|
|
'Phase-0 artifact aggregation: Windows x64 and Linux x64; macOS x64/arm64 deferred to PI-150 by user decision on 2026-08-22',
|
|
];
|
|
}
|
|
|
|
export function buildEvidenceWaivers() {
|
|
return [
|
|
'macOS x64/arm64 Phase-0 execution deferred to PI-150 by user decision on 2026-08-22; no macOS Pass is claimed',
|
|
'QG-004/QG-005 real Provider Account compatibility and real-provider concurrency/isolation explicitly waived by user decision on 2026-08-22; real authentication, endpoint/proxy/rate-limit/provider variation, protocol compatibility, concurrency, abort/event/session/model/credential isolation, and image behavior remain unverified',
|
|
];
|
|
}
|
|
|
|
const DEFAULT_TIMEOUT_MS = 10_000;
|
|
const DEFAULT_SAMPLE_COUNT = 5;
|
|
const MAX_STDERR_LENGTH = 16_000;
|
|
|
|
function parsePositiveInteger(value, flag) {
|
|
const parsed = Number.parseInt(value, 10);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
throw new Error(`${flag} must be a positive integer`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
export function parseProbeArgs(argv) {
|
|
const options = {
|
|
samples: DEFAULT_SAMPLE_COUNT,
|
|
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
stage: false,
|
|
keepStage: false,
|
|
reportPath: undefined,
|
|
providerFixturePath: undefined,
|
|
imagePath: undefined,
|
|
electronExecutablePath: undefined,
|
|
cliPath: undefined,
|
|
artifactLabel: undefined,
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index];
|
|
const next = () => {
|
|
const value = argv[index + 1];
|
|
if (!value || value.startsWith('--')) {
|
|
throw new Error(`${argument} requires a value`);
|
|
}
|
|
index += 1;
|
|
return value;
|
|
};
|
|
|
|
if (argument === '--') continue;
|
|
if (argument === '--samples') options.samples = parsePositiveInteger(next(), argument);
|
|
else if (argument === '--timeout-ms') options.timeoutMs = parsePositiveInteger(next(), argument);
|
|
else if (argument === '--stage') options.stage = true;
|
|
else if (argument === '--keep-stage') options.keepStage = true;
|
|
else if (argument === '--report') options.reportPath = resolve(next());
|
|
else if (argument === '--provider-fixture') options.providerFixturePath = resolve(next());
|
|
else if (argument === '--image') options.imagePath = resolve(next());
|
|
else if (argument === '--electron-executable') options.electronExecutablePath = resolve(next());
|
|
else if (argument === '--cli-path') options.cliPath = resolve(next());
|
|
else if (argument === '--artifact-label') options.artifactLabel = next();
|
|
else if (argument === '--help') options.help = true;
|
|
else throw new Error(`Unknown argument: ${argument}`);
|
|
}
|
|
|
|
if (options.keepStage && !options.stage) {
|
|
throw new Error('--keep-stage requires --stage');
|
|
}
|
|
if (options.imagePath && !options.providerFixturePath) {
|
|
throw new Error('--image requires --provider-fixture');
|
|
}
|
|
if (Boolean(options.electronExecutablePath) !== Boolean(options.cliPath)) {
|
|
throw new Error('--electron-executable and --cli-path must be provided together');
|
|
}
|
|
if (options.stage && options.electronExecutablePath) {
|
|
throw new Error('--stage cannot be combined with a packaged runtime override');
|
|
}
|
|
return options;
|
|
}
|
|
|
|
export function createStrictJsonlParser(onRecord) {
|
|
const decoder = new StringDecoder('utf8');
|
|
let buffer = '';
|
|
|
|
const emitLines = () => {
|
|
while (true) {
|
|
const newlineIndex = buffer.indexOf('\n');
|
|
if (newlineIndex === -1) return;
|
|
let line = buffer.slice(0, newlineIndex);
|
|
buffer = buffer.slice(newlineIndex + 1);
|
|
if (line.endsWith('\r')) line = line.slice(0, -1);
|
|
if (line.length > 0) onRecord(JSON.parse(line));
|
|
}
|
|
};
|
|
|
|
return {
|
|
push(chunk) {
|
|
buffer += typeof chunk === 'string' ? chunk : decoder.write(chunk);
|
|
emitLines();
|
|
},
|
|
finish() {
|
|
buffer += decoder.end();
|
|
if (buffer.length > 0) {
|
|
const line = buffer.endsWith('\r') ? buffer.slice(0, -1) : buffer;
|
|
buffer = '';
|
|
if (line.length > 0) onRecord(JSON.parse(line));
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
export function percentile(values, requestedPercentile) {
|
|
if (values.length === 0) return null;
|
|
const sorted = [...values].sort((left, right) => left - right);
|
|
const rank = Math.ceil((requestedPercentile / 100) * sorted.length) - 1;
|
|
return sorted[Math.max(0, Math.min(sorted.length - 1, rank))];
|
|
}
|
|
|
|
export function summarizeMeasurements(values) {
|
|
if (values.length === 0) {
|
|
return { samples: 0, p50: null, p95: null, max: null };
|
|
}
|
|
return {
|
|
samples: values.length,
|
|
p50: Math.round(percentile(values, 50)),
|
|
p95: Math.round(percentile(values, 95)),
|
|
max: Math.round(Math.max(...values)),
|
|
};
|
|
}
|
|
|
|
function readJson(path) {
|
|
return readFile(path, 'utf8').then((source) => JSON.parse(source));
|
|
}
|
|
|
|
async function pathExists(path) {
|
|
try {
|
|
await stat(path);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export async function resolvePiRuntime(projectRoot = process.cwd()) {
|
|
const resolvedProjectRoot = resolve(projectRoot);
|
|
const requireFromProject = createRequire(join(resolvedProjectRoot, 'package.json'));
|
|
const electronExecutable = requireFromProject('electron');
|
|
const packageRoot = realpathSync(join(
|
|
resolvedProjectRoot,
|
|
'node_modules',
|
|
...PI_RUNTIME_IDENTITY.packageName.split('/'),
|
|
));
|
|
return {
|
|
projectRoot: resolvedProjectRoot,
|
|
electronExecutable,
|
|
packageRoot,
|
|
cliPath: join(packageRoot, 'dist', 'cli.js'),
|
|
};
|
|
}
|
|
|
|
export async function validatePiIdentity(runtime) {
|
|
const rootPackage = await readJson(join(runtime.projectRoot, 'package.json'));
|
|
const piPackage = await readJson(join(runtime.packageRoot, 'package.json'));
|
|
const pinnedVersion = rootPackage.dependencies?.[PI_RUNTIME_IDENTITY.packageName];
|
|
const problems = [];
|
|
|
|
if (pinnedVersion !== PI_RUNTIME_IDENTITY.version) {
|
|
problems.push(`root dependency must be exactly ${PI_RUNTIME_IDENTITY.version}, got ${pinnedVersion ?? 'missing'}`);
|
|
}
|
|
if (piPackage.version !== PI_RUNTIME_IDENTITY.version) {
|
|
problems.push(`installed package must be ${PI_RUNTIME_IDENTITY.version}, got ${piPackage.version ?? 'missing'}`);
|
|
}
|
|
if (piPackage.engines?.node !== PI_RUNTIME_IDENTITY.nodeEngine) {
|
|
problems.push(`Pi Node engine must be ${PI_RUNTIME_IDENTITY.nodeEngine}, got ${piPackage.engines?.node ?? 'missing'}`);
|
|
}
|
|
if (piPackage.bin?.pi !== 'dist/cli.js') {
|
|
problems.push(`Pi CLI entry must be dist/cli.js, got ${piPackage.bin?.pi ?? 'missing'}`);
|
|
}
|
|
if (!await pathExists(runtime.cliPath)) problems.push(`Pi CLI does not exist at ${runtime.cliPath}`);
|
|
if (!await pathExists(runtime.electronExecutable)) {
|
|
problems.push(`Electron executable does not exist at ${runtime.electronExecutable}`);
|
|
}
|
|
if (problems.length > 0) throw new Error(problems.join('; '));
|
|
|
|
return {
|
|
...PI_RUNTIME_IDENTITY,
|
|
installedPackageRoot: runtime.packageRoot,
|
|
cliEntry: relative(runtime.packageRoot, runtime.cliPath),
|
|
};
|
|
}
|
|
|
|
function runCommand(executable, args, options = {}) {
|
|
return new Promise((resolvePromise, reject) => {
|
|
const child = spawn(executable, args, {
|
|
cwd: options.cwd,
|
|
env: options.env ?? process.env,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
windowsHide: true,
|
|
});
|
|
let stdout = '';
|
|
let stderr = '';
|
|
child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
|
|
child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
|
|
child.once('error', reject);
|
|
child.once('exit', (code, signal) => {
|
|
if (code === 0) resolvePromise({ stdout, stderr });
|
|
else reject(new Error(`${executable} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}: ${stderr || stdout}`));
|
|
});
|
|
});
|
|
}
|
|
|
|
async function inspectElectronNodeRuntime(executable) {
|
|
const { stdout } = await runCommand(
|
|
executable,
|
|
['-p', 'JSON.stringify(process.versions)'],
|
|
{ env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' } },
|
|
);
|
|
const versions = JSON.parse(stdout.trim());
|
|
const [major, minor] = versions.node.split('.').map((value) => Number.parseInt(value, 10));
|
|
if (major < 22 || (major === 22 && minor < 19)) {
|
|
throw new Error(`Electron Node ${versions.node} does not satisfy Pi ${PI_RUNTIME_IDENTITY.nodeEngine}`);
|
|
}
|
|
return {
|
|
electron: versions.electron,
|
|
node: versions.node,
|
|
modules: versions.modules,
|
|
napi: versions.napi,
|
|
};
|
|
}
|
|
|
|
export async function stagePiRuntime(runtime, destination) {
|
|
await mkdir(destination, { recursive: true });
|
|
await cp(runtime.packageRoot, destination, {
|
|
recursive: true,
|
|
filter(source) {
|
|
return relative(runtime.packageRoot, source).split(sep)[0] !== 'node_modules';
|
|
},
|
|
});
|
|
|
|
// The published shrinkwrap intentionally contains the production graph only,
|
|
// while the published package.json still lists development dependencies. npm
|
|
// validates those dev entries before applying --omit=dev, so make the copied
|
|
// qualification root match the production-only shrinkwrap before npm ci.
|
|
const stagedPackagePath = join(destination, 'package.json');
|
|
const stagedPackage = await readJson(stagedPackagePath);
|
|
const omittedDevDependencies = Object.keys(stagedPackage.devDependencies ?? {}).length;
|
|
delete stagedPackage.devDependencies;
|
|
await writeFile(stagedPackagePath, `${JSON.stringify(stagedPackage, null, 2)}\n`);
|
|
|
|
const npmCli = join(
|
|
realpathSync(join(runtime.projectRoot, 'node_modules', 'npm')),
|
|
'bin',
|
|
'npm-cli.js',
|
|
);
|
|
const install = await runCommand(process.execPath, [
|
|
npmCli,
|
|
'ci',
|
|
'--omit=dev',
|
|
'--ignore-scripts',
|
|
'--no-audit',
|
|
'--no-fund',
|
|
], {
|
|
cwd: destination,
|
|
env: {
|
|
...process.env,
|
|
npm_config_update_notifier: 'false',
|
|
},
|
|
});
|
|
return { destination, installStderr: install.stderr.trim(), omittedDevDependencies };
|
|
}
|
|
|
|
function currentLibc() {
|
|
if (platform() !== 'linux') return undefined;
|
|
return process.report?.getReport?.().header?.glibcVersionRuntime ? 'glibc' : 'musl';
|
|
}
|
|
|
|
function constraintMatches(constraints, current) {
|
|
if (!constraints || constraints.length === 0 || !current) return true;
|
|
const positive = constraints.filter((value) => !value.startsWith('!'));
|
|
const negative = constraints.filter((value) => value.startsWith('!')).map((value) => value.slice(1));
|
|
return !negative.includes(current) && (positive.length === 0 || positive.includes(current));
|
|
}
|
|
|
|
export function lockEntrySupportsCurrentPlatform(entry) {
|
|
return constraintMatches(entry.os, platform())
|
|
&& constraintMatches(entry.cpu, arch())
|
|
&& constraintMatches(entry.libc, currentLibc());
|
|
}
|
|
|
|
async function collectRuntimeAssets(root) {
|
|
const assets = [];
|
|
const visit = async (directory) => {
|
|
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
const path = join(directory, entry.name);
|
|
if (entry.isDirectory()) await visit(path);
|
|
else if (entry.name.endsWith('.wasm') || entry.name.endsWith('.node')) {
|
|
assets.push(relative(root, path).split(sep).join('/'));
|
|
}
|
|
}
|
|
};
|
|
for (const runtimeDirectory of ['dist', 'node_modules']) {
|
|
const path = join(root, runtimeDirectory);
|
|
if (await pathExists(path)) await visit(path);
|
|
}
|
|
return assets.sort();
|
|
}
|
|
|
|
export async function validateStagedClosure(stageRoot) {
|
|
const shrinkwrap = await readJson(join(stageRoot, 'npm-shrinkwrap.json'));
|
|
const missing = [];
|
|
let expectedPackages = 0;
|
|
let platformSkippedPackages = 0;
|
|
|
|
for (const [packagePath, entry] of Object.entries(shrinkwrap.packages ?? {})) {
|
|
if (!packagePath || entry.dev) continue;
|
|
if (!lockEntrySupportsCurrentPlatform(entry)) {
|
|
platformSkippedPackages += 1;
|
|
continue;
|
|
}
|
|
expectedPackages += 1;
|
|
if (!await pathExists(join(stageRoot, ...packagePath.split('/')))) missing.push(packagePath);
|
|
}
|
|
|
|
const requiredWasm = 'node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm';
|
|
if (!await pathExists(join(stageRoot, ...requiredWasm.split('/')))) missing.push(requiredWasm);
|
|
if (missing.length > 0) {
|
|
throw new Error(`Staged Pi production closure is incomplete: ${missing.join(', ')}`);
|
|
}
|
|
|
|
return {
|
|
lockfileVersion: shrinkwrap.lockfileVersion,
|
|
lockedPackages: Math.max(0, Object.keys(shrinkwrap.packages ?? {}).length - 1),
|
|
expectedPackages,
|
|
platformSkippedPackages,
|
|
assets: await collectRuntimeAssets(stageRoot),
|
|
};
|
|
}
|
|
|
|
function timeoutAfter(milliseconds, message) {
|
|
let timer;
|
|
const promise = new Promise((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(message)), milliseconds);
|
|
});
|
|
return { promise, cancel: () => clearTimeout(timer) };
|
|
}
|
|
|
|
export class PiRpcWorker {
|
|
constructor({ executable, cliPath, cwd, env, args, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
this.executable = executable;
|
|
this.cliPath = cliPath;
|
|
this.cwd = cwd;
|
|
this.env = env;
|
|
this.args = args;
|
|
this.timeoutMs = timeoutMs;
|
|
this.pending = new Map();
|
|
this.waiters = [];
|
|
this.events = [];
|
|
this.stderr = '';
|
|
this.sequence = 0;
|
|
}
|
|
|
|
async start() {
|
|
if (this.child) throw new Error('Pi RPC worker already started');
|
|
this.startedAt = performance.now();
|
|
this.child = spawn(this.executable, [this.cliPath, ...this.args], {
|
|
cwd: this.cwd,
|
|
env: { ...process.env, ...this.env, ELECTRON_RUN_AS_NODE: '1' },
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
windowsHide: true,
|
|
});
|
|
this.exitResult = new Promise((resolveExit, rejectExit) => {
|
|
this.child.once('error', rejectExit);
|
|
this.child.once('exit', (code, signal) => {
|
|
this.exitedAt = performance.now();
|
|
const result = { code, signal };
|
|
resolveExit(result);
|
|
for (const pending of this.pending.values()) {
|
|
pending.reject(new Error(`Pi RPC worker exited before response (code=${code}, signal=${signal})`));
|
|
}
|
|
this.pending.clear();
|
|
});
|
|
});
|
|
|
|
const parser = createStrictJsonlParser((record) => this.handleRecord(record));
|
|
this.child.stdout.on('data', (chunk) => parser.push(chunk));
|
|
this.child.stdout.on('end', () => parser.finish());
|
|
this.child.stderr.on('data', (chunk) => {
|
|
this.stderr = `${this.stderr}${chunk.toString()}`.slice(-MAX_STDERR_LENGTH);
|
|
});
|
|
|
|
await new Promise((resolveSpawn, rejectSpawn) => {
|
|
this.child.once('spawn', () => {
|
|
this.spawnedAt = performance.now();
|
|
resolveSpawn();
|
|
});
|
|
this.child.once('error', rejectSpawn);
|
|
});
|
|
return this;
|
|
}
|
|
|
|
handleRecord(record) {
|
|
record.receivedAt = performance.now();
|
|
if (record.type === 'response' && record.id && this.pending.has(record.id)) {
|
|
const pending = this.pending.get(record.id);
|
|
this.pending.delete(record.id);
|
|
pending.resolve(record);
|
|
return;
|
|
}
|
|
this.events.push(record);
|
|
for (const waiter of [...this.waiters]) {
|
|
if (!waiter.predicate(record)) continue;
|
|
this.waiters.splice(this.waiters.indexOf(waiter), 1);
|
|
waiter.cancelTimeout();
|
|
waiter.resolve(record);
|
|
}
|
|
}
|
|
|
|
async request(command, timeoutMs = this.timeoutMs) {
|
|
const id = `probe-${++this.sequence}`;
|
|
const timeout = timeoutAfter(timeoutMs, `Pi RPC command ${command.type} timed out after ${timeoutMs}ms`);
|
|
const response = new Promise((resolveResponse, reject) => {
|
|
this.pending.set(id, { resolve: resolveResponse, reject });
|
|
});
|
|
this.child.stdin.write(`${JSON.stringify({ ...command, id })}\n`);
|
|
try {
|
|
const record = await Promise.race([response, timeout.promise]);
|
|
if (!record.success) throw new Error(`Pi RPC ${record.command} failed: ${record.error}`);
|
|
return record;
|
|
} finally {
|
|
timeout.cancel();
|
|
this.pending.delete(id);
|
|
}
|
|
}
|
|
|
|
waitForEvent(predicate, timeoutMs = this.timeoutMs) {
|
|
const existing = this.events.find(predicate);
|
|
if (existing) return Promise.resolve(existing);
|
|
return new Promise((resolveEvent, reject) => {
|
|
const timeout = timeoutAfter(timeoutMs, `Pi RPC event timed out after ${timeoutMs}ms`);
|
|
const waiter = {
|
|
predicate,
|
|
resolve: resolveEvent,
|
|
cancelTimeout: timeout.cancel,
|
|
};
|
|
this.waiters.push(waiter);
|
|
timeout.promise.catch((error) => {
|
|
const index = this.waiters.indexOf(waiter);
|
|
if (index !== -1) this.waiters.splice(index, 1);
|
|
reject(error);
|
|
});
|
|
});
|
|
}
|
|
|
|
stop(graceMs = 2_000) {
|
|
if (!this.stopPromise) this.stopPromise = this.performStop(graceMs);
|
|
return this.stopPromise;
|
|
}
|
|
|
|
async performStop(graceMs) {
|
|
if (!this.child) return { mode: 'not-started', code: null, signal: null, exitMs: 0 };
|
|
const stopStartedAt = performance.now();
|
|
this.child.stdin.end();
|
|
const gracefulTimeout = timeoutAfter(graceMs, 'Pi RPC stdin close did not exit');
|
|
try {
|
|
const result = await Promise.race([this.exitResult, gracefulTimeout.promise]);
|
|
return { mode: 'stdin-close', ...result, exitMs: Math.round(performance.now() - stopStartedAt) };
|
|
} catch {
|
|
this.child.kill('SIGTERM');
|
|
const terminateTimeout = timeoutAfter(graceMs, 'Pi RPC SIGTERM did not exit');
|
|
try {
|
|
const result = await Promise.race([this.exitResult, terminateTimeout.promise]);
|
|
return { mode: 'sigterm', ...result, exitMs: Math.round(performance.now() - stopStartedAt) };
|
|
} catch {
|
|
if (platform() === 'win32') {
|
|
await runCommand('taskkill.exe', ['/pid', String(this.child.pid), '/t', '/f']).catch(() => undefined);
|
|
} else {
|
|
this.child.kill('SIGKILL');
|
|
}
|
|
const result = await this.exitResult;
|
|
return {
|
|
mode: platform() === 'win32' ? 'taskkill' : 'sigkill',
|
|
...result,
|
|
exitMs: Math.round(performance.now() - stopStartedAt),
|
|
};
|
|
} finally {
|
|
terminateTimeout.cancel();
|
|
}
|
|
} finally {
|
|
gracefulTimeout.cancel();
|
|
}
|
|
}
|
|
}
|
|
|
|
async function getProcessRssKb(pid) {
|
|
if (!pid) return null;
|
|
try {
|
|
if (platform() === 'win32') {
|
|
const { stdout } = await runCommand('tasklist.exe', ['/fi', `PID eq ${pid}`, '/fo', 'csv', '/nh']);
|
|
const columns = stdout.trim().replace(/^"|"$/g, '').split('","');
|
|
const digits = columns.at(-1)?.replace(/[^0-9]/g, '');
|
|
return digits ? Number.parseInt(digits, 10) : null;
|
|
}
|
|
const { stdout } = await runCommand('ps', ['-o', 'rss=', '-p', String(pid)]);
|
|
return Number.parseInt(stdout.trim(), 10) || null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function baseRpcArgs(sessionDir, extraArgs = []) {
|
|
return [
|
|
'--mode', 'rpc',
|
|
'--offline',
|
|
'--session-dir', sessionDir,
|
|
'--no-extensions',
|
|
'--no-skills',
|
|
'--no-prompt-templates',
|
|
'--no-themes',
|
|
'--no-context-files',
|
|
'--no-approve',
|
|
'--no-tools',
|
|
...extraArgs,
|
|
];
|
|
}
|
|
|
|
async function makeWorkerPaths(root, prefix) {
|
|
const base = await mkdtemp(join(root, `${prefix}-`));
|
|
const configDir = join(base, 'config');
|
|
const sessionDir = join(base, 'sessions');
|
|
const cwd = join(base, 'project');
|
|
await Promise.all([mkdir(configDir), mkdir(sessionDir), mkdir(cwd)]);
|
|
return { base, configDir, sessionDir, cwd };
|
|
}
|
|
|
|
function workerEnv(configDir) {
|
|
return {
|
|
PI_CODING_AGENT_DIR: configDir,
|
|
PI_OFFLINE: '1',
|
|
PI_TELEMETRY: '0',
|
|
};
|
|
}
|
|
|
|
async function runReadySample(runtime, paths, timeoutMs) {
|
|
const worker = new PiRpcWorker({
|
|
executable: runtime.electronExecutable,
|
|
cliPath: runtime.cliPath,
|
|
cwd: paths.cwd,
|
|
env: workerEnv(paths.configDir),
|
|
args: baseRpcArgs(paths.sessionDir, ['--session-id', `probe-${Date.now()}-${Math.random()}`]),
|
|
timeoutMs,
|
|
});
|
|
try {
|
|
await worker.start();
|
|
const response = await worker.request({ type: 'get_state' });
|
|
const readyMs = response.receivedAt - worker.startedAt;
|
|
const workerSpawnMs = worker.spawnedAt - worker.startedAt;
|
|
const rpcReadyMs = response.receivedAt - worker.spawnedAt;
|
|
const rssKb = await getProcessRssKb(worker.child.pid);
|
|
const stop = await worker.stop();
|
|
return { readyMs, workerSpawnMs, rpcReadyMs, rssKb, stop, sessionId: response.data.sessionId };
|
|
} finally {
|
|
await worker.stop().catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
async function runPerformanceSamples(runtime, scratchRoot, sampleCount, timeoutMs) {
|
|
const cold = [];
|
|
for (let index = 0; index < sampleCount; index += 1) {
|
|
const paths = await makeWorkerPaths(scratchRoot, `cold-${index}`);
|
|
cold.push(await runReadySample(runtime, paths, timeoutMs));
|
|
}
|
|
|
|
const warmConfig = join(scratchRoot, 'warm-config');
|
|
await mkdir(warmConfig);
|
|
const warm = [];
|
|
for (let index = 0; index < sampleCount; index += 1) {
|
|
const paths = await makeWorkerPaths(scratchRoot, `warm-${index}`);
|
|
paths.configDir = warmConfig;
|
|
warm.push(await runReadySample(runtime, paths, timeoutMs));
|
|
}
|
|
|
|
return {
|
|
definition: {
|
|
cold: 'fresh Pi config, session, and project directories per process',
|
|
warm: 'shared primed Pi config directory with fresh session and project directories per process',
|
|
workerSpawn: 'final product executable spawn event minus spawn request',
|
|
rpcReady: 'first successful get_state response minus child spawn event',
|
|
},
|
|
coldReadyMs: summarizeMeasurements(cold.map((sample) => sample.readyMs)),
|
|
warmReadyMs: summarizeMeasurements(warm.map((sample) => sample.readyMs)),
|
|
coldWorkerSpawnMs: summarizeMeasurements(cold.map((sample) => sample.workerSpawnMs)),
|
|
warmWorkerSpawnMs: summarizeMeasurements(warm.map((sample) => sample.workerSpawnMs)),
|
|
coldRpcReadyMs: summarizeMeasurements(cold.map((sample) => sample.rpcReadyMs)),
|
|
warmRpcReadyMs: summarizeMeasurements(warm.map((sample) => sample.rpcReadyMs)),
|
|
rssKb: summarizeMeasurements([...cold, ...warm].flatMap((sample) => sample.rssKb == null ? [] : [sample.rssKb])),
|
|
exitMs: summarizeMeasurements([...cold, ...warm].map((sample) => sample.stop.exitMs)),
|
|
exitModes: [...new Set([...cold, ...warm].map((sample) => sample.stop.mode))],
|
|
};
|
|
}
|
|
|
|
async function runSessionLifecycle(runtime, scratchRoot, timeoutMs) {
|
|
const paths = await makeWorkerPaths(scratchRoot, 'session');
|
|
const failureFixture = {
|
|
id: 'makelore-deterministic-failure',
|
|
apiProtocol: 'openai-completions',
|
|
baseUrl: 'http://127.0.0.1:1/v1',
|
|
apiKeyEnv: 'PI_PROBE_FAILURE_KEY',
|
|
model: { id: 'pi-probe-failure-model' },
|
|
};
|
|
await writeFile(
|
|
join(paths.configDir, 'models.json'),
|
|
`${JSON.stringify(buildPiProviderConfig(failureFixture), null, 2)}\n`,
|
|
);
|
|
const common = {
|
|
executable: runtime.electronExecutable,
|
|
cliPath: runtime.cliPath,
|
|
cwd: paths.cwd,
|
|
env: { ...workerEnv(paths.configDir), PI_PROBE_FAILURE_KEY: 'qualification-only' },
|
|
timeoutMs,
|
|
};
|
|
const first = new PiRpcWorker({
|
|
...common,
|
|
args: baseRpcArgs(paths.sessionDir, [
|
|
'--session-id', `probe-session-${Date.now()}`,
|
|
'--provider', failureFixture.id,
|
|
'--model', failureFixture.model.id,
|
|
]),
|
|
});
|
|
let reopened;
|
|
try {
|
|
await first.start();
|
|
const initial = await first.request({ type: 'get_state' });
|
|
await first.request({ type: 'set_session_name', name: 'PI runtime qualification' });
|
|
const shell = await first.request({ type: 'bash', command: 'echo PI_RPC_SHELL_OK' });
|
|
await first.request({ type: 'abort' });
|
|
await first.request({ type: 'set_auto_retry', enabled: false });
|
|
const failureStartedAt = performance.now();
|
|
const failedTurn = await promptAndSettle(
|
|
first,
|
|
'This request must fail against the closed local probe endpoint.',
|
|
undefined,
|
|
timeoutMs,
|
|
);
|
|
const deterministicFailureMs = performance.now() - failureStartedAt;
|
|
const firstStop = await first.stop();
|
|
if (!initial.data.sessionFile || !await pathExists(initial.data.sessionFile)) {
|
|
throw new Error('Pi RPC session file was not persisted');
|
|
}
|
|
|
|
reopened = new PiRpcWorker({
|
|
...common,
|
|
args: baseRpcArgs(paths.sessionDir, [
|
|
'--session', initial.data.sessionFile,
|
|
'--provider', failureFixture.id,
|
|
'--model', failureFixture.model.id,
|
|
]),
|
|
});
|
|
await reopened.start();
|
|
const reopenedState = await reopened.request({ type: 'get_state' });
|
|
const entries = await reopened.request({ type: 'get_entries' });
|
|
const reopenedStop = await reopened.stop();
|
|
if (reopenedState.data.sessionId !== initial.data.sessionId) {
|
|
throw new Error('Pi RPC reopened a different session id');
|
|
}
|
|
|
|
return {
|
|
sessionIdPreserved: true,
|
|
sessionFilePersisted: true,
|
|
entryCount: entries.data.entries.length,
|
|
shellOutputMatched: shell.data.output.includes('PI_RPC_SHELL_OK'),
|
|
idleAbortAccepted: true,
|
|
promptAccepted: true,
|
|
agentSettled: true,
|
|
deterministicFailureStopReason: failedTurn.stopReason,
|
|
deterministicFailureMs: Math.round(deterministicFailureMs),
|
|
exitModes: [firstStop.mode, reopenedStop.mode],
|
|
};
|
|
} finally {
|
|
await Promise.all([
|
|
first.stop().catch(() => undefined),
|
|
reopened?.stop().catch(() => undefined),
|
|
]);
|
|
}
|
|
}
|
|
|
|
function longShellCommand(label, seconds) {
|
|
if (platform() === 'win32') return `ping 127.0.0.1 -n ${seconds + 1} > nul && echo ${label}`;
|
|
return `sleep ${seconds}; echo ${label}`;
|
|
}
|
|
|
|
async function runLocalWorkerIsolation(runtime, scratchRoot, timeoutMs) {
|
|
const [leftPaths, rightPaths] = await Promise.all([
|
|
makeWorkerPaths(scratchRoot, 'worker-left'),
|
|
makeWorkerPaths(scratchRoot, 'worker-right'),
|
|
]);
|
|
const create = (paths) => new PiRpcWorker({
|
|
executable: runtime.electronExecutable,
|
|
cliPath: runtime.cliPath,
|
|
cwd: paths.cwd,
|
|
env: workerEnv(paths.configDir),
|
|
args: baseRpcArgs(paths.sessionDir),
|
|
timeoutMs,
|
|
});
|
|
const left = create(leftPaths);
|
|
const right = create(rightPaths);
|
|
try {
|
|
await Promise.all([left.start(), right.start()]);
|
|
const [leftState, rightState] = await Promise.all([
|
|
left.request({ type: 'get_state' }),
|
|
right.request({ type: 'get_state' }),
|
|
]);
|
|
|
|
const startedAt = performance.now();
|
|
const leftBash = left.request({ type: 'bash', command: longShellCommand('PI_LEFT_ABORTED', 4) });
|
|
const rightBash = right.request({ type: 'bash', command: longShellCommand('PI_RIGHT_OK', 2) });
|
|
await new Promise((resolveWait) => setTimeout(resolveWait, 500));
|
|
await left.request({ type: 'abort_bash' });
|
|
const [leftResult, rightResult] = await Promise.all([leftBash, rightBash]);
|
|
const elapsedMs = performance.now() - startedAt;
|
|
const stops = await Promise.all([left.stop(), right.stop()]);
|
|
|
|
if (leftState.data.sessionId === rightState.data.sessionId) {
|
|
throw new Error('Independent Pi workers shared a session id');
|
|
}
|
|
if (!leftResult.data.cancelled) throw new Error('Aborted Pi bash command was not marked cancelled');
|
|
if (rightResult.data.cancelled || !rightResult.data.output.includes('PI_RIGHT_OK')) {
|
|
throw new Error('Aborting one Pi worker affected the other worker');
|
|
}
|
|
|
|
return {
|
|
distinctSessionIds: true,
|
|
abortIsolated: true,
|
|
overlappingLocalWork: elapsedMs < 3_500,
|
|
elapsedMs: Math.round(elapsedMs),
|
|
exitModes: stops.map((stop) => stop.mode),
|
|
limitation: 'This proves local worker and shell isolation, not overlapping real-provider turns.',
|
|
};
|
|
} finally {
|
|
await Promise.all([
|
|
left.stop().catch(() => undefined),
|
|
right.stop().catch(() => undefined),
|
|
]);
|
|
}
|
|
}
|
|
|
|
export function mapMakeloreProtocolToPi(protocol) {
|
|
if (!MAKELore_PROVIDER_PROTOCOLS.includes(protocol)) {
|
|
throw new Error(`Unsupported Makelore provider protocol: ${protocol}`);
|
|
}
|
|
if (protocol === 'openrouter') {
|
|
return {
|
|
api: 'openai-completions',
|
|
compat: { thinkingFormat: 'openrouter', sessionAffinityFormat: 'openrouter' },
|
|
};
|
|
}
|
|
return { api: protocol, compat: undefined };
|
|
}
|
|
|
|
export function buildPiProviderConfig(fixture) {
|
|
const mapping = mapMakeloreProtocolToPi(fixture.apiProtocol);
|
|
const model = fixture.model ?? {};
|
|
return {
|
|
providers: {
|
|
[fixture.id]: {
|
|
baseUrl: fixture.baseUrl,
|
|
apiKey: `$${fixture.apiKeyEnv}`,
|
|
api: mapping.api,
|
|
...(fixture.headers ? { headers: fixture.headers } : {}),
|
|
models: [{
|
|
id: model.id,
|
|
name: model.name ?? model.id,
|
|
reasoning: model.reasoning ?? false,
|
|
input: model.input ?? ['text'],
|
|
cost: model.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
contextWindow: model.contextWindow ?? 128_000,
|
|
maxTokens: model.maxTokens ?? 8_192,
|
|
...(mapping.compat ? { compat: mapping.compat } : {}),
|
|
}],
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
async function loadProviderFixture(path) {
|
|
const fixture = await readJson(path);
|
|
for (const field of ['id', 'apiProtocol', 'baseUrl', 'apiKeyEnv']) {
|
|
if (!fixture[field]) throw new Error(`Provider fixture is missing ${field}`);
|
|
}
|
|
if (!fixture.model?.id) throw new Error('Provider fixture is missing model.id');
|
|
mapMakeloreProtocolToPi(fixture.apiProtocol);
|
|
if (!process.env[fixture.apiKeyEnv]) {
|
|
throw new Error(`Provider credential environment variable is not set: ${fixture.apiKeyEnv}`);
|
|
}
|
|
return fixture;
|
|
}
|
|
|
|
async function imageContentFromPath(path) {
|
|
const data = await readFile(path);
|
|
const extension = path.toLowerCase().split('.').at(-1);
|
|
const mimeType = extension === 'png' ? 'image/png' : extension === 'webp' ? 'image/webp' : 'image/jpeg';
|
|
return { type: 'image', data: data.toString('base64'), mimeType };
|
|
}
|
|
|
|
function lastAssistantStopReason(messagesResponse) {
|
|
const messages = messagesResponse.data.messages;
|
|
return [...messages].reverse().find((message) => message.role === 'assistant')?.stopReason ?? null;
|
|
}
|
|
|
|
async function runProviderWorker(runtime, paths, fixture, timeoutMs) {
|
|
await writeFile(
|
|
join(paths.configDir, 'models.json'),
|
|
`${JSON.stringify(buildPiProviderConfig(fixture), null, 2)}\n`,
|
|
);
|
|
const worker = new PiRpcWorker({
|
|
executable: runtime.electronExecutable,
|
|
cliPath: runtime.cliPath,
|
|
cwd: paths.cwd,
|
|
env: workerEnv(paths.configDir),
|
|
args: baseRpcArgs(paths.sessionDir, ['--provider', fixture.id, '--model', fixture.model.id]),
|
|
timeoutMs,
|
|
});
|
|
await worker.start();
|
|
return worker;
|
|
}
|
|
|
|
async function promptAndSettle(worker, message, images, timeoutMs) {
|
|
const eventStartIndex = worker.events.length;
|
|
const sentAt = performance.now();
|
|
const accepted = await worker.request({ type: 'prompt', message, ...(images ? { images } : {}) });
|
|
const agentStart = await worker.waitForEvent(
|
|
(event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_start',
|
|
timeoutMs,
|
|
);
|
|
const settled = await worker.waitForEvent(
|
|
(event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_settled',
|
|
timeoutMs,
|
|
);
|
|
const firstProviderEvent = worker.events
|
|
.slice(eventStartIndex)
|
|
.find((event) => event !== agentStart && event.receivedAt >= agentStart.receivedAt);
|
|
const messagesResponse = await worker.request({ type: 'get_messages' });
|
|
return {
|
|
acceptedMs: accepted.receivedAt - sentAt,
|
|
agentStartMs: agentStart.receivedAt - accepted.receivedAt,
|
|
providerFirstEventMs: firstProviderEvent
|
|
? firstProviderEvent.receivedAt - agentStart.receivedAt
|
|
: settled.receivedAt - agentStart.receivedAt,
|
|
agentSettledMs: settled.receivedAt - accepted.receivedAt,
|
|
startedAt: agentStart.receivedAt,
|
|
settledAt: settled.receivedAt,
|
|
stopReason: lastAssistantStopReason(messagesResponse),
|
|
};
|
|
}
|
|
|
|
export async function runProviderQualification(runtime, scratchRoot, fixturePath, imagePath, timeoutMs) {
|
|
const fixture = await loadProviderFixture(fixturePath);
|
|
const [leftPaths, rightPaths] = await Promise.all([
|
|
makeWorkerPaths(scratchRoot, 'provider-left'),
|
|
makeWorkerPaths(scratchRoot, 'provider-right'),
|
|
]);
|
|
const [left, right] = await Promise.all([
|
|
runProviderWorker(runtime, leftPaths, fixture, timeoutMs),
|
|
runProviderWorker(runtime, rightPaths, fixture, timeoutMs),
|
|
]);
|
|
try {
|
|
const state = await Promise.all([
|
|
left.request({ type: 'get_state' }),
|
|
right.request({ type: 'get_state' }),
|
|
]);
|
|
await Promise.all([
|
|
left.request({ type: 'set_auto_retry', enabled: false }),
|
|
right.request({ type: 'set_auto_retry', enabled: false }),
|
|
]);
|
|
const images = imagePath ? [await imageContentFromPath(imagePath)] : undefined;
|
|
const [leftTurn, rightTurn] = await Promise.all([
|
|
promptAndSettle(left, 'Reply with exactly PI_PROVIDER_LEFT_OK. Do not use tools.', images, timeoutMs),
|
|
promptAndSettle(right, 'Reply with exactly PI_PROVIDER_RIGHT_OK. Do not use tools.', images, timeoutMs),
|
|
]);
|
|
const overlapMs = Math.min(leftTurn.settledAt, rightTurn.settledAt)
|
|
- Math.max(leftTurn.startedAt, rightTurn.startedAt);
|
|
const successfulReasons = new Set(['stop', 'length', 'toolUse']);
|
|
if (!successfulReasons.has(leftTurn.stopReason) || !successfulReasons.has(rightTurn.stopReason)) {
|
|
throw new Error(`Provider turns did not succeed: left=${leftTurn.stopReason}, right=${rightTurn.stopReason}`);
|
|
}
|
|
if (overlapMs <= 0) throw new Error('Provider turns did not overlap');
|
|
if (state[0].data.sessionId === state[1].data.sessionId) {
|
|
throw new Error('Provider workers shared a session id');
|
|
}
|
|
|
|
const [leftWarmTurn, rightWarmTurn] = await Promise.all([
|
|
promptAndSettle(left, 'Reply with exactly PI_PROVIDER_LEFT_WARM_OK. Do not use tools.', undefined, timeoutMs),
|
|
promptAndSettle(right, 'Reply with exactly PI_PROVIDER_RIGHT_WARM_OK. Do not use tools.', undefined, timeoutMs),
|
|
]);
|
|
if (!successfulReasons.has(leftWarmTurn.stopReason)
|
|
|| !successfulReasons.has(rightWarmTurn.stopReason)) {
|
|
throw new Error(
|
|
`Warm provider turns did not succeed: left=${leftWarmTurn.stopReason}, right=${rightWarmTurn.stopReason}`,
|
|
);
|
|
}
|
|
const warmOverlapMs = Math.min(leftWarmTurn.settledAt, rightWarmTurn.settledAt)
|
|
- Math.max(leftWarmTurn.startedAt, rightWarmTurn.startedAt);
|
|
if (warmOverlapMs <= 0) throw new Error('Warm provider turns did not overlap');
|
|
|
|
const abortStartIndex = left.events.length;
|
|
const abortedPrompt = left.request({
|
|
type: 'prompt',
|
|
message: 'Start a normal response. This turn will be aborted by the qualification probe.',
|
|
});
|
|
const unaffectedTurn = promptAndSettle(
|
|
right,
|
|
'Reply with exactly PI_PROVIDER_ABORT_ISOLATION_OK. Do not use tools.',
|
|
undefined,
|
|
timeoutMs,
|
|
);
|
|
await abortedPrompt;
|
|
await left.waitForEvent(
|
|
(event) => left.events.indexOf(event) >= abortStartIndex && event.type === 'agent_start',
|
|
timeoutMs,
|
|
);
|
|
await left.request({ type: 'abort' });
|
|
await left.waitForEvent(
|
|
(event) => left.events.indexOf(event) >= abortStartIndex && event.type === 'agent_settled',
|
|
timeoutMs,
|
|
);
|
|
const [abortedMessages, unaffected] = await Promise.all([
|
|
left.request({ type: 'get_messages' }),
|
|
unaffectedTurn,
|
|
]);
|
|
const abortedStopReason = lastAssistantStopReason(abortedMessages);
|
|
if (abortedStopReason !== 'aborted') {
|
|
throw new Error(`Aborted provider turn ended with ${abortedStopReason ?? 'no assistant message'}`);
|
|
}
|
|
if (!successfulReasons.has(unaffected.stopReason)) {
|
|
throw new Error(`Aborting one provider worker affected the other: ${unaffected.stopReason}`);
|
|
}
|
|
const stops = await Promise.all([left.stop(), right.stop()]);
|
|
|
|
return {
|
|
protocol: fixture.apiProtocol,
|
|
providerId: fixture.id,
|
|
modelId: fixture.model.id,
|
|
headerNames: Object.keys(fixture.headers ?? {}).sort(),
|
|
apiKeyEnv: fixture.apiKeyEnv,
|
|
imageInput: Boolean(imagePath),
|
|
distinctSessionIds: true,
|
|
cold: {
|
|
promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round),
|
|
promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]),
|
|
agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round),
|
|
agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]),
|
|
providerFirstEventSamplesMs: [
|
|
leftTurn.providerFirstEventMs,
|
|
rightTurn.providerFirstEventMs,
|
|
].map(Math.round),
|
|
providerFirstEventMs: summarizeMeasurements([
|
|
leftTurn.providerFirstEventMs,
|
|
rightTurn.providerFirstEventMs,
|
|
]),
|
|
agentSettledSamplesMs: [leftTurn.agentSettledMs, rightTurn.agentSettledMs].map(Math.round),
|
|
agentSettledMs: summarizeMeasurements([leftTurn.agentSettledMs, rightTurn.agentSettledMs]),
|
|
},
|
|
warm: {
|
|
promptAcceptedSamplesMs: [leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs].map(Math.round),
|
|
promptAcceptedMs: summarizeMeasurements([leftWarmTurn.acceptedMs, rightWarmTurn.acceptedMs]),
|
|
agentStartSamplesMs: [leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs].map(Math.round),
|
|
agentStartMs: summarizeMeasurements([leftWarmTurn.agentStartMs, rightWarmTurn.agentStartMs]),
|
|
providerFirstEventSamplesMs: [
|
|
leftWarmTurn.providerFirstEventMs,
|
|
rightWarmTurn.providerFirstEventMs,
|
|
].map(Math.round),
|
|
providerFirstEventMs: summarizeMeasurements([
|
|
leftWarmTurn.providerFirstEventMs,
|
|
rightWarmTurn.providerFirstEventMs,
|
|
]),
|
|
agentSettledSamplesMs: [leftWarmTurn.agentSettledMs, rightWarmTurn.agentSettledMs].map(Math.round),
|
|
agentSettledMs: summarizeMeasurements([
|
|
leftWarmTurn.agentSettledMs,
|
|
rightWarmTurn.agentSettledMs,
|
|
]),
|
|
},
|
|
overlapMs: Math.round(overlapMs),
|
|
warmOverlapMs: Math.round(warmOverlapMs),
|
|
stopReasons: [
|
|
leftTurn.stopReason,
|
|
rightTurn.stopReason,
|
|
leftWarmTurn.stopReason,
|
|
rightWarmTurn.stopReason,
|
|
],
|
|
abortIsolation: {
|
|
abortedStopReason,
|
|
unaffectedStopReason: unaffected.stopReason,
|
|
passed: true,
|
|
},
|
|
exitModes: stops.map((stop) => stop.mode),
|
|
};
|
|
} finally {
|
|
await Promise.all([
|
|
left.stop().catch(() => undefined),
|
|
right.stop().catch(() => undefined),
|
|
]);
|
|
}
|
|
}
|
|
|
|
function printHelp() {
|
|
process.stdout.write(`Usage: node scripts/probe-pi-runtime.mjs [options]\n\n`);
|
|
process.stdout.write(' --stage Install the package shrinkwrap into a temporary staged runtime\n');
|
|
process.stdout.write(' --keep-stage Preserve the temporary staged runtime for inspection\n');
|
|
process.stdout.write(' --samples <count> Cold and warm startup sample count (default: 5)\n');
|
|
process.stdout.write(' --timeout-ms <ms> RPC/provider timeout (default: 10000)\n');
|
|
process.stdout.write(' --report <path> Write the JSON report to this path\n');
|
|
process.stdout.write(' --provider-fixture <path> Opt in to a real-provider overlap qualification run\n');
|
|
process.stdout.write(' --image <path> Add an image to provider prompts (requires fixture)\n');
|
|
process.stdout.write(' --electron-executable <p> Use a packaged Electron executable (requires --cli-path)\n');
|
|
process.stdout.write(' --cli-path <path> Pi CLI path visible to packaged Electron\n');
|
|
process.stdout.write(' --artifact-label <label> Report label for a packaged runtime override\n');
|
|
}
|
|
|
|
export async function runProbe(options, projectRoot = process.cwd()) {
|
|
const sourceRuntime = await resolvePiRuntime(projectRoot);
|
|
const identity = await validatePiIdentity(sourceRuntime);
|
|
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-probe-'));
|
|
let stageRoot;
|
|
let runError;
|
|
try {
|
|
let runtime = sourceRuntime;
|
|
let closure = null;
|
|
if (options.stage) {
|
|
stageRoot = join(scratchRoot, 'staged-runtime');
|
|
const stage = await stagePiRuntime(sourceRuntime, stageRoot);
|
|
closure = {
|
|
...await validateStagedClosure(stageRoot),
|
|
stagingPreparation: {
|
|
omittedPublishedDevDependencies: stage.omittedDevDependencies,
|
|
reason: 'Published npm-shrinkwrap.json contains the production graph only.',
|
|
},
|
|
};
|
|
runtime = { ...sourceRuntime, packageRoot: stageRoot, cliPath: join(stageRoot, 'dist', 'cli.js') };
|
|
}
|
|
else if (options.electronExecutablePath) {
|
|
runtime = {
|
|
...sourceRuntime,
|
|
electronExecutable: options.electronExecutablePath,
|
|
cliPath: options.cliPath,
|
|
};
|
|
}
|
|
const electronVersions = await inspectElectronNodeRuntime(runtime.electronExecutable);
|
|
|
|
const performanceResult = await runPerformanceSamples(
|
|
runtime,
|
|
scratchRoot,
|
|
options.samples,
|
|
options.timeoutMs,
|
|
);
|
|
const session = await runSessionLifecycle(runtime, scratchRoot, options.timeoutMs);
|
|
const localIsolation = await runLocalWorkerIsolation(runtime, scratchRoot, options.timeoutMs);
|
|
const provider = options.providerFixturePath
|
|
? await runProviderQualification(
|
|
runtime,
|
|
scratchRoot,
|
|
options.providerFixturePath,
|
|
options.imagePath,
|
|
options.timeoutMs,
|
|
)
|
|
: null;
|
|
|
|
const coldBudgetPassed = performanceResult.coldReadyMs.p95 <= 3_000;
|
|
const warmBudgetPassed = performanceResult.warmReadyMs.p95 <= 1_500;
|
|
const localPassed = coldBudgetPassed
|
|
&& warmBudgetPassed
|
|
&& session.sessionIdPreserved
|
|
&& session.shellOutputMatched
|
|
&& localIsolation.abortIsolated
|
|
&& localIsolation.overlappingLocalWork;
|
|
const report = {
|
|
schemaVersion: 1,
|
|
generatedAt: new Date().toISOString(),
|
|
identity,
|
|
artifact: options.artifactLabel
|
|
?? (options.stage ? 'temporary-staged-production-closure' : 'workspace-install'),
|
|
platform: { platform: platform(), arch: arch(), release: release() },
|
|
electron: { executable: runtime.electronExecutable, ...electronVersions },
|
|
closure,
|
|
performance: performanceResult,
|
|
session,
|
|
localIsolation,
|
|
provider,
|
|
providerMatrix: MAKELore_PROVIDER_PROTOCOLS.map((protocol) => ({
|
|
protocol,
|
|
pi: mapMakeloreProtocolToPi(protocol),
|
|
realTurnVerified: provider?.protocol === protocol,
|
|
})),
|
|
budgets: { coldReadyPassed: coldBudgetPassed, warmReadyPassed: warmBudgetPassed },
|
|
result: localPassed ? 'partial-pass' : 'fail',
|
|
decision: 'incomplete',
|
|
missingEvidence: buildMissingEvidence(),
|
|
waivers: buildEvidenceWaivers(),
|
|
};
|
|
if (options.reportPath) {
|
|
await mkdir(dirname(options.reportPath), { recursive: true });
|
|
await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
return report;
|
|
} catch (error) {
|
|
runError = error;
|
|
throw error;
|
|
} finally {
|
|
if (!options.keepStage) {
|
|
try {
|
|
await rm(scratchRoot, {
|
|
recursive: true,
|
|
force: true,
|
|
maxRetries: 5,
|
|
retryDelay: 200,
|
|
});
|
|
} catch (cleanupError) {
|
|
if (!runError) throw cleanupError;
|
|
process.stderr.write(`Pi probe cleanup also failed: ${cleanupError.message}\n`);
|
|
}
|
|
}
|
|
else process.stderr.write(`Staged Pi runtime preserved at ${stageRoot ?? scratchRoot}\n`);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const options = parseProbeArgs(process.argv.slice(2));
|
|
if (options.help) {
|
|
printHelp();
|
|
return;
|
|
}
|
|
const report = await runProbe(options);
|
|
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
if (report.result === 'fail') process.exitCode = 1;
|
|
}
|
|
|
|
const isMain = process.argv[1]
|
|
&& pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
|
|
if (isMain) {
|
|
main().catch((error) => {
|
|
process.stderr.write(`${error.stack ?? error.message}\n`);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|