feat: add Pi process and RPC foundation

This commit is contained in:
2026-08-22 20:05:59 +08:00
parent b6f693048d
commit 9a31dacb2a
14 changed files with 1955 additions and 26 deletions

View File

@@ -0,0 +1,123 @@
import { realpathSync } from 'node:fs';
import { arch, platform } from 'node:os';
import { join, relative, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import {
PI_RUNTIME_PACKAGE,
defaultPiBundleTarget,
stagePiRuntimeBundle,
} from './lib/pi-runtime-bundle.mjs';
function parseTarget(value) {
const separator = value.lastIndexOf('-');
if (separator <= 0 || separator === value.length - 1) {
throw new Error(`Invalid Pi runtime target: ${value}`);
}
const target = {
platform: value.slice(0, separator),
arch: value.slice(separator + 1),
};
if (!['win32', 'darwin', 'linux'].includes(target.platform)) {
throw new Error(`Unsupported Pi runtime platform: ${target.platform}`);
}
if (!['x64', 'arm64'].includes(target.arch)) {
throw new Error(`Unsupported Pi runtime architecture: ${target.arch}`);
}
return target.platform === 'linux' ? { ...target, libc: 'glibc' } : target;
}
export function parseBundleArgs(argv) {
const options = { outputRoot: resolve('build/pi-runtime'), targets: [] };
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index];
if (argument === '--output') {
const value = argv[index + 1];
if (!value || value.startsWith('--')) throw new Error('--output requires a value');
options.outputRoot = resolve(value);
index += 1;
}
else if (argument === '--target') {
const value = argv[index + 1];
if (!value || value.startsWith('--')) throw new Error('--target requires a value');
options.targets.push(parseTarget(value));
index += 1;
}
else if (argument === '--release-targets') {
const currentPlatform = platform();
const arches = currentPlatform === 'win32' ? ['x64'] : ['x64', 'arm64'];
options.targets.push(...arches.map((targetArch) => parseTarget(`${currentPlatform}-${targetArch}`)));
}
else if (argument === '--help') options.help = true;
else throw new Error(`Unknown argument: ${argument}`);
}
if (options.targets.length === 0) options.targets.push(defaultPiBundleTarget());
const unique = new Map(options.targets.map((target) => [
`${target.platform}-${target.arch}`,
target,
]));
options.targets = [...unique.values()];
for (const target of options.targets) {
if (target.platform !== platform()) {
throw new Error(
`Pi runtime ${target.platform}-${target.arch} must be staged on ${target.platform}, current host is ${platform()}-${arch()}`,
);
}
}
return options;
}
function printHelp() {
process.stdout.write('Usage: node scripts/bundle-pi-runtime.mjs [options]\n\n');
process.stdout.write(' --target <platform-arch> Stage one runtime target (repeatable)\n');
process.stdout.write(' --release-targets Stage every configured architecture for the host OS\n');
process.stdout.write(' --output <directory> Output root (default: build/pi-runtime)\n');
}
export async function bundlePiRuntime(options, projectRoot = process.cwd()) {
const resolvedProjectRoot = resolve(projectRoot);
const sourcePackageRoot = realpathSync(join(
resolvedProjectRoot,
'node_modules',
...PI_RUNTIME_PACKAGE.split('/'),
));
const results = [];
for (const target of options.targets) {
const destination = join(options.outputRoot, `${target.platform}-${target.arch}`);
const destinationRelative = relative(options.outputRoot, destination);
if (destinationRelative.startsWith('..') || destinationRelative === '') {
throw new Error(`Unsafe Pi runtime destination: ${destination}`);
}
const manifest = await stagePiRuntimeBundle({
projectRoot: resolvedProjectRoot,
sourcePackageRoot,
destination,
target,
});
results.push({ destination, manifest });
}
return results;
}
async function main() {
const options = parseBundleArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const results = await bundlePiRuntime(options);
process.stdout.write(`${JSON.stringify(results.map(({ destination, manifest }) => ({
destination,
target: manifest.target,
packageCount: manifest.productionPackages.length,
assetCount: manifest.runtimeAssets.length,
})), null, 2)}\n`);
}
const isMain = process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
if (isMain) {
main().catch((error) => {
process.stderr.write(`${error.stack ?? error.message}\n`);
process.exitCode = 1;
});
}

View File

@@ -0,0 +1,301 @@
import { spawn } from 'node:child_process';
import { cp, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
import { realpathSync } from 'node:fs';
import { createRequire } from 'node:module';
import { arch as hostArch, platform as hostPlatform } from 'node:os';
import { dirname, join, relative, resolve, sep } from 'node:path';
export const PI_RUNTIME_PACKAGE = '@earendil-works/pi-coding-agent';
export const PI_RUNTIME_VERSION = '0.84.2';
export const PI_RUNTIME_CLI_ENTRY = 'dist/cli.js';
export const PI_RUNTIME_MANIFEST = 'makelore-pi-runtime.json';
export const PI_RUNTIME_STAGING_NPM_VERSION = '11.6.2';
function readJson(path) {
return readFile(path, 'utf8').then((source) => JSON.parse(source));
}
async function pathExists(path) {
try {
await stat(path);
return true;
} catch {
return false;
}
}
function portable(path) {
return path.split(sep).join('/');
}
function constraintMatches(constraints, current) {
if (!constraints || constraints.length === 0 || !current) return true;
const positive = constraints.filter((value) => !value.startsWith('!'));
const negative = constraints.filter((value) => value.startsWith('!')).map((value) => value.slice(1));
return !negative.includes(current) && (positive.length === 0 || positive.includes(current));
}
export function lockEntrySupportsTarget(entry, target) {
return constraintMatches(entry.os, target.platform)
&& constraintMatches(entry.cpu, target.arch)
&& constraintMatches(entry.libc, target.libc);
}
function packageNameFromLockPath(packagePath) {
const marker = packagePath.lastIndexOf('node_modules/');
if (marker === -1) return null;
const remainder = packagePath.slice(marker + 'node_modules/'.length).split('/');
return remainder[0]?.startsWith('@') ? remainder.slice(0, 2).join('/') : remainder[0];
}
function packageIdentity(name, version) {
return `${name}@${version}`;
}
export function requiredPackageIdentities(shrinkwrap, target) {
const identities = new Set();
let platformSkippedPackages = 0;
for (const [packagePath, entry] of Object.entries(shrinkwrap.packages ?? {})) {
if (!packagePath || entry.dev) continue;
if (!lockEntrySupportsTarget(entry, target)) {
platformSkippedPackages += 1;
continue;
}
const name = entry.name ?? packageNameFromLockPath(packagePath);
if (!name || !entry.version) {
throw new Error(`Pi shrinkwrap package identity is incomplete at ${packagePath}`);
}
identities.add(packageIdentity(name, entry.version));
}
return {
identities: [...identities].sort(),
platformSkippedPackages,
};
}
async function visitInstalledPackages(nodeModulesPath, identities) {
if (!await pathExists(nodeModulesPath)) return;
for (const entry of await readdir(nodeModulesPath, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name === '.bin') continue;
if (entry.name.startsWith('@')) {
const scopePath = join(nodeModulesPath, entry.name);
for (const scopedEntry of await readdir(scopePath, { withFileTypes: true })) {
if (scopedEntry.isDirectory()) {
await visitInstalledPackage(join(scopePath, scopedEntry.name), identities);
}
}
continue;
}
await visitInstalledPackage(join(nodeModulesPath, entry.name), identities);
}
}
async function visitInstalledPackage(packageRoot, identities) {
const packagePath = join(packageRoot, 'package.json');
if (!await pathExists(packagePath)) return;
const packageJson = await readJson(packagePath);
if (packageJson.name && packageJson.version) {
identities.add(packageIdentity(packageJson.name, packageJson.version));
}
await visitInstalledPackages(join(packageRoot, 'node_modules'), identities);
}
export async function installedPackageIdentities(stageRoot) {
const identities = new Set();
const rootPackage = await readJson(join(stageRoot, 'package.json'));
identities.add(packageIdentity(rootPackage.name, rootPackage.version));
await visitInstalledPackages(join(stageRoot, 'node_modules'), identities);
return [...identities].sort();
}
export function assertPackageIdentities(required, installed) {
const installedSet = new Set(installed);
const missing = required.filter((identity) => !installedSet.has(identity));
if (missing.length > 0) {
throw new Error(`Pi production closure is missing package identities: ${missing.join(', ')}`);
}
}
async function collectRuntimeAssets(stageRoot) {
const assets = [];
const visit = async (directory) => {
if (!await pathExists(directory)) return;
for (const entry of await readdir(directory, { withFileTypes: true })) {
const path = join(directory, entry.name);
if (entry.isDirectory()) await visit(path);
else if (entry.name.endsWith('.node') || entry.name.endsWith('.wasm')) {
assets.push(portable(relative(stageRoot, path)));
}
}
};
await visit(join(stageRoot, 'dist'));
await visit(join(stageRoot, 'node_modules'));
return assets.sort();
}
export async function preparePublishedPackageRoot(sourcePackageRoot, destination) {
await mkdir(dirname(destination), { recursive: true });
await rm(destination, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
await cp(sourcePackageRoot, destination, {
recursive: true,
filter(source) {
return relative(sourcePackageRoot, source).split(sep)[0] !== 'node_modules';
},
});
const packagePath = join(destination, 'package.json');
const packageJson = await readJson(packagePath);
const omittedDevDependencies = Object.keys(packageJson.devDependencies ?? {}).sort();
delete packageJson.devDependencies;
await writeFile(packagePath, `${JSON.stringify(packageJson, null, 2)}\n`);
return { omittedDevDependencies };
}
function runCommand(executable, args, options) {
return new Promise((resolvePromise, reject) => {
const child = spawn(executable, args, {
cwd: options.cwd,
env: options.env,
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
child.once('error', reject);
child.once('exit', (code, signal) => {
if (code === 0) resolvePromise({ stdout, stderr });
else {
reject(new Error(
`${executable} exited with code ${code ?? 'null'} signal ${signal ?? 'none'}: ${stderr || stdout}`,
));
}
});
});
}
export function resolvePinnedNpmCli(projectRoot) {
const requireFromProject = createRequire(join(resolve(projectRoot), 'package.json'));
return join(realpathSync(dirname(requireFromProject.resolve('npm/package.json'))), 'bin', 'npm-cli.js');
}
export async function validatePinnedNpm(projectRoot) {
const resolvedRoot = resolve(projectRoot);
const requireFromProject = createRequire(join(resolvedRoot, 'package.json'));
const projectPackage = await readJson(join(resolvedRoot, 'package.json'));
const npmPackagePath = requireFromProject.resolve('npm/package.json');
const npmPackage = await readJson(npmPackagePath);
if (projectPackage.dependencies?.npm !== PI_RUNTIME_STAGING_NPM_VERSION) {
throw new Error(`Root npm dependency must be exactly ${PI_RUNTIME_STAGING_NPM_VERSION}`);
}
if (npmPackage.version !== PI_RUNTIME_STAGING_NPM_VERSION) {
throw new Error(
`Installed npm must be ${PI_RUNTIME_STAGING_NPM_VERSION}, got ${npmPackage.version ?? 'missing'}`,
);
}
return npmPackagePath;
}
export async function installProductionShrinkwrap({
projectRoot,
stageRoot,
target,
npmCli = resolvePinnedNpmCli(projectRoot),
}) {
await validatePinnedNpm(projectRoot);
return await runCommand(process.execPath, [
npmCli,
'ci',
'--omit=dev',
'--ignore-scripts',
'--no-audit',
'--no-fund',
`--os=${target.platform}`,
`--cpu=${target.arch}`,
], {
cwd: stageRoot,
env: {
...process.env,
npm_config_update_notifier: 'false',
},
});
}
export async function createPiRuntimeManifest(stageRoot, target, omittedDevDependencies) {
const rootPackage = await readJson(join(stageRoot, 'package.json'));
const shrinkwrap = await readJson(join(stageRoot, 'npm-shrinkwrap.json'));
if (rootPackage.name !== PI_RUNTIME_PACKAGE || rootPackage.version !== PI_RUNTIME_VERSION) {
throw new Error(
`Pi runtime must be ${PI_RUNTIME_PACKAGE}@${PI_RUNTIME_VERSION}, got ${rootPackage.name}@${rootPackage.version}`,
);
}
if (rootPackage.bin?.pi !== PI_RUNTIME_CLI_ENTRY) {
throw new Error(`Pi CLI entry must be ${PI_RUNTIME_CLI_ENTRY}`);
}
if (!await pathExists(join(stageRoot, ...PI_RUNTIME_CLI_ENTRY.split('/')))) {
throw new Error(`Pi CLI is missing at ${PI_RUNTIME_CLI_ENTRY}`);
}
const required = requiredPackageIdentities(shrinkwrap, target);
const installed = await installedPackageIdentities(stageRoot);
assertPackageIdentities(required.identities, installed);
const assets = await collectRuntimeAssets(stageRoot);
const requiredPhotonAsset = 'node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm';
if (!assets.includes(requiredPhotonAsset)) {
throw new Error(`Pi runtime asset is missing: ${requiredPhotonAsset}`);
}
return {
schemaVersion: 1,
runtime: {
packageName: PI_RUNTIME_PACKAGE,
version: PI_RUNTIME_VERSION,
cliEntry: PI_RUNTIME_CLI_ENTRY,
nodeEngine: rootPackage.engines?.node ?? null,
},
target: {
platform: target.platform,
arch: target.arch,
...(target.libc ? { libc: target.libc } : {}),
},
lockfileVersion: shrinkwrap.lockfileVersion,
productionPackages: required.identities,
platformSkippedPackages: required.platformSkippedPackages,
runtimeAssets: assets,
staging: {
npmVersion: PI_RUNTIME_STAGING_NPM_VERSION,
omitDev: true,
ignoreScripts: true,
omittedPublishedDevDependencies: [...omittedDevDependencies].sort(),
},
};
}
export async function stagePiRuntimeBundle({
projectRoot,
sourcePackageRoot,
destination,
target,
}) {
const prepared = await preparePublishedPackageRoot(sourcePackageRoot, destination);
await installProductionShrinkwrap({ projectRoot, stageRoot: destination, target });
const manifest = await createPiRuntimeManifest(
destination,
target,
prepared.omittedDevDependencies,
);
await writeFile(
join(destination, PI_RUNTIME_MANIFEST),
`${JSON.stringify(manifest, null, 2)}\n`,
);
return manifest;
}
export function defaultPiBundleTarget() {
return {
platform: hostPlatform(),
arch: hostArch(),
...(hostPlatform() === 'linux' ? { libc: 'glibc' } : {}),
};
}

View File

@@ -8,13 +8,16 @@ import {
parseProbeArgs,
resolvePiRuntime,
runProbe,
stagePiRuntime,
validatePiIdentity,
validateStagedClosure,
} from './probe-pi-runtime.mjs';
import {
PI_RUNTIME_MANIFEST,
defaultPiBundleTarget,
stagePiRuntimeBundle,
} from './lib/pi-runtime-bundle.mjs';
const PRODUCT_NAME = 'MakelorePiProbe';
const ARTIFACT_LABEL = 'controlled-electron-builder-dir-app-asar';
const ARTIFACT_LABEL = 'controlled-electron-builder-dir-extra-resources';
async function pathExists(path) {
try {
@@ -92,9 +95,12 @@ async function inspectPackagedClosure(executable, resourcesDirectory, assets) {
const path = require('node:path');
const resources = process.env.PI_PROBE_RESOURCES_DIRECTORY;
if (!resources) throw new Error('PI_PROBE_RESOURCES_DIRECTORY is required');
const root = path.join(resources, 'app.asar');
const unpackedRoot = path.join(resources, 'app.asar.unpacked');
const root = path.join(resources, 'pi-runtime');
const packageJson = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
const runtimeManifestPath = path.join(root, '${PI_RUNTIME_MANIFEST}');
const runtimeManifest = fs.existsSync(runtimeManifestPath)
? JSON.parse(fs.readFileSync(runtimeManifestPath, 'utf8'))
: null;
const shrinkwrap = JSON.parse(fs.readFileSync(path.join(root, 'npm-shrinkwrap.json'), 'utf8'));
const assets = JSON.parse(process.env.PI_PROBE_ASSETS_JSON);
const libc = process.platform === 'linux'
@@ -140,12 +146,11 @@ async function inspectPackagedClosure(executable, resourcesDirectory, assets) {
missingPackages.push(packagePath);
}
const missingAssets = assets.filter((asset) => !fs.existsSync(path.join(root, ...asset.split('/'))));
const missingUnpackedNativeAssets = assets
.filter((asset) => asset.endsWith('.node'))
.filter((asset) => !fs.existsSync(path.join(unpackedRoot, ...asset.split('/'))));
process.stdout.write(JSON.stringify({
packageName: packageJson.name,
packageVersion: packageJson.version,
runtimeManifestExists: Boolean(runtimeManifest),
runtimeManifestTarget: runtimeManifest?.target ?? null,
cliExists: fs.existsSync(path.join(root, 'dist', 'cli.js')),
lockedPackages: Math.max(0, Object.keys(shrinkwrap.packages || {}).length - 1),
expectedPackages,
@@ -154,7 +159,7 @@ async function inspectPackagedClosure(executable, resourcesDirectory, assets) {
relocatedPackages,
assetCount: assets.length,
missingAssets,
missingUnpackedNativeAssets,
nativeAssetsOutsideAsar: assets.filter((asset) => asset.endsWith('.node')).length,
}));
`;
const { stdout } = await runCommand(executable, ['-e', script], {
@@ -168,9 +173,9 @@ async function inspectPackagedClosure(executable, resourcesDirectory, assets) {
const result = JSON.parse(stdout.trim());
if (
!result.cliExists
|| !result.runtimeManifestExists
|| result.missingPackages.length > 0
|| result.missingAssets.length > 0
|| result.missingUnpackedNativeAssets.length > 0
) {
throw new Error(`Packaged Pi closure is incomplete: ${JSON.stringify(result)}`);
}
@@ -187,14 +192,20 @@ function assertControlledOutput(projectRoot, outputDirectory) {
}
}
async function buildControlledArtifact(projectRoot, appDirectory, outputDirectory) {
async function buildControlledArtifact(projectRoot, appDirectory, runtimeDirectory, outputDirectory) {
const electronPackage = JSON.parse(await readFile(join(projectRoot, 'node_modules', 'electron', 'package.json'), 'utf8'));
const { platformTarget, architecture } = currentTarget();
const buildResources = join(appDirectory, 'build-resources');
await mkdir(buildResources);
await mkdir(buildResources, { recursive: true });
await writeFile(join(appDirectory, 'package.json'), `${JSON.stringify({
name: 'makelore-pi-production-shape-probe',
version: '1.0.0',
main: 'main.js',
}, null, 2)}\n`);
await writeFile(join(appDirectory, 'main.js'), 'process.exit(0);\n');
const builderArtifacts = await build({
// Use the staged Pi root as the project root so this controlled probe does
// not inherit Makelore's production hooks or unrelated extraResources.
// Keep the probe application minimal while copying the permanent stage to
// the same resources/pi-runtime location used by the product configuration.
projectDir: appDirectory,
targets: platformTarget.createTarget('dir', architecture),
publish: 'never',
@@ -208,13 +219,22 @@ async function buildControlledArtifact(projectRoot, appDirectory, outputDirector
buildResources,
},
files: [
'dist/**/*',
'main.js',
'package.json',
'npm-shrinkwrap.json',
'node_modules/**/*',
],
extraResources: [
{
from: runtimeDirectory,
to: 'pi-runtime',
filter: ['**/*', '!node_modules{,/**/*}'],
},
{
from: join(runtimeDirectory, 'node_modules'),
to: 'pi-runtime/node_modules',
filter: ['**/*'],
},
],
asar: true,
asarUnpack: ['**/*.node'],
npmRebuild: false,
win: {
executableName: PRODUCT_NAME,
@@ -239,22 +259,37 @@ export async function runPackagedProbe(options, projectRoot = process.cwd()) {
assertControlledOutput(resolvedProjectRoot, outputDirectory);
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-packaged-probe-'));
const appDirectory = join(scratchRoot, 'app');
const runtimeDirectory = join(scratchRoot, 'runtime');
try {
const sourceRuntime = await resolvePiRuntime(resolvedProjectRoot);
const identity = await validatePiIdentity(sourceRuntime);
await stagePiRuntime(sourceRuntime, appDirectory);
const stagedClosure = await validateStagedClosure(appDirectory);
const runtimeManifest = await stagePiRuntimeBundle({
projectRoot: resolvedProjectRoot,
sourcePackageRoot: sourceRuntime.packageRoot,
destination: runtimeDirectory,
target: defaultPiBundleTarget(),
});
const stagedClosure = {
lockfileVersion: runtimeManifest.lockfileVersion,
expectedPackages: runtimeManifest.productionPackages.length,
platformSkippedPackages: runtimeManifest.platformSkippedPackages,
assets: runtimeManifest.runtimeAssets,
omittedPublishedDevDependencies: runtimeManifest.staging.omittedPublishedDevDependencies,
manifest: runtimeManifest,
};
await rm(outputDirectory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
await mkdir(dirname(outputDirectory), { recursive: true });
const builderArtifacts = await buildControlledArtifact(
resolvedProjectRoot,
appDirectory,
runtimeDirectory,
outputDirectory,
);
const executable = await findPackagedExecutable(outputDirectory);
const resourcesDirectory = packagedResourcesDirectory(executable);
const appAsar = join(resourcesDirectory, 'app.asar');
const cliPath = join(appAsar, 'dist', 'cli.js');
const packagedRuntime = join(resourcesDirectory, 'pi-runtime');
const cliPath = join(packagedRuntime, 'dist', 'cli.js');
if (!await pathExists(appAsar)) throw new Error(`Packaged app.asar is missing: ${appAsar}`);
const packagedClosure = await inspectPackagedClosure(
executable,
@@ -279,7 +314,7 @@ export async function runPackagedProbe(options, projectRoot = process.cwd()) {
outputDirectory,
executable,
appAsar,
appAsarUnpacked: join(resourcesDirectory, 'app.asar.unpacked'),
runtimeRoot: packagedRuntime,
builderArtifacts,
},
stagedClosure,
@@ -317,6 +352,6 @@ const isMain = process.argv[1]
if (isMain) {
main().catch((error) => {
process.stderr.write(`${error.stack ?? error.message}\n`);
process.exitCode = 1;
process.exit(1);
});
}