fix(coding): log sanitized Pi initialization failures

This commit is contained in:
鲨鱼辣椒 committed 2026-10-14 14:50:49 +08:00
1 parent e3e26b6bd6
commit 5da90e4a06
6 files changed
+815 -7

No files matched your search

@@ -0,0 +1,267 @@
import { createHash } from 'node:crypto';
import { sanitizePiDiagnostic } from './worker-process';
const STAGES = new Set([
'registry', 'provider-input', 'provider-catalog', 'provider-credentials',
'worker-resources', 'device-resources', 'resource-materialization',
'extension-registration', 'worker-start', 'session-bind', 'initial-hydration',
'snapshot', 'prepare',
]);
const EXCEPTIONS = new Set([
'Error', 'EvalError', 'RangeError', 'ReferenceError', 'SyntaxError', 'TypeError',
'URIError', 'AggregateError', 'AbortError', 'PiProcessError',
'CodingRuntimeContractError', 'PiProviderConfigError', 'PiSessionProjectionError',
'CodingConversationServiceError', 'CodingProjectServiceError',
]);
const CODES = new Set([
'ENOENT', 'ENOTDIR', 'EACCES', 'EPERM', 'EINVAL', 'EEXIST', 'EMFILE', 'ENFILE',
'ENOSPC', 'EROFS', 'EIO', 'ENAMETOOLONG', 'ETIMEDOUT', 'EPIPE', 'ECONNRESET',
'ECONNREFUSED', 'MODULE_NOT_FOUND', 'ERR_MODULE_NOT_FOUND', 'ERR_REQUIRE_ESM',
'ERR_PACKAGE_PATH_NOT_EXPORTED', 'ERR_INVALID_ARG_TYPE', 'ERR_INVALID_URL',
'ERR_IMPORT_ATTRIBUTE_MISSING', 'ERR_UNKNOWN_FILE_EXTENSION',
'ERR_DLOPEN_FAILED', 'ERR_WORKER_INIT_FAILED', 'ABORT_ERR',
'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_SOCKET',
'PI_RPC_PROTOCOL_ERROR', 'PI_RPC_TIMEOUT', 'PI_RPC_ABORTED',
'PI_RPC_RESPONSE_ERROR', 'PI_RPC_WRITE_FAILED', 'PI_RPC_EXITED',
'PI_WORKER_START_FAILED', 'PI_WORKER_STOPPED', 'PI_WORKER_STOP_FAILED',
'CODING_RUNTIME_START_FAILED', 'CODING_RUNTIME_READY_TIMEOUT',
'CODING_RUNTIME_PROTOCOL_ERROR', 'CODING_RUNTIME_UNAVAILABLE',
'CODING_CONVERSATION_NOT_FOUND', 'CODING_SESSION_UNREADABLE',
'CODING_STORAGE_WRITE_FAILED', 'CODING_MIGRATION_MODEL_REQUIRED',
'CODING_MODEL_UNAVAILABLE', 'CODING_PROVIDER_AUTH_REQUIRED',
'CODING_AGENT_NOT_FOUND', 'CODING_REQUEST_UNCERTAIN',
'PROVIDER_INVALID', 'PROVIDER_AUTH_REQUIRED', 'MODEL_UNAVAILABLE',
]);
const MAX_BYTES = 4_096;
const MAX_TEXT_BYTES = 32_768;
const MAX_FRAME_BYTES = 160;
const MAX_FRAMES = 6;
const MAX_CAUSE_DEPTH = 2;
const REDACTED_FRAME = '[REDACTED_FRAME]';
const MAX_RPC_TAIL_BYTES = 2_048;
// Server control commands plus PiRpcClient's read-only initialization commands.
const RPC_OPERATIONS = [
'server_initialize', 'thread_open', 'thread_close', 'server_shutdown',
'get_available_models', 'get_available_thinking_levels', 'get_commands',
'get_entries', 'get_fork_messages', 'get_last_assistant_text', 'get_messages',
'get_session_stats', 'get_state', 'get_tree',
] as const;
export interface PiInitializationDiagnostic {
version: 1;
stage: string;
exception: string;
code?: string;
category: string;
message: string;
fingerprint?: string;
rpcOperation?: (typeof RPC_OPERATIONS)[number];
rpcFailureSignature?: 'undefined-property' | 'null-property' | 'not-callable' | 'module-not-found' | 'missing-local-resource';
frames?: string[];
causes?: PiInitializationDiagnostic[];
truncated?: true;
}
export interface PiInitializationDiagnosticOptions {
stage: string;
sensitiveValues?: readonly string[];
/** Bundled application/package roots only; never user project roots. */
trustedRoots?: readonly string[];
}
function own(value: unknown, key: string): PropertyDescriptor | undefined {
if ((typeof value !== 'object' || value === null) && typeof value !== 'function') return undefined;
try { return Object.getOwnPropertyDescriptor(value, key); } catch { return undefined; }
}
function data(value: unknown, key: string): unknown {
const descriptor = own(value, key);
return descriptor && 'value' in descriptor ? descriptor.value : undefined;
}
function strings(value: unknown): string[] {
try {
if (!Array.isArray(value)) return [];
const length = data(value, 'length');
if (typeof length !== 'number' || !Number.isSafeInteger(length)) return [];
const result: string[] = [];
for (let index = 0; index < Math.min(length, 128); index++) {
const item = data(value, String(index));
if (typeof item === 'string' && item && item.length <= MAX_TEXT_BYTES && Buffer.byteLength(item) <= MAX_TEXT_BYTES) result.push(item);
}
return result;
} catch { return []; }
}
function exceptionName(error: unknown): string {
let value = error;
for (let depth = 0; depth < 3; depth++) {
const name = data(value, 'name');
if (typeof name === 'string') return EXCEPTIONS.has(name) ? name : 'UnknownError';
if (own(value, 'name')) return 'UnknownError';
try {
if ((typeof value !== 'object' || value === null) && typeof value !== 'function') break;
value = Object.getPrototypeOf(value);
} catch { break; }
}
return 'UnknownError';
}
function knownCode(error: unknown): string | undefined {
const direct = data(error, 'code');
if (typeof direct === 'string' && CODES.has(direct)) return direct;
const publicCode = data(data(error, 'publicError'), 'code');
return typeof publicCode === 'string' && CODES.has(publicCode) ? publicCode : undefined;
}
function rpcProjection(error: unknown): Pick<PiInitializationDiagnostic, 'rpcOperation' | 'rpcFailureSignature'> {
if (data(error, 'code') !== 'PI_RPC_RESPONSE_ERROR') return {};
const message = data(error, 'message');
if (typeof message !== 'string') return {};
// Inspect a bounded prefix first; never scan/copy an unbounded response tail.
const wrapper = message.slice(0, 128).match(/^Pi RPC ([a-z_]{1,48}) failed: /);
const operation = wrapper ? RPC_OPERATIONS.find(candidate => candidate === wrapper[1]) : undefined;
if (!wrapper || !operation) return {};
const result: Pick<PiInitializationDiagnostic, 'rpcOperation' | 'rpcFailureSignature'> = { rpcOperation: operation };
if (message.length - wrapper[0].length > MAX_RPC_TAIL_BYTES) return result;
const tail = message.slice(wrapper[0].length);
if (Buffer.byteLength(tail) > MAX_RPC_TAIL_BYTES) return result;
// Match known runtime shapes, but never retain their private property names,
// function expressions, package names, paths, URLs or other response suffixes.
const property = tail.match(/^(?:TypeError: )?Cannot read properties of (undefined|null) \(reading (?:'[^'\r\n]{1,256}'|"[^"\r\n]{1,256}")\)$/);
if (property) result.rpcFailureSignature = property[1] === 'undefined' ? 'undefined-property' : 'null-property';
else if (/^(?:TypeError: )?[^\r\n]{1,256} is not a function$/.test(tail)) result.rpcFailureSignature = 'not-callable';
else if (/^(?:Error: )?Cannot find module '[^'\r\n]{1,512}'(?:\nRequire stack:(?:\n- [^\r\n]{1,512}){1,8})?$/.test(tail)
|| /^(?:Error: )?Cannot find package '[^'\r\n]{1,256}' imported from [^\r\n]{1,1024}$/.test(tail)) result.rpcFailureSignature = 'module-not-found';
else if (/^(?:Error: )?ENOENT: no such file or directory(?:, (?:open|access|stat|lstat|scandir|readdir|realpath|readlink) '[^'\r\n]{1,1024}')?$/.test(tail)) result.rpcFailureSignature = 'missing-local-resource';
return result;
}
function classify(exception: string, code: string | undefined): Pick<PiInitializationDiagnostic, 'category' | 'message'> {
if (code && ['MODULE_NOT_FOUND', 'ERR_MODULE_NOT_FOUND', 'ERR_REQUIRE_ESM', 'ERR_PACKAGE_PATH_NOT_EXPORTED', 'ERR_UNKNOWN_FILE_EXTENSION', 'ERR_DLOPEN_FAILED'].includes(code))
return { category: 'module-loading', message: 'A required runtime module could not be loaded.' };
if (code && ['ENOENT', 'ENOTDIR', 'EACCES', 'EPERM', 'EINVAL', 'EEXIST', 'EMFILE', 'ENFILE', 'ENOSPC', 'EROFS', 'EIO', 'ENAMETOOLONG', 'CODING_STORAGE_WRITE_FAILED'].includes(code))
return { category: 'local-storage', message: 'A local runtime resource could not be read or written.' };
if (code && ['PROVIDER_AUTH_REQUIRED', 'CODING_PROVIDER_AUTH_REQUIRED'].includes(code))
return { category: 'provider-authentication', message: 'Provider authentication preparation failed.' };
if (code && ['PROVIDER_INVALID', 'MODEL_UNAVAILABLE', 'CODING_MODEL_UNAVAILABLE', 'CODING_MIGRATION_MODEL_REQUIRED'].includes(code))
return { category: 'provider-configuration', message: 'The runtime model configuration could not be prepared.' };
if (code && ['PI_RPC_TIMEOUT', 'CODING_RUNTIME_READY_TIMEOUT', 'ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT'].includes(code))
return { category: 'initialization-timeout', message: 'Runtime preparation or communication timed out.' };
if (code && ['PI_RPC_PROTOCOL_ERROR', 'PI_RPC_RESPONSE_ERROR', 'PI_RPC_WRITE_FAILED', 'CODING_RUNTIME_PROTOCOL_ERROR'].includes(code))
return { category: 'worker-protocol', message: 'The runtime worker communication failed.' };
if (code && ['PI_RPC_EXITED', 'PI_WORKER_START_FAILED', 'PI_WORKER_STOPPED', 'PI_WORKER_STOP_FAILED', 'CODING_RUNTIME_START_FAILED', 'ERR_WORKER_INIT_FAILED'].includes(code))
return { category: 'worker-lifecycle', message: 'The runtime worker could not start or remain available.' };
if (code === 'CODING_SESSION_UNREADABLE')
return { category: 'session-projection', message: 'Stored runtime session data could not be projected.' };
if (exception === 'TypeError' || exception === 'ReferenceError' || code === 'ERR_INVALID_ARG_TYPE')
return { category: 'runtime-contract', message: 'Runtime initialization encountered a type or reference mismatch.' };
if (exception === 'SyntaxError')
return { category: 'invalid-runtime-data', message: 'Runtime initialization encountered invalid syntax or data.' };
return { category: 'unclassified-initialization', message: 'Runtime initialization failed without a recognized error classification.' };
}
function secretAliases(values: string[]): string[] {
const result = new Set<string>();
for (const value of values) {
result.add(value);
try {
result.add(encodeURIComponent(value));
result.add(encodeURI(value));
result.add(encodeURIComponent(value).replace(/%20/g, '+'));
} catch { /* Invalid UTF-16 is still covered by its literal value. */ }
result.add(JSON.stringify(value).slice(1, -1));
result.add(Buffer.from(value).toString('base64'));
result.add(Buffer.from(value).toString('base64url'));
}
return [...result];
}
function decodedLocation(value: string): string | null {
if (Buffer.byteLength(value) > 2_048) return null;
let result = value;
try {
for (let depth = 0; depth < 2; depth++) {
const next = decodeURIComponent(result);
if (next === result) break;
result = next;
}
} catch { return null; }
if (result.startsWith('file:///')) result = result.slice(7);
else if (result.startsWith('file://localhost/')) result = result.slice(16);
else if (result.includes('://')) return null;
if (/^\/[A-Za-z]:\//.test(result)) result = result.slice(1);
result = result.replace(/\\/g, '/');
if (result.split('/').some(segment => segment === '..' || segment === '.')) return null;
return result;
}
function frameLocation(line: string, roots: string[], sensitive: string[]): string {
const match = line.match(/^\s*at\s+(?:.*?\()?([^()\r\n]+):(\d{1,9}):(\d{1,9})\)?\s*$/);
if (!match) return REDACTED_FRAME;
const location = decodedLocation(match[1]);
if (!location) return REDACTED_FRAME;
let relative: string | undefined;
if (/^node:internal\/[A-Za-z0-9_./-]+$/.test(location)) relative = location;
else {
for (let index = 0; index < roots.length; index++) {
const root = decodedLocation(roots[index])?.replace(/\/+$/, '');
if (!root || !/^(?:\/|[A-Za-z]:\/)/.test(root)) continue;
const windows = /^[A-Za-z]:\//.test(root);
const candidate = windows ? location.toLowerCase() : location;
const prefix = windows ? root.toLowerCase() : root;
if (candidate.startsWith(prefix + '/')) {
const suffix = location.slice(root.length + 1);
// Drop function names and absolute roots; retain only code locations.
if (/^[A-Za-z0-9_@+.,$-]+(?:\/[A-Za-z0-9_@+.,$-]+)*\.(?:[cm]?[jt]sx?|mjs|cjs)$/.test(suffix)) relative = `root${index}/${suffix}`;
break;
}
}
}
if (!relative || /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/.test(relative)) return REDACTED_FRAME;
if (sanitizePiDiagnostic(relative, sensitive) !== relative) return REDACTED_FRAME;
const frame = `${relative}:${match[2]}:${match[3]}`;
return Buffer.byteLength(frame) <= MAX_FRAME_BYTES ? frame : REDACTED_FRAME;
}
/** Produce bounded technical evidence only; arbitrary exception text is never logged. */
export function buildPiInitializationDiagnostic(
error: unknown,
options: PiInitializationDiagnosticOptions,
): PiInitializationDiagnostic {
const requestedStage = data(options, 'stage');
const stage = typeof requestedStage === 'string' && STAGES.has(requestedStage) ? requestedStage : 'unknown';
const roots = strings(data(options, 'trustedRoots'));
const sensitive = secretAliases(strings(data(options, 'sensitiveValues')));
const seen = new Set<unknown>();
const build = (value: unknown, depth: number): PiInitializationDiagnostic => {
const exception = exceptionName(value);
const code = knownCode(value);
const result: PiInitializationDiagnostic = {
version: 1, stage, exception, ...(code ? { code } : {}), ...classify(exception, code), ...rpcProjection(value),
};
const message = typeof value === 'string' ? value : data(value, 'message');
if (typeof message === 'string' && message) {
const bytes = Buffer.from(message.slice(0, MAX_TEXT_BYTES));
result.fingerprint = createHash('sha256').update(bytes.subarray(0, MAX_TEXT_BYTES)).digest('hex').slice(0, 16);
if (message.length > MAX_TEXT_BYTES || bytes.length > MAX_TEXT_BYTES) result.truncated = true;
}
// Even native lazy stack accessors can call global formatting hooks. Only
// inspect an already available string data property; never invoke a getter.
const stack = data(value, 'stack');
if (typeof stack === 'string') {
const bytes = Buffer.from(stack.slice(0, MAX_TEXT_BYTES));
const bounded = bytes.subarray(0, MAX_TEXT_BYTES).toString('utf8');
const lines = bounded.split(/\r?\n/).filter(line => /^\s*at\s/.test(line));
result.frames = lines.slice(0, MAX_FRAMES).map(line => frameLocation(line, roots, sensitive));
if (lines.length > MAX_FRAMES || stack.length > MAX_TEXT_BYTES || bytes.length > MAX_TEXT_BYTES) result.truncated = true;
}
seen.add(value);
const cause = data(value, 'cause');
if (cause !== undefined) {
if (depth < MAX_CAUSE_DEPTH && !seen.has(cause)) result.causes = [build(cause, depth + 1)];
else result.truncated = true;
}
return result;
};
const diagnostic = build(error, 0);
const nodes = [diagnostic, ...(diagnostic.causes ?? []), ...(diagnostic.causes?.[0]?.causes ?? [])];
while (Buffer.byteLength(JSON.stringify(diagnostic)) > MAX_BYTES) {
diagnostic.truncated = true;
const removable = [...nodes].reverse().find(node => node.frames?.length);
if (removable?.frames) removable.frames.pop();
else { delete diagnostic.causes; break; }
}
return diagnostic;
}
+73 -2
View File
@@ -1,6 +1,8 @@
import { randomUUID } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { realpathSync } from 'node:fs';
import path from 'node:path';
import { logger } from '../../utils/logger';
import { buildPiInitializationDiagnostic } from './initialization-diagnostic';
import type { ModelSummary, ProviderAccount } from '../../shared/providers/types';
import { validateSessionKey } from '../../coding-projects/conversation-store';
import {
@@ -233,9 +235,18 @@ export function createPiManagedWorkerOpener(
const createProcess = options.createProcess ?? ((processOptions) => new PiWorkerProcess(processOptions));
const createSessionKey = options.createSessionKey ?? randomUUID;
const now = options.now ?? Date.now;
return async (input) => {
const openWorker = async (
input: PiWorkerOpenInput,
observation: {
stage(value: string): void;
sensitive(...values: Array<string | undefined>): void;
failed(error: unknown): void;
},
): Promise<PiWorkerOpenResult> => {
const resourcesStartedAt = now();
const registered = await options.registry.prepare(input.conversation);
observation.sensitive(registered.projectPath, registered.agent.prompt, registered.agent.name,
registered.agent.roleName, registered.conversation.title);
const existingSession = input.existingSession ?? registered.session ?? undefined;
const model = registered.conversation.model;
if (!model || registered.conversation.modelResolution !== 'resolved') {
@@ -245,7 +256,9 @@ export function createPiManagedWorkerOpener(
true,
);
}
observation.stage('provider-input');
const providerInput = await options.loadProviderInput();
observation.stage('provider-catalog');
const catalog = buildPiProviderCatalog(providerInput);
const selection = selectPiProviderModel(catalog, model);
const account = providerInput.accounts.find((candidate) => candidate.id === selection.accountId);
@@ -253,6 +266,7 @@ export function createPiManagedWorkerOpener(
if (!account || !descriptor) throw new Error('Selected Provider account is unavailable');
const managedPaths = await ensurePiManagedPaths(options.userDataDir);
await writePiProviderCatalog(managedPaths.modelsFile, catalog, model);
observation.stage('worker-resources');
const workerResources = options.capabilityRegistry
? await options.capabilityRegistry.resolveWorkerResources({
projectPath: registered.projectPath,
@@ -261,6 +275,7 @@ export function createPiManagedWorkerOpener(
role: 'parent',
})
: fallbackWorkerResources(registered.agent.skillIds);
observation.stage('device-resources');
const deviceResources = options.devicePackageManager
? await options.devicePackageManager.resolveEnabledResources()
: undefined;
@@ -268,6 +283,7 @@ export function createPiManagedWorkerOpener(
...workerResources.skillEntries,
...(deviceResources?.skillEntries ?? []),
];
observation.stage('resource-materialization');
const resources = await materializePiAgentResources({
userDataDir: options.userDataDir,
projectId: input.conversation.projectId,
@@ -290,6 +306,8 @@ export function createPiManagedWorkerOpener(
: {}),
revision: input.revision,
});
observation.sensitive(resources.paths.configDir, resources.projectSessionsDir, managedPaths.rootDir);
observation.stage('provider-credentials');
const credential = await buildPiWorkerCredentialProjection({
account,
descriptor,
@@ -298,6 +316,8 @@ export function createPiManagedWorkerOpener(
? { localProxyCredential: await options.getLocalProxyCredential() }
: {}),
});
observation.sensitive(...credential.sensitiveValues);
observation.stage('extension-registration');
const modelTools = options.modelToolRegistry?.registerWorker({
conversationId: input.conversation.conversationId,
generation: input.generation,
@@ -335,6 +355,7 @@ export function createPiManagedWorkerOpener(
extensionsDir: managedPaths.extensionsDir,
});
} catch (error) {
observation.failed(error);
modelTools?.dispose();
throw error;
}
@@ -346,12 +367,15 @@ export function createPiManagedWorkerOpener(
now(),
!existingSession,
);
observation.sensitive(extension.extensionPath, ...extension.sensitiveValues);
observation.stage('session-bind');
const sessionKey = validateSessionKey(
existingSession?.sessionKey ?? createSessionKey(),
);
if (existingSession && input.fork) {
throw new Error('Pi worker cannot reopen and fork a session at the same time');
}
observation.stage('worker-start');
const process = createProcess({
executablePath: options.executablePath,
cliPath: options.cliPath,
@@ -417,6 +441,7 @@ export function createPiManagedWorkerOpener(
if (input.fork?.sourceEntryId) {
await process.request({ type: 'fork', entryId: input.fork.sourceEntryId });
}
observation.stage('initial-hydration');
const readyStartedAt = now();
const response = await process.request<PiRpcSessionStateProjection>(
{ type: 'get_state' },
@@ -451,6 +476,7 @@ export function createPiManagedWorkerOpener(
);
}
}
observation.stage('session-bind');
const sessionStartedAt = now();
const bound = await options.registry.ensureBinding(input.conversation, async () => ({
piSessionId,
@@ -485,12 +511,57 @@ export function createPiManagedWorkerOpener(
session: clone(bound.session),
};
} catch (error) {
observation.failed(error);
unsubscribeExtensionInvalidation();
await process.stop('open_failure').catch(() => undefined);
await disposeManagedResources();
throw error;
}
};
return async (input) => {
let stage = 'registry';
let failureObserved = false;
const sensitiveValues = [input.conversation.title, options.userDataDir,
options.executablePath, options.cliPath, options.bundledSkillsDir];
const observation = {
stage: (value: string) => { stage = value; },
sensitive: (...values: Array<string | undefined>) => {
sensitiveValues.push(...values.filter((value): value is string => typeof value === 'string'));
},
failed: (error: unknown) => {
if (failureObserved) return;
failureObserved = true;
try {
const piPackageRoot = path.resolve(path.dirname(options.cliPath), '..');
const trustedRoots = [piPackageRoot, path.resolve(__dirname, '..', '..'), options.bundledSkillsDir];
try {
// pnpm package stacks use the real path behind the bundled dependency link.
const resolvedPiRoot = realpathSync.native(piPackageRoot);
if (!trustedRoots.includes(resolvedPiRoot)) trustedRoots.push(resolvedPiRoot);
} catch {
// Missing resources must still produce a diagnostic using known roots.
}
logger.warn('[PiWorkerInitializationFailed]', {
conversationRef: createHash('sha256').update(input.conversation.conversationId).digest('hex').slice(0, 16),
workerGeneration: Number.isSafeInteger(input.generation) ? input.generation : 0,
diagnostic: buildPiInitializationDiagnostic(error, {
stage,
sensitiveValues,
trustedRoots,
}),
});
} catch {
// Diagnostic construction/output must never change initialization or cleanup.
}
},
};
try {
return await openWorker(input, observation);
} catch (error) {
observation.failed(error);
throw error;
}
};
}
function recordManagedMilestone(