wangxuming/makelore
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(pi): converge worker failures and thinking state

Browse Source
This commit is contained in:
brother7
2026-08-25 11:45:14 +08:00
parent 274187e3cf
commit 61817b161f
28 changed files with 2019 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
electron/coding-runtime/pi/worker-process.ts
View File

@@ -4,7 +4,8 @@ import {
type ChildProcessWithoutNullStreams,
} from 'node:child_process';
import { platform } from 'node:os';
import { PiProcessError } from './process-errors';
import { logger } from '../../utils/logger';
import { PiProcessError, type PiProcessErrorCode } from './process-errors';
import {
PiRpcClient,
type PiRpcCommand,
@@ -46,6 +47,59 @@ export type PiWorkerStopResult = {
signal: NodeJS.Signals | null;
};
export type PiWorkerProofFailure = 'unexpected_exit' | 'protocol_invalidation';
export type PiWorkerStopReason =
| 'app_shutdown'
| 'idle_eviction'
| 'queued_suspension'
| 'stale_resource_rebuild'
| 'recover'
| 'dispose'
| 'model_reconfiguration'
| 'process_capacity_reopen'
| 'fork_replacement'
| 'protocol_invalidation'
| 'unexpected_exit_cleanup'
| 'open_failure'
| 'session_binding_mismatch'
| 'subagent_complete'
| 'subagent_abort'
| 'subagent_open_failure'
| 'test_injection';
const PI_WORKER_STOP_REASONS = new Set<PiWorkerStopReason>([
'app_shutdown',
'idle_eviction',
'queued_suspension',
'stale_resource_rebuild',
'recover',
'dispose',
'model_reconfiguration',
'process_capacity_reopen',
'fork_replacement',
'protocol_invalidation',
'unexpected_exit_cleanup',
'open_failure',
'session_binding_mismatch',
'subagent_complete',
'subagent_abort',
'subagent_open_failure',
'test_injection',
]);
export type PiWorkerLifecycleEvent = {
classification: 'unexpected_exit' | 'protocol_invalidation' | 'intentional_stop';
stage: 'protocol' | 'close' | 'stop_requested' | 'stop_completed' | 'stop_failed';
conversationId?: string;
generation: number;
code: PiProcessErrorCode;
reason?: PiWorkerStopReason;
exitCode: number | null;
signal: NodeJS.Signals | null;
diagnostic?: string;
};
export type PiWorkerProcessOptions = {
executablePath: string;
cliPath: string;
@@ -60,6 +114,9 @@ export type PiWorkerProcessOptions = {
shutdownGraceMs?: number;
maxLineBytes?: number;
diagnosticBytes?: number;
conversationId?: string;
workerGeneration?: number;
onLifecycleEvent?(event: PiWorkerLifecycleEvent): void;
};
export function buildPiRpcArgs(
@@ -174,11 +231,11 @@ export class PiWorkerProcess {
private readonly commandTimeoutMs: number;
private readonly shutdownGraceMs: number;
private readonly diagnosticBytes: number;
private readonly diagnosticSensitiveValues: string[];
private child: ChildProcessWithoutNullStreams | null = null;
private rpc: PiRpcClient | null = null;
private generationValue = 1;
private generationValue: number;
private diagnostic = '';
private stdoutTail = '';
private invalidation: PiProcessError | null = null;
private stopping = false;
private stopPromise: Promise<PiWorkerStopResult> | null = null;
@@ -187,6 +244,7 @@ export class PiWorkerProcess {
constructor(options: PiWorkerProcessOptions) {
this.options = options;
this.generationValue = positiveInteger(options.workerGeneration, 1, 'workerGeneration');
this.commandTimeoutMs = positiveInteger(
options.commandTimeoutMs,
DEFAULT_COMMAND_TIMEOUT_MS,
@@ -202,6 +260,17 @@ export class PiWorkerProcess {
DEFAULT_DIAGNOSTIC_BYTES,
'diagnosticBytes',
);
this.diagnosticSensitiveValues = [...new Set([
...(options.sensitiveValues ?? []),
options.cwd,
options.cliPath,
options.configDir,
options.sessionDir,
options.env?.HOME,
options.env?.USERPROFILE,
options.env?.APPDATA,
options.env?.LOCALAPPDATA,
].filter((value): value is string => Boolean(value)))];
}
get generation(): number {
@@ -267,11 +336,26 @@ export class PiWorkerProcess {
});
child.once('close', (code, signal) => {
if (!this.stopping && !this.invalidation) {
this.invalidate(new PiProcessError(
const failure = new PiProcessError(
'PI_RPC_EXITED',
`Pi worker exited unexpectedly (code=${code ?? 'null'}, signal=${signal ?? 'none'})`,
{ generation },
));
{
generation,
diagnostic: this.diagnostic || undefined,
exitCode: code,
signal,
},
);
this.recordLifecycle({
classification: 'unexpected_exit',
stage: 'close',
generation,
code: failure.code,
exitCode: code,
signal,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
this.invalidate(failure);
}
});
@@ -280,10 +364,6 @@ export class PiWorkerProcess {
onRecord: (record) => this.acceptRecord(record),
});
child.stdout.on('data', (chunk: Buffer) => {
this.stdoutTail = boundedUtf8Tail(
`${this.stdoutTail}${sanitizePiDiagnostic(chunk.toString('utf8'), this.options.sensitiveValues)}`,
Math.min(this.diagnosticBytes, 2_048),
);
try {
framer.push(chunk);
} catch (error) {
@@ -351,11 +431,26 @@ export class PiWorkerProcess {
return () => this.invalidationListeners.delete(listener);
}
stop(): Promise<PiWorkerStopResult> {
if (!this.stopPromise) this.stopPromise = this.performStop();
stop(reason: PiWorkerStopReason): Promise<PiWorkerStopResult> {
if (!PI_WORKER_STOP_REASONS.has(reason)) {
return Promise.reject(new Error('Pi worker stop reason is required'));
}
if (!this.stopPromise) this.stopPromise = this.performStop(reason);
return this.stopPromise;
}
async injectFailureForProof(failure: PiWorkerProofFailure): Promise<void> {
if (!this.isRunning) throw new Error('Pi worker proof failure requires a running process');
this.appendDiagnostic(
'[release-proof] worker failure Authorization: Bearer packaged-proof-secret\n',
);
if (failure === 'protocol_invalidation') {
this.handleProtocolFailure(new Error('Injected strict JSONL protocol failure'));
return;
}
await this.forceKillTree();
}
private acceptRecord(record: unknown): void {
try {
this.rpc?.accept(record);
@@ -367,12 +462,21 @@ export class PiWorkerProcess {
private handleProtocolFailure(error: unknown): void {
if (this.invalidation) return;
const message = error instanceof Error ? error.message : 'Pi RPC protocol failure';
this.appendDiagnostic(`[stdout-protocol] ${this.stdoutTail}\n`);
this.appendDiagnostic('[stdout-protocol]\n');
const failure = new PiProcessError('PI_RPC_PROTOCOL_ERROR', message, {
cause: error,
generation: this.generationValue,
diagnostic: this.diagnostic,
});
this.recordLifecycle({
classification: 'protocol_invalidation',
stage: 'protocol',
generation: this.generationValue,
code: failure.code,
exitCode: this.child?.exitCode ?? null,
signal: this.child?.signalCode ?? null,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
this.invalidate(failure);
void this.forceKillTree();
}
@@ -394,14 +498,40 @@ export class PiWorkerProcess {
}
private appendDiagnostic(source: string): void {
const sanitized = sanitizePiDiagnostic(source, this.options.sensitiveValues);
const sanitized = sanitizePiDiagnostic(source, this.diagnosticSensitiveValues);
this.diagnostic = boundedUtf8Tail(`${this.diagnostic}${sanitized}`, this.diagnosticBytes);
}
private async performStop(): Promise<PiWorkerStopResult> {
private async performStop(reason: PiWorkerStopReason): Promise<PiWorkerStopResult> {
const child = this.child;
const exitResult = this.exitResult;
const generation = this.invalidation?.generation ?? this.generationValue;
const classification = this.invalidation?.code === 'PI_RPC_PROTOCOL_ERROR'
? 'protocol_invalidation'
: this.invalidation?.code === 'PI_RPC_EXITED'
? 'unexpected_exit'
: 'intentional_stop';
this.recordLifecycle({
classification,
stage: 'stop_requested',
generation,
code: this.invalidation?.code ?? 'PI_WORKER_STOPPED',
reason,
exitCode: child?.exitCode ?? null,
signal: child?.signalCode ?? null,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
if (!child || !exitResult) {
this.recordLifecycle({
classification,
stage: 'stop_completed',
generation,
code: this.invalidation?.code ?? 'PI_WORKER_STOPPED',
reason,
exitCode: null,
signal: null,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
return { mode: 'not-started', code: null, signal: null };
}
@@ -411,17 +541,71 @@ export class PiWorkerProcess {
}));
child.stdin.end();
const graceful = await this.waitForExit(this.shutdownGraceMs);
if (graceful) return { mode: 'stdin-close', ...graceful };
if (graceful) {
this.recordLifecycle({
classification,
stage: 'stop_completed',
generation,
code: this.invalidation?.code ?? 'PI_WORKER_STOPPED',
reason,
exitCode: graceful.code,
signal: graceful.signal,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
return { mode: 'stdin-close', ...graceful };
}
await this.forceKillTree();
const forced = await this.waitForExit(this.shutdownGraceMs);
if (forced) return { mode: 'forced-tree-kill', ...forced };
if (forced) {
this.recordLifecycle({
classification,
stage: 'stop_completed',
generation,
code: this.invalidation?.code ?? 'PI_WORKER_STOPPED',
reason,
exitCode: forced.code,
signal: forced.signal,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
return { mode: 'forced-tree-kill', ...forced };
}
this.recordLifecycle({
classification,
stage: 'stop_failed',
generation,
code: 'PI_WORKER_STOP_FAILED',
reason,
exitCode: child.exitCode,
signal: child.signalCode,
...(this.diagnostic ? { diagnostic: this.diagnostic } : {}),
});
throw new PiProcessError('PI_WORKER_STOP_FAILED', 'Pi worker did not exit after forced tree kill', {
generation: this.generationValue,
diagnostic: this.diagnostic,
});
}
private recordLifecycle(event: PiWorkerLifecycleEvent): void {
const diagnostic = event.diagnostic
? boundedUtf8Tail(
sanitizePiDiagnostic(event.diagnostic, this.diagnosticSensitiveValues),
this.diagnosticBytes,
)
: undefined;
const safeEvent: PiWorkerLifecycleEvent = {
...event,
...(this.options.conversationId ? { conversationId: this.options.conversationId } : {}),
...(diagnostic ? { diagnostic } : {}),
};
try {
this.options.onLifecycleEvent?.(structuredClone(safeEvent));
} catch {
// Lifecycle observers must not affect process cleanup.
}
logger.warn('[PiWorkerLifecycle]', safeEvent);
}
private async waitForExit(
timeoutMs: number,
): Promise<{ code: number | null; signal: NodeJS.Signals | null } | null> {