fix(pi): converge worker failures and thinking state
This commit is contained in:
1 parent
274187e3cf
commit
61817b161f
28 files changed
+2019
-118
No files matched your search
@@ -13,6 +13,7 @@ import {
|
||||
buildPiRpcArgs,
|
||||
sanitizePiDiagnostic,
|
||||
buildPiWorkerEnvironment,
|
||||
type PiWorkerLifecycleEvent,
|
||||
} from '../../electron/coding-runtime/pi/worker-process';
|
||||
|
||||
const fakeChildPath = resolve('tests/fixtures/fake-pi-rpc-child.mjs');
|
||||
@@ -56,7 +57,7 @@ async function processAlive(pid: number): Promise<boolean> {
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(workers.splice(0).map((worker) => worker.stop().catch(() => undefined)));
|
||||
await Promise.all(workers.splice(0).map((worker) => worker.stop('test_injection').catch(() => undefined)));
|
||||
await Promise.all(scratchRoots.splice(0).map((root) => rm(root, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
@@ -259,6 +260,87 @@ describe('Pi worker process', () => {
|
||||
expect(invalidations).toEqual(['PI_RPC_EXITED']);
|
||||
});
|
||||
|
||||
it('classifies unexpected exit with bounded redacted stderr and generation correlation', async () => {
|
||||
const secret = 'unexpected-exit-secret';
|
||||
const lifecycle: PiWorkerLifecycleEvent[] = [];
|
||||
const worker = await makeWorker({
|
||||
conversationId: 'conversation-exit',
|
||||
workerGeneration: 7,
|
||||
env: { FAKE_PI_SECRET: secret },
|
||||
sensitiveValues: [secret],
|
||||
diagnosticBytes: 160,
|
||||
onLifecycleEvent: (event) => lifecycle.push(event),
|
||||
});
|
||||
|
||||
await expect(worker.request({ type: 'crash_with_stderr' })).rejects.toMatchObject({
|
||||
code: 'PI_RPC_EXITED',
|
||||
generation: 7,
|
||||
exitCode: 9,
|
||||
signal: null,
|
||||
diagnostic: expect.stringContaining('worker-exit-marker'),
|
||||
});
|
||||
const close = lifecycle.find((event) => (
|
||||
event.classification === 'unexpected_exit' && event.stage === 'close'
|
||||
));
|
||||
expect(close).toMatchObject({
|
||||
conversationId: 'conversation-exit',
|
||||
generation: 7,
|
||||
code: 'PI_RPC_EXITED',
|
||||
exitCode: 9,
|
||||
signal: null,
|
||||
});
|
||||
expect(JSON.stringify(close)).not.toContain(secret);
|
||||
expect(JSON.stringify(close)).not.toContain(scratchRoots.at(-1));
|
||||
expect(close?.diagnostic).toContain('cwd=[REDACTED]');
|
||||
expect(Buffer.byteLength(close?.diagnostic ?? '')).toBeLessThanOrEqual(160);
|
||||
});
|
||||
|
||||
it('keeps protocol invalidation and deliberate stop as distinct reasoned lifecycle events', async () => {
|
||||
const protocolLifecycle: PiWorkerLifecycleEvent[] = [];
|
||||
const protocolWorker = await makeWorker({
|
||||
conversationId: 'conversation-protocol',
|
||||
workerGeneration: 3,
|
||||
onLifecycleEvent: (event) => protocolLifecycle.push(event),
|
||||
});
|
||||
await expect(protocolWorker.request({ type: 'malformed' })).rejects.toMatchObject({
|
||||
code: 'PI_RPC_PROTOCOL_ERROR',
|
||||
generation: 3,
|
||||
});
|
||||
expect(protocolLifecycle).toContainEqual(expect.objectContaining({
|
||||
classification: 'protocol_invalidation',
|
||||
stage: 'protocol',
|
||||
conversationId: 'conversation-protocol',
|
||||
generation: 3,
|
||||
code: 'PI_RPC_PROTOCOL_ERROR',
|
||||
}));
|
||||
expect(protocolLifecycle).not.toContainEqual(expect.objectContaining({
|
||||
classification: 'unexpected_exit',
|
||||
}));
|
||||
|
||||
const stopLifecycle: PiWorkerLifecycleEvent[] = [];
|
||||
const stoppedWorker = await makeWorker({
|
||||
conversationId: 'conversation-stop',
|
||||
workerGeneration: 5,
|
||||
onLifecycleEvent: (event) => stopLifecycle.push(event),
|
||||
});
|
||||
await expect(stoppedWorker.stop('test_injection')).resolves.toMatchObject({
|
||||
mode: 'stdin-close',
|
||||
});
|
||||
expect(stopLifecycle).toContainEqual(expect.objectContaining({
|
||||
classification: 'intentional_stop',
|
||||
stage: 'stop_completed',
|
||||
reason: 'test_injection',
|
||||
conversationId: 'conversation-stop',
|
||||
generation: 5,
|
||||
code: 'PI_WORKER_STOPPED',
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
}));
|
||||
expect(stopLifecycle).not.toContainEqual(expect.objectContaining({
|
||||
classification: 'unexpected_exit',
|
||||
}));
|
||||
});
|
||||
|
||||
it('keeps only bounded redacted stderr diagnostics', async () => {
|
||||
const secret = 'credential-that-must-not-leak';
|
||||
const worker = await makeWorker({
|
||||
@@ -314,7 +396,7 @@ describe('Pi worker process', () => {
|
||||
const descendantPid = response.data?.pid;
|
||||
expect(descendantPid).toBeTypeOf('number');
|
||||
|
||||
await expect(worker.stop()).resolves.toMatchObject({ mode: 'forced-tree-kill' });
|
||||
await expect(worker.stop('test_injection')).resolves.toMatchObject({ mode: 'forced-tree-kill' });
|
||||
for (let attempt = 0; attempt < 20 && await processAlive(descendantPid!); attempt += 1) {
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, 25));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user