fix(pi): retain ownership for uncertain mutations
This commit is contained in:
@@ -25,6 +25,7 @@ const MAX_ATTACHMENTS = 16;
|
||||
const MAX_ACCEPTANCES = 512;
|
||||
const REQUEST_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const ATTACHMENT_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/;
|
||||
const REQUEST_UNCERTAIN_MESSAGE = '请求确认延迟,可能仍在执行。请等待结果,或中止/恢复后再重试。';
|
||||
|
||||
export class CodingConversationServiceError extends Error {
|
||||
constructor(
|
||||
@@ -442,8 +443,13 @@ export class CodingConversationService {
|
||||
try {
|
||||
selected = await this.runtime.validateModel(model);
|
||||
} catch (error) { runtimeError(error); }
|
||||
await this.ensurePrepared(conversationId);
|
||||
let snapshot: ConversationSnapshot;
|
||||
try {
|
||||
snapshot = await this.runtime.getSnapshot(conversationId);
|
||||
} catch (error) { runtimeError(error); }
|
||||
this.assertSnapshotAllowsMutation(snapshot);
|
||||
if (conversation.modelResolution === 'resolved' && conversation.model) {
|
||||
await this.ensurePrepared(conversationId);
|
||||
let state: ConversationModelState;
|
||||
try {
|
||||
state = await this.runtime.setModel({
|
||||
@@ -499,6 +505,7 @@ export class CodingConversationService {
|
||||
} catch (error) {
|
||||
runtimeError(error);
|
||||
}
|
||||
this.assertSnapshotAllowsMutation(sourceSnapshot);
|
||||
const sourceNode = sourceSnapshot.nodes.find((node) => (
|
||||
node.kind === 'message'
|
||||
&& node.sourceEntryId === entryId
|
||||
@@ -696,6 +703,16 @@ export class CodingConversationService {
|
||||
}
|
||||
}
|
||||
|
||||
private assertSnapshotAllowsMutation(snapshot: ConversationSnapshot): void {
|
||||
if (snapshot.run.error?.code !== 'CODING_REQUEST_UNCERTAIN') return;
|
||||
if (!['queued', 'running', 'retrying', 'compacting', 'aborting'].includes(snapshot.run.status)) return;
|
||||
throw new CodingConversationServiceError(
|
||||
409,
|
||||
'CODING_REQUEST_UNCERTAIN',
|
||||
REQUEST_UNCERTAIN_MESSAGE,
|
||||
);
|
||||
}
|
||||
|
||||
private async archiveSession(projectId: string, sessionKey: string | undefined): Promise<void> {
|
||||
if (!sessionKey) return;
|
||||
if (!this.options.archiveSession) {
|
||||
|
||||
@@ -74,6 +74,7 @@ type LocalProofProvider = {
|
||||
releaseChildren(): void;
|
||||
releaseParents(): void;
|
||||
releaseAll(): void;
|
||||
armDelayedCompaction(): void;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
@@ -239,6 +240,8 @@ export interface PiReleaseResilienceStatus {
|
||||
errorCode: string | null;
|
||||
recoverable: boolean;
|
||||
bindingPreserved: boolean;
|
||||
contextCompaction: 'idle' | 'running';
|
||||
completedCompactions: number;
|
||||
};
|
||||
other: {
|
||||
conversationId: string;
|
||||
@@ -295,6 +298,7 @@ const PROOF_AGENT_ID = 'release-proof-agent';
|
||||
const PROOF_MODEL_ID = 'release-proof-model';
|
||||
const PROXY_PROOF_MODEL_ID = 'deepseek-v4-pro';
|
||||
const PROOF_PROVIDER_FIRST_EVENT_DELAY_MS = 75;
|
||||
const PROOF_MUTATION_CONFIRMATION_DELAY_MS = 12_000;
|
||||
const EXPECTED_TURN_MILESTONES: readonly ProofMilestone[] = [
|
||||
'worker.queue_wait',
|
||||
'resources.ready',
|
||||
@@ -482,6 +486,7 @@ async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalPr
|
||||
const held = new Set<HeldProviderResponse>();
|
||||
let closed = false;
|
||||
let closeFlight: Promise<void> | null = null;
|
||||
let delayedCompactionArmed = false;
|
||||
const server: Server = createServer(async (request, response) => {
|
||||
response.once('error', () => undefined);
|
||||
try {
|
||||
@@ -545,6 +550,26 @@ async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalPr
|
||||
respondWithSubagentCall(response, model, 'coding');
|
||||
return;
|
||||
}
|
||||
if (role === 'parent' && delayedCompactionArmed) {
|
||||
delayedCompactionArmed = false;
|
||||
const entry = { role, response } satisfies HeldProviderResponse;
|
||||
held.add(entry);
|
||||
response.once('close', () => held.delete(entry));
|
||||
await delay(PROOF_MUTATION_CONFIRMATION_DELAY_MS);
|
||||
if (!held.delete(entry)) return;
|
||||
respondWithText(response, model, 'RESILIENCE_COMPACTION_SUMMARY');
|
||||
return;
|
||||
}
|
||||
if (role === 'parent' && latestUserMessageContains(body, 'BACKGROUND_ACTIVE')) {
|
||||
const entry = { role, response } satisfies HeldProviderResponse;
|
||||
held.add(entry);
|
||||
response.once('close', () => held.delete(entry));
|
||||
await delay(PROOF_MUTATION_CONFIRMATION_DELAY_MS);
|
||||
if (!held.has(entry)) return;
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
writeChunk(response, model, { role: 'assistant', content: 'RESILIENCE_ACTIVE' }, null);
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
writeChunk(response, model, { role: 'assistant', content: 'RESILIENCE_ACTIVE' }, null);
|
||||
const entry = { role, response };
|
||||
@@ -634,6 +659,7 @@ async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalPr
|
||||
releaseChildren: () => release('child'),
|
||||
releaseParents: () => release('parent'),
|
||||
releaseAll: () => release(),
|
||||
armDelayedCompaction: () => { delayedCompactionArmed = true; },
|
||||
close: async () => {
|
||||
if (closed) return;
|
||||
if (!closeFlight) {
|
||||
@@ -1666,6 +1692,10 @@ export async function getFinalAsarResilienceStatus(): Promise<PiReleaseResilienc
|
||||
errorCode: targetError?.code ?? null,
|
||||
recoverable: targetError?.recoverable ?? false,
|
||||
bindingPreserved: targetBindingPreserved,
|
||||
contextCompaction: target.context.compaction,
|
||||
completedCompactions: target.nodes.filter((node) => (
|
||||
node.kind === 'compaction' && node.status === 'complete'
|
||||
)).length,
|
||||
},
|
||||
other: {
|
||||
conversationId: run.otherConversationId,
|
||||
@@ -1860,6 +1890,13 @@ export function releaseFinalAsarResilienceParents(): void {
|
||||
run.provider.releaseParents();
|
||||
}
|
||||
|
||||
export function armFinalAsarResilienceCompactDelay(): { delayMs: number } {
|
||||
const run = resilienceCompositionRun;
|
||||
if (!run) throw new Error('PI resilience proof is not running');
|
||||
run.provider.armDelayedCompaction();
|
||||
return { delayMs: PROOF_MUTATION_CONFIRMATION_DELAY_MS };
|
||||
}
|
||||
|
||||
export async function restartFinalAsarResilienceOther(): Promise<PiReleaseResilienceStatus> {
|
||||
const run = resilienceCompositionRun;
|
||||
if (!run) throw new Error('PI resilience proof is not running');
|
||||
|
||||
@@ -18,16 +18,24 @@ export type PiRpcResponse<T = unknown> = {
|
||||
export type PiRpcEvent = Record<string, unknown> & { type: string };
|
||||
export type PiRpcRetryPolicy = 'none' | 'read-only-once';
|
||||
|
||||
export type PiRpcLateResult =
|
||||
| { response: PiRpcResponse; error?: never }
|
||||
| { response?: never; error: PiProcessError };
|
||||
|
||||
export type PiRpcRequestOptions = {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
retry?: PiRpcRetryPolicy;
|
||||
retainAfterTimeout?: boolean;
|
||||
onLateResult?(result: PiRpcLateResult): void;
|
||||
};
|
||||
|
||||
type PendingRequest = {
|
||||
commandType: string;
|
||||
timedOut: boolean;
|
||||
resolve(response: PiRpcResponse): void;
|
||||
reject(error: PiProcessError): void;
|
||||
notifyLate(result: PiRpcLateResult): void;
|
||||
cancel(): void;
|
||||
};
|
||||
|
||||
@@ -155,17 +163,30 @@ export class PiRpcClient {
|
||||
}
|
||||
pending.cancel();
|
||||
this.pending.delete(value.id);
|
||||
if (value.success) pending.resolve(value);
|
||||
else {
|
||||
pending.reject(new PiProcessError(
|
||||
if (value.success) {
|
||||
if (pending.timedOut) pending.notifyLate({ response: value });
|
||||
else pending.resolve(value);
|
||||
} else {
|
||||
const error = new PiProcessError(
|
||||
'PI_RPC_RESPONSE_ERROR',
|
||||
`Pi RPC ${value.command ?? pending.commandType} failed: ${value.error ?? 'unknown error'}`,
|
||||
{ generation: this.generation },
|
||||
));
|
||||
);
|
||||
if (pending.timedOut) pending.notifyLate({ error });
|
||||
else pending.reject(error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.type === 'agent_settled') {
|
||||
for (const [id, pending] of this.pending) {
|
||||
if (!pending.timedOut) continue;
|
||||
pending.cancel();
|
||||
this.pending.delete(id);
|
||||
this.retire(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const listener of this.listeners) {
|
||||
try {
|
||||
listener(value as PiRpcEvent);
|
||||
@@ -185,7 +206,8 @@ export class PiRpcClient {
|
||||
for (const [id, pending] of this.pending) {
|
||||
pending.cancel();
|
||||
this.retire(id);
|
||||
pending.reject(error);
|
||||
if (pending.timedOut) pending.notifyLate({ error });
|
||||
else pending.reject(error);
|
||||
}
|
||||
this.pending.clear();
|
||||
}
|
||||
@@ -209,8 +231,12 @@ export class PiRpcClient {
|
||||
|
||||
const response = new Promise<PiRpcResponse<T>>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.pending.delete(id);
|
||||
this.retire(id);
|
||||
const pending = this.pending.get(id);
|
||||
if (options.retainAfterTimeout && pending) pending.timedOut = true;
|
||||
else {
|
||||
this.pending.delete(id);
|
||||
this.retire(id);
|
||||
}
|
||||
reject(new PiProcessError(
|
||||
'PI_RPC_TIMEOUT',
|
||||
`Pi RPC ${command.type} timed out after ${timeoutMs}ms`,
|
||||
@@ -218,18 +244,33 @@ export class PiRpcClient {
|
||||
));
|
||||
}, timeoutMs);
|
||||
const abort = (): void => {
|
||||
const pending = this.pending.get(id);
|
||||
this.pending.delete(id);
|
||||
this.retire(id);
|
||||
clearTimeout(timeout);
|
||||
reject(new PiProcessError('PI_RPC_ABORTED', `Pi RPC ${command.type} was aborted`, {
|
||||
const error = new PiProcessError('PI_RPC_ABORTED', `Pi RPC ${command.type} was aborted`, {
|
||||
generation: this.generation,
|
||||
}));
|
||||
});
|
||||
if (pending?.timedOut) pending.notifyLate({ error });
|
||||
else reject(error);
|
||||
};
|
||||
options.signal?.addEventListener('abort', abort, { once: true });
|
||||
this.pending.set(id, {
|
||||
commandType: command.type,
|
||||
timedOut: false,
|
||||
resolve: (value) => resolve(value as PiRpcResponse<T>),
|
||||
reject,
|
||||
notifyLate: (result) => {
|
||||
try {
|
||||
options.onLateResult?.(result);
|
||||
} catch (error) {
|
||||
try {
|
||||
this.onEventListenerError?.(error);
|
||||
} catch {
|
||||
// Diagnostics must not affect transport state.
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel: () => {
|
||||
clearTimeout(timeout);
|
||||
options.signal?.removeEventListener('abort', abort);
|
||||
@@ -248,6 +289,12 @@ export class PiRpcClient {
|
||||
return record;
|
||||
} catch (error) {
|
||||
const pending = this.pending.get(id);
|
||||
if (error instanceof PiProcessError
|
||||
&& error.code === 'PI_RPC_TIMEOUT'
|
||||
&& options.retainAfterTimeout
|
||||
&& pending?.timedOut) {
|
||||
throw error;
|
||||
}
|
||||
if (pending) {
|
||||
pending.cancel();
|
||||
this.pending.delete(id);
|
||||
|
||||
@@ -426,6 +426,16 @@ function runIsTerminal(status: ConversationSnapshot['run']['status']): boolean {
|
||||
return status === 'idle' || status === 'error';
|
||||
}
|
||||
|
||||
const REQUEST_UNCERTAIN_MESSAGE = '请求确认延迟,可能仍在执行。请等待结果,或中止/恢复后再重试。';
|
||||
|
||||
function requestUncertainError(): CodingRuntimeContractError {
|
||||
return new CodingRuntimeContractError(
|
||||
'CODING_REQUEST_UNCERTAIN',
|
||||
REQUEST_UNCERTAIN_MESSAGE,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
function publicWorkerState(state: PiWorkerPoolState): ConversationSnapshot['worker'] {
|
||||
if (state.state === 'spawning') return { status: 'starting', generation: state.generation };
|
||||
if (state.state === 'crashed') {
|
||||
@@ -460,7 +470,7 @@ function runtimeFailure(error: unknown): CodingRuntimePublicError {
|
||||
if (error.code === 'PI_RPC_TIMEOUT') {
|
||||
return {
|
||||
code: 'CODING_REQUEST_UNCERTAIN',
|
||||
message: 'The local Agent did not confirm the request',
|
||||
message: REQUEST_UNCERTAIN_MESSAGE,
|
||||
recoverable: true,
|
||||
};
|
||||
}
|
||||
@@ -626,6 +636,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
|
||||
async prompt(input: PromptConversationInput): Promise<PromptAcceptance> {
|
||||
await this.waitForProjection(input.conversationId);
|
||||
this.assertNoUncertainMutation(input.conversationId);
|
||||
if (input.mode === 'steer') {
|
||||
const acceptance = await this.steer(input);
|
||||
return {
|
||||
@@ -765,6 +776,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
|
||||
async setModel(input: SetConversationModelInput): Promise<ConversationModelState> {
|
||||
await this.waitForProjection(input.conversationId);
|
||||
this.assertNoUncertainMutation(input.conversationId);
|
||||
const snapshot = this.snapshot(input.conversationId);
|
||||
const thinkingLevel = snapshot.conversation.model.model?.thinkingLevel ?? 'off';
|
||||
const selection = await this.resolveModel({
|
||||
@@ -825,6 +837,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
|
||||
async setThinking(input: SetThinkingLevelInput): Promise<ConversationModelState> {
|
||||
await this.waitForProjection(input.conversationId);
|
||||
this.assertNoUncertainMutation(input.conversationId);
|
||||
const current = this.snapshot(input.conversationId).conversation.model;
|
||||
if (!current.model) {
|
||||
throw new CodingRuntimeContractError(
|
||||
@@ -876,6 +889,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
|
||||
async compact(conversationId: string): Promise<void> {
|
||||
await this.waitForProjection(conversationId);
|
||||
this.assertNoUncertainMutation(conversationId);
|
||||
const runId = this.id('run');
|
||||
this.acquireRunBackgroundLease(conversationId, runId);
|
||||
const generation = this.pool.getState(conversationId)?.generation;
|
||||
@@ -905,6 +919,11 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
try {
|
||||
await ticket.accepted;
|
||||
} catch (error) {
|
||||
if (error instanceof PiProcessError && error.code === 'PI_RPC_TIMEOUT') {
|
||||
const failure = requestUncertainError();
|
||||
this.markRunUncertain(conversationId, runId, failure.publicError);
|
||||
throw failure;
|
||||
}
|
||||
await this.failRun(conversationId, runId, error);
|
||||
throw error;
|
||||
}
|
||||
@@ -912,6 +931,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
|
||||
async fork(input: ForkConversationInput): Promise<ForkResult> {
|
||||
await this.waitForProjection(input.sourceConversationId);
|
||||
this.assertNoUncertainMutation(input.sourceConversationId);
|
||||
this.snapshot(input.sourceConversationId);
|
||||
const registered = await this.registry.prepare(input.conversation);
|
||||
const canonicalInput: PrepareConversationInput = {
|
||||
@@ -1091,6 +1111,14 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
return Boolean(snapshot && !runIsTerminal(snapshot.run.status));
|
||||
}
|
||||
|
||||
private assertNoUncertainMutation(conversationId: string): void {
|
||||
const run = this.states.get(conversationId)?.snapshot?.run;
|
||||
if (!runIsTerminal(run?.status ?? 'idle')
|
||||
&& run?.error?.code === 'CODING_REQUEST_UNCERTAIN') {
|
||||
throw requestUncertainError();
|
||||
}
|
||||
}
|
||||
|
||||
markProviderStale(): void {
|
||||
this.pool.markProviderStale();
|
||||
}
|
||||
@@ -1137,6 +1165,7 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
input: QueueMessageInput,
|
||||
): Promise<QueueAcceptance> {
|
||||
await this.waitForProjection(input.conversationId);
|
||||
this.assertNoUncertainMutation(input.conversationId);
|
||||
const snapshot = this.snapshot(input.conversationId);
|
||||
if (snapshot.run.runId) {
|
||||
this.acquireRunBackgroundLease(input.conversationId, snapshot.run.runId);
|
||||
@@ -1255,6 +1284,11 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
true,
|
||||
)
|
||||
: error;
|
||||
if (failure instanceof PiProcessError && failure.code === 'PI_RPC_TIMEOUT') {
|
||||
const uncertain = requestUncertainError();
|
||||
this.markRunUncertain(conversationId, runId, uncertain.publicError);
|
||||
throw uncertain;
|
||||
}
|
||||
this.pool.failTopLevel(
|
||||
conversationId,
|
||||
runId,
|
||||
@@ -1265,6 +1299,19 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
private markRunUncertain(
|
||||
conversationId: string,
|
||||
runId: string,
|
||||
error: CodingRuntimePublicError,
|
||||
): void {
|
||||
const current = this.states.get(conversationId)?.snapshot?.run;
|
||||
if (!current || current.runId !== runId || runIsTerminal(current.status)) return;
|
||||
this.emit(conversationId, {
|
||||
op: 'run.state',
|
||||
run: { ...current, error: clone(error) },
|
||||
}, runId);
|
||||
}
|
||||
|
||||
private snapshot(conversationId: string): ConversationSnapshot {
|
||||
const snapshot = this.states.get(conversationId)?.snapshot;
|
||||
if (!snapshot) {
|
||||
@@ -1314,6 +1361,40 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
}
|
||||
|
||||
private onPoolEvent(event: PiWorkerPoolEvent): void {
|
||||
if (event.type === 'top-level.confirmed') {
|
||||
void this.enqueueProjection(event.conversationId, async () => {
|
||||
const snapshot = this.states.get(event.conversationId)?.snapshot;
|
||||
const current = snapshot?.run;
|
||||
if (!snapshot
|
||||
|| snapshot.cursor.workerGeneration !== event.generation
|
||||
|| current?.runId !== event.runId
|
||||
|| runIsTerminal(current.status)) return;
|
||||
const { error: _error, ...confirmed } = current;
|
||||
this.emit(event.conversationId, {
|
||||
op: 'run.state',
|
||||
run: {
|
||||
...confirmed,
|
||||
status: confirmed.status === 'queued' ? 'running' : confirmed.status,
|
||||
},
|
||||
}, event.runId);
|
||||
}).catch((error) => {
|
||||
this.recordProjectionFailure(event.conversationId, event.generation, error);
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (event.type === 'top-level.failed') {
|
||||
void this.enqueueProjection(event.conversationId, async () => {
|
||||
await this.failRun(
|
||||
event.conversationId,
|
||||
event.runId,
|
||||
event.error,
|
||||
event.generation,
|
||||
);
|
||||
}).catch((error) => {
|
||||
this.recordProjectionFailure(event.conversationId, event.generation, error);
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (event.type === 'worker.replaced') {
|
||||
if (!this.states.has(event.conversationId)) return;
|
||||
this.replaceWorkerGeneration(event.conversationId, event.state, true);
|
||||
|
||||
@@ -3,10 +3,11 @@ import type {
|
||||
ConversationModelState,
|
||||
PrepareConversationInput,
|
||||
} from '../contracts';
|
||||
import type { PiProcessError, PiProcessErrorCode } from './process-errors';
|
||||
import { PiProcessError, type PiProcessErrorCode } from './process-errors';
|
||||
import type {
|
||||
PiRpcCommand,
|
||||
PiRpcEvent,
|
||||
PiRpcLateResult,
|
||||
PiRpcRequestOptions,
|
||||
PiRpcResponse,
|
||||
} from './rpc-client';
|
||||
@@ -108,6 +109,19 @@ export type PiWorkerPoolEvent =
|
||||
generation: number;
|
||||
reason: PiWorkerReplacementReason;
|
||||
state: PiWorkerPoolState;
|
||||
}
|
||||
| {
|
||||
type: 'top-level.confirmed';
|
||||
conversationId: string;
|
||||
generation: number;
|
||||
runId: string;
|
||||
}
|
||||
| {
|
||||
type: 'top-level.failed';
|
||||
conversationId: string;
|
||||
generation: number;
|
||||
runId: string;
|
||||
error: PiProcessError;
|
||||
};
|
||||
|
||||
export type PiWorkerReplacementReason =
|
||||
@@ -247,9 +261,13 @@ export class PiWorkerPool {
|
||||
private readonly waitingRuns: PendingTopLevelRun[] = [];
|
||||
private readonly activeRuns = new Map<string, {
|
||||
runId: string;
|
||||
run: PendingTopLevelRun;
|
||||
generation: number;
|
||||
cold: boolean;
|
||||
acceptedAt?: number;
|
||||
confirmationStartedAt?: number;
|
||||
confirmation: 'pending' | 'uncertain' | 'confirmed';
|
||||
cancelConfirmation?: () => void;
|
||||
}>();
|
||||
private readonly generations = new Map<string, number>();
|
||||
private readonly listeners = new Set<(event: PiWorkerPoolEvent) => void>();
|
||||
@@ -498,13 +516,14 @@ export class PiWorkerPool {
|
||||
|
||||
getActiveRun(conversationId: string): { runId: string; generation: number } | null {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
return active ? { ...active } : null;
|
||||
return active ? { runId: active.runId, generation: active.generation } : null;
|
||||
}
|
||||
|
||||
failTopLevel(conversationId: string, runId: string, error: Error): void {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active?.runId === runId) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
const record = this.workers.get(conversationId);
|
||||
if (record && record.state !== 'crashed') {
|
||||
@@ -667,8 +686,10 @@ export class PiWorkerPool {
|
||||
await Promise.allSettled([...this.rebuildFlights]);
|
||||
const records = [...this.workers.values()];
|
||||
this.workers.clear();
|
||||
const activeRuns = [...this.activeRuns.values()];
|
||||
this.activeRuns.clear();
|
||||
this.runningCount = 0;
|
||||
for (const active of activeRuns) active.cancelConfirmation?.();
|
||||
await Promise.all(records.map(async (record) => {
|
||||
record.unsubscribeEvent();
|
||||
record.unsubscribeInvalidation();
|
||||
@@ -684,8 +705,10 @@ export class PiWorkerPool {
|
||||
this.runningCount += 1;
|
||||
this.activeRuns.set(run.conversationId, {
|
||||
runId: run.runId,
|
||||
run,
|
||||
generation: record.generation,
|
||||
cold: record.coldStart && record.acceptedPromptCount === 0,
|
||||
confirmation: 'pending',
|
||||
});
|
||||
void this.acceptTopLevel(record, run);
|
||||
}
|
||||
@@ -708,29 +731,34 @@ export class PiWorkerPool {
|
||||
'worker.queue_wait',
|
||||
run.queuedAt === undefined ? 0 : this.now() - run.queuedAt,
|
||||
);
|
||||
const acceptedAt = this.now();
|
||||
const response = await current.worker.request(run.command);
|
||||
if (run.command.type === 'prompt') {
|
||||
const acceptedFinishedAt = this.now();
|
||||
active.acceptedAt = acceptedFinishedAt;
|
||||
this.recordMilestone(
|
||||
current,
|
||||
run,
|
||||
'prompt.accepted',
|
||||
acceptedFinishedAt - acceptedAt,
|
||||
active.cold,
|
||||
);
|
||||
current.acceptedPromptCount += 1;
|
||||
}
|
||||
active.confirmationStartedAt = this.now();
|
||||
const confirmationController = new AbortController();
|
||||
active.cancelConfirmation = () => confirmationController.abort();
|
||||
const response = await current.worker.request(run.command, {
|
||||
signal: confirmationController.signal,
|
||||
retainAfterTimeout: true,
|
||||
onLateResult: (result) => this.handleLateTopLevelResult(current!, run, result),
|
||||
});
|
||||
this.confirmTopLevel(current, run);
|
||||
run.resolve(response);
|
||||
} catch (error) {
|
||||
const active = this.activeRuns.get(run.conversationId);
|
||||
if (error instanceof PiProcessError
|
||||
&& error.code === 'PI_RPC_TIMEOUT'
|
||||
&& active?.runId === run.runId
|
||||
&& current
|
||||
&& active.generation === current.generation) {
|
||||
if (active.confirmation !== 'confirmed') active.confirmation = 'uncertain';
|
||||
run.reject(error);
|
||||
return;
|
||||
}
|
||||
if (active) {
|
||||
this.activeRuns.delete(run.conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
this.launchWaitingRuns();
|
||||
}
|
||||
if (current && this.workers.get(run.conversationId) === current
|
||||
if (active && current && this.workers.get(run.conversationId) === current
|
||||
&& current.state !== 'crashed') {
|
||||
current.state = 'idle';
|
||||
this.notifyReclaimableWorker();
|
||||
@@ -741,10 +769,84 @@ export class PiWorkerPool {
|
||||
}
|
||||
}
|
||||
|
||||
private confirmTopLevel(record: WorkerRecord, run: PendingTopLevelRun): boolean {
|
||||
const active = this.activeRuns.get(run.conversationId);
|
||||
if (!active
|
||||
|| active.runId !== run.runId
|
||||
|| active.generation !== record.generation
|
||||
|| active.confirmation === 'confirmed') return false;
|
||||
active.confirmation = 'confirmed';
|
||||
active.cancelConfirmation = undefined;
|
||||
if (run.command.type === 'prompt') {
|
||||
const acceptedFinishedAt = this.now();
|
||||
active.acceptedAt = acceptedFinishedAt;
|
||||
this.recordMilestone(
|
||||
record,
|
||||
run,
|
||||
'prompt.accepted',
|
||||
acceptedFinishedAt - (active.confirmationStartedAt ?? acceptedFinishedAt),
|
||||
active.cold,
|
||||
);
|
||||
record.acceptedPromptCount += 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private handleLateTopLevelResult(
|
||||
record: WorkerRecord,
|
||||
run: PendingTopLevelRun,
|
||||
result: PiRpcLateResult,
|
||||
): void {
|
||||
const active = this.activeRuns.get(run.conversationId);
|
||||
if (!active || active.runId !== run.runId || active.generation !== record.generation) return;
|
||||
if (result.response) {
|
||||
if (!this.confirmTopLevel(record, run)) return;
|
||||
this.emit({
|
||||
type: 'top-level.confirmed',
|
||||
conversationId: run.conversationId,
|
||||
generation: record.generation,
|
||||
runId: run.runId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (result.error.code === 'PI_RPC_EXITED'
|
||||
|| result.error.code === 'PI_RPC_PROTOCOL_ERROR') return;
|
||||
this.activeRuns.delete(run.conversationId);
|
||||
active.cancelConfirmation = undefined;
|
||||
this.runningCount -= 1;
|
||||
if (this.workers.get(run.conversationId) === record && record.state !== 'crashed') {
|
||||
record.state = 'idle';
|
||||
this.notifyReclaimableWorker();
|
||||
try {
|
||||
this.revisions.settleRun(record.revisionWorkerId);
|
||||
} catch {
|
||||
// Recover/crash may already have removed this generation.
|
||||
}
|
||||
}
|
||||
this.launchWaitingRuns();
|
||||
void this.trimIdleWorkers();
|
||||
this.emit({
|
||||
type: 'top-level.failed',
|
||||
conversationId: run.conversationId,
|
||||
generation: record.generation,
|
||||
runId: run.runId,
|
||||
error: result.error,
|
||||
});
|
||||
}
|
||||
|
||||
private settleTopLevel(record: WorkerRecord): void {
|
||||
const conversationId = record.conversation.conversationId;
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (!active || active.generation !== record.generation) return;
|
||||
const cancelConfirmation = active.cancelConfirmation;
|
||||
if (active.confirmation !== 'confirmed') {
|
||||
this.confirmTopLevel(record, active.run);
|
||||
active.run.resolve({
|
||||
type: 'response',
|
||||
id: `agent-settled:${active.runId}`,
|
||||
success: true,
|
||||
});
|
||||
}
|
||||
if (active.acceptedAt !== undefined) {
|
||||
this.recordMilestone(
|
||||
record,
|
||||
@@ -755,6 +857,7 @@ export class PiWorkerPool {
|
||||
);
|
||||
}
|
||||
this.activeRuns.delete(conversationId);
|
||||
cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
record.state = 'idle';
|
||||
this.notifyReclaimableWorker();
|
||||
@@ -1033,6 +1136,7 @@ export class PiWorkerPool {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active?.generation === record.generation) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
}
|
||||
for (let index = this.waitingRuns.length - 1; index >= 0; index -= 1) {
|
||||
@@ -1060,6 +1164,7 @@ export class PiWorkerPool {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
}
|
||||
for (let index = this.waitingRuns.length - 1; index >= 0; index -= 1) {
|
||||
|
||||
Reference in New Issue
Block a user