merge: integrate remote main
This commit is contained in:
@@ -6,8 +6,8 @@ import type {
|
||||
ConversationToolNode,
|
||||
PublicUsage,
|
||||
} from '../contracts';
|
||||
import { isAIGatewayUserContextMissing } from '../../../shared/ai-gateway-error-details';
|
||||
import type { PiRpcEvent } from './rpc-client';
|
||||
import { projectPiProviderFailure } from './provider-failure';
|
||||
import { subagentDetailsOfResult } from '../subagent-protocol';
|
||||
import {
|
||||
isProductToolName,
|
||||
@@ -45,17 +45,6 @@ function retryFailure(message: string) {
|
||||
};
|
||||
}
|
||||
|
||||
function providerFailure(message: unknown) {
|
||||
if (typeof message === 'string' && isAIGatewayUserContextMissing(message)) {
|
||||
return {
|
||||
code: 'CODING_PROVIDER_AUTH_REQUIRED' as const,
|
||||
message: '模型服务身份上下文无效,请重试;若仍失败请重新登录。',
|
||||
recoverable: true,
|
||||
};
|
||||
}
|
||||
return retryFailure('模型服务请求失败,请稍后重试。');
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
@@ -294,7 +283,7 @@ export class PiEventProjector {
|
||||
...run,
|
||||
status: 'error',
|
||||
terminalReason: 'failed',
|
||||
error: providerFailure(event.finalError),
|
||||
error: projectPiProviderFailure(event.finalError),
|
||||
},
|
||||
}];
|
||||
}
|
||||
@@ -485,7 +474,7 @@ export class PiEventProjector {
|
||||
...snapshot.run,
|
||||
status: 'error',
|
||||
terminalReason: 'failed',
|
||||
error: providerFailure(assistant.errorMessage),
|
||||
error: projectPiProviderFailure(assistant.errorMessage),
|
||||
},
|
||||
}];
|
||||
}
|
||||
|
||||
@@ -180,12 +180,13 @@ export function createMakeloreRuntime(runtimeDefaults = {}) {
|
||||
return response.result;
|
||||
}
|
||||
|
||||
function registerProductTool(name, label, description, parameters) {
|
||||
function registerProductTool(name, label, description, parameters, projectWriteLease = false) {
|
||||
pi.registerTool({
|
||||
name,
|
||||
label,
|
||||
description,
|
||||
parameters,
|
||||
...(projectWriteLease ? { executionMode: 'sequential' } : {}),
|
||||
async execute(toolCallId, params, signal) {
|
||||
return await invokeProduct(toolCallId, name, params, signal);
|
||||
},
|
||||
@@ -227,6 +228,7 @@ export function createMakeloreRuntime(runtimeDefaults = {}) {
|
||||
declaration.label,
|
||||
declaration.description,
|
||||
declaration.inputSchema,
|
||||
dynamicLeaseTools.has(declaration.name),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
25
electron/coding-runtime/pi/provider-failure.ts
Normal file
25
electron/coding-runtime/pi/provider-failure.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import type { CodingRuntimePublicError } from '../contracts';
|
||||
import { isAIGatewayUserContextMissing } from '../../../shared/ai-gateway-error-details';
|
||||
import { getAIGatewayErrorKind } from '../../../shared/ai-gateway-error-kind';
|
||||
|
||||
export function projectPiProviderFailure(message: unknown): CodingRuntimePublicError {
|
||||
if (typeof message === 'string' && isAIGatewayUserContextMissing(message)) {
|
||||
return {
|
||||
code: 'CODING_PROVIDER_AUTH_REQUIRED',
|
||||
message: '模型服务身份上下文无效,请重试;若仍失败请重新登录。',
|
||||
recoverable: true,
|
||||
};
|
||||
}
|
||||
if (typeof message === 'string' && getAIGatewayErrorKind(message) === 'quota_exhausted') {
|
||||
return {
|
||||
code: 'CODING_PROVIDER_QUOTA_EXHAUSTED',
|
||||
message: '词元点数余额不足,请充值后重试。',
|
||||
recoverable: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
code: 'CODING_RUNTIME_START_FAILED',
|
||||
message: '模型服务请求失败,请稍后重试。',
|
||||
recoverable: true,
|
||||
};
|
||||
}
|
||||
@@ -1416,7 +1416,7 @@ export async function runFinalAsarExtensionProof(): Promise<PiReleaseExtensionPr
|
||||
providerRequests: requestCounts,
|
||||
subagentStatus: 'complete',
|
||||
subagentSummary: 'REAL_CHILD_COMPLETE',
|
||||
materializedExtension: 'makelore-runtime-v4.mjs',
|
||||
materializedExtension: 'makelore-runtime-v5.mjs',
|
||||
providerFirstEventDelayMs: PROOF_PROVIDER_FIRST_EVENT_DELAY_MS,
|
||||
managedTurns,
|
||||
managedWorkerMilestones: composition.telemetry,
|
||||
|
||||
@@ -1536,19 +1536,20 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
const current = snapshot?.run;
|
||||
if (!snapshot
|
||||
|| snapshot.cursor.workerGeneration !== event.generation
|
||||
|| current?.runId !== event.runId
|
||||
|| runIsTerminal(current.status)) return;
|
||||
this.emit(event.conversationId, {
|
||||
op: 'run.state',
|
||||
run: {
|
||||
status: 'idle',
|
||||
runId: event.runId,
|
||||
...(current.mode ? { mode: current.mode } : {}),
|
||||
...(current.startedAt !== undefined ? { startedAt: current.startedAt } : {}),
|
||||
settledAt: this.now(),
|
||||
terminalReason: 'completed',
|
||||
},
|
||||
}, event.runId);
|
||||
|| current?.runId !== event.runId) return;
|
||||
if (!runIsTerminal(current.status)) {
|
||||
this.emit(event.conversationId, {
|
||||
op: 'run.state',
|
||||
run: {
|
||||
status: 'idle',
|
||||
runId: event.runId,
|
||||
...(current.mode ? { mode: current.mode } : {}),
|
||||
...(current.startedAt !== undefined ? { startedAt: current.startedAt } : {}),
|
||||
settledAt: this.now(),
|
||||
terminalReason: current.status === 'aborting' ? 'aborted' : 'completed',
|
||||
},
|
||||
}, event.runId);
|
||||
}
|
||||
this.extensionUi.endRun(event.conversationId, event.runId);
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
@@ -1562,6 +1563,12 @@ export class PiConversationRuntime implements CodingConversationRuntime {
|
||||
} finally {
|
||||
this.releaseRunBackgroundLease(event.conversationId, event.runId);
|
||||
}
|
||||
if (event.source === 'state_probe') {
|
||||
const worker = this.pool.getState(event.conversationId);
|
||||
if (worker?.generation === event.generation && worker.state !== 'crashed') {
|
||||
await this.hydrateGenerationNow(event.conversationId, worker, false);
|
||||
}
|
||||
}
|
||||
}).catch((error) => {
|
||||
this.recordProjectionFailure(event.conversationId, event.generation, error);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
} from './event-projector';
|
||||
import { isProductToolName, productToolDetailsOfResult } from '../product-tool-protocol';
|
||||
import { subagentDetailsOfResult } from '../subagent-protocol';
|
||||
import { projectPiProviderFailure } from './provider-failure';
|
||||
|
||||
export interface PiSessionSnapshotInput {
|
||||
snapshot: ConversationSnapshot;
|
||||
@@ -106,23 +107,6 @@ function activePath(entriesValue: unknown, leafId: unknown): SessionEntryRecord[
|
||||
return path.reverse();
|
||||
}
|
||||
|
||||
function retainedTail(path: SessionEntryRecord[]): SessionEntryRecord[] {
|
||||
const compactionIndex = path.findLastIndex((entry) => entry.type === 'compaction');
|
||||
if (compactionIndex < 0) return path;
|
||||
const compaction = path[compactionIndex];
|
||||
const firstKeptEntryId = compaction.firstKeptEntryId;
|
||||
if (typeof firstKeptEntryId !== 'string') unreadable();
|
||||
const firstKeptIndex = path.findIndex(
|
||||
(entry, index) => index < compactionIndex && entry.id === firstKeptEntryId,
|
||||
);
|
||||
if (firstKeptIndex < 0) unreadable();
|
||||
return [
|
||||
compaction,
|
||||
...path.slice(firstKeptIndex, compactionIndex),
|
||||
...path.slice(compactionIndex + 1),
|
||||
];
|
||||
}
|
||||
|
||||
async function contentBlocks(
|
||||
messageId: string,
|
||||
content: unknown,
|
||||
@@ -265,6 +249,19 @@ async function projectEntries(
|
||||
...(stopReason ? { stopReason } : {}),
|
||||
};
|
||||
nodes.push(node);
|
||||
if (message.stopReason === 'error') {
|
||||
const failure = projectPiProviderFailure(message.errorMessage);
|
||||
if (failure.code === 'CODING_PROVIDER_AUTH_REQUIRED'
|
||||
|| failure.code === 'CODING_PROVIDER_QUOTA_EXHAUSTED') {
|
||||
nodes.push({
|
||||
kind: 'notice',
|
||||
id: `${messageId}:provider-error`,
|
||||
code: failure.code,
|
||||
level: 'error',
|
||||
message: failure.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!Array.isArray(message.content)) continue;
|
||||
for (const value of message.content) {
|
||||
const block = asRecord(value);
|
||||
@@ -310,6 +307,16 @@ function reconcileLiveIds(
|
||||
liveNodes: ConversationNode[],
|
||||
): ConversationNode[] {
|
||||
const used = new Set<string>();
|
||||
const durableCompactions = durableNodes.filter((node) => node.kind === 'compaction');
|
||||
const liveCompactions = liveNodes.filter((node) => node.kind === 'compaction');
|
||||
const compactionMatches = new Map<string, Extract<ConversationNode, { kind: 'compaction' }>>();
|
||||
for (
|
||||
let durableIndex = durableCompactions.length - 1, liveIndex = liveCompactions.length - 1;
|
||||
durableIndex >= 0 && liveIndex >= 0;
|
||||
durableIndex -= 1, liveIndex -= 1
|
||||
) {
|
||||
compactionMatches.set(durableCompactions[durableIndex].id, liveCompactions[liveIndex]);
|
||||
}
|
||||
return durableNodes.map((node) => {
|
||||
if (node.kind === 'message') {
|
||||
const live = liveNodes.find((candidate) => candidate.kind === 'message'
|
||||
@@ -338,10 +345,8 @@ function reconcileLiveIds(
|
||||
};
|
||||
}
|
||||
if (node.kind === 'compaction') {
|
||||
const live = liveNodes.findLast(
|
||||
(candidate) => candidate.kind === 'compaction' && !used.has(candidate.id),
|
||||
);
|
||||
if (!live || live.kind !== 'compaction') return node;
|
||||
const live = compactionMatches.get(node.id);
|
||||
if (!live) return node;
|
||||
used.add(live.id);
|
||||
return { ...node, id: live.id, runId: live.runId };
|
||||
}
|
||||
@@ -375,7 +380,7 @@ export async function projectPiSessionSnapshot(
|
||||
const response = asRecord(input.entries);
|
||||
const state = asRecord(input.state);
|
||||
if (!response || !state) unreadable();
|
||||
const path = retainedTail(activePath(response.entries, response.leafId));
|
||||
const path = activePath(response.entries, response.leafId);
|
||||
const durableNodes = await projectEntries(path, input);
|
||||
const nodes = reconcileLiveIds(durableNodes, input.snapshot.nodes);
|
||||
return {
|
||||
|
||||
@@ -26,6 +26,45 @@ import {
|
||||
} from './telemetry';
|
||||
import { logger } from '../../utils/logger';
|
||||
|
||||
const SETTLEMENT_PROBE_INTERVAL_MS = 3_000;
|
||||
const SETTLEMENT_PROBE_TIMEOUT_MS = 5_000;
|
||||
const TERMINAL_SETTLEMENT_TIMEOUT_MS = 30_000;
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: null;
|
||||
}
|
||||
|
||||
function sessionIsAuthoritativelyIdle(value: unknown): boolean {
|
||||
const state = recordValue(value);
|
||||
return state?.isStreaming === false
|
||||
&& state.isCompacting === false
|
||||
&& state.pendingMessageCount === 0;
|
||||
}
|
||||
|
||||
function sessionIsTerminallyStalled(value: unknown): boolean {
|
||||
const state = recordValue(value);
|
||||
return state?.isStreaming === true
|
||||
&& state.isCompacting === false
|
||||
&& state.pendingMessageCount === 0
|
||||
&& state.retryAttempt === 0;
|
||||
}
|
||||
|
||||
function isTerminalAssistantMessage(event: PiRpcEvent): boolean {
|
||||
if (event.type !== 'message_end') return false;
|
||||
const message = recordValue(event.message);
|
||||
return message?.role === 'assistant'
|
||||
&& ['stop', 'length', 'error', 'aborted'].includes(String(message.stopReason));
|
||||
}
|
||||
|
||||
function continuesAfterTerminalCandidate(event: PiRpcEvent): boolean {
|
||||
return event.type === 'agent_start'
|
||||
|| event.type === 'auto_retry_start'
|
||||
|| event.type === 'compaction_start'
|
||||
|| (event.type === 'message_start' && recordValue(event.message)?.role === 'user');
|
||||
}
|
||||
|
||||
export interface PiConversationWorker {
|
||||
readonly id: string;
|
||||
readonly generation: number;
|
||||
@@ -122,6 +161,7 @@ export type PiWorkerPoolEvent =
|
||||
conversationId: string;
|
||||
generation: number;
|
||||
runId: string;
|
||||
source: 'agent_settled' | 'compact_rpc' | 'state_probe';
|
||||
}
|
||||
| {
|
||||
type: 'top-level.failed';
|
||||
@@ -147,6 +187,9 @@ export interface PiWorkerPoolOptions {
|
||||
revisionCoordinator?: PiManagedInputRevisionCoordinator;
|
||||
now?: () => number;
|
||||
onTelemetry?: (event: PiRuntimeTelemetryEvent) => void;
|
||||
settlementProbeIntervalMs?: number;
|
||||
settlementProbeTimeoutMs?: number;
|
||||
terminalSettlementTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface PiProcessLease {
|
||||
@@ -255,6 +298,20 @@ interface PendingTopLevelRun extends PiTopLevelRunInput {
|
||||
queuedAt?: number;
|
||||
}
|
||||
|
||||
interface ActiveTopLevelRun {
|
||||
runId: string;
|
||||
run: PendingTopLevelRun;
|
||||
generation: number;
|
||||
cold: boolean;
|
||||
acceptedAt?: number;
|
||||
confirmationStartedAt?: number;
|
||||
confirmation: 'pending' | 'uncertain' | 'confirmed';
|
||||
cancelConfirmation?: () => void;
|
||||
settlementProbeTimer?: ReturnType<typeof setTimeout>;
|
||||
settlementProbeInFlight?: boolean;
|
||||
terminalObservedAt?: number;
|
||||
}
|
||||
|
||||
export class PiWorkerPool {
|
||||
private readonly openWorker: PiWorkerPoolOptions['openWorker'];
|
||||
private readonly maxRunning: number;
|
||||
@@ -264,20 +321,14 @@ export class PiWorkerPool {
|
||||
private readonly revisions: PiManagedInputRevisionCoordinator;
|
||||
private readonly now: () => number;
|
||||
private readonly onTelemetry: ((event: PiRuntimeTelemetryEvent) => void) | undefined;
|
||||
private readonly settlementProbeIntervalMs: number;
|
||||
private readonly settlementProbeTimeoutMs: number;
|
||||
private readonly terminalSettlementTimeoutMs: number;
|
||||
private readonly workers = new Map<string, WorkerRecord>();
|
||||
private readonly prepareFlights = new Map<string, Promise<PiWorkerPoolState>>();
|
||||
private readonly rebuildFlights = new Set<Promise<WorkerRecord>>();
|
||||
private readonly waitingRuns: PendingTopLevelRun[] = [];
|
||||
private readonly activeRuns = new Map<string, {
|
||||
runId: string;
|
||||
run: PendingTopLevelRun;
|
||||
generation: number;
|
||||
cold: boolean;
|
||||
acceptedAt?: number;
|
||||
confirmationStartedAt?: number;
|
||||
confirmation: 'pending' | 'uncertain' | 'confirmed';
|
||||
cancelConfirmation?: () => void;
|
||||
}>();
|
||||
private readonly activeRuns = new Map<string, ActiveTopLevelRun>();
|
||||
private readonly generations = new Map<string, number>();
|
||||
private readonly listeners = new Set<(event: PiWorkerPoolEvent) => void>();
|
||||
private readonly reclaimWaiters = new Set<() => void>();
|
||||
@@ -297,12 +348,27 @@ export class PiWorkerPool {
|
||||
this.revisions = options.revisionCoordinator ?? new PiManagedInputRevisionCoordinator();
|
||||
this.now = options.now ?? Date.now;
|
||||
this.onTelemetry = options.onTelemetry;
|
||||
this.settlementProbeIntervalMs = options.settlementProbeIntervalMs
|
||||
?? SETTLEMENT_PROBE_INTERVAL_MS;
|
||||
this.settlementProbeTimeoutMs = options.settlementProbeTimeoutMs
|
||||
?? SETTLEMENT_PROBE_TIMEOUT_MS;
|
||||
this.terminalSettlementTimeoutMs = options.terminalSettlementTimeoutMs
|
||||
?? TERMINAL_SETTLEMENT_TIMEOUT_MS;
|
||||
if (!Number.isSafeInteger(this.maxRunning) || this.maxRunning <= 0) {
|
||||
throw new Error('maxRunning must be a positive safe integer');
|
||||
}
|
||||
if (!Number.isSafeInteger(this.maxIdle) || this.maxIdle < 0) {
|
||||
throw new Error('maxIdle must be a non-negative safe integer');
|
||||
}
|
||||
for (const [name, value] of [
|
||||
['settlementProbeIntervalMs', this.settlementProbeIntervalMs],
|
||||
['settlementProbeTimeoutMs', this.settlementProbeTimeoutMs],
|
||||
['terminalSettlementTimeoutMs', this.terminalSettlementTimeoutMs],
|
||||
] as const) {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prepare(conversation: PrepareConversationInput): Promise<PiWorkerPoolState> {
|
||||
@@ -544,7 +610,7 @@ export class PiWorkerPool {
|
||||
failTopLevel(conversationId: string, runId: string, error: Error): void {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active?.runId === runId) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
this.removeActiveRun(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
const record = this.workers.get(conversationId);
|
||||
@@ -724,7 +790,7 @@ export class PiWorkerPool {
|
||||
const records = [...this.workers.values()];
|
||||
this.workers.clear();
|
||||
const activeRuns = [...this.activeRuns.values()];
|
||||
this.activeRuns.clear();
|
||||
for (const active of activeRuns) this.removeActiveRun(active.run.conversationId);
|
||||
this.runningCount = 0;
|
||||
for (const active of activeRuns) active.cancelConfirmation?.();
|
||||
await Promise.all(records.map(async (record) => {
|
||||
@@ -778,13 +844,7 @@ export class PiWorkerPool {
|
||||
});
|
||||
this.confirmTopLevel(current, run);
|
||||
if (run.command.type === 'compact') {
|
||||
this.settleTopLevel(current);
|
||||
this.emit({
|
||||
type: 'top-level.settled',
|
||||
conversationId: run.conversationId,
|
||||
generation: current.generation,
|
||||
runId: run.runId,
|
||||
});
|
||||
this.settleTopLevelAndNotify(current, 'compact_rpc');
|
||||
}
|
||||
run.resolve(response);
|
||||
} catch (error) {
|
||||
@@ -799,7 +859,7 @@ export class PiWorkerPool {
|
||||
return;
|
||||
}
|
||||
if (active) {
|
||||
this.activeRuns.delete(run.conversationId);
|
||||
this.removeActiveRun(run.conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
this.launchWaitingRuns();
|
||||
@@ -834,6 +894,7 @@ export class PiWorkerPool {
|
||||
active.cold,
|
||||
);
|
||||
record.acceptedPromptCount += 1;
|
||||
this.scheduleSettlementProbe(record, active);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -854,19 +915,13 @@ export class PiWorkerPool {
|
||||
runId: run.runId,
|
||||
});
|
||||
if (run.command.type === 'compact') {
|
||||
this.settleTopLevel(record);
|
||||
this.emit({
|
||||
type: 'top-level.settled',
|
||||
conversationId: run.conversationId,
|
||||
generation: record.generation,
|
||||
runId: run.runId,
|
||||
});
|
||||
this.settleTopLevelAndNotify(record, 'compact_rpc');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (result.error.code === 'PI_RPC_EXITED'
|
||||
|| result.error.code === 'PI_RPC_PROTOCOL_ERROR') return;
|
||||
this.activeRuns.delete(run.conversationId);
|
||||
this.removeActiveRun(run.conversationId);
|
||||
active.cancelConfirmation = undefined;
|
||||
this.runningCount -= 1;
|
||||
if (this.workers.get(run.conversationId) === record && record.state !== 'crashed') {
|
||||
@@ -889,10 +944,10 @@ export class PiWorkerPool {
|
||||
});
|
||||
}
|
||||
|
||||
private settleTopLevel(record: WorkerRecord): void {
|
||||
private settleTopLevel(record: WorkerRecord): ActiveTopLevelRun | null {
|
||||
const conversationId = record.conversation.conversationId;
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (!active || active.generation !== record.generation) return;
|
||||
if (!active || active.generation !== record.generation) return null;
|
||||
const cancelConfirmation = active.cancelConfirmation;
|
||||
if (active.confirmation !== 'confirmed') {
|
||||
this.confirmTopLevel(record, active.run);
|
||||
@@ -911,7 +966,7 @@ export class PiWorkerPool {
|
||||
active.cold,
|
||||
);
|
||||
}
|
||||
this.activeRuns.delete(conversationId);
|
||||
this.removeActiveRun(conversationId);
|
||||
cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
record.state = 'idle';
|
||||
@@ -932,6 +987,110 @@ export class PiWorkerPool {
|
||||
}
|
||||
this.launchWaitingRuns();
|
||||
void this.trimIdleWorkers();
|
||||
return active;
|
||||
}
|
||||
|
||||
private settleTopLevelAndNotify(
|
||||
record: WorkerRecord,
|
||||
source: Extract<PiWorkerPoolEvent, { type: 'top-level.settled' }>['source'],
|
||||
): boolean {
|
||||
const active = this.settleTopLevel(record);
|
||||
if (!active) return false;
|
||||
this.emit({
|
||||
type: 'top-level.settled',
|
||||
conversationId: record.conversation.conversationId,
|
||||
generation: record.generation,
|
||||
runId: active.runId,
|
||||
source,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private observeTopLevelEvent(record: WorkerRecord, event: PiRpcEvent): void {
|
||||
const conversationId = record.conversation.conversationId;
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (!active || active.generation !== record.generation) return;
|
||||
if (continuesAfterTerminalCandidate(event)) {
|
||||
active.terminalObservedAt = undefined;
|
||||
return;
|
||||
}
|
||||
if (isTerminalAssistantMessage(event)
|
||||
|| (event.type === 'agent_end' && event.willRetry === false)) {
|
||||
active.terminalObservedAt ??= this.now();
|
||||
this.scheduleSettlementProbe(record, active);
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleSettlementProbe(record: WorkerRecord, active: ActiveTopLevelRun): void {
|
||||
if (this.shuttingDown
|
||||
|| active.run.command.type !== 'prompt'
|
||||
|| active.settlementProbeTimer
|
||||
|| active.settlementProbeInFlight
|
||||
|| this.activeRuns.get(record.conversation.conversationId) !== active) return;
|
||||
active.settlementProbeTimer = setTimeout(() => {
|
||||
active.settlementProbeTimer = undefined;
|
||||
void this.probeTopLevelSettlement(record, active);
|
||||
}, this.settlementProbeIntervalMs);
|
||||
active.settlementProbeTimer.unref();
|
||||
}
|
||||
|
||||
private async probeTopLevelSettlement(
|
||||
record: WorkerRecord,
|
||||
active: ActiveTopLevelRun,
|
||||
): Promise<void> {
|
||||
const conversationId = record.conversation.conversationId;
|
||||
if (this.activeRuns.get(conversationId) !== active
|
||||
|| active.generation !== record.generation
|
||||
|| this.workers.get(conversationId) !== record
|
||||
|| record.state === 'crashed') return;
|
||||
active.settlementProbeInFlight = true;
|
||||
try {
|
||||
const response = await record.worker.request({ type: 'get_state' }, {
|
||||
retry: 'read-only-once',
|
||||
timeoutMs: this.settlementProbeTimeoutMs,
|
||||
});
|
||||
if (this.activeRuns.get(conversationId) !== active) return;
|
||||
if (sessionIsAuthoritativelyIdle(response.data)) {
|
||||
this.settleTopLevelAndNotify(record, 'state_probe');
|
||||
return;
|
||||
}
|
||||
if (this.terminalSettlementExpired(active)
|
||||
&& sessionIsTerminallyStalled(response.data)) {
|
||||
this.handleInvalidation(record, this.settlementProtocolError(record));
|
||||
}
|
||||
} catch {
|
||||
if (this.activeRuns.get(conversationId) === active
|
||||
&& this.terminalSettlementExpired(active)) {
|
||||
this.handleInvalidation(record, this.settlementProtocolError(record));
|
||||
}
|
||||
} finally {
|
||||
active.settlementProbeInFlight = false;
|
||||
if (this.activeRuns.get(conversationId) === active) {
|
||||
this.scheduleSettlementProbe(record, active);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private terminalSettlementExpired(active: ActiveTopLevelRun): boolean {
|
||||
return active.terminalObservedAt !== undefined
|
||||
&& this.now() - active.terminalObservedAt >= this.terminalSettlementTimeoutMs;
|
||||
}
|
||||
|
||||
private settlementProtocolError(record: WorkerRecord): PiProcessError {
|
||||
return new PiProcessError(
|
||||
'PI_RPC_PROTOCOL_ERROR',
|
||||
'Pi prompt produced a terminal response but did not settle',
|
||||
{ generation: record.generation },
|
||||
);
|
||||
}
|
||||
|
||||
private removeActiveRun(conversationId: string): ActiveTopLevelRun | undefined {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (!active) return undefined;
|
||||
this.activeRuns.delete(conversationId);
|
||||
if (active.settlementProbeTimer) clearTimeout(active.settlementProbeTimer);
|
||||
active.settlementProbeTimer = undefined;
|
||||
return active;
|
||||
}
|
||||
|
||||
private async ensureFresh(record: WorkerRecord): Promise<WorkerRecord> {
|
||||
@@ -1098,7 +1257,17 @@ export class PiWorkerPool {
|
||||
reconfigureAfterSettled: false,
|
||||
};
|
||||
record.unsubscribeEvent = worker.subscribe((event) => {
|
||||
if (event.type === 'agent_settled') this.settleTopLevel(record);
|
||||
if (event.type === 'makelore_thread_error'
|
||||
&& event.code === 'PROMPT_FAILED_AFTER_ACCEPTANCE') {
|
||||
this.handleInvalidation(record, new PiProcessError(
|
||||
'PI_RPC_PROTOCOL_ERROR',
|
||||
'Pi prompt failed after it was accepted',
|
||||
{ generation },
|
||||
));
|
||||
return;
|
||||
}
|
||||
this.observeTopLevelEvent(record, event);
|
||||
if (event.type === 'agent_settled') this.settleTopLevelAndNotify(record, 'agent_settled');
|
||||
this.emit({
|
||||
type: 'worker.event',
|
||||
conversationId: conversation.conversationId,
|
||||
@@ -1190,7 +1359,7 @@ export class PiWorkerPool {
|
||||
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active?.generation === record.generation) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
this.removeActiveRun(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
}
|
||||
@@ -1218,7 +1387,7 @@ export class PiWorkerPool {
|
||||
private cancelConversationRuns(conversationId: string, error: Error): void {
|
||||
const active = this.activeRuns.get(conversationId);
|
||||
if (active) {
|
||||
this.activeRuns.delete(conversationId);
|
||||
this.removeActiveRun(conversationId);
|
||||
active.cancelConfirmation?.();
|
||||
this.runningCount -= 1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user