fix(pi): converge worker failures and thinking state

2026-08-25 11:45:14 +08:00
parent 274187e3cf
commit 61817b161f
28 changed files with 2019 additions and 118 deletions
--- a/electron/coding-runtime/pi/release-proof.ts
+++ b/electron/coding-runtime/pi/release-proof.ts
@@ -6,6 +6,7 @@ import path from 'node:path';
 import { promisify } from 'node:util';

 import type { CodingProductComposition } from '../../api/coding-product-services';
+import { getRecentLogs } from '../../utils/logger';
 import { createCodingConversationStore } from '../../coding-projects/conversation-store';
 import { createCodingProjectAgent } from '../../coding-projects/project-config';
 import {
@@ -25,7 +26,7 @@ import { PiManagedExtensionHost } from './extension-host';
 import { PiManagedInputRevisionCoordinator } from './managed-input-revision';
 import { runPiReleasePressureCleanup } from './release-proof-cleanup';
 import type { PiRpcEvent } from './rpc-client';
-import { createPiManagedWorkerOpener } from './runtime';
+import { createPiManagedWorkerOpener, PiConversationRuntime } from './runtime';
 import { PiSessionRegistry } from './session-registry';
 import { createPiManagedSubagentChildOpener } from './subagent-child';
 import { PiSubagentScheduler } from './subagent';
@@ -35,7 +36,7 @@ import { PiProcessBudget, PiWorkerPool, type PiWorkerPoolEvent } from './worker-
 import { PiProjectWriteLeaseCoordinator, type PiProjectWriteLease } from './write-lease';

 type ProofWorkerRole = 'parent' | 'child';
-type ProofProviderMode = 'subagent' | 'pressure';
+type ProofProviderMode = 'subagent' | 'pressure' | 'resilience';
 type ProofMilestone = PiRuntimeTelemetryEvent['milestone'] | 'agent.start' | 'provider.first_event';
 type ProofMilestoneSource = 'main.telemetry' | 'pi.agent_start' | 'pi.assistant_message_start';

@@ -67,6 +68,7 @@ type LocalProofProvider = {
  requests: ProviderRequest[];
  activeCounts(): { parent: number; child: number };
  releaseChildren(): void;
+  releaseParents(): void;
  releaseAll(): void;
  close(): Promise<void>;
 };
@@ -102,6 +104,18 @@ type ProxyCompositionRun = {
  activeStatus?: PiReleaseProxyCompositionStatus;
 };

+type ResilienceCompositionRun = {
+  composition: CodingProductComposition;
+  provider: LocalProofProvider;
+  projectId: string;
+  projectPath: string;
+  targetConversationId: string;
+  otherConversationId: string;
+  targetBinding: { piSessionId: string; sessionKey: string };
+  otherBinding: { piSessionId: string; sessionKey: string };
+  hostToken: string;
+};
+
 type InspectedPiProcess = {
  processId: number;
  role: ProofWorkerRole;
@@ -187,6 +201,49 @@ export interface PiReleaseProxyCompositionProof extends PiReleaseProxyCompositio
  released: { workers: number };
 }

+export interface PiReleaseResilienceStatus {
+  target: {
+    conversationId: string;
+    workerStatus: string;
+    workerGeneration: number;
+    runStatus: string;
+    errorCode: string | null;
+    recoverable: boolean;
+    bindingPreserved: boolean;
+  };
+  other: {
+    conversationId: string;
+    workerStatus: string;
+    workerGeneration: number;
+    runStatus: string;
+    bindingPreserved: boolean;
+  };
+  providerRequests: { parent: number; child: number };
+  activeProviderRequests: { parent: number; child: number };
+  resources: ReturnType<PiConversationRuntime['getResilienceProofDiagnostics']>;
+  processes: { supported: boolean; parent: number[]; child: number[] };
+  realTurnVerified: false;
+}
+
+export interface PiReleaseResilienceProof extends PiReleaseResilienceStatus {
+  lifecycle: {
+    unexpectedExit: boolean;
+    protocolInvalidation: boolean;
+    intentionalStop: boolean;
+    everyStopHasReason: boolean;
+    everyReplacementHasReason: boolean;
+    diagnosticRedacted: boolean;
+    promptFree: boolean;
+    projectPathFree: boolean;
+  };
+  released: {
+    workers: number;
+    processes: number;
+    resources: ReturnType<PiConversationRuntime['getResilienceProofDiagnostics']>;
+  };
+  realTurnVerified: false;
+}
+
 const PROOF_ACCOUNT_ID = 'release-proof-account';
 const PROOF_AGENT_ID = 'release-proof-agent';
 const PROOF_MODEL_ID = 'release-proof-model';
@@ -205,6 +262,7 @@ const EXPECTED_TURN_MILESTONES: readonly ProofMilestone[] = [

 let pressureRun: PressureRun | null = null;
 let proxyCompositionRun: ProxyCompositionRun | null = null;
+let resilienceCompositionRun: ResilienceCompositionRun | null = null;
 const execFileAsync = promisify(execFile);

 async function waitFor(predicate: () => boolean, message: string): Promise<void> {
@@ -298,7 +356,11 @@ function respondWithText(response: ServerResponse, model: string, text: string):
  finishResponse(response, model);
 }

-function respondWithSubagentCall(response: ServerResponse, model: string): void {
+function respondWithSubagentCall(
+  response: ServerResponse,
+  model: string,
+  toolProfile: 'read-only' | 'coding' = 'read-only',
+): void {
  response.writeHead(200, { 'content-type': 'text/event-stream' });
  writeChunk(response, model, {
    role: 'assistant',
@@ -313,7 +375,7 @@ function respondWithSubagentCall(response: ServerResponse, model: string): void
          tasks: [{
            agentId: PROOF_AGENT_ID,
            task: 'Return REAL_CHILD_COMPLETE for final packaged qualification.',
-            toolProfile: 'read-only',
+            toolProfile,
          }],
        }),
      },
@@ -323,6 +385,30 @@ function respondWithSubagentCall(response: ServerResponse, model: string): void
  response.end('data: [DONE]\n\n');
 }

+function respondWithHoldingBashCall(response: ServerResponse, model: string): void {
+  response.writeHead(200, { 'content-type': 'text/event-stream' });
+  writeChunk(response, model, {
+    role: 'assistant',
+    tool_calls: [{
+      index: 0,
+      id: 'release-proof-holding-bash',
+      type: 'function',
+      function: {
+        name: 'bash',
+        arguments: JSON.stringify({
+          command: 'powershell.exe -NoProfile -NonInteractive -Command "Start-Sleep -Seconds 120"',
+        }),
+      },
+    }],
+  }, null);
+  writeChunk(response, model, {}, 'tool_calls');
+  response.end('data: [DONE]\n\n');
+}
+
+function requestContains(body: Record<string, unknown>, marker: string): boolean {
+  return JSON.stringify(body.messages ?? []).includes(marker);
+}
+
 async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalProofProvider> {
  const requests: ProviderRequest[] = [];
  const held = new Set<HeldProviderResponse>();
@@ -342,6 +428,32 @@ async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalPr
      const toolResult = hasToolResult(body);
      requests.push({ role, toolNames, hasToolResult: toolResult });

+      if (mode === 'resilience') {
+        if (role === 'child') {
+          if (toolResult) respondWithText(response, model, 'RESILIENCE_CHILD_COMPLETE');
+          else respondWithHoldingBashCall(response, model);
+          return;
+        }
+        if (requestContains(body, 'RESILIENCE_TARGET_ACTIVE')) {
+          respondWithSubagentCall(response, model, 'coding');
+          return;
+        }
+        if (requestContains(body, 'RESILIENCE_RECOVERED_TURN')) {
+          respondWithText(response, model, 'RESILIENCE_RECOVERED_COMPLETE');
+          return;
+        }
+        if (requestContains(body, 'RESILIENCE_SETTLED_BEFORE_CLOSE')) {
+          respondWithText(response, model, 'RESILIENCE_SETTLED_COMPLETE');
+          return;
+        }
+        response.writeHead(200, { 'content-type': 'text/event-stream' });
+        writeChunk(response, model, { role: 'assistant', content: 'RESILIENCE_ACTIVE' }, null);
+        const entry = { role, response };
+        held.add(entry);
+        response.once('close', () => held.delete(entry));
+        return;
+      }
+
      if (mode === 'subagent' && role === 'parent' && !toolResult) {
        await delay(PROOF_PROVIDER_FIRST_EVENT_DELAY_MS);
        respondWithSubagentCall(response, model);
@@ -393,6 +505,7 @@ async function startLocalProofProvider(mode: ProofProviderMode): Promise<LocalPr
      child: [...held].filter(({ role }) => role === 'child').length,
    }),
    releaseChildren: () => release('child'),
+    releaseParents: () => release('parent'),
    releaseAll: () => release(),
    close: async () => {
      if (closed) return;
@@ -1276,6 +1389,296 @@ export async function finishFinalAsarProxyCompositionProof(): Promise<PiReleaseP
  }
 }

+function resilienceRuntime(run: ResilienceCompositionRun): PiConversationRuntime {
+  if (!(run.composition.runtime instanceof PiConversationRuntime)) {
+    throw new Error('Packaged Main does not own the Pi Conversation runtime');
+  }
+  return run.composition.runtime;
+}
+
+async function resilienceBindingPreserved(
+  run: ResilienceCompositionRun,
+  conversationId: string,
+  expected: { piSessionId: string; sessionKey: string },
+): Promise<boolean> {
+  const binding = await run.composition.projects.conversationStore(run.projectPath).get(conversationId);
+  return binding?.piSessionId === expected.piSessionId && binding.sessionKey === expected.sessionKey;
+}
+
+export async function getFinalAsarResilienceStatus(): Promise<PiReleaseResilienceStatus> {
+  const run = resilienceCompositionRun;
+  if (!run) throw new Error('PI resilience proof is not running');
+  const runtime = resilienceRuntime(run);
+  const [target, other, processInspection, targetBindingPreserved, otherBindingPreserved] = await Promise.all([
+    run.composition.conversations.getSnapshot(run.targetConversationId),
+    run.composition.conversations.getSnapshot(run.otherConversationId),
+    inspectWindowsPiProcesses(run.hostToken),
+    resilienceBindingPreserved(run, run.targetConversationId, run.targetBinding),
+    resilienceBindingPreserved(run, run.otherConversationId, run.otherBinding),
+  ]);
+  const targetError = target.run.error ?? target.worker.error;
+  return {
+    target: {
+      conversationId: run.targetConversationId,
+      workerStatus: target.worker.status,
+      workerGeneration: target.cursor.workerGeneration,
+      runStatus: target.run.status,
+      errorCode: targetError?.code ?? null,
+      recoverable: targetError?.recoverable ?? false,
+      bindingPreserved: targetBindingPreserved,
+    },
+    other: {
+      conversationId: run.otherConversationId,
+      workerStatus: other.worker.status,
+      workerGeneration: other.cursor.workerGeneration,
+      runStatus: other.run.status,
+      bindingPreserved: otherBindingPreserved,
+    },
+    providerRequests: providerRequestCounts(run.provider),
+    activeProviderRequests: run.provider.activeCounts(),
+    resources: runtime.getResilienceProofDiagnostics(),
+    processes: {
+      supported: processInspection.supported,
+      parent: processInspection.processes
+        .filter(({ role }) => role === 'parent')
+        .map(({ processId }) => processId)
+        .sort((left, right) => left - right),
+      child: processInspection.processes
+        .filter(({ role }) => role === 'child')
+        .map(({ processId }) => processId)
+        .sort((left, right) => left - right),
+    },
+    realTurnVerified: false,
+  };
+}
+
+async function waitForResilienceStatus(
+  predicate: (status: PiReleaseResilienceStatus) => boolean,
+  message: string,
+): Promise<PiReleaseResilienceStatus> {
+  const deadline = Date.now() + 30_000;
+  let latest: PiReleaseResilienceStatus | null = null;
+  while (Date.now() < deadline) {
+    latest = await getFinalAsarResilienceStatus();
+    if (predicate(latest)) return latest;
+    await delay(20);
+  }
+  throw new Error(`${message}: ${JSON.stringify(latest)}`);
+}
+
+export async function startFinalAsarResilienceProof(input: {
+  composition: CodingProductComposition;
+  projectPath: string;
+  hostProxyBaseUrl: string;
+  hostToken: string;
+}): Promise<{
+  projectId: string;
+  targetConversationId: string;
+  otherConversationId: string;
+  bindingsEstablished: true;
+  otherRunAccepted: true;
+  realTurnVerified: false;
+}> {
+  if (resilienceCompositionRun) throw new Error('PI resilience proof is already running');
+  const hostToken = input.hostToken.trim();
+  if (!hostToken) throw new Error('Current Main Host token is unavailable');
+  const provider = await startLocalProofProvider('resilience');
+  let projectId: string | null = null;
+  try {
+    seedWorksSquareAIGatewayCredential({
+      accessToken: 'release-proof-upstream-only',
+      oneApiBaseUrl: provider.baseUrl,
+    });
+    const providerService = getProviderService();
+    await providerService.createAccount(proxyProviderAccount(input.hostProxyBaseUrl));
+    await providerService.setDefaultAccount(PROOF_ACCOUNT_ID);
+    const project = await input.composition.projects.createProject({ projectPath: input.projectPath });
+    projectId = project.project.id;
+    await createCodingProjectAgent(input.projectPath, {
+      id: PROOF_AGENT_ID,
+      avatarId: 'avatar-01',
+      roleName: 'Packaged resilience proof',
+      name: 'Packaged resilience proof agent',
+      model: {
+        accountId: PROOF_ACCOUNT_ID,
+        modelId: PROOF_MODEL_ID,
+        thinkingLevel: 'medium',
+      },
+      modelResolution: 'resolved',
+      responsibility: {
+        mission: 'Exercise packaged worker failure convergence',
+        owns: [],
+        boundaries: [],
+        collaborators: [],
+        principles: [],
+      },
+      prompt: 'Follow the controlled loopback resilience qualification Provider.',
+      skillIds: [],
+    });
+    const other = await input.composition.conversations.createConversation({
+      projectId,
+      agentId: PROOF_AGENT_ID,
+      title: 'Resilience isolation control',
+    });
+    const target = await input.composition.conversations.createConversation({
+      projectId,
+      agentId: PROOF_AGENT_ID,
+      title: 'Resilience fault target',
+    });
+    await Promise.all([
+      input.composition.conversations.getSnapshot(other.id),
+      input.composition.conversations.getSnapshot(target.id),
+    ]);
+    const store = input.composition.projects.conversationStore(input.projectPath);
+    const [targetStored, otherStored] = await Promise.all([store.get(target.id), store.get(other.id)]);
+    if (!targetStored?.piSessionId || !targetStored.sessionKey
+      || !otherStored?.piSessionId || !otherStored.sessionKey) {
+      throw new Error('Resilience proof did not establish both Pi session bindings');
+    }
+    resilienceCompositionRun = {
+      composition: input.composition,
+      provider,
+      projectId,
+      projectPath: input.projectPath,
+      targetConversationId: target.id,
+      otherConversationId: other.id,
+      targetBinding: { piSessionId: targetStored.piSessionId, sessionKey: targetStored.sessionKey },
+      otherBinding: { piSessionId: otherStored.piSessionId, sessionKey: otherStored.sessionKey },
+      hostToken,
+    };
+    const accepted = await input.composition.conversations.acceptPrompt({
+      conversationId: other.id,
+      clientRequestId: 'release-proof-other-active',
+      mode: 'prompt',
+      text: 'RESILIENCE_OTHER_ACTIVE',
+      attachments: [],
+    });
+    if (!accepted.accepted) throw new Error('Resilience isolation run was not accepted');
+    await waitForResilienceStatus(
+      (status) => status.other.runStatus === 'running'
+        && status.activeProviderRequests.parent === 1,
+      'Resilience isolation run did not become active',
+    );
+    return {
+      projectId,
+      targetConversationId: target.id,
+      otherConversationId: other.id,
+      bindingsEstablished: true,
+      otherRunAccepted: true,
+      realTurnVerified: false,
+    };
+  } catch (error) {
+    provider.releaseAll();
+    if (projectId) await input.composition.projects.removeProject(projectId).catch(() => undefined);
+    await provider.close().catch(() => undefined);
+    clearWorksSquareAIGatewayCredential();
+    await getProviderService().deleteAccount(PROOF_ACCOUNT_ID).catch(() => undefined);
+    resilienceCompositionRun = null;
+    throw error;
+  }
+}
+
+export async function injectFinalAsarResilienceFailure(
+  failure: 'unexpected_exit' | 'protocol_invalidation',
+): Promise<{ generation: number; terminalizationMs: number; status: PiReleaseResilienceStatus }> {
+  const run = resilienceCompositionRun;
+  if (!run) throw new Error('PI resilience proof is not running');
+  const startedAt = Date.now();
+  const injected = await resilienceRuntime(run).injectWorkerFailureForProof(
+    run.targetConversationId,
+    failure,
+  );
+  const status = await waitForResilienceStatus(
+    (candidate) => candidate.target.workerStatus === 'error'
+      && candidate.target.workerGeneration === injected.generation,
+    'Injected Pi worker failure did not reach the target Conversation',
+  );
+  const terminalizationMs = Date.now() - startedAt;
+  if (terminalizationMs > 2_000) {
+    throw new Error(`Injected Pi worker failure exceeded the 2s convergence bound: ${terminalizationMs}ms`);
+  }
+  return { ...injected, terminalizationMs, status };
+}
+
+export async function abortFinalAsarResilienceTarget(): Promise<PiReleaseResilienceStatus> {
+  const run = resilienceCompositionRun;
+  if (!run) throw new Error('PI resilience proof is not running');
+  await run.composition.conversations.abort(run.targetConversationId);
+  return await getFinalAsarResilienceStatus();
+}
+
+export function releaseFinalAsarResilienceParents(): void {
+  const run = resilienceCompositionRun;
+  if (!run) throw new Error('PI resilience proof is not running');
+  run.provider.releaseParents();
+}
+
+export async function finishFinalAsarResilienceProof(): Promise<PiReleaseResilienceProof> {
+  const run = resilienceCompositionRun;
+  if (!run) throw new Error('PI resilience proof is not running');
+  try {
+    const status = await getFinalAsarResilienceStatus();
+    run.provider.releaseAll();
+    await run.composition.projects.removeProject(run.projectId);
+    const processInspection = await inspectWindowsPiProcesses(run.hostToken);
+    const resources = resilienceRuntime(run).getResilienceProofDiagnostics();
+    const lifecycleLogs = getRecentLogs().filter((line) => (
+      line.includes('[PiWorkerLifecycle]')
+        && (line.includes(run.targetConversationId) || line.includes(run.otherConversationId))
+    ));
+    const stopLogs = lifecycleLogs.filter((line) => line.includes('"stage": "stop_requested"'));
+    const replacementLogs = lifecycleLogs.filter((line) => line.includes('worker.replacement_'));
+    const lifecycle = {
+      unexpectedExit: lifecycleLogs.some((line) => line.includes('"classification": "unexpected_exit"')),
+      protocolInvalidation: lifecycleLogs.some((line) => line.includes('"classification": "protocol_invalidation"')),
+      intentionalStop: lifecycleLogs.some((line) => line.includes('"classification": "intentional_stop"')),
+      everyStopHasReason: stopLogs.length > 0 && stopLogs.every((line) => line.includes('"reason":')),
+      everyReplacementHasReason: replacementLogs.every((line) => line.includes('"reason":')),
+      diagnosticRedacted: lifecycleLogs.some((line) => line.includes('[REDACTED]'))
+        && lifecycleLogs.every((line) => !line.includes('packaged-proof-secret')),
+      promptFree: lifecycleLogs.every((line) => !line.includes('RESILIENCE_')),
+      projectPathFree: lifecycleLogs.every((line) => !line.includes(run.projectPath)),
+    };
+    if (Object.values(lifecycle).some((value) => value !== true)) {
+      throw new Error(`Packaged Pi lifecycle evidence is incomplete: ${JSON.stringify(lifecycle)}`);
+    }
+    const released = {
+      workers: run.composition.runtime.getDiagnostics().workers.length,
+      processes: processInspection.processes.length,
+      resources,
+    };
+    if (released.workers !== 0
+      || released.processes !== 0
+      || released.resources.pool.processBudget.active !== 0
+      || released.resources.pool.processBudget.waiting !== 0
+      || released.resources.pool.runs.active !== 0
+      || released.resources.pool.runs.waiting !== 0
+      || released.resources.subagents?.activeChildPermits !== 0
+      || released.resources.subagents?.waitingChildPermits !== 0
+      || released.resources.subagents?.activeDispatches !== 0
+      || released.resources.subagents?.activeParents !== 0
+      || released.resources.extension?.registrations.parent !== 0
+      || released.resources.extension?.registrations.child !== 0
+      || released.resources.extension?.writeLeases.active !== 0
+      || released.resources.extension?.writeLeases.waiting !== 0) {
+      throw new Error(`Packaged Pi resources were not fully released: ${JSON.stringify(released)}`);
+    }
+    await run.provider.close();
+    clearWorksSquareAIGatewayCredential();
+    await getProviderService().deleteAccount(PROOF_ACCOUNT_ID).catch(() => undefined);
+    resilienceCompositionRun = null;
+    return { ...status, lifecycle, released, realTurnVerified: false };
+  } catch (error) {
+    run.provider.releaseAll();
+    await run.composition.projects.removeProject(run.projectId).catch(() => undefined);
+    await run.provider.close().catch(() => undefined);
+    clearWorksSquareAIGatewayCredential();
+    await getProviderService().deleteAccount(PROOF_ACCOUNT_ID).catch(() => undefined);
+    resilienceCompositionRun = null;
+    throw error;
+  }
+}
+
 export async function startFinalAsarPressureProof(): Promise<PiReleasePressureSnapshot> {
  if (pressureRun) throw new Error('PI release pressure proof is already running');
  pressureRun = await startPressureRun();