wangxuming/makelore
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(pi): converge worker failures and thinking state

Browse Source
This commit is contained in:
brother7
2026-08-25 11:45:14 +08:00
parent 274187e3cf
commit 61817b161f
28 changed files with 2019 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
scripts/run-pi-subagent-packaged-smoke.mjs
View File

@@ -1,15 +1,19 @@
#!/usr/bin/env node
import { _electron as electron } from '@playwright/test';
import { execFile } from 'node:child_process';
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { createServer } from 'node:net';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { performance } from 'node:perf_hooks';
import { promisify } from 'node:util';
import { defaultProductExecutable } from './lib/pi-product-artifact.mjs';
const execFileAsync = promisify(execFile);
function parseArgs(argv, projectRoot = process.cwd()) {
const options = {
projectRoot: resolve(projectRoot),
@@ -175,6 +179,41 @@ async function waitForActiveProxyProof(electronApplication) {
throw new Error(`Packaged Main proxy child did not become active: ${JSON.stringify(latest?.proxy)}`);
}
async function waitForResilienceProof(electronApplication, predicate, message) {
const deadline = Date.now() + 30_000;
let latest;
while (Date.now() < deadline) {
latest = await evaluateProof(electronApplication, 'resilience.status');
if (predicate(latest?.resilience)) return latest;
await new Promise((resolveWait) => setTimeout(resolveWait, 100));
}
throw new Error(`${message}: ${JSON.stringify(latest?.resilience)}`);
}
function assertResilienceTerminal(status, expectedRunStatus = 'error') {
if (status?.target?.workerStatus !== 'error'
|| status?.target?.runStatus !== expectedRunStatus
|| status?.target?.errorCode !== 'CODING_RUNTIME_START_FAILED'
|| status?.target?.recoverable !== true
|| status?.target?.bindingPreserved !== true
|| status?.other?.bindingPreserved !== true) {
throw new Error(`Packaged resilience target did not converge: ${JSON.stringify(status)}`);
}
}
async function recoverSelectedConversation(page, electronApplication) {
await page.getByRole('button', { name: '恢复' }).first().click({ timeout: 10_000 });
const recovered = await waitForResilienceProof(
electronApplication,
(status) => status?.target?.workerStatus === 'ready' && status?.target?.runStatus === 'idle',
'Packaged resilience target did not recover',
);
const composer = page.getByRole('textbox');
await composer.waitFor({ state: 'visible', timeout: 10_000 });
if (!await composer.isEnabled()) throw new Error('Recovered resilience composer is disabled');
return recovered;
}
async function evaluateProof(electronApplication, action) {
return await electronApplication.evaluate(async (_electron, requestedAction) => {
const proof = globalThis.__niancodeRunPiReleaseProofE2E;
@@ -190,6 +229,42 @@ async function closeApplication(electronApplication) {
]);
}
async function windowsDescendantProcessIds(rootPid) {
if (process.platform !== 'win32') return [];
const script = 'Get-CimInstance Win32_Process | Select-Object ProcessId, ParentProcessId | ConvertTo-Json -Compress';
const { stdout } = await execFileAsync(
'powershell.exe',
['-NoProfile', '-NonInteractive', '-Command', script],
{ windowsHide: true, maxBuffer: 4 * 1024 * 1024 },
);
const parsed = stdout.trim() ? JSON.parse(stdout) : [];
const processes = (Array.isArray(parsed) ? parsed : [parsed]).flatMap((entry) => {
const pid = Number(entry?.ProcessId);
const parentPid = Number(entry?.ParentProcessId);
return Number.isSafeInteger(pid) && Number.isSafeInteger(parentPid) ? [{ pid, parentPid }] : [];
});
const descendants = new Set();
const frontier = [rootPid];
while (frontier.length > 0) {
const parentPid = frontier.shift();
for (const processEntry of processes) {
if (processEntry.parentPid !== parentPid || descendants.has(processEntry.pid)) continue;
descendants.add(processEntry.pid);
frontier.push(processEntry.pid);
}
}
return [...descendants];
}
function processIsAlive(pid) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
export async function runPackagedProductProof(options) {
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-final-product-proof-'));
const homeDir = join(scratchRoot, 'home');
@@ -204,6 +279,7 @@ export async function runPackagedProductProof(options) {
let electronApplication;
let pressureActive = false;
let proxyActive = false;
let resilienceActive = false;
try {
electronApplication = await electron.launch({
executablePath: options.electronExecutable,
@@ -332,6 +408,143 @@ export async function runPackagedProductProof(options) {
assertPackagedMain(proxyFinish);
assertProxyProof(proxyFinish.proxy);
const resilienceStart = await evaluateProof(electronApplication, 'resilience.start');
resilienceActive = true;
assertPackagedMain(resilienceStart);
if (resilienceStart?.resilience?.bindingsEstablished !== true
|| resilienceStart?.resilience?.otherRunAccepted !== true
|| resilienceStart?.resilience?.realTurnVerified !== false) {
throw new Error(`Packaged resilience setup failed: ${JSON.stringify(resilienceStart?.resilience)}`);
}
await page.reload();
await page.waitForLoadState('domcontentloaded');
if (await page.getByTestId('ai-module-selection-page').count()) {
await page.getByTestId('ai-module-option-programming').click();
await page.getByTestId('main-layout').waitFor({ state: 'visible', timeout: 10_000 });
}
await page.evaluate(() => { window.location.hash = '/chat'; });
const resilienceComposer = page.getByRole('textbox');
await resilienceComposer.waitFor({ state: 'visible', timeout: 30_000 });
await resilienceComposer.fill('RESILIENCE_TARGET_ACTIVE');
await page.getByRole('button', { name: '发送' }).click({ timeout: 30_000 });
const activeResilience = await waitForResilienceProof(
electronApplication,
(status) => status?.target?.runStatus === 'running'
&& status?.other?.runStatus === 'running'
&& status?.resources?.subagents?.activeChildPermits === 1
&& status?.resources?.subagents?.activeDispatches === 1
&& status?.resources?.extension?.registrations?.child === 1
&& status?.resources?.extension?.writeLeases?.active === 1
&& status?.processes?.parent?.length === 2
&& status?.processes?.child?.length === 1,
'Packaged resilience parent/child/write-lease fault state did not become active',
);
const requestsBeforeExit = activeResilience.resilience.providerRequests;
const exitFailure = await evaluateProof(electronApplication, 'resilience.inject-exit');
assertPackagedMain(exitFailure);
assertResilienceTerminal(exitFailure.resilience.status);
if (exitFailure.resilience.terminalizationMs > 2_000) {
throw new Error(`Packaged worker exit convergence exceeded 2s: ${exitFailure.resilience.terminalizationMs}`);
}
const releasedAfterExit = await waitForResilienceProof(
electronApplication,
(status) => status?.other?.runStatus === 'running'
&& status?.resources?.subagents?.activeChildPermits === 0
&& status?.resources?.subagents?.activeDispatches === 0
&& status?.resources?.extension?.registrations?.child === 0
&& status?.resources?.extension?.writeLeases?.active === 0
&& status?.processes?.child?.length === 0,
'Target crash did not release child and write-lease resources or preserve the other run',
);
await page.getByText('本地 Agent 已中断，原请求未自动重发。').waitFor({ state: 'visible', timeout: 10_000 });
const composerAfterExit = await page.getByTestId('coding-message-composer').innerText();
if (composerAfterExit.includes('当前对话正在处理') || composerAfterExit.includes('生成中')) {
throw new Error(`Packaged resilience UI remained busy after worker exit: ${composerAfterExit}`);
}
if (!await resilienceComposer.isEnabled()) {
throw new Error('Packaged resilience composer stayed disabled after worker exit');
}
const abortAfterExit = await evaluateProof(electronApplication, 'resilience.abort-after-exit');
assertPackagedMain(abortAfterExit);
assertResilienceTerminal(abortAfterExit.resilience);
const recoveredAfterExit = await recoverSelectedConversation(page, electronApplication);
if (JSON.stringify(recoveredAfterExit.resilience.providerRequests) !== JSON.stringify(requestsBeforeExit)) {
throw new Error('Recover replayed the uncertain accepted prompt after worker exit');
}
await resilienceComposer.fill('RESILIENCE_RECOVERED_TURN');
await page.getByRole('button', { name: '发送' }).click({ timeout: 30_000 });
await page.getByText('RESILIENCE_RECOVERED_COMPLETE').waitFor({ state: 'visible', timeout: 30_000 });
await evaluateProof(electronApplication, 'resilience.release-parents');
await waitForResilienceProof(
electronApplication,
(status) => status?.other?.runStatus === 'idle' && status?.target?.runStatus === 'idle',
'Isolation control Conversation did not settle after release',
);
await resilienceComposer.fill('RESILIENCE_PROTOCOL_ACTIVE');
await page.getByRole('button', { name: '发送' }).click({ timeout: 30_000 });
const protocolActive = await waitForResilienceProof(
electronApplication,
(status) => status?.target?.runStatus === 'running'
&& status?.activeProviderRequests?.parent === 1,
'Protocol invalidation target did not become active',
);
const protocolFailure = await evaluateProof(electronApplication, 'resilience.inject-protocol');
assertPackagedMain(protocolFailure);
assertResilienceTerminal(protocolFailure.resilience.status);
if (protocolFailure.resilience.terminalizationMs > 2_000) {
throw new Error(`Packaged protocol convergence exceeded 2s: ${protocolFailure.resilience.terminalizationMs}`);
}
await page.getByText('本地 Agent 已中断，原请求未自动重发。').waitFor({ state: 'visible', timeout: 10_000 });
const recoveredAfterProtocol = await recoverSelectedConversation(page, electronApplication);
if (JSON.stringify(recoveredAfterProtocol.resilience.providerRequests)
!== JSON.stringify(protocolFailure.resilience.status.providerRequests)) {
throw new Error('Recover replayed the uncertain prompt after protocol invalidation');
}
await resilienceComposer.fill('RESILIENCE_SETTLED_BEFORE_CLOSE');
await page.getByRole('button', { name: '发送' }).click({ timeout: 30_000 });
await page.getByText('RESILIENCE_SETTLED_COMPLETE').waitFor({ state: 'visible', timeout: 30_000 });
const settledBeforeClose = await waitForResilienceProof(
electronApplication,
(status) => status?.target?.runStatus === 'idle' && status?.target?.workerStatus === 'ready',
'Settled-before-close target did not reach its authoritative terminal event',
);
const settledClose = await evaluateProof(electronApplication, 'resilience.inject-exit');
assertPackagedMain(settledClose);
assertResilienceTerminal(settledClose.resilience.status, 'idle');
await recoverSelectedConversation(page, electronApplication);
const resilienceFinish = await evaluateProof(electronApplication, 'resilience.finish');
resilienceActive = false;
assertPackagedMain(resilienceFinish);
if (resilienceFinish?.resilience?.released?.workers !== 0
|| resilienceFinish?.resilience?.released?.processes !== 0
|| Object.values(resilienceFinish?.resilience?.lifecycle ?? {}).some((value) => value !== true)
|| resilienceFinish?.resilience?.realTurnVerified !== false) {
throw new Error(`Packaged resilience proof failed: ${JSON.stringify(resilienceFinish?.resilience)}`);
}
const mainProcessId = electronApplication.process().pid;
const proofProcessIds = [
mainProcessId,
...await windowsDescendantProcessIds(mainProcessId),
];
await closeApplication(electronApplication);
electronApplication = undefined;
const exitDeadline = Date.now() + 5_000;
let lingeringProcessIds = proofProcessIds.filter(processIsAlive);
while (lingeringProcessIds.length > 0 && Date.now() < exitDeadline) {
await new Promise((resolveWait) => setTimeout(resolveWait, 100));
lingeringProcessIds = proofProcessIds.filter(processIsAlive);
}
if (lingeringProcessIds.length > 0) {
throw new Error(`Packaged proof left Electron/Pi processes alive: ${lingeringProcessIds.join(',')}`);
}
const report = {
schemaVersion: 1,
generatedAt: new Date().toISOString(),
@@ -362,6 +575,33 @@ export async function runPackagedProductProof(options) {
},
realTurnVerified: false,
},
resilience: {
setup: resilienceStart.resilience,
active: activeResilience.resilience,
exit: exitFailure.resilience,
abortAfterExit: abortAfterExit.resilience,
releasedAfterExit: releasedAfterExit.resilience,
protocol: {
active: protocolActive.resilience,
failure: protocolFailure.resilience,
},
settledBeforeClose: {
settled: settledBeforeClose.resilience,
close: settledClose.resilience,
},
completed: resilienceFinish.resilience,
ui: {
terminalErrorVisible: true,
composerUnlocked: true,
recoverAvailable: true,
uncertainPromptReplayed: false,
},
realTurnVerified: false,
},
cleanExit: {
trackedProcessIds: proofProcessIds,
lingeringProcessIds,
},
result: 'pass',
};
if (options.reportPath) {
@@ -377,6 +617,10 @@ export async function runPackagedProductProof(options) {
await evaluateProof(electronApplication, 'proxy.release-child').catch(() => undefined);
await evaluateProof(electronApplication, 'proxy.finish').catch(() => undefined);
}
if (electronApplication && resilienceActive) {
await evaluateProof(electronApplication, 'resilience.release-parents').catch(() => undefined);
await evaluateProof(electronApplication, 'resilience.finish').catch(() => undefined);
}
if (electronApplication) await closeApplication(electronApplication);
await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
}