test: add PI release proof harness

This commit is contained in:
2026-08-24 13:44:39 +08:00
parent 977445ba45
commit 8820e82530
19 changed files with 1407 additions and 44 deletions

View File

@@ -857,9 +857,16 @@ async function promptAndSettle(worker, message, images, timeoutMs) {
(event) => worker.events.indexOf(event) >= eventStartIndex && event.type === 'agent_settled',
timeoutMs,
);
const firstProviderEvent = worker.events
.slice(eventStartIndex)
.find((event) => event !== agentStart && event.receivedAt >= agentStart.receivedAt);
const messagesResponse = await worker.request({ type: 'get_messages' });
return {
acceptedMs: accepted.receivedAt - sentAt,
agentStartMs: agentStart.receivedAt - accepted.receivedAt,
providerFirstEventMs: firstProviderEvent
? firstProviderEvent.receivedAt - agentStart.receivedAt
: settled.receivedAt - agentStart.receivedAt,
startedAt: agentStart.receivedAt,
settledAt: settled.receivedAt,
stopReason: lastAssistantStopReason(messagesResponse),
@@ -944,7 +951,18 @@ export async function runProviderQualification(runtime, scratchRoot, fixturePath
imageInput: Boolean(imagePath),
distinctSessionIds: true,
overlapMs: Math.round(overlapMs),
promptAcceptedSamplesMs: [leftTurn.acceptedMs, rightTurn.acceptedMs].map(Math.round),
promptAcceptedMs: summarizeMeasurements([leftTurn.acceptedMs, rightTurn.acceptedMs]),
agentStartSamplesMs: [leftTurn.agentStartMs, rightTurn.agentStartMs].map(Math.round),
agentStartMs: summarizeMeasurements([leftTurn.agentStartMs, rightTurn.agentStartMs]),
providerFirstEventSamplesMs: [
leftTurn.providerFirstEventMs,
rightTurn.providerFirstEventMs,
].map(Math.round),
providerFirstEventMs: summarizeMeasurements([
leftTurn.providerFirstEventMs,
rightTurn.providerFirstEventMs,
]),
stopReasons: [leftTurn.stopReason, rightTurn.stopReason],
abortIsolation: {
abortedStopReason,