import { writeFileSync } from 'fs';
import { join } from 'path';
import { renderHeader, renderSummaryTable } from '../shared/markdown-report';
import type { EvalReport } from './types';
function pct(rate: number): string {
return `${Math.round(rate * 100)}%`;
}
function countErrors(samples: { error?: string }[]): number {
return samples.filter((s) => s.error).length;
}
/**
* Write `report.md` summarising pre-fix vs post-fix END rates per scenario.
* Returns the absolute path of the written report.
*/
export function writeReport(runDir: string, report: EvalReport): string {
const lines: string[] = [];
lines.push(
...renderHeader({
title: 'Director Premature-END Regression Eval',
timestamp: new Date().toISOString(),
model: report.model,
extra: {
'Samples per variant': report.samplesPerVariant,
'Post-fix END threshold (regression guard)': pct(report.postFixEndThreshold),
'Discrimination threshold (Δ END-rate, informational)': pct(report.thresholdDelta),
Method:
'A/B director prompt + summary: post-fix = current main; pre-fix = pre-#554 [User]/[Assistant] summary labels AND system.md without rules 10/11/12',
'Post-fix regression guard (must hold)': report.allPostFixPass ? 'PASS' : 'FAIL',
'Any scenario discriminates? (informational)': report.anyDiscriminates ? 'YES' : 'NO',
},
}),
);
lines.push(`## Detail`, ``);
for (const r of report.results) {
const pass = r.postFixPasses ? 'PASS' : '**FAIL**';
const disc = r.discriminates ? ' (Δ ≥ threshold)' : '';
lines.push(`### ${pass}${disc} ${r.case_id}`, ``);
lines.push(`- **Description**: ${r.description}`);
const preErr = countErrors(r.preFix.samples);
const postErr = countErrors(r.postFix.samples);
lines.push(`- **Samples per variant**: ${r.samples} (rates exclude errored samples)`);
lines.push(
`- **Pre-fix END rate**: ${pct(r.preFix.endRate)}${preErr ? ` — ${preErr} error(s)` : ''}`,
);
lines.push(
`- **Post-fix END rate**: ${pct(r.postFix.endRate)}${postErr ? ` — ${postErr} error(s)` : ''}`,
);
lines.push(`- **Δ (pre − post)**: ${pct(r.delta)}`);
lines.push(``);
lines.push(`Pre-fix raw decisions
`, ``);
for (const s of r.preFix.samples) {
const label = s.error ? `ERROR: ${s.error}` : s.isEnd ? '**END**' : s.decision;
lines.push(`- ${label}`);
}
lines.push(``, ` `, ``);
lines.push(`Post-fix raw decisions
`, ``);
for (const s of r.postFix.samples) {
const label = s.error ? `ERROR: ${s.error}` : s.isEnd ? '**END**' : s.decision;
lines.push(`- ${label}`);
}
lines.push(``, ` `, ``);
}
lines.push(`## Summary`, ``);
const rows: string[][] = report.results.map((r, i) => [
String(i + 1),
r.case_id,
pct(r.preFix.endRate),
pct(r.postFix.endRate),
pct(r.delta),
r.postFixPasses ? 'PASS' : 'FAIL',
r.discriminates ? 'YES' : 'no',
]);
lines.push(
...renderSummaryTable(
['#', 'Scenario', 'Pre-fix END', 'Post-fix END', 'Δ', 'Regression guard', 'Discriminates'],
rows,
),
);
const reportPath = join(runDir, 'report.md');
writeFileSync(reportPath, lines.join('\n'));
return reportPath;
}