409 lines
14 KiB
TypeScript
409 lines
14 KiB
TypeScript
/**
|
|
* Director Question-Answering Eval (#598 / #511 follow-up)
|
|
*
|
|
* Tests whether the director routes correctly when the conversation contains
|
|
* an unanswered user question. The bug observed in production: when agents
|
|
* have drifted off-topic — whether the user has expressed frustration yet
|
|
* or not — the director keeps picking peer agents for "variety" instead of
|
|
* routing to the teacher to actually answer the literal question.
|
|
*
|
|
* Scenarios cover both shapes:
|
|
* - first-turn drift, no frustration yet (the root case)
|
|
* - escalated frustration after multiple complaints (the recovery case)
|
|
*
|
|
* Per-decision classification (deterministic, no LLM judge):
|
|
* - TEACHER → ✓ correct (teacher answers, or asks a clarifying question
|
|
* when the user's message is too vague)
|
|
* - USER → ✗ wrong (cue_user makes no agent speak — the user faces
|
|
* dead air; the teacher should ask the clarifying question)
|
|
* - OTHER_AGENT → ✗ wrong (peer-agent "variety" routing)
|
|
* - END → ✗ wrong
|
|
*
|
|
* A/B:
|
|
* - baseline : current director template with rule 13 stripped
|
|
* - with_rule : current director template as-shipped (rule 13 in place)
|
|
*
|
|
* Pass criterion: with_rule.correctRate ≥ EVAL_PASS_THRESHOLD (default 0.7).
|
|
* The pre-vs-post Δ is reported as informational only — scenarios where the
|
|
* baseline already routes correctly shouldn't fail just because there is no
|
|
* room to lift.
|
|
*
|
|
* Required env:
|
|
* EVAL_DIRECTOR_MODEL
|
|
*
|
|
* Optional env:
|
|
* EVAL_SAMPLES Samples per (scenario, variant). Default 5.
|
|
* EVAL_PASS_THRESHOLD Min with_rule correct rate per scenario. Default 0.7.
|
|
* EVAL_SCENARIO Filter to a single scenario by case_id.
|
|
*
|
|
* Output: eval/orchestration/results-answering/<model>/<timestamp>/report.md
|
|
*/
|
|
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
import { callLLM } from '@/lib/ai/llm';
|
|
import { parseDirectorDecision } from '@/lib/orchestration/director-prompt';
|
|
import {
|
|
summarizeConversation,
|
|
type OpenAIMessage,
|
|
} from '@/lib/orchestration/summarizers/conversation-summary';
|
|
import {
|
|
processSnippets,
|
|
processConditionalBlocks,
|
|
interpolateVariables,
|
|
} from '@/lib/prompts/loader';
|
|
import { resolveEvalModel } from '../shared/resolve-model';
|
|
import { createRunDir } from '../shared/run-dir';
|
|
import type { AgentTurnSummary } from '@/lib/orchestration/types';
|
|
import type { ScenarioAgent } from './types';
|
|
|
|
const OUTPUT_DIR = 'eval/orchestration/results-answering';
|
|
|
|
// ==================== Types ====================
|
|
|
|
interface AnsweringScenario {
|
|
case_id: string;
|
|
description: string;
|
|
agents: ScenarioAgent[];
|
|
teacherAgentId: string;
|
|
messages: OpenAIMessage[];
|
|
agentResponses: AgentTurnSummary[];
|
|
turnCount: number;
|
|
whiteboardOpen?: boolean;
|
|
}
|
|
|
|
type Variant = 'baseline' | 'with_rule';
|
|
type DecisionClass = 'USER' | 'TEACHER' | 'OTHER_AGENT' | 'END' | 'ERROR';
|
|
|
|
interface SampleResult {
|
|
variant: Variant;
|
|
raw: string;
|
|
classification: DecisionClass;
|
|
rawAgentId: string | null;
|
|
error?: string;
|
|
}
|
|
|
|
interface ScenarioResult {
|
|
case_id: string;
|
|
description: string;
|
|
samples: number;
|
|
baseline: { samples: SampleResult[]; rates: Record<DecisionClass, number>; correctRate: number };
|
|
withRule: { samples: SampleResult[]; rates: Record<DecisionClass, number>; correctRate: number };
|
|
delta: number;
|
|
passes: boolean;
|
|
}
|
|
|
|
// ==================== Prompt building ====================
|
|
|
|
function readDirectorTemplate(): string {
|
|
const p = path.join(process.cwd(), 'lib', 'prompts', 'templates', 'director', 'system.md');
|
|
return fs.readFileSync(p, 'utf-8').trim();
|
|
}
|
|
|
|
/**
|
|
* Rule 13 was injected directly into director/system.md. To A/B against a
|
|
* pre-rule baseline, strip rule 13 (and its indented continuation block) out
|
|
* of the current template.
|
|
*/
|
|
function withoutAnsweringRule(template: string): string {
|
|
// Match rule 13 by its number (heading text is reworded often) up to the
|
|
// next blank-line + section header. Decoupled from the heading wording.
|
|
const stripped = template.replace(/^13\. \*\*[\s\S]*?(?=\n\n# )/m, '');
|
|
if (stripped === template) {
|
|
throw new Error(
|
|
'answering-runner: rule 13 not found in director template; eval baseline cannot be constructed',
|
|
);
|
|
}
|
|
return stripped.replace(/\n{3,}/g, '\n\n');
|
|
}
|
|
|
|
function buildPromptFromTemplate(
|
|
template: string,
|
|
scenario: AnsweringScenario,
|
|
conversationSummary: string,
|
|
): string {
|
|
const agentList = scenario.agents
|
|
.map((a) => `- id: "${a.id}", name: "${a.name}", role: ${a.role}, priority: ${a.priority}`)
|
|
.join('\n');
|
|
|
|
const respondedList =
|
|
scenario.agentResponses.length > 0
|
|
? scenario.agentResponses
|
|
.map(
|
|
(r) =>
|
|
`- ${r.agentName} (${r.agentId}): "${r.contentPreview}" [${r.actionCount} actions]`,
|
|
)
|
|
.join('\n')
|
|
: 'None yet.';
|
|
|
|
const rule1 =
|
|
"1. The teacher (role: teacher, highest priority) should usually speak first to address the user's question or topic.";
|
|
|
|
const vars: Record<string, unknown> = {
|
|
agentList,
|
|
respondedList,
|
|
conversationSummary,
|
|
discussionSection: '',
|
|
whiteboardSection: '',
|
|
studentProfileSection: '',
|
|
rule1,
|
|
turnCountPlusOne: scenario.turnCount + 1,
|
|
whiteboardOpenText: scenario.whiteboardOpen
|
|
? 'OPEN (slide canvas is hidden — spotlight/laser will not work)'
|
|
: 'CLOSED (slide canvas is visible)',
|
|
};
|
|
|
|
const withSnippets = processSnippets(template);
|
|
const withConditionals = processConditionalBlocks(withSnippets, vars);
|
|
return interpolateVariables(withConditionals, vars);
|
|
}
|
|
|
|
function buildVariants(scenario: AnsweringScenario): { baseline: string; with_rule: string } {
|
|
const current = readDirectorTemplate();
|
|
const summary = summarizeConversation(scenario.messages);
|
|
return {
|
|
baseline: buildPromptFromTemplate(withoutAnsweringRule(current), scenario, summary),
|
|
with_rule: buildPromptFromTemplate(current, scenario, summary),
|
|
};
|
|
}
|
|
|
|
// ==================== Classifier ====================
|
|
|
|
function classify(
|
|
raw: string,
|
|
scenario: AnsweringScenario,
|
|
): {
|
|
classification: DecisionClass;
|
|
rawAgentId: string | null;
|
|
} {
|
|
const parsed = parseDirectorDecision(raw);
|
|
if (parsed.shouldEnd || !parsed.nextAgentId) {
|
|
return { classification: 'END', rawAgentId: null };
|
|
}
|
|
if (parsed.nextAgentId === 'USER') {
|
|
return { classification: 'USER', rawAgentId: 'USER' };
|
|
}
|
|
if (parsed.nextAgentId === scenario.teacherAgentId) {
|
|
return { classification: 'TEACHER', rawAgentId: parsed.nextAgentId };
|
|
}
|
|
return { classification: 'OTHER_AGENT', rawAgentId: parsed.nextAgentId };
|
|
}
|
|
|
|
function emptyRates(): Record<DecisionClass, number> {
|
|
return { USER: 0, TEACHER: 0, OTHER_AGENT: 0, END: 0, ERROR: 0 };
|
|
}
|
|
|
|
function computeRates(samples: SampleResult[]): {
|
|
rates: Record<DecisionClass, number>;
|
|
correctRate: number;
|
|
} {
|
|
const rates = emptyRates();
|
|
const usable = samples.filter((s) => !s.error);
|
|
for (const s of usable) rates[s.classification]++;
|
|
const total = usable.length || 1;
|
|
for (const k of Object.keys(rates) as DecisionClass[]) {
|
|
rates[k] = rates[k] / total;
|
|
}
|
|
rates.ERROR = (samples.length - usable.length) / samples.length;
|
|
// Only TEACHER is correct: the teacher answers, or asks a clarifying question
|
|
// for vague input. USER cue is dead air (no agent speaks); peer/END are wrong.
|
|
const correctRate = rates.TEACHER;
|
|
return { rates, correctRate };
|
|
}
|
|
|
|
// ==================== Sampling ====================
|
|
|
|
async function sampleVariant(
|
|
scenario: AnsweringScenario,
|
|
variant: Variant,
|
|
systemPrompt: string,
|
|
model: Awaited<ReturnType<typeof resolveEvalModel>>['model'],
|
|
samples: number,
|
|
): Promise<SampleResult[]> {
|
|
const tasks = Array.from({ length: samples }, async (): Promise<SampleResult> => {
|
|
try {
|
|
const result = await callLLM(
|
|
{
|
|
model,
|
|
messages: [
|
|
{ role: 'system', content: systemPrompt },
|
|
{ role: 'user', content: 'Decide which agent should speak next.' },
|
|
],
|
|
},
|
|
'eval-orchestration-answering',
|
|
);
|
|
const raw = result.text;
|
|
const { classification, rawAgentId } = classify(raw, scenario);
|
|
return { variant, raw, classification, rawAgentId };
|
|
} catch (err) {
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
return {
|
|
variant,
|
|
raw: '',
|
|
classification: 'ERROR',
|
|
rawAgentId: null,
|
|
error: msg,
|
|
};
|
|
}
|
|
});
|
|
return Promise.all(tasks);
|
|
}
|
|
|
|
// ==================== Reporting ====================
|
|
|
|
function pct(x: number): string {
|
|
return `${Math.round(x * 100)}%`;
|
|
}
|
|
|
|
function writeReport(
|
|
runDir: string,
|
|
results: ScenarioResult[],
|
|
modelStr: string,
|
|
samples: number,
|
|
threshold: number,
|
|
): string {
|
|
const lines: string[] = [];
|
|
const overallPass = results.every((r) => r.passes);
|
|
const meanBaseline = results.reduce((acc, r) => acc + r.baseline.correctRate, 0) / results.length;
|
|
const meanWithRule = results.reduce((acc, r) => acc + r.withRule.correctRate, 0) / results.length;
|
|
|
|
lines.push(`# Director Question-Answering Eval`, ``);
|
|
lines.push(`- **Date**: ${new Date().toISOString()}`);
|
|
lines.push(`- **Model**: ${modelStr}`);
|
|
lines.push(`- **Samples per (scenario, variant)**: ${samples}`);
|
|
lines.push(`- **with_rule correct-rate threshold**: ${pct(threshold)}`);
|
|
lines.push(`- **Δ (pre vs post)**: informational — PASS depends only on with_rule rate`);
|
|
lines.push(``);
|
|
lines.push(`## Aggregate`);
|
|
lines.push(``);
|
|
lines.push(`| Variant | Mean correct rate (TEACHER) |`);
|
|
lines.push(`|---|---|`);
|
|
lines.push(`| baseline | ${pct(meanBaseline)} |`);
|
|
lines.push(`| with_rule | ${pct(meanWithRule)} |`);
|
|
lines.push(`| Δ | ${pct(meanWithRule - meanBaseline)} |`);
|
|
lines.push(``);
|
|
lines.push(`Overall verdict: **${overallPass ? 'PASS' : 'FAIL'}**`);
|
|
lines.push(``);
|
|
|
|
lines.push(`## Per scenario`);
|
|
lines.push(``);
|
|
lines.push(
|
|
`| # | Scenario | Baseline USER% TEACHER% OTHER% END% | with_rule USER% TEACHER% OTHER% END% | Δ correct | pass? |`,
|
|
);
|
|
lines.push(`|---|---|---|---|---|---|`);
|
|
results.forEach((r, i) => {
|
|
const b = r.baseline.rates;
|
|
const w = r.withRule.rates;
|
|
const bStr = `${pct(b.USER)}/${pct(b.TEACHER)}/${pct(b.OTHER_AGENT)}/${pct(b.END)}`;
|
|
const wStr = `${pct(w.USER)}/${pct(w.TEACHER)}/${pct(w.OTHER_AGENT)}/${pct(w.END)}`;
|
|
lines.push(
|
|
`| ${i + 1} | ${r.case_id} | ${bStr} | ${wStr} | ${pct(r.delta)} | ${r.passes ? '✓' : '✗'} |`,
|
|
);
|
|
});
|
|
lines.push(``);
|
|
|
|
lines.push(`## Detail`);
|
|
for (const r of results) {
|
|
lines.push(``, `### ${r.case_id} ${r.passes ? '✓' : '✗'}`, ``);
|
|
lines.push(`- ${r.description}`);
|
|
lines.push(
|
|
`- Baseline correct: ${pct(r.baseline.correctRate)}; with_rule correct: ${pct(r.withRule.correctRate)}; Δ: ${pct(r.delta)}`,
|
|
);
|
|
lines.push(``);
|
|
lines.push(`<details><summary>baseline samples</summary>`, ``);
|
|
for (const s of r.baseline.samples) {
|
|
const label = s.error
|
|
? `ERROR: ${s.error}`
|
|
: `${s.classification}${s.rawAgentId && s.classification === 'OTHER_AGENT' ? ` (${s.rawAgentId})` : ''}`;
|
|
lines.push(`- ${label}`);
|
|
}
|
|
lines.push(``, `</details>`, ``);
|
|
lines.push(`<details><summary>with_rule samples</summary>`, ``);
|
|
for (const s of r.withRule.samples) {
|
|
const label = s.error
|
|
? `ERROR: ${s.error}`
|
|
: `${s.classification}${s.rawAgentId && s.classification === 'OTHER_AGENT' ? ` (${s.rawAgentId})` : ''}`;
|
|
lines.push(`- ${label}`);
|
|
}
|
|
lines.push(``, `</details>`, ``);
|
|
}
|
|
|
|
const reportPath = path.join(runDir, 'report.md');
|
|
fs.writeFileSync(reportPath, lines.join('\n'));
|
|
return reportPath;
|
|
}
|
|
|
|
// ==================== Main ====================
|
|
|
|
function getCurrentDir(): string {
|
|
return typeof __dirname !== 'undefined'
|
|
? __dirname
|
|
: path.dirname(fileURLToPath(import.meta.url));
|
|
}
|
|
|
|
function loadScenarios(): AnsweringScenario[] {
|
|
const p = path.join(getCurrentDir(), 'scenarios/answering.json');
|
|
const scenarios = JSON.parse(fs.readFileSync(p, 'utf-8')) as AnsweringScenario[];
|
|
const filter = process.env.EVAL_SCENARIO;
|
|
return filter ? scenarios.filter((s) => s.case_id === filter) : scenarios;
|
|
}
|
|
|
|
async function main() {
|
|
const modelStr = process.env.EVAL_DIRECTOR_MODEL || process.env.DEFAULT_MODEL;
|
|
if (!modelStr) {
|
|
console.error(
|
|
'Error: EVAL_DIRECTOR_MODEL must be set. Example: EVAL_DIRECTOR_MODEL=google:gemini-3-flash-preview',
|
|
);
|
|
process.exit(1);
|
|
}
|
|
const samples = Number(process.env.EVAL_SAMPLES || '5');
|
|
const threshold = Number(process.env.EVAL_PASS_THRESHOLD || '0.7');
|
|
|
|
console.log('=== Director Question-Answering Eval ===');
|
|
console.log(`Model: ${modelStr} | Samples/variant: ${samples} | pass threshold: ${threshold}`);
|
|
|
|
const { model } = await resolveEvalModel('EVAL_DIRECTOR_MODEL', process.env.DEFAULT_MODEL);
|
|
const scenarios = loadScenarios();
|
|
console.log(`Loaded ${scenarios.length} scenario(s)`);
|
|
const runDir = createRunDir(OUTPUT_DIR, modelStr);
|
|
console.log(`Output: ${runDir}`);
|
|
|
|
const results: ScenarioResult[] = [];
|
|
for (const sc of scenarios) {
|
|
process.stdout.write(` - ${sc.case_id} ... `);
|
|
const variants = buildVariants(sc);
|
|
const [bs, ws] = await Promise.all([
|
|
sampleVariant(sc, 'baseline', variants.baseline, model, samples),
|
|
sampleVariant(sc, 'with_rule', variants.with_rule, model, samples),
|
|
]);
|
|
const bAgg = computeRates(bs);
|
|
const wAgg = computeRates(ws);
|
|
const lift = wAgg.correctRate - bAgg.correctRate;
|
|
const passes = wAgg.correctRate >= threshold;
|
|
results.push({
|
|
case_id: sc.case_id,
|
|
description: sc.description,
|
|
samples,
|
|
baseline: { samples: bs, rates: bAgg.rates, correctRate: bAgg.correctRate },
|
|
withRule: { samples: ws, rates: wAgg.rates, correctRate: wAgg.correctRate },
|
|
delta: lift,
|
|
passes,
|
|
});
|
|
console.log(
|
|
`baseline=${pct(bAgg.correctRate)} with_rule=${pct(wAgg.correctRate)} Δ=${pct(lift)} ${passes ? 'PASS' : 'FAIL'}`,
|
|
);
|
|
}
|
|
|
|
const reportPath = writeReport(runDir, results, modelStr, samples, threshold);
|
|
const overallPass = results.every((r) => r.passes);
|
|
console.log(`\nReport: ${reportPath}`);
|
|
console.log(`Verdict: ${overallPass ? 'PASS' : 'FAIL'}`);
|
|
process.exit(overallPass ? 0 : 1);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error('Fatal error:', err);
|
|
process.exit(1);
|
|
});
|