/** * PBL v2 — Evaluator prompt builders. * * Pure functions that assemble the {system, user} pair the * evaluator agent feeds into `streamText`. Three kinds: * * - `buildTaskEvalPrompt` → after a microtask with submissions * - `buildMilestoneEvalPrompt` → after a milestone completes * - `buildFinalEvalPrompt` → after the project completes * * The `system` half loads the matching markdown prompt * (`evaluator-task.md` / `-milestone.md` / `-final.md`) via the * shared loader, with `{{language}}` interpolated to the project's * resolved language. The `user` half packs the evidence: project * context, the specific milestone/microtask, engagement telemetry, * existing task evaluations, and the latest submission summary. * * The split mirrors the v1 repo's evaluator: prompt = system rules, * user = evidence stream. Keeping the rules in a markdown file (not * inlined) lets us iterate on prompt language without touching * TypeScript, and lets a human review the prompt by reading one file * end to end. */ import { loadPBLV2Prompt } from '../../prompts/loader'; import { microtaskEngagement } from '../kernel/engagement'; import { scenarioActGoalsScaffold } from './completion-stats'; import { PBL_SIMULATOR_AGENT_ID } from '../kernel/progress'; import { listEvaluationsForMicrotask } from './evaluation'; import { summarizeLatestSubmissionForMicrotask } from './submission'; import type { PBLMicrotask, PBLMilestone, PBLProjectV2, PBLScenarioConfig } from '../../types'; import { trimmedPBLText } from '../../readers'; export interface EvalPromptPair { system: string; user: string; } function resolveLanguage(project: PBLProjectV2): string { return project.languageDirective || project.language || 'en-US'; } function laterMicrotasksInMilestone(milestone: PBLMilestone, task: PBLMicrotask): PBLMicrotask[] { const currentOrder = task.order ?? 0; return milestone.microtasks .filter((t) => t.id !== task.id && (t.order ?? 0) > currentOrder) .sort((a, b) => (a.order ?? 0) - (b.order ?? 0)); } // --------------------------------------------------------------------------- // Task evaluation // --------------------------------------------------------------------------- /** Build the prompt pair for a TASK-level evaluation. * * Caller responsibility: only invoke this when the learner has at * least one submission for the task (per PR 6 D1-B decision: tasks * without submissions get no LLM evaluation, just an Instructor * follow-on). The function does NOT enforce that — it will still * produce a usable prompt with an empty submission section, which * is fine for testing. */ export function buildTaskEvalPrompt( project: PBLProjectV2, milestone: PBLMilestone, task: PBLMicrotask, opts: { recentChatSummary?: string } = {}, ): EvalPromptPair { const language = resolveLanguage(project); const system = loadPBLV2Prompt('evaluator-task', { language }); const sections: string[] = [ `## Project\n${project.title}\n\n${project.description}`, `## Milestone\n${milestone.title} — ${milestone.description ?? ''}`.trim(), `## Microtask just completed\n${task.title}\n${task.description ?? ''}`.trim(), ]; const futureTasks = laterMicrotasksInMilestone(milestone, task); if (futureTasks.length) { sections.push( [ '## Later microtasks in this milestone — exclusion boundary', 'These are listed only so you DO NOT grade this submission against them. Do not mention these later-task requirements as missing work, score penalties, or items in `improvements` for the current task.', ...futureTasks.map((t) => `- ${t.title}${t.description ? ` — ${t.description}` : ''}`), ].join('\n'), ); } if (task.hints?.length) { sections.push('## Hints that were given\n' + task.hints.map((h) => `- ${h}`).join('\n')); } const submissions = summarizeLatestSubmissionForMicrotask(project, task.id); if (submissions) { sections.push( [ '## What the learner produced in the latest submission', 'Grade this latest submission only. Earlier drafts are context, not evidence to re-score.', submissions, ].join('\n'), ); } const previousTaskEvals = listEvaluationsForMicrotask(project, task.id) .filter((e) => e.kind === 'task') .slice(-3); if (previousTaskEvals.length) { sections.push( [ '## Prior task evaluations for context only', 'Use these only to understand revision history. Do not penalize the latest submission for issues that only appeared in older drafts.', ...previousTaskEvals.map((e) => { const score = typeof e.score === 'number' ? `${e.score}/100` : 'not scored'; const improvements = e.improvements?.length ? `; prior improvements=${e.improvements.slice(0, 3).join('; ')}` : ''; return `- ${e.createdAt}: score=${score}${improvements}`; }), ].join('\n'), ); } if (opts.recentChatSummary) { sections.push(`## Recent conversation context\n${opts.recentChatSummary}`); } return { system, user: sections.join('\n\n') }; } // --------------------------------------------------------------------------- // Milestone evaluation — narrative reflection card // --------------------------------------------------------------------------- /** Build the prompt pair for a MILESTONE-level reflection card. * * This prompt is explicitly framed as a *reflection* moment (not a * judgement) and is fed real engagement telemetry. The engagement * per-microtask block is the difference between generic "Great * work!" feedback and feedback that actually references what the * learner did — see the user's repeated requirement that milestone * feedback come from data that "自然涌现" during the stage. */ export function buildMilestoneEvalPrompt( project: PBLProjectV2, milestone: PBLMilestone, opts: { recentChatSummary?: string } = {}, ): EvalPromptPair { const language = resolveLanguage(project); const system = loadPBLV2Prompt('evaluator-milestone', { language }); const taskBlocks: string[] = []; for (const task of milestone.microtasks) { const bits: string[] = [`### ${task.title}`]; if (task.description) bits.push(task.description); // Per-microtask telemetry from the cached snapshot first (set on // advance, see progress.ts), with a live recomputation as a // fallback. The cache exists because the engagement ledger is // capped (ring buffer, 500 entries) and a long project can // overflow — once a microtask completes we freeze its summary // onto `microtask.engagement` so the milestone evaluator always // has data even if the underlying ledger has rolled over. const summary = task.engagement ?? microtaskEngagement(project, task.id); const teleLines: string[] = []; if (summary.durationSeconds) { teleLines.push(`- time on task: ${summary.durationSeconds}s`); } if (summary.learnerTurnCount) { teleLines.push(`- learner messages: ${summary.learnerTurnCount}`); } if (summary.errorCount) { const repeats = summary.repeatErrorCount ? `, ${summary.repeatErrorCount} repeated` : ''; const sigs = summary.errorSignatures?.slice(0, 5).join(', ') || '—'; teleLines.push(`- errors seen: ${summary.errorCount}${repeats} (signatures: ${sigs})`); } if (summary.conceptsUnlocked?.length) { teleLines.push('- concepts unlocked: ' + summary.conceptsUnlocked.slice(0, 6).join(', ')); } if (summary.struggles?.length) { teleLines.push('- struggle notes: ' + summary.struggles.slice(0, 3).join(' | ')); } if (summary.questionsRaised) { teleLines.push(`- learner questions raised: ${summary.questionsRaised}`); } if (summary.closingQuality) { const cq = summary.closingQuality; let ans = trimmedPBLText(summary.closingAnswer); if (ans.length > 140) ans = ans.slice(0, 140) + '…'; teleLines.push(`- closing check: quality=${cq}, answer="${ans}"`); } if (teleLines.length) { bits.push('Engagement signals:\n' + teleLines.join('\n')); } // Per-task LLM evaluation recap (only present if D1-B fired — // i.e. the learner submitted something on this task). Useful as // secondary signal alongside raw engagement events. const evs = listEvaluationsForMicrotask(project, task.id).filter((e) => e.kind === 'task'); if (evs.length) { const last = evs[evs.length - 1]; const extras: string[] = []; if (last.strengths.length) { extras.push('strengths=' + last.strengths.slice(0, 3).join('; ')); } if (last.improvements.length) { extras.push('growth-edges=' + last.improvements.slice(0, 3).join('; ')); } if (extras.length) bits.push('Task eval recap: ' + extras.join(' · ')); } taskBlocks.push(bits.join('\n')); } const sections: string[] = [ `## Project\n${project.title}\n\n${project.description}`, `## Milestone just completed\n${milestone.title}\n${milestone.description ?? ''}`.trim(), '## How the stage went — per microtask\n\n' + taskBlocks.join('\n\n'), ]; if (opts.recentChatSummary) { sections.push(`## Recent conversation context\n${opts.recentChatSummary}`); } return { system, user: sections.join('\n\n') }; } // --------------------------------------------------------------------------- // Final evaluation — completion report // --------------------------------------------------------------------------- /** Compute project-level analytics rollup for the final-eval prompt. * Aggregates per-milestone engagement summaries so the LLM has * structured factual evidence to draw on, instead of inventing * generic "great work" prose. */ export function formatProjectEngagementRollup(project: PBLProjectV2): string { // Per-milestone aggregation type MilestoneRollup = { title: string; status: string; microtaskCount: number; microtasksCompleted: number; durationSeconds: number; learnerTurnCount: number; errorCount: number; repeatErrorCount: number; conceptsUnlocked: string[]; }; const msRollups: MilestoneRollup[] = []; let totalDuration = 0; let totalLearnerTurns = 0; let totalErrors = 0; let totalRepeatErrors = 0; let totalMicrotasksCompleted = 0; let totalMilestonesCompleted = 0; const allConcepts = new Set(); const closingHisto: Record = { weak: 0, ok: 0, strong: 0 }; for (const ms of project.milestones) { let msDuration = 0; let msTurns = 0; let msErrors = 0; let msRepeatErrors = 0; let msCompleted = 0; const msConcepts = new Set(); for (const t of ms.microtasks) { const s = t.engagement ?? microtaskEngagement(project, t.id); msDuration += s.durationSeconds ?? 0; msTurns += s.learnerTurnCount ?? 0; msErrors += s.errorCount ?? 0; msRepeatErrors += s.repeatErrorCount ?? 0; if (t.status === 'completed') msCompleted++; for (const c of s.conceptsUnlocked ?? []) { msConcepts.add(c); allConcepts.add(c); } if (s.closingQuality && s.closingQuality in closingHisto) { closingHisto[s.closingQuality]++; } } if (ms.status === 'completed') totalMilestonesCompleted++; totalDuration += msDuration; totalLearnerTurns += msTurns; totalErrors += msErrors; totalRepeatErrors += msRepeatErrors; totalMicrotasksCompleted += msCompleted; msRollups.push({ title: ms.title, status: ms.status, microtaskCount: ms.microtasks.length, microtasksCompleted: msCompleted, durationSeconds: msDuration, learnerTurnCount: msTurns, errorCount: msErrors, repeatErrorCount: msRepeatErrors, conceptsUnlocked: Array.from(msConcepts), }); } const selfRecoveredErrors = Math.max(0, totalErrors - totalRepeatErrors); const msLines = msRollups.map((ms) => { const dMin = Math.round(ms.durationSeconds / 60); const conceptsPreview = ms.conceptsUnlocked.slice(0, 6).join(', ') || '(none recorded)'; return ( `- ${ms.title} [${ms.status}] · ` + `${ms.microtasksCompleted}/${ms.microtaskCount} tasks · ` + `~${dMin} min · ` + `${ms.learnerTurnCount} learner turns · ` + `${ms.errorCount} errors (${ms.repeatErrorCount} repeats) · ` + `concepts: ${conceptsPreview}` ); }); const totalMin = Math.round(totalDuration / 60); const conceptsPreview = Array.from(allConcepts).slice(0, 12).join(', ') || '(none recorded)'; return ( `### Project totals\n` + `- Wall time: ~${totalMin} min\n` + `- Learner turns: ${totalLearnerTurns}\n` + `- Microtasks completed: ${totalMicrotasksCompleted}\n` + `- Milestones completed: ${totalMilestonesCompleted}\n` + `- Errors hit: ${totalErrors} (self-recovered: ${selfRecoveredErrors}, repeats: ${totalRepeatErrors})\n` + `- Closing-check quality: weak=${closingHisto.weak}, ok=${closingHisto.ok}, strong=${closingHisto.strong}\n` + `- Distinct concepts unlocked (${allConcepts.size}): ${conceptsPreview}\n\n` + `### Per-milestone\n` + msLines.join('\n') ); } function promptText(value: unknown, maxLength = 700): string { const text = String(value ?? '') .replace(/\s+/g, ' ') .trim(); if (!text) return ''; return text.length > maxLength ? text.slice(0, maxLength) + '…' : text; } function lastMicrotaskInMilestone(milestone: PBLMilestone): PBLMicrotask | undefined { return milestone.microtasks.slice().sort((a, b) => (b.order ?? 0) - (a.order ?? 0))[0]; } /** Render the learner's optional integrative stage-check answers for * the final report evaluator. These are not part of task readiness; * they are completion-report evidence so the report can acknowledge * whole-stage synthesis when it happened. */ export function formatProjectSynthesisChecks(project: PBLProjectV2): string { const coreMilestones = project.milestones.filter((ms) => !!ms.synthesisCheck); if (coreMilestones.length === 0) return '(no integrative checks configured)'; const lines: string[] = []; for (const ms of coreMilestones) { const microtaskIds = new Set(ms.microtasks.map((t) => t.id)); const stageEvent = project.engagementEvents .filter( (ev) => ev.kind === 'stage_synthesis_check' && (ev.milestoneId === ms.id || (!!ev.microtaskId && microtaskIds.has(ev.microtaskId))), ) .at(-1); const lastTask = lastMicrotaskInMilestone(ms); const fallbackClosingEvent = !stageEvent && lastTask ? project.engagementEvents .filter((ev) => ev.kind === 'closing_check' && ev.microtaskId === lastTask.id) .at(-1) : undefined; const cachedClosing = !stageEvent && !fallbackClosingEvent && lastTask?.engagement?.closingAnswer ? lastTask.engagement : undefined; const coreConcept = promptText(stageEvent?.payload?.coreConcept, 240) || promptText(ms.synthesisCheck?.coreConcept, 240) || '(not specified)'; const question = promptText(stageEvent?.payload?.question) || promptText(fallbackClosingEvent?.payload?.question) || promptText(cachedClosing?.closingQuestion); const learnerAnswer = promptText(stageEvent?.payload?.learner_answer) || promptText(fallbackClosingEvent?.payload?.learner_answer) || promptText(cachedClosing?.closingAnswer); const quality = promptText(stageEvent?.payload?.quality, 40) || promptText(fallbackClosingEvent?.payload?.quality, 40) || promptText(cachedClosing?.closingQuality, 40); if (!learnerAnswer) { lines.push(`- ${ms.title} · core concept: ${coreConcept} · no learner answer recorded`); continue; } lines.push( [ `- ${ms.title} · core concept: ${coreConcept}`, question ? ` question: ${question}` : '', ` learner answer: ${learnerAnswer}`, quality ? ` quality: ${quality}` : '', ` source: ${ stageEvent ? 'stage_synthesis_check' : fallbackClosingEvent ? 'closing_check on final microtask' : 'cached final-microtask engagement' }`, ] .filter(Boolean) .join('\n'), ); } return lines.join('\n'); } // --------------------------------------------------------------------------- // Scenario (role-play) final evaluation — SCENARIO ONLY // --------------------------------------------------------------------------- /** Max characters of the role-play transcript fed to the final evaluator * (keeps the prompt bounded; we keep the TAIL — the most recent exchanges). */ const SCENARIO_TRANSCRIPT_BUDGET = 6000; /** Render the role-play transcript (the Simulator thread) for the evaluator: * neutral narration, character lines (by name), and the learner's lines. * Exported so the wrapup Instructor can ground its debrief in what the learner * ACTUALLY said/did (not just scored decisions). */ export function formatScenarioTranscript(project: PBLProjectV2): string { const thread = project.threads.find((t) => t.agentId === PBL_SIMULATOR_AGENT_ID); const messages = thread?.messages ?? []; if (messages.length === 0) return '(no role-play conversation recorded)'; const nameOf = (characterId?: string) => project.scenario?.characters?.find((c) => c.id === characterId)?.name ?? 'Character'; const lines = messages .map((message) => ({ message, content: trimmedPBLText(message.content) })) .filter(({ content }) => content) .map(({ message, content }) => { if (message.roleType === 'system') return `[narration] ${content}`; if (message.roleType === 'user') return `Learner: ${content}`; if (message.roleType === 'simulator') { return `${nameOf(message.characterId)}: ${content}`; } return null; }) .filter((l): l is string => !!l); let text = lines.join('\n'); if (text.length > SCENARIO_TRANSCRIPT_BUDGET) { text = '…(earlier exchanges trimmed)…\n' + text.slice(text.length - SCENARIO_TRANSCRIPT_BUDGET); } return text; } /** SCENARIO ONLY. The user-evidence half for a role-play completion report: * the scenario premise, the actual transcript, the act goals to assess, and a * light engagement total — framed for SKILL feedback, not knowledge recap. */ function buildScenarioFinalUser(project: PBLProjectV2, sc: PBLScenarioConfig): string { const cast = (sc.characters ?? []).map((c) => `${c.name} (${c.persona})`).join('; ') || '(none)'; const ctx = [ `Setting: ${sc.setting}`, sc.goal ? `Skill being practised: ${sc.goal}` : '', sc.rules ? `Rules: ${sc.rules}` : '', sc.learnerRole ? `The learner's role: ${sc.learnerRole}` : '', `Character(s) they interacted with: ${cast}`, ] .filter(Boolean) .join('\n'); // ACT MODEL: a roleplay act is one continuous scene the learner ends // manually, so every beat is marked completed on finish — "beats done" is no // longer a real signal. The genuine signal is whether the learner actually // covered each act's GOALS (the authored per-beat `successWhen`), judged from // the transcript. We list them per act FROM THE SHARED SCAFFOLD (single // source of truth in completion-stats), tagged with milestoneId + goal index, // so the model can both (a) assess coverage in prose AND (b) return a // structured `act_goals` verdict that `normalizeActGoals` overlays back onto // the very same scaffold. This drives the "what you accomplished" credit and // the completion page's per-act review — never progression. const scaffold = scenarioActGoalsScaffold(project); const checklistLines: string[] = []; for (const act of scaffold) { checklistLines.push(`### Act (milestoneId: ${act.milestoneId}): ${act.actTitle}`); act.goals.forEach((g, i) => { const skill = g.skillFocus ? ` [skill: ${g.skillFocus}]` : ''; // Tag each goal with its goalIndex — the model MUST echo this index back // in `act_goals[].goals[].goalIndex` so the verdict is aligned by index, // never by array position (guards against same-act goal reordering). checklistLines.push(`- goalIndex ${i}: ${g.goal}${skill}`); }); } const checklist = checklistLines.length > 0 ? checklistLines.join('\n') : '(this scenario authored no explicit goals — judge holistically from the transcript)'; // Light engagement totals (turns / minutes) across roleplay. let turns = 0; let durationSeconds = 0; for (const ms of project.milestones) { if (ms.scenarioStage !== 'roleplay') continue; for (const t of ms.microtasks) { const s = t.engagement ?? microtaskEngagement(project, t.id); turns += s.learnerTurnCount ?? 0; durationSeconds += s.durationSeconds ?? 0; } } const engagement = `learner turns: ${turns} · ~${Math.round(durationSeconds / 60)} min`; return [ `## The scenario\n${ctx}`, `## How it actually went (role-play transcript)\n${formatScenarioTranscript(project)}`, `## The act goals to assess (for each, judge from the transcript whether the learner covered it — see act_goals in the output)\n${checklist}`, `## Engagement\n${engagement}`, ].join('\n\n'); } /** Build the prompt pair for the FINAL completion report. * * Grounding strategy: feed BOTH the per-milestone narrative * evaluations AND the analytics rollup so the LLM has structured, * factual evidence. Without the rollup the LLM defaults to generic * "great job, you learned a lot" — we want it to reference real * moments. * * SCENARIO ONLY: role-play projects are SKILL practice, not knowledge * building — their per-milestone reflection cards and concept rollup are * empty/meaningless. They get a dedicated prompt + skill-oriented evidence * (premise + transcript + decisions). Ordinary projects are byte-identical * to before. */ export function buildFinalEvalPrompt( project: PBLProjectV2, opts: { recentChatSummary?: string } = {}, ): EvalPromptPair { const language = resolveLanguage(project); if (project.scenario) { const system = loadPBLV2Prompt('evaluator-final-scenario', { language }); const sections = [ `## Project\n${project.title}\n\n${project.description}`, buildScenarioFinalUser(project, project.scenario), ]; if (opts.recentChatSummary) { sections.push(`## Recent conversation context\n${opts.recentChatSummary}`); } return { system, user: sections.join('\n\n') }; } const system = loadPBLV2Prompt('evaluator-final', { language }); // Milestone narrative recap — pull prose + key signals so the // closing arc can echo them. Use the milestone evaluation's prose, // NOT just its strengths list — the narrative captures the human // moments we want the final card to reflect back. const msSummaries: string[] = []; for (const ms of project.milestones) { const evs = project.evaluations.filter( (e) => e.kind === 'milestone' && e.milestoneId === ms.id, ); if (evs.length) { const last = evs[evs.length - 1]; const learned = last.strengths.slice(0, 4).join(', ') || '(no bullets)'; const starsStr = typeof last.stars === 'number' ? `${last.stars}★` : '(unrated)'; const feedback = trimmedPBLText(last.feedback); const prose = feedback.slice(0, 280) + (feedback.length > 280 ? '…' : ''); msSummaries.push(`- ${ms.title} [${starsStr}]\n learned: ${learned}\n prose: ${prose}`); } else { msSummaries.push(`- ${ms.title}: (no milestone evaluation recorded)`); } } const sections = [ `## Project\n${project.title}\n\n${project.description}`, `## Per-milestone reflection cards\n${msSummaries.join('\n')}`, `## Engagement rollup (from the analytics ledger)\n${formatProjectEngagementRollup(project)}`, `## Integrative checks (stage synthesis)\n${formatProjectSynthesisChecks(project)}`, ]; if (opts.recentChatSummary) { sections.push(`## Recent conversation context\n${opts.recentChatSummary}`); } return { system, user: sections.join('\n\n') }; }