/** * PBL v2 — Evaluator agent. * * Three modes, one streaming pattern: * * runTaskEvaluation — fires after a microtask completes WITH * a submission (D1-B). Streams short * feedback + {strengths, improvements, * score?} JSON tail. * runMilestoneEvaluation — fires after the last microtask of a * milestone advances. Streams a reflection * card narrative + {learned, performance, * stars} JSON tail. Drives the * MilestoneCard UI. * runFinalEvaluation — fires after the last milestone * completes. Streams a short intro * narrative + {stars, what_you_built, * what_you_learned, whats_next} JSON tail. * Drives the Completion page hero. * * Why a separate agent (not a tool on Instructor): * - Different system prompts (reflection / report tone, not * teaching). Same system prompt would fight itself. * - Different output contract (narrative + JSON tail vs * conversational reply with optional tool calls). * - Separate SSE call lets the client display "导师正在生成阶段反 * 馈…" as a distinct phase, not as a mysterious extra Instructor * turn. * * All three functions are async generators of PBLSSEEvent — the same * shape Instructor uses — so they slot into the same SSE wrapping * (`createSSEResponse`) and the same client-side stream consumer * (`use-instructor-stream.ts` will get a small additive parser for * the `evaluation` patch in PR 6.5). * * No tool calls. The evaluator does not modify the project mid-stream; * it only appends to `project.evaluations` once, at the end, after the * JSON tail parses. This keeps the streaming layer simple and means a * failed parse leaves the project untouched (the narrative is still * shown — the LLM said useful things — just no structured payload to * persist). */ import type { LanguageModel } from 'ai'; import { createLogger } from '@/lib/logger'; import { streamLLM } from '@/lib/ai/llm'; import { buildVisionUserContent } from '@openmaic/generation'; import type { ThinkingConfig } from '@/lib/types/provider'; import type { PBLEvaluation, PBLEvaluationKind, PBLMicrotask, PBLMilestone, PBLProjectV2, } from '../types'; import type { PBLSSEEvent } from '../api/sse'; import { addEvaluation } from '../operations/runtime/evaluation'; import { latestSubmissionForMicrotask } from '../operations/runtime/submission'; import { buildFinalEvalPrompt, buildMilestoneEvalPrompt, buildTaskEvalPrompt, } from '../operations/runtime/eval-prompts'; import { normalizeOptionalString, normalizeScore, normalizeStars, normalizeStringList, parseEvaluationTail, sanitizeMilestoneEvaluationFeedback, stripEvaluationTail, } from '../operations/runtime/eval-tail-parser'; import { normalizeActGoals } from '../operations/runtime/completion-stats'; const log = createLogger('PBL v2 Evaluator'); export interface RunTaskEvaluationArgs { project: PBLProjectV2; milestoneId: string; microtaskId: string; languageModel: LanguageModel; thinkingConfig?: ThinkingConfig; recentChatSummary?: string; /** True when the resolved model can read images. Gates whether an image * submission is sent multimodally (else it degrades to text-only eval). */ hasVision?: boolean; signal?: AbortSignal; } export interface RunMilestoneEvaluationArgs { project: PBLProjectV2; milestoneId: string; languageModel: LanguageModel; thinkingConfig?: ThinkingConfig; recentChatSummary?: string; signal?: AbortSignal; } export interface RunFinalEvaluationArgs { project: PBLProjectV2; languageModel: LanguageModel; thinkingConfig?: ThinkingConfig; recentChatSummary?: string; signal?: AbortSignal; } // --------------------------------------------------------------------------- // Shared streaming loop // --------------------------------------------------------------------------- interface RunSharedArgs { project: PBLProjectV2; kind: PBLEvaluationKind; microtaskId?: string; milestoneId?: string; systemPrompt: string; userPrompt: string; languageModel: LanguageModel; thinkingConfig?: ThinkingConfig; /** When set (image submission + vision-capable model), the user turn is sent * as a multimodal message carrying this image instead of a plain prompt. * Accepts an http(s) URL or a base64 data URL. */ visionImageSrc?: string; signal?: AbortSignal; } async function* runShared(args: RunSharedArgs): AsyncGenerator { const { project, kind, microtaskId, milestoneId, systemPrompt, userPrompt, languageModel, thinkingConfig, visionImageSrc, signal, } = args; let fullText = ''; // Evaluations are JSON-only and render only after the structured // payload is persisted. Do not stream raw JSON into the chat. const shouldStreamTokens = false; // Milestone evaluations progressively sanitise the streaming text so // the learner never sees a flash of "Continue to next stage" / next- // milestone setup text before it's stripped by the final sanitizer. // We keep `lastSanitizedLength` to yield only the newly-sanitised // portion each chunk (avoiding re-sending already-shown text). const isMilestone = kind === 'milestone'; let lastSanitizedLength = 0; try { const result = streamLLM( { model: languageModel, system: systemPrompt, // Image submission on a vision-capable model → send the picture as a // multimodal user turn (reusing OpenMAIC's buildVisionUserContent). // Everything else keeps the plain text prompt unchanged. ...(visionImageSrc ? { messages: [ { role: 'user' as const, content: buildVisionUserContent(userPrompt, [ { id: 'submission', src: visionImageSrc }, ]), }, ], } : { prompt: userPrompt }), ...(signal ? { abortSignal: signal } : {}), }, `pbl-v2-evaluator-${kind}`, thinkingConfig, ); for await (const part of result.fullStream) { switch (part.type) { case 'text-delta': { // AI SDK 6.x uses `text` for the chunk content; fall back to // legacy `textDelta` for safety with older provider shims. const delta = (part as unknown as { text?: string; textDelta?: string }).text ?? (part as unknown as { textDelta?: string }).textDelta ?? ''; if (delta) { fullText += delta; if (!shouldStreamTokens) { break; } if (isMilestone) { const sanitized = sanitizeMilestoneEvaluationFeedback(fullText); const next = sanitized.slice(lastSanitizedLength); if (next) { yield { type: 'token', delta: next }; lastSanitizedLength = sanitized.length; } // When sanitized is shorter than fullText the LLM has // started a disallowed section — stop yielding tokens // but keep accumulating so the final evaluation still // gets the complete text for structured-tail parsing. } else { yield { type: 'token', delta }; } } break; } case 'error': { const errAny = (part as unknown as { error?: unknown }).error; yield { type: 'error', code: 'LLM_ERROR', message: errAny instanceof Error ? errAny.message : String(errAny ?? 'LLM error'), }; yield { type: 'done' }; return; } case 'finish': default: break; } } } catch (err) { log.warn(`Evaluator turn threw: ${err instanceof Error ? err.message : String(err)}`); yield { type: 'error', code: 'STREAM_ERROR', message: err instanceof Error ? err.message : String(err), }; yield { type: 'done' }; return; } // Parse the structured tail. Failure is non-fatal: we still // persist the prose feedback so the user sees what the LLM said, // we just don't have structured strengths/stars/etc to render. // This is intentional — partial success beats throwing the whole // evaluation away on a malformed JSON tail. const tail = parseEvaluationTail(fullText) ?? {}; const evaluation = persistEvaluation({ project, kind, microtaskId, milestoneId, fullText, tail }); yield { type: 'project_patch', patch: { kind: 'evaluation', evaluation }, }; yield { type: 'done' }; } function persistEvaluation(args: { project: PBLProjectV2; kind: PBLEvaluationKind; microtaskId?: string; milestoneId?: string; fullText: string; tail: Record; }): PBLEvaluation { const { project, kind, microtaskId, milestoneId, tail } = args; const fullText = kind === 'milestone' ? sanitizeMilestoneEvaluationFeedback(args.fullText) : args.fullText; // Three kinds share the same Evaluation storage shape; each kind // populates a different subset of fields. This funneling lets the // UI branch on `kind` instead of three parallel storage paths. if (kind === 'milestone') { // {learned, performance, stars} — no score, no improvements list. // Funnel: // tail.learned -> Evaluation.strengths // tail.performance -> Evaluation.improvements[0] // (single element so the UI doesn't // need a second field) const learned = normalizeStringList(tail.learned, 6); const performance = normalizeOptionalString(tail.performance); return addEvaluation(project, { kind, microtaskId, milestoneId, feedback: normalizeOptionalString(tail.feedback) ?? sanitizeMilestoneEvaluationFeedback(stripEvaluationTail(fullText)), strengths: learned, improvements: performance ? [performance] : [], stars: normalizeStars(tail.stars) ?? undefined, }); } if (kind === 'final') { // SCENARIO ONLY: overlay the per-act goal verdict onto the authored // scaffold. The guard keeps normal projects completely out of this path // (they also never emit `act_goals`, but the explicit check makes the // isolation obvious and defends against a stray tail field). const actGoals = project.scenario ? normalizeActGoals(tail.act_goals, project) : undefined; return addEvaluation(project, { kind, microtaskId, milestoneId, feedback: normalizeOptionalString(tail.feedback) ?? stripEvaluationTail(fullText), strengths: [], improvements: [], stars: normalizeStars(tail.stars) ?? undefined, whatYouBuilt: normalizeStringList(tail.what_you_built, 6), whatYouLearned: normalizeStringList(tail.what_you_learned, 6), whatsNext: normalizeOptionalString(tail.whats_next) ?? undefined, ...(actGoals ? { actGoals } : {}), }); } // task return addEvaluation(project, { kind, microtaskId, milestoneId, feedback: normalizeOptionalString(tail.feedback) ?? stripEvaluationTail(fullText), strengths: normalizeStringList(tail.strengths, 4), improvements: normalizeStringList(tail.improvements, 4), score: normalizeScore(tail.score) ?? undefined, }); } // --------------------------------------------------------------------------- // Public entry points // --------------------------------------------------------------------------- export async function* runTaskEvaluation( args: RunTaskEvaluationArgs, ): AsyncGenerator { const { project, milestoneId, microtaskId, languageModel, thinkingConfig, recentChatSummary, signal, } = args; const lookup = findMilestoneAndTask(project, milestoneId, microtaskId); if (!lookup) { yield { type: 'error', code: 'NOT_FOUND', message: 'Milestone or microtask not found for task evaluation', }; yield { type: 'done' }; return; } const { milestone, microtask } = lookup; const { system, user } = buildTaskEvalPrompt(project, milestone, microtask, { recentChatSummary, }); // Image submission + vision-capable model → grade the picture itself. // Otherwise (text/PDF, or a non-vision model) keep the text-only path. const latest = latestSubmissionForMicrotask(project, microtaskId); const visionImageSrc = args.hasVision && latest?.fileUrl && (latest.mimeType?.startsWith('image/') ?? false) ? latest.fileUrl : undefined; log.info( `Task eval start: project=${project.title.slice(0, 40)} milestone=${milestone.title.slice(0, 40)} task=${microtask.title.slice(0, 40)}${visionImageSrc ? ' [vision]' : ''}`, ); for await (const ev of runShared({ project, kind: 'task', microtaskId, milestoneId, systemPrompt: system, userPrompt: user, languageModel, thinkingConfig, visionImageSrc, signal, })) { yield ev; } } export async function* runMilestoneEvaluation( args: RunMilestoneEvaluationArgs, ): AsyncGenerator { const { project, milestoneId, languageModel, thinkingConfig, recentChatSummary, signal } = args; const milestone = project.milestones.find((m) => m.id === milestoneId); if (!milestone) { yield { type: 'error', code: 'NOT_FOUND', message: 'Milestone not found for milestone evaluation', }; yield { type: 'done' }; return; } const { system, user } = buildMilestoneEvalPrompt(project, milestone, { recentChatSummary }); log.info( `Milestone eval start: project=${project.title.slice(0, 40)} milestone=${milestone.title.slice(0, 40)}`, ); for await (const ev of runShared({ project, kind: 'milestone', milestoneId, systemPrompt: system, userPrompt: user, languageModel, thinkingConfig, signal, })) { yield ev; } } export async function* runFinalEvaluation( args: RunFinalEvaluationArgs, ): AsyncGenerator { const { project, languageModel, thinkingConfig, recentChatSummary, signal } = args; const { system, user } = buildFinalEvalPrompt(project, { recentChatSummary }); log.info(`Final eval start: project=${project.title.slice(0, 40)}`); for await (const ev of runShared({ project, kind: 'final', systemPrompt: system, userPrompt: user, languageModel, thinkingConfig, signal, })) { yield ev; } } function findMilestoneAndTask( project: PBLProjectV2, milestoneId: string, microtaskId: string, ): { milestone: PBLMilestone; microtask: PBLMicrotask } | null { const milestone = project.milestones.find((m) => m.id === milestoneId); if (!milestone) return null; const microtask = milestone.microtasks.find((t) => t.id === microtaskId); if (!microtask) return null; return { milestone, microtask }; }