chore: establish learning module baseline

OpenMAIC/eval/whiteboard-layout/scorer.ts

@@ -0,0 +1,142 @@
+/**
+ * VLM Scorer for whiteboard layout quality.
+ *
+ * Uses the project's LLM infrastructure (resolveModel + generateText from AI SDK)
+ * so model configuration follows the same `provider:model` convention as the rest
+ * of the codebase. Supports all providers (OpenAI, Google, Anthropic, etc.).
+ *
+ * The caller supplies the model string explicitly (typically from EVAL_SCORER_MODEL);
+ * this function no longer has a hardcoded default.
+ */
+
+import { readFileSync } from 'fs';
+import { generateText } from 'ai';
+import { resolveModel } from '@/lib/server/resolve-model';
+import type { VlmScore } from './types';
+
+const RUBRIC_PROMPT = `You are evaluating a classroom whiteboard screenshot from an AI teaching assistant. Score like a teacher reviewing their own board work for a student's benefit.
+
+Context: This is a real-time teaching whiteboard, NOT a poster or infographic.
+- Empty space is NORMAL and NOT a problem — teachers write in one area at a time.
+- What matters: would a student be confused, misled, or unable to read the content?
+- Ignore the small dark circle "N" in the corner — it is a page UI element, not whiteboard content.
+
+Score each dimension from 1 to 10 (10 = perfect, 1 = broken):
+
+1. readability — Can a student read every element easily?
+   - Font size CONSISTENCY is critical: penalize heavily if some text is 2x+ larger than other text on the same board (e.g., one giant title + tiny formulas).
+   - Are characters crisp? Any Chinese rendered as boxes or missing glyphs?
+   - Penalize text styled like UI components (gray boxes, card backgrounds) that don't match handwritten whiteboard feel.
+
+2. overlap — Are elements clear of each other, AND does new content respect existing content?
+   - Penalize any occlusion (shapes over text, text stacked on text, arrows piercing labels).
+   - CRITICAL: penalize "writing over existing content" — if a new formula is placed directly on top of an existing table row when empty space was available nearby, that is a layout failure, not just overlap.
+   - 10 = everything distinct; 1 = multiple elements unreadable due to occlusion.
+
+3. rendering_correctness — Are formulas, shapes, and symbols drawn correctly?
+   - LaTeX must render: raw source like "\\\\frac", "\\\\theta", or garbled chunks like "0ext", "Gsinheta", "heta" = major penalty.
+   - Subscripts/superscripts must render: "G_x" shown as raw underscore (not Gₓ) = penalty.
+   - Chinese inside LaTeX math mode (e.g., "口诀(当 a > 0 ext 时)") = penalty.
+   - Diagram ACCURACY matters: a parabola drawn as V-shape straight lines, a circle drawn as ellipse-when-should-be-circle, an angle labeled wrong = penalty.
+   - 10 = all math/shapes render correctly and match the concept; 1 = multiple broken renders OR fundamentally wrong diagrams.
+
+4. content_completeness — Is the content whole, bounded, and annotated?
+   - Edge clipping: any element cut off at canvas edge (formula missing its left character, table column cut, arrow head beyond edge) = major penalty.
+   - Unexpected clearing: if previous turns' content has vanished in a later turn with no reason, penalize.
+   - Bare diagrams with no labels (a circle with no annotation of what it represents) = penalty.
+   - 10 = all content fully visible and annotated; 1 = significant content lost, truncated, or unlabeled.
+
+5. layout_logic — Does the arrangement support teaching flow?
+   - Related elements grouped (a diagram with its labels/formulas together)?
+   - Natural reading order for the concept (cause → effect, equation → graph → solution)?
+   - Spatial planning: does new content go to sensibly-chosen empty areas rather than crammed near or over existing elements?
+
+overall: 1–10 holistic teaching-quality score. Weight overlap and rendering_correctness more heavily since they directly block comprehension.
+
+issues: 1-5 short concrete problem descriptions a teacher would call out.
+
+Output ONLY a JSON object with this exact structure (no markdown, no code fences):
+{"readability":{"score":N,"reason":"..."},"overlap":{"score":N,"reason":"..."},"rendering_correctness":{"score":N,"reason":"..."},"content_completeness":{"score":N,"reason":"..."},"layout_logic":{"score":N,"reason":"..."},"overall":N,"issues":["..."]}`;
+
+/**
+ * Score a whiteboard screenshot using a VLM.
+ *
+ * The caller must provide the model string explicitly (typically from EVAL_SCORER_MODEL);
+ * this function no longer has a hardcoded default.
+ */
+export async function scoreScreenshot(
+  screenshotPath: string,
+  modelString: string,
+): Promise<VlmScore> {
+  const imageBuffer = readFileSync(screenshotPath);
+
+  const { model } = await resolveModel({ modelString });
+
+  const result = await generateText({
+    model,
+    messages: [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: RUBRIC_PROMPT },
+          { type: 'image', image: imageBuffer },
+        ],
+      },
+    ],
+    temperature: 0,
+    maxOutputTokens: 3000,
+  });
+
+  const content = result.text;
+
+  // Extract JSON from response (may be wrapped in markdown code fences)
+  const jsonMatch = content.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) {
+    throw new Error(`VLM returned non-JSON response: ${content.slice(0, 200)}`);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let raw: any;
+  try {
+    raw = JSON.parse(jsonMatch[0]);
+  } catch {
+    // VLM sometimes produces unescaped quotes or trailing content — attempt cleanup
+    const cleaned = jsonMatch[0]
+      .replace(/,\s*}/g, '}') // trailing commas
+      .replace(/,\s*]/g, ']');
+    try {
+      raw = JSON.parse(cleaned);
+    } catch (e2) {
+      throw new Error(
+        `VLM returned invalid JSON: ${(e2 as Error).message}\n${jsonMatch[0].slice(0, 300)}`,
+      );
+    }
+  }
+
+  const dimensions = [
+    'readability',
+    'overlap',
+    'rendering_correctness',
+    'content_completeness',
+    'layout_logic',
+  ] as const;
+  for (const dim of dimensions) {
+    if (!raw[dim] || typeof raw[dim].score !== 'number') {
+      throw new Error(`VLM response missing or invalid dimension: ${dim}`);
+    }
+  }
+  if (typeof raw.overall !== 'number') {
+    throw new Error('VLM response missing overall score');
+  }
+
+  const score: VlmScore = {
+    readability: raw.readability,
+    overlap: raw.overlap,
+    rendering_correctness: raw.rendering_correctness,
+    content_completeness: raw.content_completeness,
+    layout_logic: raw.layout_logic,
+    overall: raw.overall,
+    issues: Array.isArray(raw.issues) ? raw.issues : [],
+  };
+  return score;
+}