// Courseware knowledge loading + zero-LLM retrieval for the teaching assistant. // // The knowledge pack (`knowledge/knowledge.json` inside the frozen bundle) is // extracted at publish time; retrieval here is pure local scoring — no LLM, // no network — so the L2 hot path for a Q&A request starts at zero model cost // until the answer generation itself. import { createFileBundleByteStore, COURSEWARE_BUNDLES_DIR } from '@/lib/courseware-repo/bundle-store'; import { readFrozenBundleDocuments } from '@/lib/bundle/packager'; import type { KnowledgePack, QuizPack } from '@/lib/bundle/types'; export interface CoursewareKnowledge { coursewareId: string; /** Present for legacy multi-publish courseware records; unique Makelore courses omit it. */ version?: number; language?: string; title: string; knowledge: KnowledgePack; quiz: QuizPack; } export interface RetrievedChunk { sceneId: string; order: number; title: string; type: string; text: string; narration?: string; /** Retrieval score (term overlap); higher is better. */ score: number; } // Bounded in-memory cache: one entry per (courseware, version). A published // version is immutable, so the cache never goes stale; only new versions or a // process restart miss it. const knowledgeCache = new Map>(); const KNOWLEDGE_CACHE_MAX = 64; /** * Load the knowledge/quiz packs for a published courseware version from its * frozen bundle. Returns null when the bundle or its knowledge pack is * missing (unpublished / pre-P3 bundles). */ export async function loadCoursewareKnowledge( coursewareId: string, version: number, store = createFileBundleByteStore(COURSEWARE_BUNDLES_DIR), ): Promise { const cacheKey = `${coursewareId}@${version}`; const cached = knowledgeCache.get(cacheKey); if (cached) return cached; const pending = (async (): Promise => { const bytes = await store.read(coursewareId, version); if (!bytes) return null; const documents = await readFrozenBundleDocuments(bytes); return { coursewareId, version, language: documents.meta.language, title: documents.meta.stageName, knowledge: documents.knowledge, quiz: documents.quiz, }; })(); if (knowledgeCache.size >= KNOWLEDGE_CACHE_MAX) { const oldest = knowledgeCache.keys().next().value; if (oldest !== undefined) knowledgeCache.delete(oldest); } knowledgeCache.set(cacheKey, pending); return pending; } /** Test hook. */ export function clearKnowledgeCache(): void { knowledgeCache.clear(); } // ─── Retrieval ──────────────────────────────────────────────── /** * Split a Chinese/English query into terms: ASCII words plus CJK character * bigrams. CJK text has no spaces, so bigrams are the standard cheap tokenizer * for overlap scoring. */ export function tokenizeQuery(query: string): string[] { const terms = new Set(); for (const match of query.toLowerCase().match(/[a-z0-9_]+/g) ?? []) { if (match.length >= 2) terms.add(match); } const cjk = query.replace(/[^\u4e00-\u9fff]/g, ''); if (cjk.length > 0) { if (cjk.length === 1) terms.add(cjk); for (let i = 0; i < cjk.length - 1; i++) { terms.add(cjk.slice(i, i + 2)); } } return [...terms].filter((term) => term.length >= 2); } /** * Score scenes by query-term overlap. Title matches weigh more than body text * matches; narration counts as body text (it is the spoken explanation). * Returns the top `limit` chunks that matched at least one term. */ export function retrieveChunks( knowledge: KnowledgePack, query: string, limit = 3, ): RetrievedChunk[] { const terms = tokenizeQuery(query); if (terms.length === 0) return []; const scored: RetrievedChunk[] = []; for (const scene of knowledge.scenes) { const title = scene.title.toLowerCase(); const text = `${scene.text}\n${scene.narration ?? ''}`.toLowerCase(); let score = 0; for (const term of terms) { if (title.includes(term)) score += 3; else if (text.includes(term)) score += 1; } if (score > 0) { scored.push({ ...scene, score }); } } scored.sort((a, b) => b.score - a.score || a.order - b.order); return scored.slice(0, limit); }