131 lines
4.3 KiB
TypeScript
131 lines
4.3 KiB
TypeScript
// Courseware knowledge loading + zero-LLM retrieval for the teaching assistant.
|
|
//
|
|
// The knowledge pack (`knowledge/knowledge.json` inside the frozen bundle) is
|
|
// extracted at publish time; retrieval here is pure local scoring — no LLM,
|
|
// no network — so the L2 hot path for a Q&A request starts at zero model cost
|
|
// until the answer generation itself.
|
|
|
|
import { createFileBundleByteStore, COURSEWARE_BUNDLES_DIR } from '@/lib/courseware-repo/bundle-store';
|
|
import { readFrozenBundleDocuments } from '@/lib/bundle/packager';
|
|
import type { KnowledgePack, QuizPack } from '@/lib/bundle/types';
|
|
|
|
export interface CoursewareKnowledge {
|
|
coursewareId: string;
|
|
/** Present for legacy multi-publish courseware records; unique Makelore courses omit it. */
|
|
version?: number;
|
|
language?: string;
|
|
title: string;
|
|
knowledge: KnowledgePack;
|
|
quiz: QuizPack;
|
|
}
|
|
|
|
export interface RetrievedChunk {
|
|
sceneId: string;
|
|
order: number;
|
|
title: string;
|
|
type: string;
|
|
text: string;
|
|
narration?: string;
|
|
/** Retrieval score (term overlap); higher is better. */
|
|
score: number;
|
|
}
|
|
|
|
// Bounded in-memory cache: one entry per (courseware, version). A published
|
|
// version is immutable, so the cache never goes stale; only new versions or a
|
|
// process restart miss it.
|
|
const knowledgeCache = new Map<string, Promise<CoursewareKnowledge | null>>();
|
|
const KNOWLEDGE_CACHE_MAX = 64;
|
|
|
|
/**
|
|
* Load the knowledge/quiz packs for a published courseware version from its
|
|
* frozen bundle. Returns null when the bundle or its knowledge pack is
|
|
* missing (unpublished / pre-P3 bundles).
|
|
*/
|
|
export async function loadCoursewareKnowledge(
|
|
coursewareId: string,
|
|
version: number,
|
|
store = createFileBundleByteStore(COURSEWARE_BUNDLES_DIR),
|
|
): Promise<CoursewareKnowledge | null> {
|
|
const cacheKey = `${coursewareId}@${version}`;
|
|
const cached = knowledgeCache.get(cacheKey);
|
|
if (cached) return cached;
|
|
|
|
const pending = (async (): Promise<CoursewareKnowledge | null> => {
|
|
const bytes = await store.read(coursewareId, version);
|
|
if (!bytes) return null;
|
|
const documents = await readFrozenBundleDocuments(bytes);
|
|
return {
|
|
coursewareId,
|
|
version,
|
|
language: documents.meta.language,
|
|
title: documents.meta.stageName,
|
|
knowledge: documents.knowledge,
|
|
quiz: documents.quiz,
|
|
};
|
|
})();
|
|
|
|
if (knowledgeCache.size >= KNOWLEDGE_CACHE_MAX) {
|
|
const oldest = knowledgeCache.keys().next().value;
|
|
if (oldest !== undefined) knowledgeCache.delete(oldest);
|
|
}
|
|
knowledgeCache.set(cacheKey, pending);
|
|
return pending;
|
|
}
|
|
|
|
/** Test hook. */
|
|
export function clearKnowledgeCache(): void {
|
|
knowledgeCache.clear();
|
|
}
|
|
|
|
// ─── Retrieval ────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Split a Chinese/English query into terms: ASCII words plus CJK character
|
|
* bigrams. CJK text has no spaces, so bigrams are the standard cheap tokenizer
|
|
* for overlap scoring.
|
|
*/
|
|
export function tokenizeQuery(query: string): string[] {
|
|
const terms = new Set<string>();
|
|
for (const match of query.toLowerCase().match(/[a-z0-9_]+/g) ?? []) {
|
|
if (match.length >= 2) terms.add(match);
|
|
}
|
|
const cjk = query.replace(/[^\u4e00-\u9fff]/g, '');
|
|
if (cjk.length > 0) {
|
|
if (cjk.length === 1) terms.add(cjk);
|
|
for (let i = 0; i < cjk.length - 1; i++) {
|
|
terms.add(cjk.slice(i, i + 2));
|
|
}
|
|
}
|
|
return [...terms].filter((term) => term.length >= 2);
|
|
}
|
|
|
|
/**
|
|
* Score scenes by query-term overlap. Title matches weigh more than body text
|
|
* matches; narration counts as body text (it is the spoken explanation).
|
|
* Returns the top `limit` chunks that matched at least one term.
|
|
*/
|
|
export function retrieveChunks(
|
|
knowledge: KnowledgePack,
|
|
query: string,
|
|
limit = 3,
|
|
): RetrievedChunk[] {
|
|
const terms = tokenizeQuery(query);
|
|
if (terms.length === 0) return [];
|
|
|
|
const scored: RetrievedChunk[] = [];
|
|
for (const scene of knowledge.scenes) {
|
|
const title = scene.title.toLowerCase();
|
|
const text = `${scene.text}\n${scene.narration ?? ''}`.toLowerCase();
|
|
let score = 0;
|
|
for (const term of terms) {
|
|
if (title.includes(term)) score += 3;
|
|
else if (text.includes(term)) score += 1;
|
|
}
|
|
if (score > 0) {
|
|
scored.push({ ...scene, score });
|
|
}
|
|
}
|
|
scored.sort((a, b) => b.score - a.score || a.order - b.order);
|
|
return scored.slice(0, limit);
|
|
}
|