feat: productionize learning engine and classroom

This commit is contained in:
inman
2026-08-16 21:44:52 +08:00
parent 2d04197f3f
commit ba35adfbfa
124 changed files with 9071 additions and 796 deletions

View File

@@ -47,6 +47,10 @@ export interface PackagePersistedClassroomOptions {
/** Test seam. Production uses the SSRF-guarded server fetch below. */
fetchImpl?: typeof fetch;
publishedAt?: string;
/** Require interactive HTML only when the product request asked for it. */
requireInteractiveHtml?: boolean;
/** Strict when generation requested TTS; false allows intentional silent narration. */
requireNarrationAudio?: boolean;
}
export interface PublishPersistedClassroomOptions extends Omit<
@@ -426,7 +430,8 @@ export async function packagePersistedClassroom(
publishedAt: options.publishedAt ?? new Date().toISOString(),
appVersion: process.env.npm_package_version ?? '0.0.0',
requireAgentRoster: true,
requireInteractiveHtml: true,
requireInteractiveHtml: options.requireInteractiveHtml ?? true,
requireNarrationAudio: options.requireNarrationAudio ?? true,
strictInteractiveAssets: true,
requireComplete: true,
});

View File

@@ -62,6 +62,8 @@ export interface GenerateClassroomInput {
enableTTS?: boolean;
/** Use the original app Interactive Mode outline prompt for this classroom. */
interactiveMode?: boolean;
/** Enable the original vocational Task Engine outline and scene path. */
taskEngineMode?: boolean;
}
export type ClassroomGenerationStep =
@@ -343,6 +345,7 @@ export async function generateClassroom(
const requirements: UserRequirements = {
requirement,
...(input.interactiveMode ? { interactiveMode: true } : {}),
...(input.taskEngineMode ? { taskEngineMode: true } : {}),
};
const vocationalActive = resolveVocationalActive(requirements);
const pdfText = pdfContent?.text || undefined;

View File

@@ -44,7 +44,7 @@ export async function resumeClassroomGenerationJob(
): Promise<boolean> {
const job = await readClassroomGenerationJob(jobId);
if (!job) return false;
if (job.status === 'running' || job.status === 'queued') return false;
if (job.status !== 'failed' && job.status !== 'cancelled') return false;
if (!job.input) return false;
await updateClassroomGenerationJob(jobId, {

View File

@@ -11,9 +11,11 @@ import {
} from '@/lib/server/authz/owner-binding';
export const CLASSROOMS_DIR =
process.env.CLASSROOM_DATA_DIR ?? path.join(process.cwd(), 'data', 'classrooms');
process.env.CLASSROOM_DATA_DIR ??
path.join(process.env.LEARNING_DATA_DIR ?? path.join(process.cwd(), 'data'), 'classrooms');
export const CLASSROOM_JOBS_DIR =
process.env.CLASSROOM_JOBS_DIR ?? path.join(process.cwd(), 'data', 'classroom-jobs');
process.env.CLASSROOM_JOBS_DIR ??
path.join(process.env.LEARNING_DATA_DIR ?? path.join(process.cwd(), 'data'), 'classroom-jobs');
async function ensureDir(dir: string) {
await fs.mkdir(dir, { recursive: true });

View File

@@ -0,0 +1,285 @@
// Server-only transactional large-course ingestion.
//
// One multipart request carries `metadata` JSON plus `module-{index}` ZIPs.
// The server stages every frozen bundle as unpublished, promotes the complete
// batch, commits one schema-v2 manifest, and compensates this batch back to
// unpublished on failure. Learner APIs never expose sourceClassroomId.
import { type NextRequest, NextResponse } from 'next/server';
import { CoursePublishInProgressError } from '@/lib/course-framework/publish-state';
import type { CourseManifestRepo } from '@/lib/course-manifest-repo/types';
import {
COURSEWARE_BUNDLES_DIR,
createFileBundleByteStore,
type BundleByteStore,
} from '@/lib/courseware-repo/bundle-store';
import { COURSEWARES_DIR, createFileCoursewareRepo } from '@/lib/courseware-repo/store';
import {
COURSE_PUBLISH_MAX_UPLOAD_BYTES_ENV,
COURSEWARE_PUBLIC_BASE_URL_ENV,
CoursePublishTransportError,
DEFAULT_COURSE_PUBLISH_MAX_UPLOAD_BYTES,
DEFAULT_COURSEWARE_MAX_UPLOAD_BYTES,
MAX_COURSE_PUBLISH_METADATA_BYTES,
parseRemoteCoursePublishMetadata,
positiveIntegerEnv,
resolveCoursewarePublicBaseUrl,
type RemoteCoursePublishErrorBody,
type RemoteCoursePublishSuccessBody,
} from '@/lib/server/course-publish-contract';
import {
commitRemoteCoursePublish,
type CoursePublishTransactionRepos,
} from '@/lib/server/course-publish-transaction';
import { buildRequestOrigin } from '@/lib/server/classroom-storage';
import { capBodyStream } from '@/lib/server/capped-stream';
import { createLogger } from '@/lib/logger';
import {
COURSEWARE_PUBLISH_TOKEN_ENV,
isPublishTokenConfigured,
verifyPublishToken,
} from '@/lib/courseware-repo';
import type { CoursewareRepo } from '@/lib/courseware-repo/types';
import { getServerDeploymentRole } from '@/lib/server/ops-access';
const log = createLogger('Internal Course Publish API');
export interface CoursePublishRouteDependencies {
coursewares?: CoursewareRepo;
bundles?: BundleByteStore;
manifests?: CourseManifestRepo;
publicBaseUrl?: string;
}
function bearerToken(request: NextRequest): string | null {
const header = request.headers.get('authorization');
if (!header?.startsWith('Bearer ')) return null;
return header.slice('Bearer '.length).trim() || null;
}
function errorResponse(error: CoursePublishTransportError): NextResponse {
const body: RemoteCoursePublishErrorBody = {
success: false,
errorCode: error.errorCode,
error: error.message,
phase: error.phase,
...(error.moduleIndex ? { moduleIndex: error.moduleIndex } : {}),
...(error.details ? { details: error.details } : {}),
};
return NextResponse.json(body, { status: error.status });
}
function routeRepos(deps: CoursePublishRouteDependencies): Partial<CoursePublishTransactionRepos> {
return {
coursewares: deps.coursewares ?? createFileCoursewareRepo(COURSEWARES_DIR),
bundles: deps.bundles ?? createFileBundleByteStore(COURSEWARE_BUNDLES_DIR),
...(deps.manifests ? { manifests: deps.manifests } : {}),
};
}
function canonicalPublicBaseUrl(
request: NextRequest,
deps: CoursePublishRouteDependencies,
): string {
const configured =
deps.publicBaseUrl?.trim() || process.env[COURSEWARE_PUBLIC_BASE_URL_ENV]?.trim();
if (!configured && process.env.NODE_ENV === 'production') {
throw new CoursePublishTransportError(
'PUBLIC_URL_MISSING',
`${COURSEWARE_PUBLIC_BASE_URL_ENV} is required on a production server`,
'request',
503,
);
}
return resolveCoursewarePublicBaseUrl(configured || buildRequestOrigin(request));
}
export async function handleCoursePublishRequest(
request: NextRequest,
deps: CoursePublishRouteDependencies = {},
): Promise<NextResponse> {
const role = getServerDeploymentRole();
if (role !== 'server' && role !== 'all') {
return errorResponse(
new CoursePublishTransportError(
'FORBIDDEN',
'Transactional course publishing is only available on the server deployment',
'request',
403,
),
);
}
if (!isPublishTokenConfigured()) {
return errorResponse(
new CoursePublishTransportError(
'PUBLISH_DISABLED',
`Publish is disabled: ${COURSEWARE_PUBLISH_TOKEN_ENV} is not configured`,
'request',
503,
),
);
}
const token = bearerToken(request);
if (!verifyPublishToken(token)) {
return errorResponse(
new CoursePublishTransportError(
'UNAUTHORIZED',
'Invalid or missing publish token',
'request',
401,
),
);
}
try {
if (!request.headers.get('content-type')?.toLowerCase().startsWith('multipart/form-data')) {
throw new CoursePublishTransportError(
'INVALID_CONTENT_TYPE',
'Course publish requires multipart/form-data',
'request',
415,
);
}
const maxUploadBytes = positiveIntegerEnv(
COURSE_PUBLISH_MAX_UPLOAD_BYTES_ENV,
DEFAULT_COURSE_PUBLISH_MAX_UPLOAD_BYTES,
);
const declaredLength = Number(request.headers.get('content-length') ?? '');
if (Number.isFinite(declaredLength) && declaredLength > maxUploadBytes) {
throw new CoursePublishTransportError(
'COURSE_UPLOAD_TOO_LARGE',
`Course publish exceeds ${maxUploadBytes} bytes`,
'request',
413,
);
}
if (!request.body) {
throw new CoursePublishTransportError(
'BODY_MISSING',
'Course publish body is missing',
'request',
400,
);
}
const capped = capBodyStream(request.body, maxUploadBytes);
let form: FormData;
try {
form = await new Response(capped.stream, {
headers: { 'content-type': request.headers.get('content-type')! },
}).formData();
} catch (error) {
if (capped.exceeded()) {
throw new CoursePublishTransportError(
'COURSE_UPLOAD_TOO_LARGE',
`Course publish exceeds ${maxUploadBytes} bytes`,
'request',
413,
);
}
throw new CoursePublishTransportError(
'MULTIPART_INVALID',
'Course publish multipart body could not be parsed',
'request',
400,
undefined,
error instanceof Error ? error.message : undefined,
);
}
const metadataField = form.get('metadata');
if (typeof metadataField !== 'string') {
throw new CoursePublishTransportError(
'METADATA_MISSING',
'Course publish metadata is missing',
'request',
400,
);
}
if (new TextEncoder().encode(metadataField).byteLength > MAX_COURSE_PUBLISH_METADATA_BYTES) {
throw new CoursePublishTransportError(
'METADATA_TOO_LARGE',
'Course publish metadata exceeds the size limit',
'request',
413,
);
}
let metadataValue: unknown;
try {
metadataValue = JSON.parse(metadataField) as unknown;
} catch {
throw new CoursePublishTransportError(
'INVALID_METADATA',
'Course publish metadata is not valid JSON',
'request',
400,
);
}
const metadata = parseRemoteCoursePublishMetadata(metadataValue);
const perModuleLimit = positiveIntegerEnv(
'COURSEWARE_MAX_UPLOAD_BYTES',
DEFAULT_COURSEWARE_MAX_UPLOAD_BYTES,
);
const archives = metadata.modules.map((moduleRecord) => {
const fieldName = `module-${moduleRecord.index}`;
const fields = form.getAll(fieldName);
if (fields.length !== 1 || !(fields[0] instanceof File)) {
throw new CoursePublishTransportError(
'MODULE_ARCHIVE_MISSING',
`Module ${moduleRecord.index} requires exactly one ZIP`,
'request',
400,
moduleRecord.index,
);
}
const file = fields[0];
if (file.size > perModuleLimit) {
throw new CoursePublishTransportError(
'MODULE_ARCHIVE_TOO_LARGE',
`Module ${moduleRecord.index} ZIP exceeds ${perModuleLimit} bytes`,
'request',
413,
moduleRecord.index,
);
}
return { index: moduleRecord.index, file };
});
const loadedArchives = await Promise.all(
archives.map(async ({ index, file }) => ({
index,
zipBytes: new Uint8Array(await file.arrayBuffer()),
})),
);
const result = await commitRemoteCoursePublish({
metadata,
archives: loadedArchives,
token,
publicBaseUrl: canonicalPublicBaseUrl(request, deps),
repos: routeRepos(deps),
});
const body: RemoteCoursePublishSuccessBody = {
success: true,
record: result.record,
idempotent: result.idempotent,
};
return NextResponse.json(body, { status: result.idempotent ? 200 : 201 });
} catch (error) {
if (error instanceof CoursePublishTransportError) return errorResponse(error);
if (error instanceof CoursePublishInProgressError) {
return errorResponse(
new CoursePublishTransportError('PUBLISH_IN_PROGRESS', error.message, 'request', 409),
);
}
log.error('Transactional course publish failed:', error);
return errorResponse(
new CoursePublishTransportError(
'INTERNAL_ERROR',
'Transactional course publish failed',
'commit',
500,
undefined,
error instanceof Error ? error.message : undefined,
),
);
}
}

View File

@@ -0,0 +1,229 @@
import { nanoid } from 'nanoid';
import {
buildDocumentBundle,
documentArtifactToParsedPdfContent,
getDocumentExtractorProviders,
getMediaExtractorProviders,
MAX_DOCUMENT_BUNDLE_FILES,
MAX_DOCUMENT_BUNDLE_TOTAL_SIZE_BYTES,
} from '@/lib/document';
import { normalizeDocumentMimeType, SUPPORTED_MEDIA_MIME_TYPES } from '@/lib/document/mime';
import type { MediaArtifact } from '@/lib/document/types';
import {
getServerPDFProviders,
isServerConfiguredProvider,
resolveManagedAliDocMindCredentials,
resolvePDFApiKey,
resolvePDFBaseUrl,
} from '@/lib/server/provider-config';
export const MAX_COURSE_MATERIAL_FILE_SIZE_BYTES = 50 * 1024 * 1024;
export class CourseMaterialError extends Error {
constructor(
readonly code: string,
readonly status: number,
message: string,
) {
super(message);
this.name = 'CourseMaterialError';
}
}
function mediaArtifactText(artifact: MediaArtifact): string {
const sections: string[] = [];
const synopsis =
artifact.providerRaw &&
typeof artifact.providerRaw === 'object' &&
'synopsis' in artifact.providerRaw
? String((artifact.providerRaw as { synopsis?: unknown }).synopsis ?? '').trim()
: '';
if (synopsis) sections.push(`## Synopsis\n\n${synopsis}`);
if (artifact.transcript?.length) {
const transcript = artifact.transcript
.filter((segment) => segment.text.trim())
.map((segment) => `[${Math.floor(segment.startMs / 1000)}s] ${segment.text.trim()}`)
.join('\n');
if (transcript) sections.push(`## Transcript\n\n${transcript}`);
}
if (artifact.keyframes?.length) {
const keyframes = artifact.keyframes
.map((keyframe) => keyframe.description || keyframe.ocrText || '')
.filter(Boolean)
.join('\n');
if (keyframes) sections.push(`## Keyframes\n\n${keyframes}`);
}
return sections.join('\n\n');
}
function managedDocumentProvider(mimeType: string) {
const configured = new Set(Object.keys(getServerPDFProviders()));
const candidates = getDocumentExtractorProviders().filter(
(provider) =>
provider.supportedMimeTypes.includes(mimeType) &&
(provider.id === 'plain-text' || provider.id === 'unpdf' || configured.has(provider.id)),
);
// Prefer an operator-managed high-fidelity extractor. Plain text and unpdf
// are safe in-process fallbacks and are selected only when no managed
// provider for the MIME exists.
return (
candidates.find((provider) => isServerConfiguredProvider('pdf', provider.id)) ?? candidates[0]
);
}
function managedMediaProvider(mimeType: string) {
return getMediaExtractorProviders().find(
(provider) =>
provider.supportedMimeTypes.includes(mimeType) &&
isServerConfiguredProvider('pdf', provider.id),
);
}
function managedExtractorConfig(providerId: string) {
const ali = providerId === 'alidocmind' ? resolveManagedAliDocMindCredentials() : undefined;
return {
providerId,
apiKey: resolvePDFApiKey(providerId),
baseUrl: ali?.baseUrl ?? resolvePDFBaseUrl(providerId),
accessKeyId: ali?.accessKeyId,
accessKeySecret: ali?.accessKeySecret,
allowEnvFallback: isServerConfiguredProvider('pdf', providerId),
};
}
/**
* Extract files with server-managed providers and fold them into the exact
* pdfContent shape consumed by the original classroom/framework runners.
* Caller-supplied provider credentials and storage keys never enter this API.
*/
export async function extractManagedCourseMaterials(
files: readonly File[],
): Promise<{ text: string; images: string[] }> {
if (files.length === 0) return { text: '', images: [] };
if (files.length > MAX_DOCUMENT_BUNDLE_FILES) {
throw new CourseMaterialError(
'too_many_materials',
413,
`At most ${MAX_DOCUMENT_BUNDLE_FILES} course materials are allowed`,
);
}
const totalBytes = files.reduce((total, file) => total + file.size, 0);
if (totalBytes > MAX_DOCUMENT_BUNDLE_TOTAL_SIZE_BYTES) {
throw new CourseMaterialError(
'materials_too_large',
413,
'Course materials exceed the 150 MiB aggregate limit',
);
}
const parts = await Promise.all(
files.map(async (file, index) => {
if (file.size <= 0 || file.size > MAX_COURSE_MATERIAL_FILE_SIZE_BYTES) {
throw new CourseMaterialError(
'material_too_large',
413,
`Material "${file.name}" must be between 1 byte and 50 MiB`,
);
}
const mimeType = normalizeDocumentMimeType({ mimeType: file.type, fileName: file.name });
if (!mimeType) {
throw new CourseMaterialError(
'unsupported_material',
415,
`Unsupported course material type for "${file.name}"`,
);
}
const buffer = Buffer.from(await file.arrayBuffer());
const source = {
id: `material_${nanoid(8)}`,
name: file.name || `material-${index + 1}`,
size: file.size,
mimeType,
order: index + 1,
};
if (SUPPORTED_MEDIA_MIME_TYPES.includes(mimeType)) {
const provider = managedMediaProvider(mimeType);
if (!provider) {
throw new CourseMaterialError(
'extractor_not_configured',
422,
`No server-managed media extractor supports "${file.name}"`,
);
}
const artifact = await provider.extract({
buffer,
fileName: file.name,
fileSize: file.size,
mimeType,
config: managedExtractorConfig(provider.id),
});
const text = mediaArtifactText(artifact);
if (!text.trim()) {
throw new CourseMaterialError(
'material_parse_failed',
422,
`No usable transcript or synopsis was extracted from "${file.name}"`,
);
}
return { source, text, rawTextLength: text.length, images: [] };
}
const provider = managedDocumentProvider(mimeType);
if (!provider) {
throw new CourseMaterialError(
'extractor_not_configured',
422,
`No server-managed document extractor supports "${file.name}"`,
);
}
const artifact = await provider.extract({
buffer,
fileName: file.name,
fileSize: file.size,
mimeType,
config: managedExtractorConfig(provider.id),
});
const parsed = documentArtifactToParsedPdfContent(artifact);
if (!parsed.text.trim() && parsed.images.length === 0) {
throw new CourseMaterialError(
'material_parse_failed',
422,
`No usable content was extracted from "${file.name}"`,
);
}
const fallbackImages = parsed.images.map((src, imageIndex) => ({
id: `img_${imageIndex + 1}`,
src,
pageNumber: 0,
description: undefined,
width: undefined,
height: undefined,
}));
const images = (parsed.metadata?.pdfImages ?? fallbackImages).map((image) => ({
id: image.id,
src: image.src,
pageNumber: image.pageNumber,
description: image.description,
width: image.width,
height: image.height,
}));
return {
source,
text: parsed.text,
rawTextLength: parsed.text.length,
pageCount: parsed.metadata?.pageCount,
images,
};
}),
);
const bundle = buildDocumentBundle(parts);
return {
text: bundle.text,
images: bundle.images
.filter((image) => image.visionPriority > 0)
.sort((a, b) => b.visionPriority - a.visionPriority)
.map((image) => image.src),
};
}

View File

@@ -6,6 +6,8 @@ import { apiError, API_ERROR_CODES } from '@/lib/server/api-response';
const SAFE_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
export const OPS_PUBLIC_ORIGIN_ENV = 'OPS_PUBLIC_ORIGIN';
export const WORKS_SQUARE_API_BASE_URL_ENV = 'WORKS_SQUARE_API_BASE_URL';
export const WORKS_OPERATIONS_ADMIN_USERNAME = 'jiaoyuop';
function configuredOpsOrigin(): string | null | undefined {
const configured = process.env[OPS_PUBLIC_ORIGIN_ENV]?.trim();
@@ -69,31 +71,85 @@ export function isOpsDeployment(role = getServerDeploymentRole()): boolean {
* then authenticates the browser session. A future account/role system can
* replace this helper without changing every course route again.
*/
export function requireOpsAccess(request: NextRequest) {
if (!isOpsDeployment()) {
return apiError(API_ERROR_CODES.FORBIDDEN, 403, 'Operations capability is disabled');
}
const accessCode = process.env.ACCESS_CODE;
if (!accessCode) {
if (process.env.NODE_ENV === 'production') {
return apiError(
API_ERROR_CODES.INTERNAL_ERROR,
503,
'Operations deployment is not configured with ACCESS_CODE',
);
export function configuredWorksSquareApiOrigin(): string | null {
const raw = process.env[WORKS_SQUARE_API_BASE_URL_ENV]?.trim();
if (!raw) return null;
try {
const url = new URL(raw);
if (
(url.protocol !== 'https:' && url.protocol !== 'http:') ||
url.username ||
url.password ||
url.search ||
url.hash ||
(url.pathname !== '/' && url.pathname !== '')
) {
return null;
}
return url.origin;
} catch {
return null;
}
}
const token = request.cookies.get(ACCESS_CODE_COOKIE_NAME)?.value;
if (!token || !verifyAccessToken(token, accessCode)) {
return apiError(API_ERROR_CODES.INVALID_CREDENTIALS, 401, 'Operations access required');
async function authorizeWithWorksSession(request: NextRequest, origin: string) {
const authorization = request.headers.get('authorization')?.trim() ?? '';
if (!/^Bearer\s+\S+$/.test(authorization) || authorization.length > 8192) {
return apiError(API_ERROR_CODES.INVALID_CREDENTIALS, 401, 'Works administrator session required');
}
try {
const response = await fetch(`${origin}/api/auth/me`, {
method: 'GET',
headers: { Authorization: authorization, Accept: 'application/json' },
redirect: 'error',
cache: 'no-store',
signal: AbortSignal.timeout(5000),
});
if (!response.ok) {
return apiError(API_ERROR_CODES.INVALID_CREDENTIALS, 401, 'Works administrator session is invalid');
}
const identity = (await response.json()) as { username?: unknown; site_role?: unknown };
if (
identity.username !== WORKS_OPERATIONS_ADMIN_USERNAME ||
identity.site_role !== 'admin'
) {
return apiError(API_ERROR_CODES.FORBIDDEN, 403, 'Works administrator role required');
}
return null;
} catch {
return apiError(API_ERROR_CODES.INTERNAL_ERROR, 503, 'Works identity service is unavailable');
}
}
export async function requireOpsAccess(request: NextRequest) {
if (!isOpsDeployment()) {
return apiError(API_ERROR_CODES.FORBIDDEN, 403, 'Operations capability is disabled');
}
if (!hasValidOpsMutationOrigin(request)) {
return apiError(API_ERROR_CODES.FORBIDDEN, 403, 'Cross-origin operations request denied');
}
const worksOrigin = configuredWorksSquareApiOrigin();
if (worksOrigin) return authorizeWithWorksSession(request, worksOrigin);
// Production Learning Ops always delegates identity to Works. The legacy
// ACCESS_CODE fallback is deliberately development-only.
if (process.env.NODE_ENV === 'production') {
return apiError(
API_ERROR_CODES.INTERNAL_ERROR,
503,
`Operations identity is not configured (${WORKS_SQUARE_API_BASE_URL_ENV})`,
);
}
const accessCode = process.env.ACCESS_CODE;
if (!accessCode) return null;
const token = request.cookies.get(ACCESS_CODE_COOKIE_NAME)?.value;
if (!token || !verifyAccessToken(token, accessCode)) {
return apiError(API_ERROR_CODES.INVALID_CREDENTIALS, 401, 'Operations access required');
}
return null;
}