chore: establish learning module baseline

OpenMAIC/tests/eval/shared/resolve-model.test.ts

@@ -0,0 +1,51 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+
+describe('resolveEvalModel', () => {
+  const ORIG_ENV = process.env;
+
+  beforeEach(() => {
+    process.env = { ...ORIG_ENV };
+    vi.resetModules();
+  });
+
+  afterEach(() => {
+    process.env = ORIG_ENV;
+    vi.restoreAllMocks();
+  });
+
+  it('throws a helpful error when env var is unset and no fallback is given', async () => {
+    delete process.env.EVAL_FOO_MODEL;
+    vi.doMock('@/lib/server/resolve-model', () => ({
+      resolveModel: vi.fn(),
+    }));
+    const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
+    await expect(resolveEvalModel('EVAL_FOO_MODEL')).rejects.toThrow(/EVAL_FOO_MODEL/);
+  });
+
+  it('uses the env var when set', async () => {
+    process.env.EVAL_FOO_MODEL = 'openai:gpt-4.1';
+    const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
+    vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
+    const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
+    await resolveEvalModel('EVAL_FOO_MODEL');
+    expect(resolveModel).toHaveBeenCalledWith({ modelString: 'openai:gpt-4.1' });
+  });
+
+  it('uses the explicit fallback when env var is unset', async () => {
+    delete process.env.EVAL_FOO_MODEL;
+    const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
+    vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
+    const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
+    await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
+    expect(resolveModel).toHaveBeenCalledWith({ modelString: 'google:gemini-2.5-flash' });
+  });
+
+  it('env var takes precedence over fallback', async () => {
+    process.env.EVAL_FOO_MODEL = 'anthropic:claude-haiku-4-5';
+    const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
+    vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
+    const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
+    await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
+    expect(resolveModel).toHaveBeenCalledWith({ modelString: 'anthropic:claude-haiku-4-5' });
+  });
+});

OpenMAIC/tests/eval/shared/run-dir.test.ts

@@ -0,0 +1,39 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { existsSync, rmSync, mkdtempSync } from 'fs';
+import { tmpdir } from 'os';
+import { join, sep } from 'path';
+import { createRunDir } from '@/eval/shared/run-dir';
+
+describe('createRunDir', () => {
+  let tempRoot: string;
+
+  beforeEach(() => {
+    tempRoot = mkdtempSync(join(tmpdir(), 'run-dir-test-'));
+  });
+
+  afterEach(() => {
+    rmSync(tempRoot, { recursive: true, force: true });
+  });
+
+  it('creates <base>/<sanitized-model>/<timestamp>/ and returns the path', () => {
+    const runDir = createRunDir(tempRoot, 'openai:gpt-4.1');
+    expect(runDir.startsWith(join(tempRoot, 'openai-gpt-4.1'))).toBe(true);
+    expect(existsSync(runDir)).toBe(true);
+  });
+
+  it('sanitizes both : and / from the model string', () => {
+    const runDir = createRunDir(tempRoot, 'google:gemini-2.5-flash/latest');
+    expect(runDir).toContain('google-gemini-2.5-flash-latest');
+    expect(runDir).not.toMatch(/[:/]gemini/);
+  });
+
+  it('timestamp segment has no colons or dots', () => {
+    const runDir = createRunDir(tempRoot, 'x');
+    const segments = runDir.split(sep);
+    const timestamp = segments[segments.length - 1];
+    const timestampWithoutDrive = timestamp.replace(/^[A-Za-z]:/, '');
+    expect(timestampWithoutDrive).not.toContain(':');
+    expect(timestampWithoutDrive).not.toContain('.');
+    expect(timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}$/);
+  });
+});