wangxuming/makelore
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'codex/20260912-model-capabilities-c48271f9-model-capabilities'

Browse Source
This commit is contained in:
brother7
2026-09-12 22:09:09 +08:00
parent a06909e78b 26cbb29aa9
commit 2888aaedac
29 changed files with 681 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
tests/unit/managed-model-capabilities.test.ts Normal file
View File

@@ -0,0 +1,112 @@
// @vitest-environment node
import { createServer } from 'node:http';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { describe, expect, it, vi } from 'vitest';
import { streamSimple } from '@earendil-works/pi-ai/api/openai-completions';
import type { Model } from '@earendil-works/pi-ai';
import {
buildManagedModelRequest, managedPiThinkingLevel, managedReasoningOptions,
unknownManagedModelCapability,
type ManagedModelCapability, type ManagedReasoningChoice,
} from '../../shared/managed-model-capabilities';
import { materializeMakelorePiExtension } from '../../electron/coding-runtime/pi/extensions/makelore-runtime';
function capability(format: 'qwen' | 'deepseek'): ManagedModelCapability {
return {
...unknownManagedModelCapability(),
inputModalities: ['text', 'image'], outputModalities: ['text'], resolutionStatus: 'ready',
reasoning: {
supported: true, canDisable: true, defaultEnabled: true,
effortValues: ['low', 'medium', 'xhigh', 'max', 'future-native'],
defaultEffort: 'low', controlFormat: format, budget: null,
},
};
}
describe('managed model reasoning', () => {
it('distinguishes unknown, switch-only and always-on capabilities', () => {
expect(managedReasoningOptions(unknownManagedModelCapability()).map((x) => x.value)).toEqual(['default']);
const profile = capability('qwen');
profile.reasoning.effortValues = [];
expect(managedReasoningOptions(profile).map((x) => x.value)).toEqual(['default', 'disabled', 'enabled']);
profile.reasoning.canDisable = false;
profile.reasoning.effortValues = ['max'];
expect(managedReasoningOptions(profile).map((x) => x.value)).toEqual(['default', 'effort:max']);
expect(buildManagedModelRequest('model', { mode: 'enabled', effort: 'max' }, profile).reasoningFields)
.toEqual({ reasoning_effort: 'max' });
expect(() => buildManagedModelRequest('model', { mode: 'disabled' }, profile)).toThrow('不可用');
expect(() => buildManagedModelRequest('model', { mode: 'enabled', effort: 'high' }, profile)).toThrow('不可用');
});
it.each(['qwen', 'deepseek'] as const)('sends exact %s controls through Pi and the bundled extension', async (format) => {
const root = await mkdtemp(path.join(tmpdir(), 'makelore-managed-wire-'));
const bodies: Record<string, unknown>[] = [];
const server = createServer(async (request, response) => {
let raw = '';
for await (const chunk of request) raw += String(chunk);
bodies.push(JSON.parse(raw));
response.writeHead(200, { 'content-type': 'text/event-stream' });
response.end('data: ' + JSON.stringify({
id: 'test-response', object: 'chat.completion.chunk', created: 0, model: 'catalog-model',
choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
}) + '\n\ndata: [DONE]\n\n');
});
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
try {
const address = server.address();
if (!address || typeof address === 'string') throw new Error('No test server address');
const contextFile = path.join(root, 'worker.json');
const profile = capability(format);
await writeFile(contextFile, JSON.stringify({ runId: 'test-run', tools: [], allowedToolNames: [] }));
const extensionPath = await materializeMakelorePiExtension(root);
const extension = await import(/* @vite-ignore */ pathToFileURL(extensionPath).href);
const hooks = new Map<string, (event: { payload: unknown }) => Promise<unknown>>();
await extension.createMakeloreRuntime({ contextFile })({
registerFlag: vi.fn(), registerTool: vi.fn(),
on: (name: string, handler: (event: { payload: unknown }) => Promise<unknown>) => hooks.set(name, handler),
});
const beforeRequest = hooks.get('before_provider_request');
expect(beforeRequest).toBeTypeOf('function');
const model: Model<'openai-completions'> = {
id: 'catalog-model', name: 'Catalog model', api: 'openai-completions',
provider: 'managed-test', baseUrl: 'http://127.0.0.1:' + address.port + '/v1',
reasoning: true, input: ['text', 'image'], contextWindow: 4096, maxTokens: 512,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
thinkingLevelMap: { off: 'off', low: 'low', medium: 'medium', xhigh: 'xhigh', max: 'max' },
compat: { thinkingFormat: format, supportsReasoningEffort: true },
};
const choices: ManagedReasoningChoice[] = [
{ mode: 'default' }, { mode: 'disabled' }, { mode: 'enabled' },
...['low', 'medium', 'xhigh', 'max', 'future-native'].map((effort) => ({ mode: 'enabled' as const, effort })),
];
for (const choice of choices) {
const prepared = buildManagedModelRequest(model.id, choice, profile);
await writeFile(contextFile, JSON.stringify({
runId: 'test-run', tools: [], allowedToolNames: [], managedModelRequest: prepared,
}));
const result = await streamSimple(model, {
messages: [{ role: 'user', content: 'test', timestamp: 0 }],
}, {
apiKey: 'synthetic-key', reasoning: managedPiThinkingLevel(choice),
onPayload: (payload) => beforeRequest!({ payload }),
}).result();
expect(result.stopReason, result.errorMessage).not.toBe('error');
const body = bodies.at(-1)!;
for (const field of ['thinking', 'enable_thinking', 'reasoning_effort', 'thinking_budget']) {
expect(body[field], field + ' for ' + JSON.stringify(choice)).toEqual(prepared.reasoningFields[field]);
}
expect(body.model).toBe(model.id);
expect(body.messages).toEqual([{ role: 'user', content: 'test' }]);
}
expect(bodies).toHaveLength(choices.length);
} finally {
server.closeAllConnections();
await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
await rm(root, { recursive: true, force: true });
}
});
});

56
tests/unit/pi-conversation-runtime.test.ts
View File

@@ -28,6 +28,7 @@ import type {
PiRpcResponse,
} from '../../electron/coding-runtime/pi/rpc-client';
import { PI_084_TEXT_TURN } from '../fixtures/pi-0.84.2-projector-fixtures';
import { unknownManagedModelCapability, type ManagedModelRequest } from '../../shared/managed-model-capabilities';
const roots: string[] = [];
const NOW = '2026-08-22T15:00:00.000Z';
@@ -152,9 +153,11 @@ class RuntimeFakeWorker implements PiConversationWorker {
}
class TrackingExtensionHost extends PiManagedExtensionHost {
managedRequest?: ManagedModelRequest;
readonly runs = new Map<string, { generation: number; runId: string }>();
override async bindRun(conversationId: string, generation: number, runId: string): Promise<void> {
override async bindRun(conversationId: string, generation: number, runId: string, request?: ManagedModelRequest): Promise<void> {
this.managedRequest = request;
this.runs.set(conversationId, { generation, runId });
}
@@ -168,6 +171,57 @@ afterEach(async () => {
});
describe('Pi Conversation runtime', () => {
it('persists native managed choices and rejects removed effort or unsupported images before prompt', async () => {
const projectPath = await mkdtemp(path.join(tmpdir(), 'makelore-managed-runtime-'));
roots.push(projectPath);
const projectStore = createCodingProjectStore(createMemoryCodingProjectStorage(), {
createId: () => 'project-managed', now: () => NOW,
});
await createLocalCodingProject({ projectPath, now: NOW }, projectStore);
const model = { accountId: 'niancode-user-models', modelId: 'unfamiliar-model', thinkingLevel: 'high' as const };
await createCodingProjectAgent(projectPath, {
id: 'agent-a', avatarId: 'avatar-01', roleName: 'Implementer', name: 'Agent A',
model, modelResolution: 'resolved',
responsibility: { mission: 'Implement', owns: [], boundaries: [], collaborators: [], principles: [] },
}, { now: NOW });
const store = createCodingConversationStore(projectPath);
const conversation = await store.create({ agentId: 'agent-a', title: 'Managed', model, modelResolution: 'resolved' });
let worker: RuntimeFakeWorker;
const pool = new PiWorkerPool({ openWorker: async ({generation}) => {
worker = new RuntimeFakeWorker('managed-worker', generation);
return { worker, session: { piSessionId: 'session', sessionKey: 'key' } };
} });
const capability = unknownManagedModelCapability();
capability.inputModalities = ['text'];
capability.reasoning = { supported: true, canDisable: true, defaultEnabled: true,
controlFormat: 'qwen', effortValues: ['xhigh', 'future-native'], defaultEffort: 'xhigh', budget: null };
const host = new TrackingExtensionHost();
const runtime = new PiConversationRuntime({
pool, registry: new PiSessionRegistry({ projectStore }), extensionHost: host,
resolveImages: async () => [],
resolveModel: async candidate => ({ ...candidate, input: ['text'],
runtimeProviderId: 'managed', managedCapability: structuredClone(capability) }),
});
try {
await runtime.prepare({ conversationId: conversation.id, projectId: 'project-managed', agentId: 'agent-a',
title: conversation.title, model: { model, modelResolution: 'resolved' } });
expect((await runtime.getSnapshot(conversation.id)).conversation.model.model?.reasoningChoice).toEqual({ mode: 'default' });
await runtime.setThinking({ conversationId: conversation.id, thinkingLevel: 'off',
reasoningChoice: { mode: 'enabled', effort: 'future-native' } });
expect((await store.get(conversation.id))?.model?.reasoningChoice).toEqual({ mode: 'enabled', effort: 'future-native' });
capability.reasoning.effortValues = ['xhigh'];
const prompt = { conversationId: conversation.id, clientRequestId: 'request-managed', mode: 'prompt' as const, text: 'Hello', attachments: [] };
await expect(runtime.prompt(prompt)).rejects.toThrow('思考选项已不可用');
expect(worker!.requests.filter(r => r.type === 'prompt')).toHaveLength(0);
await runtime.setThinking({ conversationId: conversation.id, thinkingLevel: 'off', reasoningChoice: { mode: 'default' } });
await expect(runtime.prompt({ ...prompt, attachments: [{ attachmentId: 'image' }] })).rejects.toThrow('图片输入');
await runtime.prompt(prompt);
expect(host.managedRequest).toEqual({ modelId: 'unfamiliar-model', choice: { mode: 'default' }, reasoningFields: {} });
for (const event of PI_084_TEXT_TURN.events) worker!.emit(structuredClone(event));
await runtime.getSnapshot(conversation.id);
} finally { await runtime.shutdown(); }
});
it('separates RPC acceptance from settle and changes only the target Conversation model', async () => {
const projectPath = await mkdtemp(path.join(tmpdir(), 'makelore-pi-runtime-'));
roots.push(projectPath);

241
tests/unit/pi-provider-config.test.ts
View File

@@ -1,3 +1,4 @@
import { normalizeManagedModelCatalog } from '../../shared/managed-model-capabilities';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
@@ -107,23 +108,10 @@ describe('Pi Provider catalog', () => {
'X-Works-Square-AI-Token',
]);
expect(descriptor.models[0]).toMatchObject({
id: 'qwen3.6-plus',
input: ['text', 'image'],
reasoning: true,
contextWindow: 1_000_000,
maxOutputTokens: 65_536,
thinkingLevelMap: {
minimal: null,
low: null,
medium: null,
high: 'high',
},
compat: {
thinkingFormat: 'qwen',
supportsDeveloperRole: false,
supportsReasoningEffort: false,
supportsStore: false,
},
id: 'qwen3.6-plus', input: ['text'], reasoning: false,
contextWindow: 1_000_000, maxOutputTokens: 65_536,
managedCapability: { resolutionStatus: 'unknown' },
compat: { supportsDeveloperRole: false, supportsReasoningEffort: false, supportsStore: false },
});
expect(catalog.modelsFile.providers[descriptor.runtimeProviderId]?.models[0]?.compat)
.toMatchObject({ supportsDeveloperRole: false });
@@ -170,48 +158,6 @@ describe('Pi Provider catalog', () => {
expect(descriptor.models[0]?.compat?.supportsDeveloperRole).toBe(false);
});
it('projects Qwen3.8 Max reasoning effort levels into Pi models.json', () => {
const provider = account({
id: 'niancode-user-models',
vendorId: 'custom',
apiProtocol: 'openai-completions',
baseUrl: 'http://127.0.0.1:54321/api/ai-proxy/v1',
model: 'qwen3.8-max',
metadata: {
worksSquareCredentialMode: 'works_square_ai_gateway_proxy',
customModels: ['qwen3.8-max'],
},
});
const catalog = buildPiProviderCatalog({ accounts: [provider] });
const descriptor = catalog.descriptors[0]!.models[0]!;
const written = catalog.modelsFile.providers[resolvePiRuntimeProviderId(provider.id)]!.models[0]!;
expect(descriptor).toMatchObject({
id: 'qwen3.8-max',
input: ['text', 'image'],
reasoning: true,
contextWindow: 1_000_000,
maxOutputTokens: 131_072,
thinkingLevelMap: {
minimal: null,
low: 'low',
medium: 'medium',
high: 'xhigh',
},
compat: {
thinkingFormat: 'qwen',
supportsDeveloperRole: false,
supportsReasoningEffort: true,
supportsStore: false,
},
});
expect(written).toMatchObject({
reasoning: true,
thinkingLevelMap: descriptor.thinkingLevelMap,
compat: descriptor.compat,
});
});
it('uses account-scoped model capability metadata and rejects unavailable models', () => {
const provider = account({ id: 'account-with-vision', model: 'vision-model' });
@@ -244,165 +190,40 @@ describe('Pi Provider catalog', () => {
})).toThrowError(PiProviderConfigError);
});
it('projects the managed DeepSeek capability contract into Pi models.json', () => {
it.each(['qwen', 'deepseek'] as const)('uses v2 %s capabilities for unfamiliar models and native effort values', format => {
const provider = account({
id: 'niancode-user-models',
vendorId: 'custom',
apiProtocol: 'openai-completions',
baseUrl: 'http://127.0.0.1:54321/api/ai-proxy/v1',
model: 'deepseek/deepseek-v4-pro',
id: 'niancode-user-models', vendorId: 'custom', apiProtocol: 'openai-completions',
model: 'future-model', baseUrl: 'https://gateway.test/v1',
metadata: {
worksSquareCredentialMode: 'works_square_ai_gateway_proxy',
customModels: ['deepseek/deepseek-v4-pro'],
worksSquareModelCapabilitiesV2: normalizeManagedModelCatalog({
schema_version: 2, models: { 'future-model': {
input_modalities: ['text', 'image'], output_modalities: ['text'],
reasoning: { supported: true, can_disable: true, default_enabled: true,
control_format: format, effort_values: ['low', 'medium', 'xhigh', 'max', 'future'] },
limits: { context_window: 123456, max_output_tokens: 1234 }, resolution_status: 'ready',
} },
}),
},
});
const catalog = buildPiProviderCatalog({ accounts: [provider] });
const descriptor = catalog.descriptors[0]!.models[0]!;
const written = catalog.modelsFile.providers[resolvePiRuntimeProviderId(provider.id)]!.models[0]!;
expect(descriptor).toMatchObject({
id: 'deepseek-v4-pro',
reasoning: true,
contextWindow: 1_000_000,
maxOutputTokens: 384_000,
compat: {
thinkingFormat: 'deepseek',
requiresReasoningContentOnAssistantMessages: true,
},
thinkingLevelMap: {
minimal: null,
low: 'low',
medium: null,
high: 'high',
max: 'max',
},
});
expect(written).toMatchObject({
reasoning: true,
thinkingLevelMap: descriptor.thinkingLevelMap,
compat: descriptor.compat,
});
});
it('uses server reasoning capabilities while retaining local model metadata', () => {
const provider = account({
id: 'niancode-user-models',
vendorId: 'custom',
apiProtocol: 'openai-completions',
baseUrl: 'http://127.0.0.1:54321/api/ai-proxy/v1',
model: 'qwen3.8-max',
metadata: {
worksSquareCredentialMode: 'works_square_ai_gateway_proxy',
customModels: ['qwen3.8-max'],
worksSquareModelCapabilities: {
'qwen3.8-max': {
reasoningEfforts: ['low'],
reasoningCanDisable: false,
},
},
},
});
const descriptor = buildPiProviderCatalog({ accounts: [provider] }).descriptors[0]!.models[0]!;
expect(descriptor).toMatchObject({
id: 'qwen3.8-max',
input: ['text', 'image'],
reasoning: true,
contextWindow: 1_000_000,
maxOutputTokens: 131_072,
thinkingLevelMap: {
off: null,
minimal: null,
low: 'low',
medium: null,
high: null,
max: null,
},
compat: {
thinkingFormat: 'qwen',
supportsDeveloperRole: false,
supportsReasoningEffort: true,
supportsStore: false,
},
});
});
it('enables reasoning-effort serialization for a server model without a local profile', () => {
const provider = account({
id: 'niancode-user-models',
vendorId: 'custom',
apiProtocol: 'openai-completions',
baseUrl: 'http://127.0.0.1:54321/api/ai-proxy/v1',
model: 'future-reasoning-model',
metadata: {
worksSquareCredentialMode: 'works_square_ai_gateway_proxy',
customModels: ['future-reasoning-model'],
worksSquareModelCapabilities: {
'future-reasoning-model': {
reasoningEfforts: ['low', 'high', 'max'],
reasoningCanDisable: true,
},
},
},
});
const descriptor = buildPiProviderCatalog({ accounts: [provider] }).descriptors[0]!.models[0]!;
expect(descriptor).toMatchObject({
id: 'future-reasoning-model',
reasoning: true,
compat: {
supportsDeveloperRole: false,
supportsReasoningEffort: true,
},
thinkingLevelMap: {
minimal: null,
low: 'low',
medium: null,
high: 'high',
max: 'max',
},
});
expect(descriptor.thinkingLevelMap).not.toHaveProperty('off');
});
it('lets an explicit empty server effort list override a locally reasoning model', () => {
const provider = account({
id: 'niancode-user-models',
vendorId: 'custom',
apiProtocol: 'openai-completions',
baseUrl: 'http://127.0.0.1:54321/api/ai-proxy/v1',
model: 'deepseek-v4-pro',
metadata: {
worksSquareCredentialMode: 'works_square_ai_gateway_proxy',
customModels: ['deepseek-v4-pro'],
worksSquareModelCapabilities: {
'deepseek-v4-pro': {
reasoningEfforts: [],
reasoningCanDisable: false,
},
},
},
});
const descriptor = buildPiProviderCatalog({ accounts: [provider] }).descriptors[0]!.models[0]!;
expect(descriptor.reasoning).toBe(false);
expect(descriptor.thinkingLevelMap).toEqual({
off: null,
minimal: null,
low: null,
medium: null,
high: null,
max: null,
});
expect(descriptor.compat).toMatchObject({
thinkingFormat: 'deepseek',
supportsReasoningEffort: true,
requiresReasoningContentOnAssistantMessages: true,
input: ['text', 'image'], reasoning: true, contextWindow: 123456, maxOutputTokens: 1234,
compat: { thinkingFormat: format, supportsReasoningEffort: false, requiresReasoningContentOnAssistantMessages: true },
managedCapability: { reasoning: { effortValues: ['low', 'medium', 'xhigh', 'max', 'future'] } },
});
expect(descriptor.thinkingLevelMap).toBeUndefined();
expect(catalog.modelsFile.providers[resolvePiRuntimeProviderId(provider.id)]!.models[0])
.not.toHaveProperty('managedCapability');
const profile = provider.metadata!.worksSquareModelCapabilitiesV2!.models['future-model']!;
profile.inputModalities = ['text'];
profile.reasoning.supported = false;
expect(buildPiProviderCatalog({ accounts: [provider] }).descriptors[0]!.models[0])
.toMatchObject({ input: ['text'], reasoning: false });
});
it('serializes DeepSeek off without sending a reasoning effort', async () => {

19
tests/unit/provider-routes.test.ts
View File

@@ -120,6 +120,21 @@ describe('provider host api routes', () => {
});
});
it('preserves exact v2 public aliases and native effort values on import', () => {
const config = normalizeImportedUserModelConfig({
base_url: 'https://gateway.test/v1', api_key: 'test', models: ['deepseek/public-model'],
model_capabilities_v2: { schema_version: 2, models: {
'deepseek/public-model': { input_modalities: ['text', 'image'],
reasoning: { supported: true, can_disable: true, effort_values: ['medium', 'xhigh', 'new-native'], control_format: 'qwen' } },
'not-authorized': { input_modalities: ['image'] },
} },
});
expect(config.models).toEqual(['deepseek/public-model']);
expect(Object.keys(config.modelCapabilitiesV2!.models)).toEqual(['deepseek/public-model']);
expect(config.modelCapabilitiesV2!.models['deepseek/public-model']!.reasoning.effortValues)
.toEqual(['medium', 'xhigh', 'new-native']);
});
it('persists server capabilities and invalidates the runtime when they change', async () => {
const existing = account({
id: 'niancode-user-models',
@@ -152,9 +167,13 @@ describe('provider host api routes', () => {
reasoning_can_disable: true,
},
},
model_capabilities_v2: { schema_version: 2, models: { 'deepseek/deepseek-v4-pro': {
input_modalities: ['text'], reasoning: { supported: true, can_disable: true, effort_values: ['xhigh'] },
} } },
}), { status: 200, headers: { 'content-type': 'application/json' } }));
const imported = await importCurrentUserModelConfig(context, 'access-token');
expect(imported.account.metadata?.worksSquareModelCapabilitiesV2?.models['deepseek/deepseek-v4-pro']?.reasoning.effortValues).toEqual(['xhigh']);
expect(imported.account.metadata?.worksSquareModelCapabilities).toEqual({
'deepseek-v4-pro': {