feat: qualify Pi runtime cutover foundation
This commit is contained in:
324
scripts/probe-pi-provider-contracts.mjs
Normal file
324
scripts/probe-pi-provider-contracts.mjs
Normal file
@@ -0,0 +1,324 @@
|
||||
import { createServer } from 'node:http';
|
||||
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import {
|
||||
MAKELore_PROVIDER_PROTOCOLS,
|
||||
parseProbeArgs,
|
||||
resolvePiRuntime,
|
||||
runProviderQualification,
|
||||
validatePiIdentity,
|
||||
} from './probe-pi-runtime.mjs';
|
||||
|
||||
const LOCAL_API_KEY_ENV = 'PI_PROBE_LOCAL_CONTRACT_KEY';
|
||||
const LOCAL_API_KEY = 'pi-local-contract-only';
|
||||
const CUSTOM_HEADER = 'x-makelore-pi-probe';
|
||||
const IMAGE_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Z4N8AAAAASUVORK5CYII=';
|
||||
|
||||
function readRequestBody(request) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const chunks = [];
|
||||
request.on('data', (chunk) => chunks.push(chunk));
|
||||
request.once('end', () => {
|
||||
try {
|
||||
resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
request.once('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function writeSse(response, event) {
|
||||
if (response.destroyed || response.writableEnded) return;
|
||||
response.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
||||
function finishResponse(response, events) {
|
||||
setTimeout(() => {
|
||||
if (response.destroyed || response.writableEnded) return;
|
||||
for (const event of events) writeSse(response, event);
|
||||
response.end();
|
||||
}, 250);
|
||||
}
|
||||
|
||||
function respondOpenAiCompletions(response, body) {
|
||||
const id = `chatcmpl-local-${Date.now()}`;
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
response.write(`data: ${JSON.stringify({
|
||||
id,
|
||||
object: 'chat.completion.chunk',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: body.model,
|
||||
choices: [{ index: 0, delta: { role: 'assistant', content: 'PI_PROVIDER_OK' }, finish_reason: null }],
|
||||
})}\n\n`);
|
||||
setTimeout(() => {
|
||||
if (response.destroyed || response.writableEnded) return;
|
||||
response.write(`data: ${JSON.stringify({
|
||||
id,
|
||||
object: 'chat.completion.chunk',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: body.model,
|
||||
choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
|
||||
})}\n\n`);
|
||||
response.end('data: [DONE]\n\n');
|
||||
}, 250);
|
||||
}
|
||||
|
||||
function responseEnvelope(id, model, status, output = []) {
|
||||
return {
|
||||
id,
|
||||
object: 'response',
|
||||
created_at: Math.floor(Date.now() / 1000),
|
||||
status,
|
||||
model,
|
||||
output,
|
||||
error: null,
|
||||
incomplete_details: null,
|
||||
usage: status === 'completed'
|
||||
? {
|
||||
input_tokens: 1,
|
||||
input_tokens_details: { cached_tokens: 0 },
|
||||
output_tokens: 1,
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
total_tokens: 2,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function respondOpenAiResponses(response, body) {
|
||||
const id = `resp_local_${Date.now()}`;
|
||||
const item = {
|
||||
id: `msg_local_${Date.now()}`,
|
||||
type: 'message',
|
||||
status: 'completed',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'output_text', text: 'PI_PROVIDER_OK', annotations: [] }],
|
||||
};
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
writeSse(response, { type: 'response.created', response: responseEnvelope(id, body.model, 'in_progress') });
|
||||
writeSse(response, { type: 'response.output_item.added', output_index: 0, item: { ...item, status: 'in_progress', content: [] } });
|
||||
writeSse(response, {
|
||||
type: 'response.output_text.delta',
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
delta: 'PI_PROVIDER_OK',
|
||||
});
|
||||
finishResponse(response, [
|
||||
{ type: 'response.output_item.done', output_index: 0, item },
|
||||
{ type: 'response.completed', response: responseEnvelope(id, body.model, 'completed', [item]) },
|
||||
]);
|
||||
}
|
||||
|
||||
function respondAnthropic(response, body) {
|
||||
const id = `msg_local_${Date.now()}`;
|
||||
response.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
writeSse(response, {
|
||||
type: 'message_start',
|
||||
message: {
|
||||
id,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: body.model,
|
||||
content: [],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 1, output_tokens: 0 },
|
||||
},
|
||||
});
|
||||
writeSse(response, { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } });
|
||||
writeSse(response, { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'PI_PROVIDER_OK' } });
|
||||
finishResponse(response, [
|
||||
{ type: 'content_block_stop', index: 0 },
|
||||
{
|
||||
type: 'message_delta',
|
||||
delta: { stop_reason: 'end_turn', stop_sequence: null },
|
||||
usage: { output_tokens: 1 },
|
||||
},
|
||||
{ type: 'message_stop' },
|
||||
]);
|
||||
}
|
||||
|
||||
function protocolFromPath(pathname) {
|
||||
return MAKELore_PROVIDER_PROTOCOLS.find((protocol) => pathname.startsWith(`/${protocol}/`));
|
||||
}
|
||||
|
||||
async function startContractServer(requests) {
|
||||
const server = createServer(async (request, response) => {
|
||||
response.once('error', () => undefined);
|
||||
try {
|
||||
const body = await readRequestBody(request);
|
||||
const protocol = protocolFromPath(request.url ?? '');
|
||||
if (!protocol) {
|
||||
response.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
requests.push({
|
||||
protocol,
|
||||
method: request.method,
|
||||
path: request.url,
|
||||
headers: request.headers,
|
||||
model: body.model,
|
||||
hasImage: JSON.stringify(body).includes(IMAGE_BASE64),
|
||||
});
|
||||
if (protocol === 'anthropic-messages') respondAnthropic(response, body);
|
||||
else if (protocol === 'openai-responses') respondOpenAiResponses(response, body);
|
||||
else respondOpenAiCompletions(response, body);
|
||||
} catch (error) {
|
||||
if (!response.headersSent) response.writeHead(400, { 'content-type': 'application/json' });
|
||||
if (!response.destroyed && !response.writableEnded) {
|
||||
response.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
|
||||
}
|
||||
}
|
||||
});
|
||||
await new Promise((resolvePromise, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', resolvePromise);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') throw new Error('Local provider server did not expose a TCP port');
|
||||
return {
|
||||
port: address.port,
|
||||
close: () => new Promise((resolvePromise, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolvePromise()));
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function expectedPath(protocol) {
|
||||
const endpoint = protocol === 'openai-responses'
|
||||
? 'responses'
|
||||
: protocol === 'anthropic-messages'
|
||||
? 'messages'
|
||||
: 'chat/completions';
|
||||
return `/${protocol}/v1/${endpoint}`;
|
||||
}
|
||||
|
||||
function validateCapturedRequests(protocol, modelId, requests) {
|
||||
const matching = requests.filter((request) => request.protocol === protocol);
|
||||
const authHeader = protocol === 'anthropic-messages' ? 'x-api-key' : 'authorization';
|
||||
const expectedAuth = protocol === 'anthropic-messages' ? LOCAL_API_KEY : `Bearer ${LOCAL_API_KEY}`;
|
||||
const problems = [];
|
||||
if (matching.length !== 4) problems.push(`expected 4 requests, got ${matching.length}`);
|
||||
if (matching.some((request) => request.method !== 'POST')) problems.push('non-POST request');
|
||||
if (matching.some((request) => request.path !== expectedPath(protocol))) problems.push('unexpected endpoint path');
|
||||
if (matching.some((request) => request.headers[authHeader] !== expectedAuth)) problems.push('credential header mismatch');
|
||||
if (matching.some((request) => request.headers[CUSTOM_HEADER] !== protocol)) problems.push('custom header mismatch');
|
||||
if (matching.some((request) => request.model !== modelId)) problems.push('model mismatch');
|
||||
if (matching.filter((request) => request.hasImage).length < 2) problems.push('image payload missing from initial turns');
|
||||
if (problems.length > 0) throw new Error(`${protocol} local contract failed: ${problems.join('; ')}`);
|
||||
return {
|
||||
requestCount: matching.length,
|
||||
method: 'POST',
|
||||
path: expectedPath(protocol),
|
||||
credentialHeader: authHeader,
|
||||
customHeader: CUSTOM_HEADER,
|
||||
modelId,
|
||||
imageRequests: matching.filter((request) => request.hasImage).length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runLocalProviderContracts(options, projectRoot = process.cwd()) {
|
||||
const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-provider-contracts-'));
|
||||
const requests = [];
|
||||
const previousApiKey = process.env[LOCAL_API_KEY_ENV];
|
||||
const server = await startContractServer(requests);
|
||||
process.env[LOCAL_API_KEY_ENV] = LOCAL_API_KEY;
|
||||
try {
|
||||
const sourceRuntime = await resolvePiRuntime(projectRoot);
|
||||
const identity = await validatePiIdentity(sourceRuntime);
|
||||
const runtime = options.electronExecutablePath
|
||||
? {
|
||||
...sourceRuntime,
|
||||
electronExecutable: options.electronExecutablePath,
|
||||
cliPath: options.cliPath,
|
||||
}
|
||||
: sourceRuntime;
|
||||
const imagePath = join(scratchRoot, 'pixel.png');
|
||||
await writeFile(imagePath, Buffer.from(IMAGE_BASE64, 'base64'));
|
||||
const protocols = [];
|
||||
for (const protocol of MAKELore_PROVIDER_PROTOCOLS) {
|
||||
const protocolRoot = join(scratchRoot, protocol);
|
||||
await mkdir(protocolRoot);
|
||||
const fixturePath = join(protocolRoot, 'fixture.json');
|
||||
const modelId = `makelore-local-${protocol}`;
|
||||
const baseUrl = protocol === 'anthropic-messages'
|
||||
? `http://127.0.0.1:${server.port}/${protocol}`
|
||||
: `http://127.0.0.1:${server.port}/${protocol}/v1`;
|
||||
await writeFile(fixturePath, `${JSON.stringify({
|
||||
id: `makelore-local-${protocol}`,
|
||||
apiProtocol: protocol,
|
||||
baseUrl,
|
||||
apiKeyEnv: LOCAL_API_KEY_ENV,
|
||||
headers: { [CUSTOM_HEADER]: protocol },
|
||||
model: { id: modelId, input: ['text', 'image'] },
|
||||
}, null, 2)}\n`);
|
||||
const qualification = await runProviderQualification(
|
||||
runtime,
|
||||
protocolRoot,
|
||||
fixturePath,
|
||||
imagePath,
|
||||
options.timeoutMs,
|
||||
);
|
||||
protocols.push({
|
||||
protocol,
|
||||
qualification,
|
||||
requestContract: validateCapturedRequests(protocol, modelId, requests),
|
||||
});
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
identity,
|
||||
runtime: {
|
||||
artifact: options.artifactLabel ?? 'workspace-install',
|
||||
executable: runtime.electronExecutable,
|
||||
cliPath: runtime.cliPath,
|
||||
},
|
||||
scope: {
|
||||
network: '127.0.0.1 only',
|
||||
realProvider: false,
|
||||
limitation: 'Validates Pi HTTP/SSE request contracts and worker behavior; does not qualify a real provider account.',
|
||||
},
|
||||
protocols,
|
||||
result: 'pass',
|
||||
};
|
||||
if (options.reportPath) {
|
||||
await mkdir(dirname(options.reportPath), { recursive: true });
|
||||
await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
return report;
|
||||
} finally {
|
||||
if (previousApiKey === undefined) delete process.env[LOCAL_API_KEY_ENV];
|
||||
else process.env[LOCAL_API_KEY_ENV] = previousApiKey;
|
||||
await server.close();
|
||||
await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseProbeArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
process.stdout.write('Usage: node scripts/probe-pi-provider-contracts.mjs [probe options]\n');
|
||||
return;
|
||||
}
|
||||
if (options.stage || options.keepStage || options.providerFixturePath || options.imagePath) {
|
||||
throw new Error('Local provider contract probe owns its fixtures and does not accept staging/provider fixture flags');
|
||||
}
|
||||
const report = await runLocalProviderContracts(options);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
const isMain = process.argv[1]
|
||||
&& pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
|
||||
if (isMain) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`${error.stack ?? error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user