makelore/scripts/probe-pi-provider-contracts.mjs

import { createServer } from 'node:http';
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import {
  MAKELore_PROVIDER_PROTOCOLS,
  parseProbeArgs,
  resolvePiRuntime,
  runProviderQualification,
  validatePiIdentity,
} from './probe-pi-runtime.mjs';

const LOCAL_API_KEY_ENV = 'PI_PROBE_LOCAL_CONTRACT_KEY';
const LOCAL_API_KEY = 'pi-local-contract-only';
const LOCAL_HEADER_ENV = 'PI_PROBE_LOCAL_CONTRACT_HEADER';
const CUSTOM_HEADER = 'x-makelore-pi-probe';
const IMAGE_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Z4N8AAAAASUVORK5CYII=';

function readRequestBody(request) {
  return new Promise((resolvePromise, reject) => {
    const chunks = [];
    request.on('data', (chunk) => chunks.push(chunk));
    request.once('end', () => {
      try {
        resolvePromise(JSON.parse(Buffer.concat(chunks).toString('utf8')));
      } catch (error) {
        reject(error);
      }
    });
    request.once('error', reject);
  });
}

function writeSse(response, event) {
  if (response.destroyed || response.writableEnded) return;
  response.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
}

function finishResponse(response, events) {
  setTimeout(() => {
    if (response.destroyed || response.writableEnded) return;
    for (const event of events) writeSse(response, event);
    response.end();
  }, 250);
}

function respondOpenAiCompletions(response, body) {
  const id = `chatcmpl-local-${Date.now()}`;
  response.writeHead(200, { 'content-type': 'text/event-stream' });
  response.write(`data: ${JSON.stringify({
    id,
    object: 'chat.completion.chunk',
    created: Math.floor(Date.now() / 1000),
    model: body.model,
    choices: [{ index: 0, delta: { role: 'assistant', content: 'PI_PROVIDER_OK' }, finish_reason: null }],
  })}\n\n`);
  setTimeout(() => {
    if (response.destroyed || response.writableEnded) return;
    response.write(`data: ${JSON.stringify({
      id,
      object: 'chat.completion.chunk',
      created: Math.floor(Date.now() / 1000),
      model: body.model,
      choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
      usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
    })}\n\n`);
    response.end('data: [DONE]\n\n');
  }, 250);
}

function responseEnvelope(id, model, status, output = []) {
  return {
    id,
    object: 'response',
    created_at: Math.floor(Date.now() / 1000),
    status,
    model,
    output,
    error: null,
    incomplete_details: null,
    usage: status === 'completed'
      ? {
        input_tokens: 1,
        input_tokens_details: { cached_tokens: 0 },
        output_tokens: 1,
        output_tokens_details: { reasoning_tokens: 0 },
        total_tokens: 2,
      }
      : null,
  };
}

function respondOpenAiResponses(response, body) {
  const id = `resp_local_${Date.now()}`;
  const item = {
    id: `msg_local_${Date.now()}`,
    type: 'message',
    status: 'completed',
    role: 'assistant',
    content: [{ type: 'output_text', text: 'PI_PROVIDER_OK', annotations: [] }],
  };
  response.writeHead(200, { 'content-type': 'text/event-stream' });
  writeSse(response, { type: 'response.created', response: responseEnvelope(id, body.model, 'in_progress') });
  writeSse(response, { type: 'response.output_item.added', output_index: 0, item: { ...item, status: 'in_progress', content: [] } });
  writeSse(response, {
    type: 'response.output_text.delta',
    item_id: item.id,
    output_index: 0,
    content_index: 0,
    delta: 'PI_PROVIDER_OK',
  });
  finishResponse(response, [
    { type: 'response.output_item.done', output_index: 0, item },
    { type: 'response.completed', response: responseEnvelope(id, body.model, 'completed', [item]) },
  ]);
}

function respondAnthropic(response, body) {
  const id = `msg_local_${Date.now()}`;
  response.writeHead(200, { 'content-type': 'text/event-stream' });
  writeSse(response, {
    type: 'message_start',
    message: {
      id,
      type: 'message',
      role: 'assistant',
      model: body.model,
      content: [],
      stop_reason: null,
      stop_sequence: null,
      usage: { input_tokens: 1, output_tokens: 0 },
    },
  });
  writeSse(response, { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } });
  writeSse(response, { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'PI_PROVIDER_OK' } });
  finishResponse(response, [
    { type: 'content_block_stop', index: 0 },
    {
      type: 'message_delta',
      delta: { stop_reason: 'end_turn', stop_sequence: null },
      usage: { output_tokens: 1 },
    },
    { type: 'message_stop' },
  ]);
}

function protocolFromPath(pathname) {
  return MAKELore_PROVIDER_PROTOCOLS.find((protocol) => pathname.startsWith(`/${protocol}/`));
}

async function startContractServer(requests) {
  const server = createServer(async (request, response) => {
    response.once('error', () => undefined);
    try {
      const body = await readRequestBody(request);
      const protocol = protocolFromPath(request.url ?? '');
      if (!protocol) {
        response.writeHead(404).end();
        return;
      }
      requests.push({
        protocol,
        method: request.method,
        path: request.url,
        headers: request.headers,
        model: body.model,
        hasImage: JSON.stringify(body).includes(IMAGE_BASE64),
      });
      if (protocol === 'anthropic-messages') respondAnthropic(response, body);
      else if (protocol === 'openai-responses') respondOpenAiResponses(response, body);
      else respondOpenAiCompletions(response, body);
    } catch (error) {
      if (!response.headersSent) response.writeHead(400, { 'content-type': 'application/json' });
      if (!response.destroyed && !response.writableEnded) {
        response.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
      }
    }
  });
  await new Promise((resolvePromise, reject) => {
    server.once('error', reject);
    server.listen(0, '127.0.0.1', resolvePromise);
  });
  const address = server.address();
  if (!address || typeof address === 'string') throw new Error('Local provider server did not expose a TCP port');
  return {
    port: address.port,
    close: () => new Promise((resolvePromise, reject) => {
      server.close((error) => (error ? reject(error) : resolvePromise()));
    }),
  };
}

function expectedPath(protocol) {
  const endpoint = protocol === 'openai-responses'
    ? 'responses'
    : protocol === 'anthropic-messages'
      ? 'messages'
      : 'chat/completions';
  return `/${protocol}/v1/${endpoint}`;
}

function validateCapturedRequests(protocol, modelId, requests) {
  const matching = requests.filter((request) => request.protocol === protocol);
  const authHeader = protocol === 'anthropic-messages' ? 'x-api-key' : 'authorization';
  const expectedAuth = protocol === 'anthropic-messages' ? LOCAL_API_KEY : `Bearer ${LOCAL_API_KEY}`;
  const problems = [];
  if (matching.length !== 6) problems.push(`expected 6 requests, got ${matching.length}`);
  if (matching.some((request) => request.method !== 'POST')) problems.push('non-POST request');
  if (matching.some((request) => request.path !== expectedPath(protocol))) problems.push('unexpected endpoint path');
  if (matching.some((request) => request.headers[authHeader] !== expectedAuth)) problems.push('credential header mismatch');
  if (matching.some((request) => request.headers[CUSTOM_HEADER] !== protocol)) problems.push('custom header mismatch');
  if (matching.some((request) => request.model !== modelId)) problems.push('model mismatch');
  if (matching.filter((request) => request.hasImage).length < 2) problems.push('image payload missing from initial turns');
  if (problems.length > 0) throw new Error(`${protocol} local contract failed: ${problems.join('; ')}`);
  return {
    requestCount: matching.length,
    method: 'POST',
    path: expectedPath(protocol),
    credentialHeader: authHeader,
    customHeader: CUSTOM_HEADER,
    modelId,
    imageRequests: matching.filter((request) => request.hasImage).length,
  };
}

export async function runLocalProviderContracts(options, projectRoot = process.cwd()) {
  const scratchRoot = await mkdtemp(join(tmpdir(), 'makelore-pi-provider-contracts-'));
  const requests = [];
  const previousApiKey = process.env[LOCAL_API_KEY_ENV];
  const previousHeader = process.env[LOCAL_HEADER_ENV];
  const server = await startContractServer(requests);
  process.env[LOCAL_API_KEY_ENV] = LOCAL_API_KEY;
  try {
    const sourceRuntime = await resolvePiRuntime(projectRoot);
    const identity = await validatePiIdentity(sourceRuntime);
    const runtime = options.electronExecutablePath
      ? {
        ...sourceRuntime,
        electronExecutable: options.electronExecutablePath,
        cliPath: options.cliPath,
      }
      : sourceRuntime;
    const imagePath = join(scratchRoot, 'pixel.png');
    await writeFile(imagePath, Buffer.from(IMAGE_BASE64, 'base64'));
    const protocols = [];
    for (const protocol of MAKELore_PROVIDER_PROTOCOLS) {
      process.env[LOCAL_HEADER_ENV] = protocol;
      const protocolRoot = join(scratchRoot, protocol);
      await mkdir(protocolRoot);
      const fixturePath = join(protocolRoot, 'fixture.json');
      const modelId = `makelore-local-${protocol}`;
      const baseUrl = protocol === 'anthropic-messages'
        ? `http://127.0.0.1:${server.port}/${protocol}`
        : `http://127.0.0.1:${server.port}/${protocol}/v1`;
      await writeFile(fixturePath, `${JSON.stringify({
        id: `makelore-local-${protocol}`,
        apiProtocol: protocol,
        baseUrl,
        apiKeyEnv: LOCAL_API_KEY_ENV,
        headers: { [CUSTOM_HEADER]: `$${LOCAL_HEADER_ENV}` },
        model: { id: modelId, input: ['text', 'image'] },
      }, null, 2)}\n`);
      const qualification = await runProviderQualification(
        runtime,
        protocolRoot,
        fixturePath,
        imagePath,
        options.timeoutMs,
      );
      protocols.push({
        protocol,
        qualification,
        requestContract: validateCapturedRequests(protocol, modelId, requests),
      });
    }
    const report = {
      schemaVersion: 1,
      generatedAt: new Date().toISOString(),
      identity,
      runtime: {
        artifact: options.artifactLabel ?? 'workspace-install',
        executable: runtime.electronExecutable,
        cliPath: runtime.cliPath,
      },
      scope: {
        network: '127.0.0.1 only',
        realProvider: false,
        realTurnVerified: false,
        realProviderDecision: 'explicitly-waived-accepted-risk',
        limitation: 'Validates Pi HTTP/SSE request contracts and worker behavior; does not qualify a real provider account.',
      },
      protocols,
      result: 'pass',
    };
    if (options.reportPath) {
      await mkdir(dirname(options.reportPath), { recursive: true });
      await writeFile(options.reportPath, `${JSON.stringify(report, null, 2)}\n`);
    }
    return report;
  } finally {
    if (previousApiKey === undefined) delete process.env[LOCAL_API_KEY_ENV];
    else process.env[LOCAL_API_KEY_ENV] = previousApiKey;
    if (previousHeader === undefined) delete process.env[LOCAL_HEADER_ENV];
    else process.env[LOCAL_HEADER_ENV] = previousHeader;
    await server.close();
    await rm(scratchRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
  }
}

async function main() {
  const options = parseProbeArgs(process.argv.slice(2));
  if (options.help) {
    process.stdout.write('Usage: node scripts/probe-pi-provider-contracts.mjs [probe options]\n');
    return;
  }
  if (options.stage || options.keepStage || options.providerFixturePath || options.imagePath) {
    throw new Error('Local provider contract probe owns its fixtures and does not accept staging/provider fixture flags');
  }
  const report = await runLocalProviderContracts(options);
  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
}

const isMain = process.argv[1]
  && pathToFileURL(resolve(process.argv[1])).href === import.meta.url;
if (isMain) {
  main().catch((error) => {
    process.stderr.write(`${error.stack ?? error.message}\n`);
    process.exitCode = 1;
  });
}