openmaic/OpenMAIC/lib/server/public-url-fetch.ts

import { promises as dns, type LookupAddress } from 'node:dns';
import { isIP, type LookupFunction } from 'node:net';
import {
  Agent,
  fetch as undiciFetch,
  type BodyInit as UndiciBodyInit,
  type Response as UndiciResponse,
} from 'undici';
import { isGlobalUnicastIP } from '@/lib/server/ssrf-guard';

export const PUBLIC_MEDIA_MAX_RESPONSE_BYTES = 25 * 1024 * 1024;
export const PUBLIC_URL_MAX_REQUEST_BYTES = 50 * 1024 * 1024;

export type PublicUrlFetchErrorCode =
  | 'INVALID_URL'
  | 'BLOCKED_TARGET'
  | 'DNS_FAILURE'
  | 'REQUEST_TOO_LARGE'
  | 'UPSTREAM_FAILURE';

/** An intentionally non-sensitive error suitable for mapping at an API boundary. */
export class PublicUrlFetchError extends Error {
  constructor(
    readonly code: PublicUrlFetchErrorCode,
    message: string,
  ) {
    super(message);
    this.name = 'PublicUrlFetchError';
  }
}

interface PinnedAddress {
  address: string;
  family: 4 | 6;
}

export interface ResolvedPublicTarget {
  url: URL;
  hostname: string;
  addresses: readonly PinnedAddress[];
}

export interface PinnedPublicResponse {
  response: UndiciResponse;
  dispose: () => Promise<void>;
}

export type PinnedPublicRequestBody = string | ArrayBuffer | NodeJS.ArrayBufferView | Blob;

export interface FetchPinnedPublicUrlOptions {
  signal?: AbortSignal;
  maxResponseBytes?: number;
  maxRequestBytes?: number;
  headersTimeoutMs?: number;
  bodyTimeoutMs?: number;
  headers?: Readonly<Record<string, string>>;
  method?: 'GET' | 'PUT';
  body?: PinnedPublicRequestBody;
}

function normalizeHostname(value: string): string {
  let hostname = value.trim().toLowerCase();
  if (hostname.startsWith('[') && hostname.endsWith(']')) {
    hostname = hostname.slice(1, -1);
  }
  return hostname.replace(/\.+$/, '');
}

function blockedTarget(): never {
  throw new PublicUrlFetchError(
    'BLOCKED_TARGET',
    'URL target is not an allowed public network address',
  );
}

/**
 * Resolve a URL exactly once and retain only the verified address set.
 *
 * This public-proxy policy intentionally ignores ALLOW_LOCAL_NETWORKS. That
 * switch is useful for explicitly configured self-hosted providers, but must
 * never turn a user-supplied media URL into a general-purpose intranet proxy.
 */
export async function resolvePublicTarget(rawUrl: string): Promise<ResolvedPublicTarget> {
  let url: URL;
  try {
    url = new URL(rawUrl);
  } catch {
    throw new PublicUrlFetchError('INVALID_URL', 'Invalid public URL');
  }

  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
    throw new PublicUrlFetchError('INVALID_URL', 'Only HTTP(S) public URLs are allowed');
  }
  if (url.username || url.password) {
    throw new PublicUrlFetchError('INVALID_URL', 'Credentialed URLs are not allowed');
  }

  const hostname = normalizeHostname(url.hostname);
  if (!hostname) {
    throw new PublicUrlFetchError('INVALID_URL', 'Invalid public URL hostname');
  }

  const literalFamily = isIP(hostname);
  if (literalFamily) {
    if (!isGlobalUnicastIP(hostname)) blockedTarget();
    return {
      url,
      hostname,
      addresses: [{ address: hostname, family: literalFamily as 4 | 6 }],
    };
  }

  let answers: LookupAddress[];
  try {
    answers = await dns.lookup(hostname, { all: true, verbatim: true });
  } catch {
    throw new PublicUrlFetchError('DNS_FAILURE', 'Unable to resolve a public URL target');
  }

  if (answers.length === 0) {
    throw new PublicUrlFetchError('DNS_FAILURE', 'Unable to resolve a public URL target');
  }

  const addresses: PinnedAddress[] = [];
  const seen = new Set<string>();
  for (const answer of answers) {
    const family = isIP(answer.address);
    if ((family !== 4 && family !== 6) || !isGlobalUnicastIP(answer.address)) {
      blockedTarget();
    }
    const key = `${family}:${answer.address.toLowerCase()}`;
    if (!seen.has(key)) {
      seen.add(key);
      addresses.push({ address: answer.address, family });
    }
  }

  if (addresses.length === 0) blockedTarget();
  return { url, hostname, addresses };
}

function lookupError(code: string): NodeJS.ErrnoException {
  return Object.assign(new Error('Pinned public address is unavailable'), { code });
}

function requestBodyByteLength(body: PinnedPublicRequestBody): number {
  if (typeof body === 'string') return Buffer.byteLength(body);
  if (body instanceof ArrayBuffer) return body.byteLength;
  if (ArrayBuffer.isView(body)) return body.byteLength;
  return body.size;
}

function positiveLimit(value: number | undefined, fallback: number): number {
  if (value === undefined) return fallback;
  if (Number.isSafeInteger(value) && value > 0) return value;
  throw new PublicUrlFetchError('INVALID_URL', 'Fetch limits must be positive integers');
}

/**
 * A connector lookup that never calls DNS. It may only return addresses from
 * the set resolved and checked by `resolvePublicTarget`.
 */
export function createPinnedLookup(target: ResolvedPublicTarget): LookupFunction {
  const pinned = target.addresses.map((entry) => ({ ...entry }));

  return (requestedHostname, options, callback) => {
    if (normalizeHostname(requestedHostname) !== target.hostname) {
      callback(lookupError('ENOTFOUND'), options.all ? [] : '', 0);
      return;
    }

    const requestedFamily = options.family === 4 || options.family === 6 ? options.family : 0;
    const candidates = requestedFamily
      ? pinned.filter(({ family }) => family === requestedFamily)
      : pinned;
    if (candidates.length === 0) {
      callback(lookupError('EAFNOSUPPORT'), options.all ? [] : '', 0);
      return;
    }

    if (options.all) {
      callback(null, candidates);
      return;
    }
    callback(null, candidates[0].address, candidates[0].family);
  };
}

/**
 * Perform one redirect-disabled fetch through a DNS-pinned dispatcher.
 * Redirect handling belongs to the caller so every hop receives a fresh
 * resolution, global-unicast check and independently pinned connection.
 */
export async function fetchPinnedPublicUrl(
  rawUrl: string,
  options: FetchPinnedPublicUrlOptions = {},
): Promise<PinnedPublicResponse> {
  const method: string = options.method ?? 'GET';
  if (method !== 'GET' && method !== 'PUT') {
    throw new PublicUrlFetchError('INVALID_URL', 'Unsupported public URL request method');
  }
  if (method === 'GET' && options.body !== undefined) {
    throw new PublicUrlFetchError('INVALID_URL', 'GET requests cannot include a request body');
  }

  const managedHeaders = new Set(['host', 'content-length', 'transfer-encoding']);
  if (Object.keys(options.headers ?? {}).some((name) => managedHeaders.has(name.toLowerCase()))) {
    throw new PublicUrlFetchError('INVALID_URL', 'Transport-managed headers cannot be overridden');
  }

  if (options.body !== undefined) {
    const maxRequestBytes = positiveLimit(options.maxRequestBytes, PUBLIC_URL_MAX_REQUEST_BYTES);
    if (requestBodyByteLength(options.body) > maxRequestBytes) {
      throw new PublicUrlFetchError('REQUEST_TOO_LARGE', 'Public URL request body is too large');
    }
  }

  const target = await resolvePublicTarget(rawUrl);
  const dispatcher = new Agent({
    connect: {
      lookup: createPinnedLookup(target),
      autoSelectFamily: true,
      autoSelectFamilyAttemptTimeout: 250,
    },
    connectTimeout: 10_000,
    headersTimeout: positiveLimit(options.headersTimeoutMs, 15_000),
    bodyTimeout: positiveLimit(options.bodyTimeoutMs, 30_000),
    maxResponseSize: positiveLimit(options.maxResponseBytes, PUBLIC_MEDIA_MAX_RESPONSE_BYTES),
  });

  let response: UndiciResponse;
  try {
    response = await undiciFetch(target.url, {
      method,
      redirect: 'manual',
      dispatcher,
      signal: options.signal,
      headers: options.headers ? { ...options.headers } : undefined,
      body: options.body as UndiciBodyInit | undefined,
    });
  } catch {
    await dispatcher.close().catch(() => {});
    throw new PublicUrlFetchError('UPSTREAM_FAILURE', 'Unable to fetch upstream media');
  }

  let disposed = false;
  return {
    response,
    dispose: async () => {
      if (disposed) return;
      disposed = true;
      if (response.body && !response.bodyUsed) {
        await response.body.cancel().catch(() => {});
      }
      await dispatcher.close().catch(() => {});
    },
  };
}