feat: add official game audio generation client

This commit is contained in:
2026-09-21 12:17:32 +08:00
parent d222d17500
commit 09e0ce5cf3
28 changed files with 1835 additions and 31 deletions

View File

@@ -0,0 +1,233 @@
import type { CapabilityBillingReceiptV1 } from '../../shared/data-service';
import { WORKS_SQUARE_CONFIG } from '../api/works-config';
import { proxyAwareFetch } from '../utils/proxy-fetch';
import { getValidWorksSquareAccessToken, getWorksSquareAccountBinding } from './works-square-session';
const ROOT = '/api/plugins/v1/hosted/game-audio/generations';
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u;
const DECIMAL = /^(?:0|[1-9]\d*)\.\d{2}$/u;
const AUDIO_EXTENSIONS: Readonly<Record<string, string>> = {
'audio/mpeg': 'mp3', 'audio/wav': 'wav', 'audio/x-wav': 'wav',
'audio/ogg': 'ogg', 'audio/flac': 'flac', 'audio/mp4': 'm4a',
};
export type AudioBilling = Extract<CapabilityBillingReceiptV1, { mode: 'platform_metered' }>;
export interface AudioGeneration {
schema_version: 1;
plugin_id: 'makelore.game-audio';
execution_id: string;
release_id: string;
project_id: string;
logical_operation_id: string;
kind: 'music' | 'sound';
provider_status: 'accepted' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'submission_unknown';
output_ready: boolean;
outputs: { index: number; name: string }[];
progress_percent: number | null;
poll_interval_seconds: number;
error_code: string | null;
billing: AudioBilling;
}
interface AudioIdentity {
release_id: string;
release_admission_id: string;
project_id: string;
logical_operation_id: string;
pricing_version_id: string;
prompt: string;
confirmed: true;
}
export type AudioCommand = AudioIdentity & ({
kind: 'music'; mode: 'full' | 'demo';
reference_files?: { name: string; mime_type: string; data_base64: string }[];
} | {
kind: 'sound'; mode: 'single' | 'pack' | 'variants';
duration: number; count: number; loop: boolean;
});
export class GameAudioClientError extends Error {
constructor(readonly code: string, readonly status: number, message: string) {
super(message);
this.name = 'GameAudioClientError';
}
}
export function audioRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object' && !Array.isArray(value);
}
function invalid(): never {
throw new GameAudioClientError('plugin_backend_invalid', 502, 'Audio response is invalid');
}
function bounded(value: unknown, max: number): value is string {
return typeof value === 'string' && value.length > 0 && value.length <= max;
}
function integer(value: unknown, min: number, max: number): value is number {
return typeof value === 'number' && Number.isSafeInteger(value) && value >= min && value <= max;
}
export function parseAudioGeneration(value: unknown): AudioGeneration {
if (!audioRecord(value) || value.schema_version !== 1 || value.plugin_id !== 'makelore.game-audio'
|| !bounded(value.execution_id, 36) || !UUID.test(value.execution_id)
|| !bounded(value.release_id, 36) || !UUID.test(value.release_id)
|| !bounded(value.project_id, 36) || !UUID.test(value.project_id)
|| !bounded(value.logical_operation_id, 128)
|| !['music', 'sound'].includes(String(value.kind))
|| !['accepted', 'running', 'succeeded', 'failed', 'cancelled', 'submission_unknown'].includes(String(value.provider_status))
|| typeof value.output_ready !== 'boolean'
|| !Array.isArray(value.outputs) || value.outputs.length > 10
|| !value.outputs.every((item, index) => audioRecord(item)
&& Object.keys(item).every((key) => ['index', 'name'].includes(key))
&& item.index === index && bounded(item.name, 160))
|| value.output_ready !== (value.outputs.length > 0)
|| (value.output_ready && value.provider_status !== 'succeeded')
|| !integer(value.poll_interval_seconds, 1, 300)
|| (value.progress_percent !== null && !integer(value.progress_percent, 0, 100))
|| (value.error_code !== null && !bounded(value.error_code, 100))) invalid();
const bill = value.billing;
if (!audioRecord(bill) || bill.mode !== 'platform_metered'
|| !['reserved', 'dispatched', 'settled', 'released', 'expired', 'pending_review', 'refunded'].includes(String(bill.status))
|| typeof bill.reserved_points !== 'string' || !DECIMAL.test(bill.reserved_points)
|| (bill.actual_points != null && (typeof bill.actual_points !== 'string' || !DECIMAL.test(bill.actual_points)))
|| !integer(bill.usage_amount, 0, 200)
|| bill.unit !== (value.kind === 'sound' ? 'half_second' : 'generation')
|| (['settled', 'refunded'].includes(String(bill.status)) && bill.actual_points == null)
|| (['accepted', 'running'].includes(String(value.provider_status)) && !['reserved', 'dispatched'].includes(String(bill.status)))
|| (value.provider_status === 'succeeded' && !['settled', 'pending_review', 'refunded'].includes(String(bill.status)))
|| (value.provider_status === 'submission_unknown' && bill.status !== 'pending_review')
|| (['failed', 'cancelled'].includes(String(value.provider_status)) && !['released', 'expired', 'refunded', 'pending_review'].includes(String(bill.status)))) invalid();
const fields = ['schema_version', 'plugin_id', 'execution_id', 'release_id', 'project_id',
'logical_operation_id', 'kind', 'provider_status', 'output_ready', 'outputs',
'progress_percent', 'poll_interval_seconds', 'error_code', 'billing'];
if (Object.keys(value).some((key) => !fields.includes(key))
|| Object.keys(bill).some((key) => !['mode', 'status', 'reserved_points', 'actual_points', 'usage_amount', 'unit'].includes(key))) invalid();
// Hosted JSON uses null; shared capability receipts use an absent optional amount.
return { ...value, billing: Object.fromEntries(Object.entries(bill).filter(([, field]) => field !== null)) } as unknown as AudioGeneration;
}
async function bytes(response: Response, limit: number): Promise<Uint8Array> {
if (Number(response.headers.get('content-length')) > limit) {
await response.body?.cancel();
invalid();
}
const reader = response.body?.getReader();
if (!reader) invalid();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const next = await reader.read();
if (next.done) break;
total += next.value.byteLength;
if (total > limit) invalid();
chunks.push(next.value);
}
} finally {
await reader.cancel().catch(() => undefined);
reader.releaseLock();
}
const result = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) { result.set(chunk, offset); offset += chunk.byteLength; }
return result;
}
export interface GameAudioClientOptions {
fetchImpl?: typeof fetch;
getAccessToken?: typeof getValidWorksSquareAccessToken;
getAccountBinding?: typeof getWorksSquareAccountBinding;
apiBaseUrl?: string;
}
export class GameAudioClient {
constructor(private readonly options: GameAudioClientOptions = {}) {}
async generate(command: AudioCommand, account: string): Promise<AudioGeneration> {
const result = await this.json(ROOT, account, 'POST', command);
if (result.project_id !== command.project_id || result.release_id !== command.release_id
|| result.logical_operation_id !== command.logical_operation_id || result.kind !== command.kind) invalid();
return result;
}
async get(id: string, account: string): Promise<AudioGeneration> {
if (!UUID.test(id)) invalid();
const result = await this.json(ROOT + '/' + id, account);
if (result.execution_id !== id) invalid();
return result;
}
async byOperation(id: string, account: string): Promise<AudioGeneration> {
if (!bounded(id, 128)) invalid();
const result = await this.json(ROOT + '/by-operation/' + encodeURIComponent(id), account);
if (result.logical_operation_id !== id) invalid();
return result;
}
async cancel(id: string, account: string): Promise<AudioGeneration> {
if (!UUID.test(id)) invalid();
const result = await this.json(ROOT + '/' + id + '/cancel', account, 'POST');
if (result.execution_id !== id) invalid();
return result;
}
async download(id: string, index: number, account: string) {
if (!UUID.test(id) || !integer(index, 0, 9)) invalid();
return await this.request(ROOT + '/' + id + '/content?output_index=' + index, account, 'GET', undefined, async (response) => {
const mime = response.headers.get('content-type')?.split(';', 1)[0]?.trim().toLowerCase() ?? '';
const extension = AUDIO_EXTENSIONS[mime];
if (!extension) {
await response.body?.cancel().catch(() => undefined);
throw new GameAudioClientError('game_audio_content_invalid', 502, 'Unsupported audio media');
}
const content = await bytes(response, 32 * 1024 * 1024);
if (content.length === 0) invalid();
return { bytes: content, extension, mimeType: mime };
});
}
private async json(route: string, account: string, method = 'GET', command?: AudioCommand) {
return await this.request(route, account, method, command, async (response) => {
const content = await bytes(response, 1024 * 1024);
let value: unknown;
try { value = JSON.parse(new TextDecoder().decode(content)); } catch { invalid(); }
return parseAudioGeneration(value);
});
}
private async request<T>(
route: string, account: string, method: string, command: AudioCommand | undefined,
consume: (response: Response) => Promise<T>,
): Promise<T> {
const binding = this.options.getAccountBinding ?? getWorksSquareAccountBinding;
const original = binding();
const assertAccount = () => {
const current = binding();
if (!original || original.accountKey !== account || current?.accountKey !== account || current.epoch !== original.epoch) {
throw new GameAudioClientError('plugin_account_changed', 409, 'Original signed-in account is required');
}
};
assertAccount();
const fetchImpl = this.options.fetchImpl ?? proxyAwareFetch;
const getToken = this.options.getAccessToken ?? getValidWorksSquareAccessToken;
const body = command ? JSON.stringify(command) : undefined;
if (body && Buffer.byteLength(body) > 24 * 1024 * 1024) invalid();
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), method === 'POST' ? 90_000 : 40_000);
try {
for (let attempt = 0; attempt < 2; attempt++) {
const token = await getToken({ fetchImpl, ...(attempt ? { forceRefresh: true } : {}) });
assertAccount();
if (!token) throw new GameAudioClientError('authentication_required', 401, 'Sign-in is required');
const response = await fetchImpl((this.options.apiBaseUrl ?? WORKS_SQUARE_CONFIG.apiBaseUrl).replace(/\/$/u, '') + route, {
method, redirect: 'manual', signal: controller.signal,
headers: { Accept: 'application/json', Authorization: 'Bearer ' + token,
...(body ? { 'Content-Type': 'application/json' } : {}) },
...(body ? { body } : {}),
});
assertAccount();
if (response.status === 401 && attempt === 0) { await response.body?.cancel(); continue; }
if (!response.ok) {
let code = response.status === 429 ? 'plugin_rate_limited' : 'plugin_backend_unavailable';
try {
const error: unknown = JSON.parse(new TextDecoder().decode(await bytes(response, 1024 * 1024)));
if (response.status !== 429 && audioRecord(error) && audioRecord(error.detail)
&& bounded(error.detail.error_code, 100)) code = error.detail.error_code;
} catch { /* HTTP status still owns the failure even if the error body is invalid. */ }
throw new GameAudioClientError(code, response.status, 'Audio request failed (' + response.status + ')');
}
const value = await consume(response);
assertAccount();
return value;
}
throw new GameAudioClientError('authentication_required', 401, 'Sign-in is required');
} finally {
clearTimeout(timeout);
}
}
}

View File

@@ -0,0 +1,268 @@
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { atomicWriteJson, readJsonFile } from '../coding-projects/atomic-json';
import type { PiProjectWriteLeaseCoordinator } from '../coding-runtime/pi/write-lease';
import { audioRecord, GameAudioClientError, parseAudioGeneration,
type AudioCommand, type AudioGeneration, type GameAudioClient, type AudioBilling } from './game-audio-client';
import { getWorksSquareAccountBinding } from './works-square-session';
import { readMediaDataUrl } from '../coding-projects/game-asset-browser';
export interface AudioDeliveryContext {
accountKey: string;
conversationId: string;
runId: string;
localProjectId: string;
durableProjectId: string;
projectPath: string;
logicalOperationId: string;
}
export interface AudioDeliveredFile { index: number; path: string; bytes: number; mimeType: string }
export interface AudioDeliveryResult {
executionId: string | null;
logicalOperationId: string;
providerStatus: AudioGeneration['provider_status'];
deliveryStatus: 'waiting' | 'saving' | 'saved' | 'delivery_failed' | 'needs_reconciliation';
outputCount: number;
files: AudioDeliveredFile[];
billing: AudioBilling;
phase: 'submitted' | 'generating' | 'saving' | 'saved' | 'paused';
progressPercent: number | null;
errorCode?: string;
}
interface Receipt {
context: AudioDeliveryContext;
generation: AudioGeneration | null;
deliveryStatus: AudioDeliveryResult['deliveryStatus'];
files: AudioDeliveredFile[];
errorCode?: string;
submissionRejected?: boolean;
}
function sameIdentity(left: AudioDeliveryContext, right: AudioDeliveryContext) {
return left.accountKey === right.accountKey && left.logicalOperationId === right.logicalOperationId;
}
function validReceipt(value: unknown): value is Receipt {
if (!audioRecord(value) || !audioRecord(value.context)
|| !['accountKey', 'conversationId', 'runId', 'localProjectId', 'durableProjectId', 'projectPath', 'logicalOperationId']
.every((key) => typeof value.context[key] === 'string' && value.context[key].length > 0)
|| !path.isAbsolute(String(value.context.projectPath))
|| !['waiting', 'saving', 'saved', 'delivery_failed', 'needs_reconciliation'].includes(String(value.deliveryStatus))
|| !Array.isArray(value.files) || value.files.length > 10) return false;
try {
if (value.generation !== null) parseAudioGeneration(value.generation);
} catch { return false; }
return value.files.every((file) => audioRecord(file) && Number.isInteger(file.index)
&& typeof file.bytes === 'number' && file.bytes > 0 && typeof file.path === 'string'
&& /^assets\/generated\/game-audio\/[0-9a-f-]{36}\/(?:music|sound)-\d+\.(?:mp3|wav|ogg|flac|m4a)$/u.test(file.path));
}
/** One atomic userData file; no project file serves as the job database. */
export class GameAudioReceiptStore {
private writes: Promise<void> = Promise.resolve();
constructor(private readonly filePath: string) {}
async list(): Promise<Receipt[]> {
let value: unknown;
try { value = await readJsonFile(this.filePath); } catch (error) {
if (audioRecord(error) && error.code === 'ENOENT') return [];
throw error;
}
if (!audioRecord(value) || value.schemaVersion !== 1 || !Array.isArray(value.receipts)
|| !value.receipts.every(validReceipt)) throw new Error('Audio receipt file is invalid');
return value.receipts;
}
async put(receipt: Receipt): Promise<void> {
const operation = this.writes.then(async () => {
const rows = (await this.list()).filter((row) => !sameIdentity(row.context, receipt.context));
rows.push(receipt);
await atomicWriteJson(this.filePath, { schemaVersion: 1, receipts: rows });
});
this.writes = operation.catch(() => undefined);
await operation;
}
}
interface Options {
client: Pick<GameAudioClient, 'generate' | 'get' | 'byOperation' | 'download'>;
receipts: GameAudioReceiptStore;
leases: PiProjectWriteLeaseCoordinator;
getAccountKey?: () => string | null;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
recordTouchedPaths?: (conversationId: string, runId: string, paths: readonly string[]) => Promise<void>;
acquireBackgroundLease?: (lease: { id: string; kind: 'coding-run' }) => () => void;
}
type Progress = (value: AudioDeliveryResult) => void;
export class GameAudioDeliveryCoordinator {
private readonly flights = new Map<string, Promise<AudioDeliveryResult>>();
private readonly shutdown = new AbortController();
private recoveryTimer: ReturnType<typeof setTimeout> | undefined;
constructor(private readonly options: Options) {}
dispose(): void {
this.shutdown.abort();
clearTimeout(this.recoveryTimer);
}
private scheduleRecovery(): void {
if (this.shutdown.signal.aborted || this.recoveryTimer) return;
this.recoveryTimer = setTimeout(() => {
this.recoveryTimer = undefined;
void this.resumePending().catch(() => this.scheduleRecovery());
}, 30_000);
this.recoveryTimer.unref?.();
}
async readSavedOutput(executionId: string, index: number): Promise<{ path: string; dataUrl: string }> {
const account = this.account();
const receipt = (await this.options.receipts.list()).find((row) =>
row.context.accountKey === account && row.generation?.execution_id === executionId);
const file = receipt?.files.find((item) => item.index === index);
if (!receipt || !file) throw new GameAudioClientError('game_audio_output_not_found', 404, 'Saved audio is unavailable');
this.assertAccount(receipt.context);
const dataUrl = await readMediaDataUrl(receipt.context.projectPath, file.path, 32 * 1024 * 1024);
this.assertAccount(receipt.context);
if (!dataUrl?.startsWith('data:audio/')) throw new GameAudioClientError('game_audio_content_invalid', 422, 'Saved audio cannot be read');
return { path: file.path, dataUrl };
}
async generateAndMaterialize(context: AudioDeliveryContext, command: AudioCommand, progress?: Progress) {
return await this.singleFlight(context, async () => {
const old = (await this.options.receipts.list()).find((row) => sameIdentity(row.context, context));
if (old) return await this.advance(old, progress);
this.assertAccount(context);
const receipt: Receipt = { context: { ...context }, generation: null, deliveryStatus: 'waiting', files: [] };
// The durable identity must exist BEFORE the first possibly charged request.
await this.options.receipts.put(receipt);
try {
receipt.generation = await this.options.client.generate(command, context.accountKey);
this.checkGeneration(receipt);
await this.options.receipts.put(receipt);
} catch (error) {
if (error instanceof GameAudioClientError && (error.status === 401 || error.status === 422
|| error.code === 'plugin_pricing_changed')) {
receipt.deliveryStatus = 'needs_reconciliation';
receipt.submissionRejected = true;
receipt.errorCode = error.code;
await this.options.receipts.put(receipt);
return this.result(receipt, 'paused');
}
// Response loss is not permission to submit again.
}
return await this.advance(receipt, progress);
});
}
async resumePending(): Promise<void> {
const account = this.account();
if (!account || this.shutdown.signal.aborted) return;
const rows = await this.options.receipts.list();
await Promise.allSettled(rows.filter((row) => row.context.accountKey === account
&& !row.submissionRejected
&& row.deliveryStatus !== 'saved'
&& !['failed', 'cancelled'].includes(row.generation?.provider_status ?? ''))
.map((row) => this.singleFlight(row.context, () => this.advance(row))));
}
private async singleFlight(context: AudioDeliveryContext, run: () => Promise<AudioDeliveryResult>) {
const key = JSON.stringify([context.accountKey, context.logicalOperationId]);
const old = this.flights.get(key);
if (old) return await old;
const release = this.options.acquireBackgroundLease?.({ id: 'audio:' + key, kind: 'coding-run' });
const flight = run().finally(() => { this.flights.delete(key); release?.(); });
this.flights.set(key, flight);
return await flight;
}
private account(): string | null {
return this.options.getAccountKey ? this.options.getAccountKey() : getWorksSquareAccountBinding()?.accountKey ?? null;
}
private assertAccount(context: AudioDeliveryContext): void {
if (this.shutdown.signal.aborted || this.account() !== context.accountKey) {
throw new GameAudioClientError('plugin_account_changed', 409, 'Audio recovery waits for the original account');
}
}
private checkGeneration(receipt: Receipt): void {
const job = receipt.generation;
if (job && (job.project_id !== receipt.context.durableProjectId
|| job.logical_operation_id !== receipt.context.logicalOperationId)) {
throw new GameAudioClientError('plugin_backend_invalid', 502, 'Audio execution identity changed');
}
}
private async sleep(ms: number): Promise<void> {
if (this.options.sleep) await this.options.sleep(ms, this.shutdown.signal);
else await delay(ms, undefined, { signal: this.shutdown.signal });
}
private async advance(receipt: Receipt, progress?: Progress): Promise<AudioDeliveryResult> {
if (receipt.submissionRejected) return this.result(receipt, 'paused');
if (receipt.deliveryStatus === 'saved') return this.result(receipt, 'saved');
const report = (phase: AudioDeliveryResult['phase']) => {
try { progress?.(this.result(receipt, phase)); } catch { /* Observational only. */ }
};
try {
this.assertAccount(receipt.context);
let errors = 0;
while (true) {
this.assertAccount(receipt.context);
const job = receipt.generation;
if (job && !['accepted', 'running'].includes(job.provider_status)) break;
report(job ? 'generating' : 'submitted');
try {
receipt.generation = job
? await this.options.client.get(job.execution_id, receipt.context.accountKey)
: await this.options.client.byOperation(receipt.context.logicalOperationId, receipt.context.accountKey);
this.checkGeneration(receipt);
await this.options.receipts.put(receipt);
errors = 0;
} catch (error) {
errors++;
if (errors >= 3 || (error instanceof GameAudioClientError && error.code === 'plugin_account_changed')) throw error;
await this.sleep(1000 * errors);
continue;
}
if (['accepted', 'running'].includes(receipt.generation.provider_status)) {
await this.sleep(receipt.generation.poll_interval_seconds * 1000);
}
}
const job = receipt.generation;
if (!job?.output_ready) return this.result(receipt, 'paused');
this.assertAccount(receipt.context);
const project = await stat(receipt.context.projectPath);
if (!project.isDirectory()) throw new Error('Original project is unavailable');
receipt.deliveryStatus = 'saving';
report('saving');
for (const output of job.outputs) {
if (receipt.files.some((file) => file.index === output.index)) continue;
this.assertAccount(receipt.context);
const content = await this.options.client.download(job.execution_id, output.index, receipt.context.accountKey);
const relative = 'assets/generated/game-audio/' + job.execution_id + '/' + job.kind + '-' + (output.index + 1) + '.' + content.extension;
const target = path.join(receipt.context.projectPath, relative);
const lease = await this.options.leases.acquire(receipt.context.localProjectId, 'audio:' + job.execution_id, this.shutdown.signal);
try {
this.assertAccount(receipt.context);
await mkdir(path.dirname(target), { recursive: true });
try { await writeFile(target, content.bytes, { flag: 'wx' }); } catch (error) {
if (!audioRecord(error) || error.code !== 'EEXIST') throw error;
const existing = await readFile(target);
if (!existing.equals(Buffer.from(content.bytes))) throw new Error('Audio target already contains a different file', { cause: error });
}
receipt.files.push({ index: output.index, path: relative, bytes: content.bytes.byteLength, mimeType: content.mimeType });
await this.options.receipts.put(receipt);
try {
await this.options.recordTouchedPaths?.(receipt.context.conversationId, receipt.context.runId, [relative]);
} catch { /* Restored runs may no longer have an in-memory change tracker. Saved paths remain durable. */ }
} finally { lease.release(); }
}
receipt.deliveryStatus = 'saved';
delete receipt.errorCode;
await this.options.receipts.put(receipt);
report('saved');
return this.result(receipt, 'saved');
} catch (error) {
receipt.deliveryStatus = receipt.generation?.output_ready ? 'delivery_failed' : 'needs_reconciliation';
receipt.errorCode = error instanceof GameAudioClientError ? error.code : 'game_audio_delivery_paused';
await this.options.receipts.put(receipt);
report('paused');
if (!(error instanceof GameAudioClientError && error.code === 'plugin_account_changed')) this.scheduleRecovery();
return this.result(receipt, 'paused');
}
}
private result(receipt: Receipt, phase: AudioDeliveryResult['phase']): AudioDeliveryResult {
const job = receipt.generation;
return { executionId: job?.execution_id ?? null, logicalOperationId: receipt.context.logicalOperationId,
providerStatus: job?.provider_status ?? (receipt.submissionRejected ? 'failed' : 'submission_unknown'), deliveryStatus: receipt.deliveryStatus,
outputCount: job?.outputs.length ?? 0, files: [...receipt.files], phase,
progressPercent: job?.progress_percent ?? null,
billing: job?.billing ?? { mode: 'platform_metered', status: receipt.submissionRejected ? 'not_started' : 'receipt_unavailable' },
...(receipt.errorCode || job?.error_code ? { errorCode: receipt.errorCode ?? job?.error_code ?? undefined } : {}) };
}
}