feat: add official game audio generation client

This commit is contained in:
2026-09-21 12:17:32 +08:00
parent d222d17500
commit 09e0ce5cf3
28 changed files with 1835 additions and 31 deletions

View File

@@ -48,6 +48,10 @@ import {
import { createDataServicePluginAdapter } from '../coding-plugins/adapters/data-service';
import { createGameResourcePluginAdapter } from '../coding-plugins/adapters/game-resource';
import { GameResourceClient } from '../services/game-resource-client';
import { GameAudioClient } from '../services/game-audio-client';
import { GameAudioDeliveryCoordinator, GameAudioReceiptStore } from '../services/game-audio-delivery';
import { GameAudioPluginAdapter } from '../coding-plugins/adapters/game-audio';
import { MarketplaceHostedAdmissionResolver } from '../coding-plugins/hosted-admission';
import {
GameResourceDeliveryCoordinator,
GameResourceDeliveryReceiptStore,
@@ -339,6 +343,27 @@ export function createCodingComposition(
bundledReleases: CODE_OWNED_OPTIONAL_BUNDLED_RELEASES,
});
const policyClient = options.policyClient ?? new PluginPolicyClient();
const gameAudioClient = new GameAudioClient();
const gameAudioDelivery = new GameAudioDeliveryCoordinator({
client: gameAudioClient,
receipts: new GameAudioReceiptStore(path.join(options.paths.userDataDir, 'coding-runtime', 'game-audio', 'receipts.json')),
leases: projectWriteLeases,
recordTouchedPaths: (conversationId, runId, paths) => productTools.recordTouchedPaths(conversationId, runId, paths),
...(options.acquireBackgroundLease ? { acquireBackgroundLease: options.acquireBackgroundLease } : {}),
});
const gameAudioAdapter = new GameAudioPluginAdapter({
client: gameAudioClient,
delivery: gameAudioDelivery,
admission: new MarketplaceHostedAdmissionResolver({
marketplace: marketplaceClient, packageStore,
makeloreVersion: options.clientVersion ?? '2.0.1',
bundledReleases: CODE_OWNED_OPTIONAL_BUNDLED_RELEASES,
}),
getPricingVersion: () => {
const state = policyClient.getState();
return state.status === 'current' ? state.catalog?.pricing_version?.version_id ?? null : null;
},
});
const knownPluginIds = new Set(pluginDefinitions.map(({ id }) => id));
// Existing user Releases are discovered from the device index at startup;
// unlike the bundled catalog they cannot be enumerated synchronously. Keep
@@ -373,7 +398,7 @@ export function createCodingComposition(
const capabilityRegistry = createCodingCapabilityRegistry({
policyClient,
projectPlugins,
adapters: [dataServiceAdapter, gameResourceAdapter],
adapters: [dataServiceAdapter, gameResourceAdapter, gameAudioAdapter],
definitions: pluginDefinitions,
effectiveResolver,
getDurableProjectId: async (projectPath, localProjectId) => {
@@ -389,7 +414,7 @@ export function createCodingComposition(
projects,
projectPlugins,
policyClient,
adapters: [dataServiceAdapter, gameResourceAdapter],
adapters: [dataServiceAdapter, gameResourceAdapter, gameAudioAdapter],
definitions: pluginDefinitions,
effectiveResolver,
getDefinitions: async () => {
@@ -505,9 +530,11 @@ export function createCodingComposition(
},
});
void gameResourceDelivery.resumePending().catch(() => undefined);
void gameAudioDelivery.resumePending().catch(() => undefined);
const unsubscribeMarketplaceSession = subscribeWorksSquareSession(() => {
void invalidateManagedResources();
void gameResourceDelivery.resumePending().catch(() => undefined);
void gameAudioDelivery.resumePending().catch(() => undefined);
});
const host = createCodingProductHost({
projects,
@@ -526,6 +553,7 @@ export function createCodingComposition(
})
: () => undefined;
return {
gameAudio: gameAudioDelivery,
attachments,
dataService,
devicePackages: devicePackageManager,
@@ -557,6 +585,7 @@ export function createCodingComposition(
async shutdown() {
conversations.dispose();
gameResourceDelivery.dispose();
gameAudioDelivery.dispose();
previewDataSession?.dispose();
if (typeof options.browser.configurePreviewDataSession === 'function') {
options.browser.configurePreviewDataSession(undefined);

View File

@@ -48,6 +48,7 @@ import type {
import type { MarketplaceLibrarySnapshot } from '../coding-plugins/account-plugin-cache';
import type { EffectivePluginResolver } from '../coding-plugins/effective-resolver';
import type { DevicePackageManager } from '../coding-packages/device-package-manager';
import type { GameAudioDeliveryCoordinator } from '../services/game-audio-delivery';
export interface ActiveCodingProject {
id: string;
@@ -65,6 +66,7 @@ export interface CodingProductHost {
}
export interface CodingProductComposition {
gameAudio?: Pick<GameAudioDeliveryCoordinator, 'readSavedOutput'>;
attachments: CodingAttachmentStore;
dataService: DataServiceOperations;
devicePackages: DevicePackageManager;

View File

@@ -4,6 +4,7 @@ import { CodingProjectServiceError } from '../../coding-projects/project-service
import type { HostApiContext } from '../context';
import { parseJsonBody, sendJson } from '../route-utils';
import { CodingProjectPluginServiceError } from '../coding-product-services';
import { GameAudioClientError } from '../../services/game-audio-client';
const PLUGIN_ROUTE = /^\/api\/coding\/plugins\/([^/]+)$/u;
const PLUGIN_ID_PATTERN = /^[a-z][a-z0-9.-]{0,127}$/u;
@@ -96,6 +97,20 @@ export async function handleCodingPluginRoutes(
url: URL,
ctx: HostApiContext,
): Promise<boolean> {
const savedAudio = /^\/api\/coding\/game-audio\/([0-9a-f-]{36})\/outputs\/([0-9])$/u.exec(url.pathname);
if (savedAudio && req.method === 'GET') {
try {
const delivery = ctx.codingProducts?.gameAudio;
if (!delivery) throw new CodingPluginRouteError(503, 'game_audio_unavailable', 'Audio delivery is unavailable');
if (url.search) requestError('Saved audio lookup does not accept query parameters');
sendJson(res, 200, await delivery.readSavedOutput(savedAudio[1], Number(savedAudio[2])));
} catch (error) {
if (error instanceof GameAudioClientError) {
sendJson(res, error.status, { code: error.code, error: error.message });
} else sendError(res, error);
}
return true;
}
if (url.pathname !== '/api/coding/plugins' && !PLUGIN_ROUTE.test(url.pathname)) return false;
const plugins = ctx.codingProducts?.plugins;
if (!plugins) {

View File

@@ -0,0 +1,119 @@
import { readFile, stat } from 'node:fs/promises';
import path from 'node:path';
import type { CodingPluginToolDefinition } from '../../../shared/coding-plugins';
import type { MarketplaceHostedAdmissionResolver } from '../hosted-admission';
import { MarketplaceHostedAdmissionError } from '../hosted-admission';
import type { AdapterInvocationResult, CodingPluginAdapter, TrustedCodingCapabilityContext } from '../registry';
import { audioRecord, GameAudioClientError, type AudioCommand, type GameAudioClient } from '../../services/game-audio-client';
import type { AudioDeliveryResult, GameAudioDeliveryCoordinator } from '../../services/game-audio-delivery';
import { getWorksSquareAccountBinding } from '../../services/works-square-session';
interface Options {
client: Pick<GameAudioClient, 'cancel'>;
delivery: Pick<GameAudioDeliveryCoordinator, 'generateAndMaterialize'>;
admission: Pick<MarketplaceHostedAdmissionResolver, 'resolve'>;
getPricingVersion: () => string | null;
getAccountKey?: () => string | null;
}
function invalid(): never { throw new GameAudioClientError('plugin_input_invalid', 422, 'Audio generation input is invalid'); }
function result(value: AudioDeliveryResult, projectId: string): AdapterInvocationResult {
const { billing, executionId, progressPercent, ...remaining } = value;
const data = { ...remaining, projectId,
...(executionId === null ? {} : { executionId }),
...(progressPercent === null ? {} : { progressPercent }) };
if (value.providerStatus === 'failed') return {
success: false, status: 503, code: value.errorCode ?? 'game_audio_generation_failed',
error: 'Audio generation failed; inspect the billing receipt before a new attempt',
retryable: false, payload_schema: 'game-audio.v1', data: null, billing,
};
return { success: true, status: value.deliveryStatus === 'saved' ? 200 : 202,
code: null, error: null,
retryable: false, payload_schema: 'game-audio.v1', data, billing };
}
async function references(project: string, value: unknown) {
if (value === undefined) return [];
if (!Array.isArray(value) || value.length > 4) invalid();
const items: { name: string; mime_type: string; data_base64: string }[] = [];
let total = 0;
for (const candidate of value) {
if (typeof candidate !== 'string' || !candidate || candidate.length > 1024) invalid();
const absolute = path.resolve(project, candidate);
const relative = path.relative(path.resolve(project), absolute);
if (!relative || relative === '..' || relative.startsWith('..' + path.sep) || path.isAbsolute(relative)) invalid();
const mime = ({ '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp' } as Record<string, string>)[path.extname(absolute).toLowerCase()];
const metadata = await stat(absolute);
if (!mime || !metadata.isFile() || metadata.size <= 0 || metadata.size > 8 * 1024 * 1024) invalid();
const content = await readFile(absolute);
total += content.byteLength;
if (!content.length || content.length > 8 * 1024 * 1024 || total > 16 * 1024 * 1024) invalid();
items.push({ name: path.basename(absolute).slice(0, 160), mime_type: mime, data_base64: content.toString('base64') });
}
return items;
}
export class GameAudioPluginAdapter implements CodingPluginAdapter {
readonly pluginId = 'makelore.game-audio';
constructor(private readonly options: Options) {}
async inspect() { return { status: 'ready' as const }; }
async invoke(context: TrustedCodingCapabilityContext, tool: CodingPluginToolDefinition,
input: unknown, onProgress?: (value: AdapterInvocationResult) => void): Promise<AdapterInvocationResult> {
let submitted = false;
try {
if (!audioRecord(input)) invalid();
const account = this.options.getAccountKey ? this.options.getAccountKey() : getWorksSquareAccountBinding()?.accountKey;
if (!account) throw new GameAudioClientError('authentication_required', 401, 'Sign-in is required');
if (tool.name === 'game_audio_cancel') {
if (typeof input.executionId !== 'string' || Object.keys(input).some((key) => key !== 'executionId')) invalid();
const generation = await this.options.client.cancel(input.executionId, account);
return { success: true, status: 200, code: null, error: null, retryable: false,
payload_schema: 'game-audio.v1', data: { executionId: generation.execution_id,
providerStatus: generation.provider_status, outputCount: generation.outputs.length,
generationBilling: Object.fromEntries(Object.entries(generation.billing).filter(([, value]) => value !== null)) } };
}
if (input.confirmed !== true) throw new GameAudioClientError('confirmation_required', 400, 'Confirm Token Point generation and automatic project save first');
const sound = tool.name === 'game_sound_generate';
if (!sound && tool.name !== 'game_music_generate' && tool.name !== 'game_music_preview') invalid();
const allowed = sound ? ['prompt', 'confirmed', 'mode', 'duration', 'count', 'loop'] : ['prompt', 'confirmed', 'referencePaths'];
if (Object.keys(input).some((key) => !allowed.includes(key)) || typeof input.prompt !== 'string'
|| !input.prompt.trim() || input.prompt.trim().length > 4000) invalid();
const pricing = this.options.getPricingVersion();
if (!pricing) throw new GameAudioClientError('plugin_billing_unavailable', 503, 'Current audio pricing is unavailable');
let specification: Pick<Extract<AudioCommand, { kind: 'sound' }>, 'kind' | 'mode' | 'duration' | 'count' | 'loop'>
| Pick<Extract<AudioCommand, { kind: 'music' }>, 'kind' | 'mode' | 'reference_files'>;
if (sound) {
const mode = input.mode ?? 'single';
const duration = input.duration ?? 2;
const count = input.count ?? (mode === 'single' ? 1 : 4);
const loop = input.loop ?? false;
if (!['single', 'pack', 'variants'].includes(String(mode)) || typeof mode !== 'string'
|| typeof duration !== 'number' || !(duration === 0.5 || Number.isInteger(duration) && duration >= 1 && duration <= 10)
|| typeof count !== 'number' || !Number.isInteger(count) || count < 1 || count > 10
|| (mode === 'single' && count !== 1) || typeof loop !== 'boolean') invalid();
specification = { kind: 'sound', mode: mode as 'single' | 'pack' | 'variants', duration, count, loop };
} else {
specification = { kind: 'music', mode: tool.name === 'game_music_preview' ? 'demo' : 'full',
reference_files: await references(context.projectPath, input.referencePaths) };
}
const admission = await this.options.admission.resolve({ pluginId: this.pluginId,
workerSnapshot: { requestId: context.requestId, pluginReleaseId: context.pluginReleaseId } });
submitted = true;
const delivered = await this.options.delivery.generateAndMaterialize({
accountKey: account, conversationId: context.conversationId, runId: context.runId,
localProjectId: context.localProjectId, durableProjectId: context.durableProjectId,
projectPath: context.projectPath, logicalOperationId: context.requestId,
}, { ...specification, release_id: admission.releaseId, release_admission_id: admission.releaseAdmissionId,
project_id: context.durableProjectId, logical_operation_id: context.requestId,
pricing_version_id: pricing, prompt: input.prompt.trim(), confirmed: true },
(progress) => onProgress?.(result(progress, context.localProjectId)));
return result(delivered, context.localProjectId);
} catch (error) {
const known = error instanceof GameAudioClientError || error instanceof MarketplaceHostedAdmissionError;
return { success: false, status: known ? error.status : 503,
code: known ? error.code : 'plugin_backend_unavailable',
error: known ? error.message : submitted
? 'Audio receipt is unavailable; recover the original operation instead of generating again'
: 'Audio service is unavailable; no new generation was started',
retryable: false, payload_schema: 'game-audio.v1', data: null,
billing: { mode: 'platform_metered', status: submitted ? 'receipt_unavailable' : 'not_started' } };
}
}
}

View File

@@ -12,6 +12,8 @@ import {
CODE_OWNED_PLUGIN_PERMISSION_IDS,
GAME_RESOURCE_BUNDLED_RELEASE_ID,
GAME_RESOURCE_PLUGIN_ID,
GAME_AUDIO_PLUGIN_ID,
GAME_AUDIO_BUNDLED_RELEASE_ID,
PROJECT_SCAFFOLD_BUNDLED_RELEASE_ID,
PROJECT_SCAFFOLD_PLUGIN_ID,
DATA_SERVICE_CAPABILITY_IDS,
@@ -38,9 +40,17 @@ export const BUNDLED_CODING_PLUGIN_ROOTS = Object.freeze([
'data-service',
'game-resource',
'project-scaffold',
'game-audio',
] as const);
const BUNDLED_CODING_PLUGIN_METADATA = Object.freeze({
'game-audio': Object.freeze({
pluginId: GAME_AUDIO_PLUGIN_ID,
runtimeKind: 'platform_hosted' as const,
acquisitionMode: 'user_acquired' as const,
releaseId: GAME_AUDIO_BUNDLED_RELEASE_ID,
bundledV2: true,
}),
'data-service': Object.freeze({
pluginId: DATA_SERVICE_PLUGIN_ID,
runtimeKind: 'bundled_typed' as const,
@@ -1182,9 +1192,10 @@ function validateBundledDefinitions(definitions: readonly CodingPluginDefinition
}
for (const tool of definition.tools) {
if (toolIds.has(tool.name)) throw new Error(`Duplicate bundled tool identifier: ${tool.name}`);
if (operationIds.has(tool.operation)) throw new Error(`Duplicate bundled operation identifier: ${tool.operation}`);
const operationKey = `${definition.id}:${tool.capabilityId}:${tool.operation}`;
if (operationIds.has(operationKey)) throw new Error(`Duplicate bundled operation identifier: ${operationKey}`);
toolIds.add(tool.name);
operationIds.add(tool.operation);
operationIds.add(operationKey);
capabilityIds.add(tool.capabilityId);
}
}

View File

@@ -145,7 +145,7 @@ async function resolveInsideProject(projectPath: string, candidatePath: string):
}
}
async function readMediaDataUrl(projectPath: string, candidatePath: string, maxBytes: number): Promise<string | null> {
export async function readMediaDataUrl(projectPath: string, candidatePath: string, maxBytes: number): Promise<string | null> {
const resolved = await resolveInsideProject(projectPath, candidatePath);
if (!resolved) return null;
const fileStat = await stat(resolved);

View File

@@ -0,0 +1,233 @@
import type { CapabilityBillingReceiptV1 } from '../../shared/data-service';
import { WORKS_SQUARE_CONFIG } from '../api/works-config';
import { proxyAwareFetch } from '../utils/proxy-fetch';
import { getValidWorksSquareAccessToken, getWorksSquareAccountBinding } from './works-square-session';
const ROOT = '/api/plugins/v1/hosted/game-audio/generations';
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u;
const DECIMAL = /^(?:0|[1-9]\d*)\.\d{2}$/u;
const AUDIO_EXTENSIONS: Readonly<Record<string, string>> = {
'audio/mpeg': 'mp3', 'audio/wav': 'wav', 'audio/x-wav': 'wav',
'audio/ogg': 'ogg', 'audio/flac': 'flac', 'audio/mp4': 'm4a',
};
export type AudioBilling = Extract<CapabilityBillingReceiptV1, { mode: 'platform_metered' }>;
export interface AudioGeneration {
schema_version: 1;
plugin_id: 'makelore.game-audio';
execution_id: string;
release_id: string;
project_id: string;
logical_operation_id: string;
kind: 'music' | 'sound';
provider_status: 'accepted' | 'running' | 'succeeded' | 'failed' | 'cancelled' | 'submission_unknown';
output_ready: boolean;
outputs: { index: number; name: string }[];
progress_percent: number | null;
poll_interval_seconds: number;
error_code: string | null;
billing: AudioBilling;
}
interface AudioIdentity {
release_id: string;
release_admission_id: string;
project_id: string;
logical_operation_id: string;
pricing_version_id: string;
prompt: string;
confirmed: true;
}
export type AudioCommand = AudioIdentity & ({
kind: 'music'; mode: 'full' | 'demo';
reference_files?: { name: string; mime_type: string; data_base64: string }[];
} | {
kind: 'sound'; mode: 'single' | 'pack' | 'variants';
duration: number; count: number; loop: boolean;
});
export class GameAudioClientError extends Error {
constructor(readonly code: string, readonly status: number, message: string) {
super(message);
this.name = 'GameAudioClientError';
}
}
export function audioRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object' && !Array.isArray(value);
}
function invalid(): never {
throw new GameAudioClientError('plugin_backend_invalid', 502, 'Audio response is invalid');
}
function bounded(value: unknown, max: number): value is string {
return typeof value === 'string' && value.length > 0 && value.length <= max;
}
function integer(value: unknown, min: number, max: number): value is number {
return typeof value === 'number' && Number.isSafeInteger(value) && value >= min && value <= max;
}
export function parseAudioGeneration(value: unknown): AudioGeneration {
if (!audioRecord(value) || value.schema_version !== 1 || value.plugin_id !== 'makelore.game-audio'
|| !bounded(value.execution_id, 36) || !UUID.test(value.execution_id)
|| !bounded(value.release_id, 36) || !UUID.test(value.release_id)
|| !bounded(value.project_id, 36) || !UUID.test(value.project_id)
|| !bounded(value.logical_operation_id, 128)
|| !['music', 'sound'].includes(String(value.kind))
|| !['accepted', 'running', 'succeeded', 'failed', 'cancelled', 'submission_unknown'].includes(String(value.provider_status))
|| typeof value.output_ready !== 'boolean'
|| !Array.isArray(value.outputs) || value.outputs.length > 10
|| !value.outputs.every((item, index) => audioRecord(item)
&& Object.keys(item).every((key) => ['index', 'name'].includes(key))
&& item.index === index && bounded(item.name, 160))
|| value.output_ready !== (value.outputs.length > 0)
|| (value.output_ready && value.provider_status !== 'succeeded')
|| !integer(value.poll_interval_seconds, 1, 300)
|| (value.progress_percent !== null && !integer(value.progress_percent, 0, 100))
|| (value.error_code !== null && !bounded(value.error_code, 100))) invalid();
const bill = value.billing;
if (!audioRecord(bill) || bill.mode !== 'platform_metered'
|| !['reserved', 'dispatched', 'settled', 'released', 'expired', 'pending_review', 'refunded'].includes(String(bill.status))
|| typeof bill.reserved_points !== 'string' || !DECIMAL.test(bill.reserved_points)
|| (bill.actual_points != null && (typeof bill.actual_points !== 'string' || !DECIMAL.test(bill.actual_points)))
|| !integer(bill.usage_amount, 0, 200)
|| bill.unit !== (value.kind === 'sound' ? 'half_second' : 'generation')
|| (['settled', 'refunded'].includes(String(bill.status)) && bill.actual_points == null)
|| (['accepted', 'running'].includes(String(value.provider_status)) && !['reserved', 'dispatched'].includes(String(bill.status)))
|| (value.provider_status === 'succeeded' && !['settled', 'pending_review', 'refunded'].includes(String(bill.status)))
|| (value.provider_status === 'submission_unknown' && bill.status !== 'pending_review')
|| (['failed', 'cancelled'].includes(String(value.provider_status)) && !['released', 'expired', 'refunded', 'pending_review'].includes(String(bill.status)))) invalid();
const fields = ['schema_version', 'plugin_id', 'execution_id', 'release_id', 'project_id',
'logical_operation_id', 'kind', 'provider_status', 'output_ready', 'outputs',
'progress_percent', 'poll_interval_seconds', 'error_code', 'billing'];
if (Object.keys(value).some((key) => !fields.includes(key))
|| Object.keys(bill).some((key) => !['mode', 'status', 'reserved_points', 'actual_points', 'usage_amount', 'unit'].includes(key))) invalid();
// Hosted JSON uses null; shared capability receipts use an absent optional amount.
return { ...value, billing: Object.fromEntries(Object.entries(bill).filter(([, field]) => field !== null)) } as unknown as AudioGeneration;
}
async function bytes(response: Response, limit: number): Promise<Uint8Array> {
if (Number(response.headers.get('content-length')) > limit) {
await response.body?.cancel();
invalid();
}
const reader = response.body?.getReader();
if (!reader) invalid();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const next = await reader.read();
if (next.done) break;
total += next.value.byteLength;
if (total > limit) invalid();
chunks.push(next.value);
}
} finally {
await reader.cancel().catch(() => undefined);
reader.releaseLock();
}
const result = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) { result.set(chunk, offset); offset += chunk.byteLength; }
return result;
}
export interface GameAudioClientOptions {
fetchImpl?: typeof fetch;
getAccessToken?: typeof getValidWorksSquareAccessToken;
getAccountBinding?: typeof getWorksSquareAccountBinding;
apiBaseUrl?: string;
}
export class GameAudioClient {
constructor(private readonly options: GameAudioClientOptions = {}) {}
async generate(command: AudioCommand, account: string): Promise<AudioGeneration> {
const result = await this.json(ROOT, account, 'POST', command);
if (result.project_id !== command.project_id || result.release_id !== command.release_id
|| result.logical_operation_id !== command.logical_operation_id || result.kind !== command.kind) invalid();
return result;
}
async get(id: string, account: string): Promise<AudioGeneration> {
if (!UUID.test(id)) invalid();
const result = await this.json(ROOT + '/' + id, account);
if (result.execution_id !== id) invalid();
return result;
}
async byOperation(id: string, account: string): Promise<AudioGeneration> {
if (!bounded(id, 128)) invalid();
const result = await this.json(ROOT + '/by-operation/' + encodeURIComponent(id), account);
if (result.logical_operation_id !== id) invalid();
return result;
}
async cancel(id: string, account: string): Promise<AudioGeneration> {
if (!UUID.test(id)) invalid();
const result = await this.json(ROOT + '/' + id + '/cancel', account, 'POST');
if (result.execution_id !== id) invalid();
return result;
}
async download(id: string, index: number, account: string) {
if (!UUID.test(id) || !integer(index, 0, 9)) invalid();
return await this.request(ROOT + '/' + id + '/content?output_index=' + index, account, 'GET', undefined, async (response) => {
const mime = response.headers.get('content-type')?.split(';', 1)[0]?.trim().toLowerCase() ?? '';
const extension = AUDIO_EXTENSIONS[mime];
if (!extension) {
await response.body?.cancel().catch(() => undefined);
throw new GameAudioClientError('game_audio_content_invalid', 502, 'Unsupported audio media');
}
const content = await bytes(response, 32 * 1024 * 1024);
if (content.length === 0) invalid();
return { bytes: content, extension, mimeType: mime };
});
}
private async json(route: string, account: string, method = 'GET', command?: AudioCommand) {
return await this.request(route, account, method, command, async (response) => {
const content = await bytes(response, 1024 * 1024);
let value: unknown;
try { value = JSON.parse(new TextDecoder().decode(content)); } catch { invalid(); }
return parseAudioGeneration(value);
});
}
private async request<T>(
route: string, account: string, method: string, command: AudioCommand | undefined,
consume: (response: Response) => Promise<T>,
): Promise<T> {
const binding = this.options.getAccountBinding ?? getWorksSquareAccountBinding;
const original = binding();
const assertAccount = () => {
const current = binding();
if (!original || original.accountKey !== account || current?.accountKey !== account || current.epoch !== original.epoch) {
throw new GameAudioClientError('plugin_account_changed', 409, 'Original signed-in account is required');
}
};
assertAccount();
const fetchImpl = this.options.fetchImpl ?? proxyAwareFetch;
const getToken = this.options.getAccessToken ?? getValidWorksSquareAccessToken;
const body = command ? JSON.stringify(command) : undefined;
if (body && Buffer.byteLength(body) > 24 * 1024 * 1024) invalid();
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), method === 'POST' ? 90_000 : 40_000);
try {
for (let attempt = 0; attempt < 2; attempt++) {
const token = await getToken({ fetchImpl, ...(attempt ? { forceRefresh: true } : {}) });
assertAccount();
if (!token) throw new GameAudioClientError('authentication_required', 401, 'Sign-in is required');
const response = await fetchImpl((this.options.apiBaseUrl ?? WORKS_SQUARE_CONFIG.apiBaseUrl).replace(/\/$/u, '') + route, {
method, redirect: 'manual', signal: controller.signal,
headers: { Accept: 'application/json', Authorization: 'Bearer ' + token,
...(body ? { 'Content-Type': 'application/json' } : {}) },
...(body ? { body } : {}),
});
assertAccount();
if (response.status === 401 && attempt === 0) { await response.body?.cancel(); continue; }
if (!response.ok) {
let code = response.status === 429 ? 'plugin_rate_limited' : 'plugin_backend_unavailable';
try {
const error: unknown = JSON.parse(new TextDecoder().decode(await bytes(response, 1024 * 1024)));
if (response.status !== 429 && audioRecord(error) && audioRecord(error.detail)
&& bounded(error.detail.error_code, 100)) code = error.detail.error_code;
} catch { /* HTTP status still owns the failure even if the error body is invalid. */ }
throw new GameAudioClientError(code, response.status, 'Audio request failed (' + response.status + ')');
}
const value = await consume(response);
assertAccount();
return value;
}
throw new GameAudioClientError('authentication_required', 401, 'Sign-in is required');
} finally {
clearTimeout(timeout);
}
}
}

View File

@@ -0,0 +1,268 @@
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { atomicWriteJson, readJsonFile } from '../coding-projects/atomic-json';
import type { PiProjectWriteLeaseCoordinator } from '../coding-runtime/pi/write-lease';
import { audioRecord, GameAudioClientError, parseAudioGeneration,
type AudioCommand, type AudioGeneration, type GameAudioClient, type AudioBilling } from './game-audio-client';
import { getWorksSquareAccountBinding } from './works-square-session';
import { readMediaDataUrl } from '../coding-projects/game-asset-browser';
export interface AudioDeliveryContext {
accountKey: string;
conversationId: string;
runId: string;
localProjectId: string;
durableProjectId: string;
projectPath: string;
logicalOperationId: string;
}
export interface AudioDeliveredFile { index: number; path: string; bytes: number; mimeType: string }
export interface AudioDeliveryResult {
executionId: string | null;
logicalOperationId: string;
providerStatus: AudioGeneration['provider_status'];
deliveryStatus: 'waiting' | 'saving' | 'saved' | 'delivery_failed' | 'needs_reconciliation';
outputCount: number;
files: AudioDeliveredFile[];
billing: AudioBilling;
phase: 'submitted' | 'generating' | 'saving' | 'saved' | 'paused';
progressPercent: number | null;
errorCode?: string;
}
interface Receipt {
context: AudioDeliveryContext;
generation: AudioGeneration | null;
deliveryStatus: AudioDeliveryResult['deliveryStatus'];
files: AudioDeliveredFile[];
errorCode?: string;
submissionRejected?: boolean;
}
function sameIdentity(left: AudioDeliveryContext, right: AudioDeliveryContext) {
return left.accountKey === right.accountKey && left.logicalOperationId === right.logicalOperationId;
}
function validReceipt(value: unknown): value is Receipt {
if (!audioRecord(value) || !audioRecord(value.context)
|| !['accountKey', 'conversationId', 'runId', 'localProjectId', 'durableProjectId', 'projectPath', 'logicalOperationId']
.every((key) => typeof value.context[key] === 'string' && value.context[key].length > 0)
|| !path.isAbsolute(String(value.context.projectPath))
|| !['waiting', 'saving', 'saved', 'delivery_failed', 'needs_reconciliation'].includes(String(value.deliveryStatus))
|| !Array.isArray(value.files) || value.files.length > 10) return false;
try {
if (value.generation !== null) parseAudioGeneration(value.generation);
} catch { return false; }
return value.files.every((file) => audioRecord(file) && Number.isInteger(file.index)
&& typeof file.bytes === 'number' && file.bytes > 0 && typeof file.path === 'string'
&& /^assets\/generated\/game-audio\/[0-9a-f-]{36}\/(?:music|sound)-\d+\.(?:mp3|wav|ogg|flac|m4a)$/u.test(file.path));
}
/** One atomic userData file; no project file serves as the job database. */
export class GameAudioReceiptStore {
private writes: Promise<void> = Promise.resolve();
constructor(private readonly filePath: string) {}
async list(): Promise<Receipt[]> {
let value: unknown;
try { value = await readJsonFile(this.filePath); } catch (error) {
if (audioRecord(error) && error.code === 'ENOENT') return [];
throw error;
}
if (!audioRecord(value) || value.schemaVersion !== 1 || !Array.isArray(value.receipts)
|| !value.receipts.every(validReceipt)) throw new Error('Audio receipt file is invalid');
return value.receipts;
}
async put(receipt: Receipt): Promise<void> {
const operation = this.writes.then(async () => {
const rows = (await this.list()).filter((row) => !sameIdentity(row.context, receipt.context));
rows.push(receipt);
await atomicWriteJson(this.filePath, { schemaVersion: 1, receipts: rows });
});
this.writes = operation.catch(() => undefined);
await operation;
}
}
interface Options {
client: Pick<GameAudioClient, 'generate' | 'get' | 'byOperation' | 'download'>;
receipts: GameAudioReceiptStore;
leases: PiProjectWriteLeaseCoordinator;
getAccountKey?: () => string | null;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
recordTouchedPaths?: (conversationId: string, runId: string, paths: readonly string[]) => Promise<void>;
acquireBackgroundLease?: (lease: { id: string; kind: 'coding-run' }) => () => void;
}
type Progress = (value: AudioDeliveryResult) => void;
export class GameAudioDeliveryCoordinator {
private readonly flights = new Map<string, Promise<AudioDeliveryResult>>();
private readonly shutdown = new AbortController();
private recoveryTimer: ReturnType<typeof setTimeout> | undefined;
constructor(private readonly options: Options) {}
dispose(): void {
this.shutdown.abort();
clearTimeout(this.recoveryTimer);
}
private scheduleRecovery(): void {
if (this.shutdown.signal.aborted || this.recoveryTimer) return;
this.recoveryTimer = setTimeout(() => {
this.recoveryTimer = undefined;
void this.resumePending().catch(() => this.scheduleRecovery());
}, 30_000);
this.recoveryTimer.unref?.();
}
async readSavedOutput(executionId: string, index: number): Promise<{ path: string; dataUrl: string }> {
const account = this.account();
const receipt = (await this.options.receipts.list()).find((row) =>
row.context.accountKey === account && row.generation?.execution_id === executionId);
const file = receipt?.files.find((item) => item.index === index);
if (!receipt || !file) throw new GameAudioClientError('game_audio_output_not_found', 404, 'Saved audio is unavailable');
this.assertAccount(receipt.context);
const dataUrl = await readMediaDataUrl(receipt.context.projectPath, file.path, 32 * 1024 * 1024);
this.assertAccount(receipt.context);
if (!dataUrl?.startsWith('data:audio/')) throw new GameAudioClientError('game_audio_content_invalid', 422, 'Saved audio cannot be read');
return { path: file.path, dataUrl };
}
async generateAndMaterialize(context: AudioDeliveryContext, command: AudioCommand, progress?: Progress) {
return await this.singleFlight(context, async () => {
const old = (await this.options.receipts.list()).find((row) => sameIdentity(row.context, context));
if (old) return await this.advance(old, progress);
this.assertAccount(context);
const receipt: Receipt = { context: { ...context }, generation: null, deliveryStatus: 'waiting', files: [] };
// The durable identity must exist BEFORE the first possibly charged request.
await this.options.receipts.put(receipt);
try {
receipt.generation = await this.options.client.generate(command, context.accountKey);
this.checkGeneration(receipt);
await this.options.receipts.put(receipt);
} catch (error) {
if (error instanceof GameAudioClientError && (error.status === 401 || error.status === 422
|| error.code === 'plugin_pricing_changed')) {
receipt.deliveryStatus = 'needs_reconciliation';
receipt.submissionRejected = true;
receipt.errorCode = error.code;
await this.options.receipts.put(receipt);
return this.result(receipt, 'paused');
}
// Response loss is not permission to submit again.
}
return await this.advance(receipt, progress);
});
}
async resumePending(): Promise<void> {
const account = this.account();
if (!account || this.shutdown.signal.aborted) return;
const rows = await this.options.receipts.list();
await Promise.allSettled(rows.filter((row) => row.context.accountKey === account
&& !row.submissionRejected
&& row.deliveryStatus !== 'saved'
&& !['failed', 'cancelled'].includes(row.generation?.provider_status ?? ''))
.map((row) => this.singleFlight(row.context, () => this.advance(row))));
}
private async singleFlight(context: AudioDeliveryContext, run: () => Promise<AudioDeliveryResult>) {
const key = JSON.stringify([context.accountKey, context.logicalOperationId]);
const old = this.flights.get(key);
if (old) return await old;
const release = this.options.acquireBackgroundLease?.({ id: 'audio:' + key, kind: 'coding-run' });
const flight = run().finally(() => { this.flights.delete(key); release?.(); });
this.flights.set(key, flight);
return await flight;
}
private account(): string | null {
return this.options.getAccountKey ? this.options.getAccountKey() : getWorksSquareAccountBinding()?.accountKey ?? null;
}
private assertAccount(context: AudioDeliveryContext): void {
if (this.shutdown.signal.aborted || this.account() !== context.accountKey) {
throw new GameAudioClientError('plugin_account_changed', 409, 'Audio recovery waits for the original account');
}
}
private checkGeneration(receipt: Receipt): void {
const job = receipt.generation;
if (job && (job.project_id !== receipt.context.durableProjectId
|| job.logical_operation_id !== receipt.context.logicalOperationId)) {
throw new GameAudioClientError('plugin_backend_invalid', 502, 'Audio execution identity changed');
}
}
private async sleep(ms: number): Promise<void> {
if (this.options.sleep) await this.options.sleep(ms, this.shutdown.signal);
else await delay(ms, undefined, { signal: this.shutdown.signal });
}
private async advance(receipt: Receipt, progress?: Progress): Promise<AudioDeliveryResult> {
if (receipt.submissionRejected) return this.result(receipt, 'paused');
if (receipt.deliveryStatus === 'saved') return this.result(receipt, 'saved');
const report = (phase: AudioDeliveryResult['phase']) => {
try { progress?.(this.result(receipt, phase)); } catch { /* Observational only. */ }
};
try {
this.assertAccount(receipt.context);
let errors = 0;
while (true) {
this.assertAccount(receipt.context);
const job = receipt.generation;
if (job && !['accepted', 'running'].includes(job.provider_status)) break;
report(job ? 'generating' : 'submitted');
try {
receipt.generation = job
? await this.options.client.get(job.execution_id, receipt.context.accountKey)
: await this.options.client.byOperation(receipt.context.logicalOperationId, receipt.context.accountKey);
this.checkGeneration(receipt);
await this.options.receipts.put(receipt);
errors = 0;
} catch (error) {
errors++;
if (errors >= 3 || (error instanceof GameAudioClientError && error.code === 'plugin_account_changed')) throw error;
await this.sleep(1000 * errors);
continue;
}
if (['accepted', 'running'].includes(receipt.generation.provider_status)) {
await this.sleep(receipt.generation.poll_interval_seconds * 1000);
}
}
const job = receipt.generation;
if (!job?.output_ready) return this.result(receipt, 'paused');
this.assertAccount(receipt.context);
const project = await stat(receipt.context.projectPath);
if (!project.isDirectory()) throw new Error('Original project is unavailable');
receipt.deliveryStatus = 'saving';
report('saving');
for (const output of job.outputs) {
if (receipt.files.some((file) => file.index === output.index)) continue;
this.assertAccount(receipt.context);
const content = await this.options.client.download(job.execution_id, output.index, receipt.context.accountKey);
const relative = 'assets/generated/game-audio/' + job.execution_id + '/' + job.kind + '-' + (output.index + 1) + '.' + content.extension;
const target = path.join(receipt.context.projectPath, relative);
const lease = await this.options.leases.acquire(receipt.context.localProjectId, 'audio:' + job.execution_id, this.shutdown.signal);
try {
this.assertAccount(receipt.context);
await mkdir(path.dirname(target), { recursive: true });
try { await writeFile(target, content.bytes, { flag: 'wx' }); } catch (error) {
if (!audioRecord(error) || error.code !== 'EEXIST') throw error;
const existing = await readFile(target);
if (!existing.equals(Buffer.from(content.bytes))) throw new Error('Audio target already contains a different file', { cause: error });
}
receipt.files.push({ index: output.index, path: relative, bytes: content.bytes.byteLength, mimeType: content.mimeType });
await this.options.receipts.put(receipt);
try {
await this.options.recordTouchedPaths?.(receipt.context.conversationId, receipt.context.runId, [relative]);
} catch { /* Restored runs may no longer have an in-memory change tracker. Saved paths remain durable. */ }
} finally { lease.release(); }
}
receipt.deliveryStatus = 'saved';
delete receipt.errorCode;
await this.options.receipts.put(receipt);
report('saved');
return this.result(receipt, 'saved');
} catch (error) {
receipt.deliveryStatus = receipt.generation?.output_ready ? 'delivery_failed' : 'needs_reconciliation';
receipt.errorCode = error instanceof GameAudioClientError ? error.code : 'game_audio_delivery_paused';
await this.options.receipts.put(receipt);
report('paused');
if (!(error instanceof GameAudioClientError && error.code === 'plugin_account_changed')) this.scheduleRecovery();
return this.result(receipt, 'paused');
}
}
private result(receipt: Receipt, phase: AudioDeliveryResult['phase']): AudioDeliveryResult {
const job = receipt.generation;
return { executionId: job?.execution_id ?? null, logicalOperationId: receipt.context.logicalOperationId,
providerStatus: job?.provider_status ?? (receipt.submissionRejected ? 'failed' : 'submission_unknown'), deliveryStatus: receipt.deliveryStatus,
outputCount: job?.outputs.length ?? 0, files: [...receipt.files], phase,
progressPercent: job?.progress_percent ?? null,
billing: job?.billing ?? { mode: 'platform_metered', status: receipt.submissionRejected ? 'not_started' : 'receipt_unavailable' },
...(receipt.errorCode || job?.error_code ? { errorCode: receipt.errorCode ?? job?.error_code ?? undefined } : {}) };
}
}