merge: integrate AI design input
This commit is contained in:
@@ -466,6 +466,21 @@ Gate result:
|
||||
- This resumption intentionally performs a local merge only. It does not fetch or push, and the existing remote-authentication follow-up for this long-running integration task remains unchanged.
|
||||
- Created no-ff merge commit `4ff1a7a8461c01de4d1630a3db55b0942fc37ec7` with `3b4669722e0ea671909beb039da1e07c2483341a` as first parent and reviewed source `87a95b4cecf20fce7912305d2e6e2c063ddcbae4` as second parent.
|
||||
|
||||
### 2026-08-18 AI Design Enter/Voice Input Integration Resume
|
||||
|
||||
- Reused this existing Integration owner after the user's explicit request to merge the reviewed AI Design input task. The feature task `20260818-ai-design-input-9f3a` is `ready_for_integration`; its source worktree is clean after forming commit `1ba5b9cc84347bf37cf8a18148a1dcc6b88b3ad2`, based on `13bdc0c`. The parallel video-download merge advanced local `main` to `31fb4ff` before this merge was completed.
|
||||
- Started a normal `--no-ff --no-commit` merge of `1ba5b9c` onto `31fb4ff`; Git reported no textual conflicts. The feature task record remains on its source branch and is deliberately excluded from the integrated `main` tree to preserve task-document ownership boundaries.
|
||||
- The merge keeps AI Design's Enter/Shift+Enter/IME composer contract and E2E Enter coverage, enables Works Speech voice input with 16 kHz mono WAV conversion, authentication retry, context-lifecycle cleanup, and an always-available stop control while other composer operations are busy.
|
||||
- No canonical architecture, domain-rule, or decision promotion is required; the source task records no promotion candidate. Real microphone/Works-account smoke remains an external release validation item.
|
||||
- This resumption is local-only. It does not fetch or push the remote; the existing Git credential follow-up remains unchanged.
|
||||
|
||||
### 2026-08-18 AI Design Video Download Extension Integration Resume
|
||||
|
||||
- Reused this existing Integration owner after the user's explicit request to merge the reviewed AI Design video-download fix. The feature task `20260818-video-download-extension-a4f1` is `ready_for_integration`; its source commit is `43e8a60`, based on the local `main` tip `13bdc0c`.
|
||||
- Created normal no-ff merge commit `31fb4ff`, with `13bdc0c` as first parent and `43e8a60` as second parent. The source task record remains on its feature worktree and is excluded from `main`.
|
||||
- No canonical architecture, domain-rule, or decision promotion is required. The merge is local-only and does not fetch or push the remote.
|
||||
- The merge preserves the separately staged AI Design input task changes already present in the integration worktree; those unrelated changes were not included in `31fb4ff` and were not cleaned or overwritten.
|
||||
|
||||
## Verification
|
||||
|
||||
- Source closeout before integration: Main focused 164/164, full unit 176 files / 2100 tests, typecheck, full lint with zero errors and seven pre-existing warnings, `build:vite`, focused Electron E2E 1/1, project-document gates and seventh independent Sol Standards/Spec review all passed.
|
||||
@@ -625,6 +640,10 @@ Gate result:
|
||||
- Post-review Electron E2E selection — 3/3 passed.
|
||||
- Final context-compaction integration re-review — `PASS`; no remaining P0-P3 findings. The reviewer confirmed polling-idle completion precedes queue release, cold hydration requires a matching current running identity, `main` is clean, and merge/fix/documentation topology is correct.
|
||||
|
||||
- 2026-08-18 video-download merged-main route regression — 1 file / 14 tests passed; `.mp4`, `.mov`, and `.webm` preserve their extensions and non-media MIME responses remain rejected.
|
||||
- 2026-08-18 video-download merged-main focused selection — 3 files / 72 tests passed; typecheck and scoped ESLint for the route/test files passed.
|
||||
- Merge topology and `git diff --check` passed for `31fb4ff`; the source task record is absent from `main`. The integration worktree retains unrelated staged AI Design input changes, so no clean-worktree claim is made.
|
||||
|
||||
## Follow-ups
|
||||
|
||||
- Before claiming true simultaneous model execution, run the opt-in real bundled OpenCode two-Session smoke with an explicitly configured test provider. Current automation proves application isolation, bounded failure and no replay, not provider/runtime concurrency.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Send,
|
||||
Square,
|
||||
Sparkles,
|
||||
Upload,
|
||||
WandSparkles,
|
||||
@@ -42,9 +44,11 @@ import {
|
||||
uploadImageWorkspaceAsset,
|
||||
} from '@/lib/image-workspace';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square';
|
||||
import { useImageWorkspaceStore } from '@/stores/image-workspace';
|
||||
import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum';
|
||||
import { useAuthStore } from '@/stores/auth';
|
||||
import { getProfileAccountKey } from '@/stores/user-profile';
|
||||
import logoSvg from '@/assets/logo.svg';
|
||||
import type {
|
||||
DesignAsset,
|
||||
@@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([
|
||||
]);
|
||||
const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片';
|
||||
const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24;
|
||||
const VOICE_WAV_SAMPLE_RATE = 16_000;
|
||||
const VOICE_WAV_CHANNELS = 1;
|
||||
const VOICE_WAV_BITS_PER_SAMPLE = 16;
|
||||
|
||||
type OfflineAudioContextConstructor = new (
|
||||
numberOfChannels: number,
|
||||
length: number,
|
||||
sampleRate: number,
|
||||
) => OfflineAudioContext;
|
||||
|
||||
type AudioContextWindow = Window & {
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
webkitOfflineAudioContext?: OfflineAudioContextConstructor;
|
||||
};
|
||||
|
||||
async function blobToBase64(blob: Blob): Promise<string> {
|
||||
const bytes = new Uint8Array(await blob.arrayBuffer());
|
||||
let binary = '';
|
||||
const chunkSize = 0x8000;
|
||||
for (let index = 0; index < bytes.length; index += chunkSize) {
|
||||
binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function writeAscii(view: DataView, offset: number, value: string): void {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
view.setUint8(offset + index, value.charCodeAt(index));
|
||||
}
|
||||
}
|
||||
|
||||
function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob {
|
||||
const samples = audioBuffer.getChannelData(0);
|
||||
const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8;
|
||||
const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample;
|
||||
const dataSize = samples.length * blockAlign;
|
||||
const wavBuffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(wavBuffer);
|
||||
|
||||
writeAscii(view, 0, 'RIFF');
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeAscii(view, 8, 'WAVE');
|
||||
writeAscii(view, 12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, VOICE_WAV_CHANNELS, true);
|
||||
view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true);
|
||||
view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true);
|
||||
writeAscii(view, 36, 'data');
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
let offset = 44;
|
||||
for (const sample of samples) {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
|
||||
offset += bytesPerSample;
|
||||
}
|
||||
return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' });
|
||||
}
|
||||
|
||||
async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise<Blob> {
|
||||
const audioWindow = window as AudioContextWindow;
|
||||
const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext;
|
||||
const OfflineAudioContextCtor = window.OfflineAudioContext
|
||||
?? audioWindow.webkitOfflineAudioContext;
|
||||
if (!AudioContextCtor || !OfflineAudioContextCtor) {
|
||||
throw new Error('当前环境无法转换录音');
|
||||
}
|
||||
|
||||
const audioContext = new AudioContextCtor();
|
||||
let decodedBuffer: AudioBuffer;
|
||||
try {
|
||||
decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer());
|
||||
} finally {
|
||||
await audioContext.close().catch(() => undefined);
|
||||
}
|
||||
const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE));
|
||||
const offlineContext = new OfflineAudioContextCtor(
|
||||
VOICE_WAV_CHANNELS,
|
||||
frameCount,
|
||||
VOICE_WAV_SAMPLE_RATE,
|
||||
);
|
||||
const source = offlineContext.createBufferSource();
|
||||
source.buffer = decodedBuffer;
|
||||
source.connect(offlineContext.destination);
|
||||
source.start(0);
|
||||
return audioBufferTo16BitPcmWav(await offlineContext.startRendering());
|
||||
}
|
||||
|
||||
function isUnauthorizedWorksError(error: unknown): boolean {
|
||||
if (error instanceof WorksSquareApiError && error.statusCode === 401) return true;
|
||||
if (!(error instanceof Error)) return false;
|
||||
const record = error as Error & { code?: unknown; details?: { status?: unknown } };
|
||||
const message = error.message.toLowerCase();
|
||||
return record.details?.status === 401
|
||||
|| record.code === 'AUTH_INVALID'
|
||||
|| message.includes('401')
|
||||
|| message.includes('unauthorized')
|
||||
|| message.includes('invalid access token');
|
||||
}
|
||||
|
||||
function currentAuthAccountKey(): string | null {
|
||||
const user = useAuthStore.getState().user;
|
||||
return getProfileAccountKey(user?.userId ?? user?.username);
|
||||
}
|
||||
|
||||
function isGenerationConfirmationIntent(message: string): boolean {
|
||||
const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, '');
|
||||
@@ -782,6 +893,12 @@ function QuoteCard({
|
||||
|
||||
export function ImageCanvas() {
|
||||
const authenticated = useAuthStore((state) => state.isAuthenticated());
|
||||
const accountKey = useAuthStore((state) => (
|
||||
getProfileAccountKey(state.user?.userId ?? state.user?.username)
|
||||
));
|
||||
const accessToken = useAuthStore((state) => state.accessToken);
|
||||
const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken);
|
||||
const refreshSession = useAuthStore((state) => state.refreshSession);
|
||||
const status = useImageWorkspaceStore((state) => state.status);
|
||||
const bootstrap = useImageWorkspaceStore((state) => state.bootstrap);
|
||||
const workspace = useImageWorkspaceStore((state) => state.workspace);
|
||||
@@ -817,12 +934,24 @@ export function ImageCanvas() {
|
||||
const [imageSourceBusyKey, setImageSourceBusyKey] = useState<string | null>(null);
|
||||
const [imageSourceError, setImageSourceError] = useState<string | null>(null);
|
||||
const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState<DesignAsset | null>(null);
|
||||
const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle');
|
||||
const conversationEndRef = useRef<HTMLDivElement | null>(null);
|
||||
const promptRef = useRef<HTMLTextAreaElement | null>(null);
|
||||
const referenceUploadInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const promptSelectionRef = useRef({ start: 0, end: 0 });
|
||||
const quoteRepriceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const quoteDraftVersionRef = useRef(0);
|
||||
const voiceRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const voiceStreamRef = useRef<MediaStream | null>(null);
|
||||
const voiceChunksRef = useRef<Blob[]>([]);
|
||||
const voiceOperationRef = useRef(0);
|
||||
const voiceLifecycleRevisionRef = useRef(0);
|
||||
const renderedAccountKeyRef = useRef(accountKey);
|
||||
if (renderedAccountKeyRef.current !== accountKey) {
|
||||
renderedAccountKeyRef.current = accountKey;
|
||||
voiceLifecycleRevisionRef.current += 1;
|
||||
}
|
||||
const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||
|
||||
const quote = useMemo(
|
||||
() => conversation ? activeQuote(conversation.messages) : null,
|
||||
@@ -859,6 +988,13 @@ export function ImageCanvas() {
|
||||
});
|
||||
}, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]);
|
||||
const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0);
|
||||
const canUseVoiceInput = voiceInputState === 'recording' || (
|
||||
Boolean(workspace && conversation)
|
||||
&& !submittingConversationId
|
||||
&& !imageSourceBusyKey
|
||||
&& !referenceUploadBusy
|
||||
&& voiceInputState !== 'transcribing'
|
||||
);
|
||||
const selectingImageReference = conversation?.brief.medium === 'image';
|
||||
const conversationMessages = useMemo<RenderedDesignMessage[]>(() => {
|
||||
if (!conversation) return [];
|
||||
@@ -1148,6 +1284,184 @@ export function ImageCanvas() {
|
||||
}
|
||||
};
|
||||
|
||||
const stopVoiceStream = useCallback(() => {
|
||||
for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop();
|
||||
voiceStreamRef.current = null;
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
voiceOperationRef.current += 1;
|
||||
const recorder = voiceRecorderRef.current;
|
||||
if (recorder) {
|
||||
recorder.ondataavailable = null;
|
||||
recorder.onstop = null;
|
||||
if (recorder.state !== 'inactive') recorder.stop();
|
||||
voiceRecorderRef.current = null;
|
||||
}
|
||||
voiceChunksRef.current = [];
|
||||
stopVoiceStream();
|
||||
setVoiceInputState('idle');
|
||||
};
|
||||
}, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]);
|
||||
|
||||
const finishVoiceTranscription = useCallback(async (
|
||||
mimeType: string,
|
||||
operationId: number,
|
||||
requestedAccountKey: string,
|
||||
lifecycleRevision: number,
|
||||
requestedWorkspaceId: string,
|
||||
requestedConversationId: string,
|
||||
) => {
|
||||
const isActive = () => voiceOperationRef.current === operationId
|
||||
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||
&& currentAuthAccountKey() === requestedAccountKey
|
||||
&& useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId
|
||||
&& useImageWorkspaceStore.getState().activeConversationId === requestedConversationId;
|
||||
const chunks = voiceChunksRef.current;
|
||||
voiceChunksRef.current = [];
|
||||
stopVoiceStream();
|
||||
|
||||
if (chunks.length === 0) {
|
||||
if (isActive()) {
|
||||
setVoiceInputState('idle');
|
||||
toast.warning('没有录到声音');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isActive()) setVoiceInputState('transcribing');
|
||||
try {
|
||||
if (!isActive()) return;
|
||||
const validAccessToken = await getValidAccessToken();
|
||||
if (!isActive()) return;
|
||||
if (!validAccessToken) {
|
||||
toast.error('请先登录后再使用语音输入');
|
||||
return;
|
||||
}
|
||||
const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' });
|
||||
const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob);
|
||||
if (!isActive()) return;
|
||||
const audioBase64 = await blobToBase64(wavBlob);
|
||||
if (!isActive()) return;
|
||||
const transcribeWithToken = (token: string) => transcribeWorksSpeech({
|
||||
accessToken: token,
|
||||
audioBase64,
|
||||
fileName: 'voice.wav',
|
||||
mimeType: 'audio/wav',
|
||||
language: 'zh',
|
||||
});
|
||||
let transcription;
|
||||
try {
|
||||
if (!isActive()) return;
|
||||
transcription = await transcribeWithToken(validAccessToken);
|
||||
if (!isActive()) return;
|
||||
} catch (error) {
|
||||
if (!isUnauthorizedWorksError(error)) throw error;
|
||||
if (!isActive()) return;
|
||||
const refreshedAccessToken = await refreshSession();
|
||||
if (!isActive()) return;
|
||||
if (!refreshedAccessToken) throw error;
|
||||
transcription = await transcribeWithToken(refreshedAccessToken);
|
||||
if (!isActive()) return;
|
||||
}
|
||||
const transcript = transcription.text.trim();
|
||||
if (transcript) {
|
||||
setPrompt((currentPrompt) => currentPrompt.trim()
|
||||
? `${currentPrompt.trim()}\n${transcript}`
|
||||
: transcript);
|
||||
setActionError(null);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isActive()) {
|
||||
toast.error('语音识别失败', {
|
||||
description: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
if (isActive()) {
|
||||
voiceRecorderRef.current = null;
|
||||
setVoiceInputState('idle');
|
||||
}
|
||||
}
|
||||
}, [getValidAccessToken, refreshSession, stopVoiceStream]);
|
||||
|
||||
const handleVoiceInputClick = useCallback(async () => {
|
||||
if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision
|
||||
|| renderedAccountKeyRef.current !== accountKey) return;
|
||||
if (voiceInputState === 'recording') {
|
||||
const recorder = voiceRecorderRef.current;
|
||||
if (!recorder || recorder.state === 'inactive') {
|
||||
stopVoiceStream();
|
||||
setVoiceInputState('idle');
|
||||
return;
|
||||
}
|
||||
setVoiceInputState('transcribing');
|
||||
recorder.stop();
|
||||
return;
|
||||
}
|
||||
if (!canUseVoiceInput || !workspace || !conversation) return;
|
||||
if (!accountKey || !accessToken) {
|
||||
toast.error('请先登录后再使用语音输入');
|
||||
return;
|
||||
}
|
||||
if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') {
|
||||
toast.error('当前环境不支持语音输入');
|
||||
return;
|
||||
}
|
||||
|
||||
const operationId = voiceOperationRef.current + 1;
|
||||
voiceOperationRef.current = operationId;
|
||||
const lifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||
const requestedAccountKey = accountKey;
|
||||
const isActive = () => voiceOperationRef.current === operationId
|
||||
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||
&& currentAuthAccountKey() === requestedAccountKey;
|
||||
const requestedWorkspaceId = workspace.workspaceId;
|
||||
const requestedConversationId = conversation.conversationId;
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!isActive()) {
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
return;
|
||||
}
|
||||
const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm')
|
||||
? 'audio/webm'
|
||||
: '';
|
||||
const recorder = new window.MediaRecorder(
|
||||
stream,
|
||||
preferredMimeType ? { mimeType: preferredMimeType } : undefined,
|
||||
);
|
||||
voiceStreamRef.current = stream;
|
||||
voiceChunksRef.current = [];
|
||||
voiceRecorderRef.current = recorder;
|
||||
recorder.ondataavailable = (event) => {
|
||||
if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data);
|
||||
};
|
||||
recorder.onstop = () => {
|
||||
void finishVoiceTranscription(
|
||||
recorder.mimeType || preferredMimeType || 'audio/webm',
|
||||
operationId,
|
||||
requestedAccountKey,
|
||||
lifecycleRevision,
|
||||
requestedWorkspaceId,
|
||||
requestedConversationId,
|
||||
);
|
||||
};
|
||||
recorder.start();
|
||||
setVoiceInputState('recording');
|
||||
} catch (error) {
|
||||
stopVoiceStream();
|
||||
if (isActive()) {
|
||||
setVoiceInputState('idle');
|
||||
toast.error('无法开始录音', {
|
||||
description: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]);
|
||||
|
||||
const handleSend = async (override?: string) => {
|
||||
const message = (override ?? prompt).trim();
|
||||
if (!workspace || !conversation || !message || submitting) return;
|
||||
@@ -1605,12 +1919,27 @@ export function ImageCanvas() {
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
disabled
|
||||
className="motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none"
|
||||
aria-label="语音输入"
|
||||
title="AI 绘画暂不支持语音输入"
|
||||
disabled={!canUseVoiceInput}
|
||||
className={cn(
|
||||
'motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none hover:bg-surface-subtle hover:text-foreground',
|
||||
voiceInputState !== 'idle' && 'bg-surface-subtle text-foreground',
|
||||
)}
|
||||
aria-label={voiceInputState === 'recording'
|
||||
? '停止语音输入'
|
||||
: voiceInputState === 'transcribing'
|
||||
? '正在识别语音'
|
||||
: '语音输入'}
|
||||
title={typeof window.MediaRecorder === 'undefined'
|
||||
|| !navigator.mediaDevices?.getUserMedia
|
||||
? '当前环境不支持语音输入'
|
||||
: '语音输入'}
|
||||
onClick={() => void handleVoiceInputClick()}
|
||||
>
|
||||
<Mic className="h-4 w-4" />
|
||||
{voiceInputState === 'transcribing'
|
||||
? <Loader2 className="h-4 w-4 animate-spin" />
|
||||
: voiceInputState === 'recording'
|
||||
? <Square className="h-4 w-4 fill-current" />
|
||||
: <Mic className="h-4 w-4" />}
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
|
||||
@@ -81,7 +81,7 @@ test.describe('AI Design workspace', () => {
|
||||
|
||||
const firstPrompt = '为项目设计一张蓝色星空海报';
|
||||
await page.getByLabel('设计需求').fill(firstPrompt);
|
||||
await page.getByRole('button', { name: '发送给设计 Agent' }).click();
|
||||
await page.getByLabel('设计需求').press('Enter');
|
||||
await expect(page.getByTestId('image-workspace-conversation')).toContainText(firstPrompt);
|
||||
await expect(page.getByText('设计 Agent', { exact: true })).toHaveCount(0);
|
||||
await expect(page.getByText('你', { exact: true })).toHaveCount(0);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { toast } from 'sonner';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ImageWorkspaceApiError } from '@/lib/image-workspace';
|
||||
import { ImageCanvas } from '@/pages/ImageCanvas';
|
||||
@@ -27,6 +28,9 @@ const openImageWorkspaceTaskEventsMock = vi.hoisted(() => vi.fn());
|
||||
const resolveImageWorkspaceAssetUrlMock = vi.hoisted(() => vi.fn());
|
||||
const saveImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const uploadImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const transcribeWorksSpeechMock = vi.hoisted(() => vi.fn());
|
||||
const originalGetValidAccessToken = useAuthStore.getState().getValidAccessToken;
|
||||
const originalRefreshSession = useAuthStore.getState().refreshSession;
|
||||
|
||||
type EventListener = (event: MessageEvent<string>) => void;
|
||||
|
||||
@@ -88,6 +92,42 @@ vi.mock('@/lib/image-workspace', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('@/lib/works-square', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@/lib/works-square')>();
|
||||
return {
|
||||
...actual,
|
||||
transcribeWorksSpeech: (...args: unknown[]) => transcribeWorksSpeechMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
function installMockVoiceWavTranscoder(): void {
|
||||
const pcm = Float32Array.from([0, 0.5, -0.5, 1, -1]);
|
||||
class MockAudioContext {
|
||||
decodeAudioData = vi.fn().mockResolvedValue({
|
||||
duration: pcm.length / 48_000,
|
||||
getChannelData: () => pcm,
|
||||
});
|
||||
close = vi.fn().mockResolvedValue(undefined);
|
||||
}
|
||||
class MockOfflineAudioContext {
|
||||
destination = {};
|
||||
createBufferSource = vi.fn(() => ({
|
||||
buffer: null,
|
||||
connect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
}));
|
||||
startRendering = vi.fn().mockResolvedValue({ getChannelData: () => pcm });
|
||||
}
|
||||
Object.defineProperty(window, 'AudioContext', {
|
||||
configurable: true,
|
||||
value: MockAudioContext,
|
||||
});
|
||||
Object.defineProperty(window, 'OfflineAudioContext', {
|
||||
configurable: true,
|
||||
value: MockOfflineAudioContext,
|
||||
});
|
||||
}
|
||||
|
||||
const bootstrapFixture: DesignWorkspaceBootstrap = {
|
||||
capabilities: {
|
||||
conversation: true,
|
||||
@@ -243,6 +283,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useAuthStore.setState({
|
||||
getValidAccessToken: originalGetValidAccessToken,
|
||||
refreshSession: originalRefreshSession,
|
||||
});
|
||||
useImageWorkspaceStore.getState().reset();
|
||||
useImagePromptMuseumStore.getState().clearPendingPrompt();
|
||||
fetchImageWorkspaceMock.mockResolvedValue(bootstrapFixture);
|
||||
@@ -283,6 +327,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
...taskFixture.resultAssets[0],
|
||||
assetId: 'asset-uploaded',
|
||||
});
|
||||
transcribeWorksSpeechMock.mockResolvedValue({
|
||||
text: '让天空更通透',
|
||||
model: 'gpt-4o-mini-transcribe',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows an honest unavailable state without fabricating projects', async () => {
|
||||
@@ -754,7 +802,7 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(screen.getByTestId('image-workspace-composer-surface'))
|
||||
.toHaveClass('rounded-lg', 'flex', 'min-w-0');
|
||||
expect(screen.getByRole('button', { name: '上传参考图' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled();
|
||||
expect(screen.queryByText('Enter 发送 · Shift+Enter 换行')).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '上传参考图' }));
|
||||
@@ -831,6 +879,316 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(composer).toHaveValue('正在输入中文设计描述');
|
||||
});
|
||||
|
||||
it('reports fixed login and environment guards before starting voice input', async () => {
|
||||
const toastError = vi.spyOn(toast, 'error').mockImplementation(() => 'toast-id');
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: undefined,
|
||||
});
|
||||
useAuthStore.setState({ accessToken: null, canRefresh: false });
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
expect(toastError).toHaveBeenLastCalledWith('请先登录后再使用语音输入');
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
});
|
||||
});
|
||||
const voiceButton = screen.getByRole('button', { name: '语音输入' });
|
||||
expect(voiceButton).toHaveAttribute('title', '当前环境不支持语音输入');
|
||||
fireEvent.click(voiceButton);
|
||||
expect(toastError).toHaveBeenLastCalledWith('当前环境不支持语音输入');
|
||||
toastError.mockRestore();
|
||||
});
|
||||
|
||||
it('records Works Speech input into the current draft and releases the microphone', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const pendingSend = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(pendingSend.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
initialized: true,
|
||||
loading: false,
|
||||
error: null,
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
legacyRefreshToken: null,
|
||||
user: {
|
||||
username: 'designer',
|
||||
userId: '42',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
const composer = screen.getByLabelText('设计需求');
|
||||
fireEvent.change(composer, { target: { value: '保留当前构图' } });
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
|
||||
await waitFor(() => expect(getUserMedia).toHaveBeenCalledWith({ audio: true }));
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '发送给设计 Agent' }));
|
||||
await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce());
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
|
||||
await waitFor(() => expect(composer).toHaveValue('让天空更通透'));
|
||||
expect(transcribeWorksSpeechMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||
accessToken: 'access-token',
|
||||
fileName: 'voice.wav',
|
||||
mimeType: 'audio/wav',
|
||||
language: 'zh',
|
||||
}));
|
||||
const audioBase64 = transcribeWorksSpeechMock.mock.calls[0]?.[0]?.audioBase64 as string;
|
||||
const wavBytes = Uint8Array.from(atob(audioBase64), (character) => character.charCodeAt(0));
|
||||
const wavView = new DataView(wavBytes.buffer);
|
||||
expect(new TextDecoder().decode(wavBytes.slice(0, 4))).toBe('RIFF');
|
||||
expect(wavView.getUint16(22, true)).toBe(1);
|
||||
expect(wavView.getUint32(24, true)).toBe(16_000);
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce();
|
||||
expect(MockMediaRecorder.latest?.state).toBe('inactive');
|
||||
await act(async () => {
|
||||
pendingSend.resolve(conversationFixture());
|
||||
await pendingSend.promise;
|
||||
});
|
||||
});
|
||||
|
||||
it('discards a late transcription after Conversation switch', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const transcription = deferred<{ text: string; model: string }>();
|
||||
transcribeWorksSpeechMock.mockReturnValueOnce(transcription.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(transcribeWorksSpeechMock).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
await act(async () => {
|
||||
transcription.resolve({ text: '不应写入新会话', model: 'gpt-4o-mini-transcribe' });
|
||||
await transcription.promise;
|
||||
});
|
||||
await waitFor(() => expect(screen.getByLabelText('设计需求')).toHaveValue(''));
|
||||
});
|
||||
|
||||
it('stops an active recorder and releases its stream when Conversation changes', async () => {
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
readonly stop = vi.fn(() => {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['late audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
});
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
expect(MockMediaRecorder.latest?.state).toBe('recording');
|
||||
expect(stopTrack).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await waitFor(() => expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled());
|
||||
expect(MockMediaRecorder.latest?.stop).toHaveBeenCalledOnce();
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('does not use a switched account token for an earlier voice recording', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const tokenLookup = deferred<string>();
|
||||
const getValidAccessToken = vi.fn().mockReturnValue(tokenLookup.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['old account audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'old-account-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
getValidAccessToken,
|
||||
user: {
|
||||
username: 'old-account',
|
||||
userId: 'old-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(getValidAccessToken).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'new-account-token',
|
||||
user: {
|
||||
username: 'new-account',
|
||||
userId: 'new-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await act(async () => {
|
||||
tokenLookup.resolve('new-account-token');
|
||||
await tokenLookup.promise;
|
||||
});
|
||||
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('renders the pending user turn and assistant deltas while the command is running', async () => {
|
||||
const response = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(response.promise);
|
||||
|
||||
Reference in New Issue
Block a user