feat: enable AI design voice input
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
# Task: Fix AI design dialog Enter and voice input
|
||||
|
||||
## Identity
|
||||
|
||||
- Task ID: 20260818-ai-design-input-9f3a
|
||||
- Mode: Feature
|
||||
- Branch: codex/20260818-ai-design-input-9f3a-ai-design-input
|
||||
- Worktree: D:\Datas\OthersProjects\makelore-ai-design-input-9f3a
|
||||
- Base commit: 13bdc0c9bab656aa9ff6c990bf6359f7d3210750
|
||||
- Owner: developer
|
||||
- Status: Completed
|
||||
|
||||
## Scope
|
||||
|
||||
- 保留并验收 AI 设计对话框现有的 Enter 发送、Shift+Enter 换行和输入法组合保护。
|
||||
- 打开 AI 设计对话框的语音输入：采集麦克风、转为 16 kHz 单声道 WAV，经既有 Works Speech Host API 转写并回填当前草稿。
|
||||
- 为账号、Workspace、Conversation 切换和组件卸载增加录音/异步转写生命周期清理与迟到结果保护；录音中始终保留停止按钮。
|
||||
- 增加组件回归测试和 AI Design Electron E2E 的 Enter 发送覆盖。
|
||||
|
||||
## Intent And Constraints
|
||||
|
||||
- 复用现有 `transcribeWorksSpeech` 与 Renderer → Host API → Main → Works 调用边界，不新增直连 IPC 或本地运行时 HTTP。
|
||||
- 不改变消息、Quote 确认、任务创建和多会话数据链路；Enter 仍由现有 `handleSend` 提交，Shift+Enter 和 IME 行为保持不变。
|
||||
- 录音期间即使发送或附件操作进入 busy，也必须可以停止录音并释放 MediaStream；账号、Workspace、Conversation 变化后不得使用旧录音结果回填新上下文。
|
||||
- 真实麦克风和 Works 账号的 Electron smoke 未纳入本次自动化验证。
|
||||
|
||||
## Plan
|
||||
|
||||
1. 核对 AI 设计 composer 的键盘处理、语音入口和 Works Speech 复用边界。
|
||||
2. 实现语音采集、WAV 转码、认证重试及上下文生命周期守卫，并补齐回归测试。
|
||||
3. 更新 Enter 交互 E2E 覆盖，运行聚焦/全量测试、类型检查、Lint、构建和文档漂移检查。
|
||||
|
||||
## Outcome
|
||||
|
||||
- AI 设计 composer 的 Enter 提交行为已由单测和 Electron E2E 覆盖；Shift+Enter 换行及 IME 组合期间不误发保持不变。
|
||||
- 语音输入按钮已接入 MediaRecorder；录音停止后生成 16 kHz、单声道、16-bit PCM WAV，调用 Works Speech 并将识别文本追加到当前草稿。
|
||||
- 语音流程支持未登录/环境不支持提示、401 强制刷新后单次重试，并在账号、Workspace、Conversation 变化或卸载时停止 recorder、释放 tracks、丢弃迟到结果。
|
||||
- 录音状态下停止按钮不受发送、参考图上传或素材操作 busy 状态禁用。
|
||||
|
||||
## Verification
|
||||
|
||||
- `node node_modules/vitest/vitest.mjs run tests/unit/image-canvas-page.test.tsx`：44 passed。
|
||||
- `node node_modules/vitest/vitest.mjs run`：176 files / 2114 tests passed。
|
||||
- `node node_modules/typescript/bin/tsc --noEmit`：通过。
|
||||
- `node node_modules/eslint/bin/eslint.js src/pages/ImageCanvas/index.tsx tests/unit/image-canvas-page.test.tsx tests/e2e/image-workspace-conversations.spec.ts`：通过。
|
||||
- `node --max-old-space-size=6144 node_modules/vite/bin/vite.js build`：Renderer、Electron Main、Preload 均构建通过；仅有既有动态导入和 chunk 体积警告。
|
||||
- `git diff --check`：通过。
|
||||
- `check_doc_drift.py --task-id 20260818-ai-design-input-9f3a`：通过。
|
||||
|
||||
## Follow-ups
|
||||
|
||||
- 需要真实麦克风权限和 Works 账号的 Electron smoke，确认不同浏览器/系统的 MediaRecorder 编码兼容性。
|
||||
- 音频转码 helper 目前与 AI 编程 composer 各自维护，后续可在独立任务中评估抽到共享 `src/lib` 工具。
|
||||
|
||||
## Promotion Candidates
|
||||
|
||||
- None。此次改动复用既有 Host API 和产品交互契约，不需要修改 canonical 架构或决策文档。
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Send,
|
||||
Square,
|
||||
Sparkles,
|
||||
Upload,
|
||||
WandSparkles,
|
||||
@@ -42,9 +44,11 @@ import {
|
||||
uploadImageWorkspaceAsset,
|
||||
} from '@/lib/image-workspace';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square';
|
||||
import { useImageWorkspaceStore } from '@/stores/image-workspace';
|
||||
import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum';
|
||||
import { useAuthStore } from '@/stores/auth';
|
||||
import { getProfileAccountKey } from '@/stores/user-profile';
|
||||
import logoSvg from '@/assets/logo.svg';
|
||||
import type {
|
||||
DesignAsset,
|
||||
@@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([
|
||||
]);
|
||||
const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片';
|
||||
const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24;
|
||||
const VOICE_WAV_SAMPLE_RATE = 16_000;
|
||||
const VOICE_WAV_CHANNELS = 1;
|
||||
const VOICE_WAV_BITS_PER_SAMPLE = 16;
|
||||
|
||||
type OfflineAudioContextConstructor = new (
|
||||
numberOfChannels: number,
|
||||
length: number,
|
||||
sampleRate: number,
|
||||
) => OfflineAudioContext;
|
||||
|
||||
type AudioContextWindow = Window & {
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
webkitOfflineAudioContext?: OfflineAudioContextConstructor;
|
||||
};
|
||||
|
||||
async function blobToBase64(blob: Blob): Promise<string> {
|
||||
const bytes = new Uint8Array(await blob.arrayBuffer());
|
||||
let binary = '';
|
||||
const chunkSize = 0x8000;
|
||||
for (let index = 0; index < bytes.length; index += chunkSize) {
|
||||
binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function writeAscii(view: DataView, offset: number, value: string): void {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
view.setUint8(offset + index, value.charCodeAt(index));
|
||||
}
|
||||
}
|
||||
|
||||
function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob {
|
||||
const samples = audioBuffer.getChannelData(0);
|
||||
const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8;
|
||||
const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample;
|
||||
const dataSize = samples.length * blockAlign;
|
||||
const wavBuffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(wavBuffer);
|
||||
|
||||
writeAscii(view, 0, 'RIFF');
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeAscii(view, 8, 'WAVE');
|
||||
writeAscii(view, 12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, VOICE_WAV_CHANNELS, true);
|
||||
view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true);
|
||||
view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true);
|
||||
writeAscii(view, 36, 'data');
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
let offset = 44;
|
||||
for (const sample of samples) {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
|
||||
offset += bytesPerSample;
|
||||
}
|
||||
return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' });
|
||||
}
|
||||
|
||||
async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise<Blob> {
|
||||
const audioWindow = window as AudioContextWindow;
|
||||
const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext;
|
||||
const OfflineAudioContextCtor = window.OfflineAudioContext
|
||||
?? audioWindow.webkitOfflineAudioContext;
|
||||
if (!AudioContextCtor || !OfflineAudioContextCtor) {
|
||||
throw new Error('当前环境无法转换录音');
|
||||
}
|
||||
|
||||
const audioContext = new AudioContextCtor();
|
||||
let decodedBuffer: AudioBuffer;
|
||||
try {
|
||||
decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer());
|
||||
} finally {
|
||||
await audioContext.close().catch(() => undefined);
|
||||
}
|
||||
const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE));
|
||||
const offlineContext = new OfflineAudioContextCtor(
|
||||
VOICE_WAV_CHANNELS,
|
||||
frameCount,
|
||||
VOICE_WAV_SAMPLE_RATE,
|
||||
);
|
||||
const source = offlineContext.createBufferSource();
|
||||
source.buffer = decodedBuffer;
|
||||
source.connect(offlineContext.destination);
|
||||
source.start(0);
|
||||
return audioBufferTo16BitPcmWav(await offlineContext.startRendering());
|
||||
}
|
||||
|
||||
function isUnauthorizedWorksError(error: unknown): boolean {
|
||||
if (error instanceof WorksSquareApiError && error.statusCode === 401) return true;
|
||||
if (!(error instanceof Error)) return false;
|
||||
const record = error as Error & { code?: unknown; details?: { status?: unknown } };
|
||||
const message = error.message.toLowerCase();
|
||||
return record.details?.status === 401
|
||||
|| record.code === 'AUTH_INVALID'
|
||||
|| message.includes('401')
|
||||
|| message.includes('unauthorized')
|
||||
|| message.includes('invalid access token');
|
||||
}
|
||||
|
||||
function currentAuthAccountKey(): string | null {
|
||||
const user = useAuthStore.getState().user;
|
||||
return getProfileAccountKey(user?.userId ?? user?.username);
|
||||
}
|
||||
|
||||
function isGenerationConfirmationIntent(message: string): boolean {
|
||||
const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, '');
|
||||
@@ -782,6 +893,12 @@ function QuoteCard({
|
||||
|
||||
export function ImageCanvas() {
|
||||
const authenticated = useAuthStore((state) => state.isAuthenticated());
|
||||
const accountKey = useAuthStore((state) => (
|
||||
getProfileAccountKey(state.user?.userId ?? state.user?.username)
|
||||
));
|
||||
const accessToken = useAuthStore((state) => state.accessToken);
|
||||
const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken);
|
||||
const refreshSession = useAuthStore((state) => state.refreshSession);
|
||||
const status = useImageWorkspaceStore((state) => state.status);
|
||||
const bootstrap = useImageWorkspaceStore((state) => state.bootstrap);
|
||||
const workspace = useImageWorkspaceStore((state) => state.workspace);
|
||||
@@ -817,12 +934,24 @@ export function ImageCanvas() {
|
||||
const [imageSourceBusyKey, setImageSourceBusyKey] = useState<string | null>(null);
|
||||
const [imageSourceError, setImageSourceError] = useState<string | null>(null);
|
||||
const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState<DesignAsset | null>(null);
|
||||
const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle');
|
||||
const conversationEndRef = useRef<HTMLDivElement | null>(null);
|
||||
const promptRef = useRef<HTMLTextAreaElement | null>(null);
|
||||
const referenceUploadInputRef = useRef<HTMLInputElement | null>(null);
|
||||
const promptSelectionRef = useRef({ start: 0, end: 0 });
|
||||
const quoteRepriceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const quoteDraftVersionRef = useRef(0);
|
||||
const voiceRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const voiceStreamRef = useRef<MediaStream | null>(null);
|
||||
const voiceChunksRef = useRef<Blob[]>([]);
|
||||
const voiceOperationRef = useRef(0);
|
||||
const voiceLifecycleRevisionRef = useRef(0);
|
||||
const renderedAccountKeyRef = useRef(accountKey);
|
||||
if (renderedAccountKeyRef.current !== accountKey) {
|
||||
renderedAccountKeyRef.current = accountKey;
|
||||
voiceLifecycleRevisionRef.current += 1;
|
||||
}
|
||||
const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||
|
||||
const quote = useMemo(
|
||||
() => conversation ? activeQuote(conversation.messages) : null,
|
||||
@@ -859,6 +988,13 @@ export function ImageCanvas() {
|
||||
});
|
||||
}, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]);
|
||||
const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0);
|
||||
const canUseVoiceInput = voiceInputState === 'recording' || (
|
||||
Boolean(workspace && conversation)
|
||||
&& !submittingConversationId
|
||||
&& !imageSourceBusyKey
|
||||
&& !referenceUploadBusy
|
||||
&& voiceInputState !== 'transcribing'
|
||||
);
|
||||
const selectingImageReference = conversation?.brief.medium === 'image';
|
||||
const conversationMessages = useMemo<RenderedDesignMessage[]>(() => {
|
||||
if (!conversation) return [];
|
||||
@@ -1148,6 +1284,184 @@ export function ImageCanvas() {
|
||||
}
|
||||
};
|
||||
|
||||
const stopVoiceStream = useCallback(() => {
|
||||
for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop();
|
||||
voiceStreamRef.current = null;
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
voiceOperationRef.current += 1;
|
||||
const recorder = voiceRecorderRef.current;
|
||||
if (recorder) {
|
||||
recorder.ondataavailable = null;
|
||||
recorder.onstop = null;
|
||||
if (recorder.state !== 'inactive') recorder.stop();
|
||||
voiceRecorderRef.current = null;
|
||||
}
|
||||
voiceChunksRef.current = [];
|
||||
stopVoiceStream();
|
||||
setVoiceInputState('idle');
|
||||
};
|
||||
}, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]);
|
||||
|
||||
const finishVoiceTranscription = useCallback(async (
|
||||
mimeType: string,
|
||||
operationId: number,
|
||||
requestedAccountKey: string,
|
||||
lifecycleRevision: number,
|
||||
requestedWorkspaceId: string,
|
||||
requestedConversationId: string,
|
||||
) => {
|
||||
const isActive = () => voiceOperationRef.current === operationId
|
||||
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||
&& currentAuthAccountKey() === requestedAccountKey
|
||||
&& useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId
|
||||
&& useImageWorkspaceStore.getState().activeConversationId === requestedConversationId;
|
||||
const chunks = voiceChunksRef.current;
|
||||
voiceChunksRef.current = [];
|
||||
stopVoiceStream();
|
||||
|
||||
if (chunks.length === 0) {
|
||||
if (isActive()) {
|
||||
setVoiceInputState('idle');
|
||||
toast.warning('没有录到声音');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isActive()) setVoiceInputState('transcribing');
|
||||
try {
|
||||
if (!isActive()) return;
|
||||
const validAccessToken = await getValidAccessToken();
|
||||
if (!isActive()) return;
|
||||
if (!validAccessToken) {
|
||||
toast.error('请先登录后再使用语音输入');
|
||||
return;
|
||||
}
|
||||
const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' });
|
||||
const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob);
|
||||
if (!isActive()) return;
|
||||
const audioBase64 = await blobToBase64(wavBlob);
|
||||
if (!isActive()) return;
|
||||
const transcribeWithToken = (token: string) => transcribeWorksSpeech({
|
||||
accessToken: token,
|
||||
audioBase64,
|
||||
fileName: 'voice.wav',
|
||||
mimeType: 'audio/wav',
|
||||
language: 'zh',
|
||||
});
|
||||
let transcription;
|
||||
try {
|
||||
if (!isActive()) return;
|
||||
transcription = await transcribeWithToken(validAccessToken);
|
||||
if (!isActive()) return;
|
||||
} catch (error) {
|
||||
if (!isUnauthorizedWorksError(error)) throw error;
|
||||
if (!isActive()) return;
|
||||
const refreshedAccessToken = await refreshSession();
|
||||
if (!isActive()) return;
|
||||
if (!refreshedAccessToken) throw error;
|
||||
transcription = await transcribeWithToken(refreshedAccessToken);
|
||||
if (!isActive()) return;
|
||||
}
|
||||
const transcript = transcription.text.trim();
|
||||
if (transcript) {
|
||||
setPrompt((currentPrompt) => currentPrompt.trim()
|
||||
? `${currentPrompt.trim()}\n${transcript}`
|
||||
: transcript);
|
||||
setActionError(null);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isActive()) {
|
||||
toast.error('语音识别失败', {
|
||||
description: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
if (isActive()) {
|
||||
voiceRecorderRef.current = null;
|
||||
setVoiceInputState('idle');
|
||||
}
|
||||
}
|
||||
}, [getValidAccessToken, refreshSession, stopVoiceStream]);
|
||||
|
||||
const handleVoiceInputClick = useCallback(async () => {
|
||||
if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision
|
||||
|| renderedAccountKeyRef.current !== accountKey) return;
|
||||
if (voiceInputState === 'recording') {
|
||||
const recorder = voiceRecorderRef.current;
|
||||
if (!recorder || recorder.state === 'inactive') {
|
||||
stopVoiceStream();
|
||||
setVoiceInputState('idle');
|
||||
return;
|
||||
}
|
||||
setVoiceInputState('transcribing');
|
||||
recorder.stop();
|
||||
return;
|
||||
}
|
||||
if (!canUseVoiceInput || !workspace || !conversation) return;
|
||||
if (!accountKey || !accessToken) {
|
||||
toast.error('请先登录后再使用语音输入');
|
||||
return;
|
||||
}
|
||||
if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') {
|
||||
toast.error('当前环境不支持语音输入');
|
||||
return;
|
||||
}
|
||||
|
||||
const operationId = voiceOperationRef.current + 1;
|
||||
voiceOperationRef.current = operationId;
|
||||
const lifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||
const requestedAccountKey = accountKey;
|
||||
const isActive = () => voiceOperationRef.current === operationId
|
||||
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||
&& currentAuthAccountKey() === requestedAccountKey;
|
||||
const requestedWorkspaceId = workspace.workspaceId;
|
||||
const requestedConversationId = conversation.conversationId;
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
if (!isActive()) {
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
return;
|
||||
}
|
||||
const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm')
|
||||
? 'audio/webm'
|
||||
: '';
|
||||
const recorder = new window.MediaRecorder(
|
||||
stream,
|
||||
preferredMimeType ? { mimeType: preferredMimeType } : undefined,
|
||||
);
|
||||
voiceStreamRef.current = stream;
|
||||
voiceChunksRef.current = [];
|
||||
voiceRecorderRef.current = recorder;
|
||||
recorder.ondataavailable = (event) => {
|
||||
if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data);
|
||||
};
|
||||
recorder.onstop = () => {
|
||||
void finishVoiceTranscription(
|
||||
recorder.mimeType || preferredMimeType || 'audio/webm',
|
||||
operationId,
|
||||
requestedAccountKey,
|
||||
lifecycleRevision,
|
||||
requestedWorkspaceId,
|
||||
requestedConversationId,
|
||||
);
|
||||
};
|
||||
recorder.start();
|
||||
setVoiceInputState('recording');
|
||||
} catch (error) {
|
||||
stopVoiceStream();
|
||||
if (isActive()) {
|
||||
setVoiceInputState('idle');
|
||||
toast.error('无法开始录音', {
|
||||
description: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]);
|
||||
|
||||
const handleSend = async (override?: string) => {
|
||||
const message = (override ?? prompt).trim();
|
||||
if (!workspace || !conversation || !message || submitting) return;
|
||||
@@ -1605,12 +1919,27 @@ export function ImageCanvas() {
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
disabled
|
||||
className="motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none"
|
||||
aria-label="语音输入"
|
||||
title="AI 绘画暂不支持语音输入"
|
||||
disabled={!canUseVoiceInput}
|
||||
className={cn(
|
||||
'motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none hover:bg-surface-subtle hover:text-foreground',
|
||||
voiceInputState !== 'idle' && 'bg-surface-subtle text-foreground',
|
||||
)}
|
||||
aria-label={voiceInputState === 'recording'
|
||||
? '停止语音输入'
|
||||
: voiceInputState === 'transcribing'
|
||||
? '正在识别语音'
|
||||
: '语音输入'}
|
||||
title={typeof window.MediaRecorder === 'undefined'
|
||||
|| !navigator.mediaDevices?.getUserMedia
|
||||
? '当前环境不支持语音输入'
|
||||
: '语音输入'}
|
||||
onClick={() => void handleVoiceInputClick()}
|
||||
>
|
||||
<Mic className="h-4 w-4" />
|
||||
{voiceInputState === 'transcribing'
|
||||
? <Loader2 className="h-4 w-4 animate-spin" />
|
||||
: voiceInputState === 'recording'
|
||||
? <Square className="h-4 w-4 fill-current" />
|
||||
: <Mic className="h-4 w-4" />}
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
|
||||
@@ -81,7 +81,7 @@ test.describe('AI Design workspace', () => {
|
||||
|
||||
const firstPrompt = '为项目设计一张蓝色星空海报';
|
||||
await page.getByLabel('设计需求').fill(firstPrompt);
|
||||
await page.getByRole('button', { name: '发送给设计 Agent' }).click();
|
||||
await page.getByLabel('设计需求').press('Enter');
|
||||
await expect(page.getByTestId('image-workspace-conversation')).toContainText(firstPrompt);
|
||||
await expect(page.getByText('设计 Agent', { exact: true })).toHaveCount(0);
|
||||
await expect(page.getByText('你', { exact: true })).toHaveCount(0);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { toast } from 'sonner';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ImageWorkspaceApiError } from '@/lib/image-workspace';
|
||||
import { ImageCanvas } from '@/pages/ImageCanvas';
|
||||
@@ -27,6 +28,9 @@ const openImageWorkspaceTaskEventsMock = vi.hoisted(() => vi.fn());
|
||||
const resolveImageWorkspaceAssetUrlMock = vi.hoisted(() => vi.fn());
|
||||
const saveImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const uploadImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const transcribeWorksSpeechMock = vi.hoisted(() => vi.fn());
|
||||
const originalGetValidAccessToken = useAuthStore.getState().getValidAccessToken;
|
||||
const originalRefreshSession = useAuthStore.getState().refreshSession;
|
||||
|
||||
type EventListener = (event: MessageEvent<string>) => void;
|
||||
|
||||
@@ -88,6 +92,42 @@ vi.mock('@/lib/image-workspace', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('@/lib/works-square', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@/lib/works-square')>();
|
||||
return {
|
||||
...actual,
|
||||
transcribeWorksSpeech: (...args: unknown[]) => transcribeWorksSpeechMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
function installMockVoiceWavTranscoder(): void {
|
||||
const pcm = Float32Array.from([0, 0.5, -0.5, 1, -1]);
|
||||
class MockAudioContext {
|
||||
decodeAudioData = vi.fn().mockResolvedValue({
|
||||
duration: pcm.length / 48_000,
|
||||
getChannelData: () => pcm,
|
||||
});
|
||||
close = vi.fn().mockResolvedValue(undefined);
|
||||
}
|
||||
class MockOfflineAudioContext {
|
||||
destination = {};
|
||||
createBufferSource = vi.fn(() => ({
|
||||
buffer: null,
|
||||
connect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
}));
|
||||
startRendering = vi.fn().mockResolvedValue({ getChannelData: () => pcm });
|
||||
}
|
||||
Object.defineProperty(window, 'AudioContext', {
|
||||
configurable: true,
|
||||
value: MockAudioContext,
|
||||
});
|
||||
Object.defineProperty(window, 'OfflineAudioContext', {
|
||||
configurable: true,
|
||||
value: MockOfflineAudioContext,
|
||||
});
|
||||
}
|
||||
|
||||
const bootstrapFixture: DesignWorkspaceBootstrap = {
|
||||
capabilities: {
|
||||
conversation: true,
|
||||
@@ -243,6 +283,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useAuthStore.setState({
|
||||
getValidAccessToken: originalGetValidAccessToken,
|
||||
refreshSession: originalRefreshSession,
|
||||
});
|
||||
useImageWorkspaceStore.getState().reset();
|
||||
useImagePromptMuseumStore.getState().clearPendingPrompt();
|
||||
fetchImageWorkspaceMock.mockResolvedValue(bootstrapFixture);
|
||||
@@ -283,6 +327,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
...taskFixture.resultAssets[0],
|
||||
assetId: 'asset-uploaded',
|
||||
});
|
||||
transcribeWorksSpeechMock.mockResolvedValue({
|
||||
text: '让天空更通透',
|
||||
model: 'gpt-4o-mini-transcribe',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows an honest unavailable state without fabricating projects', async () => {
|
||||
@@ -754,7 +802,7 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(screen.getByTestId('image-workspace-composer-surface'))
|
||||
.toHaveClass('rounded-lg', 'flex', 'min-w-0');
|
||||
expect(screen.getByRole('button', { name: '上传参考图' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled();
|
||||
expect(screen.queryByText('Enter 发送 · Shift+Enter 换行')).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '上传参考图' }));
|
||||
@@ -831,6 +879,316 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(composer).toHaveValue('正在输入中文设计描述');
|
||||
});
|
||||
|
||||
it('reports fixed login and environment guards before starting voice input', async () => {
|
||||
const toastError = vi.spyOn(toast, 'error').mockImplementation(() => 'toast-id');
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: undefined,
|
||||
});
|
||||
useAuthStore.setState({ accessToken: null, canRefresh: false });
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
expect(toastError).toHaveBeenLastCalledWith('请先登录后再使用语音输入');
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
});
|
||||
});
|
||||
const voiceButton = screen.getByRole('button', { name: '语音输入' });
|
||||
expect(voiceButton).toHaveAttribute('title', '当前环境不支持语音输入');
|
||||
fireEvent.click(voiceButton);
|
||||
expect(toastError).toHaveBeenLastCalledWith('当前环境不支持语音输入');
|
||||
toastError.mockRestore();
|
||||
});
|
||||
|
||||
it('records Works Speech input into the current draft and releases the microphone', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const pendingSend = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(pendingSend.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
initialized: true,
|
||||
loading: false,
|
||||
error: null,
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
legacyRefreshToken: null,
|
||||
user: {
|
||||
username: 'designer',
|
||||
userId: '42',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
const composer = screen.getByLabelText('设计需求');
|
||||
fireEvent.change(composer, { target: { value: '保留当前构图' } });
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
|
||||
await waitFor(() => expect(getUserMedia).toHaveBeenCalledWith({ audio: true }));
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '发送给设计 Agent' }));
|
||||
await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce());
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
|
||||
await waitFor(() => expect(composer).toHaveValue('让天空更通透'));
|
||||
expect(transcribeWorksSpeechMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||
accessToken: 'access-token',
|
||||
fileName: 'voice.wav',
|
||||
mimeType: 'audio/wav',
|
||||
language: 'zh',
|
||||
}));
|
||||
const audioBase64 = transcribeWorksSpeechMock.mock.calls[0]?.[0]?.audioBase64 as string;
|
||||
const wavBytes = Uint8Array.from(atob(audioBase64), (character) => character.charCodeAt(0));
|
||||
const wavView = new DataView(wavBytes.buffer);
|
||||
expect(new TextDecoder().decode(wavBytes.slice(0, 4))).toBe('RIFF');
|
||||
expect(wavView.getUint16(22, true)).toBe(1);
|
||||
expect(wavView.getUint32(24, true)).toBe(16_000);
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce();
|
||||
expect(MockMediaRecorder.latest?.state).toBe('inactive');
|
||||
await act(async () => {
|
||||
pendingSend.resolve(conversationFixture());
|
||||
await pendingSend.promise;
|
||||
});
|
||||
});
|
||||
|
||||
it('discards a late transcription after Conversation switch', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const transcription = deferred<{ text: string; model: string }>();
|
||||
transcribeWorksSpeechMock.mockReturnValueOnce(transcription.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(transcribeWorksSpeechMock).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
await act(async () => {
|
||||
transcription.resolve({ text: '不应写入新会话', model: 'gpt-4o-mini-transcribe' });
|
||||
await transcription.promise;
|
||||
});
|
||||
await waitFor(() => expect(screen.getByLabelText('设计需求')).toHaveValue(''));
|
||||
});
|
||||
|
||||
it('stops an active recorder and releases its stream when Conversation changes', async () => {
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
readonly stop = vi.fn(() => {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['late audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
});
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
expect(MockMediaRecorder.latest?.state).toBe('recording');
|
||||
expect(stopTrack).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await waitFor(() => expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled());
|
||||
expect(MockMediaRecorder.latest?.stop).toHaveBeenCalledOnce();
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('does not use a switched account token for an earlier voice recording', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const tokenLookup = deferred<string>();
|
||||
const getValidAccessToken = vi.fn().mockReturnValue(tokenLookup.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['old account audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'old-account-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
getValidAccessToken,
|
||||
user: {
|
||||
username: 'old-account',
|
||||
userId: 'old-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(getValidAccessToken).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'new-account-token',
|
||||
user: {
|
||||
username: 'new-account',
|
||||
userId: 'new-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await act(async () => {
|
||||
tokenLookup.resolve('new-account-token');
|
||||
await tokenLookup.promise;
|
||||
});
|
||||
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('renders the pending user turn and assistant deltas while the command is running', async () => {
|
||||
const response = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(response.promise);
|
||||
|
||||
Reference in New Issue
Block a user