From 1ba5b9cc84347bf37cf8a18148a1dcc6b88b3ad2 Mon Sep 17 00:00:00 2001 From: brother7 <7brother7@gmail.com> Date: Tue, 18 Aug 2026 02:30:39 +0800 Subject: [PATCH] feat: enable AI design voice input --- .../tasks/20260818-ai-design-input-9f3a.md | 57 +++ src/pages/ImageCanvas/index.tsx | 339 ++++++++++++++++- .../e2e/image-workspace-conversations.spec.ts | 2 +- tests/unit/image-canvas-page.test.tsx | 360 +++++++++++++++++- 4 files changed, 751 insertions(+), 7 deletions(-) create mode 100644 .project-docs/30-worklog/tasks/20260818-ai-design-input-9f3a.md diff --git a/.project-docs/30-worklog/tasks/20260818-ai-design-input-9f3a.md b/.project-docs/30-worklog/tasks/20260818-ai-design-input-9f3a.md new file mode 100644 index 0000000..3e884e3 --- /dev/null +++ b/.project-docs/30-worklog/tasks/20260818-ai-design-input-9f3a.md @@ -0,0 +1,57 @@ +# Task: Fix AI design dialog Enter and voice input + +## Identity + +- Task ID: 20260818-ai-design-input-9f3a +- Mode: Feature +- Branch: codex/20260818-ai-design-input-9f3a-ai-design-input +- Worktree: D:\Datas\OthersProjects\makelore-ai-design-input-9f3a +- Base commit: 13bdc0c9bab656aa9ff6c990bf6359f7d3210750 +- Owner: developer +- Status: Completed + +## Scope + +- 保留并验收 AI 设计对话框现有的 Enter 发送、Shift+Enter 换行和输入法组合保护。 +- 打开 AI 设计对话框的语音输入：采集麦克风、转为 16 kHz 单声道 WAV，经既有 Works Speech Host API 转写并回填当前草稿。 +- 为账号、Workspace、Conversation 切换和组件卸载增加录音/异步转写生命周期清理与迟到结果保护；录音中始终保留停止按钮。 +- 增加组件回归测试和 AI Design Electron E2E 的 Enter 发送覆盖。 + +## Intent And Constraints + +- 复用现有 `transcribeWorksSpeech` 与 Renderer → Host API → Main → Works 调用边界，不新增直连 IPC 或本地运行时 HTTP。 +- 不改变消息、Quote 确认、任务创建和多会话数据链路；Enter 仍由现有 `handleSend` 提交，Shift+Enter 和 IME 行为保持不变。 +- 录音期间即使发送或附件操作进入 busy，也必须可以停止录音并释放 MediaStream；账号、Workspace、Conversation 变化后不得使用旧录音结果回填新上下文。 +- 真实麦克风和 Works 账号的 Electron smoke 未纳入本次自动化验证。 + +## Plan + +1. 核对 AI 设计 composer 的键盘处理、语音入口和 Works Speech 复用边界。 +2. 实现语音采集、WAV 转码、认证重试及上下文生命周期守卫，并补齐回归测试。 +3. 更新 Enter 交互 E2E 覆盖，运行聚焦/全量测试、类型检查、Lint、构建和文档漂移检查。 + +## Outcome + +- AI 设计 composer 的 Enter 提交行为已由单测和 Electron E2E 覆盖；Shift+Enter 换行及 IME 组合期间不误发保持不变。 +- 语音输入按钮已接入 MediaRecorder；录音停止后生成 16 kHz、单声道、16-bit PCM WAV，调用 Works Speech 并将识别文本追加到当前草稿。 +- 语音流程支持未登录/环境不支持提示、401 强制刷新后单次重试，并在账号、Workspace、Conversation 变化或卸载时停止 recorder、释放 tracks、丢弃迟到结果。 +- 录音状态下停止按钮不受发送、参考图上传或素材操作 busy 状态禁用。 + +## Verification + +- `node node_modules/vitest/vitest.mjs run tests/unit/image-canvas-page.test.tsx`：44 passed。 +- `node node_modules/vitest/vitest.mjs run`：176 files / 2114 tests passed。 +- `node node_modules/typescript/bin/tsc --noEmit`：通过。 +- `node node_modules/eslint/bin/eslint.js src/pages/ImageCanvas/index.tsx tests/unit/image-canvas-page.test.tsx tests/e2e/image-workspace-conversations.spec.ts`：通过。 +- `node --max-old-space-size=6144 node_modules/vite/bin/vite.js build`：Renderer、Electron Main、Preload 均构建通过；仅有既有动态导入和 chunk 体积警告。 +- `git diff --check`：通过。 +- `check_doc_drift.py --task-id 20260818-ai-design-input-9f3a`：通过。 + +## Follow-ups + +- 需要真实麦克风权限和 Works 账号的 Electron smoke，确认不同浏览器/系统的 MediaRecorder 编码兼容性。 +- 音频转码 helper 目前与 AI 编程 composer 各自维护，后续可在独立任务中评估抽到共享 `src/lib` 工具。 + +## Promotion Candidates + +- None。此次改动复用既有 Host API 和产品交互契约，不需要修改 canonical 架构或决策文档。 diff --git a/src/pages/ImageCanvas/index.tsx b/src/pages/ImageCanvas/index.tsx index add615e..cdb4992 100644 --- a/src/pages/ImageCanvas/index.tsx +++ b/src/pages/ImageCanvas/index.tsx @@ -1,4 +1,5 @@ import { + useCallback, useEffect, useMemo, useRef, @@ -18,6 +19,7 @@ import { Plus, RefreshCw, Send, + Square, Sparkles, Upload, WandSparkles, @@ -42,9 +44,11 @@ import { uploadImageWorkspaceAsset, } from '@/lib/image-workspace'; import { cn } from '@/lib/utils'; +import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square'; import { useImageWorkspaceStore } from '@/stores/image-workspace'; import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum'; import { useAuthStore } from '@/stores/auth'; +import { getProfileAccountKey } from '@/stores/user-profile'; import logoSvg from '@/assets/logo.svg'; import type { DesignAsset, @@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([ ]); const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片'; const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24; +const VOICE_WAV_SAMPLE_RATE = 16_000; +const VOICE_WAV_CHANNELS = 1; +const VOICE_WAV_BITS_PER_SAMPLE = 16; + +type OfflineAudioContextConstructor = new ( + numberOfChannels: number, + length: number, + sampleRate: number, +) => OfflineAudioContext; + +type AudioContextWindow = Window & { + webkitAudioContext?: typeof AudioContext; + webkitOfflineAudioContext?: OfflineAudioContextConstructor; +}; + +async function blobToBase64(blob: Blob): Promise { + const bytes = new Uint8Array(await blob.arrayBuffer()); + let binary = ''; + const chunkSize = 0x8000; + for (let index = 0; index < bytes.length; index += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize)); + } + return btoa(binary); +} + +function writeAscii(view: DataView, offset: number, value: string): void { + for (let index = 0; index < value.length; index += 1) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} + +function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob { + const samples = audioBuffer.getChannelData(0); + const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8; + const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample; + const dataSize = samples.length * blockAlign; + const wavBuffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(wavBuffer); + + writeAscii(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeAscii(view, 8, 'WAVE'); + writeAscii(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, VOICE_WAV_CHANNELS, true); + view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true); + view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true); + writeAscii(view, 36, 'data'); + view.setUint32(40, dataSize, true); + + let offset = 44; + for (const sample of samples) { + const clamped = Math.max(-1, Math.min(1, sample)); + view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true); + offset += bytesPerSample; + } + return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' }); +} + +async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise { + const audioWindow = window as AudioContextWindow; + const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext; + const OfflineAudioContextCtor = window.OfflineAudioContext + ?? audioWindow.webkitOfflineAudioContext; + if (!AudioContextCtor || !OfflineAudioContextCtor) { + throw new Error('当前环境无法转换录音'); + } + + const audioContext = new AudioContextCtor(); + let decodedBuffer: AudioBuffer; + try { + decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer()); + } finally { + await audioContext.close().catch(() => undefined); + } + const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE)); + const offlineContext = new OfflineAudioContextCtor( + VOICE_WAV_CHANNELS, + frameCount, + VOICE_WAV_SAMPLE_RATE, + ); + const source = offlineContext.createBufferSource(); + source.buffer = decodedBuffer; + source.connect(offlineContext.destination); + source.start(0); + return audioBufferTo16BitPcmWav(await offlineContext.startRendering()); +} + +function isUnauthorizedWorksError(error: unknown): boolean { + if (error instanceof WorksSquareApiError && error.statusCode === 401) return true; + if (!(error instanceof Error)) return false; + const record = error as Error & { code?: unknown; details?: { status?: unknown } }; + const message = error.message.toLowerCase(); + return record.details?.status === 401 + || record.code === 'AUTH_INVALID' + || message.includes('401') + || message.includes('unauthorized') + || message.includes('invalid access token'); +} + +function currentAuthAccountKey(): string | null { + const user = useAuthStore.getState().user; + return getProfileAccountKey(user?.userId ?? user?.username); +} function isGenerationConfirmationIntent(message: string): boolean { const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, ''); @@ -782,6 +893,12 @@ function QuoteCard({ export function ImageCanvas() { const authenticated = useAuthStore((state) => state.isAuthenticated()); + const accountKey = useAuthStore((state) => ( + getProfileAccountKey(state.user?.userId ?? state.user?.username) + )); + const accessToken = useAuthStore((state) => state.accessToken); + const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken); + const refreshSession = useAuthStore((state) => state.refreshSession); const status = useImageWorkspaceStore((state) => state.status); const bootstrap = useImageWorkspaceStore((state) => state.bootstrap); const workspace = useImageWorkspaceStore((state) => state.workspace); @@ -817,12 +934,24 @@ export function ImageCanvas() { const [imageSourceBusyKey, setImageSourceBusyKey] = useState(null); const [imageSourceError, setImageSourceError] = useState(null); const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState(null); + const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle'); const conversationEndRef = useRef(null); const promptRef = useRef(null); const referenceUploadInputRef = useRef(null); const promptSelectionRef = useRef({ start: 0, end: 0 }); const quoteRepriceTimerRef = useRef | null>(null); const quoteDraftVersionRef = useRef(0); + const voiceRecorderRef = useRef(null); + const voiceStreamRef = useRef(null); + const voiceChunksRef = useRef([]); + const voiceOperationRef = useRef(0); + const voiceLifecycleRevisionRef = useRef(0); + const renderedAccountKeyRef = useRef(accountKey); + if (renderedAccountKeyRef.current !== accountKey) { + renderedAccountKeyRef.current = accountKey; + voiceLifecycleRevisionRef.current += 1; + } + const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current; const quote = useMemo( () => conversation ? activeQuote(conversation.messages) : null, @@ -859,6 +988,13 @@ export function ImageCanvas() { }); }, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]); const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0); + const canUseVoiceInput = voiceInputState === 'recording' || ( + Boolean(workspace && conversation) + && !submittingConversationId + && !imageSourceBusyKey + && !referenceUploadBusy + && voiceInputState !== 'transcribing' + ); const selectingImageReference = conversation?.brief.medium === 'image'; const conversationMessages = useMemo(() => { if (!conversation) return []; @@ -1148,6 +1284,184 @@ export function ImageCanvas() { } }; + const stopVoiceStream = useCallback(() => { + for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop(); + voiceStreamRef.current = null; + }, []); + + useEffect(() => { + return () => { + voiceOperationRef.current += 1; + const recorder = voiceRecorderRef.current; + if (recorder) { + recorder.ondataavailable = null; + recorder.onstop = null; + if (recorder.state !== 'inactive') recorder.stop(); + voiceRecorderRef.current = null; + } + voiceChunksRef.current = []; + stopVoiceStream(); + setVoiceInputState('idle'); + }; + }, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]); + + const finishVoiceTranscription = useCallback(async ( + mimeType: string, + operationId: number, + requestedAccountKey: string, + lifecycleRevision: number, + requestedWorkspaceId: string, + requestedConversationId: string, + ) => { + const isActive = () => voiceOperationRef.current === operationId + && voiceLifecycleRevisionRef.current === lifecycleRevision + && renderedAccountKeyRef.current === requestedAccountKey + && currentAuthAccountKey() === requestedAccountKey + && useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId + && useImageWorkspaceStore.getState().activeConversationId === requestedConversationId; + const chunks = voiceChunksRef.current; + voiceChunksRef.current = []; + stopVoiceStream(); + + if (chunks.length === 0) { + if (isActive()) { + setVoiceInputState('idle'); + toast.warning('没有录到声音'); + } + return; + } + if (isActive()) setVoiceInputState('transcribing'); + try { + if (!isActive()) return; + const validAccessToken = await getValidAccessToken(); + if (!isActive()) return; + if (!validAccessToken) { + toast.error('请先登录后再使用语音输入'); + return; + } + const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' }); + const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob); + if (!isActive()) return; + const audioBase64 = await blobToBase64(wavBlob); + if (!isActive()) return; + const transcribeWithToken = (token: string) => transcribeWorksSpeech({ + accessToken: token, + audioBase64, + fileName: 'voice.wav', + mimeType: 'audio/wav', + language: 'zh', + }); + let transcription; + try { + if (!isActive()) return; + transcription = await transcribeWithToken(validAccessToken); + if (!isActive()) return; + } catch (error) { + if (!isUnauthorizedWorksError(error)) throw error; + if (!isActive()) return; + const refreshedAccessToken = await refreshSession(); + if (!isActive()) return; + if (!refreshedAccessToken) throw error; + transcription = await transcribeWithToken(refreshedAccessToken); + if (!isActive()) return; + } + const transcript = transcription.text.trim(); + if (transcript) { + setPrompt((currentPrompt) => currentPrompt.trim() + ? `${currentPrompt.trim()}\n${transcript}` + : transcript); + setActionError(null); + } + } catch (error) { + if (isActive()) { + toast.error('语音识别失败', { + description: error instanceof Error ? error.message : String(error), + }); + } + } finally { + if (isActive()) { + voiceRecorderRef.current = null; + setVoiceInputState('idle'); + } + } + }, [getValidAccessToken, refreshSession, stopVoiceStream]); + + const handleVoiceInputClick = useCallback(async () => { + if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision + || renderedAccountKeyRef.current !== accountKey) return; + if (voiceInputState === 'recording') { + const recorder = voiceRecorderRef.current; + if (!recorder || recorder.state === 'inactive') { + stopVoiceStream(); + setVoiceInputState('idle'); + return; + } + setVoiceInputState('transcribing'); + recorder.stop(); + return; + } + if (!canUseVoiceInput || !workspace || !conversation) return; + if (!accountKey || !accessToken) { + toast.error('请先登录后再使用语音输入'); + return; + } + if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') { + toast.error('当前环境不支持语音输入'); + return; + } + + const operationId = voiceOperationRef.current + 1; + voiceOperationRef.current = operationId; + const lifecycleRevision = voiceLifecycleRevisionRef.current; + const requestedAccountKey = accountKey; + const isActive = () => voiceOperationRef.current === operationId + && voiceLifecycleRevisionRef.current === lifecycleRevision + && renderedAccountKeyRef.current === requestedAccountKey + && currentAuthAccountKey() === requestedAccountKey; + const requestedWorkspaceId = workspace.workspaceId; + const requestedConversationId = conversation.conversationId; + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (!isActive()) { + for (const track of stream.getTracks()) track.stop(); + return; + } + const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm') + ? 'audio/webm' + : ''; + const recorder = new window.MediaRecorder( + stream, + preferredMimeType ? { mimeType: preferredMimeType } : undefined, + ); + voiceStreamRef.current = stream; + voiceChunksRef.current = []; + voiceRecorderRef.current = recorder; + recorder.ondataavailable = (event) => { + if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data); + }; + recorder.onstop = () => { + void finishVoiceTranscription( + recorder.mimeType || preferredMimeType || 'audio/webm', + operationId, + requestedAccountKey, + lifecycleRevision, + requestedWorkspaceId, + requestedConversationId, + ); + }; + recorder.start(); + setVoiceInputState('recording'); + } catch (error) { + stopVoiceStream(); + if (isActive()) { + setVoiceInputState('idle'); + toast.error('无法开始录音', { + description: error instanceof Error ? error.message : String(error), + }); + } + } + }, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]); + const handleSend = async (override?: string) => { const message = (override ?? prompt).trim(); if (!workspace || !conversation || !message || submitting) return; @@ -1605,12 +1919,27 @@ export function ImageCanvas() {