diff --git a/.project-docs/30-worklog/tasks/20260813-sync-push-main-9c2f71.md b/.project-docs/30-worklog/tasks/20260813-sync-push-main-9c2f71.md index 550f278..40d2319 100644 --- a/.project-docs/30-worklog/tasks/20260813-sync-push-main-9c2f71.md +++ b/.project-docs/30-worklog/tasks/20260813-sync-push-main-9c2f71.md @@ -466,6 +466,21 @@ Gate result: - This resumption intentionally performs a local merge only. It does not fetch or push, and the existing remote-authentication follow-up for this long-running integration task remains unchanged. - Created no-ff merge commit `4ff1a7a8461c01de4d1630a3db55b0942fc37ec7` with `3b4669722e0ea671909beb039da1e07c2483341a` as first parent and reviewed source `87a95b4cecf20fce7912305d2e6e2c063ddcbae4` as second parent. +### 2026-08-18 AI Design Enter/Voice Input Integration Resume + +- Reused this existing Integration owner after the user's explicit request to merge the reviewed AI Design input task. The feature task `20260818-ai-design-input-9f3a` is `ready_for_integration`; its source worktree is clean after forming commit `1ba5b9cc84347bf37cf8a18148a1dcc6b88b3ad2`, based on `13bdc0c`. The parallel video-download merge advanced local `main` to `31fb4ff` before this merge was completed. +- Started a normal `--no-ff --no-commit` merge of `1ba5b9c` onto `31fb4ff`; Git reported no textual conflicts. The feature task record remains on its source branch and is deliberately excluded from the integrated `main` tree to preserve task-document ownership boundaries. +- The merge keeps AI Design's Enter/Shift+Enter/IME composer contract and E2E Enter coverage, enables Works Speech voice input with 16 kHz mono WAV conversion, authentication retry, context-lifecycle cleanup, and an always-available stop control while other composer operations are busy. +- No canonical architecture, domain-rule, or decision promotion is required; the source task records no promotion candidate. Real microphone/Works-account smoke remains an external release validation item. +- This resumption is local-only. It does not fetch or push the remote; the existing Git credential follow-up remains unchanged. + +### 2026-08-18 AI Design Video Download Extension Integration Resume + +- Reused this existing Integration owner after the user's explicit request to merge the reviewed AI Design video-download fix. The feature task `20260818-video-download-extension-a4f1` is `ready_for_integration`; its source commit is `43e8a60`, based on the local `main` tip `13bdc0c`. +- Created normal no-ff merge commit `31fb4ff`, with `13bdc0c` as first parent and `43e8a60` as second parent. The source task record remains on its feature worktree and is excluded from `main`. +- No canonical architecture, domain-rule, or decision promotion is required. The merge is local-only and does not fetch or push the remote. +- The merge preserves the separately staged AI Design input task changes already present in the integration worktree; those unrelated changes were not included in `31fb4ff` and were not cleaned or overwritten. + ## Verification - Source closeout before integration: Main focused 164/164, full unit 176 files / 2100 tests, typecheck, full lint with zero errors and seven pre-existing warnings, `build:vite`, focused Electron E2E 1/1, project-document gates and seventh independent Sol Standards/Spec review all passed. @@ -625,6 +640,10 @@ Gate result: - Post-review Electron E2E selection — 3/3 passed. - Final context-compaction integration re-review — `PASS`; no remaining P0-P3 findings. The reviewer confirmed polling-idle completion precedes queue release, cold hydration requires a matching current running identity, `main` is clean, and merge/fix/documentation topology is correct. +- 2026-08-18 video-download merged-main route regression — 1 file / 14 tests passed; `.mp4`, `.mov`, and `.webm` preserve their extensions and non-media MIME responses remain rejected. +- 2026-08-18 video-download merged-main focused selection — 3 files / 72 tests passed; typecheck and scoped ESLint for the route/test files passed. +- Merge topology and `git diff --check` passed for `31fb4ff`; the source task record is absent from `main`. The integration worktree retains unrelated staged AI Design input changes, so no clean-worktree claim is made. + ## Follow-ups - Before claiming true simultaneous model execution, run the opt-in real bundled OpenCode two-Session smoke with an explicitly configured test provider. Current automation proves application isolation, bounded failure and no replay, not provider/runtime concurrency. diff --git a/src/pages/ImageCanvas/index.tsx b/src/pages/ImageCanvas/index.tsx index add615e..cdb4992 100644 --- a/src/pages/ImageCanvas/index.tsx +++ b/src/pages/ImageCanvas/index.tsx @@ -1,4 +1,5 @@ import { + useCallback, useEffect, useMemo, useRef, @@ -18,6 +19,7 @@ import { Plus, RefreshCw, Send, + Square, Sparkles, Upload, WandSparkles, @@ -42,9 +44,11 @@ import { uploadImageWorkspaceAsset, } from '@/lib/image-workspace'; import { cn } from '@/lib/utils'; +import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square'; import { useImageWorkspaceStore } from '@/stores/image-workspace'; import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum'; import { useAuthStore } from '@/stores/auth'; +import { getProfileAccountKey } from '@/stores/user-profile'; import logoSvg from '@/assets/logo.svg'; import type { DesignAsset, @@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([ ]); const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片'; const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24; +const VOICE_WAV_SAMPLE_RATE = 16_000; +const VOICE_WAV_CHANNELS = 1; +const VOICE_WAV_BITS_PER_SAMPLE = 16; + +type OfflineAudioContextConstructor = new ( + numberOfChannels: number, + length: number, + sampleRate: number, +) => OfflineAudioContext; + +type AudioContextWindow = Window & { + webkitAudioContext?: typeof AudioContext; + webkitOfflineAudioContext?: OfflineAudioContextConstructor; +}; + +async function blobToBase64(blob: Blob): Promise { + const bytes = new Uint8Array(await blob.arrayBuffer()); + let binary = ''; + const chunkSize = 0x8000; + for (let index = 0; index < bytes.length; index += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize)); + } + return btoa(binary); +} + +function writeAscii(view: DataView, offset: number, value: string): void { + for (let index = 0; index < value.length; index += 1) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} + +function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob { + const samples = audioBuffer.getChannelData(0); + const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8; + const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample; + const dataSize = samples.length * blockAlign; + const wavBuffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(wavBuffer); + + writeAscii(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeAscii(view, 8, 'WAVE'); + writeAscii(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, VOICE_WAV_CHANNELS, true); + view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true); + view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true); + writeAscii(view, 36, 'data'); + view.setUint32(40, dataSize, true); + + let offset = 44; + for (const sample of samples) { + const clamped = Math.max(-1, Math.min(1, sample)); + view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true); + offset += bytesPerSample; + } + return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' }); +} + +async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise { + const audioWindow = window as AudioContextWindow; + const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext; + const OfflineAudioContextCtor = window.OfflineAudioContext + ?? audioWindow.webkitOfflineAudioContext; + if (!AudioContextCtor || !OfflineAudioContextCtor) { + throw new Error('当前环境无法转换录音'); + } + + const audioContext = new AudioContextCtor(); + let decodedBuffer: AudioBuffer; + try { + decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer()); + } finally { + await audioContext.close().catch(() => undefined); + } + const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE)); + const offlineContext = new OfflineAudioContextCtor( + VOICE_WAV_CHANNELS, + frameCount, + VOICE_WAV_SAMPLE_RATE, + ); + const source = offlineContext.createBufferSource(); + source.buffer = decodedBuffer; + source.connect(offlineContext.destination); + source.start(0); + return audioBufferTo16BitPcmWav(await offlineContext.startRendering()); +} + +function isUnauthorizedWorksError(error: unknown): boolean { + if (error instanceof WorksSquareApiError && error.statusCode === 401) return true; + if (!(error instanceof Error)) return false; + const record = error as Error & { code?: unknown; details?: { status?: unknown } }; + const message = error.message.toLowerCase(); + return record.details?.status === 401 + || record.code === 'AUTH_INVALID' + || message.includes('401') + || message.includes('unauthorized') + || message.includes('invalid access token'); +} + +function currentAuthAccountKey(): string | null { + const user = useAuthStore.getState().user; + return getProfileAccountKey(user?.userId ?? user?.username); +} function isGenerationConfirmationIntent(message: string): boolean { const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, ''); @@ -782,6 +893,12 @@ function QuoteCard({ export function ImageCanvas() { const authenticated = useAuthStore((state) => state.isAuthenticated()); + const accountKey = useAuthStore((state) => ( + getProfileAccountKey(state.user?.userId ?? state.user?.username) + )); + const accessToken = useAuthStore((state) => state.accessToken); + const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken); + const refreshSession = useAuthStore((state) => state.refreshSession); const status = useImageWorkspaceStore((state) => state.status); const bootstrap = useImageWorkspaceStore((state) => state.bootstrap); const workspace = useImageWorkspaceStore((state) => state.workspace); @@ -817,12 +934,24 @@ export function ImageCanvas() { const [imageSourceBusyKey, setImageSourceBusyKey] = useState(null); const [imageSourceError, setImageSourceError] = useState(null); const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState(null); + const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle'); const conversationEndRef = useRef(null); const promptRef = useRef(null); const referenceUploadInputRef = useRef(null); const promptSelectionRef = useRef({ start: 0, end: 0 }); const quoteRepriceTimerRef = useRef | null>(null); const quoteDraftVersionRef = useRef(0); + const voiceRecorderRef = useRef(null); + const voiceStreamRef = useRef(null); + const voiceChunksRef = useRef([]); + const voiceOperationRef = useRef(0); + const voiceLifecycleRevisionRef = useRef(0); + const renderedAccountKeyRef = useRef(accountKey); + if (renderedAccountKeyRef.current !== accountKey) { + renderedAccountKeyRef.current = accountKey; + voiceLifecycleRevisionRef.current += 1; + } + const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current; const quote = useMemo( () => conversation ? activeQuote(conversation.messages) : null, @@ -859,6 +988,13 @@ export function ImageCanvas() { }); }, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]); const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0); + const canUseVoiceInput = voiceInputState === 'recording' || ( + Boolean(workspace && conversation) + && !submittingConversationId + && !imageSourceBusyKey + && !referenceUploadBusy + && voiceInputState !== 'transcribing' + ); const selectingImageReference = conversation?.brief.medium === 'image'; const conversationMessages = useMemo(() => { if (!conversation) return []; @@ -1148,6 +1284,184 @@ export function ImageCanvas() { } }; + const stopVoiceStream = useCallback(() => { + for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop(); + voiceStreamRef.current = null; + }, []); + + useEffect(() => { + return () => { + voiceOperationRef.current += 1; + const recorder = voiceRecorderRef.current; + if (recorder) { + recorder.ondataavailable = null; + recorder.onstop = null; + if (recorder.state !== 'inactive') recorder.stop(); + voiceRecorderRef.current = null; + } + voiceChunksRef.current = []; + stopVoiceStream(); + setVoiceInputState('idle'); + }; + }, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]); + + const finishVoiceTranscription = useCallback(async ( + mimeType: string, + operationId: number, + requestedAccountKey: string, + lifecycleRevision: number, + requestedWorkspaceId: string, + requestedConversationId: string, + ) => { + const isActive = () => voiceOperationRef.current === operationId + && voiceLifecycleRevisionRef.current === lifecycleRevision + && renderedAccountKeyRef.current === requestedAccountKey + && currentAuthAccountKey() === requestedAccountKey + && useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId + && useImageWorkspaceStore.getState().activeConversationId === requestedConversationId; + const chunks = voiceChunksRef.current; + voiceChunksRef.current = []; + stopVoiceStream(); + + if (chunks.length === 0) { + if (isActive()) { + setVoiceInputState('idle'); + toast.warning('没有录到声音'); + } + return; + } + if (isActive()) setVoiceInputState('transcribing'); + try { + if (!isActive()) return; + const validAccessToken = await getValidAccessToken(); + if (!isActive()) return; + if (!validAccessToken) { + toast.error('请先登录后再使用语音输入'); + return; + } + const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' }); + const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob); + if (!isActive()) return; + const audioBase64 = await blobToBase64(wavBlob); + if (!isActive()) return; + const transcribeWithToken = (token: string) => transcribeWorksSpeech({ + accessToken: token, + audioBase64, + fileName: 'voice.wav', + mimeType: 'audio/wav', + language: 'zh', + }); + let transcription; + try { + if (!isActive()) return; + transcription = await transcribeWithToken(validAccessToken); + if (!isActive()) return; + } catch (error) { + if (!isUnauthorizedWorksError(error)) throw error; + if (!isActive()) return; + const refreshedAccessToken = await refreshSession(); + if (!isActive()) return; + if (!refreshedAccessToken) throw error; + transcription = await transcribeWithToken(refreshedAccessToken); + if (!isActive()) return; + } + const transcript = transcription.text.trim(); + if (transcript) { + setPrompt((currentPrompt) => currentPrompt.trim() + ? `${currentPrompt.trim()}\n${transcript}` + : transcript); + setActionError(null); + } + } catch (error) { + if (isActive()) { + toast.error('语音识别失败', { + description: error instanceof Error ? error.message : String(error), + }); + } + } finally { + if (isActive()) { + voiceRecorderRef.current = null; + setVoiceInputState('idle'); + } + } + }, [getValidAccessToken, refreshSession, stopVoiceStream]); + + const handleVoiceInputClick = useCallback(async () => { + if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision + || renderedAccountKeyRef.current !== accountKey) return; + if (voiceInputState === 'recording') { + const recorder = voiceRecorderRef.current; + if (!recorder || recorder.state === 'inactive') { + stopVoiceStream(); + setVoiceInputState('idle'); + return; + } + setVoiceInputState('transcribing'); + recorder.stop(); + return; + } + if (!canUseVoiceInput || !workspace || !conversation) return; + if (!accountKey || !accessToken) { + toast.error('请先登录后再使用语音输入'); + return; + } + if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') { + toast.error('当前环境不支持语音输入'); + return; + } + + const operationId = voiceOperationRef.current + 1; + voiceOperationRef.current = operationId; + const lifecycleRevision = voiceLifecycleRevisionRef.current; + const requestedAccountKey = accountKey; + const isActive = () => voiceOperationRef.current === operationId + && voiceLifecycleRevisionRef.current === lifecycleRevision + && renderedAccountKeyRef.current === requestedAccountKey + && currentAuthAccountKey() === requestedAccountKey; + const requestedWorkspaceId = workspace.workspaceId; + const requestedConversationId = conversation.conversationId; + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (!isActive()) { + for (const track of stream.getTracks()) track.stop(); + return; + } + const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm') + ? 'audio/webm' + : ''; + const recorder = new window.MediaRecorder( + stream, + preferredMimeType ? { mimeType: preferredMimeType } : undefined, + ); + voiceStreamRef.current = stream; + voiceChunksRef.current = []; + voiceRecorderRef.current = recorder; + recorder.ondataavailable = (event) => { + if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data); + }; + recorder.onstop = () => { + void finishVoiceTranscription( + recorder.mimeType || preferredMimeType || 'audio/webm', + operationId, + requestedAccountKey, + lifecycleRevision, + requestedWorkspaceId, + requestedConversationId, + ); + }; + recorder.start(); + setVoiceInputState('recording'); + } catch (error) { + stopVoiceStream(); + if (isActive()) { + setVoiceInputState('idle'); + toast.error('无法开始录音', { + description: error instanceof Error ? error.message : String(error), + }); + } + } + }, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]); + const handleSend = async (override?: string) => { const message = (override ?? prompt).trim(); if (!workspace || !conversation || !message || submitting) return; @@ -1605,12 +1919,27 @@ export function ImageCanvas() {