feat: enable AI design voice input
This commit is contained in:
1 parent
13bdc0c9ba
commit
1ba5b9cc84
4 files changed
+751
-7
No files matched your search
@@ -0,0 +1,57 @@
|
|||||||
|
# Task: Fix AI design dialog Enter and voice input
|
||||||
|
|
||||||
|
## Identity
|
||||||
|
|
||||||
|
- Task ID: 20260818-ai-design-input-9f3a
|
||||||
|
- Mode: Feature
|
||||||
|
- Branch: codex/20260818-ai-design-input-9f3a-ai-design-input
|
||||||
|
- Worktree: D:\Datas\OthersProjects\makelore-ai-design-input-9f3a
|
||||||
|
- Base commit: 13bdc0c9bab656aa9ff6c990bf6359f7d3210750
|
||||||
|
- Owner: developer
|
||||||
|
- Status: Completed
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- 保留并验收 AI 设计对话框现有的 Enter 发送、Shift+Enter 换行和输入法组合保护。
|
||||||
|
- 打开 AI 设计对话框的语音输入:采集麦克风、转为 16 kHz 单声道 WAV,经既有 Works Speech Host API 转写并回填当前草稿。
|
||||||
|
- 为账号、Workspace、Conversation 切换和组件卸载增加录音/异步转写生命周期清理与迟到结果保护;录音中始终保留停止按钮。
|
||||||
|
- 增加组件回归测试和 AI Design Electron E2E 的 Enter 发送覆盖。
|
||||||
|
|
||||||
|
## Intent And Constraints
|
||||||
|
|
||||||
|
- 复用现有 `transcribeWorksSpeech` 与 Renderer → Host API → Main → Works 调用边界,不新增直连 IPC 或本地运行时 HTTP。
|
||||||
|
- 不改变消息、Quote 确认、任务创建和多会话数据链路;Enter 仍由现有 `handleSend` 提交,Shift+Enter 和 IME 行为保持不变。
|
||||||
|
- 录音期间即使发送或附件操作进入 busy,也必须可以停止录音并释放 MediaStream;账号、Workspace、Conversation 变化后不得使用旧录音结果回填新上下文。
|
||||||
|
- 真实麦克风和 Works 账号的 Electron smoke 未纳入本次自动化验证。
|
||||||
|
|
||||||
|
## Plan
|
||||||
|
|
||||||
|
1. 核对 AI 设计 composer 的键盘处理、语音入口和 Works Speech 复用边界。
|
||||||
|
2. 实现语音采集、WAV 转码、认证重试及上下文生命周期守卫,并补齐回归测试。
|
||||||
|
3. 更新 Enter 交互 E2E 覆盖,运行聚焦/全量测试、类型检查、Lint、构建和文档漂移检查。
|
||||||
|
|
||||||
|
## Outcome
|
||||||
|
|
||||||
|
- AI 设计 composer 的 Enter 提交行为已由单测和 Electron E2E 覆盖;Shift+Enter 换行及 IME 组合期间不误发保持不变。
|
||||||
|
- 语音输入按钮已接入 MediaRecorder;录音停止后生成 16 kHz、单声道、16-bit PCM WAV,调用 Works Speech 并将识别文本追加到当前草稿。
|
||||||
|
- 语音流程支持未登录/环境不支持提示、401 强制刷新后单次重试,并在账号、Workspace、Conversation 变化或卸载时停止 recorder、释放 tracks、丢弃迟到结果。
|
||||||
|
- 录音状态下停止按钮不受发送、参考图上传或素材操作 busy 状态禁用。
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- `node node_modules/vitest/vitest.mjs run tests/unit/image-canvas-page.test.tsx`:44 passed。
|
||||||
|
- `node node_modules/vitest/vitest.mjs run`:176 files / 2114 tests passed。
|
||||||
|
- `node node_modules/typescript/bin/tsc --noEmit`:通过。
|
||||||
|
- `node node_modules/eslint/bin/eslint.js src/pages/ImageCanvas/index.tsx tests/unit/image-canvas-page.test.tsx tests/e2e/image-workspace-conversations.spec.ts`:通过。
|
||||||
|
- `node --max-old-space-size=6144 node_modules/vite/bin/vite.js build`:Renderer、Electron Main、Preload 均构建通过;仅有既有动态导入和 chunk 体积警告。
|
||||||
|
- `git diff --check`:通过。
|
||||||
|
- `check_doc_drift.py --task-id 20260818-ai-design-input-9f3a`:通过。
|
||||||
|
|
||||||
|
## Follow-ups
|
||||||
|
|
||||||
|
- 需要真实麦克风权限和 Works 账号的 Electron smoke,确认不同浏览器/系统的 MediaRecorder 编码兼容性。
|
||||||
|
- 音频转码 helper 目前与 AI 编程 composer 各自维护,后续可在独立任务中评估抽到共享 `src/lib` 工具。
|
||||||
|
|
||||||
|
## Promotion Candidates
|
||||||
|
|
||||||
|
- None。此次改动复用既有 Host API 和产品交互契约,不需要修改 canonical 架构或决策文档。
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import {
|
import {
|
||||||
|
useCallback,
|
||||||
useEffect,
|
useEffect,
|
||||||
useMemo,
|
useMemo,
|
||||||
useRef,
|
useRef,
|
||||||
@@ -18,6 +19,7 @@ import {
|
|||||||
Plus,
|
Plus,
|
||||||
RefreshCw,
|
RefreshCw,
|
||||||
Send,
|
Send,
|
||||||
|
Square,
|
||||||
Sparkles,
|
Sparkles,
|
||||||
Upload,
|
Upload,
|
||||||
WandSparkles,
|
WandSparkles,
|
||||||
@@ -42,9 +44,11 @@ import {
|
|||||||
uploadImageWorkspaceAsset,
|
uploadImageWorkspaceAsset,
|
||||||
} from '@/lib/image-workspace';
|
} from '@/lib/image-workspace';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
|
import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square';
|
||||||
import { useImageWorkspaceStore } from '@/stores/image-workspace';
|
import { useImageWorkspaceStore } from '@/stores/image-workspace';
|
||||||
import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum';
|
import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum';
|
||||||
import { useAuthStore } from '@/stores/auth';
|
import { useAuthStore } from '@/stores/auth';
|
||||||
|
import { getProfileAccountKey } from '@/stores/user-profile';
|
||||||
import logoSvg from '@/assets/logo.svg';
|
import logoSvg from '@/assets/logo.svg';
|
||||||
import type {
|
import type {
|
||||||
DesignAsset,
|
DesignAsset,
|
||||||
@@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([
|
|||||||
]);
|
]);
|
||||||
const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片';
|
const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片';
|
||||||
const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24;
|
const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24;
|
||||||
|
const VOICE_WAV_SAMPLE_RATE = 16_000;
|
||||||
|
const VOICE_WAV_CHANNELS = 1;
|
||||||
|
const VOICE_WAV_BITS_PER_SAMPLE = 16;
|
||||||
|
|
||||||
|
type OfflineAudioContextConstructor = new (
|
||||||
|
numberOfChannels: number,
|
||||||
|
length: number,
|
||||||
|
sampleRate: number,
|
||||||
|
) => OfflineAudioContext;
|
||||||
|
|
||||||
|
type AudioContextWindow = Window & {
|
||||||
|
webkitAudioContext?: typeof AudioContext;
|
||||||
|
webkitOfflineAudioContext?: OfflineAudioContextConstructor;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function blobToBase64(blob: Blob): Promise<string> {
|
||||||
|
const bytes = new Uint8Array(await blob.arrayBuffer());
|
||||||
|
let binary = '';
|
||||||
|
const chunkSize = 0x8000;
|
||||||
|
for (let index = 0; index < bytes.length; index += chunkSize) {
|
||||||
|
binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize));
|
||||||
|
}
|
||||||
|
return btoa(binary);
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeAscii(view: DataView, offset: number, value: string): void {
|
||||||
|
for (let index = 0; index < value.length; index += 1) {
|
||||||
|
view.setUint8(offset + index, value.charCodeAt(index));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob {
|
||||||
|
const samples = audioBuffer.getChannelData(0);
|
||||||
|
const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8;
|
||||||
|
const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample;
|
||||||
|
const dataSize = samples.length * blockAlign;
|
||||||
|
const wavBuffer = new ArrayBuffer(44 + dataSize);
|
||||||
|
const view = new DataView(wavBuffer);
|
||||||
|
|
||||||
|
writeAscii(view, 0, 'RIFF');
|
||||||
|
view.setUint32(4, 36 + dataSize, true);
|
||||||
|
writeAscii(view, 8, 'WAVE');
|
||||||
|
writeAscii(view, 12, 'fmt ');
|
||||||
|
view.setUint32(16, 16, true);
|
||||||
|
view.setUint16(20, 1, true);
|
||||||
|
view.setUint16(22, VOICE_WAV_CHANNELS, true);
|
||||||
|
view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true);
|
||||||
|
view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true);
|
||||||
|
view.setUint16(32, blockAlign, true);
|
||||||
|
view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true);
|
||||||
|
writeAscii(view, 36, 'data');
|
||||||
|
view.setUint32(40, dataSize, true);
|
||||||
|
|
||||||
|
let offset = 44;
|
||||||
|
for (const sample of samples) {
|
||||||
|
const clamped = Math.max(-1, Math.min(1, sample));
|
||||||
|
view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
|
||||||
|
offset += bytesPerSample;
|
||||||
|
}
|
||||||
|
return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise<Blob> {
|
||||||
|
const audioWindow = window as AudioContextWindow;
|
||||||
|
const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext;
|
||||||
|
const OfflineAudioContextCtor = window.OfflineAudioContext
|
||||||
|
?? audioWindow.webkitOfflineAudioContext;
|
||||||
|
if (!AudioContextCtor || !OfflineAudioContextCtor) {
|
||||||
|
throw new Error('当前环境无法转换录音');
|
||||||
|
}
|
||||||
|
|
||||||
|
const audioContext = new AudioContextCtor();
|
||||||
|
let decodedBuffer: AudioBuffer;
|
||||||
|
try {
|
||||||
|
decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer());
|
||||||
|
} finally {
|
||||||
|
await audioContext.close().catch(() => undefined);
|
||||||
|
}
|
||||||
|
const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE));
|
||||||
|
const offlineContext = new OfflineAudioContextCtor(
|
||||||
|
VOICE_WAV_CHANNELS,
|
||||||
|
frameCount,
|
||||||
|
VOICE_WAV_SAMPLE_RATE,
|
||||||
|
);
|
||||||
|
const source = offlineContext.createBufferSource();
|
||||||
|
source.buffer = decodedBuffer;
|
||||||
|
source.connect(offlineContext.destination);
|
||||||
|
source.start(0);
|
||||||
|
return audioBufferTo16BitPcmWav(await offlineContext.startRendering());
|
||||||
|
}
|
||||||
|
|
||||||
|
function isUnauthorizedWorksError(error: unknown): boolean {
|
||||||
|
if (error instanceof WorksSquareApiError && error.statusCode === 401) return true;
|
||||||
|
if (!(error instanceof Error)) return false;
|
||||||
|
const record = error as Error & { code?: unknown; details?: { status?: unknown } };
|
||||||
|
const message = error.message.toLowerCase();
|
||||||
|
return record.details?.status === 401
|
||||||
|
|| record.code === 'AUTH_INVALID'
|
||||||
|
|| message.includes('401')
|
||||||
|
|| message.includes('unauthorized')
|
||||||
|
|| message.includes('invalid access token');
|
||||||
|
}
|
||||||
|
|
||||||
|
function currentAuthAccountKey(): string | null {
|
||||||
|
const user = useAuthStore.getState().user;
|
||||||
|
return getProfileAccountKey(user?.userId ?? user?.username);
|
||||||
|
}
|
||||||
|
|
||||||
function isGenerationConfirmationIntent(message: string): boolean {
|
function isGenerationConfirmationIntent(message: string): boolean {
|
||||||
const normalized = message.trim().replace(/[\s,,。.!!??、]/g, '');
|
const normalized = message.trim().replace(/[\s,,。.!!??、]/g, '');
|
||||||
@@ -782,6 +893,12 @@ function QuoteCard({
|
|||||||
|
|
||||||
export function ImageCanvas() {
|
export function ImageCanvas() {
|
||||||
const authenticated = useAuthStore((state) => state.isAuthenticated());
|
const authenticated = useAuthStore((state) => state.isAuthenticated());
|
||||||
|
const accountKey = useAuthStore((state) => (
|
||||||
|
getProfileAccountKey(state.user?.userId ?? state.user?.username)
|
||||||
|
));
|
||||||
|
const accessToken = useAuthStore((state) => state.accessToken);
|
||||||
|
const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken);
|
||||||
|
const refreshSession = useAuthStore((state) => state.refreshSession);
|
||||||
const status = useImageWorkspaceStore((state) => state.status);
|
const status = useImageWorkspaceStore((state) => state.status);
|
||||||
const bootstrap = useImageWorkspaceStore((state) => state.bootstrap);
|
const bootstrap = useImageWorkspaceStore((state) => state.bootstrap);
|
||||||
const workspace = useImageWorkspaceStore((state) => state.workspace);
|
const workspace = useImageWorkspaceStore((state) => state.workspace);
|
||||||
@@ -817,12 +934,24 @@ export function ImageCanvas() {
|
|||||||
const [imageSourceBusyKey, setImageSourceBusyKey] = useState<string | null>(null);
|
const [imageSourceBusyKey, setImageSourceBusyKey] = useState<string | null>(null);
|
||||||
const [imageSourceError, setImageSourceError] = useState<string | null>(null);
|
const [imageSourceError, setImageSourceError] = useState<string | null>(null);
|
||||||
const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState<DesignAsset | null>(null);
|
const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState<DesignAsset | null>(null);
|
||||||
|
const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle');
|
||||||
const conversationEndRef = useRef<HTMLDivElement | null>(null);
|
const conversationEndRef = useRef<HTMLDivElement | null>(null);
|
||||||
const promptRef = useRef<HTMLTextAreaElement | null>(null);
|
const promptRef = useRef<HTMLTextAreaElement | null>(null);
|
||||||
const referenceUploadInputRef = useRef<HTMLInputElement | null>(null);
|
const referenceUploadInputRef = useRef<HTMLInputElement | null>(null);
|
||||||
const promptSelectionRef = useRef({ start: 0, end: 0 });
|
const promptSelectionRef = useRef({ start: 0, end: 0 });
|
||||||
const quoteRepriceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
const quoteRepriceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||||
const quoteDraftVersionRef = useRef(0);
|
const quoteDraftVersionRef = useRef(0);
|
||||||
|
const voiceRecorderRef = useRef<MediaRecorder | null>(null);
|
||||||
|
const voiceStreamRef = useRef<MediaStream | null>(null);
|
||||||
|
const voiceChunksRef = useRef<Blob[]>([]);
|
||||||
|
const voiceOperationRef = useRef(0);
|
||||||
|
const voiceLifecycleRevisionRef = useRef(0);
|
||||||
|
const renderedAccountKeyRef = useRef(accountKey);
|
||||||
|
if (renderedAccountKeyRef.current !== accountKey) {
|
||||||
|
renderedAccountKeyRef.current = accountKey;
|
||||||
|
voiceLifecycleRevisionRef.current += 1;
|
||||||
|
}
|
||||||
|
const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||||
|
|
||||||
const quote = useMemo(
|
const quote = useMemo(
|
||||||
() => conversation ? activeQuote(conversation.messages) : null,
|
() => conversation ? activeQuote(conversation.messages) : null,
|
||||||
@@ -859,6 +988,13 @@ export function ImageCanvas() {
|
|||||||
});
|
});
|
||||||
}, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]);
|
}, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]);
|
||||||
const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0);
|
const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0);
|
||||||
|
const canUseVoiceInput = voiceInputState === 'recording' || (
|
||||||
|
Boolean(workspace && conversation)
|
||||||
|
&& !submittingConversationId
|
||||||
|
&& !imageSourceBusyKey
|
||||||
|
&& !referenceUploadBusy
|
||||||
|
&& voiceInputState !== 'transcribing'
|
||||||
|
);
|
||||||
const selectingImageReference = conversation?.brief.medium === 'image';
|
const selectingImageReference = conversation?.brief.medium === 'image';
|
||||||
const conversationMessages = useMemo<RenderedDesignMessage[]>(() => {
|
const conversationMessages = useMemo<RenderedDesignMessage[]>(() => {
|
||||||
if (!conversation) return [];
|
if (!conversation) return [];
|
||||||
@@ -1148,6 +1284,184 @@ export function ImageCanvas() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const stopVoiceStream = useCallback(() => {
|
||||||
|
for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop();
|
||||||
|
voiceStreamRef.current = null;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
voiceOperationRef.current += 1;
|
||||||
|
const recorder = voiceRecorderRef.current;
|
||||||
|
if (recorder) {
|
||||||
|
recorder.ondataavailable = null;
|
||||||
|
recorder.onstop = null;
|
||||||
|
if (recorder.state !== 'inactive') recorder.stop();
|
||||||
|
voiceRecorderRef.current = null;
|
||||||
|
}
|
||||||
|
voiceChunksRef.current = [];
|
||||||
|
stopVoiceStream();
|
||||||
|
setVoiceInputState('idle');
|
||||||
|
};
|
||||||
|
}, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]);
|
||||||
|
|
||||||
|
const finishVoiceTranscription = useCallback(async (
|
||||||
|
mimeType: string,
|
||||||
|
operationId: number,
|
||||||
|
requestedAccountKey: string,
|
||||||
|
lifecycleRevision: number,
|
||||||
|
requestedWorkspaceId: string,
|
||||||
|
requestedConversationId: string,
|
||||||
|
) => {
|
||||||
|
const isActive = () => voiceOperationRef.current === operationId
|
||||||
|
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||||
|
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||||
|
&& currentAuthAccountKey() === requestedAccountKey
|
||||||
|
&& useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId
|
||||||
|
&& useImageWorkspaceStore.getState().activeConversationId === requestedConversationId;
|
||||||
|
const chunks = voiceChunksRef.current;
|
||||||
|
voiceChunksRef.current = [];
|
||||||
|
stopVoiceStream();
|
||||||
|
|
||||||
|
if (chunks.length === 0) {
|
||||||
|
if (isActive()) {
|
||||||
|
setVoiceInputState('idle');
|
||||||
|
toast.warning('没有录到声音');
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isActive()) setVoiceInputState('transcribing');
|
||||||
|
try {
|
||||||
|
if (!isActive()) return;
|
||||||
|
const validAccessToken = await getValidAccessToken();
|
||||||
|
if (!isActive()) return;
|
||||||
|
if (!validAccessToken) {
|
||||||
|
toast.error('请先登录后再使用语音输入');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' });
|
||||||
|
const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob);
|
||||||
|
if (!isActive()) return;
|
||||||
|
const audioBase64 = await blobToBase64(wavBlob);
|
||||||
|
if (!isActive()) return;
|
||||||
|
const transcribeWithToken = (token: string) => transcribeWorksSpeech({
|
||||||
|
accessToken: token,
|
||||||
|
audioBase64,
|
||||||
|
fileName: 'voice.wav',
|
||||||
|
mimeType: 'audio/wav',
|
||||||
|
language: 'zh',
|
||||||
|
});
|
||||||
|
let transcription;
|
||||||
|
try {
|
||||||
|
if (!isActive()) return;
|
||||||
|
transcription = await transcribeWithToken(validAccessToken);
|
||||||
|
if (!isActive()) return;
|
||||||
|
} catch (error) {
|
||||||
|
if (!isUnauthorizedWorksError(error)) throw error;
|
||||||
|
if (!isActive()) return;
|
||||||
|
const refreshedAccessToken = await refreshSession();
|
||||||
|
if (!isActive()) return;
|
||||||
|
if (!refreshedAccessToken) throw error;
|
||||||
|
transcription = await transcribeWithToken(refreshedAccessToken);
|
||||||
|
if (!isActive()) return;
|
||||||
|
}
|
||||||
|
const transcript = transcription.text.trim();
|
||||||
|
if (transcript) {
|
||||||
|
setPrompt((currentPrompt) => currentPrompt.trim()
|
||||||
|
? `${currentPrompt.trim()}\n${transcript}`
|
||||||
|
: transcript);
|
||||||
|
setActionError(null);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (isActive()) {
|
||||||
|
toast.error('语音识别失败', {
|
||||||
|
description: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (isActive()) {
|
||||||
|
voiceRecorderRef.current = null;
|
||||||
|
setVoiceInputState('idle');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [getValidAccessToken, refreshSession, stopVoiceStream]);
|
||||||
|
|
||||||
|
const handleVoiceInputClick = useCallback(async () => {
|
||||||
|
if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision
|
||||||
|
|| renderedAccountKeyRef.current !== accountKey) return;
|
||||||
|
if (voiceInputState === 'recording') {
|
||||||
|
const recorder = voiceRecorderRef.current;
|
||||||
|
if (!recorder || recorder.state === 'inactive') {
|
||||||
|
stopVoiceStream();
|
||||||
|
setVoiceInputState('idle');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setVoiceInputState('transcribing');
|
||||||
|
recorder.stop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!canUseVoiceInput || !workspace || !conversation) return;
|
||||||
|
if (!accountKey || !accessToken) {
|
||||||
|
toast.error('请先登录后再使用语音输入');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') {
|
||||||
|
toast.error('当前环境不支持语音输入');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const operationId = voiceOperationRef.current + 1;
|
||||||
|
voiceOperationRef.current = operationId;
|
||||||
|
const lifecycleRevision = voiceLifecycleRevisionRef.current;
|
||||||
|
const requestedAccountKey = accountKey;
|
||||||
|
const isActive = () => voiceOperationRef.current === operationId
|
||||||
|
&& voiceLifecycleRevisionRef.current === lifecycleRevision
|
||||||
|
&& renderedAccountKeyRef.current === requestedAccountKey
|
||||||
|
&& currentAuthAccountKey() === requestedAccountKey;
|
||||||
|
const requestedWorkspaceId = workspace.workspaceId;
|
||||||
|
const requestedConversationId = conversation.conversationId;
|
||||||
|
try {
|
||||||
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||||
|
if (!isActive()) {
|
||||||
|
for (const track of stream.getTracks()) track.stop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm')
|
||||||
|
? 'audio/webm'
|
||||||
|
: '';
|
||||||
|
const recorder = new window.MediaRecorder(
|
||||||
|
stream,
|
||||||
|
preferredMimeType ? { mimeType: preferredMimeType } : undefined,
|
||||||
|
);
|
||||||
|
voiceStreamRef.current = stream;
|
||||||
|
voiceChunksRef.current = [];
|
||||||
|
voiceRecorderRef.current = recorder;
|
||||||
|
recorder.ondataavailable = (event) => {
|
||||||
|
if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data);
|
||||||
|
};
|
||||||
|
recorder.onstop = () => {
|
||||||
|
void finishVoiceTranscription(
|
||||||
|
recorder.mimeType || preferredMimeType || 'audio/webm',
|
||||||
|
operationId,
|
||||||
|
requestedAccountKey,
|
||||||
|
lifecycleRevision,
|
||||||
|
requestedWorkspaceId,
|
||||||
|
requestedConversationId,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
recorder.start();
|
||||||
|
setVoiceInputState('recording');
|
||||||
|
} catch (error) {
|
||||||
|
stopVoiceStream();
|
||||||
|
if (isActive()) {
|
||||||
|
setVoiceInputState('idle');
|
||||||
|
toast.error('无法开始录音', {
|
||||||
|
description: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]);
|
||||||
|
|
||||||
const handleSend = async (override?: string) => {
|
const handleSend = async (override?: string) => {
|
||||||
const message = (override ?? prompt).trim();
|
const message = (override ?? prompt).trim();
|
||||||
if (!workspace || !conversation || !message || submitting) return;
|
if (!workspace || !conversation || !message || submitting) return;
|
||||||
@@ -1605,12 +1919,27 @@ export function ImageCanvas() {
|
|||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
disabled
|
disabled={!canUseVoiceInput}
|
||||||
className="motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none"
|
className={cn(
|
||||||
aria-label="语音输入"
|
'motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none hover:bg-surface-subtle hover:text-foreground',
|
||||||
title="AI 绘画暂不支持语音输入"
|
voiceInputState !== 'idle' && 'bg-surface-subtle text-foreground',
|
||||||
|
)}
|
||||||
|
aria-label={voiceInputState === 'recording'
|
||||||
|
? '停止语音输入'
|
||||||
|
: voiceInputState === 'transcribing'
|
||||||
|
? '正在识别语音'
|
||||||
|
: '语音输入'}
|
||||||
|
title={typeof window.MediaRecorder === 'undefined'
|
||||||
|
|| !navigator.mediaDevices?.getUserMedia
|
||||||
|
? '当前环境不支持语音输入'
|
||||||
|
: '语音输入'}
|
||||||
|
onClick={() => void handleVoiceInputClick()}
|
||||||
>
|
>
|
||||||
<Mic className="h-4 w-4" />
|
{voiceInputState === 'transcribing'
|
||||||
|
? <Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
: voiceInputState === 'recording'
|
||||||
|
? <Square className="h-4 w-4 fill-current" />
|
||||||
|
: <Mic className="h-4 w-4" />}
|
||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
type="submit"
|
type="submit"
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ test.describe('AI Design workspace', () => {
|
|||||||
|
|
||||||
const firstPrompt = '为项目设计一张蓝色星空海报';
|
const firstPrompt = '为项目设计一张蓝色星空海报';
|
||||||
await page.getByLabel('设计需求').fill(firstPrompt);
|
await page.getByLabel('设计需求').fill(firstPrompt);
|
||||||
await page.getByRole('button', { name: '发送给设计 Agent' }).click();
|
await page.getByLabel('设计需求').press('Enter');
|
||||||
await expect(page.getByTestId('image-workspace-conversation')).toContainText(firstPrompt);
|
await expect(page.getByTestId('image-workspace-conversation')).toContainText(firstPrompt);
|
||||||
await expect(page.getByText('设计 Agent', { exact: true })).toHaveCount(0);
|
await expect(page.getByText('设计 Agent', { exact: true })).toHaveCount(0);
|
||||||
await expect(page.getByText('你', { exact: true })).toHaveCount(0);
|
await expect(page.getByText('你', { exact: true })).toHaveCount(0);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
|
import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
|
||||||
import { MemoryRouter } from 'react-router-dom';
|
import { MemoryRouter } from 'react-router-dom';
|
||||||
|
import { toast } from 'sonner';
|
||||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
import { ImageWorkspaceApiError } from '@/lib/image-workspace';
|
import { ImageWorkspaceApiError } from '@/lib/image-workspace';
|
||||||
import { ImageCanvas } from '@/pages/ImageCanvas';
|
import { ImageCanvas } from '@/pages/ImageCanvas';
|
||||||
@@ -27,6 +28,9 @@ const openImageWorkspaceTaskEventsMock = vi.hoisted(() => vi.fn());
|
|||||||
const resolveImageWorkspaceAssetUrlMock = vi.hoisted(() => vi.fn());
|
const resolveImageWorkspaceAssetUrlMock = vi.hoisted(() => vi.fn());
|
||||||
const saveImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
const saveImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||||
const uploadImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
const uploadImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||||
|
const transcribeWorksSpeechMock = vi.hoisted(() => vi.fn());
|
||||||
|
const originalGetValidAccessToken = useAuthStore.getState().getValidAccessToken;
|
||||||
|
const originalRefreshSession = useAuthStore.getState().refreshSession;
|
||||||
|
|
||||||
type EventListener = (event: MessageEvent<string>) => void;
|
type EventListener = (event: MessageEvent<string>) => void;
|
||||||
|
|
||||||
@@ -88,6 +92,42 @@ vi.mock('@/lib/image-workspace', async (importOriginal) => {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
vi.mock('@/lib/works-square', async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import('@/lib/works-square')>();
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
transcribeWorksSpeech: (...args: unknown[]) => transcribeWorksSpeechMock(...args),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
function installMockVoiceWavTranscoder(): void {
|
||||||
|
const pcm = Float32Array.from([0, 0.5, -0.5, 1, -1]);
|
||||||
|
class MockAudioContext {
|
||||||
|
decodeAudioData = vi.fn().mockResolvedValue({
|
||||||
|
duration: pcm.length / 48_000,
|
||||||
|
getChannelData: () => pcm,
|
||||||
|
});
|
||||||
|
close = vi.fn().mockResolvedValue(undefined);
|
||||||
|
}
|
||||||
|
class MockOfflineAudioContext {
|
||||||
|
destination = {};
|
||||||
|
createBufferSource = vi.fn(() => ({
|
||||||
|
buffer: null,
|
||||||
|
connect: vi.fn(),
|
||||||
|
start: vi.fn(),
|
||||||
|
}));
|
||||||
|
startRendering = vi.fn().mockResolvedValue({ getChannelData: () => pcm });
|
||||||
|
}
|
||||||
|
Object.defineProperty(window, 'AudioContext', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockAudioContext,
|
||||||
|
});
|
||||||
|
Object.defineProperty(window, 'OfflineAudioContext', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockOfflineAudioContext,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const bootstrapFixture: DesignWorkspaceBootstrap = {
|
const bootstrapFixture: DesignWorkspaceBootstrap = {
|
||||||
capabilities: {
|
capabilities: {
|
||||||
conversation: true,
|
conversation: true,
|
||||||
@@ -243,6 +283,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
|||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
useAuthStore.setState({
|
||||||
|
getValidAccessToken: originalGetValidAccessToken,
|
||||||
|
refreshSession: originalRefreshSession,
|
||||||
|
});
|
||||||
useImageWorkspaceStore.getState().reset();
|
useImageWorkspaceStore.getState().reset();
|
||||||
useImagePromptMuseumStore.getState().clearPendingPrompt();
|
useImagePromptMuseumStore.getState().clearPendingPrompt();
|
||||||
fetchImageWorkspaceMock.mockResolvedValue(bootstrapFixture);
|
fetchImageWorkspaceMock.mockResolvedValue(bootstrapFixture);
|
||||||
@@ -283,6 +327,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
|||||||
...taskFixture.resultAssets[0],
|
...taskFixture.resultAssets[0],
|
||||||
assetId: 'asset-uploaded',
|
assetId: 'asset-uploaded',
|
||||||
});
|
});
|
||||||
|
transcribeWorksSpeechMock.mockResolvedValue({
|
||||||
|
text: '让天空更通透',
|
||||||
|
model: 'gpt-4o-mini-transcribe',
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('shows an honest unavailable state without fabricating projects', async () => {
|
it('shows an honest unavailable state without fabricating projects', async () => {
|
||||||
@@ -754,7 +802,7 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
|||||||
expect(screen.getByTestId('image-workspace-composer-surface'))
|
expect(screen.getByTestId('image-workspace-composer-surface'))
|
||||||
.toHaveClass('rounded-lg', 'flex', 'min-w-0');
|
.toHaveClass('rounded-lg', 'flex', 'min-w-0');
|
||||||
expect(screen.getByRole('button', { name: '上传参考图' })).toBeInTheDocument();
|
expect(screen.getByRole('button', { name: '上传参考图' })).toBeInTheDocument();
|
||||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeDisabled();
|
expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled();
|
||||||
expect(screen.queryByText('Enter 发送 · Shift+Enter 换行')).not.toBeInTheDocument();
|
expect(screen.queryByText('Enter 发送 · Shift+Enter 换行')).not.toBeInTheDocument();
|
||||||
|
|
||||||
fireEvent.click(screen.getByRole('button', { name: '上传参考图' }));
|
fireEvent.click(screen.getByRole('button', { name: '上传参考图' }));
|
||||||
@@ -831,6 +879,316 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
|||||||
expect(composer).toHaveValue('正在输入中文设计描述');
|
expect(composer).toHaveValue('正在输入中文设计描述');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports fixed login and environment guards before starting voice input', async () => {
|
||||||
|
const toastError = vi.spyOn(toast, 'error').mockImplementation(() => 'toast-id');
|
||||||
|
Object.defineProperty(window, 'MediaRecorder', {
|
||||||
|
configurable: true,
|
||||||
|
value: undefined,
|
||||||
|
});
|
||||||
|
useAuthStore.setState({ accessToken: null, canRefresh: false });
|
||||||
|
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||||
|
await screen.findByTestId('image-workspace-conversation');
|
||||||
|
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||||
|
expect(toastError).toHaveBeenLastCalledWith('请先登录后再使用语音输入');
|
||||||
|
|
||||||
|
act(() => {
|
||||||
|
useAuthStore.setState({
|
||||||
|
accessToken: 'access-token',
|
||||||
|
expiresAt: Date.now() + 60_000,
|
||||||
|
lastActiveAt: Date.now(),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const voiceButton = screen.getByRole('button', { name: '语音输入' });
|
||||||
|
expect(voiceButton).toHaveAttribute('title', '当前环境不支持语音输入');
|
||||||
|
fireEvent.click(voiceButton);
|
||||||
|
expect(toastError).toHaveBeenLastCalledWith('当前环境不支持语音输入');
|
||||||
|
toastError.mockRestore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records Works Speech input into the current draft and releases the microphone', async () => {
|
||||||
|
installMockVoiceWavTranscoder();
|
||||||
|
const pendingSend = deferred<DesignConversation>();
|
||||||
|
sendImageWorkspaceMessageMock.mockReturnValueOnce(pendingSend.promise);
|
||||||
|
const stopTrack = vi.fn();
|
||||||
|
const getUserMedia = vi.fn().mockResolvedValue({
|
||||||
|
getTracks: () => [{ stop: stopTrack }],
|
||||||
|
});
|
||||||
|
class MockMediaRecorder {
|
||||||
|
static latest: MockMediaRecorder | null = null;
|
||||||
|
static isTypeSupported(): boolean { return true; }
|
||||||
|
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||||
|
onstop: (() => void) | null = null;
|
||||||
|
state: RecordingState = 'inactive';
|
||||||
|
mimeType = 'audio/webm';
|
||||||
|
constructor() { MockMediaRecorder.latest = this; }
|
||||||
|
start(): void { this.state = 'recording'; }
|
||||||
|
stop(): void {
|
||||||
|
this.state = 'inactive';
|
||||||
|
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||||
|
this.onstop?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.defineProperty(navigator, 'mediaDevices', {
|
||||||
|
configurable: true,
|
||||||
|
value: { getUserMedia },
|
||||||
|
});
|
||||||
|
Object.defineProperty(window, 'MediaRecorder', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockMediaRecorder,
|
||||||
|
});
|
||||||
|
useAuthStore.setState({
|
||||||
|
initialized: true,
|
||||||
|
loading: false,
|
||||||
|
error: null,
|
||||||
|
accessToken: 'access-token',
|
||||||
|
tokenType: 'Bearer',
|
||||||
|
expiresAt: Date.now() + 60_000,
|
||||||
|
lastActiveAt: Date.now(),
|
||||||
|
canRefresh: true,
|
||||||
|
legacyRefreshToken: null,
|
||||||
|
user: {
|
||||||
|
username: 'designer',
|
||||||
|
userId: '42',
|
||||||
|
tenantId: null,
|
||||||
|
deptId: null,
|
||||||
|
authorities: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||||
|
await screen.findByTestId('image-workspace-conversation');
|
||||||
|
const composer = screen.getByLabelText('设计需求');
|
||||||
|
fireEvent.change(composer, { target: { value: '保留当前构图' } });
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||||
|
|
||||||
|
await waitFor(() => expect(getUserMedia).toHaveBeenCalledWith({ audio: true }));
|
||||||
|
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '发送给设计 Agent' }));
|
||||||
|
await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce());
|
||||||
|
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||||
|
|
||||||
|
await waitFor(() => expect(composer).toHaveValue('让天空更通透'));
|
||||||
|
expect(transcribeWorksSpeechMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
accessToken: 'access-token',
|
||||||
|
fileName: 'voice.wav',
|
||||||
|
mimeType: 'audio/wav',
|
||||||
|
language: 'zh',
|
||||||
|
}));
|
||||||
|
const audioBase64 = transcribeWorksSpeechMock.mock.calls[0]?.[0]?.audioBase64 as string;
|
||||||
|
const wavBytes = Uint8Array.from(atob(audioBase64), (character) => character.charCodeAt(0));
|
||||||
|
const wavView = new DataView(wavBytes.buffer);
|
||||||
|
expect(new TextDecoder().decode(wavBytes.slice(0, 4))).toBe('RIFF');
|
||||||
|
expect(wavView.getUint16(22, true)).toBe(1);
|
||||||
|
expect(wavView.getUint32(24, true)).toBe(16_000);
|
||||||
|
expect(stopTrack).toHaveBeenCalledOnce();
|
||||||
|
expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce();
|
||||||
|
expect(MockMediaRecorder.latest?.state).toBe('inactive');
|
||||||
|
await act(async () => {
|
||||||
|
pendingSend.resolve(conversationFixture());
|
||||||
|
await pendingSend.promise;
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('discards a late transcription after Conversation switch', async () => {
|
||||||
|
installMockVoiceWavTranscoder();
|
||||||
|
const transcription = deferred<{ text: string; model: string }>();
|
||||||
|
transcribeWorksSpeechMock.mockReturnValueOnce(transcription.promise);
|
||||||
|
const stopTrack = vi.fn();
|
||||||
|
const getUserMedia = vi.fn().mockResolvedValue({
|
||||||
|
getTracks: () => [{ stop: stopTrack }],
|
||||||
|
});
|
||||||
|
class MockMediaRecorder {
|
||||||
|
static isTypeSupported(): boolean { return true; }
|
||||||
|
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||||
|
onstop: (() => void) | null = null;
|
||||||
|
state: RecordingState = 'inactive';
|
||||||
|
mimeType = 'audio/webm';
|
||||||
|
start(): void { this.state = 'recording'; }
|
||||||
|
stop(): void {
|
||||||
|
this.state = 'inactive';
|
||||||
|
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||||
|
this.onstop?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.defineProperty(navigator, 'mediaDevices', {
|
||||||
|
configurable: true,
|
||||||
|
value: { getUserMedia },
|
||||||
|
});
|
||||||
|
Object.defineProperty(window, 'MediaRecorder', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockMediaRecorder,
|
||||||
|
});
|
||||||
|
useAuthStore.setState({
|
||||||
|
accessToken: 'access-token',
|
||||||
|
tokenType: 'Bearer',
|
||||||
|
expiresAt: Date.now() + 60_000,
|
||||||
|
lastActiveAt: Date.now(),
|
||||||
|
canRefresh: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||||
|
await screen.findByTestId('image-workspace-conversation');
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||||
|
await screen.findByRole('button', { name: '停止语音输入' });
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||||
|
await waitFor(() => expect(transcribeWorksSpeechMock).toHaveBeenCalledOnce());
|
||||||
|
|
||||||
|
act(() => {
|
||||||
|
useImageWorkspaceStore.setState({
|
||||||
|
activeConversationId: 'conversation-two',
|
||||||
|
conversation: {
|
||||||
|
...conversationFixture(),
|
||||||
|
conversationId: 'conversation-two',
|
||||||
|
title: '新会话',
|
||||||
|
messages: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
expect(stopTrack).toHaveBeenCalledOnce();
|
||||||
|
await act(async () => {
|
||||||
|
transcription.resolve({ text: '不应写入新会话', model: 'gpt-4o-mini-transcribe' });
|
||||||
|
await transcription.promise;
|
||||||
|
});
|
||||||
|
await waitFor(() => expect(screen.getByLabelText('设计需求')).toHaveValue(''));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stops an active recorder and releases its stream when Conversation changes', async () => {
|
||||||
|
const stopTrack = vi.fn();
|
||||||
|
const getUserMedia = vi.fn().mockResolvedValue({
|
||||||
|
getTracks: () => [{ stop: stopTrack }],
|
||||||
|
});
|
||||||
|
class MockMediaRecorder {
|
||||||
|
static latest: MockMediaRecorder | null = null;
|
||||||
|
static isTypeSupported(): boolean { return true; }
|
||||||
|
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||||
|
onstop: (() => void) | null = null;
|
||||||
|
state: RecordingState = 'inactive';
|
||||||
|
mimeType = 'audio/webm';
|
||||||
|
readonly stop = vi.fn(() => {
|
||||||
|
this.state = 'inactive';
|
||||||
|
this.ondataavailable?.({ data: new Blob(['late audio'], { type: 'audio/webm' }) });
|
||||||
|
this.onstop?.();
|
||||||
|
});
|
||||||
|
constructor() { MockMediaRecorder.latest = this; }
|
||||||
|
start(): void { this.state = 'recording'; }
|
||||||
|
}
|
||||||
|
Object.defineProperty(navigator, 'mediaDevices', {
|
||||||
|
configurable: true,
|
||||||
|
value: { getUserMedia },
|
||||||
|
});
|
||||||
|
Object.defineProperty(window, 'MediaRecorder', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockMediaRecorder,
|
||||||
|
});
|
||||||
|
useAuthStore.setState({
|
||||||
|
accessToken: 'access-token',
|
||||||
|
tokenType: 'Bearer',
|
||||||
|
expiresAt: Date.now() + 60_000,
|
||||||
|
lastActiveAt: Date.now(),
|
||||||
|
canRefresh: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||||
|
await screen.findByTestId('image-workspace-conversation');
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||||
|
await screen.findByRole('button', { name: '停止语音输入' });
|
||||||
|
expect(MockMediaRecorder.latest?.state).toBe('recording');
|
||||||
|
expect(stopTrack).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
act(() => {
|
||||||
|
useImageWorkspaceStore.setState({
|
||||||
|
activeConversationId: 'conversation-two',
|
||||||
|
conversation: {
|
||||||
|
...conversationFixture(),
|
||||||
|
conversationId: 'conversation-two',
|
||||||
|
title: '新会话',
|
||||||
|
messages: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await waitFor(() => expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled());
|
||||||
|
expect(MockMediaRecorder.latest?.stop).toHaveBeenCalledOnce();
|
||||||
|
expect(stopTrack).toHaveBeenCalledOnce();
|
||||||
|
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||||
|
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not use a switched account token for an earlier voice recording', async () => {
|
||||||
|
installMockVoiceWavTranscoder();
|
||||||
|
const tokenLookup = deferred<string>();
|
||||||
|
const getValidAccessToken = vi.fn().mockReturnValue(tokenLookup.promise);
|
||||||
|
const stopTrack = vi.fn();
|
||||||
|
const getUserMedia = vi.fn().mockResolvedValue({
|
||||||
|
getTracks: () => [{ stop: stopTrack }],
|
||||||
|
});
|
||||||
|
class MockMediaRecorder {
|
||||||
|
static isTypeSupported(): boolean { return true; }
|
||||||
|
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||||
|
onstop: (() => void) | null = null;
|
||||||
|
state: RecordingState = 'inactive';
|
||||||
|
mimeType = 'audio/webm';
|
||||||
|
start(): void { this.state = 'recording'; }
|
||||||
|
stop(): void {
|
||||||
|
this.state = 'inactive';
|
||||||
|
this.ondataavailable?.({ data: new Blob(['old account audio'], { type: 'audio/webm' }) });
|
||||||
|
this.onstop?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.defineProperty(navigator, 'mediaDevices', {
|
||||||
|
configurable: true,
|
||||||
|
value: { getUserMedia },
|
||||||
|
});
|
||||||
|
Object.defineProperty(window, 'MediaRecorder', {
|
||||||
|
configurable: true,
|
||||||
|
value: MockMediaRecorder,
|
||||||
|
});
|
||||||
|
useAuthStore.setState({
|
||||||
|
accessToken: 'old-account-token',
|
||||||
|
tokenType: 'Bearer',
|
||||||
|
expiresAt: Date.now() + 60_000,
|
||||||
|
lastActiveAt: Date.now(),
|
||||||
|
canRefresh: true,
|
||||||
|
getValidAccessToken,
|
||||||
|
user: {
|
||||||
|
username: 'old-account',
|
||||||
|
userId: 'old-user-id',
|
||||||
|
tenantId: null,
|
||||||
|
deptId: null,
|
||||||
|
authorities: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||||
|
await screen.findByTestId('image-workspace-conversation');
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||||
|
await screen.findByRole('button', { name: '停止语音输入' });
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||||
|
await waitFor(() => expect(getValidAccessToken).toHaveBeenCalledOnce());
|
||||||
|
|
||||||
|
act(() => {
|
||||||
|
useAuthStore.setState({
|
||||||
|
accessToken: 'new-account-token',
|
||||||
|
user: {
|
||||||
|
username: 'new-account',
|
||||||
|
userId: 'new-user-id',
|
||||||
|
tenantId: null,
|
||||||
|
deptId: null,
|
||||||
|
authorities: [],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await act(async () => {
|
||||||
|
tokenLookup.resolve('new-account-token');
|
||||||
|
await tokenLookup.promise;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(stopTrack).toHaveBeenCalledOnce();
|
||||||
|
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||||
|
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||||
|
});
|
||||||
|
|
||||||
it('renders the pending user turn and assistant deltas while the command is running', async () => {
|
it('renders the pending user turn and assistant deltas while the command is running', async () => {
|
||||||
const response = deferred<DesignConversation>();
|
const response = deferred<DesignConversation>();
|
||||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(response.promise);
|
sendImageWorkspaceMessageMock.mockReturnValueOnce(response.promise);
|
||||||
|
|||||||
Reference in new issue
Block a user