feat: enable AI design voice input
This commit is contained in:
@@ -81,7 +81,7 @@ test.describe('AI Design workspace', () => {
|
||||
|
||||
const firstPrompt = '为项目设计一张蓝色星空海报';
|
||||
await page.getByLabel('设计需求').fill(firstPrompt);
|
||||
await page.getByRole('button', { name: '发送给设计 Agent' }).click();
|
||||
await page.getByLabel('设计需求').press('Enter');
|
||||
await expect(page.getByTestId('image-workspace-conversation')).toContainText(firstPrompt);
|
||||
await expect(page.getByText('设计 Agent', { exact: true })).toHaveCount(0);
|
||||
await expect(page.getByText('你', { exact: true })).toHaveCount(0);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { toast } from 'sonner';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ImageWorkspaceApiError } from '@/lib/image-workspace';
|
||||
import { ImageCanvas } from '@/pages/ImageCanvas';
|
||||
@@ -27,6 +28,9 @@ const openImageWorkspaceTaskEventsMock = vi.hoisted(() => vi.fn());
|
||||
const resolveImageWorkspaceAssetUrlMock = vi.hoisted(() => vi.fn());
|
||||
const saveImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const uploadImageWorkspaceAssetMock = vi.hoisted(() => vi.fn());
|
||||
const transcribeWorksSpeechMock = vi.hoisted(() => vi.fn());
|
||||
const originalGetValidAccessToken = useAuthStore.getState().getValidAccessToken;
|
||||
const originalRefreshSession = useAuthStore.getState().refreshSession;
|
||||
|
||||
type EventListener = (event: MessageEvent<string>) => void;
|
||||
|
||||
@@ -88,6 +92,42 @@ vi.mock('@/lib/image-workspace', async (importOriginal) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('@/lib/works-square', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@/lib/works-square')>();
|
||||
return {
|
||||
...actual,
|
||||
transcribeWorksSpeech: (...args: unknown[]) => transcribeWorksSpeechMock(...args),
|
||||
};
|
||||
});
|
||||
|
||||
function installMockVoiceWavTranscoder(): void {
|
||||
const pcm = Float32Array.from([0, 0.5, -0.5, 1, -1]);
|
||||
class MockAudioContext {
|
||||
decodeAudioData = vi.fn().mockResolvedValue({
|
||||
duration: pcm.length / 48_000,
|
||||
getChannelData: () => pcm,
|
||||
});
|
||||
close = vi.fn().mockResolvedValue(undefined);
|
||||
}
|
||||
class MockOfflineAudioContext {
|
||||
destination = {};
|
||||
createBufferSource = vi.fn(() => ({
|
||||
buffer: null,
|
||||
connect: vi.fn(),
|
||||
start: vi.fn(),
|
||||
}));
|
||||
startRendering = vi.fn().mockResolvedValue({ getChannelData: () => pcm });
|
||||
}
|
||||
Object.defineProperty(window, 'AudioContext', {
|
||||
configurable: true,
|
||||
value: MockAudioContext,
|
||||
});
|
||||
Object.defineProperty(window, 'OfflineAudioContext', {
|
||||
configurable: true,
|
||||
value: MockOfflineAudioContext,
|
||||
});
|
||||
}
|
||||
|
||||
const bootstrapFixture: DesignWorkspaceBootstrap = {
|
||||
capabilities: {
|
||||
conversation: true,
|
||||
@@ -243,6 +283,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useAuthStore.setState({
|
||||
getValidAccessToken: originalGetValidAccessToken,
|
||||
refreshSession: originalRefreshSession,
|
||||
});
|
||||
useImageWorkspaceStore.getState().reset();
|
||||
useImagePromptMuseumStore.getState().clearPendingPrompt();
|
||||
fetchImageWorkspaceMock.mockResolvedValue(bootstrapFixture);
|
||||
@@ -283,6 +327,10 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
...taskFixture.resultAssets[0],
|
||||
assetId: 'asset-uploaded',
|
||||
});
|
||||
transcribeWorksSpeechMock.mockResolvedValue({
|
||||
text: '让天空更通透',
|
||||
model: 'gpt-4o-mini-transcribe',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows an honest unavailable state without fabricating projects', async () => {
|
||||
@@ -754,7 +802,7 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(screen.getByTestId('image-workspace-composer-surface'))
|
||||
.toHaveClass('rounded-lg', 'flex', 'min-w-0');
|
||||
expect(screen.getByRole('button', { name: '上传参考图' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeDisabled();
|
||||
expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled();
|
||||
expect(screen.queryByText('Enter 发送 · Shift+Enter 换行')).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '上传参考图' }));
|
||||
@@ -831,6 +879,316 @@ describe('ImageCanvas Workspace-first design experience', () => {
|
||||
expect(composer).toHaveValue('正在输入中文设计描述');
|
||||
});
|
||||
|
||||
it('reports fixed login and environment guards before starting voice input', async () => {
|
||||
const toastError = vi.spyOn(toast, 'error').mockImplementation(() => 'toast-id');
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: undefined,
|
||||
});
|
||||
useAuthStore.setState({ accessToken: null, canRefresh: false });
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
expect(toastError).toHaveBeenLastCalledWith('请先登录后再使用语音输入');
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
});
|
||||
});
|
||||
const voiceButton = screen.getByRole('button', { name: '语音输入' });
|
||||
expect(voiceButton).toHaveAttribute('title', '当前环境不支持语音输入');
|
||||
fireEvent.click(voiceButton);
|
||||
expect(toastError).toHaveBeenLastCalledWith('当前环境不支持语音输入');
|
||||
toastError.mockRestore();
|
||||
});
|
||||
|
||||
it('records Works Speech input into the current draft and releases the microphone', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const pendingSend = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(pendingSend.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
initialized: true,
|
||||
loading: false,
|
||||
error: null,
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
legacyRefreshToken: null,
|
||||
user: {
|
||||
username: 'designer',
|
||||
userId: '42',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
const composer = screen.getByLabelText('设计需求');
|
||||
fireEvent.change(composer, { target: { value: '保留当前构图' } });
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
|
||||
await waitFor(() => expect(getUserMedia).toHaveBeenCalledWith({ audio: true }));
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '发送给设计 Agent' }));
|
||||
await waitFor(() => expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce());
|
||||
expect(screen.getByRole('button', { name: '停止语音输入' })).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
|
||||
await waitFor(() => expect(composer).toHaveValue('让天空更通透'));
|
||||
expect(transcribeWorksSpeechMock).toHaveBeenCalledWith(expect.objectContaining({
|
||||
accessToken: 'access-token',
|
||||
fileName: 'voice.wav',
|
||||
mimeType: 'audio/wav',
|
||||
language: 'zh',
|
||||
}));
|
||||
const audioBase64 = transcribeWorksSpeechMock.mock.calls[0]?.[0]?.audioBase64 as string;
|
||||
const wavBytes = Uint8Array.from(atob(audioBase64), (character) => character.charCodeAt(0));
|
||||
const wavView = new DataView(wavBytes.buffer);
|
||||
expect(new TextDecoder().decode(wavBytes.slice(0, 4))).toBe('RIFF');
|
||||
expect(wavView.getUint16(22, true)).toBe(1);
|
||||
expect(wavView.getUint32(24, true)).toBe(16_000);
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(sendImageWorkspaceMessageMock).toHaveBeenCalledOnce();
|
||||
expect(MockMediaRecorder.latest?.state).toBe('inactive');
|
||||
await act(async () => {
|
||||
pendingSend.resolve(conversationFixture());
|
||||
await pendingSend.promise;
|
||||
});
|
||||
});
|
||||
|
||||
it('discards a late transcription after Conversation switch', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const transcription = deferred<{ text: string; model: string }>();
|
||||
transcribeWorksSpeechMock.mockReturnValueOnce(transcription.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['audio bytes'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(transcribeWorksSpeechMock).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
await act(async () => {
|
||||
transcription.resolve({ text: '不应写入新会话', model: 'gpt-4o-mini-transcribe' });
|
||||
await transcription.promise;
|
||||
});
|
||||
await waitFor(() => expect(screen.getByLabelText('设计需求')).toHaveValue(''));
|
||||
});
|
||||
|
||||
it('stops an active recorder and releases its stream when Conversation changes', async () => {
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static latest: MockMediaRecorder | null = null;
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
readonly stop = vi.fn(() => {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['late audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
});
|
||||
constructor() { MockMediaRecorder.latest = this; }
|
||||
start(): void { this.state = 'recording'; }
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'access-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
expect(MockMediaRecorder.latest?.state).toBe('recording');
|
||||
expect(stopTrack).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
useImageWorkspaceStore.setState({
|
||||
activeConversationId: 'conversation-two',
|
||||
conversation: {
|
||||
...conversationFixture(),
|
||||
conversationId: 'conversation-two',
|
||||
title: '新会话',
|
||||
messages: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await waitFor(() => expect(screen.getByRole('button', { name: '语音输入' })).toBeEnabled());
|
||||
expect(MockMediaRecorder.latest?.stop).toHaveBeenCalledOnce();
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('does not use a switched account token for an earlier voice recording', async () => {
|
||||
installMockVoiceWavTranscoder();
|
||||
const tokenLookup = deferred<string>();
|
||||
const getValidAccessToken = vi.fn().mockReturnValue(tokenLookup.promise);
|
||||
const stopTrack = vi.fn();
|
||||
const getUserMedia = vi.fn().mockResolvedValue({
|
||||
getTracks: () => [{ stop: stopTrack }],
|
||||
});
|
||||
class MockMediaRecorder {
|
||||
static isTypeSupported(): boolean { return true; }
|
||||
ondataavailable: ((event: { data: Blob }) => void) | null = null;
|
||||
onstop: (() => void) | null = null;
|
||||
state: RecordingState = 'inactive';
|
||||
mimeType = 'audio/webm';
|
||||
start(): void { this.state = 'recording'; }
|
||||
stop(): void {
|
||||
this.state = 'inactive';
|
||||
this.ondataavailable?.({ data: new Blob(['old account audio'], { type: 'audio/webm' }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia },
|
||||
});
|
||||
Object.defineProperty(window, 'MediaRecorder', {
|
||||
configurable: true,
|
||||
value: MockMediaRecorder,
|
||||
});
|
||||
useAuthStore.setState({
|
||||
accessToken: 'old-account-token',
|
||||
tokenType: 'Bearer',
|
||||
expiresAt: Date.now() + 60_000,
|
||||
lastActiveAt: Date.now(),
|
||||
canRefresh: true,
|
||||
getValidAccessToken,
|
||||
user: {
|
||||
username: 'old-account',
|
||||
userId: 'old-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
|
||||
render(<MemoryRouter><ImageCanvas /></MemoryRouter>);
|
||||
await screen.findByTestId('image-workspace-conversation');
|
||||
fireEvent.click(screen.getByRole('button', { name: '语音输入' }));
|
||||
await screen.findByRole('button', { name: '停止语音输入' });
|
||||
fireEvent.click(screen.getByRole('button', { name: '停止语音输入' }));
|
||||
await waitFor(() => expect(getValidAccessToken).toHaveBeenCalledOnce());
|
||||
|
||||
act(() => {
|
||||
useAuthStore.setState({
|
||||
accessToken: 'new-account-token',
|
||||
user: {
|
||||
username: 'new-account',
|
||||
userId: 'new-user-id',
|
||||
tenantId: null,
|
||||
deptId: null,
|
||||
authorities: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
await act(async () => {
|
||||
tokenLookup.resolve('new-account-token');
|
||||
await tokenLookup.promise;
|
||||
});
|
||||
|
||||
expect(stopTrack).toHaveBeenCalledOnce();
|
||||
expect(transcribeWorksSpeechMock).not.toHaveBeenCalled();
|
||||
expect(screen.getByLabelText('设计需求')).toHaveValue('');
|
||||
});
|
||||
|
||||
it('renders the pending user turn and assistant deltas while the command is running', async () => {
|
||||
const response = deferred<DesignConversation>();
|
||||
sendImageWorkspaceMessageMock.mockReturnValueOnce(response.promise);
|
||||
|
||||
Reference in New Issue
Block a user