wangxuming/makelore
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: enable AI design voice input

Browse Source
This commit is contained in:
brother7
2026-08-18 02:30:39 +08:00
parent 13bdc0c9ba
commit 1ba5b9cc84
4 changed files with 751 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
src/pages/ImageCanvas/index.tsx
View File

@@ -1,4 +1,5 @@
import {
useCallback,
useEffect,
useMemo,
useRef,
@@ -18,6 +19,7 @@ import {
Plus,
RefreshCw,
Send,
Square,
Sparkles,
Upload,
WandSparkles,
@@ -42,9 +44,11 @@ import {
uploadImageWorkspaceAsset,
} from '@/lib/image-workspace';
import { cn } from '@/lib/utils';
import { transcribeWorksSpeech, WorksSquareApiError } from '@/lib/works-square';
import { useImageWorkspaceStore } from '@/stores/image-workspace';
import { useImagePromptMuseumStore } from '@/stores/image-prompt-museum';
import { useAuthStore } from '@/stores/auth';
import { getProfileAccountKey } from '@/stores/user-profile';
import logoSvg from '@/assets/logo.svg';
import type {
DesignAsset,
@@ -81,6 +85,113 @@ const GENERATION_CONFIRMATION_INTENTS = new Set([
]);
const IMAGE_SOURCE_PICKER_REPLY = '从作品列表选择图片';
const TASK_CARD_DESCRIPTION_MAX_LENGTH = 24;
const VOICE_WAV_SAMPLE_RATE = 16_000;
const VOICE_WAV_CHANNELS = 1;
const VOICE_WAV_BITS_PER_SAMPLE = 16;
type OfflineAudioContextConstructor = new (
numberOfChannels: number,
length: number,
sampleRate: number,
) => OfflineAudioContext;
type AudioContextWindow = Window & {
webkitAudioContext?: typeof AudioContext;
webkitOfflineAudioContext?: OfflineAudioContextConstructor;
};
async function blobToBase64(blob: Blob): Promise<string> {
const bytes = new Uint8Array(await blob.arrayBuffer());
let binary = '';
const chunkSize = 0x8000;
for (let index = 0; index < bytes.length; index += chunkSize) {
binary += String.fromCharCode(...bytes.subarray(index, index + chunkSize));
}
return btoa(binary);
}
function writeAscii(view: DataView, offset: number, value: string): void {
for (let index = 0; index < value.length; index += 1) {
view.setUint8(offset + index, value.charCodeAt(index));
}
}
function audioBufferTo16BitPcmWav(audioBuffer: AudioBuffer): Blob {
const samples = audioBuffer.getChannelData(0);
const bytesPerSample = VOICE_WAV_BITS_PER_SAMPLE / 8;
const blockAlign = VOICE_WAV_CHANNELS * bytesPerSample;
const dataSize = samples.length * blockAlign;
const wavBuffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(wavBuffer);
writeAscii(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeAscii(view, 8, 'WAVE');
writeAscii(view, 12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, VOICE_WAV_CHANNELS, true);
view.setUint32(24, VOICE_WAV_SAMPLE_RATE, true);
view.setUint32(28, VOICE_WAV_SAMPLE_RATE * blockAlign, true);
view.setUint16(32, blockAlign, true);
view.setUint16(34, VOICE_WAV_BITS_PER_SAMPLE, true);
writeAscii(view, 36, 'data');
view.setUint32(40, dataSize, true);
let offset = 44;
for (const sample of samples) {
const clamped = Math.max(-1, Math.min(1, sample));
view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
offset += bytesPerSample;
}
return new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' });
}
async function convertAudioBlobTo16kMonoWav(audioBlob: Blob): Promise<Blob> {
const audioWindow = window as AudioContextWindow;
const AudioContextCtor = window.AudioContext ?? audioWindow.webkitAudioContext;
const OfflineAudioContextCtor = window.OfflineAudioContext
?? audioWindow.webkitOfflineAudioContext;
if (!AudioContextCtor || !OfflineAudioContextCtor) {
throw new Error('当前环境无法转换录音');
}
const audioContext = new AudioContextCtor();
let decodedBuffer: AudioBuffer;
try {
decodedBuffer = await audioContext.decodeAudioData(await audioBlob.arrayBuffer());
} finally {
await audioContext.close().catch(() => undefined);
}
const frameCount = Math.max(1, Math.ceil(decodedBuffer.duration * VOICE_WAV_SAMPLE_RATE));
const offlineContext = new OfflineAudioContextCtor(
VOICE_WAV_CHANNELS,
frameCount,
VOICE_WAV_SAMPLE_RATE,
);
const source = offlineContext.createBufferSource();
source.buffer = decodedBuffer;
source.connect(offlineContext.destination);
source.start(0);
return audioBufferTo16BitPcmWav(await offlineContext.startRendering());
}
function isUnauthorizedWorksError(error: unknown): boolean {
if (error instanceof WorksSquareApiError && error.statusCode === 401) return true;
if (!(error instanceof Error)) return false;
const record = error as Error & { code?: unknown; details?: { status?: unknown } };
const message = error.message.toLowerCase();
return record.details?.status === 401
|| record.code === 'AUTH_INVALID'
|| message.includes('401')
|| message.includes('unauthorized')
|| message.includes('invalid access token');
}
function currentAuthAccountKey(): string | null {
const user = useAuthStore.getState().user;
return getProfileAccountKey(user?.userId ?? user?.username);
}
function isGenerationConfirmationIntent(message: string): boolean {
const normalized = message.trim().replace(/[\s，,。.!！?？、]/g, '');
@@ -782,6 +893,12 @@ function QuoteCard({
export function ImageCanvas() {
const authenticated = useAuthStore((state) => state.isAuthenticated());
const accountKey = useAuthStore((state) => (
getProfileAccountKey(state.user?.userId ?? state.user?.username)
));
const accessToken = useAuthStore((state) => state.accessToken);
const getValidAccessToken = useAuthStore((state) => state.getValidAccessToken);
const refreshSession = useAuthStore((state) => state.refreshSession);
const status = useImageWorkspaceStore((state) => state.status);
const bootstrap = useImageWorkspaceStore((state) => state.bootstrap);
const workspace = useImageWorkspaceStore((state) => state.workspace);
@@ -817,12 +934,24 @@ export function ImageCanvas() {
const [imageSourceBusyKey, setImageSourceBusyKey] = useState<string | null>(null);
const [imageSourceError, setImageSourceError] = useState<string | null>(null);
const [uploadedImageSourceAsset, setUploadedImageSourceAsset] = useState<DesignAsset | null>(null);
const [voiceInputState, setVoiceInputState] = useState<'idle' | 'recording' | 'transcribing'>('idle');
const conversationEndRef = useRef<HTMLDivElement | null>(null);
const promptRef = useRef<HTMLTextAreaElement | null>(null);
const referenceUploadInputRef = useRef<HTMLInputElement | null>(null);
const promptSelectionRef = useRef({ start: 0, end: 0 });
const quoteRepriceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const quoteDraftVersionRef = useRef(0);
const voiceRecorderRef = useRef<MediaRecorder | null>(null);
const voiceStreamRef = useRef<MediaStream | null>(null);
const voiceChunksRef = useRef<Blob[]>([]);
const voiceOperationRef = useRef(0);
const voiceLifecycleRevisionRef = useRef(0);
const renderedAccountKeyRef = useRef(accountKey);
if (renderedAccountKeyRef.current !== accountKey) {
renderedAccountKeyRef.current = accountKey;
voiceLifecycleRevisionRef.current += 1;
}
const renderVoiceLifecycleRevision = voiceLifecycleRevisionRef.current;
const quote = useMemo(
() => conversation ? activeQuote(conversation.messages) : null,
@@ -859,6 +988,13 @@ export function ImageCanvas() {
});
}, [referenceAssets, referenceMentionTrigger, selectedReferenceAssetIds]);
const mentionMenuOpen = Boolean(referenceMentionTrigger && referenceMentionOptions.length > 0);
const canUseVoiceInput = voiceInputState === 'recording' || (
Boolean(workspace && conversation)
&& !submittingConversationId
&& !imageSourceBusyKey
&& !referenceUploadBusy
&& voiceInputState !== 'transcribing'
);
const selectingImageReference = conversation?.brief.medium === 'image';
const conversationMessages = useMemo<RenderedDesignMessage[]>(() => {
if (!conversation) return [];
@@ -1148,6 +1284,184 @@ export function ImageCanvas() {
}
};
const stopVoiceStream = useCallback(() => {
for (const track of voiceStreamRef.current?.getTracks() ?? []) track.stop();
voiceStreamRef.current = null;
}, []);
useEffect(() => {
return () => {
voiceOperationRef.current += 1;
const recorder = voiceRecorderRef.current;
if (recorder) {
recorder.ondataavailable = null;
recorder.onstop = null;
if (recorder.state !== 'inactive') recorder.stop();
voiceRecorderRef.current = null;
}
voiceChunksRef.current = [];
stopVoiceStream();
setVoiceInputState('idle');
};
}, [accountKey, conversation?.conversationId, stopVoiceStream, workspace?.workspaceId]);
const finishVoiceTranscription = useCallback(async (
mimeType: string,
operationId: number,
requestedAccountKey: string,
lifecycleRevision: number,
requestedWorkspaceId: string,
requestedConversationId: string,
) => {
const isActive = () => voiceOperationRef.current === operationId
&& voiceLifecycleRevisionRef.current === lifecycleRevision
&& renderedAccountKeyRef.current === requestedAccountKey
&& currentAuthAccountKey() === requestedAccountKey
&& useImageWorkspaceStore.getState().activeWorkspaceId === requestedWorkspaceId
&& useImageWorkspaceStore.getState().activeConversationId === requestedConversationId;
const chunks = voiceChunksRef.current;
voiceChunksRef.current = [];
stopVoiceStream();
if (chunks.length === 0) {
if (isActive()) {
setVoiceInputState('idle');
toast.warning('没有录到声音');
}
return;
}
if (isActive()) setVoiceInputState('transcribing');
try {
if (!isActive()) return;
const validAccessToken = await getValidAccessToken();
if (!isActive()) return;
if (!validAccessToken) {
toast.error('请先登录后再使用语音输入');
return;
}
const audioBlob = new Blob(chunks, { type: mimeType || chunks[0]?.type || 'audio/webm' });
const wavBlob = await convertAudioBlobTo16kMonoWav(audioBlob);
if (!isActive()) return;
const audioBase64 = await blobToBase64(wavBlob);
if (!isActive()) return;
const transcribeWithToken = (token: string) => transcribeWorksSpeech({
accessToken: token,
audioBase64,
fileName: 'voice.wav',
mimeType: 'audio/wav',
language: 'zh',
});
let transcription;
try {
if (!isActive()) return;
transcription = await transcribeWithToken(validAccessToken);
if (!isActive()) return;
} catch (error) {
if (!isUnauthorizedWorksError(error)) throw error;
if (!isActive()) return;
const refreshedAccessToken = await refreshSession();
if (!isActive()) return;
if (!refreshedAccessToken) throw error;
transcription = await transcribeWithToken(refreshedAccessToken);
if (!isActive()) return;
}
const transcript = transcription.text.trim();
if (transcript) {
setPrompt((currentPrompt) => currentPrompt.trim()
? `${currentPrompt.trim()}\n${transcript}`
: transcript);
setActionError(null);
}
} catch (error) {
if (isActive()) {
toast.error('语音识别失败', {
description: error instanceof Error ? error.message : String(error),
});
}
} finally {
if (isActive()) {
voiceRecorderRef.current = null;
setVoiceInputState('idle');
}
}
}, [getValidAccessToken, refreshSession, stopVoiceStream]);
const handleVoiceInputClick = useCallback(async () => {
if (voiceLifecycleRevisionRef.current !== renderVoiceLifecycleRevision
|| renderedAccountKeyRef.current !== accountKey) return;
if (voiceInputState === 'recording') {
const recorder = voiceRecorderRef.current;
if (!recorder || recorder.state === 'inactive') {
stopVoiceStream();
setVoiceInputState('idle');
return;
}
setVoiceInputState('transcribing');
recorder.stop();
return;
}
if (!canUseVoiceInput || !workspace || !conversation) return;
if (!accountKey || !accessToken) {
toast.error('请先登录后再使用语音输入');
return;
}
if (!navigator.mediaDevices?.getUserMedia || typeof window.MediaRecorder === 'undefined') {
toast.error('当前环境不支持语音输入');
return;
}
const operationId = voiceOperationRef.current + 1;
voiceOperationRef.current = operationId;
const lifecycleRevision = voiceLifecycleRevisionRef.current;
const requestedAccountKey = accountKey;
const isActive = () => voiceOperationRef.current === operationId
&& voiceLifecycleRevisionRef.current === lifecycleRevision
&& renderedAccountKeyRef.current === requestedAccountKey
&& currentAuthAccountKey() === requestedAccountKey;
const requestedWorkspaceId = workspace.workspaceId;
const requestedConversationId = conversation.conversationId;
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
if (!isActive()) {
for (const track of stream.getTracks()) track.stop();
return;
}
const preferredMimeType = window.MediaRecorder.isTypeSupported?.('audio/webm')
? 'audio/webm'
: '';
const recorder = new window.MediaRecorder(
stream,
preferredMimeType ? { mimeType: preferredMimeType } : undefined,
);
voiceStreamRef.current = stream;
voiceChunksRef.current = [];
voiceRecorderRef.current = recorder;
recorder.ondataavailable = (event) => {
if (isActive() && event.data.size > 0) voiceChunksRef.current.push(event.data);
};
recorder.onstop = () => {
void finishVoiceTranscription(
recorder.mimeType || preferredMimeType || 'audio/webm',
operationId,
requestedAccountKey,
lifecycleRevision,
requestedWorkspaceId,
requestedConversationId,
);
};
recorder.start();
setVoiceInputState('recording');
} catch (error) {
stopVoiceStream();
if (isActive()) {
setVoiceInputState('idle');
toast.error('无法开始录音', {
description: error instanceof Error ? error.message : String(error),
});
}
}
}, [accessToken, accountKey, canUseVoiceInput, conversation, finishVoiceTranscription, renderVoiceLifecycleRevision, stopVoiceStream, voiceInputState, workspace]);
const handleSend = async (override?: string) => {
const message = (override ?? prompt).trim();
if (!workspace || !conversation || !message || submitting) return;
@@ -1605,12 +1919,27 @@ export function ImageCanvas() {
<Button
type="button"
variant="ghost"
disabled
className="motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none"
aria-label="语音输入"
title="AI 绘画暂不支持语音输入"
disabled={!canUseVoiceInput}
className={cn(
'motion-press h-8 w-8 rounded-full border-0 bg-transparent p-0 text-muted-foreground shadow-none hover:bg-surface-subtle hover:text-foreground',
voiceInputState !== 'idle' && 'bg-surface-subtle text-foreground',
)}
aria-label={voiceInputState === 'recording'
? '停止语音输入'
: voiceInputState === 'transcribing'
? '正在识别语音'
: '语音输入'}
title={typeof window.MediaRecorder === 'undefined'
|| !navigator.mediaDevices?.getUserMedia
? '当前环境不支持语音输入'
: '语音输入'}
onClick={() => void handleVoiceInputClick()}
>
<Mic className="h-4 w-4" />
{voiceInputState === 'transcribing'
? <Loader2 className="h-4 w-4 animate-spin" />
: voiceInputState === 'recording'
? <Square className="h-4 w-4 fill-current" />
: <Mic className="h-4 w-4" />}
</Button>
<Button
type="submit"