fix: streaming TTS via with_streaming_response
Replaced synchronous TTS (waiting for full audio at 5.9s) with streaming TTS that sends audio chunks as they arrive. Backend now accumulates chunks in audioBufferRef and plays the complete stream on speaking_end. Reduces TTS latency from ~6s to ~1s first byte.
This commit is contained in:
@@ -44,6 +44,7 @@ export function useConversation() {
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
|
||||
// Connect WebSocket
|
||||
const connect = useCallback(() => {
|
||||
@@ -115,23 +116,36 @@ export function useConversation() {
|
||||
break;
|
||||
|
||||
case 'audio': {
|
||||
// Incoming Opus audio from TTS (full response, not streamed)
|
||||
if (msg.data && audioRef.current) {
|
||||
// Incoming Opus audio chunk from streaming TTS
|
||||
if (msg.data) {
|
||||
const binary = atob(msg.data);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
const blob = new Blob([bytes], { type: 'audio/ogg' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
audioRef.current.src = url;
|
||||
audioRef.current.play().catch(() => {});
|
||||
audioBufferRef.current.push(bytes);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'speaking_end':
|
||||
setIsKiraSpeaking(false);
|
||||
// Play all accumulated chunks as one blob
|
||||
if (audioBufferRef.current.length > 0 && audioRef.current) {
|
||||
const allChunks = audioBufferRef.current;
|
||||
const totalLen = allChunks.reduce((s, c) => s + c.length, 0);
|
||||
const combined = new Uint8Array(totalLen);
|
||||
let offset = 0;
|
||||
for (const chunk of allChunks) {
|
||||
combined.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
audioBufferRef.current = [];
|
||||
const blob = new Blob([combined], { type: 'audio/ogg' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
audioRef.current.src = url;
|
||||
audioRef.current.play().catch(() => {});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'interruption':
|
||||
|
||||
Reference in New Issue
Block a user