fix: streaming TTS via with_streaming_response

Replaced synchronous TTS (waiting for full audio at 5.9s) with
streaming TTS that sends audio chunks as they arrive. Backend now
accumulates chunks in audioBufferRef and plays the complete stream
on speaking_end. Reduces TTS latency from ~6s to ~1s first byte.
This commit is contained in:
2026-06-04 14:17:54 -04:00
parent 2cd5636ad6
commit 9cd183a83b
2 changed files with 31 additions and 18 deletions
+20 -6
View File
@@ -44,6 +44,7 @@ export function useConversation() {
const audioRef = useRef<HTMLAudioElement | null>(null);
const recorderRef = useRef<MediaRecorder | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const audioBufferRef = useRef<Uint8Array[]>([]);
// Connect WebSocket
const connect = useCallback(() => {
@@ -115,23 +116,36 @@ export function useConversation() {
break;
case 'audio': {
// Incoming Opus audio from TTS (full response, not streamed)
if (msg.data && audioRef.current) {
// Incoming Opus audio chunk from streaming TTS
if (msg.data) {
const binary = atob(msg.data);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) {
bytes[i] = binary.charCodeAt(i);
}
const blob = new Blob([bytes], { type: 'audio/ogg' });
const url = URL.createObjectURL(blob);
audioRef.current.src = url;
audioRef.current.play().catch(() => {});
audioBufferRef.current.push(bytes);
}
break;
}
case 'speaking_end':
setIsKiraSpeaking(false);
// Play all accumulated chunks as one blob
if (audioBufferRef.current.length > 0 && audioRef.current) {
const allChunks = audioBufferRef.current;
const totalLen = allChunks.reduce((s, c) => s + c.length, 0);
const combined = new Uint8Array(totalLen);
let offset = 0;
for (const chunk of allChunks) {
combined.set(chunk, offset);
offset += chunk.length;
}
audioBufferRef.current = [];
const blob = new Blob([combined], { type: 'audio/ogg' });
const url = URL.createObjectURL(blob);
audioRef.current.src = url;
audioRef.current.play().catch(() => {});
}
break;
case 'interruption':