From 83a990e838971b7efd4edea514c3aab2cb9db826 Mon Sep 17 00:00:00 2001 From: hobokenchicken Date: Fri, 5 Jun 2026 23:36:29 -0400 Subject: [PATCH] feat(audio): Gemini Live API replaces Whisper+GPT+ElevenLabs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single WebSocket proxy: frontend PCM16 16kHz → backend → Gemini Live API Gemini returns PCM16 24kHz audio + text. Playback via Web Audio API queue. Removed OpenAI/DeepSeek deps. Model: gemini-3.1-flash-live-preview. Voice: Aoede. Streaming bidirectional audio with silence gating. --- backend/config.py | 9 +- backend/main.py | 355 ++++++++++++++----------- backend/requirements.txt | 2 - frontend/src/App.tsx | 4 +- frontend/src/components/ChatBubble.tsx | 12 +- frontend/src/hooks/useConversation.ts | 235 ++++++++-------- 6 files changed, 331 insertions(+), 286 deletions(-) diff --git a/backend/config.py b/backend/config.py index 10ac30f..43a04d2 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,13 +1,8 @@ from pydantic_settings import BaseSettings class Settings(BaseSettings): - # OpenAI (used for STT + TTS) - openai_api_key: str = "" - - # DeepSeek (LLM) - deepseek_api_key: str = "" - deepseek_base_url: str = "https://api.deepseek.com/v1" - deepseek_model: str = "deepseek-chat" + # Gemini Live API + gemini_api_key: str = "" # Honcho (memory) honcho_api_key: str = "" diff --git a/backend/main.py b/backend/main.py index cc9e4d2..1a40a9d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,7 @@ """Kira — AI body double backend -REST STT (gpt-4o-transcribe) → gpt-5.4-nano + Honcho → streaming TTS (sage) +Gemini Live API (gemini-3.1-flash-live-preview) for real-time voice. +Text chat still goes through Gemini generateContent REST endpoint. """ import json @@ -8,7 +9,9 @@ import base64 import uuid import logging import asyncio +import struct +import websockets from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware @@ -33,18 +36,12 @@ BASE_SYSTEM_PROMPT = ( "You speak in a friendly, girly-pop tone. You are helping someone with ADHD " "stay focused and on task. Keep responses short, supportive, and uplifting. " "Check in on them. Remind them to take breaks. Celebrate small wins. " - "Use occasional emoji but don't overdo it. Never be judgmental." + "Use occasional emoji but don't overdo it. Never be judgmental. " + "You are speaking out loud via voice, so keep natural conversational flow." ) -_openai = None - - -def get_openai(): - global _openai - if _openai is None: - from openai import AsyncOpenAI - _openai = AsyncOpenAI(api_key=settings.openai_api_key) - return _openai +GEMINI_WS_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.BidiGenerateContent" +GEMINI_MODEL = "models/gemini-3.1-flash-live-preview" @app.on_event("startup") @@ -61,171 +58,223 @@ async def health(): return {"status": "ok", "name": "kira", "memory": mem_status} -async def transcribe_audio(client, audio_b64: str) -> str | None: - """REST transcription via gpt-4o-transcribe (full utterance blob).""" - try: - audio_bytes = base64.b64decode(audio_b64) - import io - audio_file = io.BytesIO(audio_bytes) - audio_file.name = "audio.webm" - transcript = await client.audio.transcriptions.create( - model="gpt-4o-transcribe", - file=audio_file, - ) - return transcript.text.strip() if transcript.text else None - except Exception as e: - logger.error(f"Transcription error: {e}") - return None - - -async def get_kira_response(client, user_text: str, memory_suffix: str) -> str: - """Get Kira's response from gpt-5.4-nano.""" - system_prompt = BASE_SYSTEM_PROMPT - if memory_suffix: - system_prompt += memory_suffix - resp = await client.chat.completions.create( - model="gpt-5.4-nano", - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_text}, - ], - max_completion_tokens=300, - temperature=0.7, - ) - return resp.choices[0].message.content or "Mhm, I'm here! ✨" - - -async def stream_tts(client, text: str, websocket: WebSocket): - """Stream TTS audio as Opus chunks over WebSocket.""" - await websocket.send_json({"type": "speaking_start", "text": text}) - async with client.audio.speech.with_streaming_response.create( - model="tts-1", - voice="sage", - input=text, - response_format="opus", - ) as tts_resp: - async for chunk in tts_resp.iter_bytes(): - if chunk: - b64 = base64.b64encode(chunk).decode("utf-8") - await websocket.send_json({"type": "audio", "data": b64}) - await websocket.send_json({"type": "speaking_end"}) - - @app.websocket("/api/ws") -async def conversation_ws(websocket: WebSocket): +async def gemini_voice_ws(websocket: WebSocket): + """WebSocket proxy between frontend and Gemini Live API. + + Protocol (frontend ↔ this proxy): + → {"type": "audio", "data": ""} + → {"type": "conversation_text", "text": "..."} + → {"type": "identify", "user_id": "...", "name": "..."} + → {"type": "ping"} + ← {"type": "audio", "data": ""} + ← {"type": "transcript", "role": "user"|"kira", "text": "..."} + ← {"type": "turn_complete"} + ← {"type": "interrupted"} + ← {"type": "error", "message": "..."} + """ await websocket.accept() session_id = str(uuid.uuid4())[:8] user_id = "default-user" - identified = False memory_suffix = "" logger.info(f"[{session_id}] WebSocket connected") - conversation_history: list[dict] = [] + gemini_ws = None + gemini_task = None + frontend_task = None try: - while True: - raw = await websocket.receive_text() - msg = json.loads(raw) - msg_type = msg.get("type", "") + # ── Connect to Gemini Live API ── + gemini_url = f"{GEMINI_WS_URL}?key={settings.gemini_api_key}" + gemini_ws = await websockets.connect(gemini_url, max_size=2**24) - # ── Identity & Preferences ── - if msg_type == "identify": - user_id = msg.get("user_id", "").strip() - user_name = msg.get("name", "").strip() - if user_name and user_id: - kira_memory.set_user_preference(user_id, "name", user_name) + # ── Send setup ── + system_prompt = BASE_SYSTEM_PROMPT + setup_msg = { + "setup": { + "model": GEMINI_MODEL, + "generationConfig": { + "responseModalities": ["AUDIO", "TEXT"], + "speechConfig": { + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Aoede" + } + } + }, + }, + "systemInstruction": { + "parts": [{"text": system_prompt}] + }, + } + } + await gemini_ws.send(json.dumps(setup_msg)) + logger.info(f"[{session_id}] Connected to Gemini Live API") - prefs = kira_memory.get_user_preferences(user_id) - identified = True + # Wait for setup complete + raw = await asyncio.wait_for(gemini_ws.recv(), timeout=10) + setup_resp = json.loads(raw) + if "setupComplete" in setup_resp: + logger.info(f"[{session_id}] Gemini setup complete") + else: + logger.warning(f"[{session_id}] Unexpected setup response: {list(setup_resp.keys())}") - if kira_memory.enabled: - kira_memory.ensure_peers(user_id) - kira_memory.ensure_session(session_id) - try: - ctx = kira_memory.build_system_prompt_suffix() - if ctx: - memory_suffix = ctx - except Exception: + # ── Gemini → Frontend relay ── + async def relay_gemini(): + try: + async for raw in gemini_ws: + msg = json.loads(raw) + + if "serverContent" in msg: + sc = msg["serverContent"] + model_turn = sc.get("modelTurn", {}) + parts = model_turn.get("parts", []) + + for part in parts: + # Text response + if "text" in part: + await websocket.send_json({ + "type": "transcript", + "role": "kira", + "text": part["text"], + }) + + # Audio response (PCM16 24kHz) + if "inlineData" in part: + audio_data = part["inlineData"].get("data", "") + if audio_data: + await websocket.send_json({ + "type": "audio", + "data": audio_data, + }) + + # Turn complete + if sc.get("turnComplete"): + await websocket.send_json({"type": "turn_complete"}) + + # Interrupted + if sc.get("interrupted"): + await websocket.send_json({"type": "interrupted"}) + + elif "toolCall" in msg: + pass # future: tool use + + elif "toolCallCancellation" in msg: pass - await websocket.send_json({ - "type": "identified", - "user_id": user_id, - "preferences": prefs, - }) - continue + elif "error" in msg: + err = msg["error"] + logger.error(f"[{session_id}] Gemini error: {err}") + await websocket.send_json({ + "type": "error", + "message": str(err.get("message", err)), + }) - if msg_type == "set_preference": - key = msg.get("key", "").strip() - value = msg.get("value", "").strip() - if key and user_id and user_id != "default-user": - kira_memory.set_user_preference(user_id, key, value) - await websocket.send_json({"type": "preference_saved", "key": key, "success": True}) - continue + except websockets.exceptions.ConnectionClosed: + logger.info(f"[{session_id}] Gemini WS closed") + except Exception as e: + logger.error(f"[{session_id}] Gemini relay error: {e}") - # ── Audio (full webm/opus blob from MediaRecorder) → REST STT → LLM → TTS ── - if msg_type == "audio": - audio_b64 = msg.get("data", "") - if not audio_b64: - continue + # ── Frontend → Gemini relay ── + async def relay_frontend(): + nonlocal user_id, memory_suffix + try: + while True: + raw = await websocket.receive_text() + msg = json.loads(raw) + msg_type = msg.get("type", "") - client = get_openai() + if msg_type == "identify": + user_id = msg.get("user_id", "default-user").strip() + user_name = msg.get("name", "").strip() + if user_name and user_id: + kira_memory.set_user_preference(user_id, "name", user_name) + prefs = kira_memory.get_user_preferences(user_id) + if kira_memory.enabled: + kira_memory.ensure_peers(user_id) + kira_memory.ensure_session(session_id) + try: + ctx = kira_memory.build_system_prompt_suffix() + if ctx: + memory_suffix = ctx + except Exception: + pass + await websocket.send_json({ + "type": "identified", + "user_id": user_id, + "preferences": prefs, + }) + continue - # STT (REST) - transcript = await transcribe_audio(client, audio_b64) - if not transcript: - await websocket.send_json({"type": "transcript", "role": "user", "text": "(could not transcribe)"}) - await websocket.send_json({"type": "error", "message": "Could not transcribe audio"}) - continue + if msg_type == "set_preference": + key = msg.get("key", "").strip() + value = msg.get("value", "").strip() + if key and user_id and user_id != "default-user": + kira_memory.set_user_preference(user_id, key, value) + await websocket.send_json({"type": "preference_saved", "key": key, "success": True}) + continue - logger.info(f"[{session_id}] User: {transcript}") - await websocket.send_json({"type": "transcript_delta", "text": transcript}) - await websocket.send_json({"type": "transcript", "role": "user", "text": transcript}) - conversation_history.append({"role": "user", "content": transcript}) + if msg_type == "audio": + # Forward PCM16 audio to Gemini as realtimeInput + audio_b64 = msg.get("data", "") + if audio_b64 and gemini_ws and gemini_ws.state.name == "OPEN": + gemini_msg = { + "realtimeInput": { + "audio": { + "mimeType": "audio/pcm;rate=16000", + "data": audio_b64, + } + } + } + await gemini_ws.send(json.dumps(gemini_msg)) + continue - # LLM - kira_text = await get_kira_response(client, transcript, memory_suffix) - conversation_history.append({"role": "assistant", "content": kira_text}) - logger.info(f"[{session_id}] Kira: {kira_text}") + if msg_type == "conversation_text": + text = msg.get("text", "").strip() + if not text: + continue + logger.info(f"[{session_id}] User (text): {text}") + # Send as a text turn to Gemini + if gemini_ws and gemini_ws.state.name == "OPEN": + user_part = {"text": text} + if memory_suffix: + user_part = {"text": f"[Context: {memory_suffix}]\n{text}"} + gemini_msg = { + "clientContent": { + "turns": [{"role": "user", "parts": [user_part]}], + "turnComplete": True, + } + } + await gemini_ws.send(json.dumps(gemini_msg)) + await websocket.send_json({ + "type": "transcript", + "role": "user", + "text": text, + }) + continue - # Store in Honcho - if kira_memory.enabled and identified: - try: - kira_memory.store_messages(transcript, kira_text) - except Exception: - pass + if msg_type == "ping": + await websocket.send_json({"type": "pong"}) - # TTS (streaming) - await stream_tts(client, kira_text, websocket) - continue + except WebSocketDisconnect: + pass + except Exception as e: + logger.error(f"[{session_id}] Frontend relay error: {e}") - # ── Text input → direct LLM + TTS ── - if msg_type == "conversation_text": - text = msg.get("text", "").strip() - if not text: - continue + gemini_task = asyncio.create_task(relay_gemini()) + frontend_task = asyncio.create_task(relay_frontend()) - logger.info(f"[{session_id}] User (text): {text}") - conversation_history.append({"role": "user", "content": text}) + # Wait for either to finish + done, pending = await asyncio.wait( + [gemini_task, frontend_task], + return_when=asyncio.FIRST_COMPLETED, + ) + for t in pending: + t.cancel() - client = get_openai() - kira_text = await get_kira_response(client, text, memory_suffix) - conversation_history.append({"role": "assistant", "content": kira_text}) - logger.info(f"[{session_id}] Kira: {kira_text}") - - if kira_memory.enabled and identified: - try: - kira_memory.store_messages(text, kira_text) - except Exception: - pass - - await stream_tts(client, kira_text, websocket) - continue - - if msg_type == "ping": - await websocket.send_json({"type": "pong"}) - - except WebSocketDisconnect: - logger.info(f"[{session_id}] Disconnected") except Exception as e: - logger.error(f"[{session_id}] Error: {e}") + logger.error(f"[{session_id}] Connection error: {e}") + finally: + if gemini_ws: + await gemini_ws.close() + logger.info(f"[{session_id}] Disconnected") diff --git a/backend/requirements.txt b/backend/requirements.txt index fd79264..5cc9527 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,10 +1,8 @@ fastapi>=0.115.0 uvicorn[standard]>=0.34.0 python-dotenv>=1.1.0 -openai>=1.55.0 websockets>=14.1 pydantic>=2.10.0 pydantic-settings>=2.7.0 httpx>=0.28.0 honcho-ai>=2.1.0 -openai[realtime]>=2.41.0 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4518cec..90cd653 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -27,7 +27,7 @@ export default function App() { sendText, startRecording, stopRecording, - livePartial, + } = useConversation(); const [currentSceneId, setCurrentSceneId] = useState('cozy-room'); @@ -145,7 +145,7 @@ export default function App() {
- +
(null); useEffect(() => { @@ -28,15 +27,6 @@ export default function ChatBubble({ messages, isKiraSpeaking, livePartial }: Pr - {livePartial && ( -
- 👂 - Hearing: - {livePartial} - ... -
- )} -
{messages.length === 0 && (
diff --git a/frontend/src/hooks/useConversation.ts b/frontend/src/hooks/useConversation.ts index afaa0dc..034b46b 100644 --- a/frontend/src/hooks/useConversation.ts +++ b/frontend/src/hooks/useConversation.ts @@ -20,11 +20,35 @@ const USER_ID_KEY = 'kira-user-id'; function loadUserId(): string { return localStorage.getItem(USER_ID_KEY) || ''; } - function saveUserId(id: string) { localStorage.setItem(USER_ID_KEY, id); } +/** + * Encode Float32Array (44100 Hz or any rate) to PCM16 mono at 16kHz. + * Downsamples by simple nearest-neighbour if source rate != 16000. + */ +function float32ToPcm16Base64(float32: Float32Array, srcRate: number): string { + const targetRate = 16000; + const ratio = srcRate / targetRate; + const outLen = Math.floor(float32.length / ratio); + const pcm = new Int16Array(outLen); + + for (let i = 0; i < outLen; i++) { + const s = float32[Math.floor(i * ratio)]; + const clamped = Math.max(-1, Math.min(1, s)); + pcm[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7FFF; + } + + // base64 encode raw PCM16 bytes (little-endian) + const bytes = new Uint8Array(pcm.buffer); + let binary = ''; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + export function useConversation() { const [messages, setMessages] = useState([]); const [isConnected, setIsConnected] = useState(false); @@ -39,16 +63,17 @@ export function useConversation() { }); const [loadingPrefs, setLoadingPrefs] = useState(true); const [micError, setMicError] = useState(null); - const [livePartial, setLivePartial] = useState(''); const wsRef = useRef(null); - const audioRef = useRef(null); - const captureRef = useRef<{ stop: () => void } | null>(null); - const recorderRef = useRef(null); const streamRef = useRef(null); - const audioBufferRef = useRef([]); + const audioCtxRef = useRef(null); + const processorRef = useRef(null); + + // Audio playback queue + const playbackCtxRef = useRef(null); + const playbackQueueRef = useRef([]); + const isPlayingRef = useRef(false); - // Connect WebSocket const connect = useCallback(() => { if (wsRef.current?.readyState === WebSocket.OPEN) return; setLoadingPrefs(true); @@ -75,22 +100,14 @@ export function useConversation() { try { const msg = JSON.parse(event.data); handleMessage(msg); - } catch { /* ignore parse errors */ } + } catch { /* ignore */ } }; }, []); - // Audio playback element - useEffect(() => { - if (!audioRef.current) { - audioRef.current = new Audio(); - audioRef.current.onended = () => setIsKiraSpeaking(false); - } - }, []); - - // Handle incoming WS messages + // Handle incoming messages from backend const handleMessage = useCallback((msg: any) => { switch (msg.type) { - case 'identified': { + case 'identified': setIdentified(true); setLoadingPrefs(false); if (msg.user_id) saveUserId(msg.user_id); @@ -103,63 +120,40 @@ export function useConversation() { }); } break; - } case 'transcript': addMessage(msg.role === 'user' ? 'user' : 'kira', msg.text); break; - case 'transcript_delta': - if (msg.text) { - setLivePartial(msg.text); - // Clear after short delay so it doesn't stick (for REST full-text case) - setTimeout(() => setLivePartial(''), 1500); - } - break; - - case 'speaking_start': - setIsKiraSpeaking(true); - break; - case 'audio': { - // Incoming Opus audio chunk from streaming TTS + // Incoming PCM16 24kHz audio from Gemini if (msg.data) { const binary = atob(msg.data); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) { bytes[i] = binary.charCodeAt(i); } - audioBufferRef.current.push(bytes); + // Convert PCM16 24kHz to Float32 for Web Audio API + const int16 = new Int16Array(bytes.buffer); + const float32 = new Float32Array(int16.length); + for (let i = 0; i < int16.length; i++) { + float32[i] = int16[i] / 32768; + } + enqueueAudio(float32, 24000); + setIsKiraSpeaking(true); } break; } - case 'speaking_end': + case 'turn_complete': setIsKiraSpeaking(false); - // Play all accumulated chunks as one blob - if (audioBufferRef.current.length > 0 && audioRef.current) { - const allChunks = audioBufferRef.current; - const totalLen = allChunks.reduce((s, c) => s + c.length, 0); - const combined = new Uint8Array(totalLen); - let offset = 0; - for (const chunk of allChunks) { - combined.set(chunk, offset); - offset += chunk.length; - } - // audioBufferRef no longer used for playback (incremental) - const blob = new Blob([combined], { type: 'audio/ogg' }); - const url = URL.createObjectURL(blob); - audioRef.current.src = url; - audioRef.current.play().catch(() => {}); - } break; - case 'interruption': + case 'interrupted': setIsKiraSpeaking(false); - if (audioRef.current) { - audioRef.current.pause(); - audioRef.current.currentTime = 0; - } + // Clear playback queue + playbackQueueRef.current = []; + isPlayingRef.current = false; break; case 'error': @@ -168,6 +162,42 @@ export function useConversation() { } }, []); + // Queue PCM float32 audio for playback + const enqueueAudio = useCallback((float32: Float32Array, sampleRate: number) => { + playbackQueueRef.current.push(float32.buffer as ArrayBuffer); + + if (!isPlayingRef.current) { + playNext(); + } + + function playNext() { + const next = playbackQueueRef.current.shift(); + if (!next) { + isPlayingRef.current = false; + return; + } + isPlayingRef.current = true; + + const ctx = getPlaybackCtx(); + const float32 = new Float32Array(next as ArrayBuffer); + const buf = ctx.createBuffer(1, float32.length, sampleRate); + buf.getChannelData(0).set(float32); + + const src = ctx.createBufferSource(); + src.buffer = buf; + src.connect(ctx.destination); + src.onended = playNext; + src.start(); + } + }, []); + + function getPlaybackCtx(): AudioContext { + if (!playbackCtxRef.current) { + playbackCtxRef.current = new AudioContext({ sampleRate: 24000 }); + } + return playbackCtxRef.current; + } + const addMessage = useCallback((role: 'user' | 'kira', text: string) => { setMessages((prev) => [ ...prev, @@ -176,19 +206,16 @@ export function useConversation() { }, []); // ── Identity ── - const identify = useCallback((name: string) => { const userId = `kira-${name.toLowerCase().replace(/[^a-z0-9]/g, '-')}`; saveUserId(userId); setPreferences((p) => ({ ...p, name })); - if (wsRef.current?.readyState === WebSocket.OPEN) { wsRef.current.send(JSON.stringify({ type: 'identify', user_id: userId, name })); } }, []); // ── Preferences ── - const setPreference = useCallback((key: string, value: string) => { setPreferences((p) => ({ ...p, [key]: value })); if (wsRef.current?.readyState === WebSocket.OPEN && identified) { @@ -196,17 +223,18 @@ export function useConversation() { } }, [identified]); - // ── Audio (Realtime PCM16) ── - + // ── Audio capture via ScriptProcessorNode (PCM16 16kHz stream) ── const startRecording = useCallback(async () => { - if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + if (!navigator.mediaDevices?.getUserMedia) { addMessage('kira', 'Mic requires HTTPS. Try accessing via HTTPS!'); return; } try { setMicError(null); - const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true } }); + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 48000 }, + }); streamRef.current = stream; const ws = wsRef.current; @@ -216,27 +244,32 @@ export function useConversation() { return; } - // Use MediaRecorder for full utterance blob (Opus/webm) — sent on stop for REST STT - const mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' }); - const chunks: Blob[] = []; - mediaRecorder.ondataavailable = (e) => { - if (e.data.size > 0) chunks.push(e.data); - }; - mediaRecorder.onstop = () => { - if (chunks.length > 0 && ws.readyState === WebSocket.OPEN) { - const blob = new Blob(chunks, { type: 'audio/webm' }); - blob.arrayBuffer().then((buf) => { - const base64 = arrayBufferToBase64(buf); - ws.send(JSON.stringify({ type: 'audio', data: base64 })); - }); + // Create AudioContext at native sample rate, capture via ScriptProcessor + const audioCtx = new AudioContext({ sampleRate: 48000 }); + audioCtxRef.current = audioCtx; + + const source = audioCtx.createMediaStreamSource(stream); + // 4096 buffer size → ~85ms chunks at 48kHz + const processor = audioCtx.createScriptProcessor(4096, 1, 1); + processorRef.current = processor; + + processor.onaudioprocess = (e) => { + if (ws.readyState !== WebSocket.OPEN) return; + const float32 = e.inputBuffer.getChannelData(0); + // Skip silent frames (reduces network traffic) + let maxAbs = 0; + for (let i = 0; i < float32.length; i += 4) { + const v = Math.abs(float32[i]); + if (v > maxAbs) maxAbs = v; } - chunks.length = 0; - stream.getTracks().forEach((t) => t.stop()); - streamRef.current = null; - setIsRecording(false); + if (maxAbs < 0.01) return; // silence gate + + const b64 = float32ToPcm16Base64(float32, audioCtx.sampleRate); + ws.send(JSON.stringify({ type: 'audio', data: b64 })); }; - recorderRef.current = mediaRecorder; - mediaRecorder.start(); + + source.connect(processor); + processor.connect(audioCtx.destination); setIsRecording(true); } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -246,20 +279,16 @@ export function useConversation() { }, [addMessage]); const stopRecording = useCallback(() => { - if (recorderRef.current && recorderRef.current.state === 'recording') { - recorderRef.current.stop(); - // onstop will handle sending the blob and cleanup - } else { - // fallback cleanup - streamRef.current?.getTracks().forEach((t) => t.stop()); - streamRef.current = null; - setIsRecording(false); - } - captureRef.current = null; // legacy + processorRef.current?.disconnect(); + processorRef.current = null; + audioCtxRef.current?.close().catch(() => {}); + audioCtxRef.current = null; + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + setIsRecording(false); }, []); - // ── Text ── - + // ── Text input ── const sendText = useCallback((text: string) => { if (!text.trim()) return; if (wsRef.current?.readyState === WebSocket.OPEN) { @@ -272,11 +301,9 @@ export function useConversation() { connect(); return () => { wsRef.current?.close(); - if (recorderRef.current && recorderRef.current.state === 'recording') recorderRef.current.stop(); - captureRef.current?.stop(); - streamRef.current?.getTracks().forEach((t) => t.stop()); + stopRecording(); }; - }, [connect]); + }, [connect, stopRecording]); return { messages, @@ -287,7 +314,6 @@ export function useConversation() { preferences, loadingPrefs, micError, - livePartial, identify, setPreference, sendText, @@ -295,16 +321,3 @@ export function useConversation() { stopRecording, }; } - -// ── Helpers ── - -function arrayBufferToBase64(buffer: ArrayBufferLike): string { - const bytes = new Uint8Array(buffer); - let binary = ''; - for (let i = 0; i < bytes.length; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); -} - -// (Legacy PCM capture removed - MediaRecorder full-blob path is active; eliminates ScriptProcessorNode deprecation)