diff --git a/backend/main.py b/backend/main.py index 8509f5a..82e4003 100644 --- a/backend/main.py +++ b/backend/main.py @@ -383,29 +383,6 @@ async def gemini_voice_ws(websocket: WebSocket): await gemini_ws.send(json.dumps(gemini_msg)) continue - if msg_type == "conversation_text": - text = msg.get("text", "").strip() - if not text: - continue - logger.info(f"[{session_id}] User (text): {text}") - if gemini_ws and gemini_ws.state.name == "OPEN": - user_part = {"text": text} - if memory_suffix: - user_part = {"text": f"[Context: {memory_suffix}]\n{text}"} - gemini_msg = { - "clientContent": { - "turns": [{"role": "user", "parts": [user_part]}], - "turnComplete": True, - } - } - await gemini_ws.send(json.dumps(gemini_msg)) - await websocket.send_json({ - "type": "transcript", - "role": "user", - "text": text, - }) - continue - if msg_type == "ping": await websocket.send_json({"type": "pong"}) diff --git a/backend/models/__init__.py b/backend/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/routers/__init__.py b/backend/routers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/services/memory.py b/backend/services/memory.py index 2377fd2..2b0af04 100644 --- a/backend/services/memory.py +++ b/backend/services/memory.py @@ -134,57 +134,12 @@ class KiraMemory: return "\n\n---\n### What Kira remembers:" + "".join(parts) - def store_messages( - self, - user_message: str, - kira_message: str, - ) -> None: - """Store a conversation exchange in Honcho.""" - if not self.enabled or not self._session: - return - - try: - messages = [] - if self._user_peer: - messages.append( - self._user_peer.message(user_message) - ) - if self._kira_peer: - messages.append( - self._kira_peer.message(kira_message) - ) - - if messages: - self._session.add_messages(messages) - logger.debug("Stored conversation exchange in Honcho") - except Exception as e: - logger.warning(f"Failed to store messages: {e}") - - def store_user_message(self, text: str) -> None: - """Store a single user message.""" - if not self.enabled or not self._session or not self._user_peer: - return - try: - self._session.add_messages([self._user_peer.message(text)]) - except Exception as e: - logger.warning(f"Failed to store user message: {e}") - - def store_kira_message(self, text: str) -> None: - """Store a single Kira message.""" - if not self.enabled or not self._session or not self._kira_peer: - return - try: - self._session.add_messages([self._kira_peer.message(text)]) - except Exception as e: - logger.warning(f"Failed to store Kira message: {e}") - - # ─── User preferences (stored in Honcho peer metadata) ─── - DEFAULT_PREFERENCES: dict[str, str] = { "name": "", "scene": "cozy-room", "outfit": "cozy-hoodie", "accessory": "", + "tasks": "[]", } def get_user_preferences(self, user_id: str) -> dict[str, str]: diff --git a/backend/services/whisper_stream.py b/backend/services/whisper_stream.py deleted file mode 100644 index 94788b9..0000000 --- a/backend/services/whisper_stream.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Realtime streaming transcription service via gpt-realtime-whisper. - -Connects to OpenAI Realtime API via WebSocket, configures the session -for pure transcription (no model responses), and streams word-level -transcript deltas back. Full utterances are then processed by the -cheap LLM + TTS pipeline. -""" - -import json -import base64 -import logging -import asyncio -from typing import Callable, Awaitable -from config import settings - -logger = logging.getLogger("kira.whisper") - - -class WhisperStream: - """Streaming transcription via gpt-realtime-whisper over WebSocket.""" - - def __init__( - self, - on_transcript_delta: Callable[[str], Awaitable[None]], - on_transcript_done: Callable[[str], Awaitable[None]], - on_ready: Callable[[], Awaitable[None]], - on_error: Callable[[str], Awaitable[None]], - ): - self._on_delta = on_transcript_delta - self._on_done = on_transcript_done - self._on_ready = on_ready - self._on_error = on_error - self._conn = None - self._connected = False - self._transcript = "" - - async def connect(self): - if self._connected: - return - - try: - import websockets - - url = "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ws = await websockets.connect( - url, - additional_headers={ - "Authorization": f"Bearer {settings.openai_api_key}", - }, - ) - - async with ws as conn: - self._conn = conn - self._connected = True - logger.info("Connected to Realtime transcription session") - - # Configure: transcribe only with gpt-realtime-whisper, no model responses - await self._send({ - "type": "session.update", - "session": { - "input_audio_format": "pcm16", - "input_audio_transcription": { - "model": "gpt-realtime-whisper", - "enabled": True, - }, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 600, - }, - }, - }) - - await self._on_ready() - - while self._connected: - try: - raw = await conn.recv() - if isinstance(raw, (str, bytes)): - data = json.loads(raw if isinstance(raw, str) else raw.decode()) - await self._handle(data) - except Exception as e: - if self._connected: - logger.warning(f"recv: {e}") - break - - except Exception as e: - logger.error(f"Whisper stream error: {e}") - await self._on_error(str(e)) - finally: - self._connected = False - self._conn = None - - async def _handle(self, data: dict): - et = data.get("type", "") - - if et == "input_audio_buffer.speech_started": - self._transcript = "" - logger.debug("speech_started") - - elif et in ("conversation.item.input_audio_transcription.delta", "input_audio_buffer.transcription.delta"): - # Partial streaming transcript - delta = data.get("delta", "") or data.get("transcript", "") - if delta: - self._transcript = delta # or append if cumulative - await self._on_delta(delta) - - elif et in ("conversation.item.input_audio_transcription.completed", "input_audio_buffer.transcription.completed", "conversation.item.created"): - # Final or item created with transcript - item = data.get("item", {}) - content = item.get("content", []) if item else [] - transcript = data.get("transcript", "") - if not transcript: - for part in (content or []): - if part.get("type") in ("transcript", "text"): - transcript = part.get("transcript", "") or part.get("text", "") - break - if transcript: - self._transcript = transcript - await self._on_delta(transcript) - await self._on_done(transcript.strip()) - self._transcript = "" - - elif et == "input_audio_buffer.speech_stopped": - if self._transcript.strip(): - await self._on_done(self._transcript.strip()) - self._transcript = "" - - elif et == "error": - err = data.get("error", {}) - msg = err.get("message", str(data)) - logger.warning(f"Whisper error: {msg}") - await self._on_error(msg) - - async def send_audio(self, pcm16_bytes: bytes): - if not self._connected: - return - try: - b64 = base64.b64encode(pcm16_bytes).decode("utf-8") - await self._send({"type": "input_audio_buffer.append", "audio": b64}) - except Exception as e: - logger.warning(f"send audio: {e}") - - async def _send(self, data: dict): - try: - await self._conn.send(json.dumps(data)) - except Exception as e: - logger.warning(f"send: {e}") - - async def disconnect(self): - self._connected = False - if self._conn: - try: - await self._conn.close() - except Exception: - pass - self._conn = None diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1e8d34a..26dfb78 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -16,7 +16,6 @@ import { useConversation } from './hooks/useConversation'; export default function App() { const [isLoggedIn, setIsLoggedIn] = useState(() => sessionStorage.getItem('kira-auth') === '1'); const { - messages, isConnected, isKiraSpeaking, isRecording, diff --git a/frontend/src/components/BackgroundScene.tsx b/frontend/src/components/BackgroundScene.tsx deleted file mode 100644 index f97f552..0000000 --- a/frontend/src/components/BackgroundScene.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { SCENES } from './scenes'; - -interface Props { - currentScene: string; - onSelect: (id: string) => void; -} - -export default function BackgroundScene({ currentScene, onSelect }: Props) { - return ( -
-

- 🎨 Scene -

-
- {SCENES.map((s) => ( - - ))} -
-
- ); -} diff --git a/frontend/src/components/ChatBubble.tsx b/frontend/src/components/ChatBubble.tsx deleted file mode 100644 index affe0bd..0000000 --- a/frontend/src/components/ChatBubble.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import { useRef, useEffect } from 'react'; - -interface Message { - id: string; - role: 'user' | 'kira'; - text: string; - timestamp: number; -} - -interface Props { - messages: Message[]; - isKiraSpeaking: boolean; - userName?: string; -} - -export default function ChatBubble({ messages, isKiraSpeaking }: Props) { - const bottomRef = useRef(null); - - useEffect(() => { - bottomRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [messages]); - - return ( -
-

- 💬 Conversation - -

- -
- {messages.length === 0 && ( -
- click the mic or type to talk to Kira -
- )} - - {messages.map((msg, i) => { - const isLastKira = msg.role === 'kira' && i === messages.length - 1 && isKiraSpeaking; - return ( -
- {msg.role === 'kira' && ( - 🌸 - )} -
- {msg.text} - {isLastKira && ( - - )} -
- {msg.role === 'user' && ( - 👤 - )} -
- ); - })} -
-
- - -
- ); -} diff --git a/frontend/src/components/Clock.tsx b/frontend/src/components/Clock.tsx deleted file mode 100644 index 923ed26..0000000 --- a/frontend/src/components/Clock.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import { useEffect, useState } from 'react'; - -export default function Clock() { - const [time, setTime] = useState(new Date()); - - useEffect(() => { - const id = setInterval(() => setTime(new Date()), 1000); - return () => clearInterval(id); - }, []); - - return ( -
-
- {time.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} -
-
- {time.toLocaleDateString([], { weekday: 'long', month: 'long', day: 'numeric' })} -
-
- ); -} diff --git a/frontend/src/components/Live2DCat.tsx b/frontend/src/components/Live2DCat.tsx deleted file mode 100644 index ab4c50d..0000000 --- a/frontend/src/components/Live2DCat.tsx +++ /dev/null @@ -1,76 +0,0 @@ -import { useEffect, useRef } from 'react'; - -interface Props { - className?: string; -} - -export default function Live2DCat({ className }: Props) { - const canvasRef = useRef(null); - - useEffect(() => { - let mounted = true; - const canvas = canvasRef.current; - if (!canvas) return; - - let app: any = null; - - const init = async () => { - try { - const pixi = await import('pixi.js'); - const { Live2DModel } = await import('pixi-live2d-display/cubism4'); - (Live2DModel as any).registerTicker(pixi.Ticker as any); - - const w = canvas.clientWidth || 120; - const h = canvas.clientHeight || 120; - - // Use explicit CanvasRenderer to avoid WebGL context conflict with KiraAvatar - app = new pixi.Application({ - view: canvas, - width: w, - height: h, - antialias: true, - resolution: Math.min(window.devicePixelRatio || 1, 2), - backgroundAlpha: 0, - autoDensity: true, - // Use WebGL1 context (more compatible with separate contexts) - preferWebGLVersion: 1, - } as any); - if (!mounted) { app.destroy(true); return; } - - const model = await Live2DModel.from('/live2d/models/little-cat/LittleCat.model3.json', { - autoInteract: false, - }); - if (!mounted) { app.destroy(true); return; } - - const sw = app.screen.width; - const sh = app.screen.height; - const s = Math.min((sw * 0.85) / model.width, (sh * 0.85) / model.height); - model.scale.set(s); - model.anchor.set(0.5, 0.5); - model.position.set(sw / 2, sh / 2); - - app.stage.addChild(model as any); - (model as any).isInteractive = () => false; - - try { model.motion('Idle'); } catch { /* */ } - } catch (e) { - console.warn('[Live2DCat]', e); - } - }; - - init(); - - return () => { - mounted = false; - if (app) { app.destroy(true, { children: true }); } - }; - }, []); - - return ( - - ); -} diff --git a/frontend/src/components/Live2DStage.tsx b/frontend/src/components/Live2DStage.tsx index c9e4c67..c856e88 100644 --- a/frontend/src/components/Live2DStage.tsx +++ b/frontend/src/components/Live2DStage.tsx @@ -11,7 +11,7 @@ interface Props { /** * Single full-viewport Live2D stage. One WebGL context, two models: * - Kira (center panel) - * - Mochi the cat (PetZone, bottom of right sidebar) + // Mochi the cat (next to Kira in center panel) * * Canvas sits above UI panels (z-50, pointer-events: none) so Live2D * models render on top of the frosted-glass sidebars. diff --git a/frontend/src/components/PetZone.tsx b/frontend/src/components/PetZone.tsx deleted file mode 100644 index e1dd6eb..0000000 --- a/frontend/src/components/PetZone.tsx +++ /dev/null @@ -1,12 +0,0 @@ -export default function PetZone() { - return ( -
-

- 🐱 Pet Zone -

-
- Mochi -
-
- ); -} diff --git a/frontend/src/components/Toolbar.tsx b/frontend/src/components/Toolbar.tsx deleted file mode 100644 index 3cef03e..0000000 --- a/frontend/src/components/Toolbar.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { SCENES } from './scenes'; - -interface Props { - currentScene: string; - onSceneChange: (id: string) => void; -} - -export default function Toolbar({ currentScene, onSceneChange }: Props) { - return ( -
-
- Kira - | -
- {SCENES.filter((_, i) => i < 4).map((s) => ( - - ))} -
-
- -
- 🐱 2 - focus bestie -
-
- ); -} diff --git a/frontend/src/hooks/useConversation.ts b/frontend/src/hooks/useConversation.ts index 7126f0c..2cbedb8 100644 --- a/frontend/src/hooks/useConversation.ts +++ b/frontend/src/hooks/useConversation.ts @@ -13,12 +13,6 @@ export interface Task { completed: boolean; } -interface Message { - id: string; - role: 'user' | 'kira'; - text: string; - timestamp: number; -} const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`; const USER_ID_KEY = 'kira-user-id'; @@ -56,7 +50,6 @@ function float32ToPcm16Base64(float32: Float32Array, srcRate: number): string { } export function useConversation() { - const [messages, setMessages] = useState([]); const [isConnected, setIsConnected] = useState(false); const [isKiraSpeaking, setIsKiraSpeaking] = useState(false); const [isRecording, setIsRecording] = useState(false); @@ -68,7 +61,6 @@ export function useConversation() { accessory: '', }); const [loadingPrefs, setLoadingPrefs] = useState(true); - const [micError, setMicError] = useState(null); const [tasks, setTasks] = useState([]); const [celebrateTrigger, setCelebrateTrigger] = useState(0); @@ -130,7 +122,6 @@ export function useConversation() { break; case 'transcript': - addMessage(msg.role === 'user' ? 'user' : 'kira', msg.text); break; case 'audio': { @@ -220,12 +211,6 @@ export function useConversation() { return playbackCtxRef.current; } - const addMessage = useCallback((role: 'user' | 'kira', text: string) => { - setMessages((prev) => [ - ...prev, - { id: crypto.randomUUID(), role, text, timestamp: Date.now() }, - ]); - }, []); // ── Identity ── const identify = useCallback((name: string) => { @@ -248,12 +233,10 @@ export function useConversation() { // ── Audio capture via ScriptProcessorNode (PCM16 16kHz stream) ── const startRecording = useCallback(async () => { if (!navigator.mediaDevices?.getUserMedia) { - addMessage('kira', 'Mic requires HTTPS. Try accessing via HTTPS!'); return; } try { - setMicError(null); const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, sampleRate: 48000 }, }); @@ -261,7 +244,6 @@ export function useConversation() { const ws = wsRef.current; if (!ws || ws.readyState !== WebSocket.OPEN) { - addMessage('kira', 'Not connected to server yet...'); stream.getTracks().forEach((t) => t.stop()); return; } @@ -295,10 +277,9 @@ export function useConversation() { setIsRecording(true); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - setMicError(msg); console.error('[Kira Mic]', msg); } - }, [addMessage]); + }, []); const stopRecording = useCallback(() => { processorRef.current?.disconnect(); @@ -328,14 +309,12 @@ export function useConversation() { }, [connect, stopRecording]); return { - messages, isConnected, isKiraSpeaking, isRecording, identified, preferences, loadingPrefs, - micError, identify, setPreference, sendText, diff --git a/frontend/src/index.css b/frontend/src/index.css index a27e389..d0ef443 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -7,7 +7,6 @@ --color-kira-bg: #FFF5F5; --color-kira-plum: #4A1942; --color-kira-violet: #7C3AED; - --color-kira-card: #FFFFFF; --color-kira-glow: #FDF2F8; --font-nunito: 'Nunito', sans-serif; } @@ -27,14 +26,6 @@ body { width: 100vw; } -.glass-card { - background: rgba(255, 255, 255, 0.7); - backdrop-filter: blur(12px); - border: 1px solid rgba(255, 182, 193, 0.3); - border-radius: 16px; - box-shadow: 0 4px 24px rgba(74, 25, 66, 0.08); -} - .btn-kira { background: linear-gradient(135deg, #FFB6C1, #D8B4FE); color: white; diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts deleted file mode 100644 index 3fb762e..0000000 --- a/frontend/src/types/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { SCENES, type Scene } from '../components/scenes'; - -export interface KiraState { - currentScene: Scene; - isListening: boolean; - isSpeaking: boolean; - currentOutfit: string; - currentAccessory: string | null; - sessionNotes: string[]; -} - -export type { Scene }; -export { SCENES }; diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index 781de6f..9972967 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/AnimatedAvatar.tsx","./src/components/BackgroundScene.tsx","./src/components/ChatBubble.tsx","./src/components/Clock.tsx","./src/components/KiraAvatar.tsx","./src/components/Live2DCat.tsx","./src/components/Live2DStage.tsx","./src/components/LoginScreen.tsx","./src/components/MusicPlayer.tsx","./src/components/Notes.tsx","./src/components/Particles.tsx","./src/components/PetZone.tsx","./src/components/TaskList.tsx","./src/components/Timer.tsx","./src/components/Toolbar.tsx","./src/components/Wardrobe.tsx","./src/components/WelcomeScreen.tsx","./src/components/WhiteNoise.tsx","./src/components/scenes.ts","./src/hooks/useConversation.ts","./src/types/index.ts"],"version":"6.0.3"} \ No newline at end of file +{"root":["./src/App.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/AnimatedAvatar.tsx","./src/components/KiraAvatar.tsx","./src/components/Live2DStage.tsx","./src/components/LoginScreen.tsx","./src/components/MusicPlayer.tsx","./src/components/Notes.tsx","./src/components/Particles.tsx","./src/components/TaskList.tsx","./src/components/Timer.tsx","./src/components/Wardrobe.tsx","./src/components/WelcomeScreen.tsx","./src/components/WhiteNoise.tsx","./src/components/scenes.ts","./src/hooks/useConversation.ts"],"version":"6.0.3"} \ No newline at end of file