From 78ea059f08700923b0e70ca3e89976af0cb7228c Mon Sep 17 00:00:00 2001 From: hobokenchicken Date: Thu, 4 Jun 2026 11:00:58 -0400 Subject: [PATCH] feat: user personalization with Honcho-backed preferences - WelcomeScreen: first-time name entry with cute onboarding - identify WS message: sets user_id, loads saved prefs from Honcho - set_preference WS message: saves scene/outfit/accessory to Honcho metadata - Preferences auto-load on return visits via localStorage + Honcho peer meta - Kira uses the user's name in greeting and prompts - Backend: get/set preference methods in KiraMemory service - Frontend: optimistic preference updates, synced to backend on change --- backend/main.py | 130 +++++++++++++--------- backend/services/memory.py | 48 ++++++++ frontend/src/App.tsx | 103 ++++++++++++++--- frontend/src/components/ChatBubble.tsx | 2 + frontend/src/components/WelcomeScreen.tsx | 75 +++++++++++++ frontend/src/hooks/useConversation.ts | 118 +++++++++++++++++--- 6 files changed, 396 insertions(+), 80 deletions(-) create mode 100644 frontend/src/components/WelcomeScreen.tsx diff --git a/backend/main.py b/backend/main.py index 1816ae5..66b937e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,6 +6,7 @@ Real-time speech-to-speech pipeline: Honcho memory integration: Cross-session user context injected into LLM prompts, conversation exchanges stored for continuous learning. + User preferences (name, scene, outfit, accessory) persisted in peer metadata. """ import json @@ -65,10 +66,8 @@ def build_system_prompt(user_id: str) -> dict: """Build system prompt with Honcho memory context injected.""" base = BASE_SYSTEM_PROMPT - # Append memory context if Honcho is available if kira_memory.enabled: try: - # Get user-specific context from Honcho kira_memory.ensure_peers(user_id) memory_suffix = kira_memory.build_system_prompt_suffix() if memory_suffix: @@ -79,35 +78,89 @@ def build_system_prompt(user_id: str) -> dict: return {"role": "system", "content": base} +def handle_identify(msg: dict, session_id: str) -> dict | None: + """Handle user identification. Returns user preferences or None.""" + user_id = msg.get("user_id", "").strip() + if not user_id: + return {"type": "error", "message": "user_id is required"} + + user_name = msg.get("name", "").strip() + if user_name: + kira_memory.set_user_preference(user_id, "name", user_name) + + prefs = kira_memory.get_user_preferences(user_id) + logger.info(f"[{session_id}] Identified as {user_id} (name={user_name or prefs.get('name', '')})") + + return { + "type": "identified", + "user_id": user_id, + "preferences": prefs, + } + + +def handle_set_preference(msg: dict, session_id: str, user_id: str) -> dict | None: + """Handle preference update. Returns success status.""" + if not user_id or user_id == "default-user": + return {"type": "error", "message": "Must identify first"} + + key = msg.get("key", "").strip() + value = msg.get("value", "").strip() + + if not key: + return {"type": "error", "message": "key is required"} + + ok = kira_memory.set_user_preference(user_id, key, value) + return { + "type": "preference_saved", + "key": key, + "success": ok, + } + + @app.websocket("/api/ws") async def conversation_ws(websocket: WebSocket): await websocket.accept() session_id = str(uuid.uuid4())[:8] user_id = "default-user" + identified = False logger.info(f"[{session_id}] WebSocket connected") - # Audio buffer accumulates chunks from one utterance audio_buffer = bytearray() conversation_history: list[dict] = [] - # Initialize Honcho for this session - if kira_memory.enabled: - try: - kira_memory.ensure_peers(user_id) - kira_memory.ensure_session(session_id) - logger.info(f"[{session_id}] Honcho session ready") - except Exception as e: - logger.warning(f"[{session_id}] Honcho setup failed: {e}") - try: - first_message = True - while True: raw = await websocket.receive_text() msg = json.loads(raw) msg_type = msg.get("type", "") - # Build system prompt fresh each turn to get latest Honcho context + # ── Identity & Preferences ── + + if msg_type == "identify": + response = handle_identify(msg, session_id) + if response: + await websocket.send_json(response) + if response["type"] == "identified": + user_id = response["user_id"] + identified = True + # Set up Honcho for this user + if kira_memory.enabled: + try: + kira_memory.ensure_peers(user_id) + kira_memory.ensure_session(session_id) + logger.info(f"[{session_id}] Honcho session ready for {user_id}") + except Exception as e: + logger.warning(f"[{session_id}] Honcho setup failed: {e}") + continue + + if msg_type == "set_preference": + response = handle_set_preference(msg, session_id, user_id) + if response: + await websocket.send_json(response) + continue + + # ── Conversation ── + system_prompt = build_system_prompt(user_id) if msg_type == "audio_chunk": @@ -121,7 +174,6 @@ async def conversation_ws(websocket: WebSocket): logger.info(f"[{session_id}] Transcribing {len(audio_buffer)} bytes...") - # 1. Speech-to-text transcript = await transcribe_audio(bytes(audio_buffer)) audio_buffer.clear() @@ -129,45 +181,27 @@ async def conversation_ws(websocket: WebSocket): await websocket.send_json({"type": "error", "message": "Could not transcribe audio"}) continue - # Echo transcript - await websocket.send_json({ - "type": "transcript", - "text": transcript, - }) + await websocket.send_json({"type": "transcript", "text": transcript}) - # 2. LLM call logger.info(f"[{session_id}] User: {transcript}") - user_msg = {"role": "user", "content": transcript} - conversation_history.append(user_msg) + conversation_history.append({"role": "user", "content": transcript}) messages = [system_prompt] + conversation_history[-10:] kira_text = await get_kira_response(messages) - assistant_msg = {"role": "assistant", "content": kira_text} - conversation_history.append(assistant_msg) + conversation_history.append({"role": "assistant", "content": kira_text}) logger.info(f"[{session_id}] Kira: {kira_text}") - # 3. Store in Honcho - if kira_memory.enabled: + if kira_memory.enabled and identified: try: kira_memory.store_messages(transcript, kira_text) except Exception as e: logger.warning(f"[{session_id}] Failed to store messages: {e}") - # 4. TTS - await websocket.send_json({ - "type": "speaking_start", - "text": kira_text, - }) - + await websocket.send_json({"type": "speaking_start", "text": kira_text}) audio_bytes = await synthesize_speech(kira_text) audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") - await websocket.send_json({ - "type": "audio", - "data": audio_b64, - "text": kira_text, - }) - + await websocket.send_json({"type": "audio", "data": audio_b64, "text": kira_text}) await websocket.send_json({"type": "speaking_end"}) elif msg_type == "ping": @@ -179,32 +213,24 @@ async def conversation_ws(websocket: WebSocket): continue logger.info(f"[{session_id}] User (text): {user_text}") - user_msg = {"role": "user", "content": user_text} - conversation_history.append(user_msg) + conversation_history.append({"role": "user", "content": user_text}) messages = [system_prompt] + conversation_history[-10:] kira_text = await get_kira_response(messages) - assistant_msg = {"role": "assistant", "content": kira_text} - conversation_history.append(assistant_msg) + conversation_history.append({"role": "assistant", "content": kira_text}) logger.info(f"[{session_id}] Kira: {kira_text}") - # Store in Honcho - if kira_memory.enabled: + if kira_memory.enabled and identified: try: kira_memory.store_messages(user_text, kira_text) except Exception as e: logger.warning(f"[{session_id}] Failed to store messages: {e}") - # TTS await websocket.send_json({"type": "speaking_start", "text": kira_text}) audio_bytes = await synthesize_speech(kira_text) audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") - await websocket.send_json({ - "type": "audio", - "data": audio_b64, - "text": kira_text, - }) + await websocket.send_json({"type": "audio", "data": audio_b64, "text": kira_text}) await websocket.send_json({"type": "speaking_end"}) except WebSocketDisconnect: diff --git a/backend/services/memory.py b/backend/services/memory.py index 7865fcc..2377fd2 100644 --- a/backend/services/memory.py +++ b/backend/services/memory.py @@ -178,6 +178,54 @@ class KiraMemory: except Exception as e: logger.warning(f"Failed to store Kira message: {e}") + # ─── User preferences (stored in Honcho peer metadata) ─── + + DEFAULT_PREFERENCES: dict[str, str] = { + "name": "", + "scene": "cozy-room", + "outfit": "cozy-hoodie", + "accessory": "", + } + + def get_user_preferences(self, user_id: str) -> dict[str, str]: + """Load user preferences from Honcho peer metadata.""" + if not self.enabled: + return dict(self.DEFAULT_PREFERENCES) + + try: + peer = self._honcho.peer(user_id) + meta = peer.metadata or {} + prefs = dict(self.DEFAULT_PREFERENCES) + for key in prefs: + val = meta.get(key) + if val is not None: + prefs[key] = str(val) + logger.info(f"Loaded preferences for {user_id}: {prefs}") + return prefs + except Exception as e: + logger.warning(f"Failed to load preferences: {e}") + return dict(self.DEFAULT_PREFERENCES) + + def set_user_preference( + self, user_id: str, key: str, value: str + ) -> bool: + """Save a single user preference to Honcho peer metadata.""" + if key not in self.DEFAULT_PREFERENCES: + logger.warning(f"Unknown preference key: {key}") + return False + + try: + peer = self._honcho.peer(user_id) + # Merge with existing metadata + meta = dict(peer.metadata or {}) + meta[key] = value + peer.set_metadata(meta) + logger.info(f"Saved preference {user_id}.{key} = {value}") + return True + except Exception as e: + logger.warning(f"Failed to save preference: {e}") + return False + # Singleton instance for the app kira_memory = KiraMemory() diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index be473cf..8e702d4 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useState, useEffect } from 'react'; import Clock from './components/Clock'; import BackgroundScene from './components/BackgroundScene'; import MusicPlayer from './components/MusicPlayer'; @@ -10,17 +10,54 @@ import PetZone from './components/PetZone'; import Wardrobe from './components/Wardrobe'; import Toolbar from './components/Toolbar'; import Particles from './components/Particles'; +import WelcomeScreen from './components/WelcomeScreen'; import { SCENES, type Scene } from './components/scenes'; import { useConversation } from './hooks/useConversation'; export default function App() { + const { + messages, + isConnected, + isKiraSpeaking, + isRecording, + identified, + preferences, + loadingPrefs, + identify, + setPreference, + sendText, + startRecording, + stopRecording, + } = useConversation(); + const [currentSceneId, setCurrentSceneId] = useState('cozy-room'); const [currentOutfit, setCurrentOutfit] = useState('cozy-hoodie'); const [currentAcc, setCurrentAcc] = useState(null); const [textInput, setTextInput] = useState(''); + // Apply saved preferences once they load + useEffect(() => { + if (preferences.scene) setCurrentSceneId(preferences.scene); + if (preferences.outfit) setCurrentOutfit(preferences.outfit); + if (preferences.accessory) setCurrentAcc(preferences.accessory); + }, [preferences.scene, preferences.outfit, preferences.accessory]); + const currentScene: Scene = SCENES.find((s) => s.id === currentSceneId) ?? SCENES[0]; - const { messages, isConnected, isKiraSpeaking, isRecording, sendText, startRecording, stopRecording } = useConversation(); + + const handleSceneChange = (id: string) => { + setCurrentSceneId(id); + setPreference('scene', id); + }; + + const handleOutfitChange = (id: string) => { + setCurrentOutfit(id); + setPreference('outfit', id); + }; + + const handleAccessoryChange = (id: string | null) => { + setCurrentAcc(id); + setPreference('accessory', id || ''); + }; const handleTalkToggle = () => { if (isRecording) stopRecording(); @@ -33,6 +70,42 @@ export default function App() { setTextInput(''); }; + const handleWelcome = (name: string) => { + identify(name); + }; + + // Show WelcomeScreen for first-time users + if (!identified && !loadingPrefs) { + const savedId = localStorage.getItem('kira-user-id'); + if (!savedId) { + return ; + } + // Has saved ID but not identified yet — show welcome with their name + return ( +
+
+
+
+ 🌸 +
+
+

coming back? say your name to pick up where you left off

+ +
+ +
+ ); + } + + // Main app with user's name in the greeting + const userName = preferences.name || 'there'; + return (
{/* Top toolbar */}
- +
- {/* Main grid — scrollable center */} + {/* Main grid */}
@@ -70,26 +143,26 @@ export default function App() { {/* Column 3: Chat + Text Input */}
- - + + {/* Text input fallback */}
+ {/* Subtle greeting */} +
+ hey {userName} ✨ +
setTextInput(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && handleTextSend()} - placeholder="type a message..." + placeholder={`what's up, ${userName}?`} className="flex-1 bg-white/60 rounded-xl px-3 py-2 text-sm text-kira-plum placeholder-kira-plum/30 border border-kira-pink/20 focus:outline-none focus:border-kira-pink/50" /> -
- {/* Connection indicator */}
{isConnected ? 'connected' : 'connecting...'} @@ -100,7 +173,7 @@ export default function App() { {/* Column 4: Cats + Wardrobe */}
- +
@@ -111,10 +184,10 @@ export default function App() {
- {isRecording ? 'listening...' : isKiraSpeaking ? 'kira speaking' : 'kira is here'} + {isRecording ? 'listening...' : isKiraSpeaking ? 'kira speaking' : `kira's here for you, ${userName}`}
- {isConnected ? 'body double mode' : 'offline'} + {identified ? `hi ${userName}` : 'body double mode'} 🌸
diff --git a/frontend/src/components/ChatBubble.tsx b/frontend/src/components/ChatBubble.tsx index 8a47e52..8189dfb 100644 --- a/frontend/src/components/ChatBubble.tsx +++ b/frontend/src/components/ChatBubble.tsx @@ -4,11 +4,13 @@ interface Message { id: string; role: 'user' | 'kira'; text: string; + timestamp: number; } interface Props { messages: Message[]; isKiraSpeaking: boolean; + userName?: string; } export default function ChatBubble({ messages, isKiraSpeaking }: Props) { diff --git a/frontend/src/components/WelcomeScreen.tsx b/frontend/src/components/WelcomeScreen.tsx new file mode 100644 index 0000000..93a5da3 --- /dev/null +++ b/frontend/src/components/WelcomeScreen.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react'; + +interface Props { + onComplete: (name: string) => void; +} + +export default function WelcomeScreen({ onComplete }: Props) { + const [name, setName] = useState(''); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + const trimmed = name.trim(); + if (trimmed) onComplete(trimmed); + }; + + return ( +
+
+ {/* Avatar preview */} +
+
+
+ 🌸 +
+
+
+ +
+

Welcome to Kira!

+

+ I'm your focus bestie. I'm here to body-double with you — + keep you company, cheer you on, and help you get things done. + Lo-fi beats, timers, cats, the whole deal. +

+
+ +
+
+ + setName(e.target.value)} + placeholder="Your name..." + className="w-full bg-white/60 rounded-xl px-4 py-3 text-sm text-kira-plum placeholder-kira-plum/30 border border-kira-pink/20 focus:outline-none focus:border-kira-pink/50 transition-all" + autoFocus + maxLength={30} + /> +
+ + +
+ +

+ everything stays between us — your name, your preferences, your focus vibes +

+
+ + +
+ ); +} diff --git a/frontend/src/hooks/useConversation.ts b/frontend/src/hooks/useConversation.ts index f3ac24a..29507bd 100644 --- a/frontend/src/hooks/useConversation.ts +++ b/frontend/src/hooks/useConversation.ts @@ -1,5 +1,12 @@ import { useState, useCallback, useRef, useEffect } from 'react'; +export interface UserPreferences { + name: string; + scene: string; + outfit: string; + accessory: string; +} + interface Message { id: string; role: 'user' | 'kira'; @@ -8,25 +15,54 @@ interface Message { } const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`; +const USER_ID_KEY = 'kira-user-id'; + +function loadUserId(): string { + return localStorage.getItem(USER_ID_KEY) || ''; +} + +function saveUserId(id: string) { + localStorage.setItem(USER_ID_KEY, id); +} export function useConversation() { const [messages, setMessages] = useState([]); const [isConnected, setIsConnected] = useState(false); const [isKiraSpeaking, setIsKiraSpeaking] = useState(false); + const [isRecording, setIsRecording] = useState(false); + const [identified, setIdentified] = useState(false); + const [preferences, setPreferences] = useState({ + name: '', + scene: 'cozy-room', + outfit: 'cozy-hoodie', + accessory: '', + }); + const [loadingPrefs, setLoadingPrefs] = useState(true); + const wsRef = useRef(null); const audioRef = useRef(null); const recorderRef = useRef(null); const streamRef = useRef(null); - const [isRecording, setIsRecording] = useState(false); // Connect WebSocket const connect = useCallback(() => { if (wsRef.current?.readyState === WebSocket.OPEN) return; + setLoadingPrefs(true); const ws = new WebSocket(WS_URL); wsRef.current = ws; - ws.onopen = () => setIsConnected(true); + ws.onopen = () => { + setIsConnected(true); + // Auto-identify if returning user + const savedId = loadUserId(); + if (savedId) { + ws.send(JSON.stringify({ type: 'identify', user_id: savedId })); + } else { + setLoadingPrefs(false); + } + }; + ws.onclose = () => { setIsConnected(false); setTimeout(connect, 3000); @@ -51,6 +87,25 @@ export function useConversation() { // Handle incoming WS messages const handleMessage = useCallback((msg: any) => { switch (msg.type) { + case 'identified': { + setIdentified(true); + setLoadingPrefs(false); + if (msg.user_id) saveUserId(msg.user_id); + if (msg.preferences) { + setPreferences({ + name: msg.preferences.name || '', + scene: msg.preferences.scene || 'cozy-room', + outfit: msg.preferences.outfit || 'cozy-hoodie', + accessory: msg.preferences.accessory || '', + }); + } + break; + } + + case 'preference_saved': + // Already optimistically updated locally + break; + case 'transcript': addMessage('user', msg.text); break; @@ -91,15 +146,52 @@ export function useConversation() { ]); }, []); - // Send text directly (no microphone) - const sendText = useCallback((text: string) => { - if (!text.trim()) return; + // ── Identity ── + + const identify = useCallback((name: string) => { + const userId = `kira-${name.toLowerCase().replace(/[^a-z0-9]/g, '-')}`; + saveUserId(userId); + setPreferences((p) => ({ ...p, name })); + if (wsRef.current?.readyState === WebSocket.OPEN) { - wsRef.current.send(JSON.stringify({ type: 'conversation_text', text: text.trim() })); + wsRef.current.send(JSON.stringify({ + type: 'identify', + user_id: userId, + name, + })); } }, []); - // Push-to-talk: start recording + // ── Preferences ── + + const setPreference = useCallback((key: string, value: string) => { + // Optimistic update + setPreferences((p) => ({ ...p, [key]: value })); + + // Sync to backend + if (wsRef.current?.readyState === WebSocket.OPEN && identified) { + wsRef.current.send(JSON.stringify({ + type: 'set_preference', + key, + value, + })); + } + }, [identified]); + + // ── Text ── + + const sendText = useCallback((text: string) => { + if (!text.trim()) return; + if (wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ + type: 'conversation_text', + text: text.trim(), + })); + } + }, []); + + // ── Audio ── + const startRecording = useCallback(async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -117,16 +209,12 @@ export function useConversation() { }; recorder.onstop = () => { - // Send all audio chunks as one blob const blob = new Blob(chunks, { type: 'audio/webm' }); const reader = new FileReader(); reader.onload = () => { const base64 = (reader.result as string).split(',')[1]; if (wsRef.current?.readyState === WebSocket.OPEN) { - wsRef.current.send(JSON.stringify({ - type: 'audio_chunk', - data: base64, - })); + wsRef.current.send(JSON.stringify({ type: 'audio_chunk', data: base64 })); wsRef.current.send(JSON.stringify({ type: 'transcribe' })); } }; @@ -144,7 +232,6 @@ export function useConversation() { } }, []); - // Push-to-talk: stop recording const stopRecording = useCallback(() => { recorderRef.current?.stop(); }, []); @@ -163,6 +250,11 @@ export function useConversation() { isConnected, isKiraSpeaking, isRecording, + identified, + preferences, + loadingPrefs, + identify, + setPreference, sendText, startRecording, stopRecording,