feat: user personalization with Honcho-backed preferences

- WelcomeScreen: first-time name entry with cute onboarding
- identify WS message: sets user_id, loads saved prefs from Honcho
- set_preference WS message: saves scene/outfit/accessory to Honcho metadata
- Preferences auto-load on return visits via localStorage + Honcho peer meta
- Kira uses the user's name in greeting and prompts
- Backend: get/set preference methods in KiraMemory service
- Frontend: optimistic preference updates, synced to backend on change
This commit is contained in:
2026-06-04 11:00:58 -04:00
parent 97424cb98f
commit 78ea059f08
6 changed files with 396 additions and 80 deletions
+78 -52
View File
@@ -6,6 +6,7 @@ Real-time speech-to-speech pipeline:
Honcho memory integration: Honcho memory integration:
Cross-session user context injected into LLM prompts, Cross-session user context injected into LLM prompts,
conversation exchanges stored for continuous learning. conversation exchanges stored for continuous learning.
User preferences (name, scene, outfit, accessory) persisted in peer metadata.
""" """
import json import json
@@ -65,10 +66,8 @@ def build_system_prompt(user_id: str) -> dict:
"""Build system prompt with Honcho memory context injected.""" """Build system prompt with Honcho memory context injected."""
base = BASE_SYSTEM_PROMPT base = BASE_SYSTEM_PROMPT
# Append memory context if Honcho is available
if kira_memory.enabled: if kira_memory.enabled:
try: try:
# Get user-specific context from Honcho
kira_memory.ensure_peers(user_id) kira_memory.ensure_peers(user_id)
memory_suffix = kira_memory.build_system_prompt_suffix() memory_suffix = kira_memory.build_system_prompt_suffix()
if memory_suffix: if memory_suffix:
@@ -79,35 +78,89 @@ def build_system_prompt(user_id: str) -> dict:
return {"role": "system", "content": base} return {"role": "system", "content": base}
def handle_identify(msg: dict, session_id: str) -> dict | None:
"""Handle user identification. Returns user preferences or None."""
user_id = msg.get("user_id", "").strip()
if not user_id:
return {"type": "error", "message": "user_id is required"}
user_name = msg.get("name", "").strip()
if user_name:
kira_memory.set_user_preference(user_id, "name", user_name)
prefs = kira_memory.get_user_preferences(user_id)
logger.info(f"[{session_id}] Identified as {user_id} (name={user_name or prefs.get('name', '')})")
return {
"type": "identified",
"user_id": user_id,
"preferences": prefs,
}
def handle_set_preference(msg: dict, session_id: str, user_id: str) -> dict | None:
"""Handle preference update. Returns success status."""
if not user_id or user_id == "default-user":
return {"type": "error", "message": "Must identify first"}
key = msg.get("key", "").strip()
value = msg.get("value", "").strip()
if not key:
return {"type": "error", "message": "key is required"}
ok = kira_memory.set_user_preference(user_id, key, value)
return {
"type": "preference_saved",
"key": key,
"success": ok,
}
@app.websocket("/api/ws") @app.websocket("/api/ws")
async def conversation_ws(websocket: WebSocket): async def conversation_ws(websocket: WebSocket):
await websocket.accept() await websocket.accept()
session_id = str(uuid.uuid4())[:8] session_id = str(uuid.uuid4())[:8]
user_id = "default-user" user_id = "default-user"
identified = False
logger.info(f"[{session_id}] WebSocket connected") logger.info(f"[{session_id}] WebSocket connected")
# Audio buffer accumulates chunks from one utterance
audio_buffer = bytearray() audio_buffer = bytearray()
conversation_history: list[dict] = [] conversation_history: list[dict] = []
# Initialize Honcho for this session
if kira_memory.enabled:
try: try:
kira_memory.ensure_peers(user_id)
kira_memory.ensure_session(session_id)
logger.info(f"[{session_id}] Honcho session ready")
except Exception as e:
logger.warning(f"[{session_id}] Honcho setup failed: {e}")
try:
first_message = True
while True: while True:
raw = await websocket.receive_text() raw = await websocket.receive_text()
msg = json.loads(raw) msg = json.loads(raw)
msg_type = msg.get("type", "") msg_type = msg.get("type", "")
# Build system prompt fresh each turn to get latest Honcho context # ── Identity & Preferences ──
if msg_type == "identify":
response = handle_identify(msg, session_id)
if response:
await websocket.send_json(response)
if response["type"] == "identified":
user_id = response["user_id"]
identified = True
# Set up Honcho for this user
if kira_memory.enabled:
try:
kira_memory.ensure_peers(user_id)
kira_memory.ensure_session(session_id)
logger.info(f"[{session_id}] Honcho session ready for {user_id}")
except Exception as e:
logger.warning(f"[{session_id}] Honcho setup failed: {e}")
continue
if msg_type == "set_preference":
response = handle_set_preference(msg, session_id, user_id)
if response:
await websocket.send_json(response)
continue
# ── Conversation ──
system_prompt = build_system_prompt(user_id) system_prompt = build_system_prompt(user_id)
if msg_type == "audio_chunk": if msg_type == "audio_chunk":
@@ -121,7 +174,6 @@ async def conversation_ws(websocket: WebSocket):
logger.info(f"[{session_id}] Transcribing {len(audio_buffer)} bytes...") logger.info(f"[{session_id}] Transcribing {len(audio_buffer)} bytes...")
# 1. Speech-to-text
transcript = await transcribe_audio(bytes(audio_buffer)) transcript = await transcribe_audio(bytes(audio_buffer))
audio_buffer.clear() audio_buffer.clear()
@@ -129,45 +181,27 @@ async def conversation_ws(websocket: WebSocket):
await websocket.send_json({"type": "error", "message": "Could not transcribe audio"}) await websocket.send_json({"type": "error", "message": "Could not transcribe audio"})
continue continue
# Echo transcript await websocket.send_json({"type": "transcript", "text": transcript})
await websocket.send_json({
"type": "transcript",
"text": transcript,
})
# 2. LLM call
logger.info(f"[{session_id}] User: {transcript}") logger.info(f"[{session_id}] User: {transcript}")
user_msg = {"role": "user", "content": transcript} conversation_history.append({"role": "user", "content": transcript})
conversation_history.append(user_msg)
messages = [system_prompt] + conversation_history[-10:] messages = [system_prompt] + conversation_history[-10:]
kira_text = await get_kira_response(messages) kira_text = await get_kira_response(messages)
assistant_msg = {"role": "assistant", "content": kira_text} conversation_history.append({"role": "assistant", "content": kira_text})
conversation_history.append(assistant_msg)
logger.info(f"[{session_id}] Kira: {kira_text}") logger.info(f"[{session_id}] Kira: {kira_text}")
# 3. Store in Honcho if kira_memory.enabled and identified:
if kira_memory.enabled:
try: try:
kira_memory.store_messages(transcript, kira_text) kira_memory.store_messages(transcript, kira_text)
except Exception as e: except Exception as e:
logger.warning(f"[{session_id}] Failed to store messages: {e}") logger.warning(f"[{session_id}] Failed to store messages: {e}")
# 4. TTS await websocket.send_json({"type": "speaking_start", "text": kira_text})
await websocket.send_json({
"type": "speaking_start",
"text": kira_text,
})
audio_bytes = await synthesize_speech(kira_text) audio_bytes = await synthesize_speech(kira_text)
audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
await websocket.send_json({ await websocket.send_json({"type": "audio", "data": audio_b64, "text": kira_text})
"type": "audio",
"data": audio_b64,
"text": kira_text,
})
await websocket.send_json({"type": "speaking_end"}) await websocket.send_json({"type": "speaking_end"})
elif msg_type == "ping": elif msg_type == "ping":
@@ -179,32 +213,24 @@ async def conversation_ws(websocket: WebSocket):
continue continue
logger.info(f"[{session_id}] User (text): {user_text}") logger.info(f"[{session_id}] User (text): {user_text}")
user_msg = {"role": "user", "content": user_text} conversation_history.append({"role": "user", "content": user_text})
conversation_history.append(user_msg)
messages = [system_prompt] + conversation_history[-10:] messages = [system_prompt] + conversation_history[-10:]
kira_text = await get_kira_response(messages) kira_text = await get_kira_response(messages)
assistant_msg = {"role": "assistant", "content": kira_text} conversation_history.append({"role": "assistant", "content": kira_text})
conversation_history.append(assistant_msg)
logger.info(f"[{session_id}] Kira: {kira_text}") logger.info(f"[{session_id}] Kira: {kira_text}")
# Store in Honcho if kira_memory.enabled and identified:
if kira_memory.enabled:
try: try:
kira_memory.store_messages(user_text, kira_text) kira_memory.store_messages(user_text, kira_text)
except Exception as e: except Exception as e:
logger.warning(f"[{session_id}] Failed to store messages: {e}") logger.warning(f"[{session_id}] Failed to store messages: {e}")
# TTS
await websocket.send_json({"type": "speaking_start", "text": kira_text}) await websocket.send_json({"type": "speaking_start", "text": kira_text})
audio_bytes = await synthesize_speech(kira_text) audio_bytes = await synthesize_speech(kira_text)
audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
await websocket.send_json({ await websocket.send_json({"type": "audio", "data": audio_b64, "text": kira_text})
"type": "audio",
"data": audio_b64,
"text": kira_text,
})
await websocket.send_json({"type": "speaking_end"}) await websocket.send_json({"type": "speaking_end"})
except WebSocketDisconnect: except WebSocketDisconnect:
+48
View File
@@ -178,6 +178,54 @@ class KiraMemory:
except Exception as e: except Exception as e:
logger.warning(f"Failed to store Kira message: {e}") logger.warning(f"Failed to store Kira message: {e}")
# ─── User preferences (stored in Honcho peer metadata) ───
DEFAULT_PREFERENCES: dict[str, str] = {
"name": "",
"scene": "cozy-room",
"outfit": "cozy-hoodie",
"accessory": "",
}
def get_user_preferences(self, user_id: str) -> dict[str, str]:
"""Load user preferences from Honcho peer metadata."""
if not self.enabled:
return dict(self.DEFAULT_PREFERENCES)
try:
peer = self._honcho.peer(user_id)
meta = peer.metadata or {}
prefs = dict(self.DEFAULT_PREFERENCES)
for key in prefs:
val = meta.get(key)
if val is not None:
prefs[key] = str(val)
logger.info(f"Loaded preferences for {user_id}: {prefs}")
return prefs
except Exception as e:
logger.warning(f"Failed to load preferences: {e}")
return dict(self.DEFAULT_PREFERENCES)
def set_user_preference(
self, user_id: str, key: str, value: str
) -> bool:
"""Save a single user preference to Honcho peer metadata."""
if key not in self.DEFAULT_PREFERENCES:
logger.warning(f"Unknown preference key: {key}")
return False
try:
peer = self._honcho.peer(user_id)
# Merge with existing metadata
meta = dict(peer.metadata or {})
meta[key] = value
peer.set_metadata(meta)
logger.info(f"Saved preference {user_id}.{key} = {value}")
return True
except Exception as e:
logger.warning(f"Failed to save preference: {e}")
return False
# Singleton instance for the app # Singleton instance for the app
kira_memory = KiraMemory() kira_memory = KiraMemory()
+87 -14
View File
@@ -1,4 +1,4 @@
import { useState } from 'react'; import { useState, useEffect } from 'react';
import Clock from './components/Clock'; import Clock from './components/Clock';
import BackgroundScene from './components/BackgroundScene'; import BackgroundScene from './components/BackgroundScene';
import MusicPlayer from './components/MusicPlayer'; import MusicPlayer from './components/MusicPlayer';
@@ -10,17 +10,54 @@ import PetZone from './components/PetZone';
import Wardrobe from './components/Wardrobe'; import Wardrobe from './components/Wardrobe';
import Toolbar from './components/Toolbar'; import Toolbar from './components/Toolbar';
import Particles from './components/Particles'; import Particles from './components/Particles';
import WelcomeScreen from './components/WelcomeScreen';
import { SCENES, type Scene } from './components/scenes'; import { SCENES, type Scene } from './components/scenes';
import { useConversation } from './hooks/useConversation'; import { useConversation } from './hooks/useConversation';
export default function App() { export default function App() {
const {
messages,
isConnected,
isKiraSpeaking,
isRecording,
identified,
preferences,
loadingPrefs,
identify,
setPreference,
sendText,
startRecording,
stopRecording,
} = useConversation();
const [currentSceneId, setCurrentSceneId] = useState('cozy-room'); const [currentSceneId, setCurrentSceneId] = useState('cozy-room');
const [currentOutfit, setCurrentOutfit] = useState('cozy-hoodie'); const [currentOutfit, setCurrentOutfit] = useState('cozy-hoodie');
const [currentAcc, setCurrentAcc] = useState<string | null>(null); const [currentAcc, setCurrentAcc] = useState<string | null>(null);
const [textInput, setTextInput] = useState(''); const [textInput, setTextInput] = useState('');
// Apply saved preferences once they load
useEffect(() => {
if (preferences.scene) setCurrentSceneId(preferences.scene);
if (preferences.outfit) setCurrentOutfit(preferences.outfit);
if (preferences.accessory) setCurrentAcc(preferences.accessory);
}, [preferences.scene, preferences.outfit, preferences.accessory]);
const currentScene: Scene = SCENES.find((s) => s.id === currentSceneId) ?? SCENES[0]; const currentScene: Scene = SCENES.find((s) => s.id === currentSceneId) ?? SCENES[0];
const { messages, isConnected, isKiraSpeaking, isRecording, sendText, startRecording, stopRecording } = useConversation();
const handleSceneChange = (id: string) => {
setCurrentSceneId(id);
setPreference('scene', id);
};
const handleOutfitChange = (id: string) => {
setCurrentOutfit(id);
setPreference('outfit', id);
};
const handleAccessoryChange = (id: string | null) => {
setCurrentAcc(id);
setPreference('accessory', id || '');
};
const handleTalkToggle = () => { const handleTalkToggle = () => {
if (isRecording) stopRecording(); if (isRecording) stopRecording();
@@ -33,6 +70,42 @@ export default function App() {
setTextInput(''); setTextInput('');
}; };
const handleWelcome = (name: string) => {
identify(name);
};
// Show WelcomeScreen for first-time users
if (!identified && !loadingPrefs) {
const savedId = localStorage.getItem('kira-user-id');
if (!savedId) {
return <WelcomeScreen onComplete={handleWelcome} />;
}
// Has saved ID but not identified yet — show welcome with their name
return (
<div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-kira-bg via-kira-glow to-kira-pink/20 p-4">
<div className="glass-card max-w-sm w-full p-6 text-center space-y-4">
<div className="w-16 h-16 rounded-full bg-gradient-to-br from-kira-pink via-kira-lav to-kira-mint p-1 mx-auto animate-pulse-glow">
<div className="w-full h-full rounded-full bg-white flex items-center justify-center">
<span className="text-2xl">🌸</span>
</div>
</div>
<p className="text-kira-plum/60 text-sm">coming back? say your name to pick up where you left off</p>
<WelcomeScreen onComplete={handleWelcome} />
</div>
<style>{`
@keyframes pulse-glow {
0%, 100% { box-shadow: 0 0 12px rgba(255,182,193,0.3); }
50% { box-shadow: 0 0 28px rgba(216,180,254,0.5); }
}
.animate-pulse-glow { animation: pulse-glow 3s ease-in-out infinite; }
`}</style>
</div>
);
}
// Main app with user's name in the greeting
const userName = preferences.name || 'there';
return ( return (
<div <div
className="min-h-screen relative transition-all duration-1000" className="min-h-screen relative transition-all duration-1000"
@@ -43,10 +116,10 @@ export default function App() {
<div className="relative z-20 h-screen flex flex-col"> <div className="relative z-20 h-screen flex flex-col">
{/* Top toolbar */} {/* Top toolbar */}
<div className="px-4 pt-4"> <div className="px-4 pt-4">
<Toolbar currentScene={currentSceneId} onSceneChange={setCurrentSceneId} /> <Toolbar currentScene={currentSceneId} onSceneChange={handleSceneChange} />
</div> </div>
{/* Main grid — scrollable center */} {/* Main grid */}
<div className="flex-1 overflow-y-auto px-4 py-4 scrollbar-thin"> <div className="flex-1 overflow-y-auto px-4 py-4 scrollbar-thin">
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 max-w-7xl mx-auto"> <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 max-w-7xl mx-auto">
@@ -70,26 +143,26 @@ export default function App() {
{/* Column 3: Chat + Text Input */} {/* Column 3: Chat + Text Input */}
<div className="space-y-4"> <div className="space-y-4">
<ChatBubble messages={messages} isKiraSpeaking={isKiraSpeaking} /> <ChatBubble messages={messages} isKiraSpeaking={isKiraSpeaking} userName={userName} />
{/* Text input fallback */} {/* Text input fallback */}
<div className="glass-card p-3"> <div className="glass-card p-3">
{/* Subtle greeting */}
<div className="text-[10px] text-kira-plum/30 mb-2">
hey {userName}
</div>
<div className="flex gap-2"> <div className="flex gap-2">
<input <input
value={textInput} value={textInput}
onChange={(e) => setTextInput(e.target.value)} onChange={(e) => setTextInput(e.target.value)}
onKeyDown={(e) => e.key === 'Enter' && handleTextSend()} onKeyDown={(e) => e.key === 'Enter' && handleTextSend()}
placeholder="type a message..." placeholder={`what's up, ${userName}?`}
className="flex-1 bg-white/60 rounded-xl px-3 py-2 text-sm text-kira-plum placeholder-kira-plum/30 border border-kira-pink/20 focus:outline-none focus:border-kira-pink/50" className="flex-1 bg-white/60 rounded-xl px-3 py-2 text-sm text-kira-plum placeholder-kira-plum/30 border border-kira-pink/20 focus:outline-none focus:border-kira-pink/50"
/> />
<button <button onClick={handleTextSend} className="btn-kira px-3 text-sm">
onClick={handleTextSend}
className="btn-kira px-3 text-sm"
>
Send Send
</button> </button>
</div> </div>
{/* Connection indicator */}
<div className="flex items-center gap-2 mt-2 text-[10px] text-kira-plum/30"> <div className="flex items-center gap-2 mt-2 text-[10px] text-kira-plum/30">
<span className={`w-1.5 h-1.5 rounded-full ${isConnected ? 'bg-kira-mint' : 'bg-red-300'}`} /> <span className={`w-1.5 h-1.5 rounded-full ${isConnected ? 'bg-kira-mint' : 'bg-red-300'}`} />
{isConnected ? 'connected' : 'connecting...'} {isConnected ? 'connected' : 'connecting...'}
@@ -100,7 +173,7 @@ export default function App() {
{/* Column 4: Cats + Wardrobe */} {/* Column 4: Cats + Wardrobe */}
<div className="space-y-4"> <div className="space-y-4">
<PetZone /> <PetZone />
<Wardrobe onOutfitChange={setCurrentOutfit} onAccessoryChange={setCurrentAcc} /> <Wardrobe onOutfitChange={handleOutfitChange} onAccessoryChange={handleAccessoryChange} />
</div> </div>
</div> </div>
@@ -111,10 +184,10 @@ export default function App() {
<div className="glass-card px-4 py-2 flex items-center justify-between text-xs text-kira-plum/40"> <div className="glass-card px-4 py-2 flex items-center justify-between text-xs text-kira-plum/40">
<div className="flex items-center gap-3"> <div className="flex items-center gap-3">
<span className={`w-2 h-2 rounded-full ${isRecording ? 'bg-red-400 animate-pulse' : isKiraSpeaking ? 'bg-kira-pink animate-pulse' : 'bg-kira-mint'} inline-block`} /> <span className={`w-2 h-2 rounded-full ${isRecording ? 'bg-red-400 animate-pulse' : isKiraSpeaking ? 'bg-kira-pink animate-pulse' : 'bg-kira-mint'} inline-block`} />
<span>{isRecording ? 'listening...' : isKiraSpeaking ? 'kira speaking' : 'kira is here'}</span> <span>{isRecording ? 'listening...' : isKiraSpeaking ? 'kira speaking' : `kira's here for you, ${userName}`}</span>
</div> </div>
<div className="flex items-center gap-3"> <div className="flex items-center gap-3">
<span className="hidden sm:inline">{isConnected ? 'body double mode' : 'offline'}</span> <span className="hidden sm:inline">{identified ? `hi ${userName}` : 'body double mode'}</span>
<span>🌸</span> <span>🌸</span>
</div> </div>
</div> </div>
+2
View File
@@ -4,11 +4,13 @@ interface Message {
id: string; id: string;
role: 'user' | 'kira'; role: 'user' | 'kira';
text: string; text: string;
timestamp: number;
} }
interface Props { interface Props {
messages: Message[]; messages: Message[];
isKiraSpeaking: boolean; isKiraSpeaking: boolean;
userName?: string;
} }
export default function ChatBubble({ messages, isKiraSpeaking }: Props) { export default function ChatBubble({ messages, isKiraSpeaking }: Props) {
+75
View File
@@ -0,0 +1,75 @@
import { useState } from 'react';
interface Props {
onComplete: (name: string) => void;
}
export default function WelcomeScreen({ onComplete }: Props) {
const [name, setName] = useState('');
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
const trimmed = name.trim();
if (trimmed) onComplete(trimmed);
};
return (
<div className="min-h-screen flex items-center justify-center bg-gradient-to-br from-kira-bg via-kira-glow to-kira-pink/20 p-4">
<div className="glass-card max-w-md w-full p-8 text-center space-y-6">
{/* Avatar preview */}
<div className="flex justify-center">
<div className="w-20 h-20 rounded-full bg-gradient-to-br from-kira-pink via-kira-lav to-kira-mint p-1 animate-pulse-glow">
<div className="w-full h-full rounded-full bg-white flex items-center justify-center">
<span className="text-3xl">🌸</span>
</div>
</div>
</div>
<div className="space-y-2">
<h1 className="text-2xl font-extrabold text-kira-plum">Welcome to Kira!</h1>
<p className="text-sm text-kira-violet/60 leading-relaxed">
I'm your focus bestie. I'm here to body-double with you
keep you company, cheer you on, and help you get things done.
Lo-fi beats, timers, cats, the whole deal.
</p>
</div>
<form onSubmit={handleSubmit} className="space-y-4">
<div>
<label className="block text-sm font-semibold text-kira-plum/70 mb-1.5 text-left">
What should I call you?
</label>
<input
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="Your name..."
className="w-full bg-white/60 rounded-xl px-4 py-3 text-sm text-kira-plum placeholder-kira-plum/30 border border-kira-pink/20 focus:outline-none focus:border-kira-pink/50 transition-all"
autoFocus
maxLength={30}
/>
</div>
<button
type="submit"
disabled={!name.trim()}
className="btn-kira w-full py-3 text-base disabled:opacity-40 disabled:cursor-not-allowed transition-all"
>
Let's Go!
</button>
</form>
<p className="text-[10px] text-kira-plum/30">
everything stays between us your name, your preferences, your focus vibes
</p>
</div>
<style>{`
@keyframes pulse-glow {
0%, 100% { box-shadow: 0 0 12px rgba(255,182,193,0.3); }
50% { box-shadow: 0 0 28px rgba(216,180,254,0.5); }
}
.animate-pulse-glow { animation: pulse-glow 3s ease-in-out infinite; }
`}</style>
</div>
);
}
+105 -13
View File
@@ -1,5 +1,12 @@
import { useState, useCallback, useRef, useEffect } from 'react'; import { useState, useCallback, useRef, useEffect } from 'react';
export interface UserPreferences {
name: string;
scene: string;
outfit: string;
accessory: string;
}
interface Message { interface Message {
id: string; id: string;
role: 'user' | 'kira'; role: 'user' | 'kira';
@@ -8,25 +15,54 @@ interface Message {
} }
const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`; const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`;
const USER_ID_KEY = 'kira-user-id';
function loadUserId(): string {
return localStorage.getItem(USER_ID_KEY) || '';
}
function saveUserId(id: string) {
localStorage.setItem(USER_ID_KEY, id);
}
export function useConversation() { export function useConversation() {
const [messages, setMessages] = useState<Message[]>([]); const [messages, setMessages] = useState<Message[]>([]);
const [isConnected, setIsConnected] = useState(false); const [isConnected, setIsConnected] = useState(false);
const [isKiraSpeaking, setIsKiraSpeaking] = useState(false); const [isKiraSpeaking, setIsKiraSpeaking] = useState(false);
const [isRecording, setIsRecording] = useState(false);
const [identified, setIdentified] = useState(false);
const [preferences, setPreferences] = useState<UserPreferences>({
name: '',
scene: 'cozy-room',
outfit: 'cozy-hoodie',
accessory: '',
});
const [loadingPrefs, setLoadingPrefs] = useState(true);
const wsRef = useRef<WebSocket | null>(null); const wsRef = useRef<WebSocket | null>(null);
const audioRef = useRef<HTMLAudioElement | null>(null); const audioRef = useRef<HTMLAudioElement | null>(null);
const recorderRef = useRef<MediaRecorder | null>(null); const recorderRef = useRef<MediaRecorder | null>(null);
const streamRef = useRef<MediaStream | null>(null); const streamRef = useRef<MediaStream | null>(null);
const [isRecording, setIsRecording] = useState(false);
// Connect WebSocket // Connect WebSocket
const connect = useCallback(() => { const connect = useCallback(() => {
if (wsRef.current?.readyState === WebSocket.OPEN) return; if (wsRef.current?.readyState === WebSocket.OPEN) return;
setLoadingPrefs(true);
const ws = new WebSocket(WS_URL); const ws = new WebSocket(WS_URL);
wsRef.current = ws; wsRef.current = ws;
ws.onopen = () => setIsConnected(true); ws.onopen = () => {
setIsConnected(true);
// Auto-identify if returning user
const savedId = loadUserId();
if (savedId) {
ws.send(JSON.stringify({ type: 'identify', user_id: savedId }));
} else {
setLoadingPrefs(false);
}
};
ws.onclose = () => { ws.onclose = () => {
setIsConnected(false); setIsConnected(false);
setTimeout(connect, 3000); setTimeout(connect, 3000);
@@ -51,6 +87,25 @@ export function useConversation() {
// Handle incoming WS messages // Handle incoming WS messages
const handleMessage = useCallback((msg: any) => { const handleMessage = useCallback((msg: any) => {
switch (msg.type) { switch (msg.type) {
case 'identified': {
setIdentified(true);
setLoadingPrefs(false);
if (msg.user_id) saveUserId(msg.user_id);
if (msg.preferences) {
setPreferences({
name: msg.preferences.name || '',
scene: msg.preferences.scene || 'cozy-room',
outfit: msg.preferences.outfit || 'cozy-hoodie',
accessory: msg.preferences.accessory || '',
});
}
break;
}
case 'preference_saved':
// Already optimistically updated locally
break;
case 'transcript': case 'transcript':
addMessage('user', msg.text); addMessage('user', msg.text);
break; break;
@@ -91,15 +146,52 @@ export function useConversation() {
]); ]);
}, []); }, []);
// Send text directly (no microphone) // ── Identity ──
const sendText = useCallback((text: string) => {
if (!text.trim()) return; const identify = useCallback((name: string) => {
const userId = `kira-${name.toLowerCase().replace(/[^a-z0-9]/g, '-')}`;
saveUserId(userId);
setPreferences((p) => ({ ...p, name }));
if (wsRef.current?.readyState === WebSocket.OPEN) { if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'conversation_text', text: text.trim() })); wsRef.current.send(JSON.stringify({
type: 'identify',
user_id: userId,
name,
}));
} }
}, []); }, []);
// Push-to-talk: start recording // ── Preferences ──
const setPreference = useCallback((key: string, value: string) => {
// Optimistic update
setPreferences((p) => ({ ...p, [key]: value }));
// Sync to backend
if (wsRef.current?.readyState === WebSocket.OPEN && identified) {
wsRef.current.send(JSON.stringify({
type: 'set_preference',
key,
value,
}));
}
}, [identified]);
// ── Text ──
const sendText = useCallback((text: string) => {
if (!text.trim()) return;
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({
type: 'conversation_text',
text: text.trim(),
}));
}
}, []);
// ── Audio ──
const startRecording = useCallback(async () => { const startRecording = useCallback(async () => {
try { try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
@@ -117,16 +209,12 @@ export function useConversation() {
}; };
recorder.onstop = () => { recorder.onstop = () => {
// Send all audio chunks as one blob
const blob = new Blob(chunks, { type: 'audio/webm' }); const blob = new Blob(chunks, { type: 'audio/webm' });
const reader = new FileReader(); const reader = new FileReader();
reader.onload = () => { reader.onload = () => {
const base64 = (reader.result as string).split(',')[1]; const base64 = (reader.result as string).split(',')[1];
if (wsRef.current?.readyState === WebSocket.OPEN) { if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ wsRef.current.send(JSON.stringify({ type: 'audio_chunk', data: base64 }));
type: 'audio_chunk',
data: base64,
}));
wsRef.current.send(JSON.stringify({ type: 'transcribe' })); wsRef.current.send(JSON.stringify({ type: 'transcribe' }));
} }
}; };
@@ -144,7 +232,6 @@ export function useConversation() {
} }
}, []); }, []);
// Push-to-talk: stop recording
const stopRecording = useCallback(() => { const stopRecording = useCallback(() => {
recorderRef.current?.stop(); recorderRef.current?.stop();
}, []); }, []);
@@ -163,6 +250,11 @@ export function useConversation() {
isConnected, isConnected,
isKiraSpeaking, isKiraSpeaking,
isRecording, isRecording,
identified,
preferences,
loadingPrefs,
identify,
setPreference,
sendText, sendText,
startRecording, startRecording,
stopRecording, stopRecording,