init: Kira — AI body double with Honcho memory

Full voice pipeline (Whisper STT -> DeepSeek LLM -> OpenAI TTS),
animated SVG avatar (Live2D-ready), girly-pop UI, lofi music,
timer/notes/pets/wardrobe widgets, 10 background scenes with
particle effects, Honcho cross-session memory.
This commit is contained in:
2026-06-04 10:51:38 -04:00
commit 97424cb98f
47 changed files with 5691 additions and 0 deletions
+170
View File
@@ -0,0 +1,170 @@
import { useState, useCallback, useRef, useEffect } from 'react';
interface Message {
id: string;
role: 'user' | 'kira';
text: string;
timestamp: number;
}
const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`;
export function useConversation() {
const [messages, setMessages] = useState<Message[]>([]);
const [isConnected, setIsConnected] = useState(false);
const [isKiraSpeaking, setIsKiraSpeaking] = useState(false);
const wsRef = useRef<WebSocket | null>(null);
const audioRef = useRef<HTMLAudioElement | null>(null);
const recorderRef = useRef<MediaRecorder | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const [isRecording, setIsRecording] = useState(false);
// Connect WebSocket
const connect = useCallback(() => {
if (wsRef.current?.readyState === WebSocket.OPEN) return;
const ws = new WebSocket(WS_URL);
wsRef.current = ws;
ws.onopen = () => setIsConnected(true);
ws.onclose = () => {
setIsConnected(false);
setTimeout(connect, 3000);
};
ws.onmessage = (event) => {
try {
const msg = JSON.parse(event.data);
handleMessage(msg);
} catch { /* ignore parse errors */ }
};
}, []);
// Audio playback element
useEffect(() => {
if (!audioRef.current) {
audioRef.current = new Audio();
audioRef.current.onended = () => setIsKiraSpeaking(false);
}
}, []);
// Handle incoming WS messages
const handleMessage = useCallback((msg: any) => {
switch (msg.type) {
case 'transcript':
addMessage('user', msg.text);
break;
case 'speaking_start':
setIsKiraSpeaking(true);
addMessage('kira', msg.text || '...');
break;
case 'audio':
if (msg.data && audioRef.current) {
const binary = atob(msg.data);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++) {
bytes[i] = binary.charCodeAt(i);
}
const blob = new Blob([bytes], { type: 'audio/ogg' });
const url = URL.createObjectURL(blob);
audioRef.current.src = url;
audioRef.current.play().catch(() => {});
}
break;
case 'speaking_end':
setIsKiraSpeaking(false);
break;
case 'error':
console.error('[Kira]', msg.message);
break;
}
}, []);
const addMessage = useCallback((role: 'user' | 'kira', text: string) => {
setMessages((prev) => [
...prev,
{ id: crypto.randomUUID(), role, text, timestamp: Date.now() },
]);
}, []);
// Send text directly (no microphone)
const sendText = useCallback((text: string) => {
if (!text.trim()) return;
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'conversation_text', text: text.trim() }));
}
}, []);
// Push-to-talk: start recording
const startRecording = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
streamRef.current = stream;
const recorder = new MediaRecorder(stream, {
mimeType: MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
? 'audio/webm;codecs=opus'
: 'audio/webm',
});
const chunks: BlobPart[] = [];
recorder.ondataavailable = (e) => {
if (e.data.size > 0) chunks.push(e.data);
};
recorder.onstop = () => {
// Send all audio chunks as one blob
const blob = new Blob(chunks, { type: 'audio/webm' });
const reader = new FileReader();
reader.onload = () => {
const base64 = (reader.result as string).split(',')[1];
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({
type: 'audio_chunk',
data: base64,
}));
wsRef.current.send(JSON.stringify({ type: 'transcribe' }));
}
};
reader.readAsDataURL(blob);
stream.getTracks().forEach((t) => t.stop());
setIsRecording(false);
};
recorder.start();
recorderRef.current = recorder;
setIsRecording(true);
} catch (err) {
console.error('[Kira Mic] failed:', err);
}
}, []);
// Push-to-talk: stop recording
const stopRecording = useCallback(() => {
recorderRef.current?.stop();
}, []);
// Connect on mount
useEffect(() => {
connect();
return () => {
wsRef.current?.close();
streamRef.current?.getTracks().forEach((t) => t.stop());
};
}, [connect]);
return {
messages,
isConnected,
isKiraSpeaking,
isRecording,
sendText,
startRecording,
stopRecording,
};
}