init: Kira — AI body double with Honcho memory
Full voice pipeline (Whisper STT -> DeepSeek LLM -> OpenAI TTS), animated SVG avatar (Live2D-ready), girly-pop UI, lofi music, timer/notes/pets/wardrobe widgets, 10 background scenes with particle effects, Honcho cross-session memory.
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
import { useState, useCallback, useRef, useEffect } from 'react';
|
||||
|
||||
interface Message {
|
||||
id: string;
|
||||
role: 'user' | 'kira';
|
||||
text: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
const WS_URL = `${location.protocol === 'https:' ? 'wss:' : 'ws:'}//${location.host}/api/ws`;
|
||||
|
||||
export function useConversation() {
|
||||
const [messages, setMessages] = useState<Message[]>([]);
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [isKiraSpeaking, setIsKiraSpeaking] = useState(false);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
|
||||
// Connect WebSocket
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) return;
|
||||
|
||||
const ws = new WebSocket(WS_URL);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.onopen = () => setIsConnected(true);
|
||||
ws.onclose = () => {
|
||||
setIsConnected(false);
|
||||
setTimeout(connect, 3000);
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
handleMessage(msg);
|
||||
} catch { /* ignore parse errors */ }
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Audio playback element
|
||||
useEffect(() => {
|
||||
if (!audioRef.current) {
|
||||
audioRef.current = new Audio();
|
||||
audioRef.current.onended = () => setIsKiraSpeaking(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Handle incoming WS messages
|
||||
const handleMessage = useCallback((msg: any) => {
|
||||
switch (msg.type) {
|
||||
case 'transcript':
|
||||
addMessage('user', msg.text);
|
||||
break;
|
||||
|
||||
case 'speaking_start':
|
||||
setIsKiraSpeaking(true);
|
||||
addMessage('kira', msg.text || '...');
|
||||
break;
|
||||
|
||||
case 'audio':
|
||||
if (msg.data && audioRef.current) {
|
||||
const binary = atob(msg.data);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
const blob = new Blob([bytes], { type: 'audio/ogg' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
audioRef.current.src = url;
|
||||
audioRef.current.play().catch(() => {});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'speaking_end':
|
||||
setIsKiraSpeaking(false);
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
console.error('[Kira]', msg.message);
|
||||
break;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const addMessage = useCallback((role: 'user' | 'kira', text: string) => {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{ id: crypto.randomUUID(), role, text, timestamp: Date.now() },
|
||||
]);
|
||||
}, []);
|
||||
|
||||
// Send text directly (no microphone)
|
||||
const sendText = useCallback((text: string) => {
|
||||
if (!text.trim()) return;
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({ type: 'conversation_text', text: text.trim() }));
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Push-to-talk: start recording
|
||||
const startRecording = useCallback(async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const recorder = new MediaRecorder(stream, {
|
||||
mimeType: MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
|
||||
? 'audio/webm;codecs=opus'
|
||||
: 'audio/webm',
|
||||
});
|
||||
|
||||
const chunks: BlobPart[] = [];
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunks.push(e.data);
|
||||
};
|
||||
|
||||
recorder.onstop = () => {
|
||||
// Send all audio chunks as one blob
|
||||
const blob = new Blob(chunks, { type: 'audio/webm' });
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const base64 = (reader.result as string).split(',')[1];
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify({
|
||||
type: 'audio_chunk',
|
||||
data: base64,
|
||||
}));
|
||||
wsRef.current.send(JSON.stringify({ type: 'transcribe' }));
|
||||
}
|
||||
};
|
||||
reader.readAsDataURL(blob);
|
||||
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
setIsRecording(false);
|
||||
};
|
||||
|
||||
recorder.start();
|
||||
recorderRef.current = recorder;
|
||||
setIsRecording(true);
|
||||
} catch (err) {
|
||||
console.error('[Kira Mic] failed:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Push-to-talk: stop recording
|
||||
const stopRecording = useCallback(() => {
|
||||
recorderRef.current?.stop();
|
||||
}, []);
|
||||
|
||||
// Connect on mount
|
||||
useEffect(() => {
|
||||
connect();
|
||||
return () => {
|
||||
wsRef.current?.close();
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
return {
|
||||
messages,
|
||||
isConnected,
|
||||
isKiraSpeaking,
|
||||
isRecording,
|
||||
sendText,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user