feat(phase6): video grid, noise suppression, whisper system

- New VideoGrid component: live camera feeds (local + remote) in VoicePanel
- Camera toggle in VoiceControls: enable/disable video per participant
- Noise suppression toggle in VoiceControls + voice store
- VOICE_WHISPER WS event: backend routes whisper to target user only
- Whisper UI: per-participant whisper button, notification toasts with dismiss
- Fix circular import between voice.ts and ws.ts (use lazy accessor)
This commit is contained in:
2026-06-30 12:21:53 -04:00
parent 3c063d4f56
commit 775bd953b0
8 changed files with 307 additions and 28 deletions
+106
View File
@@ -0,0 +1,106 @@
import { useEffect, useRef } from 'react';
import { Track } from 'livekit-client';
import type { Participant, TrackPublication, Room } from 'livekit-client';
import { useVoiceStore } from '../stores/voice.ts';
function VideoTile({ participant, isLocal, room }: { participant: Participant; isLocal: boolean; room: Room }) {
const videoRef = useRef<HTMLVideoElement>(null);
useEffect(() => {
const el = videoRef.current;
if (!el) return;
let pub: TrackPublication | undefined;
const attachTrack = () => {
pub = participant.getTrackPublication(Track.Source.Camera);
if (pub?.track && el) {
pub.track.attach(el);
el.muted = isLocal;
el.play().catch(() => {});
}
};
const detachTrack = () => {
if (pub?.track && el) {
pub.track.detach(el);
}
};
attachTrack();
// Listen for track publications changing on this participant via room events
const handlePublished = (pub: TrackPublication) => {
pub.track?.attach(el);
el.play().catch(() => {});
};
room.on('trackPublished' as any, handlePublished);
return () => {
detachTrack();
room.off('trackPublished' as any, handlePublished);
};
}, [participant, isLocal, room]);
const username = participant.name || participant.identity;
return (
<div className="relative bg-gb-bg rounded-sm overflow-hidden border border-gb-bg-t aspect-video flex items-center justify-center">
<video
ref={videoRef}
className="absolute inset-0 w-full h-full object-cover"
autoPlay
playsInline
/>
<div className="absolute bottom-1 left-1 text-xxs text-gb-fg-s bg-gb-bg/70 px-1 py-0.5 rounded-sm">
{username}{isLocal ? ' (you)' : ''}
</div>
</div>
);
}
export function VideoGrid() {
const room = useVoiceStore((s) => s._room);
const isVideoOn = useVoiceStore((s) => s.isVideoOn);
if (!room || !isVideoOn) return null;
const participants: Participant[] = [
room.localParticipant,
...Array.from(room.remoteParticipants.values()),
];
const hasCam = participants.filter((p) => {
const pub = p.getTrackPublication(Track.Source.Camera);
return pub && !pub.isMuted;
});
if (hasCam.length === 0) return null;
return (
<div className="px-3 py-1.5 border-b border-gb-bg-t">
<div className="text-xxs text-gb-fg-f mb-1">
video [{hasCam.length}]
</div>
<div
className={`grid gap-1 ${
hasCam.length <= 2
? 'grid-cols-2'
: hasCam.length <= 4
? 'grid-cols-2'
: 'grid-cols-3'
}`}
>
{participants.map((p) => (
<VideoTile
key={p.identity}
participant={p}
isLocal={p.identity === room.localParticipant.identity}
room={room}
/>
))}
</div>
</div>
);
}
+13
View File
@@ -5,10 +5,12 @@ export function VoiceControls() {
const isDeafened = useVoiceStore((state) => state.isDeafened);
const isVideoOn = useVoiceStore((state) => state.isVideoOn);
const isScreenSharing = useVoiceStore((state) => state.isScreenSharing);
const noiseSuppression = useVoiceStore((state) => state.noiseSuppression);
const toggleMute = useVoiceStore((state) => state.toggleMute);
const toggleDeafen = useVoiceStore((state) => state.toggleDeafen);
const toggleVideo = useVoiceStore((state) => state.toggleVideo);
const toggleScreenShare = useVoiceStore((state) => state.toggleScreenShare);
const toggleNoiseSuppression = useVoiceStore((state) => state.toggleNoiseSuppression);
const leaveVoice = useVoiceStore((state) => state.leaveVoice);
return (
@@ -57,6 +59,17 @@ export function VoiceControls() {
>
[share]
</button>
<button
onClick={toggleNoiseSuppression}
className={`px-1.5 py-0.5 rounded-sm transition-colors ${
noiseSuppression
? 'text-gb-green bg-gb-bg-s border border-gb-green'
: 'text-gb-fg-s hover:text-gb-green hover:bg-gb-bg-t border border-transparent'
}`}
title={noiseSuppression ? 'Noise suppression on' : 'Noise suppression off'}
>
[ns{noiseSuppression ? '●' : '○'}]
</button>
<div className="flex-1" />
<button
onClick={leaveVoice}
+74 -28
View File
@@ -1,5 +1,7 @@
import { useVoiceStore } from '../stores/voice.ts';
import { useAuthStore } from '../stores/auth.ts';
import { VoiceControls } from './VoiceControls.tsx';
import { VideoGrid } from './VideoGrid.tsx';
export function VoicePanel() {
const isConnected = useVoiceStore((state) => state.isConnected);
@@ -7,6 +9,10 @@ export function VoicePanel() {
const currentRoomName = useVoiceStore((state) => state.currentRoomName);
const participants = useVoiceStore((state) => state.participants);
const error = useVoiceStore((state) => state.error);
const whispers = useVoiceStore((state) => state.whispers);
const whisperTo = useVoiceStore((state) => state.whisperTo);
const dismissWhisper = useVoiceStore((state) => state.dismissWhisper);
const currentUserId = useAuthStore((state) => state.user?.id);
if (!currentRoom) return null;
@@ -38,6 +44,31 @@ export function VoicePanel() {
</div>
)}
{/* Incoming whispers */}
{whispers.length > 0 && (
<div className="px-3 py-1.5 border-b border-gb-bg-t">
<div className="text-xxs text-gb-fg-f mb-1">
whispers
</div>
{whispers.map((w) => (
<div
key={w.timestamp}
className="flex items-center gap-2 py-0.5 text-xs text-gb-orange"
>
<span>🔊</span>
<span className="font-bold">{w.fromUsername}</span>
<span className="text-gb-fg-s truncate">{w.message || 'whispered'}</span>
<button
className="ml-auto text-xxs text-gb-fg-f hover:text-gb-fg-s"
onClick={() => dismissWhisper(w.timestamp)}
>
[x]
</button>
</div>
))}
</div>
)}
{/* Participants */}
<div className="px-3 py-1.5 border-b border-gb-bg-t">
<div className="text-xxs text-gb-fg-f mb-1">
@@ -46,37 +77,52 @@ export function VoicePanel() {
{participants.length === 0 && (
<div className="text-xxs text-gb-fg-f">[empty channel]</div>
)}
{participants.map((p) => (
<div
key={p.identity}
className="flex items-center gap-2 py-0.5 text-xs"
>
<span
className={
p.isSpeaking
? 'text-gb-green'
: p.isMuted
? 'text-gb-red'
: 'text-gb-fg-s'
}
{participants.map((p) => {
const isMe = p.identity === currentUserId;
return (
<div
key={p.identity}
className="flex items-center gap-2 py-0.5 text-xs"
>
{p.isMuted ? '○' : '●'}
</span>
<span
className={p.isSpeaking ? 'text-gb-green' : 'text-gb-fg-s'}
>
{p.username}
</span>
{p.isMuted && (
<span className="text-xxs text-gb-red">[muted]</span>
)}
{p.isSpeaking && (
<span className="text-xxs text-gb-green">[speaking]</span>
)}
</div>
))}
<span
className={
p.isSpeaking
? 'text-gb-green'
: p.isMuted
? 'text-gb-red'
: 'text-gb-fg-s'
}
>
{p.isMuted ? '○' : '●'}
</span>
<span
className={p.isSpeaking ? 'text-gb-green' : 'text-gb-fg-s'}
>
{p.username}
</span>
{p.isMuted && (
<span className="text-xxs text-gb-red">[muted]</span>
)}
{p.isSpeaking && (
<span className="text-xxs text-gb-green">[speaking]</span>
)}
{!isMe && (
<button
className="ml-auto text-xxs text-gb-fg-f hover:text-gb-orange"
onClick={() => whisperTo(p.identity, p.username)}
title={`Whisper to ${p.username}`}
>
[whisper]
</button>
)}
</div>
);
})}
</div>
{/* Video grid */}
<VideoGrid />
{/* Controls */}
<div className="px-3 py-1.5">
<VoiceControls />
+62
View File
@@ -11,6 +11,7 @@ import {
} from 'livekit-client';
import { api } from '../lib/api.ts';
import { useWebSocketStore } from './ws.ts';
import { useAuthStore } from './auth.ts';
export interface VoiceParticipant {
identity: string;
@@ -19,6 +20,13 @@ export interface VoiceParticipant {
isSpeaking: boolean;
}
export interface WhisperNotification {
fromUserId: string;
fromUsername: string;
message: string;
timestamp: number;
}
interface VoiceTokenResponse {
token: string;
livekit_url: string;
@@ -32,16 +40,22 @@ interface VoiceState {
isDeafened: boolean;
isVideoOn: boolean;
isScreenSharing: boolean;
noiseSuppression: boolean;
participants: VoiceParticipant[];
error: string | null;
_room: Room | null;
isJoining: boolean;
whispers: WhisperNotification[];
joinVoice: (channelId: string, channelName: string) => Promise<void>;
leaveVoice: () => Promise<void>;
toggleMute: () => void;
toggleDeafen: () => void;
toggleVideo: () => void;
toggleScreenShare: () => void;
toggleNoiseSuppression: () => void;
_addWhisper: (w: WhisperNotification) => void;
whisperTo: (targetUserId: string, targetUsername: string, message?: string) => void;
dismissWhisper: (timestamp: number) => void;
}
function participantToVoice(p: Participant): VoiceParticipant {
@@ -62,10 +76,12 @@ export const useVoiceStore = create<VoiceState>((set, get) => ({
isDeafened: false,
isVideoOn: false,
isScreenSharing: false,
noiseSuppression: true,
participants: [],
error: null,
_room: null,
isJoining: false,
whispers: [],
joinVoice: async (channelId, channelName) => {
if (get().isJoining) {
@@ -290,4 +306,50 @@ export const useVoiceStore = create<VoiceState>((set, get) => ({
await room.localParticipant.setScreenShareEnabled(newState);
set({ isScreenSharing: newState });
},
toggleNoiseSuppression: async () => {
const room = get()._room;
if (!room) return;
const newState = !get().noiseSuppression;
set({ noiseSuppression: newState });
// Apply noise suppression + echo cancellation to local audio track
try {
const audioTrack = room.localParticipant.getTrackPublication(Track.Source.Microphone);
if (audioTrack?.track?.mediaStreamTrack) {
await audioTrack.track.mediaStreamTrack.applyConstraints({
echoCancellation: newState,
noiseSuppression: newState,
autoGainControl: newState,
} as MediaTrackConstraints);
}
} catch (err) {
console.warn('[voice] failed to update audio processing:', err);
}
},
_addWhisper: (w) => {
set((state) => ({ whispers: [...state.whispers, w] }));
},
whisperTo: (targetUserId, targetUsername, message) => {
const wsSend = useWebSocketStore.getState().send;
const me = useAuthStore.getState().user;
wsSend({
type: 'VOICE_WHISPER',
payload: {
target_user_id: targetUserId,
target_username: targetUsername,
from_username: me?.username || 'unknown',
message: message || '',
},
});
},
dismissWhisper: (timestamp) => {
set((state) => ({
whispers: state.whispers.filter((w) => w.timestamp !== timestamp),
}));
},
}));
+12
View File
@@ -4,6 +4,7 @@ import { useChannelStore } from './channel.ts';
import { useServerStore } from './server.ts';
import { usePresenceStore } from './presence.ts';
import { useTypingStore } from './typing.ts';
import { useVoiceStore } from './voice.ts';
import type { Message } from './message.ts';
import type { Channel } from './channel.ts';
import type { Server } from './server.ts';
@@ -176,6 +177,17 @@ export const useWebSocketStore = create<WebSocketState>((set, get) => ({
}
break;
}
case 'VOICE_WHISPER': {
// Forwarded from backend — only intended recipient gets this
const { from_user_id, from_username, message } = data.payload as Record<string, string>;
useVoiceStore.getState()._addWhisper({
fromUserId: from_user_id || '',
fromUsername: from_username || 'unknown',
message: message || '',
timestamp: Date.now(),
});
break;
}
default:
break;
}