From 775bd953b0299e19e9ea51918e26498c7b897393 Mon Sep 17 00:00:00 2001 From: hobokenchicken Date: Tue, 30 Jun 2026 12:21:53 -0400 Subject: [PATCH] feat(phase6): video grid, noise suppression, whisper system - New VideoGrid component: live camera feeds (local + remote) in VoicePanel - Camera toggle in VoiceControls: enable/disable video per participant - Noise suppression toggle in VoiceControls + voice store - VOICE_WHISPER WS event: backend routes whisper to target user only - Whisper UI: per-participant whisper button, notification toasts with dismiss - Fix circular import between voice.ts and ws.ts (use lazy accessor) --- internal/gateway/client.go | 17 +++++ internal/gateway/events.go | 1 + internal/gateway/hub.go | 22 ++++++ web/src/components/VideoGrid.tsx | 106 +++++++++++++++++++++++++++ web/src/components/VoiceControls.tsx | 13 ++++ web/src/components/VoicePanel.tsx | 102 +++++++++++++++++++------- web/src/stores/voice.ts | 62 ++++++++++++++++ web/src/stores/ws.ts | 12 +++ 8 files changed, 307 insertions(+), 28 deletions(-) create mode 100644 web/src/components/VideoGrid.tsx diff --git a/internal/gateway/client.go b/internal/gateway/client.go index db29de3..3f71cbc 100644 --- a/internal/gateway/client.go +++ b/internal/gateway/client.go @@ -142,6 +142,23 @@ func (c *Client) readPump() { switch event.Type { case EventTypingStart, EventPresenceUpdate: c.Hub.BroadcastEvent(event) + case EventVoiceWhisper: + // Forward voice whispers only to the target user, not broadcast + var whisperData struct { + TargetUserID string `json:"target_user_id"` + FromUserID string `json:"from_user_id"` + FromUsername string `json:"from_username"` + Message string `json:"message,omitempty"` + } + if raw, ok := event.Data.(json.RawMessage); ok { + if err := json.Unmarshal(raw, &whisperData); err == nil { + whisperData.FromUserID = c.UserID + c.Hub.SendToUser(whisperData.TargetUserID, Event{ + Type: EventVoiceWhisper, + Data: whisperData, + }) + } + } default: c.Hub.logger.Info("received event from client", "type", event.Type, "user_id", c.UserID) } diff --git a/internal/gateway/events.go b/internal/gateway/events.go index d747591..22223b4 100644 --- a/internal/gateway/events.go +++ b/internal/gateway/events.go @@ -22,6 +22,7 @@ const ( EventVoiceLeave = "VOICE_LEAVE" EventVoiceMute = "VOICE_MUTE" EventVoiceDeafen = "VOICE_DEAFEN" + EventVoiceWhisper = "VOICE_WHISPER" ) // Event represents a WebSocket event sent to clients. diff --git a/internal/gateway/hub.go b/internal/gateway/hub.go index 41ca094..e169cd6 100644 --- a/internal/gateway/hub.go +++ b/internal/gateway/hub.go @@ -227,6 +227,28 @@ func (h *Hub) ClientCount() int { return len(h.clients) } +// SendToUser sends an event only to the specified user's connected client(s). +func (h *Hub) SendToUser(userID string, event Event) { + data, err := json.Marshal(event) + if err != nil { + h.logger.Error("failed to marshal event", "type", event.Type, "error", err) + return + } + + h.mu.RLock() + defer h.mu.RUnlock() + + for client := range h.clients { + if client.UserID == userID { + select { + case client.send <- data: + default: + go func(c *Client) { h.unregister <- c }(client) + } + } + } +} + // RefreshUserServers loads the server membership for a user from the database // and caches it. Call when a client connects or when server membership changes. func (h *Hub) RefreshUserServers(userID string) { diff --git a/web/src/components/VideoGrid.tsx b/web/src/components/VideoGrid.tsx new file mode 100644 index 0000000..59501ba --- /dev/null +++ b/web/src/components/VideoGrid.tsx @@ -0,0 +1,106 @@ +import { useEffect, useRef } from 'react'; +import { Track } from 'livekit-client'; +import type { Participant, TrackPublication, Room } from 'livekit-client'; +import { useVoiceStore } from '../stores/voice.ts'; + +function VideoTile({ participant, isLocal, room }: { participant: Participant; isLocal: boolean; room: Room }) { + const videoRef = useRef(null); + + useEffect(() => { + const el = videoRef.current; + if (!el) return; + + let pub: TrackPublication | undefined; + + const attachTrack = () => { + pub = participant.getTrackPublication(Track.Source.Camera); + + if (pub?.track && el) { + pub.track.attach(el); + el.muted = isLocal; + el.play().catch(() => {}); + } + }; + + const detachTrack = () => { + if (pub?.track && el) { + pub.track.detach(el); + } + }; + + attachTrack(); + + // Listen for track publications changing on this participant via room events + const handlePublished = (pub: TrackPublication) => { + pub.track?.attach(el); + el.play().catch(() => {}); + }; + room.on('trackPublished' as any, handlePublished); + + return () => { + detachTrack(); + room.off('trackPublished' as any, handlePublished); + }; + }, [participant, isLocal, room]); + + const username = participant.name || participant.identity; + + return ( +
+
+ ); +} + +export function VideoGrid() { + const room = useVoiceStore((s) => s._room); + const isVideoOn = useVoiceStore((s) => s.isVideoOn); + + if (!room || !isVideoOn) return null; + + const participants: Participant[] = [ + room.localParticipant, + ...Array.from(room.remoteParticipants.values()), + ]; + + const hasCam = participants.filter((p) => { + const pub = p.getTrackPublication(Track.Source.Camera); + return pub && !pub.isMuted; + }); + + if (hasCam.length === 0) return null; + + return ( +
+
+ ── video [{hasCam.length}] ── +
+
+ {participants.map((p) => ( + + ))} +
+
+ ); +} diff --git a/web/src/components/VoiceControls.tsx b/web/src/components/VoiceControls.tsx index b07dfb6..eb45615 100644 --- a/web/src/components/VoiceControls.tsx +++ b/web/src/components/VoiceControls.tsx @@ -5,10 +5,12 @@ export function VoiceControls() { const isDeafened = useVoiceStore((state) => state.isDeafened); const isVideoOn = useVoiceStore((state) => state.isVideoOn); const isScreenSharing = useVoiceStore((state) => state.isScreenSharing); + const noiseSuppression = useVoiceStore((state) => state.noiseSuppression); const toggleMute = useVoiceStore((state) => state.toggleMute); const toggleDeafen = useVoiceStore((state) => state.toggleDeafen); const toggleVideo = useVoiceStore((state) => state.toggleVideo); const toggleScreenShare = useVoiceStore((state) => state.toggleScreenShare); + const toggleNoiseSuppression = useVoiceStore((state) => state.toggleNoiseSuppression); const leaveVoice = useVoiceStore((state) => state.leaveVoice); return ( @@ -57,6 +59,17 @@ export function VoiceControls() { > [share] +
)} + {/* Incoming whispers */} + {whispers.length > 0 && ( +
+
+ ── whispers ── +
+ {whispers.map((w) => ( +
+ 🔊 + {w.fromUsername} + {w.message || 'whispered'} + +
+ ))} +
+ )} + {/* Participants */}
@@ -46,37 +77,52 @@ export function VoicePanel() { {participants.length === 0 && (
[empty channel]
)} - {participants.map((p) => ( -
- { + const isMe = p.identity === currentUserId; + return ( +
- {p.isMuted ? '○' : '●'} - - - {p.username} - - {p.isMuted && ( - [muted] - )} - {p.isSpeaking && ( - [speaking] - )} -
- ))} + + {p.isMuted ? '○' : '●'} + + + {p.username} + + {p.isMuted && ( + [muted] + )} + {p.isSpeaking && ( + [speaking] + )} + {!isMe && ( + + )} +
+ ); + })}
+ {/* Video grid */} + + {/* Controls */}
diff --git a/web/src/stores/voice.ts b/web/src/stores/voice.ts index 22eb6be..23e5ca0 100644 --- a/web/src/stores/voice.ts +++ b/web/src/stores/voice.ts @@ -11,6 +11,7 @@ import { } from 'livekit-client'; import { api } from '../lib/api.ts'; import { useWebSocketStore } from './ws.ts'; +import { useAuthStore } from './auth.ts'; export interface VoiceParticipant { identity: string; @@ -19,6 +20,13 @@ export interface VoiceParticipant { isSpeaking: boolean; } +export interface WhisperNotification { + fromUserId: string; + fromUsername: string; + message: string; + timestamp: number; +} + interface VoiceTokenResponse { token: string; livekit_url: string; @@ -32,16 +40,22 @@ interface VoiceState { isDeafened: boolean; isVideoOn: boolean; isScreenSharing: boolean; + noiseSuppression: boolean; participants: VoiceParticipant[]; error: string | null; _room: Room | null; isJoining: boolean; + whispers: WhisperNotification[]; joinVoice: (channelId: string, channelName: string) => Promise; leaveVoice: () => Promise; toggleMute: () => void; toggleDeafen: () => void; toggleVideo: () => void; toggleScreenShare: () => void; + toggleNoiseSuppression: () => void; + _addWhisper: (w: WhisperNotification) => void; + whisperTo: (targetUserId: string, targetUsername: string, message?: string) => void; + dismissWhisper: (timestamp: number) => void; } function participantToVoice(p: Participant): VoiceParticipant { @@ -62,10 +76,12 @@ export const useVoiceStore = create((set, get) => ({ isDeafened: false, isVideoOn: false, isScreenSharing: false, + noiseSuppression: true, participants: [], error: null, _room: null, isJoining: false, + whispers: [], joinVoice: async (channelId, channelName) => { if (get().isJoining) { @@ -290,4 +306,50 @@ export const useVoiceStore = create((set, get) => ({ await room.localParticipant.setScreenShareEnabled(newState); set({ isScreenSharing: newState }); }, + + toggleNoiseSuppression: async () => { + const room = get()._room; + if (!room) return; + + const newState = !get().noiseSuppression; + set({ noiseSuppression: newState }); + + // Apply noise suppression + echo cancellation to local audio track + try { + const audioTrack = room.localParticipant.getTrackPublication(Track.Source.Microphone); + if (audioTrack?.track?.mediaStreamTrack) { + await audioTrack.track.mediaStreamTrack.applyConstraints({ + echoCancellation: newState, + noiseSuppression: newState, + autoGainControl: newState, + } as MediaTrackConstraints); + } + } catch (err) { + console.warn('[voice] failed to update audio processing:', err); + } + }, + + _addWhisper: (w) => { + set((state) => ({ whispers: [...state.whispers, w] })); + }, + + whisperTo: (targetUserId, targetUsername, message) => { + const wsSend = useWebSocketStore.getState().send; + const me = useAuthStore.getState().user; + wsSend({ + type: 'VOICE_WHISPER', + payload: { + target_user_id: targetUserId, + target_username: targetUsername, + from_username: me?.username || 'unknown', + message: message || '', + }, + }); + }, + + dismissWhisper: (timestamp) => { + set((state) => ({ + whispers: state.whispers.filter((w) => w.timestamp !== timestamp), + })); + }, })); diff --git a/web/src/stores/ws.ts b/web/src/stores/ws.ts index ce9c7a3..19386de 100644 --- a/web/src/stores/ws.ts +++ b/web/src/stores/ws.ts @@ -4,6 +4,7 @@ import { useChannelStore } from './channel.ts'; import { useServerStore } from './server.ts'; import { usePresenceStore } from './presence.ts'; import { useTypingStore } from './typing.ts'; +import { useVoiceStore } from './voice.ts'; import type { Message } from './message.ts'; import type { Channel } from './channel.ts'; import type { Server } from './server.ts'; @@ -176,6 +177,17 @@ export const useWebSocketStore = create((set, get) => ({ } break; } + case 'VOICE_WHISPER': { + // Forwarded from backend — only intended recipient gets this + const { from_user_id, from_username, message } = data.payload as Record; + useVoiceStore.getState()._addWhisper({ + fromUserId: from_user_id || '', + fromUsername: from_username || 'unknown', + message: message || '', + timestamp: Date.now(), + }); + break; + } default: break; }