From 239640975f37e6dc785d8f6091f0dc26f0d117f3 Mon Sep 17 00:00:00 2001 From: hobokenchicken Date: Mon, 6 Jul 2026 13:08:59 +0000 Subject: [PATCH] feat(webrtc): add background blur and virtual background options --- web/package-lock.json | 37 +++++++++++++++++ web/package.json | 1 + web/src/components/DeviceSettingsModal.tsx | 23 +++++++++++ web/src/stores/voice.ts | 47 ++++++++++++++++++++++ 4 files changed, 108 insertions(+) diff --git a/web/package-lock.json b/web/package-lock.json index a3deaf1..fe35c60 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "@livekit/components-react": "^2.9.21", + "@livekit/track-processors": "^0.7.2", "livekit-client": "^2.20.0", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -857,6 +858,25 @@ "@bufbuild/protobuf": "^1.10.0" } }, + "node_modules/@livekit/track-processors": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@livekit/track-processors/-/track-processors-0.7.2.tgz", + "integrity": "sha512-lzARBKTbBwqycdR/SwTu6//N0l20BzfDd7grxCXl07676SwRApNtZAK1GJjL1m3dCM3KBqH1aVxjMpNcbOw5uQ==", + "license": "Apache-2.0", + "dependencies": { + "@mediapipe/tasks-vision": "0.10.14" + }, + "peerDependencies": { + "@types/dom-mediacapture-transform": "^0.1.9", + "livekit-client": "^1.12.0 || ^2.1.0" + } + }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.14.tgz", + "integrity": "sha512-vOifgZhkndgybdvoRITzRkIueWWSiCKuEUXXK6Q4FaJsFvRJuwgg++vqFUMlL0Uox62U5aEXFhHxlhV7Ja5e3Q==", + "license": "Apache-2.0" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1322,6 +1342,23 @@ "license": "MIT", "peer": true }, + "node_modules/@types/dom-mediacapture-transform": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz", + "integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/dom-webcodecs": "*" + } + }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.18", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.18.tgz", + "integrity": "sha512-vAvE8C9DGWR+tkb19xyjk1TSUlJ7RUzzp4a9Anu7mwBT+fpyePWK1UxmH14tMO5zHmrnrRIMg5NutnnDztLxgg==", + "license": "MIT", + "peer": true + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", diff --git a/web/package.json b/web/package.json index 5c04deb..96e053d 100644 --- a/web/package.json +++ b/web/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@livekit/components-react": "^2.9.21", + "@livekit/track-processors": "^0.7.2", "livekit-client": "^2.20.0", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/web/src/components/DeviceSettingsModal.tsx b/web/src/components/DeviceSettingsModal.tsx index c9d0496..d77ec4a 100644 --- a/web/src/components/DeviceSettingsModal.tsx +++ b/web/src/components/DeviceSettingsModal.tsx @@ -102,6 +102,29 @@ export function DeviceSettingsModal({ onClose }: Props) { ))} + +
+ + +
diff --git a/web/src/stores/voice.ts b/web/src/stores/voice.ts index eeb092f..19de02b 100644 --- a/web/src/stores/voice.ts +++ b/web/src/stores/voice.ts @@ -11,6 +11,7 @@ import { } from 'livekit-client'; import { api } from '../lib/api.ts'; import { useWebSocketStore } from './ws.ts'; +import { BackgroundProcessor } from '@livekit/track-processors'; export interface VoiceParticipant { @@ -42,6 +43,8 @@ interface VoiceState { isVideoOn: boolean; isScreenSharing: boolean; noiseSuppression: boolean; + bgMode: 'disabled' | 'background-blur' | 'virtual-background'; + _bgProcessor: any; participants: VoiceParticipant[]; error: string | null; _room: Room | null; @@ -54,6 +57,7 @@ interface VoiceState { toggleVideo: () => Promise; toggleScreenShare: () => Promise; toggleNoiseSuppression: () => void; + setBgMode: (mode: 'disabled' | 'background-blur' | 'virtual-background', imagePath?: string) => Promise; _addWhisper: (w: WhisperNotification) => void; whisperTo: (targetUserId: string, targetUsername: string, message?: string) => void; dismissWhisper: (timestamp: number) => void; @@ -80,6 +84,8 @@ export const useVoiceStore = create((set, get) => ({ isVideoOn: false, isScreenSharing: false, noiseSuppression: true, + bgMode: 'disabled', + _bgProcessor: null, participants: [], error: null, _room: null, @@ -306,6 +312,47 @@ export const useVoiceStore = create((set, get) => ({ const newState = !get().isVideoOn; await room.localParticipant.setCameraEnabled(newState); set({ isVideoOn: newState }); + + if (newState && get().bgMode !== 'disabled') { + const mode = get().bgMode; + // re-trigger setBgMode to re-attach processor to the new track + await get().setBgMode(mode); + } + }, + + setBgMode: async (mode: 'disabled' | 'background-blur' | 'virtual-background', imagePath?: string) => { + const room = get()._room; + if (!room) return; + + let processor = get()._bgProcessor; + if (!processor && mode !== 'disabled') { + try { + processor = BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 }); + set({ _bgProcessor: processor }); + } catch (err) { + console.warn('Failed to initialize BackgroundProcessor:', err); + return; + } + } + + if (processor) { + if (mode === 'disabled') { + await processor.switchTo({ mode: 'disabled' }); + } else if (mode === 'background-blur') { + await processor.switchTo({ mode: 'background-blur', blurRadius: 10 }); + } else if (mode === 'virtual-background') { + await processor.switchTo({ mode: 'virtual-background', imagePath: imagePath || 'https://images.unsplash.com/photo-1579546929518-9e396f3cc809' }); + } + + // attach to camera track if it's running + const pub = room.localParticipant.getTrackPublication(Track.Source.Camera); + const track = pub?.track as any; + if (track && track.setProcessor) { + await track.setProcessor(processor); + } + } + + set({ bgMode: mode }); }, toggleScreenShare: async () => {