Add background filters to settings menu (#413)

2025-04-15 20:26:22 +02:00 · 2025-04-15 20:26:22 +02:00 · efac802d7b
commit efac802d7b
parent 8b2ee6c324
9 changed files with 248 additions and 36 deletions
--- a/.gitattributes
+++ b/.gitattributes
@ -0,0 +1 @@
+public/background-images/*.jpg filter=lfs diff=lfs merge=lfs -text
--- a/lib/CameraSettings.tsx
+++ b/lib/CameraSettings.tsx
@ -0,0 +1,170 @@
+import React from 'react';
+import {
+  MediaDeviceMenu,
+  TrackToggle,
+  useLocalParticipant,
+  VideoTrack,
+} from '@livekit/components-react';
+import { BackgroundBlur, VirtualBackground } from '@livekit/track-processors';
+import { isLocalTrack, LocalTrackPublication, Track } from 'livekit-client';
+import Desk from '../public/background-images/samantha-gades-BlIhVfXbi9s-unsplash.jpg';
+import Nature from '../public/background-images/ali-kazal-tbw_KQE3Cbg-unsplash.jpg';
+
+// Background image paths
+const BACKGROUND_IMAGES = [
+  { name: 'Desk', path: Desk },
+  { name: 'Nature', path: Nature },
+];
+
+// Background options
+type BackgroundType = 'none' | 'blur' | 'image';
+
+export function CameraSettings() {
+  const { cameraTrack, localParticipant } = useLocalParticipant();
+  const [backgroundType, setBackgroundType] = React.useState<BackgroundType>(
+    (cameraTrack as LocalTrackPublication)?.track?.getProcessor()?.name === 'background-blur'
+      ? 'blur'
+      : (cameraTrack as LocalTrackPublication)?.track?.getProcessor()?.name === 'virtual-background'
+      ? 'image'
+      : 'none',
+  );
+
+  const [virtualBackgroundImagePath, setVirtualBackgroundImagePath] = React.useState<string | null>(
+    null,
+  );
+
+  const camTrackRef = React.useMemo(() => {
+    return { participant: localParticipant, publication: cameraTrack, source: Track.Source.Camera };
+  }, [localParticipant, cameraTrack]);
+
+  const selectBackground = (type: BackgroundType, imagePath?: string) => {
+    setBackgroundType(type);
+    if (type === 'image' && imagePath) {
+      setVirtualBackgroundImagePath(imagePath);
+    } else if (type !== 'image') {
+      setVirtualBackgroundImagePath(null);
+    }
+  };
+
+  React.useEffect(() => {
+    if (isLocalTrack(cameraTrack?.track)) {
+      if (backgroundType === 'blur') {
+        cameraTrack.track?.setProcessor(BackgroundBlur());
+      } else if (backgroundType === 'image' && virtualBackgroundImagePath) {
+        cameraTrack.track?.setProcessor(VirtualBackground(virtualBackgroundImagePath));
+      } else {
+        cameraTrack.track?.stopProcessor();
+      }
+    }
+  }, [cameraTrack, backgroundType, virtualBackgroundImagePath]);
+
+  return (
+    <div style={{ display: 'flex', flexDirection: 'column', gap: '10px' }}>
+      <VideoTrack
+        style={{
+          maxHeight: '280px',
+          objectFit: 'contain',
+          objectPosition: 'right',
+          transform: 'scaleX(-1)',
+        }}
+        trackRef={camTrackRef}
+      />
+
+      <section className="lk-button-group">
+        <TrackToggle source={Track.Source.Camera}>Camera</TrackToggle>
+        <div className="lk-button-group-menu">
+          <MediaDeviceMenu kind="videoinput" />
+        </div>
+      </section>
+
+      <div style={{ marginTop: '10px' }}>
+        <div style={{ marginBottom: '8px' }}>Background Effects</div>
+        <div style={{ display: 'flex', gap: '10px', flexWrap: 'wrap' }}>
+          <button
+            onClick={() => selectBackground('none')}
+            className="lk-button"
+            aria-pressed={backgroundType === 'none'}
+            style={{
+              border: backgroundType === 'none' ? '2px solid #0090ff' : '1px solid #d1d1d1',
+              minWidth: '80px',
+            }}
+          >
+            None
+          </button>
+
+          <button
+            onClick={() => selectBackground('blur')}
+            className="lk-button"
+            aria-pressed={backgroundType === 'blur'}
+            style={{
+              border: backgroundType === 'blur' ? '2px solid #0090ff' : '1px solid #d1d1d1',
+              minWidth: '80px',
+              backgroundColor: '#f0f0f0',
+              position: 'relative',
+              overflow: 'hidden',
+              height: '60px',
+            }}
+          >
+            <div
+              style={{
+                position: 'absolute',
+                top: 0,
+                left: 0,
+                right: 0,
+                bottom: 0,
+                backgroundColor: '#e0e0e0',
+                filter: 'blur(8px)',
+                zIndex: 0,
+              }}
+            />
+            <span
+              style={{
+                position: 'relative',
+                zIndex: 1,
+                backgroundColor: 'rgba(0,0,0,0.6)',
+                padding: '2px 5px',
+                borderRadius: '4px',
+                fontSize: '12px',
+              }}
+            >
+              Blur
+            </span>
+          </button>
+
+          {BACKGROUND_IMAGES.map((image) => (
+            <button
+              key={image.path.src}
+              onClick={() => selectBackground('image', image.path.src)}
+              className="lk-button"
+              aria-pressed={
+                backgroundType === 'image' && virtualBackgroundImagePath === image.path.src
+              }
+              style={{
+                backgroundImage: `url(${image.path.src})`,
+                backgroundSize: 'cover',
+                backgroundPosition: 'center',
+                width: '80px',
+                height: '60px',
+                border:
+                  backgroundType === 'image' && virtualBackgroundImagePath === image.path.src
+                    ? '2px solid #0090ff'
+                    : '1px solid #d1d1d1',
+              }}
+            >
+              <span
+                style={{
+                  backgroundColor: 'rgba(0,0,0,0.6)',
+                  padding: '2px 5px',
+                  borderRadius: '4px',
+                  fontSize: '12px',
+                }}
+              >
+                {image.name}
+              </span>
+            </button>
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+}
--- a/lib/MicrophoneSettings.tsx
+++ b/lib/MicrophoneSettings.tsx
@ -0,0 +1,42 @@
+import React from 'react';
+import { useKrispNoiseFilter } from '@livekit/components-react/krisp';
+import { TrackToggle } from '@livekit/components-react';
+import { MediaDeviceMenu } from '@livekit/components-react';
+import { Track } from 'livekit-client';
+
+export function MicrophoneSettings() {
+  const { isNoiseFilterEnabled, setNoiseFilterEnabled, isNoiseFilterPending } =
+    useKrispNoiseFilter();
+
+  React.useEffect(() => {
+    // enable Krisp by default
+    setNoiseFilterEnabled(true);
+  }, []);
+  return (
+    <div
+      style={{
+        display: 'flex',
+        flexDirection: 'row',
+        gap: '10px',
+        alignItems: 'center',
+        justifyContent: 'space-between',
+      }}
+    >
+      <section className="lk-button-group">
+        <TrackToggle source={Track.Source.Microphone}>Microphone</TrackToggle>
+        <div className="lk-button-group-menu">
+          <MediaDeviceMenu kind="audioinput" />
+        </div>
+      </section>
+
+      <button
+        className="lk-button"
+        onClick={() => setNoiseFilterEnabled(!isNoiseFilterEnabled)}
+        disabled={isNoiseFilterPending}
+        aria-pressed={isNoiseFilterEnabled}
+      >
+        {isNoiseFilterEnabled ? 'Disable' : 'Enable'} Enhanced Noise Cancellation
+      </button>
+    </div>
+  );
+}
--- a/lib/SettingsMenu.tsx
+++ b/lib/SettingsMenu.tsx
@ -8,9 +8,9 @@ import {
  useRoomContext,
  useIsRecording,
 } from '@livekit/components-react';
-import { useKrispNoiseFilter } from '@livekit/components-react/krisp';
 import styles from '../styles/SettingsMenu.module.css';
-
+import { CameraSettings } from './CameraSettings';
+import { MicrophoneSettings } from './MicrophoneSettings';
 /**
 * @alpha
 */
@ -27,7 +27,6 @@ export function SettingsMenu(props: SettingsMenuProps) {
  const settings = React.useMemo(() => {
    return {
      media: { camera: true, microphone: true, label: 'Media Devices', speaker: true },
-      effects: { label: 'Effects' },
      recording: recordingEndpoint ? { label: 'Recording' } : undefined,
    };
  }, []);
@ -38,14 +37,6 @@ export function SettingsMenu(props: SettingsMenuProps) {
  );
  const [activeTab, setActiveTab] = React.useState(tabs[0]);

-  const { isNoiseFilterEnabled, setNoiseFilterEnabled, isNoiseFilterPending } =
-    useKrispNoiseFilter();
-
-  React.useEffect(() => {
-    // enable Krisp by default
-    setNoiseFilterEnabled(true);
-  }, []);
-
  const isRecording = useIsRecording();
  const [initialRecStatus, setInitialRecStatus] = React.useState(isRecording);
  const [processingRecRequest, setProcessingRecRequest] = React.useState(false);
@ -108,22 +99,16 @@ export function SettingsMenu(props: SettingsMenuProps) {
            {settings.media && settings.media.camera && (
              <>
                <h3>Camera</h3>
-                <section className="lk-button-group">
-                  <TrackToggle source={Track.Source.Camera}>Camera</TrackToggle>
-                  <div className="lk-button-group-menu">
-                    <MediaDeviceMenu kind="videoinput" />
-                  </div>
+                <section>
+                  <CameraSettings />
                </section>
              </>
            )}
            {settings.media && settings.media.microphone && (
              <>
                <h3>Microphone</h3>
-                <section className="lk-button-group">
-                  <TrackToggle source={Track.Source.Microphone}>Microphone</TrackToggle>
-                  <div className="lk-button-group-menu">
-                    <MediaDeviceMenu kind="audioinput" />
-                  </div>
+                <section>
+                  <MicrophoneSettings />
                </section>
              </>
            )}
@ -140,21 +125,6 @@ export function SettingsMenu(props: SettingsMenuProps) {
            )}
          </>
        )}
-        {activeTab === 'effects' && (
-          <>
-            <h3>Audio</h3>
-            <section>
-              <label htmlFor="noise-filter"> Enhanced Noise Cancellation</label>
-              <input
-                type="checkbox"
-                id="noise-filter"
-                onChange={(ev) => setNoiseFilterEnabled(ev.target.checked)}
-                checked={isNoiseFilterEnabled}
-                disabled={isNoiseFilterPending}
-              ></input>
-            </section>
-          </>
-        )}
        {activeTab === 'recording' && (
          <>
            <h3>Record Meeting</h3>
--- a/next.config.js
+++ b/next.config.js
@ -2,6 +2,9 @@
 const nextConfig = {
  reactStrictMode: false,
  productionBrowserSourceMaps: true,
+  images: {
+    formats: ['image/webp'],
+  },
  webpack: (config, { buildId, dev, isServer, defaultLoaders, nextRuntime, webpack }) => {
    // Important: return the modified config
    config.module.rules.push({
@ -9,6 +12,7 @@ const nextConfig = {
      enforce: 'pre',
      use: ['source-map-loader'],
    });
+
    return config;
  },
 };
--- a/package.json
+++ b/package.json
@ -13,6 +13,7 @@
    "@livekit/components-react": "2.9.2",
    "@livekit/components-styles": "1.1.5",
    "@livekit/krisp-noise-filter": "0.2.16",
+    "@livekit/track-processors": "^0.5.2",
    "livekit-client": "2.11.2",
    "livekit-server-sdk": "2.12.0",
    "next": "15.2.4",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -20,6 +20,9 @@ importers:
      '@livekit/krisp-noise-filter':
        specifier: 0.2.16
        version: 0.2.16(livekit-client@2.11.2)
+      '@livekit/track-processors':
+        specifier: ^0.5.2
+        version: 0.5.2(livekit-client@2.11.2)
      livekit-client:
        specifier: 2.11.2
        version: 2.11.2
@ -315,6 +318,14 @@ packages:
  '@livekit/protocol@1.36.1':
    resolution: {integrity: sha512-nN3QnITAQ5yXk7UKfotH7CRWIlEozNWeKVyFJ0/+dtSzvWP/ib+10l1DDnRYi3A1yICJOGAKFgJ5d6kmi1HCUA==}

+  '@livekit/track-processors@0.5.2':
+    resolution: {integrity: sha512-hnAD8PyCE3OPOohFYkPCEGGLeY4/oUa1gu3VCJ4sfelvrbZUKi0vEbUhbnnNcP9V7YiYzplktUz2w6EtYVPRLA==}
+    peerDependencies:
+      livekit-client: ^1.12.0 || ^2.1.0
+
+  '@mediapipe/tasks-vision@0.10.22-rc.20250304':
+    resolution: {integrity: sha512-dElxVXMFGthshfIj+qAVm8KE2jmNo2p8oXFib8WzEjb7GNaX/ClWBc8UJfoSZwjEMVrdHJ4YUfa7P3ifl6MIWw==}
+
  '@next/env@15.2.4':
    resolution: {integrity: sha512-+SFtMgoiYP3WoSswuNmxJOCwi06TdWE733D+WPjpXIe4LXGULwEaofiiAy6kbS0+XjM5xF5n3lKuBwN2SnqD9g==}

@ -2043,6 +2054,13 @@ snapshots:
    dependencies:
      '@bufbuild/protobuf': 1.10.0

+  '@livekit/track-processors@0.5.2(livekit-client@2.11.2)':
+    dependencies:
+      '@mediapipe/tasks-vision': 0.10.22-rc.20250304
+      livekit-client: 2.11.2
+
+  '@mediapipe/tasks-vision@0.10.22-rc.20250304': {}
+
  '@next/env@15.2.4': {}

  '@next/eslint-plugin-next@15.2.4':
--- a/public/background-images/ali-kazal-tbw_KQE3Cbg-unsplash.jpg
+++ b/public/background-images/ali-kazal-tbw_KQE3Cbg-unsplash.jpg
@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a3c9eb8da1ef3ddf2439428b49c11abd9a765e056600bd4f1d89a5dfc82778a
+size 52339
--- a/public/background-images/samantha-gades-BlIhVfXbi9s-unsplash.jpg
+++ b/public/background-images/samantha-gades-BlIhVfXbi9s-unsplash.jpg
@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bc017736e04acb0188f69cec0aafb88bd6891bfc4a6ff1530665e8dc210dbdf
+size 1273171
				`@ -0,0 +1 @@`
				`public/background-images/*.jpg filter=lfs diff=lfs merge=lfs -text`