Dynamic handling of low-power devices (#450)

app/custom/VideoConferenceClientImpl.tsx

@@ -15,6 +15,7 @@ import { useEffect, useMemo, useState } from 'react';
 import { KeyboardShortcuts } from '@/lib/KeyboardShortcuts';
 import { SettingsMenu } from '@/lib/SettingsMenu';
 import { useSetupE2EE } from '@/lib/useSetupE2EE';
+import { useLowCPUOptimizer } from '@/lib/usePerfomanceOptimiser';
 
 export function VideoConferenceClientImpl(props: {
   liveKitUrl: string;
@@ -76,6 +77,8 @@ export function VideoConferenceClientImpl(props: {
     }
   }, [room, props.liveKitUrl, props.token, connectOptions, e2eeSetupComplete]);
 
+  useLowCPUOptimizer(room);
+
   return (
     <div className="lk-room-container">
       <RoomContext.Provider value={room}>

app/rooms/[roomName]/PageClientImpl.tsx

@@ -1,7 +1,7 @@
 'use client';
 
 import React from 'react';
-import { decodePassphrase, isLowPowerDevice } from '@/lib/client-utils';
+import { decodePassphrase } from '@/lib/client-utils';
 import { DebugMode } from '@/lib/Debug';
 import { KeyboardShortcuts } from '@/lib/KeyboardShortcuts';
 import { RecordingIndicator } from '@/lib/RecordingIndicator';
@@ -28,6 +28,7 @@ import {
 } from 'livekit-client';
 import { useRouter } from 'next/navigation';
 import { useSetupE2EE } from '@/lib/useSetupE2EE';
+import { useLowCPUOptimizer } from '@/lib/usePerfomanceOptimiser';
 
 const CONN_DETAILS_ENDPOINT =
   process.env.NEXT_PUBLIC_CONN_DETAILS_ENDPOINT ?? '/api/connection-details';
@@ -119,20 +120,13 @@ function VideoConferenceComponent(props: {
       red: !e2eeEnabled,
       videoCodec,
     };
-    if (isLowPowerDevice()) {
-      // on lower end devices, publish at a lower resolution, and disable spatial layers
-      // encoding spatial layers adds to CPU overhead
-      videoCaptureDefaults.resolution = VideoPresets.h360;
-      publishDefaults.simulcast = false;
-      publishDefaults.scalabilityMode = 'L1T3';
-    }
     return {
       videoCaptureDefaults: videoCaptureDefaults,
       publishDefaults: publishDefaults,
       audioCaptureDefaults: {
         deviceId: props.userChoices.audioDeviceId ?? undefined,
       },
-      adaptiveStream: { pixelDensity: 'screen' },
+      adaptiveStream: true,
       dynacast: true,
       e2ee: keyProvider && worker && e2eeEnabled ? { keyProvider, worker } : undefined,
     };
@@ -172,6 +166,7 @@ function VideoConferenceComponent(props: {
     room.on(RoomEvent.Disconnected, handleOnLeave);
     room.on(RoomEvent.EncryptionError, handleEncryptionError);
     room.on(RoomEvent.MediaDevicesError, handleError);
+
     if (e2eeSetupComplete) {
       room
         .connect(
@@ -200,6 +195,8 @@ function VideoConferenceComponent(props: {
     };
   }, [e2eeSetupComplete, room, props.connectionDetails, props.userChoices]);
 
+  const lowPowerMode = useLowCPUOptimizer(room);
+
   const router = useRouter();
   const handleOnLeave = React.useCallback(() => router.push('/'), [router]);
   const handleError = React.useCallback((error: Error) => {
@@ -213,6 +210,12 @@ function VideoConferenceComponent(props: {
     );
   }, []);
 
+  React.useEffect(() => {
+    if (lowPowerMode) {
+      console.warn('Low power mode enabled');
+    }
+  }, [lowPowerMode]);
+
   return (
     <div className="lk-room-container">
       <RoomContext.Provider value={room}>

lib/MicrophoneSettings.tsx

@@ -11,7 +11,9 @@ export function MicrophoneSettings() {
       filterOptions: {
         quality: isLowPowerDevice() ? 'low' : 'medium',
         onBufferDrop: () => {
-          console.warn('krisp buffer dropped, either disable or reduce quality');
+          console.warn(
+            'krisp buffer dropped, noise filter versions >= 0.3.2 will automatically disable the filter',
+          );
         },
       },
     },

lib/usePerfomanceOptimiser.ts

@@ -0,0 +1,71 @@
+import {
+  Room,
+  ParticipantEvent,
+  RoomEvent,
+  RemoteTrack,
+  RemoteTrackPublication,
+  VideoQuality,
+  LocalVideoTrack,
+  isVideoTrack,
+} from 'livekit-client';
+import * as React from 'react';
+
+export type LowCPUOptimizerOptions = {
+  reducePublisherVideoQuality: boolean;
+  reduceSubscriberVideoQuality: boolean;
+  disableVideoProcessing: boolean;
+};
+
+const defaultOptions: LowCPUOptimizerOptions = {
+  reducePublisherVideoQuality: true,
+  reduceSubscriberVideoQuality: true,
+  disableVideoProcessing: false,
+} as const;
+
+/**
+ * This hook ensures that on devices with low CPU, the performance is optimised when needed.
+ * This is done by primarily reducing the video quality to low when the CPU is constrained.
+ */
+export function useLowCPUOptimizer(room: Room, options: Partial<LowCPUOptimizerOptions> = {}) {
+  const [lowPowerMode, setLowPowerMode] = React.useState(false);
+  const opts = React.useMemo(() => ({ ...defaultOptions, ...options }), [options]);
+  React.useEffect(() => {
+    const handleCpuConstrained = async (track: LocalVideoTrack) => {
+      setLowPowerMode(true);
+      console.warn('Local track CPU constrained', track);
+      if (opts.reducePublisherVideoQuality) {
+        track.prioritizePerformance();
+      }
+      if (opts.disableVideoProcessing && isVideoTrack(track)) {
+        track.stopProcessor();
+      }
+      if (opts.reduceSubscriberVideoQuality) {
+        room.remoteParticipants.forEach((participant) => {
+          participant.videoTrackPublications.forEach((publication) => {
+            publication.setVideoQuality(VideoQuality.LOW);
+          });
+        });
+      }
+    };
+
+    room.localParticipant.on(ParticipantEvent.LocalTrackCpuConstrained, handleCpuConstrained);
+    return () => {
+      room.localParticipant.off(ParticipantEvent.LocalTrackCpuConstrained, handleCpuConstrained);
+    };
+  }, [room, opts.reducePublisherVideoQuality, opts.reduceSubscriberVideoQuality]);
+
+  React.useEffect(() => {
+    const lowerQuality = (_: RemoteTrack, publication: RemoteTrackPublication) => {
+      publication.setVideoQuality(VideoQuality.LOW);
+    };
+    if (lowPowerMode && opts.reduceSubscriberVideoQuality) {
+      room.on(RoomEvent.TrackSubscribed, lowerQuality);
+    }
+
+    return () => {
+      room.off(RoomEvent.TrackSubscribed, lowerQuality);
+    };
+  }, [lowPowerMode, room, opts.reduceSubscriberVideoQuality]);
+
+  return lowPowerMode;
+}

package.json

@@ -40,5 +40,5 @@
   "engines": {
     "node": ">=18"
   },
-  "packageManager": "pnpm@9.15.9"
+  "packageManager": "pnpm@10.9.0"
 }

pnpm-lock.yaml

@@ -1947,8 +1947,8 @@ packages:
     resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==}
     hasBin: true
 
-  sdp@3.2.0:
-    resolution: {integrity: sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==}
+  sdp@3.2.1:
+    resolution: {integrity: sha512-lwsAIzOPlH8/7IIjjz3K0zYBk7aBVVcvjMwt3M4fLxpjMYyy7i3I97SLHebgn4YBjirkzfp3RvRDWSKsh/+WFw==}
 
   semver@6.3.1:
     resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
@@ -2293,8 +2293,8 @@ packages:
       webpack-cli:
         optional: true
 
-  webrtc-adapter@9.0.1:
-    resolution: {integrity: sha512-1AQO+d4ElfVSXyzNVTOewgGT/tAomwwztX/6e3totvyyzXPvXIIuUUjAmyZGbKBKbZOXauuJooZm3g6IuFuiNQ==}
+  webrtc-adapter@9.0.3:
+    resolution: {integrity: sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ==}
     engines: {node: '>=6.0.0', npm: '>=3.10.0'}
 
   which-boxed-primitive@1.0.2:
@@ -3972,7 +3972,7 @@ snapshots:
       ts-debounce: 4.0.0
       tslib: 2.8.1
       typed-emitter: 2.1.0
-      webrtc-adapter: 9.0.1
+      webrtc-adapter: 9.0.3
 
   livekit-server-sdk@2.13.1:
     dependencies:
@@ -4309,7 +4309,7 @@ snapshots:
 
   sdp-transform@2.15.0: {}
 
-  sdp@3.2.0: {}
+  sdp@3.2.1: {}
 
   semver@6.3.1: {}
 
@@ -4708,9 +4708,9 @@ snapshots:
       - esbuild
       - uglify-js
 
-  webrtc-adapter@9.0.1:
+  webrtc-adapter@9.0.3:
     dependencies:
-      sdp: 3.2.0
+      sdp: 3.2.1
 
   which-boxed-primitive@1.0.2:
     dependencies: