From 56d5126acb1891e582aaf79582c0b8e1704f8b1e Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Mon, 2 Mar 2026 17:15:29 +0100 Subject: [PATCH 1/9] frontend: update @livekit/track-processors dependency to version 0.7.2 --- meet-ce/frontend/package.json | 2 +- pnpm-lock.yaml | 108 ++++++++++++++++++++++++++++++---- 2 files changed, 96 insertions(+), 14 deletions(-) diff --git a/meet-ce/frontend/package.json b/meet-ce/frontend/package.json index 2f732ab4..51b5e088 100644 --- a/meet-ce/frontend/package.json +++ b/meet-ce/frontend/package.json @@ -28,7 +28,7 @@ "@angular/platform-browser": "20.3.15", "@angular/platform-browser-dynamic": "20.3.15", "@angular/router": "20.3.15", - "@livekit/track-processors": "0.7.0", + "@livekit/track-processors": "0.7.2", "@openvidu-meet/shared-components": "workspace:*", "@openvidu-meet/typings": "workspace:*", "autolinker": "4.1.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 76bdc1bf..c47e7ae1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -268,8 +268,8 @@ importers: specifier: 20.3.15 version: 20.3.15(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(rxjs@7.8.2) '@livekit/track-processors': - specifier: 0.7.0 - version: 0.7.0(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.16.1(@types/dom-mediacapture-record@1.0.22)) + specifier: 0.7.2 + version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.16.1(@types/dom-mediacapture-record@1.0.22)) '@openvidu-meet/shared-components': specifier: workspace:* version: link:projects/shared-meet-components @@ -306,7 +306,7 @@ importers: devDependencies: '@angular-builders/custom-webpack': specifier: 20.0.0 - version: 20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.5.1)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) + version: 20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) '@angular-devkit/build-angular': specifier: 20.3.13 version: 20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(tsx@4.20.6)(typescript@5.9.2) @@ -703,8 +703,8 @@ importers: specifier: 20.3.15 version: 20.3.15(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(rxjs@7.8.2) '@livekit/track-processors': - specifier: 0.7.0 - version: 0.7.0(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.16.1(@types/dom-mediacapture-record@1.0.22)) + specifier: 0.7.2 + version: 0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.16.1(@types/dom-mediacapture-record@1.0.22)) '@openvidu-meet/shared-components': specifier: workspace:* version: link:../../meet-ce/frontend/projects/shared-meet-components @@ -741,7 +741,7 @@ importers: devDependencies: '@angular-builders/custom-webpack': specifier: 20.0.0 - version: 20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.43.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) + version: 20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.5.1)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) '@angular-devkit/build-angular': specifier: 20.3.13 version: 20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(tsx@4.20.6)(typescript@5.9.2) @@ -2826,6 +2826,12 @@ packages: '@types/dom-mediacapture-transform': ^0.1.9 livekit-client: 2.16.1 + '@livekit/track-processors@0.7.2': + resolution: {integrity: sha512-lzARBKTbBwqycdR/SwTu6//N0l20BzfDd7grxCXl07676SwRApNtZAK1GJjL1m3dCM3KBqH1aVxjMpNcbOw5uQ==} + peerDependencies: + '@types/dom-mediacapture-transform': ^0.1.9 + livekit-client: 2.16.1 + '@lmdb/lmdb-darwin-arm64@3.4.2': resolution: {integrity: sha512-NK80WwDoODyPaSazKbzd3NEJ3ygePrkERilZshxBViBARNz21rmediktGHExoj9n5t9+ChlgLlxecdFKLCuCKg==} cpu: [arm64] @@ -9740,13 +9746,13 @@ snapshots: - chokidar - typescript - '@angular-builders/custom-webpack@20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.43.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2)': + '@angular-builders/custom-webpack@20.0.0(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2)': dependencies: '@angular-builders/common': 4.0.0(@types/node@22.18.13)(chokidar@4.0.3)(typescript@5.9.2) '@angular-devkit/architect': 0.2003.13(chokidar@4.0.3) '@angular-devkit/build-angular': 20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(browser-sync@3.0.4)(chokidar@4.0.3)(jest-environment-jsdom@29.7.0)(jest@29.7.0(@types/node@22.18.13)(ts-node@10.9.2(@types/node@22.18.13)(typescript@5.9.2)))(jiti@1.21.7)(karma@6.4.4)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(tsx@4.20.6)(typescript@5.9.2) '@angular-devkit/core': 20.3.13(chokidar@4.0.3) - '@angular/build': 20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(chokidar@4.0.3)(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.43.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) + '@angular/build': 20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(chokidar@4.0.3)(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2) '@angular/compiler-cli': 20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2) lodash: 4.17.21 webpack-merge: 6.0.1 @@ -10114,7 +10120,7 @@ snapshots: '@babel/helper-annotate-as-pure': 7.27.3 '@babel/helper-split-export-declaration': 7.24.7 '@inquirer/confirm': 5.1.14(@types/node@22.18.13) - '@vitejs/plugin-basic-ssl': 2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.43.1)(tsx@4.20.6)) + '@vitejs/plugin-basic-ssl': 2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6)) beasties: 0.3.5 browserslist: 4.28.1 esbuild: 0.25.9 @@ -10157,6 +10163,59 @@ snapshots: - tsx - yaml + '@angular/build@20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(chokidar@4.0.3)(jiti@1.21.7)(karma@6.4.4)(less@4.4.0)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2)': + dependencies: + '@ampproject/remapping': 2.3.0 + '@angular-devkit/architect': 0.2003.13(chokidar@4.0.3) + '@angular/compiler': 20.3.15 + '@angular/compiler-cli': 20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2) + '@babel/core': 7.28.3 + '@babel/helper-annotate-as-pure': 7.27.3 + '@babel/helper-split-export-declaration': 7.24.7 + '@inquirer/confirm': 5.1.14(@types/node@22.18.13) + '@vitejs/plugin-basic-ssl': 2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6)) + beasties: 0.3.5 + browserslist: 4.28.1 + esbuild: 0.25.9 + https-proxy-agent: 7.0.6 + istanbul-lib-instrument: 6.0.3 + jsonc-parser: 3.3.1 + listr2: 9.0.1 + magic-string: 0.30.17 + mrmime: 2.0.1 + parse5-html-rewriting-stream: 8.0.0 + picomatch: 4.0.3 + piscina: 5.1.3 + rollup: 4.52.3 + sass: 1.90.0 + semver: 7.7.2 + source-map-support: 0.5.21 + tinyglobby: 0.2.14 + tslib: 2.8.1 + typescript: 5.9.2 + vite: 7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6) + watchpack: 2.4.4 + optionalDependencies: + '@angular/core': 20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1) + '@angular/platform-browser': 20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)) + karma: 6.4.4 + less: 4.4.0 + lmdb: 3.4.2 + ng-packagr: 20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2) + postcss: 8.5.6 + transitivePeerDependencies: + - '@types/node' + - chokidar + - jiti + - lightningcss + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + '@angular/build@20.3.13(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(@angular/compiler@20.3.15)(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.15(@angular/animations@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@angular/common@20.3.15(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.15(@angular/compiler@20.3.15)(rxjs@7.8.2)(zone.js@0.15.1)))(@types/node@22.18.13)(chokidar@4.0.3)(jiti@1.21.7)(karma@6.4.4)(less@4.5.1)(ng-packagr@20.3.2(@angular/compiler-cli@20.3.15(@angular/compiler@20.3.15)(typescript@5.9.2))(tslib@2.8.1)(typescript@5.9.2))(postcss@8.5.6)(terser@5.44.1)(tslib@2.8.1)(tsx@4.20.6)(typescript@5.9.2)': dependencies: '@ampproject/remapping': 2.3.0 @@ -12737,6 +12796,12 @@ snapshots: '@types/dom-mediacapture-transform': 0.1.11 livekit-client: 2.16.1(@types/dom-mediacapture-record@1.0.22) + '@livekit/track-processors@0.7.2(@types/dom-mediacapture-transform@0.1.11)(livekit-client@2.16.1(@types/dom-mediacapture-record@1.0.22))': + dependencies: + '@mediapipe/tasks-vision': 0.10.14 + '@types/dom-mediacapture-transform': 0.1.11 + livekit-client: 2.16.1(@types/dom-mediacapture-record@1.0.22) + '@lmdb/lmdb-darwin-arm64@3.4.2': optional: true @@ -14302,9 +14367,9 @@ snapshots: '@ungap/structured-clone@1.3.0': {} - '@vitejs/plugin-basic-ssl@2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.43.1)(tsx@4.20.6))': + '@vitejs/plugin-basic-ssl@2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6))': dependencies: - vite: 7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.43.1)(tsx@4.20.6) + vite: 7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6) '@vitejs/plugin-basic-ssl@2.1.0(vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.5.1)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6))': dependencies: @@ -20353,7 +20418,7 @@ snapshots: - encoding - supports-color - terser-webpack-plugin@5.3.16(esbuild@0.25.9)(webpack@5.101.2(esbuild@0.25.9)): + terser-webpack-plugin@5.3.16(esbuild@0.25.9)(webpack@5.101.2): dependencies: '@jridgewell/trace-mapping': 0.3.31 jest-worker: 27.5.1 @@ -20841,6 +20906,23 @@ snapshots: terser: 5.43.1 tsx: 4.20.6 + vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.4.0)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6): + dependencies: + esbuild: 0.25.12 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.52.3 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 22.18.13 + fsevents: 2.3.3 + jiti: 1.21.7 + less: 4.4.0 + sass: 1.90.0 + terser: 5.44.1 + tsx: 4.20.6 + vite@7.1.11(@types/node@22.18.13)(jiti@1.21.7)(less@4.5.1)(sass@1.90.0)(terser@5.44.1)(tsx@4.20.6): dependencies: esbuild: 0.25.12 @@ -20980,7 +21062,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 4.3.3 tapable: 2.3.0 - terser-webpack-plugin: 5.3.16(esbuild@0.25.9)(webpack@5.101.2(esbuild@0.25.9)) + terser-webpack-plugin: 5.3.16(esbuild@0.25.9)(webpack@5.101.2) watchpack: 2.4.4 webpack-sources: 3.3.3 transitivePeerDependencies: From c808e988202c54a7bd88220aa53a4fb7584c4b83 Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Tue, 3 Mar 2026 18:43:35 +0100 Subject: [PATCH 2/9] backend(ai-assistant): implement AI assistant creation and management - Add OpenAPI components for creating and responding to AI assistant requests. - Implement AI assistant service for managing live captions capability. - Create routes and controllers for AI assistant operations (create and cancel). - Introduce request validation middleware for AI assistant requests. - Update Redis helper to manage AI assistant locks. - Integrate AI assistant cleanup in webhook service. - Enhance LiveKit service to manage agent dispatch for AI assistants. - Update token service to remove unnecessary parameters related to captions. - Add typings for AI assistant requests and responses. --- .../internal/create-ai-assistant-request.yaml | 6 + .../internal/success-create-ai-assistant.yaml | 5 + .../internal/ai-assistant-create-request.yaml | 37 +++ .../ai-assistant-create-response.yaml | 14 + .../openapi/openvidu-meet-internal-api.yaml | 8 + .../openapi/paths/internal/ai-assistant.yaml | 56 ++++ meet-ce/backend/openapi/tags/tags.yaml | 2 + .../src/config/dependency-injector.config.ts | 2 + meet-ce/backend/src/config/internal-config.ts | 2 +- .../controllers/ai-assistant.controller.ts | 69 +++++ meet-ce/backend/src/helpers/redis.helper.ts | 12 + meet-ce/backend/src/middlewares/index.ts | 1 + .../ai-assistant-validator.middleware.ts | 26 ++ meet-ce/backend/src/models/redis.model.ts | 7 +- .../models/zod-schemas/ai-assistant.schema.ts | 34 +++ .../backend/src/routes/ai-assistant.routes.ts | 26 ++ meet-ce/backend/src/routes/index.ts | 1 + meet-ce/backend/src/server.ts | 2 + .../src/services/ai-assistant.service.ts | 251 ++++++++++++++++++ .../src/services/livekit-webhook.service.ts | 11 +- .../backend/src/services/livekit.service.ts | 70 ++++- meet-ce/backend/src/services/mutex.service.ts | 27 ++ .../src/services/room-member.service.ts | 3 +- meet-ce/backend/src/services/token.service.ts | 38 +-- meet-ce/typings/src/ai-assistant.ts | 42 +++ meet-ce/typings/src/index.ts | 10 +- 26 files changed, 714 insertions(+), 48 deletions(-) create mode 100644 meet-ce/backend/openapi/components/requestBodies/internal/create-ai-assistant-request.yaml create mode 100644 meet-ce/backend/openapi/components/responses/internal/success-create-ai-assistant.yaml create mode 100644 meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-request.yaml create mode 100644 meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-response.yaml create mode 100644 meet-ce/backend/openapi/paths/internal/ai-assistant.yaml create mode 100644 meet-ce/backend/src/controllers/ai-assistant.controller.ts create mode 100644 meet-ce/backend/src/middlewares/request-validators/ai-assistant-validator.middleware.ts create mode 100644 meet-ce/backend/src/models/zod-schemas/ai-assistant.schema.ts create mode 100644 meet-ce/backend/src/routes/ai-assistant.routes.ts create mode 100644 meet-ce/backend/src/services/ai-assistant.service.ts create mode 100644 meet-ce/typings/src/ai-assistant.ts diff --git a/meet-ce/backend/openapi/components/requestBodies/internal/create-ai-assistant-request.yaml b/meet-ce/backend/openapi/components/requestBodies/internal/create-ai-assistant-request.yaml new file mode 100644 index 00000000..e8a56bb0 --- /dev/null +++ b/meet-ce/backend/openapi/components/requestBodies/internal/create-ai-assistant-request.yaml @@ -0,0 +1,6 @@ +description: Create AI assistant activation request +required: true +content: + application/json: + schema: + $ref: '../../schemas/internal/ai-assistant-create-request.yaml' diff --git a/meet-ce/backend/openapi/components/responses/internal/success-create-ai-assistant.yaml b/meet-ce/backend/openapi/components/responses/internal/success-create-ai-assistant.yaml new file mode 100644 index 00000000..caea3448 --- /dev/null +++ b/meet-ce/backend/openapi/components/responses/internal/success-create-ai-assistant.yaml @@ -0,0 +1,5 @@ +description: Successfully created or reused AI assistant activation +content: + application/json: + schema: + $ref: '../../schemas/internal/ai-assistant-create-response.yaml' diff --git a/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-request.yaml b/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-request.yaml new file mode 100644 index 00000000..50a085e8 --- /dev/null +++ b/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-request.yaml @@ -0,0 +1,37 @@ +type: object +required: + # - scope + - capabilities +properties: + # scope: + # type: object + # required: + # - resourceType + # - resourceIds + # properties: + # resourceType: + # type: string + # enum: ['meeting'] + # description: Scope resource type where assistant will be activated. + # example: meeting + # resourceIds: + # type: array + # minItems: 1 + # items: + # type: string + # minLength: 1 + # description: List of target resource ids. + # example: ['meeting_123'] + capabilities: + type: array + minItems: 1 + items: + type: object + required: + - name + properties: + name: + type: string + enum: ['live_captions'] + description: AI capability to activate. + example: live_captions diff --git a/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-response.yaml b/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-response.yaml new file mode 100644 index 00000000..03fd8844 --- /dev/null +++ b/meet-ce/backend/openapi/components/schemas/internal/ai-assistant-create-response.yaml @@ -0,0 +1,14 @@ +type: object +required: + - id + - status +properties: + id: + type: string + description: Identifier of the assistant activation. + example: asst_123 + status: + type: string + enum: ['active'] + description: Current assistant activation state. + example: active diff --git a/meet-ce/backend/openapi/openvidu-meet-internal-api.yaml b/meet-ce/backend/openapi/openvidu-meet-internal-api.yaml index 96f13ffa..573002a1 100644 --- a/meet-ce/backend/openapi/openvidu-meet-internal-api.yaml +++ b/meet-ce/backend/openapi/openvidu-meet-internal-api.yaml @@ -42,6 +42,10 @@ paths: $ref: './paths/internal/meetings.yaml#/~1meetings~1{roomId}~1participants~1{participantIdentity}' /meetings/{roomId}/participants/{participantIdentity}/role: $ref: './paths/internal/meetings.yaml#/~1meetings~1{roomId}~1participants~1{participantIdentity}~1role' + /ai/assistants: + $ref: './paths/internal/ai-assistant.yaml#/~1ai~1assistants' + /ai/assistants/{assistantId}: + $ref: './paths/internal/ai-assistant.yaml#/~1ai~1assistants~1{assistantId}' /analytics: $ref: './paths/internal/analytics.yaml#/~1analytics' @@ -63,5 +67,9 @@ components: $ref: components/schemas/meet-room.yaml MeetAnalytics: $ref: components/schemas/internal/meet-analytics.yaml + AiAssistantCreateRequest: + $ref: components/schemas/internal/ai-assistant-create-request.yaml + AiAssistantCreateResponse: + $ref: components/schemas/internal/ai-assistant-create-response.yaml Error: $ref: components/schemas/error.yaml diff --git a/meet-ce/backend/openapi/paths/internal/ai-assistant.yaml b/meet-ce/backend/openapi/paths/internal/ai-assistant.yaml new file mode 100644 index 00000000..5454791c --- /dev/null +++ b/meet-ce/backend/openapi/paths/internal/ai-assistant.yaml @@ -0,0 +1,56 @@ +/ai/assistants: + post: + operationId: createAiAssistant + summary: Create AI assistant + description: | + Activates AI assistance. + + > Currently only meeting AI Assistand and `live_captions` capability is supported. + tags: + - Internal API - AI Assistants + security: + - roomMemberTokenHeader: [] + requestBody: + $ref: '../../components/requestBodies/internal/create-ai-assistant-request.yaml' + responses: + '200': + $ref: '../../components/responses/internal/success-create-ai-assistant.yaml' + '401': + $ref: '../../components/responses/unauthorized-error.yaml' + '403': + $ref: '../../components/responses/forbidden-error.yaml' + '422': + $ref: '../../components/responses/validation-error.yaml' + '500': + $ref: '../../components/responses/internal-server-error.yaml' + +/ai/assistants/{assistantId}: + delete: + operationId: cancelAiAssistant + summary: Cancel AI assistant + description: | + Cancels AI assistant. + + The assistant process (live_captions) is stopped only when the last participant cancels it. + tags: + - Internal API - AI Assistants + security: + - roomMemberTokenHeader: [] + parameters: + - in: path + name: assistantId + required: true + schema: + type: string + minLength: 1 + description: Identifier of the assistant activation returned by create operation. + example: asst_123 + responses: + '204': + description: AI assistant canceled successfully. + '401': + $ref: '../../components/responses/unauthorized-error.yaml' + '422': + $ref: '../../components/responses/validation-error.yaml' + '500': + $ref: '../../components/responses/internal-server-error.yaml' diff --git a/meet-ce/backend/openapi/tags/tags.yaml b/meet-ce/backend/openapi/tags/tags.yaml index 9c520434..27862614 100644 --- a/meet-ce/backend/openapi/tags/tags.yaml +++ b/meet-ce/backend/openapi/tags/tags.yaml @@ -16,5 +16,7 @@ description: Operations related to managing OpenVidu Meet rooms - name: Internal API - Meetings description: Operations related to managing meetings in OpenVidu Meet rooms +- name: Internal API - AI Assistants + description: High-level operations to manage AI assistance capabilities in meetings - name: Internal API - Recordings description: Operations related to managing OpenVidu Meet recordings diff --git a/meet-ce/backend/src/config/dependency-injector.config.ts b/meet-ce/backend/src/config/dependency-injector.config.ts index 7f889ca1..1650bb85 100644 --- a/meet-ce/backend/src/config/dependency-injector.config.ts +++ b/meet-ce/backend/src/config/dependency-injector.config.ts @@ -54,6 +54,7 @@ import { LivekitWebhookService } from '../services/livekit-webhook.service.js'; import { RoomScheduledTasksService } from '../services/room-scheduled-tasks.service.js'; import { RecordingScheduledTasksService } from '../services/recording-scheduled-tasks.service.js'; import { AnalyticsService } from '../services/analytics.service.js'; +import { AiAssistantService } from '../services/ai-assistant.service.js'; export const container: Container = new Container(); @@ -113,6 +114,7 @@ export const registerDependencies = () => { container.bind(RoomScheduledTasksService).toSelf().inSingletonScope(); container.bind(RecordingScheduledTasksService).toSelf().inSingletonScope(); container.bind(AnalyticsService).toSelf().inSingletonScope(); + container.bind(AiAssistantService).toSelf().inSingletonScope(); }; const configureStorage = (storageMode: string) => { diff --git a/meet-ce/backend/src/config/internal-config.ts b/meet-ce/backend/src/config/internal-config.ts index de7baa18..3e6d96a9 100644 --- a/meet-ce/backend/src/config/internal-config.ts +++ b/meet-ce/backend/src/config/internal-config.ts @@ -49,7 +49,7 @@ export const INTERNAL_CONFIG = { PARTICIPANT_MAX_CONCURRENT_NAME_REQUESTS: '20', // Maximum number of request by the same name at the same time allowed PARTICIPANT_NAME_RESERVATION_TTL: '12h' as StringValue, // Time-to-live for participant name reservations - CAPTIONS_AGENT_NAME: 'agent-speech-processing', + CAPTIONS_AGENT_NAME: 'speech-processing', // MongoDB Schema Versions // These define the current schema version for each collection diff --git a/meet-ce/backend/src/controllers/ai-assistant.controller.ts b/meet-ce/backend/src/controllers/ai-assistant.controller.ts new file mode 100644 index 00000000..f27d77a3 --- /dev/null +++ b/meet-ce/backend/src/controllers/ai-assistant.controller.ts @@ -0,0 +1,69 @@ +import { Request, Response } from 'express'; +import { container } from '../config/dependency-injector.config.js'; +import { handleError } from '../models/error.model.js'; +import { AiAssistantService } from '../services/ai-assistant.service.js'; +import { LoggerService } from '../services/logger.service.js'; +import { RequestSessionService } from '../services/request-session.service.js'; +import { TokenService } from '../services/token.service.js'; +import { getRoomMemberToken } from '../utils/token.utils.js'; + +const getRoomMemberIdentityFromRequest = async (req: Request): Promise => { + const tokenService = container.get(TokenService); + const token = getRoomMemberToken(req); + + if (!token) { + throw new Error('Room member token not found'); + } + + const claims = await tokenService.verifyToken(token); + + if (!claims.sub) { + throw new Error('Room member token does not include participant identity'); + } + + return claims.sub; +}; + +export const createAssistant = async (req: Request, res: Response) => { + const logger = container.get(LoggerService); + const requestSessionService = container.get(RequestSessionService); + const aiAssistantService = container.get(AiAssistantService); + // const payload: MeetCreateAssistantRequest = req.body; + const roomId = requestSessionService.getRoomIdFromToken(); + + if (!roomId) { + return handleError(res, new Error('Could not resolve room from token'), 'creating assistant'); + } + + try { + const participantIdentity = await getRoomMemberIdentityFromRequest(req); + logger.verbose(`Creating assistant for participant '${participantIdentity}' in room '${roomId}'`); + const assistant = await aiAssistantService.createLiveCaptionsAssistant(roomId, participantIdentity); + return res.status(200).json(assistant); + } catch (error) { + handleError(res, error, `creating assistant in room '${roomId}'`); + } +}; + +export const cancelAssistant = async (req: Request, res: Response) => { + const logger = container.get(LoggerService); + const requestSessionService = container.get(RequestSessionService); + const aiAssistantService = container.get(AiAssistantService); + const { assistantId } = req.params; + const roomId = requestSessionService.getRoomIdFromToken(); + + if (!roomId) { + return handleError(res, new Error('Could not resolve room from token'), 'canceling assistant'); + } + + try { + const participantIdentity = await getRoomMemberIdentityFromRequest(req); + logger.verbose( + `Canceling assistant '${assistantId}' for participant '${participantIdentity}' in room '${roomId}'` + ); + await aiAssistantService.cancelAssistant(assistantId, roomId, participantIdentity); + return res.status(204).send(); + } catch (error) { + handleError(res, error, `canceling assistant '${assistantId}' in room '${roomId}'`); + } +}; diff --git a/meet-ce/backend/src/helpers/redis.helper.ts b/meet-ce/backend/src/helpers/redis.helper.ts index 62a3b0f9..1b1ef9ea 100644 --- a/meet-ce/backend/src/helpers/redis.helper.ts +++ b/meet-ce/backend/src/helpers/redis.helper.ts @@ -45,4 +45,16 @@ export class MeetLock { return `${RedisLockPrefix.BASE}${RedisLockName.WEBHOOK}_${webhookEvent.event}_${webhookEvent.id}`; } + + static getAiAssistantLock(roomId: string, capabilityName: string): string { + if (!roomId) { + throw new Error('roomId must be a non-empty string'); + } + + if (!capabilityName) { + throw new Error('capabilityName must be a non-empty string'); + } + + return `${RedisLockPrefix.BASE}${RedisLockName.AI_ASSISTANT}_${roomId}_${capabilityName}`; + } } diff --git a/meet-ce/backend/src/middlewares/index.ts b/meet-ce/backend/src/middlewares/index.ts index 7ee12db5..2c2ca197 100644 --- a/meet-ce/backend/src/middlewares/index.ts +++ b/meet-ce/backend/src/middlewares/index.ts @@ -9,6 +9,7 @@ export * from './room.middleware.js'; // Request validators export * from './request-validators/auth-validator.middleware.js'; +export * from './request-validators/ai-assistant-validator.middleware.js'; export * from './request-validators/config-validator.middleware.js'; export * from './request-validators/meeting-validator.middleware.js'; export * from './request-validators/recording-validator.middleware.js'; diff --git a/meet-ce/backend/src/middlewares/request-validators/ai-assistant-validator.middleware.ts b/meet-ce/backend/src/middlewares/request-validators/ai-assistant-validator.middleware.ts new file mode 100644 index 00000000..1a6bc0b3 --- /dev/null +++ b/meet-ce/backend/src/middlewares/request-validators/ai-assistant-validator.middleware.ts @@ -0,0 +1,26 @@ +import { NextFunction, Request, Response } from 'express'; +import { rejectUnprocessableRequest } from '../../models/error.model.js'; +import { AssistantIdSchema, CreateAssistantReqSchema } from '../../models/zod-schemas/ai-assistant.schema.js'; + +export const validateCreateAssistantReq = (req: Request, res: Response, next: NextFunction) => { + const { success, error, data } = CreateAssistantReqSchema.safeParse(req.body); + + if (!success) { + return rejectUnprocessableRequest(res, error); + } + + req.body = data; + next(); +}; + +export const validateAssistantIdPathParam = (req: Request, res: Response, next: NextFunction) => { + const { success, error, data } = AssistantIdSchema.safeParse(req.params.assistantId); + + if (!success) { + error.errors[0].path = ['assistantId']; + return rejectUnprocessableRequest(res, error); + } + + req.params.assistantId = data; + next(); +}; diff --git a/meet-ce/backend/src/models/redis.model.ts b/meet-ce/backend/src/models/redis.model.ts index 97977c00..58d82f9d 100644 --- a/meet-ce/backend/src/models/redis.model.ts +++ b/meet-ce/backend/src/models/redis.model.ts @@ -5,7 +5,9 @@ export const enum RedisKeyName { ROOM_PARTICIPANTS = `${REDIS_KEY_PREFIX}room_participants:`, // Stores released numeric suffixes (per base name) in a sorted set, so that freed numbers // can be reused efficiently instead of always incrementing to the next highest number. - PARTICIPANT_NAME_POOL = `${REDIS_KEY_PREFIX}participant_pool:` + PARTICIPANT_NAME_POOL = `${REDIS_KEY_PREFIX}participant_pool:`, + // Tracks participant-level assistant capability state in a room. + AI_ASSISTANT_PARTICIPANT_STATE = `${REDIS_KEY_PREFIX}ai_assistant:participant_state:` } export const enum RedisLockPrefix { @@ -18,5 +20,6 @@ export const enum RedisLockName { SCHEDULED_TASK = 'scheduled_task', STORAGE_INITIALIZATION = 'storage_initialization', MIGRATION = 'migration', - WEBHOOK = 'webhook' + WEBHOOK = 'webhook', + AI_ASSISTANT = 'ai_assistant' } diff --git a/meet-ce/backend/src/models/zod-schemas/ai-assistant.schema.ts b/meet-ce/backend/src/models/zod-schemas/ai-assistant.schema.ts new file mode 100644 index 00000000..f2e17049 --- /dev/null +++ b/meet-ce/backend/src/models/zod-schemas/ai-assistant.schema.ts @@ -0,0 +1,34 @@ +import { MeetAssistantCapabilityName } from '@openvidu-meet/typings'; +import { z } from 'zod'; + +export const CreateAssistantReqSchema = z.object({ + // scope: z.object({ + // resourceType: z.nativeEnum(MeetAssistantScopeResourceType), + // resourceIds: z.array(z.string().trim().min(1)).min(1) + // }), + capabilities: z + .array( + z.object({ + name: z.string() + }) + ) + .min(1) + .transform((capabilities) => { + const validValues = Object.values(MeetAssistantCapabilityName); + + // Filter out invalid capabilities + const filtered = capabilities.filter((cap) => + validValues.includes(cap.name as MeetAssistantCapabilityName) + ); + + // Remove duplicates based on capability name + const unique = Array.from(new Map(filtered.map((cap) => [cap.name, cap])).values()); + + return unique; + }) + .refine((caps) => caps.length > 0, { + message: 'At least one valid capability is required' + }) +}); + +export const AssistantIdSchema = z.string().trim().min(1); diff --git a/meet-ce/backend/src/routes/ai-assistant.routes.ts b/meet-ce/backend/src/routes/ai-assistant.routes.ts new file mode 100644 index 00000000..90c35cce --- /dev/null +++ b/meet-ce/backend/src/routes/ai-assistant.routes.ts @@ -0,0 +1,26 @@ +import bodyParser from 'body-parser'; +import { Router } from 'express'; +import * as aiAssistantCtrl from '../controllers/ai-assistant.controller.js'; +import { roomMemberTokenValidator, withAuth } from '../middlewares/auth.middleware.js'; +import { + validateAssistantIdPathParam, + validateCreateAssistantReq +} from '../middlewares/request-validators/ai-assistant-validator.middleware.js'; + +export const aiAssistantRouter: Router = Router(); +aiAssistantRouter.use(bodyParser.urlencoded({ extended: true })); +aiAssistantRouter.use(bodyParser.json()); + +aiAssistantRouter.post( + '/assistants', + withAuth(roomMemberTokenValidator), + validateCreateAssistantReq, + aiAssistantCtrl.createAssistant +); + +aiAssistantRouter.delete( + '/assistants/:assistantId', + withAuth(roomMemberTokenValidator), + validateAssistantIdPathParam, + aiAssistantCtrl.cancelAssistant +); diff --git a/meet-ce/backend/src/routes/index.ts b/meet-ce/backend/src/routes/index.ts index 226c3aa4..94d2a2aa 100644 --- a/meet-ce/backend/src/routes/index.ts +++ b/meet-ce/backend/src/routes/index.ts @@ -1,4 +1,5 @@ export * from './analytics.routes.js'; +export * from './ai-assistant.routes.js'; export * from './api-key.routes.js'; export * from './auth.routes.js'; export * from './global-config.routes.js'; diff --git a/meet-ce/backend/src/server.ts b/meet-ce/backend/src/server.ts index 1fefb73b..56a3f63b 100644 --- a/meet-ce/backend/src/server.ts +++ b/meet-ce/backend/src/server.ts @@ -9,6 +9,7 @@ import { setBaseUrlFromRequest } from './middlewares/base-url.middleware.js'; import { jsonSyntaxErrorHandler } from './middlewares/content-type.middleware.js'; import { initRequestContext } from './middlewares/request-context.middleware.js'; import { analyticsRouter } from './routes/analytics.routes.js'; +import { aiAssistantRouter } from './routes/ai-assistant.routes.js'; import { apiKeyRouter } from './routes/api-key.routes.js'; import { authRouter } from './routes/auth.routes.js'; import { configRouter } from './routes/global-config.routes.js'; @@ -99,6 +100,7 @@ const createApp = () => { // appRouter.use(`${INTERNAL_CONFIG.INTERNAL_API_BASE_PATH_V1}/recordings`, internalRecordingRouter); appRouter.use(`${INTERNAL_CONFIG.INTERNAL_API_BASE_PATH_V1}/config`, configRouter); appRouter.use(`${INTERNAL_CONFIG.INTERNAL_API_BASE_PATH_V1}/analytics`, analyticsRouter); + appRouter.use(`${INTERNAL_CONFIG.INTERNAL_API_BASE_PATH_V1}/ai`, aiAssistantRouter); appRouter.use('/health', (_req: Request, res: Response) => res.status(200).send('OK')); diff --git a/meet-ce/backend/src/services/ai-assistant.service.ts b/meet-ce/backend/src/services/ai-assistant.service.ts new file mode 100644 index 00000000..1a8aafe0 --- /dev/null +++ b/meet-ce/backend/src/services/ai-assistant.service.ts @@ -0,0 +1,251 @@ +import { MeetAssistantCapabilityName, MeetCreateAssistantResponse } from '@openvidu-meet/typings'; +import { inject, injectable } from 'inversify'; +import ms from 'ms'; +import { INTERNAL_CONFIG } from '../config/internal-config.js'; +import { MEET_ENV } from '../environment.js'; +import { MeetLock } from '../helpers/redis.helper.js'; +import { errorInsufficientPermissions } from '../models/error.model.js'; +import { RedisKeyName } from '../models/redis.model.js'; +import { LiveKitService } from './livekit.service.js'; +import { LoggerService } from './logger.service.js'; +import { MutexService } from './mutex.service.js'; +import { RedisService } from './redis.service.js'; +import { RoomService } from './room.service.js'; + +@injectable() +export class AiAssistantService { + private readonly ASSISTANT_STATE_LOCK_TTL = ms('15s'); + + constructor( + @inject(LoggerService) protected logger: LoggerService, + @inject(RoomService) protected roomService: RoomService, + @inject(LiveKitService) protected livekitService: LiveKitService, + @inject(MutexService) protected mutexService: MutexService, + @inject(RedisService) protected redisService: RedisService + ) {} + + /** + * Creates a live captions assistant for the specified room. + * If an assistant already exists for the room, it will be reused. + * @param roomId + * @param participantIdentity + * @returns + */ + async createLiveCaptionsAssistant( + roomId: string, + participantIdentity: string + ): Promise { + // ! For now, we are assuming that the only capability is live captions. + const capability = MeetAssistantCapabilityName.LIVE_CAPTIONS; + const lockName = MeetLock.getAiAssistantLock(roomId, capability); + + try { + await this.validateCreateConditions(roomId, capability); + + const lock = await this.mutexService.acquire(lockName, this.ASSISTANT_STATE_LOCK_TTL); + + if (!lock) { + this.logger.error(`Could not acquire lock '${lockName}' for creating assistant in room '${roomId}'`); + throw new Error('Could not acquire lock for creating assistant. Please try again.'); + } + + const existingAgent = await this.livekitService.getAgent(roomId, INTERNAL_CONFIG.CAPTIONS_AGENT_NAME); + + if (existingAgent) { + await this.setParticipantAssistantState(roomId, participantIdentity, capability, true); + return { id: existingAgent.id, status: 'active' }; + } + + const assistant = await this.livekitService.createAgent(roomId, INTERNAL_CONFIG.CAPTIONS_AGENT_NAME); + + await this.setParticipantAssistantState(roomId, participantIdentity, capability, true); + + return { + id: assistant.id, + status: 'active' + }; + } finally { + await this.mutexService.release(lockName); + } + } + + /** + * Stops the specified assistant for the given participant and room. + * If the assistant is not used by any other participants in the room, it will be stopped in LiveKit. + * @param assistantId + * @param roomId + * @param participantIdentity + * @returns + */ + async cancelAssistant(assistantId: string, roomId: string, participantIdentity: string): Promise { + const capability = MeetAssistantCapabilityName.LIVE_CAPTIONS; + // The lock only protects the atomic "count → stop dispatch" decision. + const lockName = MeetLock.getAiAssistantLock(roomId, capability); + + try { + await this.setParticipantAssistantState(roomId, participantIdentity, capability, false); + + const lock = await this.mutexService.acquire(lockName, this.ASSISTANT_STATE_LOCK_TTL); + + if (!lock) { + this.logger.warn( + `Could not acquire lock '${lockName}' for stopping assistant in room '${roomId}'. Participant state saved as disabled.` + ); + return; + } + + const enabledParticipants = await this.getEnabledParticipantsCount(roomId, capability); + + if (enabledParticipants > 0) { + this.logger.debug( + `Skipping assistant stop for room '${roomId}'. Remaining enabled participants: ${enabledParticipants}` + ); + return; + } + + const assistant = await this.livekitService.getAgent(roomId, assistantId); + + if (!assistant) { + this.logger.warn(`Captions assistant not found in room '${roomId}'. Skipping stop request.`); + return; + } + + await this.livekitService.stopAgent(assistantId, roomId); + } finally { + await this.mutexService.release(lockName); + } + } + + /** + * Cleanup assistant state in a room. + * - If participantIdentity is provided, removes only that participant state. + * - If participantIdentity is omitted, removes all assistant state in the room. + * + * If no enabled participants remain after cleanup, captions agent dispatch is stopped. + */ + async cleanupState(roomId: string, participantIdentity?: string): Promise { + const capability = MeetAssistantCapabilityName.LIVE_CAPTIONS; + const lockName = MeetLock.getAiAssistantLock(roomId, capability); + + try { + if (participantIdentity) { + await this.setParticipantAssistantState(roomId, participantIdentity, capability, false); + } + + // acquireWithRetry because this is called from webhooks (participantLeft / roomFinished). + // The agent may run indefinitely with no further opportunity to stop it. + const lock = await this.mutexService.acquireWithRetry(lockName, this.ASSISTANT_STATE_LOCK_TTL); + + if (!lock) { + const scope = participantIdentity ? `participant '${participantIdentity}'` : `room '${roomId}'`; + this.logger.error( + `Could not acquire lock '${lockName}' for dispatch cleanup (${scope}) after retries. ` + + (participantIdentity + ? 'Participant state was saved but dispatch stop may be skipped.' + : 'Room state cleanup and dispatch stop were skipped.') + ); + return; + } + + if (!participantIdentity) { + const pattern = `${RedisKeyName.AI_ASSISTANT_PARTICIPANT_STATE}${roomId}:${capability}:*`; + const keys = await this.redisService.getKeys(pattern); + + if (keys.length > 0) { + await this.redisService.delete(keys); + } + } + + const enabledParticipants = await this.getEnabledParticipantsCount(roomId, capability); + + if (enabledParticipants > 0) { + return; + } + + await this.stopCaptionsAssistantIfRunning(roomId); + } catch (error) { + this.logger.error(`Error occurred while cleaning up assistant state for room '${roomId}': ${error}`); + } finally { + await this.mutexService.release(lockName); + } + } + + protected async validateCreateConditions(roomId: string, capability: MeetAssistantCapabilityName): Promise { + if (capability === MeetAssistantCapabilityName.LIVE_CAPTIONS) { + if (MEET_ENV.CAPTIONS_ENABLED !== 'true') { + throw errorInsufficientPermissions(); + } + + const room = await this.roomService.getMeetRoom(roomId); + + if (!room.config.captions.enabled) { + throw errorInsufficientPermissions(); + } + } + } + + /** + * Sets or clears the assistant state for a participant in Redis. + * @param roomId + * @param participantIdentity + * @param capability + * @param enabled + */ + protected async setParticipantAssistantState( + roomId: string, + participantIdentity: string, + capability: MeetAssistantCapabilityName, + enabled: boolean + ): Promise { + const key = this.getParticipantAssistantStateKey(roomId, participantIdentity, capability); + + if (!enabled) { + await this.redisService.delete(key); + return; + } + + await this.redisService.setIfNotExists( + key, + JSON.stringify({ + enabled: true, + updatedAt: Date.now() + }) + ); + } + + /** + * Gets the count of participants that have the specified assistant capability enabled in the given room. + * @param roomId + * @param capability + * @returns + */ + protected async getEnabledParticipantsCount( + roomId: string, + capability: MeetAssistantCapabilityName + ): Promise { + const pattern = `${RedisKeyName.AI_ASSISTANT_PARTICIPANT_STATE}${roomId}:${capability}:*`; + const keys = await this.redisService.getKeys(pattern); + return keys.length; + } + + protected getParticipantAssistantStateKey( + roomId: string, + participantIdentity: string, + capability: MeetAssistantCapabilityName + ): string { + return `${RedisKeyName.AI_ASSISTANT_PARTICIPANT_STATE}${roomId}:${capability}:${participantIdentity}`; + } + + protected async stopCaptionsAssistantIfRunning(roomId: string): Promise { + const assistants = await this.livekitService.listAgents(roomId); + const captionsAssistant = assistants.find( + (assistant) => assistant.agentName === INTERNAL_CONFIG.CAPTIONS_AGENT_NAME + ); + + if (!captionsAssistant) { + return; + } + + await this.livekitService.stopAgent(captionsAssistant.id, roomId); + } +} diff --git a/meet-ce/backend/src/services/livekit-webhook.service.ts b/meet-ce/backend/src/services/livekit-webhook.service.ts index dadaa7a0..70e90ace 100644 --- a/meet-ce/backend/src/services/livekit-webhook.service.ts +++ b/meet-ce/backend/src/services/livekit-webhook.service.ts @@ -9,6 +9,7 @@ import { MeetRoomHelper } from '../helpers/room.helper.js'; import { DistributedEventType } from '../models/distributed-event.model.js'; import { RecordingRepository } from '../repositories/recording.repository.js'; import { RoomRepository } from '../repositories/room.repository.js'; +import { AiAssistantService } from './ai-assistant.service.js'; import { DistributedEventService } from './distributed-event.service.js'; import { FrontendEventService } from './frontend-event.service.js'; import { LiveKitService } from './livekit.service.js'; @@ -33,6 +34,7 @@ export class LivekitWebhookService { @inject(DistributedEventService) protected distributedEventService: DistributedEventService, @inject(FrontendEventService) protected frontendEventService: FrontendEventService, @inject(RoomMemberService) protected roomMemberService: RoomMemberService, + @inject(AiAssistantService) protected aiAssistantService: AiAssistantService, @inject(LoggerService) protected logger: LoggerService ) { this.webhookReceiver = new WebhookReceiver(MEET_ENV.LIVEKIT_API_KEY, MEET_ENV.LIVEKIT_API_SECRET); @@ -189,8 +191,10 @@ export class LivekitWebhookService { if (!this.livekitService.isStandardParticipant(participant)) return; try { - // Release the participant's reserved name - await this.roomMemberService.releaseParticipantName(room.name, participant.name); + await Promise.all([ + this.roomMemberService.releaseParticipantName(room.name, participant.name), + this.aiAssistantService.cleanupState(room.name, participant.identity) + ]); this.logger.verbose(`Released name for participant '${participant.name}' in room '${room.name}'`); } catch (error) { this.logger.error('Error releasing participant name on participant left:', error); @@ -282,7 +286,8 @@ export class LivekitWebhookService { tasks.push( this.roomMemberService.cleanupParticipantNames(roomId), - this.recordingService.releaseRecordingLockIfNoEgress(roomId) + this.recordingService.releaseRecordingLockIfNoEgress(roomId), + this.aiAssistantService.cleanupState(roomId) ); await Promise.all(tasks); } catch (error) { diff --git a/meet-ce/backend/src/services/livekit.service.ts b/meet-ce/backend/src/services/livekit.service.ts index 72debf17..676d00a4 100644 --- a/meet-ce/backend/src/services/livekit.service.ts +++ b/meet-ce/backend/src/services/livekit.service.ts @@ -1,6 +1,7 @@ -import { ParticipantInfo_Kind } from '@livekit/protocol'; +import { AgentDispatch, ParticipantInfo_Kind } from '@livekit/protocol'; import { inject, injectable } from 'inversify'; import { + AgentDispatchClient, CreateOptions, DataPacket_Kind, EgressClient, @@ -31,6 +32,7 @@ import { LoggerService } from './logger.service.js'; export class LiveKitService { private egressClient: EgressClient; private roomClient: RoomServiceClient; + private agentClient: AgentDispatchClient; constructor(@inject(LoggerService) protected logger: LoggerService) { const livekitUrlHostname = MEET_ENV.LIVEKIT_URL_PRIVATE.replace(/^ws:/, 'http:').replace(/^wss:/, 'https:'); @@ -40,6 +42,11 @@ export class LiveKitService { MEET_ENV.LIVEKIT_API_KEY, MEET_ENV.LIVEKIT_API_SECRET ); + this.agentClient = new AgentDispatchClient( + livekitUrlHostname, + MEET_ENV.LIVEKIT_API_KEY, + MEET_ENV.LIVEKIT_API_SECRET + ); } async createRoom(options: CreateOptions): Promise { @@ -270,6 +277,67 @@ export class LiveKitService { } } + /** + * Start an agent for a specific room. + * @param roomName + * @param agentName + * @returns The created AgentDispatch + */ + async createAgent( + roomName: string, + agentName: string /*, options: CreateDispatchOptions*/ + ): Promise { + try { + return await this.agentClient.createDispatch(roomName, agentName); + } catch (error) { + this.logger.error(`Error creating agent dispatch for room '${roomName}':`, error); + throw internalError(`creating agent dispatch for room '${roomName}'`); + } + } + + /** + * Lists all agents in a LiveKit room. + * @param roomName + * @returns An array of agents in the specified room + */ + async listAgents(roomName: string): Promise { + try { + return await this.agentClient.listDispatch(roomName); + } catch (error) { + this.logger.error(`Error listing agents for room '${roomName}':`, error); + throw internalError(`listing agents for room '${roomName}'`); + } + } + + /** + * Gets an agent dispatch by its ID in a LiveKit room. + * @param roomName + * @param agentId + * @returns The agent if found, otherwise undefined + */ + async getAgent(roomName: string, agentId: string): Promise { + try { + return await this.agentClient.getDispatch(agentId, roomName); + } catch (error) { + this.logger.error(`Error getting agent dispatch '${agentId}' for room '${roomName}':`, error); + throw internalError(`getting agent dispatch '${agentId}' for room '${roomName}'`); + } + } + + /** + * Stops an agent in a LiveKit room. + * @param agentId + * @param roomName + */ + async stopAgent(agentId: string, roomName: string): Promise { + try { + await this.agentClient.deleteDispatch(agentId, roomName); + } catch (error) { + this.logger.error(`Error deleting agent dispatch '${agentId}' for room '${roomName}':`, error); + throw internalError(`deleting agent dispatch '${agentId}' for room '${roomName}'`); + } + } + async startRoomComposite( roomName: string, output: EncodedFileOutput | StreamOutput, diff --git a/meet-ce/backend/src/services/mutex.service.ts b/meet-ce/backend/src/services/mutex.service.ts index 637596a0..c6ad4be1 100644 --- a/meet-ce/backend/src/services/mutex.service.ts +++ b/meet-ce/backend/src/services/mutex.service.ts @@ -105,6 +105,33 @@ export class MutexService { return locks; } + /** + * Attempts to acquire a lock, retrying up to `maxAttempts` times with a fixed delay between + * attempts. Intended for fire-and-forget flows (e.g. webhooks) where the caller has no + * opportunity to retry externally and a missed lock acquisition would leave the system in an + * inconsistent state. + * + * @param key - The resource to acquire a lock for. + * @param ttl - The time-to-live for the lock in milliseconds. + * @param maxAttempts - Maximum number of acquisition attempts. Defaults to 3. + * @param delayMs - Fixed delay in milliseconds between attempts. Defaults to 200. + * @returns A Promise that resolves to the acquired Lock, or null if all attempts fail. + */ + async acquireWithRetry(key: string, ttl: number = this.TTL_MS, maxAttempts = 3, delayMs = 200): Promise { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const lock = await this.acquire(key, ttl); + + if (lock) return lock; + + if (attempt < maxAttempts) { + this.logger.warn(`Lock '${key}' attempt ${attempt}/${maxAttempts} failed. Retrying in ${delayMs}ms...`); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + + return null; + } + lockExists(key: string): Promise { const registryKey = MeetLock.getRegistryLock(key); return this.redisService.exists(registryKey); diff --git a/meet-ce/backend/src/services/room-member.service.ts b/meet-ce/backend/src/services/room-member.service.ts index 950ae899..6fcee613 100644 --- a/meet-ce/backend/src/services/room-member.service.ts +++ b/meet-ce/backend/src/services/room-member.service.ts @@ -133,10 +133,9 @@ export class RoomMemberService { // Get participant permissions (with join meeting) const permissions = await this.getRoomMemberPermissions(roomId, role, true); - const withCaptions = room.config.captions.enabled ?? false; // Generate token with participant name - return this.tokenService.generateRoomMemberToken(role, permissions, participantName, participantIdentity, withCaptions); + return this.tokenService.generateRoomMemberToken(role, permissions, participantName, participantIdentity); } /** diff --git a/meet-ce/backend/src/services/token.service.ts b/meet-ce/backend/src/services/token.service.ts index f0af729c..7ed0536d 100644 --- a/meet-ce/backend/src/services/token.service.ts +++ b/meet-ce/backend/src/services/token.service.ts @@ -1,4 +1,3 @@ -import { RoomAgentDispatch, RoomConfiguration } from '@livekit/protocol'; import { MeetRoomMemberPermissions, MeetRoomMemberRole, @@ -42,8 +41,7 @@ export class TokenService { role: MeetRoomMemberRole, permissions: MeetRoomMemberPermissions, participantName?: string, - participantIdentity?: string, - roomWithCaptions = false + participantIdentity?: string ): Promise { const metadata: MeetRoomMemberTokenMetadata = { livekitUrl: MEET_ENV.LIVEKIT_URL, @@ -57,46 +55,16 @@ export class TokenService { ttl: INTERNAL_CONFIG.ROOM_MEMBER_TOKEN_EXPIRATION, metadata: JSON.stringify(metadata) }; - return await this.generateJwtToken(tokenOptions, permissions.livekit as VideoGrant, roomWithCaptions); + return await this.generateJwtToken(tokenOptions, permissions.livekit as VideoGrant); } - private async generateJwtToken( - tokenOptions: AccessTokenOptions, - grants?: VideoGrant, - roomWithCaptions = false - ): Promise { + private async generateJwtToken(tokenOptions: AccessTokenOptions, grants?: VideoGrant): Promise { const at = new AccessToken(MEET_ENV.LIVEKIT_API_KEY, MEET_ENV.LIVEKIT_API_SECRET, tokenOptions); if (grants) { at.addGrant(grants); } - const captionsEnabledGlobally = MEET_ENV.CAPTIONS_ENABLED === 'true'; - const captionsEnabledInRoom = Boolean(roomWithCaptions); - - // Warn if configuration is inconsistent - if (!captionsEnabledGlobally) { - if (captionsEnabledInRoom) { - this.logger.warn( - `Captions feature is disabled in environment but Room is created with captions enabled. ` + - `Please enable captions in environment by setting MEET_CAPTIONS_ENABLED=true to ensure proper functionality.` - ); - } - - return await at.toJwt(); - } - - if (captionsEnabledInRoom) { - this.logger.debug('Activating Captions Agent. Configuring Room Agent Dispatch.'); - at.roomConfig = new RoomConfiguration({ - agents: [ - new RoomAgentDispatch({ - agentName: INTERNAL_CONFIG.CAPTIONS_AGENT_NAME - }) - ] - }); - } - return await at.toJwt(); } diff --git a/meet-ce/typings/src/ai-assistant.ts b/meet-ce/typings/src/ai-assistant.ts new file mode 100644 index 00000000..50abcd42 --- /dev/null +++ b/meet-ce/typings/src/ai-assistant.ts @@ -0,0 +1,42 @@ +/** + * Assistant creation options + */ +export interface MeetCreateAssistantRequest { + // scope: MeetAssistantScope; + capabilities: MeetAssistantCapability[]; +} + +/** + * Defines the scope of an assistant, i.e. the resource(s) it is associated with. + */ +// export interface MeetAssistantScope { +// resourceType: MeetAssistantScopeResourceType; +// resourceIds: string[]; +// } + +/** + * Defines a capability that an assistant can have, such as live captions. + */ +export interface MeetAssistantCapability { + name: MeetAssistantCapabilityName; +} + +/** + * Enumeration of supported assistant capabilities. + */ +export enum MeetAssistantCapabilityName { + LIVE_CAPTIONS = 'live_captions', +} + +/** + * Enumeration of supported resource types that an assistant can be associated with. + */ +// export enum MeetAssistantScopeResourceType { +// MEETING = 'meeting', +// } + + +export interface MeetCreateAssistantResponse { + id: string; + status: 'active'; +} diff --git a/meet-ce/typings/src/index.ts b/meet-ce/typings/src/index.ts index a10c972f..e830cddd 100644 --- a/meet-ce/typings/src/index.ts +++ b/meet-ce/typings/src/index.ts @@ -1,22 +1,24 @@ export * from './api-key.js'; export * from './auth-config.js'; -export * from './global-config.js'; export * from './event.model.js'; +export * from './global-config.js'; export * from './permissions/livekit-permissions.js'; export * from './permissions/meet-permissions.js'; -export * from './sort-pagination.js'; export * from './room-member.js'; +export * from './sort-pagination.js'; export * from './user.js'; +export * from './ai-assistant.js'; +export * from './analytics.js'; +export * from './recording.model.js'; export * from './room-config.js'; export * from './room.js'; -export * from './recording.model.js'; export * from './webhook.model.js'; -export * from './analytics.js'; // Webcomponent types export * from './webcomponent/command.model.js'; export * from './webcomponent/event.model.js'; export * from './webcomponent/message.type.js'; export * from './webcomponent/properties.model.js'; + From 02703b1f837a8c2a41545d4bbd9a62a3838fd46b Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Tue, 3 Mar 2026 19:13:42 +0100 Subject: [PATCH 3/9] frontend: enhance captions button functionality and integrate AI assistant for live captions --- ...eting-toolbar-extra-buttons.component.html | 16 +++-- ...meeting-toolbar-extra-buttons.component.ts | 31 ++++++--- .../services/meeting-captions.service.ts | 67 +++++++++++++++---- .../shared/services/ai-assistant.service.ts | 33 +++++++++ .../src/lib/shared/services/index.ts | 1 + 5 files changed, 122 insertions(+), 26 deletions(-) create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/ai-assistant.service.ts diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html index ad8ecd41..5b4836da 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html @@ -11,8 +11,10 @@ > subtitles - @if (isCaptionsButtonDisabled()) { + @if (captionsStatus() === 'DISABLED_WITH_WARNING') { Live captions (disabled by admin) + } @else if (isCaptionsTogglePending()) { + {{ areCaptionsEnabledByUser() ? 'Disabling live captions...' : 'Enabling live captions...' }} } @else { {{ areCaptionsEnabledByUser() ? 'Disable live captions' : 'Enable live captions' }} } @@ -28,14 +30,16 @@ [disabledInteractive]="isCaptionsButtonDisabled()" [disableRipple]="true" [matTooltip]=" - isCaptionsButtonDisabled() + captionsStatus() === 'DISABLED_WITH_WARNING' ? 'Live captions are disabled by admin' - : areCaptionsEnabledByUser() - ? 'Disable live captions' - : 'Enable live captions' + : isCaptionsTogglePending() + ? (areCaptionsEnabledByUser() ? 'Disabling live captions...' : 'Enabling live captions...') + : areCaptionsEnabledByUser() + ? 'Disable live captions' + : 'Enable live captions' " > - @if (isCaptionsButtonDisabled()) { + @if (captionsStatus() === 'DISABLED_WITH_WARNING') { subtitles_off } @else { subtitles diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts index bcaf47be..c8dc62fb 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts @@ -44,9 +44,18 @@ export class MeetingToolbarExtraButtonsComponent { protected showCaptionsButton = computed(() => this.captionsStatus() !== 'HIDDEN'); /** - * Whether captions button is disabled (true when DISABLED_WITH_WARNING) + * Whether captions button is disabled: + * - Always disabled when captions are turned off at admin/system level (DISABLED_WITH_WARNING) + * - Also disabled while an enable/disable request is in flight to prevent concurrent calls */ - protected isCaptionsButtonDisabled = computed(() => this.captionsStatus() === 'DISABLED_WITH_WARNING'); + protected isCaptionsButtonDisabled = computed( + () => this.captionsStatus() === 'DISABLED_WITH_WARNING' || this.captionService.isCaptionsTogglePending() + ); + + /** + * Whether the captions toggle is pending a server response + */ + protected isCaptionsTogglePending = computed(() => this.captionService.isCaptionsTogglePending()); /** * Whether the device is mobile (affects button style) @@ -65,12 +74,18 @@ export class MeetingToolbarExtraButtonsComponent { this.meetingService.copyMeetingSpeakerLink(room); } - onCaptionsClick(): void { - // Don't allow toggling if captions are disabled at system level - if (this.isCaptionsButtonDisabled()) { - this.log.w('Captions are disabled at system level (MEET_CAPTIONS_ENABLED=false)'); - return; + async onCaptionsClick(): Promise { + try { + // Don't allow toggling if captions are disabled at system level + if (this.isCaptionsButtonDisabled()) { + this.log.w('Captions are disabled at system level (MEET_CAPTIONS_ENABLED=false)'); + return; + } + this.captionService.areCaptionsEnabledByUser() + ? await this.captionService.disable() + : await this.captionService.enable(); + } catch (error) { + this.log.e('Error toggling captions:', error); } - this.captionService.areCaptionsEnabledByUser() ? this.captionService.disable() : this.captionService.enable(); } } diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts index 5094763b..a63dbf49 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts @@ -1,5 +1,6 @@ import { Injectable, inject, signal } from '@angular/core'; import { ILogger, LoggerService, ParticipantService, Room, TextStreamReader } from 'openvidu-components-angular'; +import { AiAssistantService } from '../../../shared'; import { Caption, CaptionsConfig } from '../models/captions.model'; import { CustomParticipantModel } from '../models/custom-participant.model'; @@ -18,9 +19,10 @@ import { CustomParticipantModel } from '../models/custom-participant.model'; providedIn: 'root' }) export class MeetingCaptionsService { - private readonly loggerService = inject(LoggerService); private readonly logger: ILogger; + private readonly loggerService = inject(LoggerService); private readonly participantService = inject(ParticipantService); + private readonly aiAssistantService = inject(AiAssistantService); // Configuration with defaults private readonly defaultConfig: Required = { @@ -38,6 +40,8 @@ export class MeetingCaptionsService { // Reactive state private readonly _captions = signal([]); private readonly _areCaptionsEnabledByUser = signal(false); + private readonly _captionsAgentId = signal(null); + private readonly _isCaptionsTogglePending = signal(false); /** * Current list of active captions @@ -47,6 +51,11 @@ export class MeetingCaptionsService { * Whether captions are enabled by the user */ readonly areCaptionsEnabledByUser = this._areCaptionsEnabledByUser.asReadonly(); + /** + * True while an enable() or disable() call is in flight. + * Use this to prevent concurrent toggle requests. + */ + readonly isCaptionsTogglePending = this._isCaptionsTogglePending.asReadonly(); // Map to track expiration timeouts private expirationTimeouts = new Map>(); @@ -82,7 +91,7 @@ export class MeetingCaptionsService { * Enables captions by registering the transcription handler. * This is called when the user activates captions. */ - enable(): void { + async enable(): Promise { if (!this.room) { this.logger.e('Cannot enable captions: room is not initialized'); return; @@ -93,29 +102,63 @@ export class MeetingCaptionsService { return; } - // Register the LiveKit transcription handler - this.room.registerTextStreamHandler('lk.transcription', this.handleTranscription.bind(this)); + if (this._isCaptionsTogglePending()) { + this.logger.d('Captions toggle already in progress'); + return; + } - this._areCaptionsEnabledByUser.set(true); - this.logger.d('Captions enabled'); + this._isCaptionsTogglePending.set(true); + + try { + // Register the LiveKit transcription handler + const agent = await this.aiAssistantService.createLiveCaptionsAssistant(); + this._captionsAgentId.set(agent.id); + this.room.registerTextStreamHandler('lk.transcription', this.handleTranscription.bind(this)); + this._areCaptionsEnabledByUser.set(true); + this.logger.d('Captions enabled'); + } finally { + // Add a small delay before allowing another toggle to prevent rapid concurrent calls + setTimeout(() => this._isCaptionsTogglePending.set(false), 500); + } } /** * Disables captions by clearing all captions and stopping transcription. * This is called when the user deactivates captions. */ - disable(): void { + async disable(): Promise { if (!this._areCaptionsEnabledByUser()) { this.logger.d('Captions already disabled'); return; } - // Clear all active captions - this.clearAllCaptions(); + if (this._isCaptionsTogglePending()) { + this.logger.d('Captions toggle already in progress'); + return; + } - this._areCaptionsEnabledByUser.set(false); - this.room?.unregisterTextStreamHandler('lk.transcription'); - this.logger.d('Captions disabled'); + this._isCaptionsTogglePending.set(true); + + try { + const agentId = this._captionsAgentId(); + + // Clear all active captions and unregister handler immediately so the UI + // reflects the disabled state before the async server call completes. + this.clearAllCaptions(); + this._areCaptionsEnabledByUser.set(false); + this.room?.unregisterTextStreamHandler('lk.transcription'); + + if (agentId) { + await this.aiAssistantService.cancelAssistant(agentId); + } + + this.logger.d('Captions disabled'); + } catch (error) { + this.logger.e('Error disabling captions:', error); + } finally { + // Add a small delay before allowing another toggle to prevent rapid concurrent calls + setTimeout(() => this._isCaptionsTogglePending.set(false), 500); + } } /** diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/ai-assistant.service.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/ai-assistant.service.ts new file mode 100644 index 00000000..67fb2037 --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/ai-assistant.service.ts @@ -0,0 +1,33 @@ +import { Injectable } from '@angular/core'; +import { + MeetAssistantCapabilityName, + MeetCreateAssistantRequest, + MeetCreateAssistantResponse +} from '@openvidu-meet/typings'; +import { HttpService } from './http.service'; + +@Injectable({ + providedIn: 'root' +}) +export class AiAssistantService { + protected readonly AI_ASSISTANT_API = `${HttpService.INTERNAL_API_PATH_PREFIX}/ai/assistants`; + + constructor(protected httpService: HttpService) {} + + async cancelAssistant(assistantId: string): Promise { + const path = `${this.AI_ASSISTANT_API}/${assistantId}`; + await this.httpService.deleteRequest(path); + } + + async createLiveCaptionsAssistant(): Promise { + const request: MeetCreateAssistantRequest = { + capabilities: [{ name: MeetAssistantCapabilityName.LIVE_CAPTIONS }] + }; + + return this.createAssistant(request); + } + + private async createAssistant(request: MeetCreateAssistantRequest): Promise { + return this.httpService.postRequest(this.AI_ASSISTANT_API, request); + } +} diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/index.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/index.ts index 982c1175..a6adc5dd 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/index.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/shared/services/index.ts @@ -1,3 +1,4 @@ +export * from './ai-assistant.service'; export * from './analytics.service'; export * from './api-key.service'; export * from './app-config.service'; From 8953e9891c25eea666171d2ba90edb34f8379959 Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Wed, 4 Mar 2026 12:13:04 +0100 Subject: [PATCH 4/9] backend(test): Extends test sleep for webhook processing Increases the sleep duration in room deletion test helpers from 1 second to 5 seconds. This ensures that webhooks triggered by room deletion have enough time to process before test execution continues, addressing intermittent test failures caused by premature assertions. This is a temporary measure to improve test stability. --- meet-ce/backend/tests/helpers/request-helpers.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meet-ce/backend/tests/helpers/request-helpers.ts b/meet-ce/backend/tests/helpers/request-helpers.ts index 93b1a003..f435e58a 100644 --- a/meet-ce/backend/tests/helpers/request-helpers.ts +++ b/meet-ce/backend/tests/helpers/request-helpers.ts @@ -335,7 +335,7 @@ export const deleteRoom = async (roomId: string, query: Record .delete(getFullPath(`${INTERNAL_CONFIG.API_BASE_PATH_V1}/rooms/${roomId}`)) .set(INTERNAL_CONFIG.API_KEY_HEADER, MEET_ENV.INITIAL_API_KEY) .query(query); - await sleep('1s'); + await sleep('5s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests return result; }; @@ -346,7 +346,7 @@ export const bulkDeleteRooms = async (roomIds: string[], withMeeting?: string, w .delete(getFullPath(`${INTERNAL_CONFIG.API_BASE_PATH_V1}/rooms`)) .set(INTERNAL_CONFIG.API_KEY_HEADER, MEET_ENV.INITIAL_API_KEY) .query({ roomIds: roomIds.join(','), withMeeting, withRecordings }); - await sleep('1s'); + await sleep('5s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests return result; }; @@ -567,7 +567,7 @@ export const disconnectFakeParticipants = async () => { }); fakeParticipantsProcesses.clear(); - await sleep('1s'); + await sleep('1s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests }; export const updateParticipant = async ( @@ -602,7 +602,7 @@ export const endMeeting = async (roomId: string, moderatorToken: string) => { .delete(getFullPath(`${INTERNAL_CONFIG.INTERNAL_API_BASE_PATH_V1}/meetings/${roomId}`)) .set(INTERNAL_CONFIG.ROOM_MEMBER_TOKEN_HEADER, moderatorToken) .send(); - await sleep('1s'); + await sleep('5s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests return response; }; @@ -640,7 +640,7 @@ export const stopRecording = async (recordingId: string) => { .post(getFullPath(`${INTERNAL_CONFIG.API_BASE_PATH_V1}/recordings/${recordingId}/stop`)) .set(INTERNAL_CONFIG.API_KEY_HEADER, MEET_ENV.INITIAL_API_KEY) .send(); - await sleep('2.5s'); + await sleep('2.5s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests return response; }; From ba0d0b10c48d44bc404960d0b422a5b8c967aa9e Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Wed, 4 Mar 2026 12:33:36 +0100 Subject: [PATCH 5/9] backend: Enhances LiveKit agent robustness LiveKit agent service calls now gracefully handle errors instead of throwing exceptions. `listAgents` and `getAgent` return sensible defaults (empty array, undefined) on failure, preventing disruption to calling services. `stopAgent` now logs errors during deletion without halting the process. An early exit condition is also added in the AI assistant service to prevent unnecessary processing if no agents are found, further improving resilience. --- meet-ce/backend/src/services/ai-assistant.service.ts | 7 ++++--- meet-ce/backend/src/services/livekit.service.ts | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/meet-ce/backend/src/services/ai-assistant.service.ts b/meet-ce/backend/src/services/ai-assistant.service.ts index 1a8aafe0..e4be37cf 100644 --- a/meet-ce/backend/src/services/ai-assistant.service.ts +++ b/meet-ce/backend/src/services/ai-assistant.service.ts @@ -238,13 +238,14 @@ export class AiAssistantService { protected async stopCaptionsAssistantIfRunning(roomId: string): Promise { const assistants = await this.livekitService.listAgents(roomId); + + if (assistants.length === 0) return; + const captionsAssistant = assistants.find( (assistant) => assistant.agentName === INTERNAL_CONFIG.CAPTIONS_AGENT_NAME ); - if (!captionsAssistant) { - return; - } + if (!captionsAssistant) return; await this.livekitService.stopAgent(captionsAssistant.id, roomId); } diff --git a/meet-ce/backend/src/services/livekit.service.ts b/meet-ce/backend/src/services/livekit.service.ts index 676d00a4..363ad9a1 100644 --- a/meet-ce/backend/src/services/livekit.service.ts +++ b/meet-ce/backend/src/services/livekit.service.ts @@ -305,7 +305,7 @@ export class LiveKitService { return await this.agentClient.listDispatch(roomName); } catch (error) { this.logger.error(`Error listing agents for room '${roomName}':`, error); - throw internalError(`listing agents for room '${roomName}'`); + return []; } } @@ -320,7 +320,7 @@ export class LiveKitService { return await this.agentClient.getDispatch(agentId, roomName); } catch (error) { this.logger.error(`Error getting agent dispatch '${agentId}' for room '${roomName}':`, error); - throw internalError(`getting agent dispatch '${agentId}' for room '${roomName}'`); + return undefined; } } @@ -334,7 +334,6 @@ export class LiveKitService { await this.agentClient.deleteDispatch(agentId, roomName); } catch (error) { this.logger.error(`Error deleting agent dispatch '${agentId}' for room '${roomName}':`, error); - throw internalError(`deleting agent dispatch '${agentId}' for room '${roomName}'`); } } From 53f62708cef9465a7ff0907a06e06d9b8fbfce7a Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Wed, 4 Mar 2026 12:56:42 +0100 Subject: [PATCH 6/9] backend(test): increase sleep duration for webhook processing in expired rooms GC test --- meet-ce/backend/tests/helpers/request-helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meet-ce/backend/tests/helpers/request-helpers.ts b/meet-ce/backend/tests/helpers/request-helpers.ts index f435e58a..958c5833 100644 --- a/meet-ce/backend/tests/helpers/request-helpers.ts +++ b/meet-ce/backend/tests/helpers/request-helpers.ts @@ -386,7 +386,7 @@ export const runExpiredRoomsGC = async () => { const roomTaskScheduler = container.get(RoomScheduledTasksService); await roomTaskScheduler['deleteExpiredRooms'](); - await sleep('1s'); + await sleep('5s'); // TODO - replace with a more robust solution to ensure webhook is processed before proceeding with the tests }; /** From 9278260837d50ff2efdde25e1bc9200bd4f93d21 Mon Sep 17 00:00:00 2001 From: juancarmore Date: Wed, 18 Feb 2026 16:47:54 +0100 Subject: [PATCH 7/9] frontend: improve recording action buttons with consistent attributes and structure --- .../recording-lists.component.html | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/recordings/components/recording-lists/recording-lists.component.html b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/recordings/components/recording-lists/recording-lists.component.html index 11f45f5d..59d32ac2 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/recordings/components/recording-lists/recording-lists.component.html +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/recordings/components/recording-lists/recording-lists.component.html @@ -416,26 +416,34 @@ matTooltip="Play Recording" class="action-button play-button" (click)="playRecording(recording)" - [disabled]="loading()" - id="play-recording-btn-{{ recording.recordingId }}" - > - play_arrow - - } + [disabled]="loading()" + id="play-recording-btn-{{ recording.recordingId }}" + > + play_arrow + + } - - @if (canDownloadRecording(recording)) { - + } + + @if (isRecordingFailed(recording)) { +