From 5cdc49d90cc3086d1b6586d6ef51dd1cd7a6002e Mon Sep 17 00:00:00 2001 From: CSantosM <4a.santos@gmail.com> Date: Wed, 21 Jan 2026 12:45:06 +0100 Subject: [PATCH] frontend: Adds live captions component Introduces a live captions feature using LiveKit's transcription service. This adds a new component that displays real-time transcriptions of the meeting audio in. It manages caption lifecycles, handles both interim and final transcriptions, and provides reactive signals for UI updates. --- .../domains/meeting/customization/index.ts | 1 + .../meeting-captions.component.html | 23 + .../meeting-captions.component.scss | 202 +++++++++ .../meeting-captions.component.ts | 106 +++++ .../meeting-custom-layout.component.html | 83 ++-- .../meeting-custom-layout.component.scss | 10 +- .../meeting-custom-layout.component.ts | 28 +- ...eting-toolbar-extra-buttons.component.html | 24 ++ ...eting-toolbar-extra-buttons.component.scss | 4 + ...meeting-toolbar-extra-buttons.component.ts | 16 +- .../domains/meeting/models/captions.model.ts | 73 ++++ .../src/lib/domains/meeting/models/index.ts | 2 + .../pages/meeting/meeting.component.ts | 13 + .../src/lib/domains/meeting/services/index.ts | 1 + .../services/meeting-captions.service.ts | 400 ++++++++++++++++++ 15 files changed, 936 insertions(+), 50 deletions(-) create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.html create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.scss create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.ts create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/captions.model.ts create mode 100644 meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/index.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/index.ts index cd6ad358..188667b2 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/index.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/index.ts @@ -1,3 +1,4 @@ +export * from './meeting-captions/meeting-captions.component'; export * from './meeting-custom-layout/meeting-custom-layout.component'; export * from './meeting-invite-panel/meeting-invite-panel.component'; export * from './meeting-participant-item/meeting-participant-item.component'; diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.html b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.html new file mode 100644 index 00000000..0257acb4 --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.html @@ -0,0 +1,23 @@ +
+
+ @for (caption of captions(); track trackByCaption($index, caption)) { +
+ +
+
+ + {{ caption.participantName }}: + + {{ caption.text }} +
+
+
+ } +
+
diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.scss b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.scss new file mode 100644 index 00000000..cbe14515 --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.scss @@ -0,0 +1,202 @@ +@use '../../../../../../../../src/assets/styles/design-tokens'; + +// ============================================================================= +// CAPTIONS CONTAINER - Responsive to panel width changes +// ============================================================================= + +.captions-container { + pointer-events: none; + justify-content: center; + padding: 0px; + width: 100%; + height: var(--ov-footer-height, 250px) !important; + // Transition for smooth width changes when panel opens/closes + transition: padding 200ms ease-in-out; + + @include design-tokens.ov-tablet-down { + bottom: 100px; + } + + @include design-tokens.ov-mobile-down { + bottom: 90px; + } +} + +// ============================================================================= +// CAPTIONS WRAPPER +// ============================================================================= + +.captions-wrapper { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; // Center vertically when single caption + gap: 8px; + padding: 20px; + max-width: calc(100% - 40px); + width: calc(100% - 40px); + height: 100%; + margin: auto; + + // CRITICAL: Fixed max-height to prevent overflow + // This is the total available space for ALL captions combined + max-height: 230px; + overflow: hidden; // Never allow scroll or overflow + + // Wide screens: Limit width to prevent very long text lines + // Reduces eye movement required to follow captions + @media (min-width: 1400px) { + max-width: 900px; + padding: 20px 60px; + } + + @media (min-width: 1800px) { + max-width: 1000px; + padding: 20px 100px; + } + + @include design-tokens.ov-tablet-down { + max-width: calc(100% - 40px); + padding: 15px; + } + + @include design-tokens.ov-mobile-down { + padding: 10px; + } + + // When only one caption, it should fill available space and center + &.single-caption { + justify-content: center; + + .caption-item { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + } + } +} + +.caption-item { + pointer-events: auto; + display: inline-block; + text-align: center; + max-width: 100%; + padding: 0; + font-family: var(--ov-meet-font-family); + transition: + opacity 200ms ease-in-out, + transform 200ms ease-in-out; + + &.caption-entering { + opacity: 0; + transform: translateY(10px); + } + + &.caption-active { + opacity: 1; + transform: translateY(0); + } + + &.caption-leaving { + opacity: 0; + transform: translateY(-10px); + } + + &.caption-interim .caption-text { + opacity: 0.85; + } +} + +// ============================================================================= +// CAPTION TEXT CONTAINER - Scrollable wrapper with fixed height +// ============================================================================= + +.caption-text-container { + // CRITICAL: Fixed max-height with scroll to contain long monologues + max-height: 140px; + overflow-y: auto; + overflow-x: hidden; + + // Hide scrollbar but keep functionality (desktop only) + scrollbar-width: none; // Firefox + -ms-overflow-style: none; // IE/Edge + + &::-webkit-scrollbar { + display: none; // Chrome/Safari + } + + // Responsive max-heights + @include design-tokens.ov-tablet-down { + max-height: 100px; + } + + @include design-tokens.ov-mobile-down { + max-height: 100%; + // CRITICAL: Disable touch scroll on mobile to prevent accidental scrolling + overflow-y: hidden; + touch-action: none; + -webkit-overflow-scrolling: auto; + } +} + +// ============================================================================= +// CAPTION SPEAKER NAME - Inline bold text (not separate element) +// ============================================================================= + +.caption-speaker { + display: inline; + font-weight: 400; + font-size: 16px; + margin-right: 6px; + font-style: italic; + // Inherits all other styles from .caption-text parent +} + + +.caption-text { + display: inline; + font-size: 20px; + font-weight: 500; + line-height: 1.5; + color: #ffffff; + background-color: rgba(0, 0, 0, 0.7); + padding: 4px 8px; + border-radius: 4px; + -webkit-box-decoration-break: clone; + box-decoration-break: clone; + text-shadow: + 0 1px 3px rgba(0, 0, 0, 0.9), + 0 0 8px rgba(0, 0, 0, 0.5); + + // Ensure text wraps properly within the scrollable container + word-wrap: break-word; + overflow-wrap: break-word; + word-break: break-word; + + @include design-tokens.ov-tablet-down { + font-size: 18px; + padding: 3px 7px; + } + + @include design-tokens.ov-mobile-down { + font-size: 16px; + padding: 3px 6px; + line-height: 1.4; + } +} + +// ============================================================================= +// REDUCED MOTION SUPPORT +// ============================================================================= + +@media (prefers-reduced-motion: reduce) { + .caption-item { + transition: opacity 50ms ease-in-out; + + &.caption-entering, + &.caption-leaving { + transform: none; + } + } +} diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.ts new file mode 100644 index 00000000..6e6971e0 --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-captions/meeting-captions.component.ts @@ -0,0 +1,106 @@ +import { CommonModule } from '@angular/common'; +import { Component, effect, ElementRef, input, QueryList, signal, untracked, ViewChildren } from '@angular/core'; +import { Caption } from '../../models/captions.model'; + +@Component({ + selector: 'ov-meeting-captions', + imports: [CommonModule], + templateUrl: './meeting-captions.component.html', + styleUrl: './meeting-captions.component.scss' +}) +export class MeetingCaptionsComponent { + // Reactive caption data from service + captions = input([]); + + // Track animation state for each caption + protected readonly captionAnimationState = signal>(new Map()); + + // ViewChildren to access caption text containers for auto-scroll + @ViewChildren('captionTextContainer') + captionTextContainers!: QueryList>; + + constructor() { + // Monitor caption changes and update animation states + effect(() => { + const currentCaptions = this.captions(); + + // Use untracked to read current state without subscribing to it + const animationStates = new Map(untracked(() => this.captionAnimationState())); + + // Mark new captions as entering + currentCaptions.forEach((caption) => { + if (!animationStates.has(caption.id)) { + animationStates.set(caption.id, 'entering'); + // Transition to active after a brief delay + setTimeout(() => { + // Use untracked to avoid triggering the effect again + const states = new Map(untracked(() => this.captionAnimationState())); + states.set(caption.id, 'active'); + this.captionAnimationState.set(states); + }, 50); + } + }); + + // Remove states for captions that no longer exist + const currentIds = new Set(currentCaptions.map((c) => c.id)); + animationStates.forEach((_, id) => { + if (!currentIds.has(id)) { + animationStates.delete(id); + } + }); + + this.captionAnimationState.set(animationStates); + + // Auto-scroll to bottom of each caption after captions update + this.scrollCaptionsToBottom(); + }); + } + + /** + * Gets the CSS classes for a caption based on its state. + * + * @param caption The caption item + * @returns CSS class string + */ + protected getCaptionClasses(caption: Caption): string { + const classes: string[] = ['caption-item']; + + // Add final/interim class + classes.push(caption.isFinal ? 'caption-final' : 'caption-interim'); + + // Add animation state class + const animationState = this.captionAnimationState().get(caption.id); + if (animationState) { + classes.push(`caption-${animationState}`); + } + + return classes.join(' '); + } + + /** + * Tracks captions by their ID for optimal Angular rendering. + * + * @param index Item index + * @param caption Caption item + * @returns Unique identifier + */ + protected trackByCaption(index: number, caption: Caption): string { + return caption.id; + } + + /** + * Scrolls all caption text containers to the bottom to show the most recent text. + * Called automatically when captions are updated. + */ + private scrollCaptionsToBottom(): void { + // Use setTimeout to ensure DOM has updated + setTimeout(() => { + if (this.captionTextContainers) { + this.captionTextContainers.forEach((container: ElementRef) => { + const element = container.nativeElement; + element.scrollTop = element.scrollHeight; + }); + } + }, 20); + } +} diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.html b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.html index 986e44f1..ad07a49a 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.html +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.html @@ -1,40 +1,49 @@ @if (meetingContextService.lkRoom()) { - - @if (shouldShowLinkOverlay()) { - - - - } @else if (isSmartMosaicActive() && hiddenParticipantsCount() > 0) { - - -
- -
- - } +
+ + @if (shouldShowLinkOverlay()) { + + + + } @else if (isSmartMosaicActive() && hiddenParticipantsCount() > 0) { + + +
+ +
+ + } - - - - + + + + +
+ + @if (shouldShowCaptions()) { + + } } diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.scss b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.scss index 13da4604..0d0e0a2a 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.scss +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.scss @@ -1,5 +1,9 @@ @use '../../../../../../../../src/assets/styles/design-tokens'; +.main-container { + height: 100%; +} + .remote-participant { height: -webkit-fill-available; height: -moz-available; @@ -8,9 +12,11 @@ .container { height: 100%; } -.withCaptions { - height: calc(100% - var(--ov-captions-height, 250px)) !important; + +.withFooter { + height: calc(100% - var(--ov-footer-height, 250px)) !important; } + .withMargin { margin: 0px 5px; max-height: calc(100% - 5px) !important; diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.ts index c0630e5f..a1512edc 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-custom-layout/meeting-custom-layout.component.ts @@ -1,9 +1,21 @@ import { CommonModule } from '@angular/common'; import { Component, computed, effect, inject, signal, untracked } from '@angular/core'; -import { ILogger, LoggerService, OpenViduComponentsUiModule, PanelService, PanelType, ParticipantModel } from 'openvidu-components-angular'; -import { HiddenParticipantsIndicatorComponent, ShareMeetingLinkComponent } from '../../components'; -import { CustomParticipantModel } from '../../models'; -import { MeetingContextService, MeetingLayoutService, MeetingService } from '../../services'; +import { + ILogger, + LoggerService, + OpenViduComponentsUiModule, + PanelService, + PanelType, + ParticipantModel +} from 'openvidu-components-angular'; +import { HiddenParticipantsIndicatorComponent } from '../../components/hidden-participants-indicator/hidden-participants-indicator.component'; +import { ShareMeetingLinkComponent } from '../../components/share-meeting-link/share-meeting-link.component'; +import { CustomParticipantModel } from '../../models/custom-participant.model'; +import { MeetingCaptionsService } from '../../services/meeting-captions.service'; +import { MeetingContextService } from '../../services/meeting-context.service'; +import { MeetingLayoutService } from '../../services/meeting-layout.service'; +import { MeetingService } from '../../services/meeting.service'; +import { MeetingCaptionsComponent } from '../meeting-captions/meeting-captions.component'; @Component({ selector: 'ov-meeting-custom-layout', @@ -11,7 +23,8 @@ import { MeetingContextService, MeetingLayoutService, MeetingService } from '../ CommonModule, OpenViduComponentsUiModule, ShareMeetingLinkComponent, - HiddenParticipantsIndicatorComponent + HiddenParticipantsIndicatorComponent, + MeetingCaptionsComponent ], templateUrl: './meeting-custom-layout.component.html', styleUrl: './meeting-custom-layout.component.scss' @@ -22,6 +35,7 @@ export class MeetingCustomLayoutComponent { protected readonly meetingContextService = inject(MeetingContextService); protected readonly meetingService = inject(MeetingService); protected readonly panelService = inject(PanelService); + protected readonly captionsService = inject(MeetingCaptionsService); protected readonly linkOverlayConfig = { title: 'Start collaborating', subtitle: 'Share this link to bring others into the meeting', @@ -36,6 +50,10 @@ export class MeetingCustomLayoutComponent { return this.meetingContextService.canModerateRoom() && hasNoRemotes; }); + protected readonly shouldShowCaptions = computed(() => this.captionsService.areCaptionsEnabled()); + + protected readonly captions = computed(() => this.captionsService.captions()); + protected readonly isLayoutSwitchingAllowed = this.meetingContextService.allowLayoutSwitching; private displayedParticipantIds: string[] = []; diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html index 7353b467..bd189af0 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.html @@ -1,3 +1,27 @@ + +@if (isMobile()) { + + +} @else { + +} + @if (showCopyLinkButton()) { @if (isMobile()) { diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.scss b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.scss index 408d6661..8a820dcc 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.scss +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.scss @@ -1 +1,5 @@ // Additional toolbar buttons styling + +#captions-button.active { + background-color: var(--ov-accent-action-color); +} diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts index ee856d9e..7843c626 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/customization/meeting-toolbar-extra-buttons/meeting-toolbar-extra-buttons.component.ts @@ -5,6 +5,7 @@ import { MatIconModule } from '@angular/material/icon'; import { MatMenuModule } from '@angular/material/menu'; import { MatTooltipModule } from '@angular/material/tooltip'; import { LoggerService } from 'openvidu-components-angular'; +import { MeetingCaptionsService } from '../../services/meeting-captions.service'; import { MeetingContextService } from '../../services/meeting-context.service'; import { MeetingService } from '../../services/meeting.service'; @@ -22,6 +23,7 @@ export class MeetingToolbarExtraButtonsComponent { protected meetingContextService = inject(MeetingContextService); protected meetingService = inject(MeetingService); protected loggerService = inject(LoggerService); + protected captionService = inject(MeetingCaptionsService); protected log = this.loggerService.get('OpenVidu Meet - MeetingToolbarExtraButtons'); protected readonly copyLinkTooltip = 'Copy the meeting link'; protected readonly copyLinkText = 'Copy meeting link'; @@ -29,16 +31,14 @@ export class MeetingToolbarExtraButtonsComponent { /** * Whether to show the copy link button */ - protected showCopyLinkButton = computed(() => { - return this.meetingContextService.canModerateRoom(); - }); + protected showCopyLinkButton = computed(() => this.meetingContextService.canModerateRoom()); /** * Whether the device is mobile (affects button style) */ - protected isMobile = computed(() => { - return this.meetingContextService.isMobile(); - }); + protected isMobile = computed(() => this.meetingContextService.isMobile()); + + protected areCaptionsEnabled = computed(() => this.captionService.areCaptionsEnabled()); onCopyLinkClick(): void { const room = this.meetingContextService.meetRoom(); @@ -49,4 +49,8 @@ export class MeetingToolbarExtraButtonsComponent { this.meetingService.copyMeetingSpeakerLink(room); } + + onCaptionsClick(): void { + this.captionService.areCaptionsEnabled() ? this.captionService.disable() : this.captionService.enable(); + } } diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/captions.model.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/captions.model.ts new file mode 100644 index 00000000..af6ae29f --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/captions.model.ts @@ -0,0 +1,73 @@ +/** + * Represents a single caption entry with participant information + */ +export interface Caption { + /** + * Unique identifier for the caption + */ + id: string; + + /** + * Participant's identity (unique identifier) + */ + participantIdentity: string; + + /** + * Participant's display name + */ + participantName: string; + + /** + * Participant's color profile for visual representation + */ + participantColor: string; + + /** + * The transcribed text content + */ + text: string; + + /** + * Whether this is a final transcription or interim + */ + isFinal: boolean; + + /** + * The track ID being transcribed + */ + trackId: string; + + /** + * Timestamp when the caption was created + */ + timestamp: number; +} + +/** + * Configuration options for the captions display + */ +export interface CaptionsConfig { + /** + * Maximum number of captions to display simultaneously + * @default 3 + */ + maxVisibleCaptions?: number; + + /** + * Time in milliseconds before a final caption auto-expires + * @default 5000 + */ + finalCaptionDuration?: number; + + /** + * Time in milliseconds before an interim caption auto-expires + * @default 3000 + */ + interimCaptionDuration?: number; + + /** + * Whether to show interim transcriptions (partial results) + * @default true + */ + showInterimTranscriptions?: boolean; +} diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/index.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/index.ts index 418e23ae..7959167a 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/index.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/models/index.ts @@ -1,3 +1,5 @@ +export * from './captions.model'; export * from './custom-participant.model'; export * from './layout.model'; export * from './lobby.model'; + diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/pages/meeting/meeting.component.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/pages/meeting/meeting.component.ts index 97367155..05c657bd 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/pages/meeting/meeting.component.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/pages/meeting/meeting.component.ts @@ -20,6 +20,7 @@ import { NotificationService } from '../../../../shared/services/notification.se import { RoomMemberService } from '../../../rooms/services/room-member.service'; import { MeetingLobbyComponent } from '../../components/meeting-lobby/meeting-lobby.component'; import { MeetingParticipantItemComponent } from '../../customization/meeting-participant-item/meeting-participant-item.component'; +import { MeetingCaptionsService } from '../../services/meeting-captions.service'; import { MeetingContextService } from '../../services/meeting-context.service'; import { MeetingEventHandlerService } from '../../services/meeting-event-handler.service'; import { MeetingLobbyService } from '../../services/meeting-lobby.service'; @@ -76,6 +77,7 @@ export class MeetingComponent implements OnInit { protected lobbyService = inject(MeetingLobbyService); protected meetingContextService = inject(MeetingContextService); protected eventHandlerService = inject(MeetingEventHandlerService); + protected captionsService = inject(MeetingCaptionsService); protected destroy$ = new Subject(); // === LOBBY PHASE COMPUTED SIGNALS (when showLobby = true) === @@ -150,6 +152,9 @@ export class MeetingComponent implements OnInit { // Clear meeting context when component is destroyed this.meetingContextService.clearContext(); + + // Cleanup captions service + this.captionsService.destroy(); } // async onRoomConnected() { @@ -177,6 +182,14 @@ export class MeetingComponent implements OnInit { // Store LiveKit room in context this.meetingContextService.setLkRoom(lkRoom); + // Initialize captions service + this.captionsService.initialize(lkRoom, { + maxVisibleCaptions: 3, + finalCaptionDuration: 5000, + interimCaptionDuration: 3000, + showInterimTranscriptions: true + }); + // Setup LK room event listeners this.eventHandlerService.setupRoomListeners(lkRoom); } diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/index.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/index.ts index f5be6baa..73494d1b 100644 --- a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/index.ts +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/index.ts @@ -1,3 +1,4 @@ +export * from './meeting-captions.service'; export * from './meeting-context.service'; export * from './meeting-event-handler.service'; export * from './meeting-layout.service'; diff --git a/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts new file mode 100644 index 00000000..ad523233 --- /dev/null +++ b/meet-ce/frontend/projects/shared-meet-components/src/lib/domains/meeting/services/meeting-captions.service.ts @@ -0,0 +1,400 @@ +import { Injectable, inject, signal } from '@angular/core'; +import { ILogger, LoggerService, ParticipantService, Room, TextStreamReader } from 'openvidu-components-angular'; +import { Caption, CaptionsConfig } from '../models/captions.model'; +import { CustomParticipantModel } from '../models/custom-participant.model'; + +/** + * Service responsible for managing live transcription captions. + * + * This service: + * - Registers text stream handlers for LiveKit transcriptions + * - Manages caption lifecycle (creation, updates, expiration) + * - Handles both interim and final transcriptions + * - Provides reactive signals for UI consumption + * + * Follows the single responsibility principle by focusing solely on caption management. + */ +@Injectable({ + providedIn: 'root' +}) +export class MeetingCaptionsService { + private readonly loggerService = inject(LoggerService); + private readonly logger: ILogger; + private readonly participantService = inject(ParticipantService); + + // Configuration with defaults + private readonly defaultConfig: Required = { + maxVisibleCaptions: 3, + finalCaptionDuration: 5000, + interimCaptionDuration: 3000, + showInterimTranscriptions: true + }; + + private config: Required = { ...this.defaultConfig }; + + // Store room reference for dynamic subscription + private room: Room | null = null; + + // Reactive state + private readonly _captions = signal([]); + private readonly _isEnabled = signal(false); + + // Public readonly signals + readonly captions = this._captions.asReadonly(); + readonly areCaptionsEnabled = this._isEnabled.asReadonly(); + + // Map to track expiration timeouts + private expirationTimeouts = new Map>(); + + // Map to track interim captions by participant and track + private interimCaptionMap = new Map(); // key: `${participantIdentity}-${trackId}` + + constructor() { + this.logger = this.loggerService.get('OpenVidu Meet - MeetingCaptionsService'); + } + /** + * Initializes the captions service by registering text stream handlers. + * + * @param room The LiveKit Room instance + * @param config Optional configuration for caption behavior + */ + initialize(room: Room, config?: CaptionsConfig): void { + if (!room) { + this.logger.e('Cannot initialize captions: room is undefined'); + return; + } + + // Store room reference + this.room = room; + + // Merge provided config with defaults + this.config = { ...this.defaultConfig, ...config }; + + this.logger.d('Meeting Captions service initialized (ready to subscribe)'); + } + + /** + * Enables captions by registering the transcription handler. + * This is called when the user activates captions. + */ + enable(): void { + if (!this.room) { + this.logger.e('Cannot enable captions: room is not initialized'); + return; + } + + if (this._isEnabled()) { + this.logger.d('Captions already enabled'); + return; + } + + // Register the LiveKit transcription handler + this.room.registerTextStreamHandler('lk.transcription', this.handleTranscription.bind(this)); + + this._isEnabled.set(true); + this.logger.d('Captions enabled'); + } + + /** + * Disables captions by clearing all captions and stopping transcription. + * This is called when the user deactivates captions. + */ + disable(): void { + if (!this._isEnabled()) { + this.logger.d('Captions already disabled'); + return; + } + + // Clear all active captions + this.clearAllCaptions(); + + this._isEnabled.set(false); + this.room?.unregisterTextStreamHandler('lk.transcription'); + this.logger.d('Captions disabled'); + } + + /** + * Cleans up all captions and timers. + */ + destroy(): void { + this.clearAllCaptions(); + this.room = null; + this._isEnabled.set(false); + this.logger.d('Meeting Captions service destroyed'); + } + + /** + * Manually clears all active captions. + */ + clearAllCaptions(): void { + // Clear all expiration timers + this.expirationTimeouts.forEach((timeout) => clearTimeout(timeout)); + this.expirationTimeouts.clear(); + this.interimCaptionMap.clear(); + + // Clear captions + this._captions.set([]); + this.logger.d('All captions cleared'); + } + + /** + * Handles incoming transcription data. + * + * @param data Transcription data from LiveKit + */ + private async handleTranscription( + reader: TextStreamReader, + { identity: participantIdentity }: { identity: string } + ): Promise { + try { + const text = await reader.readAll(); + const isFinal = reader.info.attributes?.['lk.transcription_final'] === 'true'; + const trackId = reader.info.attributes?.['lk.transcribed_track_id'] || ''; + + if (!text || text.trim() === '') { + return; + } + + // Get full participant model from ParticipantService + const participant = this.participantService.getParticipantByIdentity( + participantIdentity + ) as CustomParticipantModel; + if (!participant) { + this.logger.e(`Participant with identity ${participantIdentity} not found for transcription`); + return; + } + + // Generate a unique key for this participant's track + const key = `${participantIdentity}-${trackId}`; + + if (isFinal) { + // Handle final transcription + this.handleFinalTranscription(key, participant, text, trackId); + } else { + // Handle interim transcription (if enabled) + if (this.config.showInterimTranscriptions) { + this.handleInterimTranscription(key, participant, text, trackId); + } + } + } catch (error) { + this.logger.e('Error reading transcription stream:', error); + } + } + + /** + * Handles final transcription by creating or updating a caption. + * + * @param key Unique key for the participant's track + * @param participantIdentity Participant identity + * @param participantName Participant display name + * @param text Transcribed text + * @param trackId Track ID being transcribed + */ + private handleFinalTranscription( + key: string, + participant: CustomParticipantModel, + text: string, + trackId: string + ): void { + const currentCaptions = this._captions(); + + const displayName = participant?.name || participant?.identity; + + // Check if there's an interim caption for this key + const interimCaptionId = this.interimCaptionMap.get(key); + + if (interimCaptionId) { + // Update existing interim caption to final + const updatedCaptions = currentCaptions.map((caption) => { + if (caption.id === interimCaptionId) { + // Clear old expiration timer + this.clearExpirationTimer(caption.id); + + // Return updated caption + return { + ...caption, + text, + isFinal: true, + timestamp: Date.now(), + participantName: participant.name || participant.identity, + participantColor: participant.colorProfile + }; + } + return caption; + }); + + this._captions.set(updatedCaptions); + + // Set new expiration timer + this.setExpirationTimer(interimCaptionId, this.config.finalCaptionDuration); + + // Remove from interim map + this.interimCaptionMap.delete(key); + } else { + // Create new final caption + this.addNewCaption(participant, text, trackId, true); + } + + this.logger.d(`Final transcription for ${displayName}: "${text}"`); + } + + /** + * Handles interim transcription by creating or updating a caption. + * + * @param key Unique key for the participant's track + * @param participantIdentity Participant identity + * @param participantName Participant display name + * @param text Transcribed text + * @param trackId Track ID being transcribed + */ + private handleInterimTranscription( + key: string, + participant: CustomParticipantModel, + text: string, + trackId: string + ): void { + const currentCaptions = this._captions(); + const participantName = participant.name || participant.identity; + + // Check if there's already an interim caption for this key + const existingInterimId = this.interimCaptionMap.get(key); + + if (existingInterimId) { + // Update existing interim caption + const updatedCaptions = currentCaptions.map((caption) => { + if (caption.id === existingInterimId) { + // Clear old expiration timer + this.clearExpirationTimer(caption.id); + + // Return updated caption + return { + ...caption, + text, + timestamp: Date.now(), + participantName: participant.name || participant.identity, + participantColor: participant.colorProfile + }; + } + return caption; + }); + + this._captions.set(updatedCaptions); + + // Reset expiration timer + this.setExpirationTimer(existingInterimId, this.config.interimCaptionDuration); + } else { + // Create new interim caption + const captionId = this.addNewCaption(participant, text, trackId, false); + + // Track this interim caption + this.interimCaptionMap.set(key, captionId); + } + + this.logger.d(`Interim transcription for ${participantName}: "${text}"`); + } + + /** + * Adds a new caption to the list. + * + * @param participantIdentity Participant identity + * @param participantName Participant display name + * @param text Transcribed text + * @param trackId Track ID being transcribed + * @param isFinal Whether this is a final transcription + * @returns The ID of the created caption + */ + private addNewCaption( + participant: CustomParticipantModel, + text: string, + trackId: string, + isFinal: boolean + ): string { + const caption: Caption = { + id: this.generateCaptionId(), + participantIdentity: participant.identity, + participantName: participant.name || participant.identity, + participantColor: participant.colorProfile, + text, + isFinal, + trackId, + timestamp: Date.now() + }; + + const currentCaptions = this._captions(); + + // Add new caption and limit total number + const updatedCaptions = [...currentCaptions, caption].slice(-this.config.maxVisibleCaptions); + + this._captions.set(updatedCaptions); + + // Set expiration timer + const duration = isFinal ? this.config.finalCaptionDuration : this.config.interimCaptionDuration; + this.setExpirationTimer(caption.id, duration); + + return caption.id; + } + + /** + * Sets an expiration timer for a caption. + * + * @param captionId Caption ID + * @param duration Duration in milliseconds + */ + private setExpirationTimer(captionId: string, duration: number): void { + // Clear existing timer if any + this.clearExpirationTimer(captionId); + + // Set new timer + const timeout = setTimeout(() => { + this.removeCaption(captionId); + }, duration); + + this.expirationTimeouts.set(captionId, timeout); + } + + /** + * Clears the expiration timer for a caption. + * + * @param captionId Caption ID + */ + private clearExpirationTimer(captionId: string): void { + const timeout = this.expirationTimeouts.get(captionId); + if (timeout) { + clearTimeout(timeout); + this.expirationTimeouts.delete(captionId); + } + } + + /** + * Removes a caption from the list. + * + * @param captionId Caption ID to remove + */ + private removeCaption(captionId: string): void { + this.clearExpirationTimer(captionId); + + const currentCaptions = this._captions(); + const updatedCaptions = currentCaptions.filter((caption) => caption.id !== captionId); + + this._captions.set(updatedCaptions); + + // Clean up interim map if necessary + for (const [key, id] of this.interimCaptionMap.entries()) { + if (id === captionId) { + this.interimCaptionMap.delete(key); + break; + } + } + + this.logger.d(`Caption ${captionId} removed`); + } + + /** + * Generates a unique caption ID. + * + * @returns Unique caption ID + */ + private generateCaptionId(): string { + return `caption-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + } +}