From 848cf2ca1735d9b0ed0cef975e59ba5a5266d914 Mon Sep 17 00:00:00 2001 From: juancarmore Date: Sun, 23 Nov 2025 17:22:25 +0100 Subject: [PATCH] test: refactor stale recordings cleanup tests for improved clarity and functionality --- .../stale-recordings-cleanup.test.ts | 551 ------------------ .../recordings/stale-recordings-gc.test.ts | 410 +++++++++++++ 2 files changed, 410 insertions(+), 551 deletions(-) delete mode 100644 meet-ce/backend/tests/integration/api/recordings/stale-recordings-cleanup.test.ts create mode 100644 meet-ce/backend/tests/integration/api/recordings/stale-recordings-gc.test.ts diff --git a/meet-ce/backend/tests/integration/api/recordings/stale-recordings-cleanup.test.ts b/meet-ce/backend/tests/integration/api/recordings/stale-recordings-cleanup.test.ts deleted file mode 100644 index 0f967b13..00000000 --- a/meet-ce/backend/tests/integration/api/recordings/stale-recordings-cleanup.test.ts +++ /dev/null @@ -1,551 +0,0 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { MeetRecordingInfo, MeetRecordingStatus } from '@openvidu-meet/typings'; -import { EgressInfo, EgressStatus, Room } from 'livekit-server-sdk'; -import ms from 'ms'; -import { container } from '../../../../src/config/index.js'; -import { INTERNAL_CONFIG } from '../../../../src/config/internal-config.js'; -import { MeetLock } from '../../../../src/helpers/index.js'; -import { LiveKitService, LoggerService, MutexService, RecordingService } from '../../../../src/services/index.js'; -import { startTestServer } from '../../../helpers/request-helpers.js'; - -describe('Recording Cleanup Tests', () => { - let recordingService: RecordingService; - let mutexService: MutexService; - let livekitService: LiveKitService; - - const getRecordingLock = (roomId: string) => MeetLock.getRecordingActiveLock(roomId); - - const testRooms = { - recentLock: 'room-recent-lock', - withPublishers: 'room-with-publishers', - withoutPublishersWithRecording: 'room-without-publishers-with-recording', - withoutPublishersNoRecording: 'room-without-publishers-no-recording', - nonExistentWithRecording: 'room-non-existent-with-recording', - nonExistentNoRecording: 'room-non-existent-no-recording', - staleRecording: 'room-stale-recording', - freshRecording: 'room-fresh-recording', - abortedRecording: 'room-aborted-recording', - noUpdatedAt: 'room-no-updated-at' - }; - - beforeAll(async () => { - await startTestServer(); - recordingService = container.get(RecordingService); - mutexService = container.get(MutexService); - livekitService = container.get(LiveKitService); - - // Mute logs for the test - const logger = container.get(LoggerService); - jest.spyOn(logger, 'debug').mockImplementation(() => {}); - jest.spyOn(logger, 'verbose').mockImplementation(() => {}); - jest.spyOn(logger, 'info').mockImplementation(() => {}); - jest.spyOn(logger, 'warn').mockImplementation(() => {}); - jest.spyOn(logger, 'error').mockImplementation(() => {}); - }); - - beforeEach(async () => { - // Clean up any existing locks before each test - for (const roomId of Object.values(testRooms)) { - try { - await mutexService.release(getRecordingLock(roomId)); - } catch (e) { - // Ignore errors if the lock does not exist - } - } - - // Setup spies - jest.spyOn(mutexService, 'getLocksByPrefix'); - jest.spyOn(mutexService, 'lockExists'); - jest.spyOn(mutexService, 'getLockCreatedAt'); - jest.spyOn(mutexService, 'release'); - jest.spyOn(livekitService, 'roomExists'); - jest.spyOn(livekitService, 'getRoom'); - jest.spyOn(livekitService, 'getInProgressRecordingsEgress'); - jest.spyOn(livekitService, 'stopEgress'); - jest.spyOn(recordingService as never, 'performActiveRecordingLocksGC'); - jest.spyOn(recordingService as never, 'evaluateAndReleaseOrphanedLock'); - jest.spyOn(recordingService as never, 'performStaleRecordingsGC'); - jest.spyOn(recordingService as never, 'evaluateAndAbortStaleRecording'); - jest.spyOn(recordingService, 'getRecording'); - jest.spyOn(recordingService as never, 'updateRecordingStatus'); - - jest.clearAllMocks(); - - // Do not set up global mocks here to improve test isolation - }); - - afterEach(async () => { - // Clean up all spies and its invocations - jest.clearAllMocks(); - jest.restoreAllMocks(); - - // Explicitly restore the mock behavior for getLockCreatedAt - if (mutexService.getLockCreatedAt && jest.isMockFunction(mutexService.getLockCreatedAt)) { - (mutexService.getLockCreatedAt as jest.Mock).mockReset(); - } - }); - - afterAll(async () => { - // Clean up all test locks - for (const roomId of Object.values(testRooms)) { - try { - await mutexService.release(getRecordingLock(roomId)); - } catch (e) { - // Ignore errors if the lock does not exist - } - } - - // Restore all mocks - jest.restoreAllMocks(); - }); - - /** - * Creates a mock EgressInfo object for testing - */ - function createMockEgressInfo( - roomId: string, - egressId: string, - status: EgressStatus, - updatedAt?: number - ): EgressInfo { - const uid = '1234567890'; - return { - egressId, - roomId, - roomName: roomId, - status, - updatedAt: updatedAt ? BigInt(updatedAt * 1_000_000) : undefined, // Convert to nanoseconds - startedAt: BigInt(Date.now() * 1_000_000), - endedAt: status === EgressStatus.EGRESS_COMPLETE ? BigInt(Date.now() * 1_000_000) : undefined, - fileResults: [ - { - filename: `${roomId}--${uid}.mp4`, - size: BigInt(1024 * 1024), // 1MB - duration: BigInt(60 * 1_000_000_000) // 60 seconds in nanoseconds - } - ], - streamResults: [], - request: { - case: 'roomComposite' - } - } as unknown as EgressInfo; - } - - /** - * Creates a mock MeetRecordingInfo object for testing - */ - function createMockRecordingInfo( - recordingId: string, - roomId: string, - status: MeetRecordingStatus - ): MeetRecordingInfo { - return { - recordingId, - roomId, - roomName: roomId, - status, - startDate: Date.now() - ms('10m'), - filename: `${recordingId}.mp4` - }; - } - - describe('Perform Stale Recordings Cleanup', () => { - it('should not process any recordings when there are no in-progress recordings', async () => { - // Simulate empty response from LiveKit - (livekitService.getInProgressRecordingsEgress as jest.Mock).mockResolvedValueOnce([] as never); - - // Execute the stale recordings cleanup - await recordingService['performStaleRecordingsGC'](); - - // Verify that we checked for recordings but didn't attempt to process any - expect(livekitService.getInProgressRecordingsEgress).toHaveBeenCalled(); - expect((recordingService as never)['evaluateAndAbortStaleRecording']).not.toHaveBeenCalled(); - }); - - it('should gracefully handle errors during in-progress recordings retrieval', async () => { - // Simulate LiveKit service failure - (livekitService.getInProgressRecordingsEgress as jest.Mock).mockRejectedValueOnce( - new Error('Failed to retrieve recordings') as never - ); - - // Execute the stale recordings cleanup - should not throw - await recordingService['performStaleRecordingsGC'](); - - // Verify the error was handled properly without further processing - expect(livekitService.getInProgressRecordingsEgress).toHaveBeenCalled(); - expect((recordingService as never)['evaluateAndAbortStaleRecording']).not.toHaveBeenCalled(); - }); - - it('should process each in-progress recording to detect and abort stale ones', async () => { - const roomIds = [testRooms.staleRecording, testRooms.freshRecording, testRooms.abortedRecording]; - const mockEgressInfos = roomIds.map((roomId) => - createMockEgressInfo(roomId, `EG_${roomId}`, EgressStatus.EGRESS_ACTIVE) - ); - - // Simulate existing in-progress recordings in the system - (livekitService.getInProgressRecordingsEgress as jest.Mock).mockResolvedValueOnce(mockEgressInfos as never); - - // Execute the stale recordings cleanup - await recordingService['performStaleRecordingsGC'](); - - // Verify that each recording was processed individually - expect((recordingService as never)['evaluateAndAbortStaleRecording']).toHaveBeenCalledTimes(3); - mockEgressInfos.forEach((egressInfo) => { - expect((recordingService as never)['evaluateAndAbortStaleRecording']).toHaveBeenCalledWith(egressInfo); - }); - }); - }); - - describe('Evaluate and Abort Stale Recording', () => { - it('should skip processing if the recording is already aborted', async () => { - const roomId = testRooms.abortedRecording; - const egressId = `EG_${roomId}`; - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE); - - // Mock recording as already aborted - const mockRecordingInfo = createMockRecordingInfo( - `${roomId}--${egressId}--1234567890`, - roomId, - MeetRecordingStatus.ABORTED - ); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the method returned true (already aborted) - expect(result).toBe(true); - expect(recordingService.getRecording).toHaveBeenCalled(); - expect(livekitService.roomExists).not.toHaveBeenCalled(); - expect(recordingService['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should skip processing if the recording has no updatedAt timestamp', async () => { - const roomId = testRooms.noUpdatedAt; - const egressId = `EG_${roomId}`; - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE); // No updatedAt - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo( - `${roomId}--${egressId}--1234567890`, - roomId, - MeetRecordingStatus.ACTIVE - ); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the method returned false (kept as fresh) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalled(); - expect(livekitService.roomExists).not.toHaveBeenCalled(); - expect(recordingService['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should keep recording as fresh if it has been updated recently', async () => { - const roomId = testRooms.freshRecording; - const egressId = `EG_${roomId}`; - const recentUpdateTime = Date.now() - ms('1m'); // 1 minute ago (fresh) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, recentUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo( - `${roomId}--${egressId}--1234567890`, - roomId, - MeetRecordingStatus.ACTIVE - ); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the method returned false (still fresh) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalled(); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - expect(recordingService['updateRecordingStatus']).not.toHaveBeenCalled(); - }); - - it('should abort recording if room does not exist and recording updated time is stale', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(false as never); - (livekitService.stopEgress as jest.Mock).mockResolvedValueOnce({} as never); - ((recordingService as never)['updateRecordingStatus'] as jest.Mock).mockResolvedValueOnce( - undefined as never - ); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was aborted - expect(result).toBe(true); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).toHaveBeenCalledWith( - recordingId, - MeetRecordingStatus.ABORTED - ); - expect(livekitService.stopEgress).toHaveBeenCalledWith(egressId); - }); - - it('should abort recording if room exists with no publishers and updated time is stale', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValue(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValue(true as never); // Room exists - (livekitService.getRoom as jest.Mock).mockResolvedValue({ - numParticipants: 0, - numPublishers: 0 - } as Room as never); - (livekitService.stopEgress as jest.Mock).mockResolvedValue({} as never); - ((recordingService as never)['updateRecordingStatus'] as jest.Mock).mockResolvedValueOnce( - undefined as never - ); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was aborted - expect(result).toBe(true); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).toHaveBeenCalledWith( - recordingId, - MeetRecordingStatus.ABORTED - ); - expect(livekitService.stopEgress).toHaveBeenCalledWith(egressId); - }); - - it('should keep recording if it has been updated recently regardless room existence', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, Date.now()); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValue(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); // Room exists - - // Execute evaluateAndAbortStaleRecording - let result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was kept fresh (not aborted) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(false as never); // Room exists - - // Execute evaluateAndAbortStaleRecording - result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was kept fresh (not aborted) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should keep recording if room exists with publishers even when updated time is stale', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); - (livekitService.getRoom as jest.Mock).mockResolvedValueOnce({ - numParticipants: 1, - numPublishers: 1 - } as Room as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was kept fresh (not aborted) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should keep recording if room exists with no publishers but updated time is recent', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const recentUpdateTime = Date.now() - ms('1m'); // 1 minute ago (recent) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, recentUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); - (livekitService.getRoom as jest.Mock).mockResolvedValueOnce({ - numParticipants: 1, - numPublishers: 0 - } as Room as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was kept fresh (not aborted) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should keep recording if room exists with publishers and updated time is recent', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const recentUpdateTime = Date.now() - ms('1m'); // 1 minute ago (recent) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, recentUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); - (livekitService.getRoom as jest.Mock).mockResolvedValueOnce({ - numParticipants: 1, - numPublishers: 1 - } as Room as never); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was kept fresh (not aborted) - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalled(); - }); - - it('should handle edge case when updatedAt is exactly on the staleAfterMs threshold', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - - // Use Jest fake timers to precisely control Date.now() - jest.useFakeTimers(); - const now = 1_000_000; - jest.setSystemTime(now); - const staleUpdateTime = now - ms(INTERNAL_CONFIG.RECORDING_STALE_GRACE_PERIOD); - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(false as never); - (livekitService.stopEgress as jest.Mock).mockResolvedValueOnce({} as never); - ((recordingService as never)['updateRecordingStatus'] as jest.Mock).mockResolvedValueOnce( - undefined as never - ); - - // Execute evaluateAndAbortStaleRecording - const result = await recordingService['evaluateAndAbortStaleRecording'](egressInfo); - - // Verify that the recording was aborted - expect(result).toBe(false); - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect((recordingService as never)['updateRecordingStatus']).not.toHaveBeenCalled(); - expect(livekitService.stopEgress).not.toHaveBeenCalledWith(egressId); - }); - - it('should handle errors during recording processing and rethrow them', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE); - - // Mock error during recording retrieval - (recordingService.getRecording as jest.Mock).mockRejectedValueOnce( - new Error('Recording not found in storage') as never - ); - - // Execute evaluateAndAbortStaleRecording and expect error to propagate - await expect(recordingService['evaluateAndAbortStaleRecording'](egressInfo)).rejects.toThrow( - 'Recording not found in storage' - ); - - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - }); - - it('should handle errors during recording abort and rethrow them', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(false as never); - (livekitService.stopEgress as jest.Mock).mockRejectedValueOnce(new Error('Failed to stop egress') as never); - - // Execute evaluateAndAbortStaleRecording and expect error to propagate - await expect(recordingService['evaluateAndAbortStaleRecording'](egressInfo)).rejects.toThrow( - 'Failed to stop egress' - ); - - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect(livekitService.stopEgress).toHaveBeenCalledWith(egressId); - }); - - it('should handle case where updatedAt is in the future due to clock skew', async () => { - const roomId = testRooms.staleRecording; - const egressId = `EG_${roomId}`; - const recordingId = `${roomId}--${egressId}--1234567890`; - const staleUpdateTime = Date.now() + ms('10m'); // 10 minutes in the future (not stale) - const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); - - // Mock recording as active - const mockRecordingInfo = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); - (recordingService.getRecording as jest.Mock).mockResolvedValueOnce(mockRecordingInfo as never); - (livekitService.roomExists as jest.Mock).mockResolvedValueOnce(true as never); - (livekitService.getRoom as jest.Mock).mockResolvedValueOnce({ - numParticipants: 0, - numPublishers: 0 - } as Room as never); - - // Execute evaluateAndAbortStaleRecording and expect it to resolve to false - await expect(recordingService['evaluateAndAbortStaleRecording'](egressInfo)).resolves.toBe(false); - - expect(recordingService.getRecording).toHaveBeenCalledWith(recordingId); - expect(livekitService.roomExists).toHaveBeenCalledWith(roomId); - expect(livekitService.getRoom).not.toBeCalled(); - }); - }); -}); diff --git a/meet-ce/backend/tests/integration/api/recordings/stale-recordings-gc.test.ts b/meet-ce/backend/tests/integration/api/recordings/stale-recordings-gc.test.ts new file mode 100644 index 00000000..864fb21e --- /dev/null +++ b/meet-ce/backend/tests/integration/api/recordings/stale-recordings-gc.test.ts @@ -0,0 +1,410 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { MeetRecordingInfo, MeetRecordingStatus } from '@openvidu-meet/typings'; +import { SpiedFunction } from 'jest-mock'; +import { EgressInfo, EgressStatus } from 'livekit-server-sdk'; +import ms from 'ms'; +import { container } from '../../../../src/config/dependency-injector.config.js'; +import { INTERNAL_CONFIG } from '../../../../src/config/internal-config.js'; +import { RecordingRepository } from '../../../../src/repositories/recording.repository.js'; +import { LiveKitService } from '../../../../src/services/livekit.service.js'; +import { LoggerService } from '../../../../src/services/logger.service.js'; +import { RecordingService } from '../../../../src/services/recording.service.js'; +import { startTestServer } from '../../../helpers/request-helpers.js'; + +describe('Stale Recordings GC Tests', () => { + let recordingService: RecordingService; + + // Mock functions + let findActiveRecordingsMock: SpiedFunction<() => Promise>; + let roomExistsMock: SpiedFunction<(roomId: string) => Promise>; + let roomHasParticipantsMock: SpiedFunction<(roomId: string) => Promise>; + let getInProgressRecordingsEgressMock: SpiedFunction<() => Promise>; + let stopEgressMock: SpiedFunction<(egressId: string) => Promise>; + let evaluateAndAbortStaleRecordingMock: SpiedFunction<(recording: MeetRecordingInfo) => Promise>; + let updateRecordingStatusMock: SpiedFunction<(recordingId: string, status: MeetRecordingStatus) => Promise>; + + beforeAll(async () => { + await startTestServer(); + recordingService = container.get(RecordingService); + const recordingRepository = container.get(RecordingRepository); + const livekitService = container.get(LiveKitService); + + // Mute logs for the test + const logger = container.get(LoggerService); + jest.spyOn(logger, 'debug').mockImplementation(() => {}); + jest.spyOn(logger, 'verbose').mockImplementation(() => {}); + jest.spyOn(logger, 'info').mockImplementation(() => {}); + jest.spyOn(logger, 'warn').mockImplementation(() => {}); + jest.spyOn(logger, 'error').mockImplementation(() => {}); + + // Setup spies and store mock references + findActiveRecordingsMock = jest.spyOn(recordingRepository, 'findActiveRecordings'); + roomExistsMock = jest.spyOn(livekitService, 'roomExists'); + roomHasParticipantsMock = jest.spyOn(livekitService, 'roomHasParticipants'); + getInProgressRecordingsEgressMock = jest.spyOn(livekitService, 'getInProgressRecordingsEgress'); + stopEgressMock = jest.spyOn(livekitService, 'stopEgress'); + evaluateAndAbortStaleRecordingMock = jest.spyOn(recordingService as never, 'evaluateAndAbortStaleRecording'); + updateRecordingStatusMock = jest.spyOn(recordingService as never, 'updateRecordingStatus'); + }); + + beforeEach(() => { + // Reset common mocks to default implementations + updateRecordingStatusMock.mockResolvedValue(); + stopEgressMock.mockResolvedValue({} as EgressInfo); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + /** + * Creates a mock EgressInfo object for testing + */ + function createMockEgressInfo( + roomId: string, + egressId: string, + status: EgressStatus, + updatedAt?: number + ): EgressInfo { + const uid = '1234567890'; + return { + egressId, + roomId, + roomName: roomId, + status, + updatedAt: updatedAt ? BigInt(updatedAt * 1_000_000) : undefined, // Convert to nanoseconds + startedAt: BigInt(Date.now() * 1_000_000), + endedAt: status === EgressStatus.EGRESS_COMPLETE ? BigInt(Date.now() * 1_000_000) : undefined, + fileResults: [ + { + filename: `${roomId}--${uid}.mp4`, + size: BigInt(1024 * 1024), // 1MB + duration: BigInt(60 * 1_000_000_000) // 60 seconds in nanoseconds + } + ], + streamResults: [], + request: { + case: 'roomComposite' + } + } as unknown as EgressInfo; + } + + /** + * Creates a mock MeetRecordingInfo object for testing + */ + function createMockRecordingInfo( + recordingId: string, + roomId: string, + status: MeetRecordingStatus + ): MeetRecordingInfo { + return { + recordingId, + roomId, + roomName: roomId, + status, + startDate: Date.now() - ms('10m'), + filename: `${recordingId}.mp4` + }; + } + + describe('performStaleRecordingsGC', () => { + it('should not process any recordings when there are no active recordings in database', async () => { + // Mock empty response from database + findActiveRecordingsMock.mockResolvedValueOnce([]); + + // Execute the stale recordings cleanup + await recordingService['performStaleRecordingsGC'](); + + // Verify that we checked for recordings but didn't attempt to process any + expect(findActiveRecordingsMock).toHaveBeenCalled(); + expect(evaluateAndAbortStaleRecordingMock).not.toHaveBeenCalled(); + }); + + it('should gracefully handle errors during active recordings retrieval from database', async () => { + // Simulate database failure + findActiveRecordingsMock.mockRejectedValueOnce(new Error('Failed to retrieve recordings')); + + // Execute the stale recordings cleanup - should not throw + await recordingService['performStaleRecordingsGC'](); + + // Verify the error was handled properly without further processing + expect(findActiveRecordingsMock).toHaveBeenCalled(); + expect(evaluateAndAbortStaleRecordingMock).not.toHaveBeenCalled(); + }); + + it('should process each active recording from database to detect and abort stale ones', async () => { + const mockRecordings: MeetRecordingInfo[] = [ + createMockRecordingInfo('room-1--EG_1--uid1', 'room-1', MeetRecordingStatus.ACTIVE), + createMockRecordingInfo('room-2--EG_2--uid2', 'room-2', MeetRecordingStatus.ACTIVE), + createMockRecordingInfo('room-3--EG_3--uid3', 'room-3', MeetRecordingStatus.ENDING) + ]; + + // Mock database response with active recordings + findActiveRecordingsMock.mockResolvedValueOnce(mockRecordings); + + // Mock that no egress exists for any recording (all stale) + getInProgressRecordingsEgressMock.mockResolvedValue([]); + + // Execute the stale recordings cleanup + await recordingService['performStaleRecordingsGC'](); + + // Verify that each recording was processed individually + expect(evaluateAndAbortStaleRecordingMock).toHaveBeenCalledTimes(3); + mockRecordings.forEach((recording) => { + expect(evaluateAndAbortStaleRecordingMock).toHaveBeenCalledWith(recording); + }); + }); + }); + + describe('evaluateAndAbortStaleRecording', () => { + it('should abort recording immediately if no corresponding egress exists in LiveKit', async () => { + const roomId = 'test-room'; + const recordingId = `${roomId}--EG_test--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + + // Mock no egress found in LiveKit + getInProgressRecordingsEgressMock.mockResolvedValueOnce([]); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the recording was aborted without calling stopEgress + expect(result).toBe(true); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(updateRecordingStatusMock).toHaveBeenCalledWith(recordingId, MeetRecordingStatus.ABORTED); + expect(stopEgressMock).not.toHaveBeenCalled(); + expect(roomExistsMock).not.toHaveBeenCalled(); + }); + + it('should skip processing if the recording has no updatedAt timestamp', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE); // No updatedAt + + // Mock egress found but without updatedAt + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the method returned false (kept as fresh) + expect(result).toBe(false); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).not.toHaveBeenCalled(); + expect(updateRecordingStatusMock).not.toHaveBeenCalled(); + expect(stopEgressMock).not.toHaveBeenCalled(); + }); + + it('should keep recording as fresh if it has been updated recently', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const recentUpdateTime = Date.now() - ms('1m'); // 1 minute ago (fresh) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, recentUpdateTime); + + // Mock egress found with recent update + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(true); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the method returned false (still fresh) + expect(result).toBe(false); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(stopEgressMock).not.toHaveBeenCalled(); + expect(updateRecordingStatusMock).not.toHaveBeenCalled(); + }); + + it('should abort recording if room does not exist and recording update time is stale', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); + + // Mock egress found with stale update and room doesn't exist + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(false); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the recording was aborted + expect(result).toBe(true); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(updateRecordingStatusMock).toHaveBeenCalledWith(recordingId, MeetRecordingStatus.ABORTED); + expect(stopEgressMock).toHaveBeenCalledWith(egressId); + }); + + it('should abort recording if room exists with no participants and updated time is stale', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); + + // Mock egress found, room exists but has no participants + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(true); + roomHasParticipantsMock.mockResolvedValueOnce(false); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the recording was aborted + expect(result).toBe(true); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(roomHasParticipantsMock).toHaveBeenCalledWith(roomId); + expect(updateRecordingStatusMock).toHaveBeenCalledWith(recordingId, MeetRecordingStatus.ABORTED); + expect(stopEgressMock).toHaveBeenCalledWith(egressId); + }); + + it('should keep recording if room exists with participants even when updated time is stale', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); + + // Mock egress found, room exists with participants + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(true); + roomHasParticipantsMock.mockResolvedValueOnce(true); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the recording was kept fresh (not aborted) + expect(result).toBe(false); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(roomHasParticipantsMock).toHaveBeenCalledWith(roomId); + expect(updateRecordingStatusMock).not.toHaveBeenCalled(); + expect(stopEgressMock).not.toHaveBeenCalled(); + }); + + it('should handle edge case when updatedAt is exactly on the staleAfterMs threshold', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + + // Use Jest fake timers to precisely control Date.now() + jest.useFakeTimers(); + const now = 1_000_000; + jest.setSystemTime(now); + const staleUpdateTime = now - ms(INTERNAL_CONFIG.RECORDING_STALE_GRACE_PERIOD); + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); + + // Mock egress found at exact threshold + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(false); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the recording was kept fresh (threshold is not inclusive) + expect(result).toBe(false); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + + jest.useRealTimers(); + }); + + it('should handle errors during recording processing and rethrow them', async () => { + const roomId = 'test-room'; + const recordingId = `${roomId}--EG_test--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + + // Mock error during egress retrieval + getInProgressRecordingsEgressMock.mockRejectedValueOnce(new Error('LiveKit service unavailable')); + + // Execute evaluateAndAbortStaleRecording and expect error to propagate + await expect(recordingService['evaluateAndAbortStaleRecording'](recording)).rejects.toThrow( + 'LiveKit service unavailable' + ); + + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + }); + + it('should handle errors during recording abort and rethrow them', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const staleUpdateTime = Date.now() - ms('10m'); // 10 minutes ago (stale) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime); + + // Mock egress found with stale update and room doesn't exist + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(false); + stopEgressMock.mockRejectedValueOnce(new Error('Failed to stop egress')); + + // Execute evaluateAndAbortStaleRecording and expect error to propagate + await expect(recordingService['evaluateAndAbortStaleRecording'](recording)).rejects.toThrow( + 'Failed to stop egress' + ); + + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(stopEgressMock).toHaveBeenCalledWith(egressId); + }); + + it('should handle case where updatedAt is in the future due to clock skew', async () => { + const roomId = 'test-room'; + const egressId = 'EG_test'; + const recordingId = `${roomId}--${egressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const futureUpdateTime = Date.now() + ms('10m'); // 10 minutes in the future (not stale) + const egressInfo = createMockEgressInfo(roomId, egressId, EgressStatus.EGRESS_ACTIVE, futureUpdateTime); + + // Mock egress found with future timestamp + getInProgressRecordingsEgressMock.mockResolvedValueOnce([egressInfo]); + roomExistsMock.mockResolvedValueOnce(true); + + // Execute evaluateAndAbortStaleRecording and expect it to resolve to false + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + expect(result).toBe(false); + expect(getInProgressRecordingsEgressMock).toHaveBeenCalledWith(roomId); + expect(roomExistsMock).toHaveBeenCalledWith(roomId); + expect(roomHasParticipantsMock).not.toHaveBeenCalled(); + }); + + it('should correctly find matching egress when multiple egresses exist for the room', async () => { + const roomId = 'test-room'; + const targetEgressId = 'EG_target'; + const recordingId = `${roomId}--${targetEgressId}--1234567890`; + const recording = createMockRecordingInfo(recordingId, roomId, MeetRecordingStatus.ACTIVE); + const staleUpdateTime = Date.now() - ms('10m'); + + // Mock multiple egresses, only one matches + const mockEgresses = [ + createMockEgressInfo(roomId, 'EG_other1', EgressStatus.EGRESS_ACTIVE, staleUpdateTime), + createMockEgressInfo(roomId, targetEgressId, EgressStatus.EGRESS_ACTIVE, staleUpdateTime), + createMockEgressInfo(roomId, 'EG_other2', EgressStatus.EGRESS_ACTIVE, staleUpdateTime) + ]; + + getInProgressRecordingsEgressMock.mockResolvedValueOnce(mockEgresses); + roomExistsMock.mockResolvedValueOnce(false); + + // Execute evaluateAndAbortStaleRecording + const result = await recordingService['evaluateAndAbortStaleRecording'](recording); + + // Verify that the correct egress was targeted + expect(result).toBe(true); + expect(stopEgressMock).toHaveBeenCalledWith(targetEgressId); + expect(stopEgressMock).toHaveBeenCalledTimes(1); + }); + }); +});