From 2c03ecdd9af1af54dca25851066bf225693f3531 Mon Sep 17 00:00:00 2001 From: Carlos Santos <4a.santos@gmail.com> Date: Thu, 29 May 2025 13:29:42 +0200 Subject: [PATCH] backend: implement bulk deletion of recording metadata files in StorageProvider and S3StorageProvider --- backend/src/services/recording.service.ts | 57 ++++++++++++++----- .../storage/providers/s3-storage.provider.ts | 17 +++++- .../src/services/storage/storage.interface.ts | 8 +-- .../src/services/storage/storage.service.ts | 11 ++++ 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/backend/src/services/recording.service.ts b/backend/src/services/recording.service.ts index d46ef9d..6537fbe 100644 --- a/backend/src/services/recording.service.ts +++ b/backend/src/services/recording.service.ts @@ -181,10 +181,25 @@ export class RecordingService { async deleteRecording(recordingId: string): Promise { try { // Get the recording metada and recording info from the S3 bucket - const { filesToDelete, recordingInfo } = await this.getDeletableRecordingFiles(recordingId); + const { binaryFilesToDelete, metadataFilesToDelete, recordingInfo } = + await this.getDeletableRecordingFiles(recordingId); const { roomId } = RecordingHelper.extractInfoFromRecordingId(recordingId); + const deleteRecordingTasks: Promise[] = []; + + if (binaryFilesToDelete.size > 0) { + // Delete video files from S3 + deleteRecordingTasks.push(this.s3Service.deleteObjects(Array.from(binaryFilesToDelete))); + } + + if (metadataFilesToDelete.size > 0) { + // Delete metadata files from storage provider + deleteRecordingTasks.push( + this.storageService.deleteRecordingMetadataByPaths(Array.from(metadataFilesToDelete)) + ); + } + + await Promise.all(deleteRecordingTasks); - await this.s3Service.deleteObjects(Array.from(filesToDelete)); this.logger.info(`Successfully deleted ${recordingId}`); const shouldDeleteRoomMetadata = await this.shouldDeleteRoomMetadata(roomId); @@ -211,7 +226,8 @@ export class RecordingService { async bulkDeleteRecordingsAndAssociatedFiles( recordingIds: string[] ): Promise<{ deleted: string[]; notDeleted: { recordingId: string; error: string }[] }> { - const allFilesToDelete: Set = new Set(); + let allMetadataFilesToDelete: Set = new Set(); + let allBinaryFilesToDelete: Set = new Set(); const deletedRecordings: Set = new Set(); const notDeletedRecordings: Set<{ recordingId: string; error: string }> = new Set(); const roomsToCheck: Set = new Set(); @@ -219,8 +235,12 @@ export class RecordingService { // Check if the recording is in progress for (const recordingId of recordingIds) { try { - const { filesToDelete } = await this.getDeletableRecordingFiles(recordingId); - filesToDelete.forEach((file) => allFilesToDelete.add(file)); + const { binaryFilesToDelete, metadataFilesToDelete } = + await this.getDeletableRecordingFiles(recordingId); + // Add files to the set of files to delete + allBinaryFilesToDelete = new Set([...allBinaryFilesToDelete, ...binaryFilesToDelete]); + allMetadataFilesToDelete = new Set([...allMetadataFilesToDelete, ...metadataFilesToDelete]); + deletedRecordings.add(recordingId); // Track the roomId for checking if the room metadata file should be deleted @@ -232,15 +252,18 @@ export class RecordingService { } } - if (allFilesToDelete.size === 0) { + if (allBinaryFilesToDelete.size === 0) { this.logger.warn(`BulkDelete: No eligible recordings found for deletion.`); return { deleted: Array.from(deletedRecordings), notDeleted: Array.from(notDeletedRecordings) }; } // Delete recordings and its metadata from S3 try { - await this.s3Service.deleteObjects(Array.from(allFilesToDelete)); - this.logger.info(`BulkDelete: Successfully deleted ${allFilesToDelete.size} objects from S3.`); + await Promise.all([ + this.s3Service.deleteObjects(Array.from(allBinaryFilesToDelete)), + this.storageService.deleteRecordingMetadataByPaths(Array.from(allMetadataFilesToDelete)) + ]); + this.logger.info(`BulkDelete: Successfully deleted ${allBinaryFilesToDelete.size} recordings.`); } catch (error) { this.logger.error(`BulkDelete: Error performing bulk deletion: ${error}`); throw error; @@ -262,7 +285,7 @@ export class RecordingService { try { this.logger.verbose(`Deleting room_metadata.json for rooms: ${roomMetadataToDelete.join(', ')}`); await Promise.all(deleteTasks); - this.logger.verbose(`BulkDelete: Successfully deleted ${allFilesToDelete.size} room metadata files.`); + this.logger.verbose(`BulkDelete: Successfully deleted ${roomMetadataToDelete.length} room metadata files.`); } catch (error) { this.logger.error(`BulkDelete: Error performing bulk deletion: ${error}`); throw error; @@ -501,11 +524,14 @@ export class RecordingService { * * @param recordingId - The unique identifier of the recording egress. */ - protected async getDeletableRecordingFiles( - recordingId: string - ): Promise<{ filesToDelete: Set; recordingInfo: MeetRecordingInfo }> { + protected async getDeletableRecordingFiles(recordingId: string): Promise<{ + binaryFilesToDelete: Set; + metadataFilesToDelete: Set; + recordingInfo: MeetRecordingInfo; + }> { const { metadataFilePath, recordingInfo } = await this.storageService.getRecordingMetadata(recordingId); - const filesToDelete: Set = new Set(); + const binaryFilesToDelete: Set = new Set(); + const metadataFilesToDelete: Set = new Set(); // Validate the recording status if (!RecordingHelper.canBeDeleted(recordingInfo)) throw errorRecordingNotStopped(recordingId); @@ -517,9 +543,10 @@ export class RecordingService { } const recordingPath = `${INTERNAL_CONFIG.S3_RECORDINGS_PREFIX}/${filename}`; - filesToDelete.add(recordingPath).add(metadataFilePath); + binaryFilesToDelete.add(recordingPath); + metadataFilesToDelete.add(metadataFilePath); - return { filesToDelete, recordingInfo }; + return { binaryFilesToDelete, metadataFilesToDelete, recordingInfo }; } // protected async getMeetRecordingInfoFromMetadata( diff --git a/backend/src/services/storage/providers/s3-storage.provider.ts b/backend/src/services/storage/providers/s3-storage.provider.ts index a043cbf..72e180c 100644 --- a/backend/src/services/storage/providers/s3-storage.provider.ts +++ b/backend/src/services/storage/providers/s3-storage.provider.ts @@ -394,7 +394,22 @@ export class S3StorageProvider< } } - async deleteRecordingMetadata(recordingId: string): Promise {} + /** + * Deletes multiple recording metadata files from S3 storage based on their file paths. + * + * @param metadataPaths - Array of file paths pointing to the metadata files to be deleted + * @returns A promise that resolves when all metadata files have been successfully deleted + * @throws May throw an error if any of the deletion operations fail + */ + async deleteRecordingMetadataByPaths(metadataPaths: string[]): Promise { + try { + await this.s3Service.deleteObjects(metadataPaths); + this.logger.verbose(`Deleted multiple recording metadata files: ${metadataPaths.join(', ')}`); + } catch (error) { + this.handleError(error, `Error deleting multiple recording metadata files: ${metadataPaths.join(', ')}`); + throw error; + } + } /** * Retrieves an object of type U from Redis by the given key. diff --git a/backend/src/services/storage/storage.interface.ts b/backend/src/services/storage/storage.interface.ts index f42caa5..453dbf3 100644 --- a/backend/src/services/storage/storage.interface.ts +++ b/backend/src/services/storage/storage.interface.ts @@ -135,10 +135,10 @@ export interface StorageProvider< getRecordingMetadata(recordingId: string): Promise<{ recordingInfo: MRec; metadataFilePath: string }>; /** - * Deletes the recording metadata for a specific recording ID. + * Deletes multiple recording metadata files by their paths. * - * @param recordingId - The unique identifier of the recording to delete. - * @returns A promise that resolves when the deletion is complete. + * @param metadataPaths - An array of metadata file paths to delete. */ - deleteRecordingMetadata(recordingId: string): Promise; + deleteRecordingMetadataByPaths(metadataPaths: string[]): Promise; + } diff --git a/backend/src/services/storage/storage.service.ts b/backend/src/services/storage/storage.service.ts index 59ce560..2fd9e25 100644 --- a/backend/src/services/storage/storage.service.ts +++ b/backend/src/services/storage/storage.service.ts @@ -210,6 +210,17 @@ export class MeetStorageService< }>; } + /** + * Deletes multiple recording metadata files by their paths. + * + * @param metadataPaths - Array of file paths to the recording metadata files to be deleted + * @returns A Promise that resolves when all metadata files have been successfully deleted + * @throws May throw an error if any of the deletion operations fail + */ + async deleteRecordingMetadataByPaths(metadataPaths: string[]): Promise { + return this.storageProvider.deleteRecordingMetadataByPaths(metadataPaths); + } + /** * Returns the default global preferences. * @returns {GPrefs}