backend: Refactor storage service operations with global preferences

This commit is contained in:
Carlos Santos 2025-04-04 12:59:14 +02:00
parent 12ef04964c
commit 2c65ec1da8
6 changed files with 121 additions and 56 deletions

View File

@ -20,7 +20,7 @@ import {
UserService UserService
} from '../services/index.js'; } from '../services/index.js';
const container: Container = new Container(); export const container: Container = new Container();
/** /**
* Registers all necessary dependencies in the container. * Registers all necessary dependencies in the container.
@ -30,7 +30,7 @@ const container: Container = new Container();
* available for injection throughout the application. * available for injection throughout the application.
* *
*/ */
const registerDependencies = () => { export const registerDependencies = () => {
console.log('Registering CE dependencies'); console.log('Registering CE dependencies');
container.bind(SystemEventService).toSelf().inSingletonScope(); container.bind(SystemEventService).toSelf().inSingletonScope();
container.bind(MutexService).toSelf().inSingletonScope(); container.bind(MutexService).toSelf().inSingletonScope();
@ -52,14 +52,12 @@ const registerDependencies = () => {
container.bind(S3Storage).toSelf().inSingletonScope(); container.bind(S3Storage).toSelf().inSingletonScope();
container.bind(StorageFactory).toSelf().inSingletonScope(); container.bind(StorageFactory).toSelf().inSingletonScope();
initializeEagerServices();
}; };
const initializeEagerServices = () => { export const initializeEagerServices = async () => {
// Force the creation of services that need to be initialized at startup // Force the creation of services that need to be initialized at startup
container.get(RecordingService); container.get(RecordingService);
await container.get(MeetStorageService).buildAndSaveDefaultPreferences();
}; };
export { injectable, inject } from 'inversify'; export { injectable, inject } from 'inversify';
export { container, registerDependencies };

View File

@ -1,3 +1,11 @@
export const enum RedisKeyPrefix {
BASE = 'ov_meet:'
}
export const enum RedisKeyName {
GLOBAL_PREFERENCES = `${RedisKeyPrefix.BASE}global_preferences`
}
export const enum RedisLockPrefix { export const enum RedisLockPrefix {
BASE = 'ov_meet_lock:', BASE = 'ov_meet_lock:',
REGISTRY = 'ov_meet_lock_registry:' REGISTRY = 'ov_meet_lock_registry:'

View File

@ -1,7 +1,7 @@
import express, { Request, Response, Express } from 'express'; import express, { Request, Response, Express } from 'express';
import cors from 'cors'; import cors from 'cors';
import chalk from 'chalk'; import chalk from 'chalk';
import { registerDependencies, container } from './config/dependency-injector.config.js'; import { registerDependencies, initializeEagerServices } from './config/dependency-injector.config.js';
import { import {
SERVER_PORT, SERVER_PORT,
SERVER_CORS_ORIGIN, SERVER_CORS_ORIGIN,
@ -25,7 +25,6 @@ import {
recordingRouter, recordingRouter,
roomRouter roomRouter
} from './routes/index.js'; } from './routes/index.js';
import { MeetStorageService } from './services/index.js';
import { internalParticipantsRouter } from './routes/participants.routes.js'; import { internalParticipantsRouter } from './routes/participants.routes.js';
import cookieParser from 'cookie-parser'; import cookieParser from 'cookie-parser';
@ -75,12 +74,6 @@ const createApp = () => {
return app; return app;
}; };
const initializeGlobalPreferences = async () => {
const globalPreferencesService = container.get(MeetStorageService);
// TODO: This should be invoked in the constructor of the service
await globalPreferencesService.ensurePreferencesInitialized();
};
const startServer = (app: express.Application) => { const startServer = (app: express.Application) => {
app.listen(SERVER_PORT, async () => { app.listen(SERVER_PORT, async () => {
console.log(' '); console.log(' ');
@ -92,7 +85,6 @@ const startServer = (app: express.Application) => {
chalk.cyanBright(`http://localhost:${SERVER_PORT}${MEET_API_BASE_PATH_V1}/docs`) chalk.cyanBright(`http://localhost:${SERVER_PORT}${MEET_API_BASE_PATH_V1}/docs`)
); );
logEnvVars(); logEnvVars();
await Promise.all([initializeGlobalPreferences()]);
}); });
}; };
@ -118,6 +110,7 @@ if (isMainModule()) {
registerDependencies(); registerDependencies();
const app = createApp(); const app = createApp();
startServer(app); startServer(app);
await initializeEagerServices();
} }
export { registerDependencies, createApp, initializeGlobalPreferences }; export { registerDependencies, createApp };

View File

@ -95,7 +95,7 @@ export class S3Service {
Body: JSON.stringify(body) Body: JSON.stringify(body)
}); });
const result = await this.retryOperation<PutObjectCommandOutput>(() => this.run(command)); const result = await this.retryOperation<PutObjectCommandOutput>(() => this.run(command));
this.logger.info(`S3: successfully saved object '${fullKey}' in bucket '${bucket}'`); this.logger.verbose(`S3: successfully saved object '${fullKey}' in bucket '${bucket}'`);
return result; return result;
} catch (error: any) { } catch (error: any) {
this.logger.error(`S3: error saving object '${fullKey}' in bucket '${bucket}': ${error}`); this.logger.error(`S3: error saving object '${fullKey}' in bucket '${bucket}': ${error}`);
@ -175,9 +175,9 @@ export class S3Service {
searchPattern = '', searchPattern = '',
bucket: string = MEET_S3_BUCKET bucket: string = MEET_S3_BUCKET
): Promise<ListObjectsV2CommandOutput> { ): Promise<ListObjectsV2CommandOutput> {
// Se construye el prefijo completo combinando el subbucket y el additionalPrefix. // The complete prefix is constructed by combining the subbucket and the additionalPrefix.
// Ejemplo: si s3Subbucket es "recordings" y additionalPrefix es ".metadata/", // Example: if s3Subbucket is "recordings" and additionalPrefix is ".metadata/",
// se listarán los objetos con key que empiece por "recordings/.metadata/". // it will list objects with keys that start with "recordings/.metadata/".
const basePrefix = this.getFullKey(additionalPrefix); const basePrefix = this.getFullKey(additionalPrefix);
this.logger.verbose(`S3 listObjectsPaginated: listing objects with prefix "${basePrefix}"`); this.logger.verbose(`S3 listObjectsPaginated: listing objects with prefix "${basePrefix}"`);
@ -191,7 +191,8 @@ export class S3Service {
try { try {
const response: ListObjectsV2CommandOutput = await this.s3.send(command); const response: ListObjectsV2CommandOutput = await this.s3.send(command);
// Si se ha proporcionado searchPattern, se filtran los resultados. // If searchPattern is provided, filter the results.
if (searchPattern) { if (searchPattern) {
const regex = new RegExp(searchPattern); const regex = new RegExp(searchPattern);
response.Contents = (response.Contents || []).filter((item) => item.Key && regex.test(item.Key)); response.Contents = (response.Contents || []).filter((item) => item.Key && regex.test(item.Key));

View File

@ -5,8 +5,8 @@ import { LoggerService } from '../../logger.service.js';
import { RedisService } from '../../redis.service.js'; import { RedisService } from '../../redis.service.js';
import { OpenViduMeetError } from '../../../models/error.model.js'; import { OpenViduMeetError } from '../../../models/error.model.js';
import { inject, injectable } from '../../../config/dependency-injector.config.js'; import { inject, injectable } from '../../../config/dependency-injector.config.js';
import { MEET_S3_ROOMS_PREFIX } from '../../../environment.js'; import { MEET_S3_ROOMS_PREFIX, MEET_S3_SUBBUCKET } from '../../../environment.js';
import { RedisKeyName } from '../../../models/redis.model.js';
/** /**
* Implementation of the StorageProvider interface using AWS S3 for persistent storage * Implementation of the StorageProvider interface using AWS S3 for persistent storage
@ -29,43 +29,75 @@ import { MEET_S3_ROOMS_PREFIX } from '../../../environment.js';
export class S3Storage<G extends GlobalPreferences = GlobalPreferences, R extends MeetRoom = MeetRoom> export class S3Storage<G extends GlobalPreferences = GlobalPreferences, R extends MeetRoom = MeetRoom>
implements StorageProvider implements StorageProvider
{ {
protected readonly GLOBAL_PREFERENCES_KEY = 'openvidu-meet-preferences'; protected readonly S3_GLOBAL_PREFERENCES_KEY = `${MEET_S3_SUBBUCKET}/global-preferences.json`;
constructor( constructor(
@inject(LoggerService) protected logger: LoggerService, @inject(LoggerService) protected logger: LoggerService,
@inject(S3Service) protected s3Service: S3Service, @inject(S3Service) protected s3Service: S3Service,
@inject(RedisService) protected redisService: RedisService @inject(RedisService) protected redisService: RedisService
) {} ) {}
/**
* Initializes global preferences. If no preferences exist, persists the provided defaults.
* If preferences exist but belong to a different project, they are replaced.
*
* @param defaultPreferences - The default preferences to initialize with.
*/
async initialize(defaultPreferences: G): Promise<void> { async initialize(defaultPreferences: G): Promise<void> {
try {
const existingPreferences = await this.getGlobalPreferences(); const existingPreferences = await this.getGlobalPreferences();
if (existingPreferences) { if (!existingPreferences) {
if (existingPreferences.projectId !== defaultPreferences.projectId) { this.logger.info('No existing preferences found. Saving default preferences to S3.');
await this.saveGlobalPreferences(defaultPreferences);
return;
}
this.logger.verbose('Global preferences found. Checking project association...');
const isDifferentProject = existingPreferences.projectId !== defaultPreferences.projectId;
if (isDifferentProject) {
this.logger.warn( this.logger.warn(
`Existing preferences are associated with a different project (Project ID: ${existingPreferences.projectId}). Replacing them with the default preferences for the current project.` `Existing global preferences belong to project [${existingPreferences.projectId}], ` +
`which differs from current project [${defaultPreferences.projectId}]. Replacing preferences.`
); );
await this.saveGlobalPreferences(defaultPreferences); await this.saveGlobalPreferences(defaultPreferences);
return;
} }
} else {
this.logger.info('Saving default preferences to S3'); this.logger.verbose(
await this.saveGlobalPreferences(defaultPreferences); 'Global preferences for the current project are already initialized. No action needed.'
);
} catch (error) {
this.logger.error('Error during global preferences initialization:', error);
} }
} }
/**
* Retrieves the global preferences.
* First attempts to retrieve from Redis; if not available, falls back to S3.
* If fetched from S3, caches the result in Redis.
*
* @returns A promise that resolves to the global preferences or null if not found.
*/
async getGlobalPreferences(): Promise<G | null> { async getGlobalPreferences(): Promise<G | null> {
try { try {
let preferences: G | null = await this.getFromRedis<G>(this.GLOBAL_PREFERENCES_KEY); // Try to get preferences from Redis cache
let preferences: G | null = await this.getFromRedis<G>(RedisKeyName.GLOBAL_PREFERENCES);
if (!preferences) { if (!preferences) {
// Fallback to fetching from S3 if Redis doesn't have it this.logger.debug('Global preferences not found in Redis. Fetching from S3...');
this.logger.debug('Preferences not found in Redis. Fetching from S3...'); preferences = await this.getFromS3<G>(this.S3_GLOBAL_PREFERENCES_KEY);
preferences = await this.getFromS3<G>(`${this.GLOBAL_PREFERENCES_KEY}.json`);
if (preferences) { if (preferences) {
// TODO: Use a key prefix for Redis this.logger.verbose('Fetched global preferences from S3. Caching them in Redis.');
await this.redisService.set(this.GLOBAL_PREFERENCES_KEY, JSON.stringify(preferences), false); const redisPayload = JSON.stringify(preferences);
await this.redisService.set(RedisKeyName.GLOBAL_PREFERENCES, redisPayload, false);
} else {
this.logger.warn('No global preferences found in S3.');
} }
} else {
this.logger.verbose('Global preferences retrieved from Redis.');
} }
return preferences; return preferences;
@ -75,16 +107,26 @@ export class S3Storage<G extends GlobalPreferences = GlobalPreferences, R extend
} }
} }
/**
* Persists the global preferences to both S3 and Redis in parallel.
* Uses Promise.all to execute both operations concurrently.
*
* @param preferences - Global preferences to store.
* @returns The saved preferences.
* @throws Rethrows any error if saving fails.
*/
async saveGlobalPreferences(preferences: G): Promise<G> { async saveGlobalPreferences(preferences: G): Promise<G> {
try { try {
const redisPayload = JSON.stringify(preferences);
await Promise.all([ await Promise.all([
this.s3Service.saveObject(`${this.GLOBAL_PREFERENCES_KEY}.json`, preferences), this.s3Service.saveObject(this.S3_GLOBAL_PREFERENCES_KEY, preferences),
// TODO: Use a key prefix for Redis this.redisService.set(RedisKeyName.GLOBAL_PREFERENCES, redisPayload, false)
this.redisService.set(this.GLOBAL_PREFERENCES_KEY, JSON.stringify(preferences), false)
]); ]);
this.logger.info('Global preferences saved successfully');
return preferences; return preferences;
} catch (error) { } catch (error) {
this.handleError(error, 'Error saving preferences'); this.handleError(error, 'Error saving global preferences');
throw error; throw error;
} }
} }
@ -233,19 +275,37 @@ export class S3Storage<G extends GlobalPreferences = GlobalPreferences, R extend
} }
} }
/**
* Retrieves an object of type U from Redis by the given key.
* Returns null if the key is not found or an error occurs.
*
* @param key - The Redis key to fetch.
* @returns A promise that resolves to an object of type U or null.
*/
protected async getFromRedis<U>(key: string): Promise<U | null> { protected async getFromRedis<U>(key: string): Promise<U | null> {
let response: string | null = null; try {
const response = await this.redisService.get(key);
response = await this.redisService.get(key);
if (response) { if (response) {
return JSON.parse(response) as U; return JSON.parse(response) as U;
} }
return null; return null;
} catch (error) {
this.logger.error(`Error fetching from Redis for key ${key}: ${error}`);
return null;
}
} }
/**
* Retrieves an object of type U from S3 at the specified path.
* Returns null if the object is not found.
*
* @param path - The S3 key or path to fetch.
* @returns A promise that resolves to an object of type U or null.
*/
protected async getFromS3<U>(path: string): Promise<U | null> { protected async getFromS3<U>(path: string): Promise<U | null> {
try {
const response = await this.s3Service.getObjectAsJson(path); const response = await this.s3Service.getObjectAsJson(path);
if (response) { if (response) {
@ -254,6 +314,10 @@ export class S3Storage<G extends GlobalPreferences = GlobalPreferences, R extend
} }
return null; return null;
} catch (error) {
this.logger.error(`Error fetching from S3 for path ${path}: ${error}`);
return null;
}
} }
protected handleError(error: any, message: string) { protected handleError(error: any, message: string) {

View File

@ -30,10 +30,11 @@ export class MeetStorageService<G extends GlobalPreferences = GlobalPreferences,
* Initializes default preferences if not already initialized. * Initializes default preferences if not already initialized.
* @returns {Promise<G>} Default global preferences. * @returns {Promise<G>} Default global preferences.
*/ */
async ensurePreferencesInitialized(): Promise<G> { async buildAndSaveDefaultPreferences(): Promise<G> {
const preferences = await this.getDefaultPreferences(); const preferences = await this.getDefaultPreferences();
try { try {
this.logger.verbose('Initializing global preferences with default values');
await this.storageProvider.initialize(preferences); await this.storageProvider.initialize(preferences);
return preferences as G; return preferences as G;
} catch (error) { } catch (error) {
@ -51,7 +52,7 @@ export class MeetStorageService<G extends GlobalPreferences = GlobalPreferences,
if (preferences) return preferences as G; if (preferences) return preferences as G;
return await this.ensurePreferencesInitialized(); return await this.buildAndSaveDefaultPreferences();
} }
/** /**
@ -117,12 +118,12 @@ export class MeetStorageService<G extends GlobalPreferences = GlobalPreferences,
} }
/** /**
* TODO: Move validation to the controller layer
* Updates room preferences in storage. * Updates room preferences in storage.
* @param {RoomPreferences} roomPreferences * @param {RoomPreferences} roomPreferences
* @returns {Promise<GlobalPreferences>} * @returns {Promise<GlobalPreferences>}
*/ */
async updateOpenViduRoomPreferences(roomId: string, roomPreferences: MeetRoomPreferences): Promise<R> { async updateOpenViduRoomPreferences(roomId: string, roomPreferences: MeetRoomPreferences): Promise<R> {
// TODO: Move validation to the controller layer
this.validateRoomPreferences(roomPreferences); this.validateRoomPreferences(roomPreferences);
const openviduRoom = await this.getOpenViduRoom(roomId); const openviduRoom = await this.getOpenViduRoom(roomId);