diff --git a/meet-ce/backend/src/config/internal-config.ts b/meet-ce/backend/src/config/internal-config.ts index b33b1a0f..9bef3257 100644 --- a/meet-ce/backend/src/config/internal-config.ts +++ b/meet-ce/backend/src/config/internal-config.ts @@ -1,4 +1,5 @@ import { StringValue } from 'ms'; +import { SchemaVersion } from '../models/migration.model.js'; export const INTERNAL_CONFIG = { // Base paths for the API @@ -45,7 +46,16 @@ export const INTERNAL_CONFIG = { // Additional intervals MIN_FUTURE_TIME_FOR_ROOM_AUTODELETION_DATE: '1h' as StringValue, // Minimum time for room auto-deletion date MEETING_EMPTY_TIMEOUT: (process.env.MEETING_EMPTY_TIMEOUT || '20s') as StringValue, // Seconds to keep the meeting (LK room) open until the first participant joins - MEETING_DEPARTURE_TIMEOUT: (process.env.MEETING_DEPARTURE_TIMEOUT || '20s') as StringValue // Seconds to keep the meeting (LK room) open after the last participant leaves + MEETING_DEPARTURE_TIMEOUT: (process.env.MEETING_DEPARTURE_TIMEOUT || '20s') as StringValue, // Seconds to keep the meeting (LK room) open after the last participant leaves + + // MongoDB Schema Versions + // These define the current schema version for each collection + // Increment when making breaking changes to the schema structure + GLOBAL_CONFIG_SCHEMA_VERSION: 1 as SchemaVersion, + USER_SCHEMA_VERSION: 1 as SchemaVersion, + API_KEY_SCHEMA_VERSION: 1 as SchemaVersion, + ROOM_SCHEMA_VERSION: 1 as SchemaVersion, + RECORDING_SCHEMA_VERSION: 1 as SchemaVersion }; // This function is used to set private configuration values for testing purposes. diff --git a/meet-ce/backend/src/migrations/README.md b/meet-ce/backend/src/migrations/README.md new file mode 100644 index 00000000..fadf16ab --- /dev/null +++ b/meet-ce/backend/src/migrations/README.md @@ -0,0 +1,195 @@ +# MongoDB Schema Migration System + +This document explains the schema migration system implemented for OpenVidu Meet's MongoDB collections. + +--- + +## Overview + +The schema migration system enables safe evolution of MongoDB document structures over time. It handles scenarios like: + +- Adding new required fields with default values +- Removing deprecated fields +- Renaming fields +- Restructuring nested objects +- Data type transformations + +### Core Features + +- ✅ **Forward-only migrations** (v1 → v2 → v3) +- ✅ **Automatic execution at startup** (before accepting requests) +- ✅ **HA-safe** (distributed locking prevents concurrent migrations) +- ✅ **Batch processing** (efficient handling of large collections) +- ✅ **Progress tracking** (migrations stored in `MeetMigration` collection) +- ✅ **Version validation** (optional runtime checks in repositories) + +--- + +## Architecture + +### Schema Version Field + +Each document includes a `schemaVersion` field: + +```typescript +{ + schemaVersion: 1, // Current version (starts at 1) + roomId: "room-123", + roomName: "My Room", + // ... other fields +} +``` + +**Important**: `schemaVersion` is **internal only** and stripped from API responses via Mongoose schema transforms. + +### Migration Components + +``` +src/ +├── migrations/ +│ ├── base-migration.ts # Base class for migrations +│ ├── migration-registry.ts # Central registry of all collections +│ ├── room-migrations.ts # Room-specific migrations +│ ├── recording-migrations.ts # Recording-specific migrations +│ ├── user-migrations.ts # User-specific migrations +│ ├── api-key-migrations.ts # API key-specific migrations +│ ├── global-config-migrations.ts # Global config-specific migrations +│ └── index.ts # Exports +└── models/ + └── migration.model.ts # Migration types and interfaces +``` + +**Note**: All migration types and interfaces (`ISchemaMigration`, `MigrationContext`, `MigrationResult`, `SchemaVersion`, `CollectionMigrationRegistry`) are defined in `src/models/migration.model.ts` for better code organization. + +--- + +## Adding New Migrations + +### Step 1: Update Schema Version in Configuration + +In `src/config/internal-config.ts`, increment the version constant: + +```typescript +// internal-config.ts +export const INTERNAL_CONFIG = { + // ... other config + ROOM_SCHEMA_VERSION: 2 // Was 1 + // ... +}; +``` + +### Step 2: Create Migration Class + +```typescript +import { BaseSchemaMigration } from './base-migration.js'; +import { MeetRoomDocument } from '../repositories/schemas/room.schema.js'; +import { MigrationContext } from '../models/migration.model.js'; +import { Model } from 'mongoose'; + +class RoomMigrationV1ToV2 extends BaseSchemaMigration { + fromVersion = 1; + toVersion = 2; + description = 'Add maxParticipants field with default value of 100'; + + protected async transform(document: MeetRoomDocument): Promise> { + // Return fields to update (schemaVersion is handled automatically) + return { + maxParticipants: 100 + }; + } + + // Optional: Add validation before migration runs + async validate(model: Model, context: MigrationContext): Promise { + // Check prerequisites, data integrity, etc. + return true; + } +} +``` + +### Step 3: Register Migration + +Add the migration instance to the migrations array in `room-migrations.ts`: + +```typescript +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetRoomDocument } from '../repositories/schemas/room.schema.js'; + +export const roomMigrations: ISchemaMigration[] = [ + new RoomMigrationV1ToV2() + // Future migrations will be added here +]; +``` + +### Step 4: Update Schema Definition + +Update the Mongoose schema default version in `internal-config.ts`: + +```typescript +// config/internal-config.ts +export const INTERNAL_CONFIG = { + // ... other config + ROOM_SCHEMA_VERSION: 2 // Updated from 1 + // ... +}; +``` + +If adding new required fields, update the Mongoose schema: + +```typescript +// repositories/schemas/room.schema.ts +import { INTERNAL_CONFIG } from '../../config/internal-config.js'; + +const MeetRoomSchema = new Schema({ + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.ROOM_SCHEMA_VERSION // Uses config value (2) + }, + // ... existing fields ... + maxParticipants: { type: Number, required: true, default: 100 } // New field +}); +``` + +### Step 5: Update TypeScript Interface + +Update the domain interface to include new fields: + +```typescript +// typings/src/room.ts +export interface MeetRoom extends MeetRoomOptions { + roomId: string; + // ... existing fields ... + maxParticipants: number; // New field +} +``` + +### Step 6: Test Migration + +1. Start application - migration runs automatically +2. Check logs for migration execution +3. Verify documents in MongoDB have correct version +4. Test API to ensure new field appears correctly + +--- + +## Migration Tracking + +Each migration is tracked in the `MeetMigration` collection: + +```json +{ + "name": "schema_room_v1_to_v2", + "status": "completed", + "startedAt": 1700000000000, + "completedAt": 1700000123000, + "metadata": { + "collectionName": "MeetRoom", + "fromVersion": 1, + "toVersion": 2, + "migratedCount": 1523, + "skippedCount": 0, + "failedCount": 0, + "durationMs": 123000 + } +} +``` diff --git a/meet-ce/backend/src/migrations/api-key-migrations.ts b/meet-ce/backend/src/migrations/api-key-migrations.ts new file mode 100644 index 00000000..7957583a --- /dev/null +++ b/meet-ce/backend/src/migrations/api-key-migrations.ts @@ -0,0 +1,24 @@ +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetApiKeyDocument } from '../repositories/schemas/api-key.schema.js'; + +/** + * All migrations for the MeetApiKey collection in chronological order. + * Add new migrations to this array as the schema evolves. + * + * Example migration (when needed in the future): + * + * class ApiKeyMigrationV1ToV2 extends BaseSchemaMigration { + * fromVersion = 1; + * toVersion = 2; + * description = 'Add expirationDate field for API key expiration'; + * + * protected async transform(document: MeetApiKeyDocument): Promise> { + * return { + * expirationDate: undefined // No expiration for existing keys + * }; + * } + * } + */ +export const apiKeyMigrations: ISchemaMigration[] = [ + // Migrations will be added here as the schema evolves +]; diff --git a/meet-ce/backend/src/migrations/base-migration.ts b/meet-ce/backend/src/migrations/base-migration.ts new file mode 100644 index 00000000..69794eb4 --- /dev/null +++ b/meet-ce/backend/src/migrations/base-migration.ts @@ -0,0 +1,124 @@ +import { Model } from 'mongoose'; +import { ISchemaMigration, MigrationContext, MigrationResult, SchemaVersion } from '../models/migration.model.js'; + +/** + * Base class for schema migrations providing common functionality. + * Extend this class to implement specific migrations for collections. + */ +export abstract class BaseSchemaMigration implements ISchemaMigration { + abstract fromVersion: SchemaVersion; + abstract toVersion: SchemaVersion; + abstract description: string; + + /** + * Default batch size for processing documents. + * Can be overridden in subclasses for collections with large documents. + */ + protected readonly defaultBatchSize = 50; + + /** + * Executes the migration in batches. + * Processes all documents at fromVersion and upgrades them to toVersion. + */ + async execute(model: Model, context: MigrationContext): Promise { + const startTime = Date.now(); + const batchSize = context.batchSize || this.defaultBatchSize; + let migratedCount = 0; + const skippedCount = 0; + let failedCount = 0; + + context.logger.info( + `Starting schema migration: ${this.description} (v${this.fromVersion} -> v${this.toVersion})` + ); + + try { + // Find all documents at the source version + const totalDocs = await model.countDocuments({ schemaVersion: this.fromVersion }).exec(); + + if (totalDocs === 0) { + context.logger.info('No documents to migrate'); + return { + migratedCount: 0, + skippedCount: 0, + failedCount: 0, + durationMs: Date.now() - startTime + }; + } + + context.logger.info(`Found ${totalDocs} documents to migrate`); + + // Process documents in batches + let processedCount = 0; + + while (processedCount < totalDocs) { + const documents = await model.find({ schemaVersion: this.fromVersion }).limit(batchSize).exec(); + + if (documents.length === 0) { + break; + } + + // Transform and update each document + for (const doc of documents) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const updates = await this.transform(doc as any); + + // Update the document with new fields and version + await model + .updateOne( + { _id: doc._id }, + { + $set: { + ...updates, + schemaVersion: this.toVersion + } + } + ) + .exec(); + + migratedCount++; + } catch (error) { + failedCount++; + context.logger.warn(`Failed to migrate document ${doc._id}:`, error); + } + } + + processedCount += documents.length; + context.logger.debug(`Processed ${processedCount}/${totalDocs} documents`); + } + + const durationMs = Date.now() - startTime; + context.logger.info( + `Migration completed: ${migratedCount} migrated, ${failedCount} failed (${durationMs}ms)` + ); + + return { + migratedCount, + skippedCount, + failedCount, + durationMs + }; + } catch (error) { + context.logger.error('Migration failed:', error); + throw error; + } + } + + /** + * Transform a single document from source version to target version. + * Override this method to implement the specific transformation logic. + * + * @param document - The document to transform + * @returns Object with fields to update (excluding schemaVersion which is handled automatically) + */ + protected abstract transform(document: TDocument): Promise>; + + /** + * Optional validation before running migration. + * Default implementation always returns true. + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async validate(_model: Model, _context: MigrationContext): Promise { + return true; + } +} diff --git a/meet-ce/backend/src/migrations/global-config-migrations.ts b/meet-ce/backend/src/migrations/global-config-migrations.ts new file mode 100644 index 00000000..ea5a7c6f --- /dev/null +++ b/meet-ce/backend/src/migrations/global-config-migrations.ts @@ -0,0 +1,27 @@ +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetGlobalConfigDocument } from '../repositories/schemas/global-config.schema.js'; + +/** + * All migrations for the MeetGlobalConfig collection in chronological order. + * Add new migrations to this array as the schema evolves. + * + * Example migration (when needed in the future): + * + * class GlobalConfigMigrationV1ToV2 extends BaseSchemaMigration { + * fromVersion = 1; + * toVersion = 2; + * description = 'Add new branding configuration section'; + * + * protected async transform(document: MeetGlobalConfigDocument): Promise> { + * return { + * brandingConfig: { + * logoUrl: '', + * companyName: 'OpenVidu Meet' + * } + * }; + * } + * } + */ +export const globalConfigMigrations: ISchemaMigration[] = [ + // Migrations will be added here as the schema evolves +]; diff --git a/meet-ce/backend/src/migrations/index.ts b/meet-ce/backend/src/migrations/index.ts new file mode 100644 index 00000000..fbafe507 --- /dev/null +++ b/meet-ce/backend/src/migrations/index.ts @@ -0,0 +1,7 @@ +export * from './base-migration.js'; +export * from './migration-registry.js'; +export * from './room-migrations.js'; +export * from './recording-migrations.js'; +export * from './user-migrations.js'; +export * from './api-key-migrations.js'; +export * from './global-config-migrations.js'; diff --git a/meet-ce/backend/src/migrations/migration-registry.ts b/meet-ce/backend/src/migrations/migration-registry.ts new file mode 100644 index 00000000..0e43b637 --- /dev/null +++ b/meet-ce/backend/src/migrations/migration-registry.ts @@ -0,0 +1,58 @@ +import { INTERNAL_CONFIG } from '../config/internal-config.js'; +import { CollectionMigrationRegistry } from '../models/migration.model.js'; +import { MeetApiKeyModel, meetApiKeyCollectionName } from '../repositories/schemas/api-key.schema.js'; +import { MeetGlobalConfigModel, meetGlobalConfigCollectionName } from '../repositories/schemas/global-config.schema.js'; +import { MeetRecordingModel, meetRecordingCollectionName } from '../repositories/schemas/recording.schema.js'; +import { MeetRoomModel, meetRoomCollectionName } from '../repositories/schemas/room.schema.js'; +import { MeetUserModel, meetUserCollectionName } from '../repositories/schemas/user.schema.js'; +import { apiKeyMigrations } from './api-key-migrations.js'; +import { globalConfigMigrations } from './global-config-migrations.js'; +import { recordingMigrations } from './recording-migrations.js'; +import { roomMigrations } from './room-migrations.js'; +import { userMigrations } from './user-migrations.js'; + +/** + * Central registry of all collection migrations. + * Defines the current version and migration path for each collection. + * + * Order matters: collections should be listed in dependency order. + * For example, if recordings depend on rooms, rooms should come first. + */ +export const migrationRegistry: CollectionMigrationRegistry[] = [ + // GlobalConfig - no dependencies, can run first + { + collectionName: meetGlobalConfigCollectionName, + model: MeetGlobalConfigModel, + currentVersion: INTERNAL_CONFIG.GLOBAL_CONFIG_SCHEMA_VERSION, + migrations: globalConfigMigrations + }, + // User - no dependencies + { + collectionName: meetUserCollectionName, + model: MeetUserModel, + currentVersion: INTERNAL_CONFIG.USER_SCHEMA_VERSION, + migrations: userMigrations + }, + // ApiKey - no dependencies + { + collectionName: meetApiKeyCollectionName, + model: MeetApiKeyModel, + currentVersion: INTERNAL_CONFIG.API_KEY_SCHEMA_VERSION, + migrations: apiKeyMigrations + }, + // Room - no dependencies on other collections + { + collectionName: meetRoomCollectionName, + model: MeetRoomModel, + currentVersion: INTERNAL_CONFIG.ROOM_SCHEMA_VERSION, + migrations: roomMigrations + }, + // Recording - depends on Room (references roomId) + // Should be migrated after rooms + { + collectionName: meetRecordingCollectionName, + model: MeetRecordingModel, + currentVersion: INTERNAL_CONFIG.RECORDING_SCHEMA_VERSION, + migrations: recordingMigrations + } +]; diff --git a/meet-ce/backend/src/migrations/recording-migrations.ts b/meet-ce/backend/src/migrations/recording-migrations.ts new file mode 100644 index 00000000..ebbde3b7 --- /dev/null +++ b/meet-ce/backend/src/migrations/recording-migrations.ts @@ -0,0 +1,24 @@ +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetRecordingDocument } from '../repositories/schemas/recording.schema.js'; + +/** + * All migrations for the MeetRecording collection in chronological order. + * Add new migrations to this array as the schema evolves. + * + * Example migration (when needed in the future): + * + * class RecordingMigrationV1ToV2 extends BaseSchemaMigration { + * fromVersion = 1; + * toVersion = 2; + * description = 'Add new optional field "quality" for recording quality tracking'; + * + * protected async transform(document: MeetRecordingDocument): Promise> { + * return { + * quality: 'standard' // Default quality for existing recordings + * }; + * } + * } + */ +export const recordingMigrations: ISchemaMigration[] = [ + // Migrations will be added here as the schema evolves +]; diff --git a/meet-ce/backend/src/migrations/room-migrations.ts b/meet-ce/backend/src/migrations/room-migrations.ts new file mode 100644 index 00000000..033556e4 --- /dev/null +++ b/meet-ce/backend/src/migrations/room-migrations.ts @@ -0,0 +1,26 @@ +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetRoomDocument } from '../repositories/schemas/room.schema.js'; + +/** + * All migrations for the MeetRoom collection in chronological order. + * Add new migrations to this array as the schema evolves. + * + * Example migration (when needed in the future): + * + * class RoomMigrationV1ToV2 extends BaseSchemaMigration { + * fromVersion = 1; + * toVersion = 2; + * description = 'Add new required field "maxParticipants" with default value'; + * + * protected async transform(document: MeetRoomDocument): Promise> { + * return { + * maxParticipants: 100 // Add default value for existing rooms + * }; + * } + * } + */ +export const roomMigrations: ISchemaMigration[] = [ + // Migrations will be added here as the schema evolves + // Example: new RoomMigrationV1ToV2(), + // Example: new RoomMigrationV2ToV3(), +]; diff --git a/meet-ce/backend/src/migrations/user-migrations.ts b/meet-ce/backend/src/migrations/user-migrations.ts new file mode 100644 index 00000000..cc8a150a --- /dev/null +++ b/meet-ce/backend/src/migrations/user-migrations.ts @@ -0,0 +1,24 @@ +import { ISchemaMigration } from '../models/migration.model.js'; +import { MeetUserDocument } from '../repositories/schemas/user.schema.js'; + +/** + * All migrations for the MeetUser collection in chronological order. + * Add new migrations to this array as the schema evolves. + * + * Example migration (when needed in the future): + * + * class UserMigrationV1ToV2 extends BaseSchemaMigration { + * fromVersion = 1; + * toVersion = 2; + * description = 'Add email field for user notifications'; + * + * protected async transform(document: MeetUserDocument): Promise> { + * return { + * email: undefined // Email will be optional initially + * }; + * } + * } + */ +export const userMigrations: ISchemaMigration[] = [ + // Migrations will be added here as the schema evolves +]; diff --git a/meet-ce/backend/src/models/migration.model.ts b/meet-ce/backend/src/models/migration.model.ts index b8293212..28515487 100644 --- a/meet-ce/backend/src/models/migration.model.ts +++ b/meet-ce/backend/src/models/migration.model.ts @@ -1,3 +1,6 @@ +import { Model } from 'mongoose'; +import { LoggerService } from '../services/logger.service.js'; + /** * Interface representing a migration document in MongoDB. */ @@ -15,12 +18,12 @@ export interface MeetMigration { /** * Timestamp when the migration started. */ - startedAt: Date; + startedAt: number; /** * Timestamp when the migration completed (success or failure). */ - completedAt?: Date; + completedAt?: number; /** * Error message if the migration failed. @@ -34,18 +37,6 @@ export interface MeetMigration { metadata?: Record; } -/** - * Enum defining all possible migration names in the system. - * Each migration should have a unique identifier. - */ -export enum MigrationName { - /** - * Migration from legacy storage (S3, ABS, GCS) to MongoDB. - * Includes: GlobalConfig, Users, ApiKeys, Rooms, and Recordings. - */ - LEGACY_STORAGE_TO_MONGODB = 'legacy_storage_to_mongodb' -} - /** * Status of a migration execution. */ @@ -65,3 +56,117 @@ export enum MigrationStatus { */ FAILED = 'failed' } + +/** + * Enum defining all possible migration names in the system. + * Each migration should have a unique identifier. + * + * Schema migrations follow the pattern: schema_{collection}_v{from}_to_v{to} + * Example: 'schema_room_v1_to_v2', 'schema_recording_v2_to_v3' + */ +export enum MigrationName { + /** + * Migration from legacy storage (S3, ABS, GCS) to MongoDB. + * Includes: GlobalConfig, Users, ApiKeys, Rooms, and Recordings. + */ + LEGACY_STORAGE_TO_MONGODB = 'legacy_storage_to_mongodb' +} + +/** + * Generates a migration name for schema version upgrades. + * + * @param collectionName - Name of the collection (e.g., 'MeetRoom', 'MeetRecording') + * @param fromVersion - Source schema version + * @param toVersion - Target schema version + * @returns Migration name string + * + * @example + * generateSchemaMigrationName('MeetRoom', 1, 2) // Returns: 'schema_room_v1_to_v2' + */ +export function generateSchemaMigrationName(collectionName: string, fromVersion: number, toVersion: number): string { + // Convert collection name to lowercase and remove 'Meet' prefix + const simpleName = collectionName.replace(/^Meet/, '').toLowerCase(); + return `schema_${simpleName}_v${fromVersion}_to_v${toVersion}`; +} + +/** + * Represents a schema version number. + * Versions start at 1 and increment sequentially. + */ +export type SchemaVersion = number; + +/** + * Context provided to migration functions. + * Contains utilities and services needed during migration. + */ +export interface MigrationContext { + /** Logger service for tracking migration progress */ + logger: LoggerService; + /** Batch size for processing documents (default: 50) */ + batchSize?: number; +} + +/** + * Result of executing a migration. + * Provides statistics about the migration execution. + */ +export interface MigrationResult { + /** Number of documents successfully migrated */ + migratedCount: number; + /** Number of documents skipped (already at target version) */ + skippedCount: number; + /** Number of documents that failed migration */ + failedCount: number; + /** Total time taken in milliseconds */ + durationMs: number; +} + +/** + * Interface for a single schema migration handler. + * Each migration transforms documents from one version to the next. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface ISchemaMigration { + /** The source schema version this migration upgrades from */ + fromVersion: SchemaVersion; + /** The target schema version this migration upgrades to */ + toVersion: SchemaVersion; + /** Short description of what this migration does */ + description: string; + + /** + * Executes the migration on a batch of documents. + * Should update documents using MongoDB bulk operations for efficiency. + * + * @param model - Mongoose model for the collection + * @param context - Migration context with logger and configuration + * @returns Migration result with statistics + */ + execute(model: Model, context: MigrationContext): Promise; + + /** + * Optional validation to check if migration is safe to run. + * Can verify prerequisites or data integrity before migration starts. + * + * @param model - Mongoose model for the collection + * @param context - Migration context with logger and configuration + * @returns true if migration can proceed, false otherwise + */ + validate?(model: Model, context: MigrationContext): Promise; +} + +/** + * Registry entry for a collection's migrations. + * Groups all migrations for a specific collection. + */ +export interface CollectionMigrationRegistry { + /** Name of the collection */ + collectionName: string; + /** Mongoose model for the collection */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + model: Model; + /** Current schema version expected by the application */ + currentVersion: SchemaVersion; + /** Array of migrations in chronological order */ + migrations: ISchemaMigration[]; +} diff --git a/meet-ce/backend/src/repositories/migration.repository.ts b/meet-ce/backend/src/repositories/migration.repository.ts index 4fd62c84..f6d7a35d 100644 --- a/meet-ce/backend/src/repositories/migration.repository.ts +++ b/meet-ce/backend/src/repositories/migration.repository.ts @@ -31,7 +31,7 @@ export class MigrationRepository extends BaseRepository( { + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.API_KEY_SCHEMA_VERSION + }, key: { type: String, required: true }, creationDate: { type: Number, required: true } }, @@ -17,6 +26,7 @@ const MeetApiKeySchema = new Schema( versionKey: false, transform: (_doc, ret) => { delete ret._id; + delete ret.schemaVersion; return ret; } } @@ -26,4 +36,9 @@ const MeetApiKeySchema = new Schema( // Create indexes for efficient querying MeetApiKeySchema.index({ key: 1 }, { unique: true }); -export const MeetApiKeyModel = model('MeetApiKey', MeetApiKeySchema); +export const meetApiKeyCollectionName = 'MeetApiKey'; + +/** + * Mongoose model for API key entity. + */ +export const MeetApiKeyModel = model(meetApiKeyCollectionName, MeetApiKeySchema); diff --git a/meet-ce/backend/src/repositories/schemas/global-config.schema.ts b/meet-ce/backend/src/repositories/schemas/global-config.schema.ts index ec81ff08..8335f4a3 100644 --- a/meet-ce/backend/src/repositories/schemas/global-config.schema.ts +++ b/meet-ce/backend/src/repositories/schemas/global-config.schema.ts @@ -1,11 +1,15 @@ import { AuthMode, AuthType, GlobalConfig, MeetRoomThemeMode } from '@openvidu-meet/typings'; import { Document, model, Schema } from 'mongoose'; +import { INTERNAL_CONFIG } from '../../config/internal-config.js'; /** * Mongoose Document interface for GlobalConfig. * Extends the GlobalConfig interface with MongoDB Document functionality. */ -export interface MeetGlobalConfigDocument extends GlobalConfig, Document {} +export interface MeetGlobalConfigDocument extends GlobalConfig, Document { + /** Schema version for migration tracking (internal use only) */ + schemaVersion?: number; +} /** * Sub-schema for authentication method. @@ -144,6 +148,11 @@ const RoomsConfigSchema = new Schema( */ const MeetGlobalConfigSchema = new Schema( { + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.GLOBAL_CONFIG_SCHEMA_VERSION + }, projectId: { type: String, required: true @@ -166,6 +175,7 @@ const MeetGlobalConfigSchema = new Schema( versionKey: false, transform: (_doc, ret) => { delete ret._id; + delete ret.schemaVersion; return ret; } } @@ -175,7 +185,12 @@ const MeetGlobalConfigSchema = new Schema( // Create indexes for efficient querying MeetGlobalConfigSchema.index({ projectId: 1 }, { unique: true }); +export const meetGlobalConfigCollectionName = 'MeetGlobalConfig'; + /** * Mongoose model for GlobalConfig entity. */ -export const MeetGlobalConfigModel = model('MeetGlobalConfig', MeetGlobalConfigSchema); +export const MeetGlobalConfigModel = model( + meetGlobalConfigCollectionName, + MeetGlobalConfigSchema +); diff --git a/meet-ce/backend/src/repositories/schemas/migration.schema.ts b/meet-ce/backend/src/repositories/schemas/migration.schema.ts index 9e2ca0bf..a92a8c7e 100644 --- a/meet-ce/backend/src/repositories/schemas/migration.schema.ts +++ b/meet-ce/backend/src/repositories/schemas/migration.schema.ts @@ -25,12 +25,12 @@ const MigrationSchema = new Schema( default: MigrationStatus.RUNNING }, startedAt: { - type: Date, + type: Number, required: true, default: Date.now }, completedAt: { - type: Date, + type: Number, required: false }, error: { diff --git a/meet-ce/backend/src/repositories/schemas/recording.schema.ts b/meet-ce/backend/src/repositories/schemas/recording.schema.ts index 3e88a1cd..e2890c3e 100644 --- a/meet-ce/backend/src/repositories/schemas/recording.schema.ts +++ b/meet-ce/backend/src/repositories/schemas/recording.schema.ts @@ -1,11 +1,14 @@ import { MeetRecordingInfo, MeetRecordingStatus } from '@openvidu-meet/typings'; import { Document, model, Schema } from 'mongoose'; +import { INTERNAL_CONFIG } from '../../config/internal-config.js'; /** * Extended interface for Recording documents in MongoDB. * Includes the base MeetRecordingInfo plus internal access secrets. */ export interface MeetRecordingDocument extends MeetRecordingInfo, Document { + /** Schema version for migration tracking (internal use only) */ + schemaVersion?: number; accessSecrets?: { public: string; private: string; @@ -18,6 +21,11 @@ export interface MeetRecordingDocument extends MeetRecordingInfo, Document { */ const MeetRecordingSchema = new Schema( { + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.RECORDING_SCHEMA_VERSION + }, recordingId: { type: String, required: true @@ -84,6 +92,7 @@ const MeetRecordingSchema = new Schema( transform: (_doc, ret) => { // Remove MongoDB internal fields delete ret._id; + delete ret.schemaVersion; // Remove access secrets before returning (they should only be accessed via specific methods) delete ret.accessSecrets; return ret; @@ -101,7 +110,9 @@ MeetRecordingSchema.index({ status: 1, startDate: -1, _id: -1 }); MeetRecordingSchema.index({ duration: -1, _id: -1 }); MeetRecordingSchema.index({ size: -1, _id: -1 }); +export const meetRecordingCollectionName = 'MeetRecording'; + /** * Mongoose model for Recording entity. */ -export const MeetRecordingModel = model('MeetRecording', MeetRecordingSchema); +export const MeetRecordingModel = model(meetRecordingCollectionName, MeetRecordingSchema); diff --git a/meet-ce/backend/src/repositories/schemas/room.schema.ts b/meet-ce/backend/src/repositories/schemas/room.schema.ts index 5703b475..c2dc9d1c 100644 --- a/meet-ce/backend/src/repositories/schemas/room.schema.ts +++ b/meet-ce/backend/src/repositories/schemas/room.schema.ts @@ -7,12 +7,16 @@ import { MeetingEndAction } from '@openvidu-meet/typings'; import { Document, Schema, model } from 'mongoose'; +import { INTERNAL_CONFIG } from '../../config/internal-config.js'; /** * Mongoose Document interface for MeetRoom. * Extends the MeetRoom interface with MongoDB Document functionality. */ -export interface MeetRoomDocument extends MeetRoom, Document {} +export interface MeetRoomDocument extends MeetRoom, Document { + /** Schema version for migration tracking (internal use only) */ + schemaVersion?: number; +} /** * Mongoose schema for MeetRoom auto-deletion policy. @@ -127,6 +131,11 @@ const MeetRoomConfigSchema = new Schema( */ const MeetRoomSchema = new Schema( { + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.ROOM_SCHEMA_VERSION + }, roomId: { type: String, required: true @@ -177,6 +186,7 @@ const MeetRoomSchema = new Schema( versionKey: false, transform: (_doc, ret) => { delete ret._id; + delete ret.schemaVersion; return ret; } } @@ -190,7 +200,9 @@ MeetRoomSchema.index({ roomName: 1, creationDate: -1, _id: -1 }); MeetRoomSchema.index({ status: 1, creationDate: -1, _id: -1 }); MeetRoomSchema.index({ autoDeletionDate: 1 }); +export const meetRoomCollectionName = 'MeetRoom'; + /** * Mongoose model for MeetRoom. */ -export const MeetRoomModel = model('MeetRoom', MeetRoomSchema); +export const MeetRoomModel = model(meetRoomCollectionName, MeetRoomSchema); diff --git a/meet-ce/backend/src/repositories/schemas/user.schema.ts b/meet-ce/backend/src/repositories/schemas/user.schema.ts index 6e80cf35..e45b7df9 100644 --- a/meet-ce/backend/src/repositories/schemas/user.schema.ts +++ b/meet-ce/backend/src/repositories/schemas/user.schema.ts @@ -1,11 +1,15 @@ import { MeetUser, MeetUserRole } from '@openvidu-meet/typings'; import { Document, model, Schema } from 'mongoose'; +import { INTERNAL_CONFIG } from '../../config/internal-config.js'; /** * Mongoose Document interface for User. * Extends the User interface with MongoDB Document functionality. */ -export interface MeetUserDocument extends MeetUser, Document {} +export interface MeetUserDocument extends MeetUser, Document { + /** Schema version for migration tracking (internal use only) */ + schemaVersion?: number; +} /** * Mongoose schema for User entity. @@ -13,6 +17,11 @@ export interface MeetUserDocument extends MeetUser, Document {} */ const MeetUserSchema = new Schema( { + schemaVersion: { + type: Number, + required: true, + default: INTERNAL_CONFIG.USER_SCHEMA_VERSION + }, username: { type: String, required: true @@ -33,6 +42,7 @@ const MeetUserSchema = new Schema( versionKey: false, transform: (_doc, ret) => { delete ret._id; + delete ret.schemaVersion; return ret; } } @@ -42,7 +52,9 @@ const MeetUserSchema = new Schema( // Create indexes for efficient querying MeetUserSchema.index({ username: 1 }, { unique: true }); +export const meetUserCollectionName = 'MeetUser'; + /** * Mongoose model for User entity. */ -export const MeetUserModel = model('MeetUser', MeetUserSchema); +export const MeetUserModel = model(meetUserCollectionName, MeetUserSchema); diff --git a/meet-ce/backend/src/services/migration.service.ts b/meet-ce/backend/src/services/migration.service.ts index 8c5e987a..2f844803 100644 --- a/meet-ce/backend/src/services/migration.service.ts +++ b/meet-ce/backend/src/services/migration.service.ts @@ -1,7 +1,15 @@ import { inject, injectable } from 'inversify'; +import { Model } from 'mongoose'; import ms from 'ms'; import { MeetLock } from '../helpers/index.js'; -import { MigrationName } from '../models/index.js'; +import { migrationRegistry } from '../migrations/index.js'; +import { + CollectionMigrationRegistry, + generateSchemaMigrationName, + ISchemaMigration, + MigrationContext, + MigrationName +} from '../models/index.js'; import { ApiKeyRepository, GlobalConfigRepository, @@ -48,16 +56,11 @@ export class MigrationService { lockAcquired = true; - // Check if legacy storage migration has already been completed - const isLegacyMigrationCompleted = await this.migrationRepository.isCompleted( - MigrationName.LEGACY_STORAGE_TO_MONGODB - ); + // Migrate data from legacy storage to MongoDB if needed + await this.runMigrationsFromLegacyStorageToMongoDB(); - if (isLegacyMigrationCompleted) { - this.logger.info('Legacy storage migration already completed. Skipping...'); - } else { - await this.migrateFromLegacyStorageToMongoDB(); - } + // Run schema migrations to upgrade document structures + await this.runSchemaMigrations(); this.logger.info('All migrations completed successfully'); } catch (error) { @@ -75,13 +78,20 @@ export class MigrationService { /** * Orchestrates the migration from legacy storage to MongoDB. * Calls individual migration methods in the correct order. - * Tracks the migration status in the database. */ - protected async migrateFromLegacyStorageToMongoDB(): Promise { - this.logger.info('Running migrations from legacy storage to MongoDB...'); - + protected async runMigrationsFromLegacyStorageToMongoDB(): Promise { const migrationName = MigrationName.LEGACY_STORAGE_TO_MONGODB; + // Check if legacy storage migration has already been completed + const isLegacyMigrationCompleted = await this.migrationRepository.isCompleted(migrationName); + + if (isLegacyMigrationCompleted) { + this.logger.info('Legacy storage migration already completed. Skipping...'); + return; + } + + this.logger.info('Running migrations from legacy storage to MongoDB...'); + try { // Mark migration as started await this.migrationRepository.markAsStarted(migrationName); @@ -388,4 +398,202 @@ export class MigrationService { throw error; } } + + /** + * Runs all schema migrations to upgrade document structures to the latest version. + * Processes each collection in the registry and executes pending migrations. + * + * Schema migrations run after data migrations and upgrade existing documents + * to match the current schema version expected by the application. + */ + protected async runSchemaMigrations(): Promise { + this.logger.info('Running schema migrations...'); + + try { + let totalMigrated = 0; + let totalSkipped = 0; + + // Process each collection in the registry + for (const registry of migrationRegistry) { + this.logger.info(`Checking schema version for collection: ${registry.collectionName}`); + + // Get the current version of documents in the collection + const currentVersionInDb = await this.getCurrentSchemaVersion(registry.model); + + if (currentVersionInDb === null) { + this.logger.info(`No documents found in ${registry.collectionName}, skipping migration`); + continue; + } + + if (currentVersionInDb === registry.currentVersion) { + this.logger.info( + `Collection ${registry.collectionName} is already at version ${registry.currentVersion}` + ); + continue; + } + + if (currentVersionInDb > registry.currentVersion) { + this.logger.warn( + `Collection ${registry.collectionName} has version ${currentVersionInDb} ` + + `but application expects ${registry.currentVersion}. ` + + `This may indicate a downgrade or inconsistent deployment.` + ); + continue; + } + + // Find migrations needed to upgrade from current to target version + const neededMigrations = this.findNeededMigrations( + registry, + currentVersionInDb, + registry.currentVersion + ); + + if (neededMigrations.length === 0) { + this.logger.info(`No migrations needed for ${registry.collectionName}`); + continue; + } + + this.logger.info( + `Found ${neededMigrations.length} migrations for ${registry.collectionName} ` + + `(v${currentVersionInDb} -> v${registry.currentVersion})` + ); + + // Execute each migration in sequence + for (const migration of neededMigrations) { + const migrationName = generateSchemaMigrationName( + registry.collectionName, + migration.fromVersion, + migration.toVersion + ); + + // Check if this specific migration was already completed + const isCompleted = await this.migrationRepository.isCompleted(migrationName as MigrationName); + + if (isCompleted) { + this.logger.info(`Migration ${migrationName} already completed, skipping`); + continue; + } + + // Mark migration as started + await this.migrationRepository.markAsStarted(migrationName as MigrationName); + + try { + const migrationContext: MigrationContext = { + logger: this.logger, + batchSize: 50 // Default batch size + }; + + // Validate migration if validation method is provided + if (migration.validate) { + const isValid = await migration.validate(registry.model, migrationContext); + + if (!isValid) { + throw new Error(`Validation failed for migration ${migrationName}`); + } + } + + // Execute the migration + this.logger.info(`Executing migration: ${migration.description}`); + const result = await migration.execute(registry.model, migrationContext); + + // Track statistics + totalMigrated += result.migratedCount; + totalSkipped += result.skippedCount; + + // Mark migration as completed with metadata + const metadata: Record = { + collectionName: registry.collectionName, + fromVersion: migration.fromVersion, + toVersion: migration.toVersion, + migratedCount: result.migratedCount, + skippedCount: result.skippedCount, + failedCount: result.failedCount, + durationMs: result.durationMs + }; + + await this.migrationRepository.markAsCompleted(migrationName as MigrationName, metadata); + + this.logger.info( + `Migration ${migrationName} completed: ${result.migratedCount} migrated, ` + + `${result.failedCount} failed (${result.durationMs}ms)` + ); + } catch (error) { + // Mark migration as failed + const errorMessage = error instanceof Error ? error.message : String(error); + await this.migrationRepository.markAsFailed(migrationName as MigrationName, errorMessage); + throw error; + } + } + } + + this.logger.info( + `Schema migrations completed successfully: ${totalMigrated} documents migrated, ${totalSkipped} skipped` + ); + } catch (error) { + this.logger.error('Error running schema migrations:', error); + throw error; + } + } + + /** + * Gets the current schema version of documents in a collection. + * Samples the database to determine the version. + * + * @param model - Mongoose model for the collection + * @returns Current version or null if collection is empty + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + protected async getCurrentSchemaVersion(model: Model): Promise { + try { + // Get a sample document to check its version + const sampleDoc = await model.findOne({}).select('schemaVersion').exec(); + + if (!sampleDoc) { + return null; // Collection is empty + } + + // If schemaVersion doesn't exist, assume version 1 (initial version) + return sampleDoc.schemaVersion ?? 1; + } catch (error) { + this.logger.error('Error getting current schema version:', error); + throw error; + } + } + + /** + * Finds the migrations needed to upgrade from one version to another. + * Returns migrations in the correct order to apply. + * + * @param registry - Collection migration registry + * @param fromVersion - Current version in database + * @param toVersion - Target version from application + * @returns Array of migrations to execute in order + */ + protected findNeededMigrations( + registry: CollectionMigrationRegistry, + fromVersion: number, + toVersion: number + ): ISchemaMigration[] { + const needed: ISchemaMigration[] = []; + + // Build a chain of migrations from fromVersion to toVersion + let currentVersion = fromVersion; + + while (currentVersion < toVersion) { + const nextMigration = registry.migrations.find((m) => m.fromVersion === currentVersion); + + if (!nextMigration) { + this.logger.warn( + `No migration found from version ${currentVersion} for ${registry.collectionName}. ` + + `Migration chain is incomplete.` + ); + break; + } + + needed.push(nextMigration); + currentVersion = nextMigration.toVersion; + } + + return needed; + } }