neoris/openvidu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: update GCS storage provider and service for improved object listing and pagination handling

Browse Source
This commit is contained in:
Piwccle 2025-09-26 18:35:34 +02:00
parent 7c8222ddb2
commit 4f97c1a1db
2 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/src/services/storage/providers/gcp/gcs-storage.provider.ts
View File

@ -11,7 +11,7 @@ export class GCSStorageProvider implements StorageProvider {
constructor( constructor(
@inject(LoggerService) protected logger: LoggerService, @inject(LoggerService) protected logger: LoggerService,
@inject(GCSService) protected gcsService: GCSService @inject(GCSService) protected gcsService: GCSService
) {} ) { }
/** /**
* Retrieves an object from GCS Storage as a JSON object. * Retrieves an object from GCS Storage as a JSON object.
@ -105,14 +105,14 @@ export class GCSStorageProvider implements StorageProvider {
// Transform GCS response to match the expected interface // Transform GCS response to match the expected interface
return { return {
Contents: result.Contents?.map((item) => ({ Contents: result.items?.map((item) => ({
Key: item.Key, Key: item.Key,
LastModified: item.LastModified, LastModified: item.LastModified,
Size: item.Size, Size: item.Size,
ETag: undefined // GCS doesn't provide ETag in the same way as S3 ETag: undefined // GCS doesn't provide ETag in the same way as S3
})), })),
IsTruncated: !!result.NextContinuationToken, IsTruncated: !!result.continuationToken,
NextContinuationToken: result.NextContinuationToken NextContinuationToken: result.continuationToken
}; };
} catch (error) { } catch (error) {
this.logger.error(`Error listing objects in GCS Storage with prefix ${prefix}: ${error}`); this.logger.error(`Error listing objects in GCS Storage with prefix ${prefix}: ${error}`);

76
backend/src/services/storage/providers/gcp/gcs.service.ts
View File

@ -116,53 +116,63 @@ export class GCSService {
*/ */
async listObjectsPaginated( async listObjectsPaginated(
additionalPrefix = '', additionalPrefix = '',
maxKeys = 50, maxResults = 50,
continuationToken?: string, continuationToken?: string,
bucket: string = MEET_S3_BUCKET bucket: string = MEET_S3_BUCKET
): Promise<{ ): Promise<{
Contents?: Array<{ Key?: string; LastModified?: Date; Size?: number; ETag?: string }>; items: Array<{ Key?: string; LastModified?: Date; Size?: number; ETag?: string }>;
NextContinuationToken?: string; continuationToken?: string;
IsTruncated?: boolean; isTruncated?: boolean;
KeyCount?: number;
}> { }> {
const basePrefix = this.getFullKey(additionalPrefix); const basePrefix = this.getFullKey(additionalPrefix);
this.logger.verbose(`GCS listObjectsPaginated: listing objects with prefix '${basePrefix}'`); this.logger.verbose(`GCS listObjectsPaginated: listing objects with prefix '${basePrefix}'`);
const options: GetFilesOptions = {
prefix: basePrefix,
maxResults: maxKeys
};
if (continuationToken && continuationToken !== 'undefined') {
options.pageToken = continuationToken;
}
try { try {
maxResults = Number(maxResults);
const bucketObj = bucket === MEET_S3_BUCKET ? this.bucket : this.storage.bucket(bucket); const bucketObj = bucket === MEET_S3_BUCKET ? this.bucket : this.storage.bucket(bucket);
const [files, , response] = await bucketObj.getFiles(options);
interface GCSFileContent { const options: GetFilesOptions = {
Key?: string; prefix: basePrefix,
LastModified?: Date; maxResults: maxResults,
Size?: number; autoPaginate: false
ETag?: string; };
if (continuationToken && continuationToken !== 'undefined') {
options.pageToken = continuationToken;
} }
const contents: GCSFileContent[] = files.map( const [files, , response] = await bucketObj.getFiles(options);
(file: File): GCSFileContent => ({
Key: file.name, const items = files.map((file: File) => ({
LastModified: file.metadata.updated ? new Date(file.metadata.updated) : undefined, Key: file.name,
Size: file.metadata.size ? parseInt(file.metadata.size as string) : undefined, LastModified: file.metadata.updated ? new Date(file.metadata.updated) : undefined,
ETag: file.metadata.etag || undefined Size: file.metadata.size ? parseInt(file.metadata.size as string) : undefined,
}) ETag: file.metadata.etag || undefined
); }));
let NextContinuationToken = (response as any)?.nextPageToken;
let isTruncated = NextContinuationToken !== undefined;
// Check if next page has items, similar to ABS implementation
if (NextContinuationToken) {
const nextOptions: GetFilesOptions = {
prefix: basePrefix,
maxResults: 1,
autoPaginate: false,
pageToken: NextContinuationToken
};
const [nextFiles] = await bucketObj.getFiles(nextOptions);
if (nextFiles.length === 0) {
NextContinuationToken = undefined;
isTruncated = false;
}
}
const nextPageToken = (response as any)?.nextPageToken;
return { return {
Contents: contents, items: items,
NextContinuationToken: nextPageToken, continuationToken: NextContinuationToken,
IsTruncated: !!nextPageToken, isTruncated: isTruncated
KeyCount: contents.length
}; };
} catch (error) { } catch (error) {
this.logger.error(`GCS listObjectsPaginated: error listing objects with prefix '${basePrefix}': ${error}`); this.logger.error(`GCS listObjectsPaginated: error listing objects with prefix '${basePrefix}': ${error}`);