neoris/openvidu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: update GCS storage provider and service for improved object listing and pagination handling

Browse Source
This commit is contained in:
Piwccle 2025-09-26 18:35:34 +02:00
parent 7c8222ddb2
commit 4f97c1a1db
2 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/src/services/storage/providers/gcp/gcs-storage.provider.ts
View File

@ -11,7 +11,7 @@ export class GCSStorageProvider implements StorageProvider {
constructor(
@inject(LoggerService) protected logger: LoggerService,
@inject(GCSService) protected gcsService: GCSService
) {}
) { }
/**
* Retrieves an object from GCS Storage as a JSON object.
@ -105,14 +105,14 @@ export class GCSStorageProvider implements StorageProvider {
// Transform GCS response to match the expected interface
return {
Contents: result.Contents?.map((item) => ({
Contents: result.items?.map((item) => ({
Key: item.Key,
LastModified: item.LastModified,
Size: item.Size,
ETag: undefined // GCS doesn't provide ETag in the same way as S3
})),
IsTruncated: !!result.NextContinuationToken,
NextContinuationToken: result.NextContinuationToken
IsTruncated: !!result.continuationToken,
NextContinuationToken: result.continuationToken
};
} catch (error) {
this.logger.error(`Error listing objects in GCS Storage with prefix ${prefix}: ${error}`);

58
backend/src/services/storage/providers/gcp/gcs.service.ts
View File

@ -116,53 +116,63 @@ export class GCSService {
*/
async listObjectsPaginated(
additionalPrefix = '',
maxKeys = 50,
maxResults = 50,
continuationToken?: string,
bucket: string = MEET_S3_BUCKET
): Promise<{
Contents?: Array<{ Key?: string; LastModified?: Date; Size?: number; ETag?: string }>;
NextContinuationToken?: string;
IsTruncated?: boolean;
KeyCount?: number;
items: Array<{ Key?: string; LastModified?: Date; Size?: number; ETag?: string }>;
continuationToken?: string;
isTruncated?: boolean;
}> {
const basePrefix = this.getFullKey(additionalPrefix);
this.logger.verbose(`GCS listObjectsPaginated: listing objects with prefix '${basePrefix}'`);
try {
maxResults = Number(maxResults);
const bucketObj = bucket === MEET_S3_BUCKET ? this.bucket : this.storage.bucket(bucket);
const options: GetFilesOptions = {
prefix: basePrefix,
maxResults: maxKeys
maxResults: maxResults,
autoPaginate: false
};
if (continuationToken && continuationToken !== 'undefined') {
options.pageToken = continuationToken;
}
try {
const bucketObj = bucket === MEET_S3_BUCKET ? this.bucket : this.storage.bucket(bucket);
const [files, , response] = await bucketObj.getFiles(options);
interface GCSFileContent {
Key?: string;
LastModified?: Date;
Size?: number;
ETag?: string;
}
const contents: GCSFileContent[] = files.map(
(file: File): GCSFileContent => ({
const items = files.map((file: File) => ({
Key: file.name,
LastModified: file.metadata.updated ? new Date(file.metadata.updated) : undefined,
Size: file.metadata.size ? parseInt(file.metadata.size as string) : undefined,
ETag: file.metadata.etag || undefined
})
);
}));
let NextContinuationToken = (response as any)?.nextPageToken;
let isTruncated = NextContinuationToken !== undefined;
// Check if next page has items, similar to ABS implementation
if (NextContinuationToken) {
const nextOptions: GetFilesOptions = {
prefix: basePrefix,
maxResults: 1,
autoPaginate: false,
pageToken: NextContinuationToken
};
const [nextFiles] = await bucketObj.getFiles(nextOptions);
if (nextFiles.length === 0) {
NextContinuationToken = undefined;
isTruncated = false;
}
}
const nextPageToken = (response as any)?.nextPageToken;
return {
Contents: contents,
NextContinuationToken: nextPageToken,
IsTruncated: !!nextPageToken,
KeyCount: contents.length
items: items,
continuationToken: NextContinuationToken,
isTruncated: isTruncated
};
} catch (error) {
this.logger.error(`GCS listObjectsPaginated: error listing objects with prefix '${basePrefix}': ${error}`);