import * as sdk from '@botpress/sdk' import type * as models from '../../definitions/models' import type * as types from '../types' import { processQueueFile } from './file-processor' import { findBatchEndCursor } from './queue-batching' export type ProcessQueueProps = { syncQueue: Readonly logger: sdk.BotLogger fileRepository: { listFiles: (params: { tags: Record }) => Promise<{ files: types.FilesApiFile[] }> deleteFile: (params: { id: string }) => Promise updateFileMetadata: (params: { id: string; tags: Record }) => Promise } integration: { name: string alias: string transferFileToBotpress: (params: { file: models.FileWithPath fileKey: string }) => Promise<{ botpressFileId: string }> } updateSyncQueue: (props: { syncQueue: types.SyncQueue }) => Promise } export const processQueue = async (props: ProcessQueueProps) => { // Short-circuit if the sync queue is empty: if (!props.syncQueue?.length) { props.logger.info('Sync queue is empty. Nothing to process.') return { finished: 'all' } as const } const syncQueue = structuredClone(props.syncQueue) as types.SyncQueue const startCursor = syncQueue.findIndex((file) => file.status === 'pending') if (startCursor < 0) { props.logger.info('No files left to process in the sync queue.') return { finished: 'all' } as const } const { endCursor } = findBatchEndCursor({ startCursor, files: syncQueue }) const filesInBatch = syncQueue.slice(startCursor, endCursor) for (const fileToSync of filesInBatch) { const processedFile = await processQueueFile({ ...props, fileToSync }) Object.assign(fileToSync, processedFile) } await props.updateSyncQueue({ syncQueue }) const hasPendingBeyondBatch = syncQueue.slice(endCursor).some((f) => f.status === 'pending') if (hasPendingBeyondBatch) { return { finished: 'batch' } as const } return { finished: 'all' } as const }