chore: import upstream snapshot with attribution

plugins/file-synchronizer/definitions/models.ts

@@ -0,0 +1,43 @@
+import * as sdk from '@botpress/sdk'
+
+const _baseItem = (itemType: 'file' | 'folder' | 'item' = 'item') =>
+  sdk.z.object({
+    id: sdk.z
+      .string()
+      .describe(
+        `The ${itemType}'s ID. This could be a unique identifier from the external service, or a relative or absolute path, so long as it's unique.`
+      ),
+    type: sdk.z.union([sdk.z.literal('file'), sdk.z.literal('folder')]).describe('The entity type'),
+    name: sdk.z
+      .string()
+      .describe(
+        `The ${itemType}'s name. This will be displayed in the Botpress UI and be used as the ${itemType}'s name on Files API."`
+      ),
+    parentId: sdk.z
+      .string()
+      .optional()
+      .describe(`The parent folder ID. Leave empty if the ${itemType} is in the root folder.`),
+  })
+
+export const FOLDER = _baseItem('folder').extend({
+  type: sdk.z.literal('folder'),
+})
+
+export const FILE = _baseItem('file').extend({
+  type: sdk.z.literal('file'),
+  sizeInBytes: sdk.z.number().optional().describe('The file size in bytes, if available'),
+  lastModifiedDate: sdk.z.string().datetime().optional().describe('The last modified date of the file, if available'),
+  contentHash: sdk.z
+    .string()
+    .optional()
+    .describe('The hash of the file content, or version/revision number, if available'),
+})
+
+export const FILE_WITH_PATH = FILE.extend({
+  absolutePath: sdk.z.string().describe('The absolute path of the file'),
+})
+
+export type Folder = sdk.z.infer<typeof FOLDER>
+export type File = sdk.z.infer<typeof FILE>
+export type FileWithPath = sdk.z.infer<typeof FILE_WITH_PATH>
+export type FolderItem = Folder | File

plugins/file-synchronizer/eslint.config.mjs

@@ -0,0 +1,13 @@
+import rootConfig from '../../eslint.config.mjs'
+
+export default [
+  ...rootConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]

plugins/file-synchronizer/hub.md

@@ -0,0 +1 @@
+# File Synchronizer Plugin

plugins/file-synchronizer/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="241" height="241" fill="none"><path fill="purple" d="M34 0A34 34 0 0 0 0 34v173a34 34 0 0 0 34 34h173a34 34 0 0 0 34-34V34a34 34 0 0 0-34-34Zm51.8 33.6h62a8.8 8.7 0 0 1 3.2.6 8.8 8.7 0 0 1 .2.1 8.8 8.7 0 0 1 2.8 1.9l44.2 43.4a8.8 8.7 0 0 1 1.9 2.8 8.8 8.7 0 0 1 0 .2 8.8 8.7 0 0 1 .7 3.2v95.7c0 14.3-12 26.1-26.5 26.1h-48.7a8.8 8.7 0 0 1-8.8-8.7 8.8 8.7 0 0 1 8.8-8.7h48.7c5 0 8.8-3.8 8.8-8.7v-87h-26.5c-9.7 0-17.7-7.9-17.7-17.4v-26H85.8c-5 0-8.9 3.7-8.9 8.6v69.6a8.8 8.7 0 0 1-8.8 8.7 8.8 8.7 0 0 1-8.9-8.7V59.7c0-14.3 12-26.1 26.6-26.1zm70.8 29.7v13.8h14zm-79.7 83.4a8.8 8.7 0 0 1 6.3 2.6l26.5 26a8.8 8.7 0 0 1 2.6 6.2 8.8 8.7 0 0 1-2.6 6.2l-26.5 26a8.8 8.7 0 0 1-12.5 0 8.8 8.7 0 0 1 0-12.2L82 190.2H41.6a8.8 8.7 0 0 1-8.9-8.7 8.8 8.7 0 0 1 8.9-8.7H82l-11.4-11.2a8.8 8.7 0 0 1 0-12.3 8.8 8.7 0 0 1 6.2-2.6z"/></svg>

plugins/file-synchronizer/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@botpresshub/file-synchronizer",
+  "scripts": {
+    "check:type": "tsc --noEmit",
+    "build": "bp add -y && bp build",
+    "test": "vitest --run"
+  },
+  "private": true,
+  "dependencies": {
+    "@botpress/client": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "picomatch": "^4.0.2"
+  },
+  "devDependencies": {
+    "@botpress/cli": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "@botpresshub/hitl": "workspace:*",
+    "@types/picomatch": "^3.0.2",
+    "@types/semver": "^7.3.11",
+    "semver": "^7.3.8"
+  },
+  "bpDependencies": {
+    "files-readonly": "../../interfaces/files-readonly"
+  }
+}

plugins/file-synchronizer/plugin.definition.ts

@@ -0,0 +1,202 @@
+import * as sdk from '@botpress/sdk'
+import filesReadonly from './bp_modules/files-readonly'
+
+const FILE_FILTER_PROPS = sdk.z.object({
+  includeFiles: sdk.z.array(
+    sdk.z
+      .object({
+        pathGlobPattern: sdk.z
+          .string()
+          .placeholder('Example: /path/to/folder/**')
+          .describe(
+            'A glob pattern to match against the file path. Only files that match the pattern will be synchronized. Any pattern supported by picomatch is supported. For example, use rule "**" to match all files, or enter a path like "/path/to/folder/**" to match all files in a specific folder.'
+          ),
+        maxSizeInBytes: sdk.z
+          .number()
+          .optional()
+          .describe(
+            'Filter by maximum size (in bytes). Only files smaller than the specified size will be synchronized.'
+          ),
+        modifiedAfter: sdk.z
+          .string()
+          .datetime()
+          .optional()
+          .describe(
+            'Filter the items by modified date. Only files modified after the specified date will be synchronized.'
+          ),
+        applyOptionsToMatchedFiles: sdk.z
+          .object({
+            addToKbId: sdk.z
+              .string()
+              .placeholder('Example: kb-2f0a7ea639')
+              .optional()
+              .title('Knowledge Base ID')
+              .describe(
+                'The ID of the knowledge base to add the file to. Note that files added to knowledge bases will count towards both the Vector DB Storage quota and the File Storage quota of the workspace.'
+              ),
+          })
+          .optional()
+          .title('Apply to Matched Files')
+          .describe('Options to apply to the matched files.'),
+      })
+      .title('Include Criteria')
+      .describe('A file must match all criteria to be synchronized.')
+  ),
+  excludeFiles: sdk.z.array(
+    sdk.z
+      .object({
+        pathGlobPattern: sdk.z
+          .string()
+          .placeholder('Example: /path/to/folder/**')
+          .describe(
+            'A glob pattern to match against the file path. Files that match the pattern will be ignored, even if they match the includeFiles configuration.'
+          ),
+      })
+      .title('Exclude Criteria')
+      .describe('A file must match all exclude criteria to be ignored.')
+  ),
+})
+
+export default new sdk.PluginDefinition({
+  name: 'file-synchronizer',
+  version: '1.1.2',
+  title: 'File Synchronizer',
+  description: 'Synchronize files from external services to Botpress',
+  icon: 'icon.svg',
+  readme: 'hub.md',
+  configuration: {
+    schema: sdk.z.object({
+      enableRealTimeSync: sdk.z
+        .boolean()
+        .default(true)
+        .describe(
+          'Enable real-time synchronization. Whenever a file is created, updated, or deleted, synchronize it to Botpress immediately. This does not work with every integration.'
+        ),
+      includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+        .title('Include Rules')
+        .describe('A list of rules to include files. Only files that match one or more rules will be synchronized.'),
+      excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+        .title('Exclude Rules')
+        .describe(
+          'A list of rules to exclude files. Files that match one or more rules will be ignored. This takes precedence over Include Rules.'
+        ),
+    }),
+  },
+  actions: {
+    syncFilesToBotpess: {
+      title: 'Sync files to Botpress',
+      description: 'Start synchronization of files from the external service to Botpress',
+      input: {
+        schema: sdk.z.object({
+          includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+            .title('Include Rules Override')
+            .describe('If omitted, the global Include Rules will be used.')
+            .optional(),
+          excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+            .title('Exclude Rules Override')
+            .describe('If omitted, the global Exclude Rules will be used')
+            .optional(),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({
+          status: sdk.z.enum(['queued', 'already-running', 'error']),
+        }),
+      },
+    },
+    listItemsInFolder: {
+      // Re-export the action from the files-readonly interface:
+      ...filesReadonly.definition.actions.listItemsInFolder,
+    },
+  },
+  workflows: {
+    buildQueue: {
+      title: 'Build file sync queue',
+      description: 'Build the file sync queue and synchronize files to Botpress',
+      input: {
+        schema: sdk.z.object({
+          includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+            .title('Include Rules Override')
+            .describe('If omitted, the global Include Rules will be used.'),
+          excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+            .title('Exclude Rules Override')
+            .describe('If omitted, the global Exclude Rules will be used'),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({}),
+      },
+      tags: {
+        syncJobId: {
+          title: 'Sync job ID',
+          description: 'The unique ID of the sync job',
+        },
+        syncType: {
+          title: 'Sync type',
+          description: 'The type of sync job',
+        },
+        syncInitiatedAt: {
+          title: 'Created at',
+          description: 'The date and time when the sync job was created',
+        },
+      },
+    },
+    processQueue: {
+      title: 'Process file sync queue',
+      description: 'Process the file sync queue and synchronize files to Botpress',
+      input: {
+        schema: sdk.z.object({
+          jobFileId: sdk.z.string().title('Job File').describe("The ID of the job's queue file"),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({}),
+      },
+      tags: {
+        syncJobId: {
+          title: 'Sync job ID',
+          description: 'The unique ID of the sync job',
+        },
+        syncType: {
+          title: 'Sync type',
+          description: 'The type of sync job',
+        },
+        syncInitiatedAt: {
+          title: 'Created at',
+          description: 'The date and time when the sync job was created',
+        },
+      },
+    },
+  },
+  states: {
+    buildQueueRuntimeState: {
+      type: 'workflow',
+      schema: sdk.z.object({
+        jobFileId: sdk.z.string().title('Job File').describe("The ID of the job's queue file"),
+        enumerationState: sdk.z
+          .object({
+            pendingFolders: sdk.z
+              .array(
+                sdk.z.object({
+                  folderId: sdk.z.string().optional().title('Folder ID').describe('The ID of the folder'),
+                  absolutePath: sdk.z.string().title('Absolute Path').describe('The absolute path of the folder'),
+                })
+              )
+              .title('Pending Folders')
+              .describe('Folders awaiting enumeration'),
+            currentFolderNextToken: sdk.z
+              .string()
+              .optional()
+              .title('Current Folder Paging Token')
+              .describe('The next token to use for pagination'),
+          })
+          .optional()
+          .title('Enumeration State')
+          .describe('The current state of the enumeration process'),
+      }),
+    },
+  },
+  interfaces: {
+    'files-readonly': sdk.version.allWithinMajorOf(filesReadonly),
+  },
+})

plugins/file-synchronizer/src/actions/index.ts

@@ -0,0 +1 @@
+export * as syncFilesToBotpess from './sync-files-to-botpress'

plugins/file-synchronizer/src/actions/sync-files-to-botpress.ts

@@ -0,0 +1,52 @@
+import * as sdk from '@botpress/sdk'
+import { randomUUID } from '../crypto'
+import * as bp from '.botpress'
+
+export const callAction: bp.PluginHandlers['actionHandlers']['syncFilesToBotpess'] = async (props) => {
+  if (await _isSyncAlreadyInProgress(props)) {
+    props.logger.info('Sync is already in progress. Ignoring sync event...')
+    return { status: 'already-running' }
+  }
+
+  const includeFiles = props.input.includeFiles ?? props.configuration.includeFiles
+  const excludeFiles = props.input.excludeFiles ?? props.configuration.excludeFiles
+
+  if (includeFiles.length === 0) {
+    throw new sdk.RuntimeError(
+      'No include rules defined. Please define at least one include rule. For example, create a rule with glob pattern "**" to include all files.'
+    )
+  }
+
+  props.logger.info('Syncing files to Botpress...', {
+    includeFiles,
+    excludeFiles,
+  })
+
+  props.logger.info('Enumerating files...')
+  await props.workflows.buildQueue.startNewInstance({
+    input: { includeFiles, excludeFiles },
+    tags: {
+      syncJobId: await randomUUID(),
+      syncType: 'manual',
+      syncInitiatedAt: new Date().toISOString(),
+    },
+  })
+
+  return { status: 'queued' }
+}
+
+const _isSyncAlreadyInProgress = async (props: bp.ActionHandlerProps) => {
+  const runningBuildQueueWorkflows = await props.workflows.buildQueue
+    .listInstances({ statuses: ['pending', 'in_progress', 'listening', 'paused'] })
+    .take(1)
+
+  if (runningBuildQueueWorkflows.length > 0) {
+    return true
+  }
+
+  const runningProcessQueueWorkflows = await props.workflows.processQueue
+    .listInstances({ statuses: ['pending', 'in_progress', 'listening', 'paused'] })
+    .take(1)
+
+  return runningProcessQueueWorkflows.length > 0
+}

plugins/file-synchronizer/src/consts.ts

@@ -0,0 +1 @@
+export const MAX_BATCH_SIZE_BYTES = 104857600 // 100MB

plugins/file-synchronizer/src/crypto.ts

@@ -0,0 +1,28 @@
+export const randomUUID = async (): Promise<string> => {
+  const crypto = await _getWebCrypto()
+  return crypto.randomUUID()
+}
+
+type _WebCrypto = { randomUUID: () => string }
+let _webCrypto: _WebCrypto | undefined
+
+export const _getWebCrypto = async (): Promise<_WebCrypto> => {
+  if (!_webCrypto) {
+    if (typeof (globalThis as any).crypto?.randomUUID === 'function') {
+      _webCrypto = (globalThis as any).crypto
+    } else if (typeof process !== 'undefined' && process.versions?.node) {
+      try {
+        const nodeCrypto = await import('crypto')
+        _webCrypto = nodeCrypto.webcrypto
+      } catch (thrown: unknown) {
+        const error: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+        throw new Error(`Failed to import 'crypto' module: ${error.message}`)
+      }
+    }
+
+    if (!_webCrypto || typeof _webCrypto.randomUUID !== 'function') {
+      throw new Error('No suitable web crypto implementation available')
+    }
+  }
+  return _webCrypto
+}

plugins/file-synchronizer/src/hooks/index.ts

@@ -0,0 +1,4 @@
+export * as onEvent from './on-event'
+export * as onWorkflowStart from './workflow-started'
+export * as onWorkflowContinue from './workflow-continued'
+export * as onWorkflowTimeout from './workflow-timed-out'

plugins/file-synchronizer/src/hooks/on-event/aggregate-file-changes.ts

@@ -0,0 +1,45 @@
+import { handleEvent as handleFileCreated } from './file-created'
+import { handleEvent as handleFileDeleted } from './file-deleted'
+import { handleEvent as handleFileUpdated } from './file-updated'
+import { handleEvent as handleFolderDeletedRecursive } from './folder-deleted-recursive'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:aggregateFileChanges'] = async (props) => {
+  const modifiedItems = props.event.payload.modifiedItems
+
+  for (const deletedItem of modifiedItems.deleted) {
+    if (deletedItem.type === 'file') {
+      await handleFileDeleted({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:fileDeleted', payload: { file: deletedItem } },
+      })
+    } else {
+      await handleFolderDeletedRecursive({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:folderDeletedRecursive', payload: { folder: deletedItem } },
+      })
+    }
+  }
+
+  for (const createdItem of modifiedItems.created) {
+    if (createdItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileCreated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileCreated', payload: { file: createdItem } },
+    })
+  }
+
+  for (const updatedItem of modifiedItems.updated) {
+    if (updatedItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileUpdated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileUpdated', payload: { file: updatedItem } },
+    })
+  }
+}

plugins/file-synchronizer/src/hooks/on-event/file-created.ts

@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileCreated'] = async (props) => {
+  const createdFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: createdFile,
+    itemPath: createdFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${createdFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...createdFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${createdFile.absolutePath} has been synchronized`)
+}

plugins/file-synchronizer/src/hooks/on-event/file-deleted.ts

@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileDeleted'] = async (props) => {
+  const deletedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFile,
+    itemPath: deletedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${deletedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalId: deletedFile.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFile.absolutePath} has been deleted`)
+    }
+  } catch {}
+}

plugins/file-synchronizer/src/hooks/on-event/file-updated.ts

@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileUpdated'] = async (props) => {
+  const updatedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: updatedFile,
+    itemPath: updatedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${updatedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...updatedFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${updatedFile.absolutePath} has been synchronized`)
+}

plugins/file-synchronizer/src/hooks/on-event/folder-deleted-recursive.ts

@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:folderDeletedRecursive'] = async (props) => {
+  const deletedFolder = props.event.payload.folder
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFolder,
+    itemPath: deletedFolder.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring folder ${deletedFolder.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalParentId: deletedFolder.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFolder.absolutePath} has been deleted`)
+    }
+  } catch {}
+}

plugins/file-synchronizer/src/hooks/on-event/index.ts

@@ -0,0 +1,5 @@
+export * as fileCreated from './file-created'
+export * as fileDeleted from './file-deleted'
+export * as fileUpdated from './file-updated'
+export * as folderDeletedRecursive from './folder-deleted-recursive'
+export * as aggregateFileChanges from './aggregate-file-changes'

plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts

@@ -0,0 +1,77 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  props.logger.info('Retrieving job file...')
+  const workflowState = await props.states.workflow.buildQueueRuntimeState.get(props.workflow.id)
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Enumerating files...')
+  const enumerationState = await SyncQueue.directoryTraversalWithBatching.enumerateAllFilesRecursive(
+    _getEnumerateAllFilesRecursiveProps(props, syncQueue, key, workflowState)
+  )
+
+  if (enumerationState !== undefined) {
+    // Enumeration is still in progress
+    props.logger.debug('Enumeration partially completed')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { ...workflowState, enumerationState })
+    return
+  }
+
+  const { syncQueue: finalSyncQueue } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  if (finalSyncQueue.length === 0) {
+    props.logger.info(
+      'Enumeration matched no files. Nothing to sync. Please check your include and exclude rules. ' +
+        `There could also be an issue with your configuration in the "${props.interfaces['files-readonly'].name}" integration. ` +
+        'For example, your access token might be missing some permissions or the integration might not be set up correctly.'
+    )
+    await props.workflow.setCompleted()
+    return
+  }
+
+  props.logger.info('Enumeration completed. Starting sync job...')
+  await props.workflow.setCompleted()
+  await props.workflows.processQueue.startNewInstance({
+    input: { jobFileId: workflowState.jobFileId },
+    tags: {
+      syncInitiatedAt: props.workflow.tags.syncInitiatedAt!,
+      syncJobId: props.workflow.tags.syncJobId!,
+      syncType: props.workflow.tags.syncType!,
+    },
+  })
+}
+
+const _getEnumerateAllFilesRecursiveProps = (
+  props: bp.WorkflowHandlerProps['buildQueue'],
+  syncQueue: SyncQueue.queueProcessor.ProcessQueueProps['syncQueue'],
+  syncFileKey: string,
+  workflowState: bp.states.States['buildQueueRuntimeState']['payload']
+): SyncQueue.directoryTraversalWithBatching.EnumerateAllFilesRecursiveProps => ({
+  ...props,
+
+  configuration: props.workflow.input,
+
+  currentEnumerationState: workflowState.enumerationState,
+  integration: { listItemsInFolder: props.actions['files-readonly'].listItemsInFolder },
+
+  async pushFilesToQueue(files) {
+    if (!files.length) {
+      return
+    }
+
+    const dedupedQueue = new Map(syncQueue.map((file) => [file.id, file]))
+    files.forEach((file) => dedupedQueue.set(file.id, { ...file, status: 'pending' } as const))
+
+    await SyncQueue.jobFileManager.updateSyncQueue(
+      props,
+      syncFileKey,
+      Array.from(dedupedQueue.values()),
+      props.workflow.tags
+    )
+  },
+})

plugins/file-synchronizer/src/hooks/workflow-continued/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-continued/process-queue.ts

@@ -0,0 +1,31 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props)
+  const logger = props.logger.withWorkflowId(props.workflow.id)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  const { finished } = await SyncQueue.queueProcessor.processQueue({
+    logger,
+    syncQueue,
+    fileRepository: props.client,
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    updateSyncQueue: (params) => SyncQueue.jobFileManager.updateSyncQueue(props, key, params.syncQueue),
+  })
+
+  if (finished === 'batch') {
+    logger.info('Batch sync success. Continuing to next batch...')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    return
+  }
+
+  logger.info('Sync completed successfully')
+  await props.workflow.setCompleted()
+}

plugins/file-synchronizer/src/hooks/workflow-started/build-queue.ts

@@ -0,0 +1,24 @@
+import * as sdk from '@botpress/sdk'
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Creating job file...')
+  const syncFileKey = _generateFileKey(props)
+  const jobFileId = await SyncQueue.jobFileManager.updateSyncQueue(props, syncFileKey, [], props.workflow.tags)
+
+  await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { jobFileId })
+}
+
+const _generateFileKey = (props: bp.WorkflowHandlerProps['buildQueue']) => {
+  const integrationName = props.interfaces['files-readonly'].name
+  const syncJobId = props.workflow.tags.syncJobId
+
+  if (!syncJobId) {
+    throw new sdk.RuntimeError('Sync job ID is not defined in workflow tags')
+  }
+
+  return `file-synchronizer:${integrationName}:/${syncJobId}.jsonl`
+}

plugins/file-synchronizer/src/hooks/workflow-started/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-started/process-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+}

plugins/file-synchronizer/src/hooks/workflow-timed-out/build-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}

plugins/file-synchronizer/src/hooks/workflow-timed-out/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-timed-out/process-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}

plugins/file-synchronizer/src/index.ts

@@ -0,0 +1,93 @@
+import * as actions from './actions'
+import * as hooks from './hooks'
+import * as bp from '.botpress'
+
+const plugin = new bp.Plugin({
+  actions: {
+    async syncFilesToBotpess(props) {
+      props.logger.info('Called action syncFilesToBotpess')
+      return await actions.syncFilesToBotpess.callAction(props)
+    },
+    async listItemsInFolder(props) {
+      props.logger.info('Called action listItemsInFolder. Redirecting to integration...')
+      return await props.actions['files-readonly'].listItemsInFolder(props.input)
+    },
+  },
+})
+
+plugin.on.event('files-readonly:fileCreated', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File created event triggered', props.event.payload.file)
+  await hooks.onEvent.fileCreated.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:fileDeleted', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File deleted event triggered', props.event.payload.file)
+  await hooks.onEvent.fileDeleted.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:fileUpdated', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File updated event triggered', props.event.payload.file)
+  await hooks.onEvent.fileUpdated.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:folderDeletedRecursive', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('Folder deleted event triggered', props.event.payload.folder)
+  await hooks.onEvent.folderDeletedRecursive.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:aggregateFileChanges', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('Aggregate file changes event triggered', props.event.payload.modifiedItems)
+  await hooks.onEvent.aggregateFileChanges.handleEvent(props)
+})
+
+plugin.on.workflowStart('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow started', props.workflow.tags)
+  await hooks.onWorkflowStart.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowContinue('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow continued', props.workflow.tags)
+  await hooks.onWorkflowContinue.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowTimeout('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow timed out', props.workflow.tags)
+  await hooks.onWorkflowTimeout.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowStart('processQueue', async (props) => {
+  props.logger.info('processQueue workflow started', props.workflow.tags)
+  await hooks.onWorkflowStart.processQueue.handleEvent(props)
+})
+
+plugin.on.workflowContinue('processQueue', async (props) => {
+  props.logger.info('processQueue workflow continued', props.workflow.tags)
+  await hooks.onWorkflowContinue.processQueue.handleEvent(props)
+})
+
+plugin.on.workflowTimeout('processQueue', async (props) => {
+  props.logger.info('processQueue workflow timed out', props.workflow.tags)
+  await hooks.onWorkflowTimeout.processQueue.handleEvent(props)
+})
+
+export default plugin

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts

@@ -0,0 +1,326 @@
+import type * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { enumerateAllFilesRecursive, type EnumerationState } from './directory-traversal-with-batching'
+
+const _getMocks = () => ({
+  logger: {
+    debug: vi.fn(),
+  } as unknown as sdk.BotLogger,
+  integration: {
+    listItemsInFolder: vi.fn().mockResolvedValue({ items: [], meta: {} }),
+  },
+  globMatcher: {
+    matchItem: vi.fn().mockReturnValue({ shouldBeIgnored: false, shouldApplyOptions: {} }),
+  },
+  configuration: {
+    includeFiles: [{ pathGlobPattern: '**' }],
+    excludeFiles: [],
+  },
+  pushFilesToQueue: vi.fn(),
+})
+
+describe.concurrent('enumerateAllFilesRecursive', () => {
+  describe('Basic enumeration', () => {
+    it('should enumerate an empty root folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.integration.listItemsInFolder).toHaveBeenNthCalledWith(1, {
+        folderId: undefined,
+        nextToken: undefined,
+      })
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should enumerate a single file in root folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+      const testFile: models.FolderItem = {
+        id: 'file1',
+        type: 'file',
+        name: 'test.txt',
+      }
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [testFile],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          ...testFile,
+          absolutePath: '/test.txt',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Recursive folder traversal', () => {
+    it('should recursively enumerate files in nested folders', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // Root folder items:
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [
+          { id: 'folder1', type: 'folder', name: 'folder1' },
+          { id: 'file1', type: 'file', name: 'root-file.txt' },
+        ],
+        meta: { nextToken: undefined },
+      })
+
+      // folder1 items:
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file2', type: 'file', name: 'nested-file.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: '/root-file.txt',
+        }),
+        expect.objectContaining({
+          absolutePath: '/folder1/nested-file.txt',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should handle deeply nested folder structures', async () => {
+      // Arrange
+      const mocks = _getMocks()
+      const folderDepth = 100
+
+      // Create a chain of nested folders
+      for (let i = 1; i <= folderDepth; i++) {
+        mocks.integration.listItemsInFolder.mockResolvedValueOnce(
+          i === folderDepth
+            ? {
+                items: [{ id: 'deepFile', type: 'file', name: 'deepFile.txt' }],
+                meta: { nextToken: undefined },
+              }
+            : {
+                items: [{ id: `folder${i}`, type: 'folder', name: `folder${i}` }],
+                meta: { nextToken: undefined },
+              }
+        )
+      }
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(folderDepth)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: `/${Array.from({ length: folderDepth - 1 }, (_, i) => `folder${i + 1}`).join('/')}/deepFile.txt`,
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Pagination handling', () => {
+    it('should support pagination within a folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // First page of root folder
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file1', type: 'file', name: 'file1.txt' }],
+        meta: { nextToken: 'page2' },
+      })
+
+      // Second page of root folder
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file2', type: 'file', name: 'file2.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
+        expect.objectContaining({
+          id: 'file1',
+        }),
+        expect.objectContaining({
+          id: 'file2',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should resume using an enumeration state', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      const previousEnumerationState: EnumerationState = {
+        pendingFolders: [{ absolutePath: '/folder1/', folderId: 'folder1' }],
+      }
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file3', type: 'file', name: 'file3.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const newEnumerationState = await enumerateAllFilesRecursive({
+        ...mocks,
+        currentEnumerationState: previousEnumerationState,
+      })
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledWith({
+        folderId: 'folder1',
+        nextToken: undefined,
+      })
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: '/folder1/file3.txt',
+        }),
+      ])
+      expect(newEnumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Glob file filtering', () => {
+    it('should properly apply file filtering', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [
+          { id: 'file1', type: 'file', name: 'included.txt' },
+          { id: 'file2', type: 'file', name: 'excluded.txt' },
+        ],
+        meta: { nextToken: undefined },
+      })
+
+      // First file is included, second is excluded
+      mocks.globMatcher.matchItem
+        .mockReturnValueOnce({
+          shouldBeIgnored: false,
+          shouldApplyOptions: { addToKbId: 'kb1' },
+        })
+        .mockReturnValueOnce({
+          shouldBeIgnored: true,
+          reason: 'matches-exclude-pattern',
+        })
+
+      // Act
+      void (await enumerateAllFilesRecursive(mocks))
+
+      // Assert
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
+        expect.objectContaining({
+          id: 'file1',
+        }),
+      ])
+      expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object), expect.any(String))
+    })
+  })
+})
+
+describe.sequential('enumerateAllFilesRecursive duration handling', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    vi.useRealTimers()
+  })
+
+  it('should not proceed to the next batch in case of a timeout', async () => {
+    // Arrange
+    const mocks = _getMocks()
+    const maximumExecutionTimeMs = 5_000 // 5 seconds
+
+    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+      items: [
+        { id: 'file1', type: 'file', name: 'file1.txt' },
+        { id: 'file2', type: 'file', name: 'file2.txt' },
+        { id: 'file3', type: 'file', name: 'file3.txt' },
+      ],
+      meta: { nextToken: 'abcd' },
+    })
+
+    const resultPromise = enumerateAllFilesRecursive({
+      ...mocks,
+      maximumExecutionTimeMs,
+    })
+
+    // Advance time after the first batch is processed:
+    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)
+
+    void (await resultPromise)
+
+    // Assert
+    expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+  })
+
+  it('should process the entire current batch in case of a timeout', async () => {
+    // Arrange
+    const mocks = _getMocks()
+    const maximumExecutionTimeMs = 5_000 // 5 seconds
+
+    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+      items: [
+        { id: 'file1', type: 'file', name: 'file1.txt' },
+        { id: 'file2', type: 'file', name: 'file2.txt' },
+        { id: 'file3', type: 'file', name: 'file3.txt' },
+      ],
+      meta: { nextToken: 'abcd' },
+    })
+
+    const resultPromise = enumerateAllFilesRecursive({
+      ...mocks,
+      maximumExecutionTimeMs,
+    })
+
+    // Advance time after the first batch is processed:
+    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)
+
+    void (await resultPromise)
+
+    // Assert
+    expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+    expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+      expect.objectContaining({
+        id: 'file1',
+      }),
+      expect.objectContaining({
+        id: 'file2',
+      }),
+      expect.objectContaining({
+        id: 'file3',
+      }),
+    ])
+  })
+})

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts

@@ -0,0 +1,132 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import * as syncQueueGlobMatcher from './glob-matcher'
+import * as bp from '.botpress'
+
+type FileWithOptions = models.FileWithPath & { shouldIndex: boolean; addToKbId?: string }
+
+export type EnumerationState = NonNullable<bp.states.States['buildQueueRuntimeState']['payload']['enumerationState']>
+
+type IntegrationActionProxy = Pick<
+  bp.WorkflowHandlerProps['processQueue']['actions']['files-readonly'],
+  'listItemsInFolder'
+>
+
+export type EnumerateAllFilesRecursiveProps = {
+  logger: sdk.BotLogger
+  integration: IntegrationActionProxy
+  configuration: Pick<bp.configuration.Configuration, 'includeFiles' | 'excludeFiles'>
+  currentEnumerationState?: EnumerationState
+  maximumExecutionTimeMs?: number
+  globMatcher?: {
+    matchItem: (props: syncQueueGlobMatcher.GlobMatcherProps) => syncQueueGlobMatcher.GlobMatchResult
+  }
+  pushFilesToQueue: (files: FileWithOptions[]) => Promise<void>
+}
+
+// 30 seconds should be a safe default for the max execution time:
+const DEFAULT_MAXIMUM_EXECUTION_TIME_MS = 30_000
+
+export const enumerateAllFilesRecursive = async ({
+  logger,
+  integration,
+  configuration,
+  currentEnumerationState,
+  maximumExecutionTimeMs = DEFAULT_MAXIMUM_EXECUTION_TIME_MS,
+  globMatcher = syncQueueGlobMatcher,
+  pushFilesToQueue: pushFilesToSyncQueue,
+}: EnumerateAllFilesRecursiveProps): Promise<EnumerationState | undefined> => {
+  const enumerationState: EnumerationState = currentEnumerationState ?? {
+    pendingFolders: [{ absolutePath: '/' }],
+  }
+
+  const includedFiles: FileWithOptions[] = []
+  const startTime = Date.now()
+
+  while (enumerationState.pendingFolders.length > 0) {
+    const { items: folderItems, nextToken } = await _listFolderContents(integration, enumerationState)
+
+    for (const folderItem of folderItems) {
+      const currentFolder = enumerationState.pendingFolders[0]!
+      const itemPath = `${currentFolder.absolutePath}${folderItem.name}`
+      const globMatchResult = globMatcher.matchItem({ configuration, item: folderItem, itemPath })
+
+      if (globMatchResult.shouldBeIgnored) {
+        logger.debug(
+          'Ignoring item',
+          { itemPath, reason: globMatchResult.reason },
+          JSON.stringify({ item: folderItem, configuration })
+        )
+        continue
+      }
+
+      if (folderItem.type === 'folder') {
+        _addFolderToEnumerationQueue(enumerationState, folderItem.id, itemPath)
+      } else {
+        _addFileToIncludedFiles(includedFiles, folderItem, itemPath, globMatchResult, logger)
+      }
+    }
+
+    if (nextToken) {
+      enumerationState.currentFolderNextToken = nextToken
+
+      if (_isTimeoutReached(startTime, maximumExecutionTimeMs)) {
+        await pushFilesToSyncQueue(includedFiles)
+        return enumerationState
+      }
+    } else {
+      enumerationState.pendingFolders.shift()
+      enumerationState.currentFolderNextToken = undefined
+    }
+  }
+
+  await pushFilesToSyncQueue(includedFiles)
+  return undefined
+}
+
+const _isTimeoutReached = (startTime: number, maximumExecutionTimeMs: number): boolean =>
+  Date.now() - startTime >= maximumExecutionTimeMs
+
+const _listFolderContents = async (integration: IntegrationActionProxy, enumerationState: EnumerationState) => {
+  const folder = enumerationState.pendingFolders[0]!
+  const response = await integration.listItemsInFolder({
+    folderId: folder.folderId,
+    nextToken: enumerationState.currentFolderNextToken,
+  })
+
+  return {
+    items: response.items,
+    nextToken: response.meta.nextToken,
+  }
+}
+
+const _addFolderToEnumerationQueue = (
+  enumerationState: EnumerationState,
+  folderId: string | undefined,
+  itemPath: string
+): void => {
+  enumerationState.pendingFolders.push({
+    folderId,
+    absolutePath: `${itemPath}/`,
+  })
+}
+
+const _addFileToIncludedFiles = (
+  includedFiles: FileWithOptions[],
+  folderItem: models.File,
+  itemPath: string,
+  globMatchResult: syncQueueGlobMatcher.GlobMatchResult,
+  logger: sdk.BotLogger
+): void => {
+  if (globMatchResult.shouldBeIgnored) {
+    return
+  }
+
+  logger.debug('Including file', itemPath)
+  includedFiles.push({
+    ...folderItem,
+    absolutePath: itemPath,
+    shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+    addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+  })
+}

plugins/file-synchronizer/src/sync-queue/file-processor.test.ts

@@ -0,0 +1,276 @@
+import * as sdk from '@botpress/sdk'
+import { describe, it, expect, vi, type Mocked } from 'vitest'
+import { processQueueFile, type ProcessFileProps } from './file-processor'
+import type * as types from '../types'
+
+const FILE_1 = {
+  id: 'file1',
+  type: 'file',
+  name: 'file1.txt',
+  absolutePath: '/path/to/file1.txt',
+  sizeInBytes: 100,
+  lastModifiedDate: '2025-01-01T00:00:00Z',
+  contentHash: 'hash1',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const FILE_2 = {
+  id: 'file2',
+  type: 'file',
+  name: 'file2.txt',
+  absolutePath: '/path/to/file2.txt',
+  sizeInBytes: 200,
+  lastModifiedDate: '2025-01-02T00:00:00Z',
+  contentHash: 'hash2',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const EXISTING_FILE = 'dummy-id'
+const FUTURE_DATE = '9999-01-01T00:00:00Z'
+const PAST_DATE = '0000-01-01T00:00:00Z'
+const DIFFERENT_HASH = 'different-hash'
+
+describe.concurrent('processQueue', () => {
+  const getMocks = () => ({
+    fileRepository: {
+      listFiles: vi.fn(),
+      deleteFile: vi.fn(),
+      updateFileMetadata: vi.fn(),
+    } as const satisfies Mocked<ProcessFileProps['fileRepository']>,
+    integration: {
+      name: 'test-integration',
+      alias: 'test-integration-alias',
+      transferFileToBotpress: vi.fn(),
+    } as const satisfies Mocked<ProcessFileProps['integration']>,
+    logger: new sdk.BotLogger(),
+  })
+
+  it('should skip files that are already synced with identical content hash', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalContentHash: FILE_1.contentHash,
+          },
+        },
+      ],
+    })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+  })
+
+  it('should upload file when more recent and content hash is different', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: PAST_DATE,
+          },
+        },
+      ],
+    })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.deleteFile).toHaveBeenCalledWith({ id: EXISTING_FILE })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+  })
+
+  it('should not upload file when existing file is same age', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: FILE_1.lastModifiedDate,
+          },
+        },
+      ],
+    })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+  })
+
+  it('should not upload file when existing file is more recent', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: FUTURE_DATE,
+          },
+        },
+      ],
+    })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+  })
+
+  it('should upload file when different hash and missing last modified date', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: PAST_DATE,
+          },
+        },
+      ],
+    })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.deleteFile).toHaveBeenCalledWith({ id: EXISTING_FILE })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+  })
+
+  it('should handle errors by modifying the queue item', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+
+    // First transfer fails, second succeeds:
+    mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'errored', errorMessage: 'Transfer failed' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+    expect(mocks.fileRepository.updateFileMetadata).not.toHaveBeenCalled()
+  })
+
+  it('should properly set metadata tags after transferring file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
+      id: FILE_1.id,
+      tags: {
+        integrationName: 'test-integration',
+        integrationAlias: 'test-integration-alias',
+        externalId: FILE_1.id,
+        externalModifiedDate: FILE_1.lastModifiedDate,
+        externalSize: FILE_1.sizeInBytes.toString(),
+        externalContentHash: FILE_1.contentHash,
+        externalPath: FILE_1.absolutePath,
+        externalParentId: FILE_1.parentId,
+      },
+    })
+  })
+
+  it('should assign to kb if needed after transferring file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: { ...FILE_1, addToKbId: 'kb1' },
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
+      id: FILE_1.id,
+      tags: {
+        integrationName: 'test-integration',
+        integrationAlias: 'test-integration-alias',
+        externalId: FILE_1.id,
+        externalModifiedDate: FILE_1.lastModifiedDate,
+        externalSize: FILE_1.sizeInBytes.toString(),
+        externalContentHash: FILE_1.contentHash,
+        externalPath: FILE_1.absolutePath,
+        externalParentId: FILE_1.parentId,
+        kbId: 'kb1',
+        modalities: '["text"]',
+        source: 'knowledge-base',
+        title: FILE_1.name,
+      },
+    })
+  })
+})

plugins/file-synchronizer/src/sync-queue/file-processor.ts

@@ -0,0 +1,149 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import type * as types from '../types'
+
+export type ProcessFileProps = {
+  fileToSync: Readonly<types.SyncQueueItem>
+  logger: sdk.BotLogger
+  fileRepository: {
+    listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
+    deleteFile: (params: { id: string }) => Promise<unknown>
+    updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
+  }
+  integration: {
+    name: string
+    alias: string
+    transferFileToBotpress: (params: {
+      file: models.FileWithPath
+      fileKey: string
+      shouldIndex: boolean
+    }) => Promise<{ botpressFileId: string }>
+  }
+}
+
+export const processQueueFile = async (props: ProcessFileProps): Promise<types.SyncQueueItem> => {
+  const fileToSync = structuredClone(props.fileToSync) as types.SyncQueueItem
+  const existingFile = await _getExistingFileFromFilesApi(props, fileToSync)
+  const shouldUploadFile = await _shouldUploadFile(props, fileToSync, existingFile)
+
+  if (!shouldUploadFile) {
+    fileToSync.status = 'already-synced'
+    return fileToSync
+  }
+
+  fileToSync.status = 'newly-synced'
+
+  await _deleteExistingFileFromFilesApi(props, existingFile)
+  await _transferFileToBotpress(props, fileToSync)
+
+  return fileToSync
+}
+
+const _getExistingFileFromFilesApi = async (
+  props: ProcessFileProps,
+  fileToSync: models.FileWithPath
+): Promise<types.FilesApiFile | undefined> => {
+  const { files: existingFiles } = await props.fileRepository.listFiles({
+    tags: {
+      externalId: fileToSync.id,
+      integrationName: props.integration.name,
+      integrationAlias: props.integration.alias,
+    },
+  })
+
+  if (existingFiles.length === 0) {
+    return
+  }
+
+  // Unfortunately, we cannot assume that there is only one file on Files API
+  // with the same externalId, because there is no unique constraint on the tag.
+
+  // However, the implementation would be more complicated if we take this into
+  // account, so we will naively assume that we have full control over the
+  // externalId tag and that it is unique. If users go out of their way to use
+  // the API to create files with the same externalId, we will not be able to
+  // handle this case correctly.
+
+  return existingFiles[0]!
+}
+
+const _shouldUploadFile = async (
+  props: ProcessFileProps,
+  fileToSync: models.FileWithPath,
+  existingFile?: types.FilesApiFile
+) => {
+  if (!existingFile) {
+    props.logger.debug(`No existing file found. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  const newFileHasIdenticalContentHash =
+    fileToSync.contentHash &&
+    existingFile.tags.externalContentHash &&
+    fileToSync.contentHash === existingFile.tags.externalContentHash
+
+  if (newFileHasIdenticalContentHash) {
+    props.logger.debug(`An identical file already exists in Botpress. Ignoring ${fileToSync.absolutePath} ...`)
+    return false
+  }
+
+  const bothFilesHaveModifiedDate = fileToSync.lastModifiedDate && existingFile.tags.externalModifiedDate
+
+  if (!bothFilesHaveModifiedDate) {
+    // Not enough information to compare the files, so we always overwrite:
+    props.logger.debug(`Not enough information to compare files. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  const newFileIsMoreRecent = new Date(fileToSync.lastModifiedDate!) > new Date(existingFile.tags.externalModifiedDate!)
+
+  if (newFileIsMoreRecent) {
+    props.logger.debug(`New file is more recent. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  props.logger.debug(`Existing file is more recent or same date. Ignoring ${fileToSync.absolutePath} ...`)
+  return false
+}
+
+const _deleteExistingFileFromFilesApi = async (props: ProcessFileProps, existingFile?: types.FilesApiFile) => {
+  if (!existingFile) {
+    return
+  }
+
+  try {
+    await props.fileRepository.deleteFile({ id: existingFile.id })
+  } catch {}
+}
+
+const _transferFileToBotpress = async (props: ProcessFileProps, fileToSync: types.SyncQueueItem) => {
+  try {
+    const { botpressFileId } = await props.integration.transferFileToBotpress({
+      file: fileToSync,
+      fileKey: `${props.integration.alias}:${fileToSync.absolutePath}`,
+      shouldIndex: fileToSync.shouldIndex,
+    })
+
+    await props.fileRepository.updateFileMetadata({
+      id: botpressFileId,
+      tags: {
+        integrationName: props.integration.name,
+        integrationAlias: props.integration.alias,
+        externalId: fileToSync.id,
+        externalModifiedDate: fileToSync.lastModifiedDate ?? null,
+        externalSize: fileToSync.sizeInBytes?.toString() ?? null,
+        externalContentHash: fileToSync.contentHash ?? null,
+        externalPath: fileToSync.absolutePath,
+        externalParentId: fileToSync.parentId ?? null,
+        ...(fileToSync.addToKbId !== undefined
+          ? { kbId: fileToSync.addToKbId, source: 'knowledge-base', title: fileToSync.name, modalities: '["text"]' }
+          : {}),
+      },
+    })
+  } catch (thrown: unknown) {
+    const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    fileToSync.status = 'errored'
+    fileToSync.errorMessage = err.message
+    props.logger.error(`Error while uploading file ${fileToSync.absolutePath}: ${err.message}`)
+  }
+}

plugins/file-synchronizer/src/sync-queue/glob-matcher.test.ts

@@ -0,0 +1,397 @@
+import { describe, it, expect } from 'vitest'
+import { matchItem } from './glob-matcher'
+import type * as models from '../../definitions/models'
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+import type * as bp from '.botpress'
+
+describe.concurrent('matchItem', () => {
+  const createConfiguration = ({
+    includeFiles,
+    excludeFiles,
+  }: {
+    includeFiles?: bp.configuration.Configuration['includeFiles']
+    excludeFiles?: bp.configuration.Configuration['excludeFiles']
+  }) =>
+    ({
+      includeFiles: includeFiles ?? [],
+      excludeFiles: excludeFiles ?? [],
+    }) as const
+
+  describe.concurrent('with file items', () => {
+    const createFileItem = (overrides: Readonly<Partial<models.File>> = {}) =>
+      ({
+        id: 'file-1',
+        type: 'file',
+        name: 'test-file.txt',
+        sizeInBytes: 1000,
+        lastModifiedDate: '1965-01-01T00:00:00Z',
+        ...overrides,
+      }) as const satisfies models.File
+
+    it('should exclude when path matches explicit exclude pattern', () => {
+      // Arrange
+      const itemPath = 'src/data/excluded-file.txt'
+      const configuration = createConfiguration({
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*.txt' }],
+      })
+      const item = createFileItem({ name: 'excluded-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should include when path matches include pattern with no restrictions', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*.txt', applyOptionsToMatchedFiles: { addToKbId: 'kbId' } }],
+      })
+      const item = createFileItem({ name: 'included-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+        shouldApplyOptions: { addToKbId: 'kbId' },
+      })
+    })
+
+    it('should exclude when path does not match any patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/unknown-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*.txt' }],
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*.txt' }],
+      })
+      const item = createFileItem({ name: 'unknown-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+
+    it('should exclude when file size exceeds maxSizeInBytes', () => {
+      // Arrange
+      const itemPath = 'src/data/large-file.txt'
+      const maxSizeInBytes = 1000
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/large-*.txt', maxSizeInBytes }],
+      })
+      const item = createFileItem({
+        name: 'large-file.txt',
+        sizeInBytes: maxSizeInBytes + 1,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'unmet-include-requirements',
+      })
+    })
+
+    it('should ignore maxSizeInBytes when set to 0', () => {
+      // Arrange
+      const itemPath = 'src/data/valid-file.txt'
+      const maxSizeInBytes = 0
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/valid-*.txt',
+            maxSizeInBytes,
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'valid-file.txt',
+        sizeInBytes: 100,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should allow files larger than MAX_BATCH_SIZE_BYTES', () => {
+      // Arrange
+      const itemPath = 'src/data/large-file.txt'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/large-*.txt' }] })
+      const item = createFileItem({
+        name: 'large-file.txt',
+        sizeInBytes: MAX_BATCH_SIZE_BYTES + 1,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should exclude when modified before modifiedAfter date', () => {
+      // Arrange
+      const itemPath = 'src/data/old-file.txt'
+      const modifiedAfter = '1965-02-01T00:00:00Z'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/old-*.txt', modifiedAfter }] })
+      const item = createFileItem({
+        name: 'old-file.txt',
+        lastModifiedDate: '1965-01-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'unmet-include-requirements',
+      })
+    })
+
+    it('should include when modified after modifiedAfter date', () => {
+      // Arrange
+      const itemPath = 'src/data/new-file.txt'
+      const modifiedAfter = '1965-01-01T00:00:00Z'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/new-*.txt', modifiedAfter }] })
+      const item = createFileItem({
+        name: 'new-file.txt',
+        lastModifiedDate: '1965-02-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should include when file meets all requirements', () => {
+      // Arrange
+      const itemPath = 'src/data/valid-file.txt'
+      const maxSizeInBytes = 2000
+      const modifiedAfter = '1965-01-01T00:00:00Z'
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/valid-*.txt',
+            maxSizeInBytes,
+            modifiedAfter,
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'valid-file.txt',
+        sizeInBytes: maxSizeInBytes - 100,
+        lastModifiedDate: '1965-02-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle missing optional properties', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/included-*.txt',
+            maxSizeInBytes: 1000,
+            modifiedAfter: '1965-01-01T00:00:00Z',
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'included-file.txt',
+        sizeInBytes: undefined,
+        lastModifiedDate: undefined,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle multiple include patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/not-matching-*.txt' }, { pathGlobPattern: '**/included-*.txt' }],
+      })
+      const item = createFileItem({ name: 'included-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle multiple include patterns with different restrictions', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [
+          { pathGlobPattern: '**/included-*.txt', maxSizeInBytes: 1 },
+          { pathGlobPattern: '**/included-*.txt', maxSizeInBytes: 100 },
+        ],
+      })
+      const item = createFileItem({ name: 'included-file.txt', sizeInBytes: 100 })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should prioritize excludeFiles over includeFiles', () => {
+      // Arrange
+      const itemPath = 'src/data/both-match.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/both-*.txt' }],
+        excludeFiles: [{ pathGlobPattern: '**/both-*.txt' }],
+      })
+      const item = createFileItem({ name: 'both-match.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should exclude by default when there are no defined include patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/any-file.txt'
+      const configuration = createConfiguration({})
+      const item = createFileItem({ name: 'any-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+  })
+
+  describe.concurrent('with folder items', () => {
+    const createFolderItem = (overrides: Readonly<Partial<models.Folder>> = {}) =>
+      ({
+        id: 'folder-1',
+        type: 'folder',
+        name: 'test-folder',
+        ...overrides,
+      }) as const satisfies models.Folder
+
+    it('should exclude when path matches explicit exclude pattern', () => {
+      // Arrange
+      const itemPath = 'src/data/__ignored'
+      const configuration = createConfiguration({ excludeFiles: [{ pathGlobPattern: '**/__ignored' }] })
+      const item = createFolderItem({ name: '__ignored' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should include when path matches include pattern', () => {
+      // Arrange
+      const itemPath = 'src/data'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/data' }] })
+      const item = createFolderItem({ name: 'data' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it.each([
+      { path: '/foo/bar', glob: '/foo/bar/baz/**' },
+      { path: '/abc', glob: '/abc/[def]/[ghi]/**' },
+      { path: '/abc/[def]', glob: '/abc/[def]/[ghi]/**' },
+    ])('should include when path matches part of include pattern', ({ path: itemPath, glob: pathGlobPattern }) => {
+      // Arrange
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern }] })
+      const item = createFolderItem({ name: 'bar' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should exclude when path does not match any patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/unknown-folder'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*' }],
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*' }],
+      })
+      const item = createFolderItem({ name: 'unknown-folder' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+  })
+})

plugins/file-synchronizer/src/sync-queue/glob-matcher.ts

@@ -0,0 +1,129 @@
+import * as picomatch from 'picomatch'
+import type * as models from '../../definitions/models'
+import * as bp from '.botpress'
+
+export type GlobMatcherProps = {
+  configuration: Pick<bp.configuration.Configuration, 'includeFiles' | 'excludeFiles'>
+  item: models.FolderItem
+  itemPath: string
+}
+
+export type GlobMatchResult =
+  | {
+      shouldBeIgnored: false
+      shouldApplyOptions: Exclude<
+        GlobMatcherProps['configuration']['includeFiles'][number]['applyOptionsToMatchedFiles'],
+        undefined
+      >
+    }
+  | {
+      shouldBeIgnored: true
+      reason: 'matches-exclude-pattern' | 'unmet-include-requirements' | 'does-not-match-any-pattern'
+    }
+
+export const matchItem = ({ configuration, item, itemPath }: GlobMatcherProps): GlobMatchResult => {
+  for (const { pathGlobPattern } of configuration.excludeFiles) {
+    if (!pathGlobPattern) {
+      continue
+    }
+
+    if (picomatch.isMatch(itemPath, pathGlobPattern)) {
+      return {
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      }
+    }
+  }
+
+  let matchesButHasUnmetRequirements = false
+
+  for (const { pathGlobPattern, applyOptionsToMatchedFiles, ...requirements } of configuration.includeFiles) {
+    const isAncestorFolder = item.type === 'folder' && _isAncestorOfGlob(itemPath, pathGlobPattern)
+
+    if (!pathGlobPattern || (!_isMatch(itemPath, pathGlobPattern) && !isAncestorFolder)) {
+      continue
+    }
+
+    if (_isFileWithUnmetRequirements(item, requirements)) {
+      matchesButHasUnmetRequirements = true
+      continue
+    }
+
+    return {
+      shouldBeIgnored: false,
+      shouldApplyOptions: applyOptionsToMatchedFiles ?? {},
+    }
+  }
+
+  return {
+    shouldBeIgnored: true,
+    reason: matchesButHasUnmetRequirements ? 'unmet-include-requirements' : 'does-not-match-any-pattern',
+  }
+}
+
+const _isMatch = (itemPath: string, globPattern: string) =>
+  picomatch.isMatch(itemPath, globPattern, {
+    // allow dotfiles to match:
+    dot: true,
+    // escape brackets in the glob pattern so that only literal brackets are matched:
+    literalBrackets: true,
+    nobracket: true,
+  })
+
+type FileRequirements = Omit<
+  GlobMatcherProps['configuration']['includeFiles'][number],
+  'pathGlobPattern' | 'applyOptionsToMatchedFiles'
+>
+
+const _isFileWithUnmetRequirements = (
+  item: models.FolderItem,
+  { maxSizeInBytes, modifiedAfter }: FileRequirements
+): boolean => {
+  if (item.type !== 'file') {
+    return false
+  }
+
+  const exceedsUserDefinedMaxSize =
+    maxSizeInBytes !== undefined &&
+    maxSizeInBytes > 0 &&
+    item.sizeInBytes !== undefined &&
+    item.sizeInBytes > maxSizeInBytes
+
+  const isItemOlderThanGivenDate =
+    modifiedAfter !== undefined &&
+    modifiedAfter.length > 0 &&
+    item.lastModifiedDate !== undefined &&
+    new Date(item.lastModifiedDate) < new Date(modifiedAfter)
+
+  return exceedsUserDefinedMaxSize || isItemOlderThanGivenDate
+}
+
+const _isAncestorOfGlob = (candidatePath: string, globPattern: string): boolean =>
+  _isAncestorPath(candidatePath, _extractStaticPrefix(globPattern))
+
+const _extractStaticPrefix = (globPattern: string): string => {
+  const wildcardIndex = globPattern.search(/[*?{}]/)
+  if (wildcardIndex === -1) {
+    return globPattern
+  }
+
+  const prefix = globPattern.substring(0, wildcardIndex)
+  const lastSlashIndex = prefix.lastIndexOf('/')
+
+  return lastSlashIndex === -1 ? '' : prefix.substring(0, lastSlashIndex)
+}
+
+const _isAncestorPath = (ancestor: string, descendant: string): boolean => {
+  if (ancestor === descendant) {
+    return true
+  }
+
+  const ancestorParts = ancestor.split('/').filter((part) => part !== '')
+  const descendantParts = descendant.split('/').filter((part) => part !== '')
+
+  if (ancestorParts.length >= descendantParts.length) {
+    return false
+  }
+
+  return ancestorParts.every((part, index) => part === descendantParts[index])
+}

plugins/file-synchronizer/src/sync-queue/index.ts

@@ -0,0 +1,5 @@
+export * as queueProcessor from './queue-processor'
+export * as fileProcessor from './file-processor'
+export * as jobFileManager from './job-file-manager'
+export * as globMatcher from './glob-matcher'
+export * as directoryTraversalWithBatching from './directory-traversal-with-batching'

plugins/file-synchronizer/src/sync-queue/job-file-manager.ts

@@ -0,0 +1,64 @@
+import * as sdk from '@botpress/sdk'
+import * as models from '../../definitions/models'
+import type * as types from '../types'
+import * as utils from '../utils'
+import * as bp from '.botpress'
+
+const QUEUE_ITEM = models.FILE_WITH_PATH.extend({
+  status: sdk.z.enum(['pending', 'newly-synced', 'already-synced', 'errored']),
+  errorMessage: sdk.z.string().optional(),
+  shouldIndex: sdk.z.boolean(),
+  addToKbId: sdk.z.string().optional(),
+})
+
+export const getSyncQueue = async (
+  props: bp.WorkflowHandlerProps['processQueue'] | bp.WorkflowHandlerProps['buildQueue'],
+  jobFileId?: string
+): Promise<{ syncQueue: types.SyncQueue; key: string }> => {
+  const { jobFileContent, key } = await _retrieveJobFile(props, jobFileId).catch(async (thrown: unknown) => {
+    const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    await props.workflow.setFailed({ failureReason: `Failed to retrieve job file: ${err.message}` })
+    throw new Error(`Failed to retrieve job file: ${thrown}`)
+  })
+
+  const syncQueue: types.SyncQueue = []
+  const syncQueueGenerator = utils.jsonl.parseJsonLines(jobFileContent, QUEUE_ITEM)
+
+  for (const item of syncQueueGenerator) {
+    if ('error' in item) {
+      props.logger.error('Error while parsing line in job file. This line will be ignored.', item)
+      continue
+    }
+
+    syncQueue.push(item.value)
+  }
+
+  return { syncQueue, key }
+}
+
+const _retrieveJobFile = async (
+  props: bp.WorkflowHandlerProps['processQueue'] | bp.WorkflowHandlerProps['buildQueue'],
+  jobFileId?: string
+): Promise<{ jobFileContent: string; key: string }> => {
+  const { file: jobFile } = await props.client.getFile({
+    id: props.workflow.input.jobFileId ?? jobFileId,
+  })
+  const jobFileContent = await fetch(jobFile.url).then((res) => res.text())
+
+  return { jobFileContent, key: jobFile.key }
+}
+
+export const updateSyncQueue = async (
+  props: types.CommonProps,
+  key: string,
+  syncQueue: types.SyncQueue,
+  tags?: Record<string, string>
+): Promise<string> => {
+  const { file: jobFile } = await props.client.uploadFile({
+    key,
+    content: syncQueue.map((item) => JSON.stringify(item)).join('\n') + '\n',
+    tags,
+  })
+
+  return jobFile.id
+}

plugins/file-synchronizer/src/sync-queue/queue-batching.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from 'vitest'
+import { findBatchEndCursor } from './queue-batching'
+import { MAX_BATCH_SIZE_BYTES as maxBatchSize } from '../consts'
+
+describe.concurrent('findBatchEndCursor', () => {
+  it.each([
+    { startCursor: 0, files: [{ sizeInBytes: 100 }, { sizeInBytes: 200 }], expectedEndCursors: [2] },
+    { startCursor: 0, files: [{ sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
+    { startCursor: 0, files: [{ sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 3, 4],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 3, 4],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize - 200 }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 4],
+    },
+    {
+      startCursor: 2,
+      files: [
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: maxBatchSize - 200 },
+        { sizeInBytes: 200 },
+      ],
+      expectedEndCursors: [4, 6],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }],
+      expectedEndCursors: [2, 3],
+    },
+  ])(
+    'should correctly calculate the end cursors for a batch of files',
+    ({ startCursor, files, expectedEndCursors }) => {
+      // Act
+      const endCursors: number[] = []
+      let currentCursor = startCursor
+
+      while (currentCursor < files.length) {
+        currentCursor = findBatchEndCursor({ startCursor: currentCursor, files }).endCursor
+        endCursors.push(currentCursor)
+      }
+
+      // Assert
+      expect(endCursors).toStrictEqual(expectedEndCursors)
+    }
+  )
+})

plugins/file-synchronizer/src/sync-queue/queue-batching.ts

@@ -0,0 +1,41 @@
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+import type * as types from '../types'
+
+type SimpleFile = Pick<types.SyncQueueItem, 'sizeInBytes'>
+
+export const findBatchEndCursor = ({
+  startCursor: unsafeStartCursor,
+  files,
+}: {
+  startCursor: number
+  files: SimpleFile[]
+}): { endCursor: number } => {
+  const startCursor = Math.min(Math.max(unsafeStartCursor, 0), files.length - 1)
+  let currentBatchSize = 0
+  let endCursor = files.length
+
+  for (let newCursor = startCursor; newCursor < files.length; newCursor++) {
+    const fileToSync = files[newCursor]
+    const fileSizeInBytes = _getFileSize(fileToSync!)
+    currentBatchSize += fileSizeInBytes
+
+    // First file is always included even if it exceeds the batch size:
+    if (newCursor > startCursor && currentBatchSize > MAX_BATCH_SIZE_BYTES) {
+      endCursor = newCursor
+      break
+    }
+  }
+
+  return { endCursor }
+}
+
+const _getFileSize = (file: SimpleFile) => {
+  if (file.sizeInBytes === undefined) {
+    // If a file has no size, we assume it takes up half the batch size limit.
+    // This should be relatively safe, since we'd process at most two files of
+    // unknown size in a batch, and we don't want to just skip them.
+    return MAX_BATCH_SIZE_BYTES / 2
+  }
+
+  return file.sizeInBytes
+}

plugins/file-synchronizer/src/sync-queue/queue-processor.test.ts

@@ -0,0 +1,187 @@
+import * as sdk from '@botpress/sdk'
+import { describe, it, expect, vi, type Mocked, type Mock } from 'vitest'
+import { processQueue, type ProcessQueueProps } from './queue-processor'
+import type * as types from '../types'
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+
+const FILE_1 = {
+  id: 'file1',
+  type: 'file',
+  name: 'file1.txt',
+  absolutePath: '/path/to/file1.txt',
+  sizeInBytes: 100,
+  lastModifiedDate: '2025-01-01T00:00:00Z',
+  contentHash: 'hash1',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const FILE_2 = {
+  id: 'file2',
+  type: 'file',
+  name: 'file2.txt',
+  absolutePath: '/path/to/file2.txt',
+  sizeInBytes: 200,
+  lastModifiedDate: '2025-01-02T00:00:00Z',
+  contentHash: 'hash2',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const LARGE_FILE = {
+  id: 'largefile',
+  type: 'file',
+  name: 'largefile.txt',
+  absolutePath: '/path/to/largefile.txt',
+  sizeInBytes: MAX_BATCH_SIZE_BYTES - 200,
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+describe.concurrent('processQueue', () => {
+  const getMocks = () => ({
+    fileRepository: {
+      listFiles: vi.fn(),
+      deleteFile: vi.fn(),
+      updateFileMetadata: vi.fn(),
+    } as const satisfies Mocked<ProcessQueueProps['fileRepository']>,
+    integration: {
+      name: 'test-integration',
+      alias: 'test-integration-alias',
+      transferFileToBotpress: vi.fn(),
+    } as const satisfies Mocked<ProcessQueueProps['integration']>,
+    updateSyncQueue: vi.fn() as Mock<ProcessQueueProps['updateSyncQueue']>,
+    logger: new sdk.BotLogger(),
+  })
+
+  it('should process all files in queue when size is within batch limit', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+  })
+
+  it('should process files in batches when size exceeds batch limit', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, LARGE_FILE, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'batch' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[2]?.status).toBe('pending')
+  })
+
+  it('should handle errors by continuing to the next file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+
+    // First transfer fails, second succeeds:
+    mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('errored')
+    expect(updatedQueue?.[0]?.errorMessage).toContain('Transfer failed')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+  })
+
+  it('should handle complex batching', async () => {
+    // >>>>>>>>>>> FIRST BATCH <<<<<<<<<<<<<
+
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const firstRun = await processQueue({
+      syncQueue: [FILE_2, FILE_1, LARGE_FILE],
+      ...mocks,
+    })
+
+    // Assert
+    expect(firstRun).toEqual({ finished: 'batch' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue!
+
+    expect(updatedQueue[0]?.status).toBe('newly-synced')
+    expect(updatedQueue[1]?.status).toBe('newly-synced')
+    expect(updatedQueue[2]?.status).toBe('pending')
+
+    // >>>>>>>>>>> SECOND BATCH <<<<<<<<<<<<<
+
+    // Arrange
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
+
+    // Act
+    const secondRun = await processQueue({
+      syncQueue: updatedQueue!,
+      ...mocks,
+    })
+
+    // Assert
+    expect(secondRun).toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(3)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(2)
+
+    const finalQueue = mocks.updateSyncQueue.mock.calls[1]?.[0].syncQueue!
+
+    expect(finalQueue[0]?.status).toBe('newly-synced')
+    expect(finalQueue[1]?.status).toBe('newly-synced')
+    expect(finalQueue[2]?.status).toBe('newly-synced')
+  })
+})

plugins/file-synchronizer/src/sync-queue/queue-processor.ts

@@ -0,0 +1,57 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import type * as types from '../types'
+import { processQueueFile } from './file-processor'
+import { findBatchEndCursor } from './queue-batching'
+
+export type ProcessQueueProps = {
+  syncQueue: Readonly<types.SyncQueue>
+  logger: sdk.BotLogger
+  fileRepository: {
+    listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
+    deleteFile: (params: { id: string }) => Promise<unknown>
+    updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
+  }
+  integration: {
+    name: string
+    alias: string
+    transferFileToBotpress: (params: {
+      file: models.FileWithPath
+      fileKey: string
+      shouldIndex: boolean
+    }) => Promise<{ botpressFileId: string }>
+  }
+  updateSyncQueue: (props: { syncQueue: types.SyncQueue }) => Promise<unknown>
+}
+
+export const processQueue = async (props: ProcessQueueProps) => {
+  // Short-circuit if the sync queue is empty:
+  if (!props.syncQueue?.length) {
+    props.logger.info('Sync queue is empty. Nothing to process.')
+    return { finished: 'all' } as const
+  }
+
+  const syncQueue = structuredClone(props.syncQueue) as types.SyncQueue
+  const startCursor = syncQueue.findIndex((file) => file.status === 'pending') ?? syncQueue.length - 1
+
+  if (startCursor < 0) {
+    props.logger.info('No files left to process in the sync queue.')
+    return { finished: 'all' } as const
+  }
+
+  const { endCursor } = findBatchEndCursor({ startCursor, files: syncQueue })
+  const filesInBatch = syncQueue.slice(startCursor, endCursor)
+
+  for (const fileToSync of filesInBatch) {
+    const processedFile = await processQueueFile({ ...props, fileToSync })
+    Object.assign(fileToSync, processedFile)
+  }
+
+  await props.updateSyncQueue({ syncQueue })
+
+  if (endCursor < syncQueue.length) {
+    return { finished: 'batch' } as const
+  }
+
+  return { finished: 'all' } as const
+}

plugins/file-synchronizer/src/types.ts

@@ -0,0 +1,21 @@
+import * as sdk from '@botpress/sdk'
+import { CommonHandlerProps } from '@botpress/sdk/dist/plugin'
+import type * as models from '../definitions/models'
+import type * as bp from '.botpress'
+
+type TPlugin = sdk.DefaultPlugin<bp.TPlugin>
+
+export type SyncQueueItem = models.FileWithPath & {
+  status: 'pending' | 'newly-synced' | 'already-synced' | 'errored'
+  errorMessage?: string
+  shouldIndex: boolean
+  addToKbId?: string
+}
+export type SyncQueue = SyncQueueItem[]
+
+export type CommonProps = CommonHandlerProps<TPlugin>
+
+export type FilesApiFile = {
+  id: string
+  tags: Record<string, string>
+}

plugins/file-synchronizer/src/utils/index.ts

@@ -0,0 +1 @@
+export * as jsonl from './json-lines'

plugins/file-synchronizer/src/utils/json-lines/index.ts

@@ -0,0 +1 @@
+export * from './parser'

plugins/file-synchronizer/src/utils/json-lines/parser.test.ts

@@ -0,0 +1,160 @@
+import { describe, it, expect } from 'vitest'
+import * as sdk from '@botpress/sdk'
+import { parseJsonLines } from './parser'
+
+describe.concurrent('parseJsonLines', () => {
+  const STRING_SCHEMA = sdk.z.string()
+  const NUMBER_SCHEMA = sdk.z.number()
+  const USER_SCHEMA = sdk.z.object({
+    id: sdk.z.number(),
+    name: sdk.z.string(),
+    email: sdk.z.string().email(),
+  })
+
+  it('should parse valid JSON lines with a string schema', () => {
+    // Arrange
+    const input = '"hello"\n"world"\n"test"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'hello' }, { value: 'world' }, { value: 'test' }])
+  })
+
+  it('should parse valid JSON lines with an object schema', () => {
+    // Arrange
+    const input =
+      '{"id": 1, "name": "John", "email": "john@example.com"}\n' +
+      '{"id": 2, "name": "Jane", "email": "jane@example.com"}\n' +
+      '{"id": 3, "name": "Bob", "email": "bob@example.com"}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, USER_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([
+      {
+        value: { id: 1, name: 'John', email: 'john@example.com' },
+      },
+      {
+        value: { id: 2, name: 'Jane', email: 'jane@example.com' },
+      },
+      { value: { id: 3, name: 'Bob', email: 'bob@example.com' } },
+    ])
+  })
+
+  it('should handle empty input strings', () => {
+    // Arrange
+    const input = ''
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([])
+  })
+
+  it('should handle input with only whitespace', () => {
+    // Arrange
+    const input = '   \n  \n\t  '
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([])
+  })
+
+  it('should skip empty lines', () => {
+    // Arrange
+    const input = '"first"\n\n"second"\n\n\n"third"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'first' }, { value: 'second' }, { value: 'third' }])
+  })
+
+  it('should handle trailing newlines', () => {
+    // Arrange
+    const input = '"line1"\n"line2"\n'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }])
+  })
+
+  it('should handle input without newlines', () => {
+    // Arrange
+    const input = '"single line"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'single line' }])
+  })
+
+  it('should return a Zod error when a line fails Zod validation', () => {
+    // Arrange
+    const input = '{"id": 1, "name": "John", "email": "invalid-email"}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, USER_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([
+      {
+        rawLine: '{"id": 1, "name": "John", "email": "invalid-email"}',
+        error: expect.objectContaining({ __type__: 'ZuiError' }),
+      },
+    ])
+  })
+
+  it('should parse to the end, even with errors', () => {
+    // Arrange
+    const input = '1\n2\n"not a number"\n4'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, NUMBER_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([
+      { rawLine: '1', value: 1 },
+      { rawLine: '2', value: 2 },
+      { rawLine: '"not a number"', error: expect.objectContaining({ __type__: 'ZuiError' }) },
+      { rawLine: '4', value: 4 },
+    ])
+  })
+
+  it('should handle different line endings (CRLF)', () => {
+    // Arrange
+    const input = '"line1"\r\n"line2"\r\n"line3"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }, { value: 'line3' }])
+  })
+
+  it('should parse properly with schema containing optional fields', () => {
+    // Arrange
+    const optionalSchema = sdk.z.object({
+      id: sdk.z.number(),
+      name: sdk.z.string(),
+      age: sdk.z.number().optional(),
+    })
+    const input = '{"id": 1, "name": "John"}\n{"id": 2, "name": "Jane", "age": 25}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, optionalSchema))
+
+    // Assert
+    expect(result).toMatchObject([{ value: { id: 1, name: 'John' } }, { value: { id: 2, name: 'Jane', age: 25 } }])
+  })
+})

plugins/file-synchronizer/src/utils/json-lines/parser.ts

@@ -0,0 +1,26 @@
+import * as sdk from '@botpress/sdk'
+
+type _JsonParseResult<T> = { rawLine: string } & ({ value: T } | { error: Error })
+
+export function* parseJsonLines<TLineSchema extends sdk.z.ZodTypeAny>(
+  rawJsonLines: string,
+  zodSchema: TLineSchema
+): Generator<_JsonParseResult<sdk.z.infer<TLineSchema>>, void, undefined> {
+  let startCursor = 0
+
+  for (let endCursor = 0; endCursor <= rawJsonLines.length; endCursor++) {
+    if (rawJsonLines[endCursor] === '\n' || endCursor === rawJsonLines.length) {
+      const line = rawJsonLines.slice(startCursor, endCursor).trim()
+      startCursor = endCursor + 1
+
+      if (line) {
+        try {
+          yield { rawLine: line, value: zodSchema.parse(JSON.parse(line)) }
+        } catch (thrown: unknown) {
+          const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+          yield { rawLine: line, error: err }
+        }
+      }
+    }
+  }
+}

plugins/file-synchronizer/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": { "*": ["./*"] },
+    "outDir": "dist"
+  },
+  "include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
+}

plugins/file-synchronizer/vitest.config.ts

@@ -0,0 +1,2 @@
+import config from '../../vitest.config'
+export default config