chore: import upstream snapshot with attribution

2026-07-13 13:34:48 +08:00
commit 77bb5bf71f
3762 changed files with 353249 additions and 0 deletions
@@ -0,0 +1,4 @@
+export * as onEvent from './on-event'
+export * as onWorkflowStart from './workflow-started'
+export * as onWorkflowContinue from './workflow-continued'
+export * as onWorkflowTimeout from './workflow-timed-out'
@@ -0,0 +1,45 @@
+import { handleEvent as handleFileCreated } from './file-created'
+import { handleEvent as handleFileDeleted } from './file-deleted'
+import { handleEvent as handleFileUpdated } from './file-updated'
+import { handleEvent as handleFolderDeletedRecursive } from './folder-deleted-recursive'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:aggregateFileChanges'] = async (props) => {
+  const modifiedItems = props.event.payload.modifiedItems
+
+  for (const deletedItem of modifiedItems.deleted) {
+    if (deletedItem.type === 'file') {
+      await handleFileDeleted({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:fileDeleted', payload: { file: deletedItem } },
+      })
+    } else {
+      await handleFolderDeletedRecursive({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:folderDeletedRecursive', payload: { folder: deletedItem } },
+      })
+    }
+  }
+
+  for (const createdItem of modifiedItems.created) {
+    if (createdItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileCreated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileCreated', payload: { file: createdItem } },
+    })
+  }
+
+  for (const updatedItem of modifiedItems.updated) {
+    if (updatedItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileUpdated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileUpdated', payload: { file: updatedItem } },
+    })
+  }
+}
@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileCreated'] = async (props) => {
+  const createdFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: createdFile,
+    itemPath: createdFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${createdFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...createdFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${createdFile.absolutePath} has been synchronized`)
+}
@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileDeleted'] = async (props) => {
+  const deletedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFile,
+    itemPath: deletedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${deletedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalId: deletedFile.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFile.absolutePath} has been deleted`)
+    }
+  } catch {}
+}
@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileUpdated'] = async (props) => {
+  const updatedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: updatedFile,
+    itemPath: updatedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${updatedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...updatedFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${updatedFile.absolutePath} has been synchronized`)
+}
@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:folderDeletedRecursive'] = async (props) => {
+  const deletedFolder = props.event.payload.folder
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFolder,
+    itemPath: deletedFolder.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring folder ${deletedFolder.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalParentId: deletedFolder.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFolder.absolutePath} has been deleted`)
+    }
+  } catch {}
+}
@@ -0,0 +1,5 @@
+export * as fileCreated from './file-created'
+export * as fileDeleted from './file-deleted'
+export * as fileUpdated from './file-updated'
+export * as folderDeletedRecursive from './folder-deleted-recursive'
+export * as aggregateFileChanges from './aggregate-file-changes'
@@ -0,0 +1,77 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  props.logger.info('Retrieving job file...')
+  const workflowState = await props.states.workflow.buildQueueRuntimeState.get(props.workflow.id)
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Enumerating files...')
+  const enumerationState = await SyncQueue.directoryTraversalWithBatching.enumerateAllFilesRecursive(
+    _getEnumerateAllFilesRecursiveProps(props, syncQueue, key, workflowState)
+  )
+
+  if (enumerationState !== undefined) {
+    // Enumeration is still in progress
+    props.logger.debug('Enumeration partially completed')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { ...workflowState, enumerationState })
+    return
+  }
+
+  const { syncQueue: finalSyncQueue } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  if (finalSyncQueue.length === 0) {
+    props.logger.info(
+      'Enumeration matched no files. Nothing to sync. Please check your include and exclude rules. ' +
+        `There could also be an issue with your configuration in the "${props.interfaces['files-readonly'].name}" integration. ` +
+        'For example, your access token might be missing some permissions or the integration might not be set up correctly.'
+    )
+    await props.workflow.setCompleted()
+    return
+  }
+
+  props.logger.info('Enumeration completed. Starting sync job...')
+  await props.workflow.setCompleted()
+  await props.workflows.processQueue.startNewInstance({
+    input: { jobFileId: workflowState.jobFileId },
+    tags: {
+      syncInitiatedAt: props.workflow.tags.syncInitiatedAt!,
+      syncJobId: props.workflow.tags.syncJobId!,
+      syncType: props.workflow.tags.syncType!,
+    },
+  })
+}
+
+const _getEnumerateAllFilesRecursiveProps = (
+  props: bp.WorkflowHandlerProps['buildQueue'],
+  syncQueue: SyncQueue.queueProcessor.ProcessQueueProps['syncQueue'],
+  syncFileKey: string,
+  workflowState: bp.states.States['buildQueueRuntimeState']['payload']
+): SyncQueue.directoryTraversalWithBatching.EnumerateAllFilesRecursiveProps => ({
+  ...props,
+
+  configuration: props.workflow.input,
+
+  currentEnumerationState: workflowState.enumerationState,
+  integration: { listItemsInFolder: props.actions['files-readonly'].listItemsInFolder },
+
+  async pushFilesToQueue(files) {
+    if (!files.length) {
+      return
+    }
+
+    const dedupedQueue = new Map(syncQueue.map((file) => [file.id, file]))
+    files.forEach((file) => dedupedQueue.set(file.id, { ...file, status: 'pending' } as const))
+
+    await SyncQueue.jobFileManager.updateSyncQueue(
+      props,
+      syncFileKey,
+      Array.from(dedupedQueue.values()),
+      props.workflow.tags
+    )
+  },
+})
@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'
@@ -0,0 +1,31 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props)
+  const logger = props.logger.withWorkflowId(props.workflow.id)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  const { finished } = await SyncQueue.queueProcessor.processQueue({
+    logger,
+    syncQueue,
+    fileRepository: props.client,
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    updateSyncQueue: (params) => SyncQueue.jobFileManager.updateSyncQueue(props, key, params.syncQueue),
+  })
+
+  if (finished === 'batch') {
+    logger.info('Batch sync success. Continuing to next batch...')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    return
+  }
+
+  logger.info('Sync completed successfully')
+  await props.workflow.setCompleted()
+}
@@ -0,0 +1,24 @@
+import * as sdk from '@botpress/sdk'
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Creating job file...')
+  const syncFileKey = _generateFileKey(props)
+  const jobFileId = await SyncQueue.jobFileManager.updateSyncQueue(props, syncFileKey, [], props.workflow.tags)
+
+  await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { jobFileId })
+}
+
+const _generateFileKey = (props: bp.WorkflowHandlerProps['buildQueue']) => {
+  const integrationName = props.interfaces['files-readonly'].name
+  const syncJobId = props.workflow.tags.syncJobId
+
+  if (!syncJobId) {
+    throw new sdk.RuntimeError('Sync job ID is not defined in workflow tags')
+  }
+
+  return `file-synchronizer:${integrationName}:/${syncJobId}.jsonl`
+}
@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'
@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+}
@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}
@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'
@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}