chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,29 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
|
||||
const _baseItem = (itemType: 'file' = 'file') =>
|
||||
sdk.z.object({
|
||||
id: sdk.z
|
||||
.string()
|
||||
.describe(
|
||||
`The ${itemType}'s ID. This could be a unique identifier from the external service, or a relative or absolute path, so long as it's unique.`
|
||||
),
|
||||
name: sdk.z
|
||||
.string()
|
||||
.describe(
|
||||
`The ${itemType}'s name. This will be displayed in the Botpress UI and be used as the ${itemType}'s name on Files API."`
|
||||
),
|
||||
type: sdk.z.union([sdk.z.literal('file'), sdk.z.literal('folder')]).describe('The entity type'),
|
||||
})
|
||||
|
||||
export const FILE = _baseItem().extend({
|
||||
type: sdk.z.literal('file'),
|
||||
sizeInBytes: sdk.z.number().optional().describe('The file size in bytes, if available'),
|
||||
contentHash: sdk.z.string().optional().describe('A content hash provided by the external service for deduplication'),
|
||||
})
|
||||
|
||||
export const FILE_WITH_PATH = FILE.extend({
|
||||
absolutePath: sdk.z.string().describe('The absolute path of the file'),
|
||||
})
|
||||
|
||||
export type File = sdk.z.infer<typeof FILE>
|
||||
export type FileWithPath = sdk.z.infer<typeof FILE_WITH_PATH>
|
||||
@@ -0,0 +1,13 @@
|
||||
import rootConfig from '../../eslint.config.mjs'
|
||||
|
||||
export default [
|
||||
...rootConfig,
|
||||
{
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
project: ['./tsconfig.json'],
|
||||
tsconfigRootDir: import.meta.dirname,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
# Knowledge Connector Plugin
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="241" height="241" fill="none"><path fill="purple" d="M34 0A34 34 0 0 0 0 34v173a34 34 0 0 0 34 34h173a34 34 0 0 0 34-34V34a34 34 0 0 0-34-34Zm51.8 33.6h62a8.8 8.7 0 0 1 3.2.6 8.8 8.7 0 0 1 .2.1 8.8 8.7 0 0 1 2.8 1.9l44.2 43.4a8.8 8.7 0 0 1 1.9 2.8 8.8 8.7 0 0 1 0 .2 8.8 8.7 0 0 1 .7 3.2v95.7c0 14.3-12 26.1-26.5 26.1h-48.7a8.8 8.7 0 0 1-8.8-8.7 8.8 8.7 0 0 1 8.8-8.7h48.7c5 0 8.8-3.8 8.8-8.7v-87h-26.5c-9.7 0-17.7-7.9-17.7-17.4v-26H85.8c-5 0-8.9 3.7-8.9 8.6v69.6a8.8 8.7 0 0 1-8.8 8.7 8.8 8.7 0 0 1-8.9-8.7V59.7c0-14.3 12-26.1 26.6-26.1zm70.8 29.7v13.8h14zm-79.7 83.4a8.8 8.7 0 0 1 6.3 2.6l26.5 26a8.8 8.7 0 0 1 2.6 6.2 8.8 8.7 0 0 1-2.6 6.2l-26.5 26a8.8 8.7 0 0 1-12.5 0 8.8 8.7 0 0 1 0-12.2L82 190.2H41.6a8.8 8.7 0 0 1-8.9-8.7 8.8 8.7 0 0 1 8.9-8.7H82l-11.4-11.2a8.8 8.7 0 0 1 0-12.3 8.8 8.7 0 0 1 6.2-2.6z"/></svg>
|
||||
|
After Width: | Height: | Size: 886 B |
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@botpresshub/knowledge-connector",
|
||||
"scripts": {
|
||||
"check:type": "tsc --noEmit",
|
||||
"build": "bp add -y && bp build",
|
||||
"test": "vitest --run"
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@botpress/client": "workspace:*",
|
||||
"@botpress/sdk": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@botpress/cli": "workspace:*",
|
||||
"@botpress/sdk": "workspace:*"
|
||||
},
|
||||
"bpDependencies": {
|
||||
"files-readonly": "../../interfaces/files-readonly"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import filesReadonly from './bp_modules/files-readonly'
|
||||
|
||||
export default new sdk.PluginDefinition({
|
||||
name: 'knowledge-connector',
|
||||
version: '1.0.0',
|
||||
title: 'Knowledge Connector',
|
||||
description: 'Synchronize files from external services to Botpress',
|
||||
icon: 'icon.svg',
|
||||
readme: 'hub.md',
|
||||
attributes: {
|
||||
// This plugin should be invisible to users:
|
||||
...sdk.WELL_KNOWN_ATTRIBUTES.HIDDEN_IN_STUDIO,
|
||||
},
|
||||
configuration: {
|
||||
schema: sdk.z.object({
|
||||
scheduledSyncIntervalHours: sdk.z
|
||||
.number()
|
||||
.default(0)
|
||||
.title('Scheduled Sync Interval (hours)')
|
||||
.describe(
|
||||
'The interval in hours between scheduled re-syncs. Set to 0 to disable. The plugin checks periodically and triggers a sync if enough time has elapsed since the last one.'
|
||||
),
|
||||
}),
|
||||
},
|
||||
events: {
|
||||
scheduledSync: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
},
|
||||
recurringEvents: {
|
||||
triggerScheduledSync: {
|
||||
type: 'scheduledSync',
|
||||
payload: {},
|
||||
schedule: {
|
||||
cron: '* * * * *', // runs every minute; handler gates execution based on scheduledSyncIntervalHours
|
||||
},
|
||||
},
|
||||
},
|
||||
states: {
|
||||
folderSyncSettings: {
|
||||
type: 'bot',
|
||||
schema: sdk.z.object({
|
||||
settings: sdk.z.record(
|
||||
sdk.z.string(),
|
||||
sdk.z.record(
|
||||
sdk.z.string(),
|
||||
sdk.z.object({
|
||||
syncNewFiles: sdk.z
|
||||
.boolean()
|
||||
.default(false)
|
||||
.title('Sync New Files')
|
||||
.describe(
|
||||
'When enabled, automatically sync new files added to this folder to Botpress. Files will be synchronized in real-time as they are added to the folder. Real-time sync is considered enabled if at least one folder has syncNewFiles set to true.'
|
||||
),
|
||||
path: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Folder Path')
|
||||
.describe(
|
||||
'The absolute path of the folder in the integration tree structure. Used to identify and locate the folder for synchronization.'
|
||||
),
|
||||
integrationInstanceAlias: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Integration Instance Alias')
|
||||
.describe('The alias of the integration instance (e.g. "dropbox") used for this folder sync.'),
|
||||
integrationDefinitionName: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Integration Definition Name')
|
||||
.describe('The name of the integration definition (e.g. "dropbox") used for this folder sync.'),
|
||||
transferFileToBotpressAlias: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Transfer File Action Alias')
|
||||
.describe('The alias of the transferFileToBotpress action for this integration.'),
|
||||
})
|
||||
)
|
||||
),
|
||||
}),
|
||||
},
|
||||
lastScheduledSync: {
|
||||
type: 'bot',
|
||||
schema: sdk.z.object({
|
||||
lastSyncAt: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Last Sync At')
|
||||
.describe('ISO timestamp of the last scheduled sync execution.'),
|
||||
}),
|
||||
},
|
||||
},
|
||||
actions: {
|
||||
listSynchronizationOperations: {
|
||||
title: 'List current synchronization operations',
|
||||
description: 'Get the list of ongoing synchronization operations',
|
||||
input: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({
|
||||
isBusy: sdk.z
|
||||
.boolean()
|
||||
.title('Is Busy')
|
||||
.describe('Indicates whether at least one synchronization operation is in progress'),
|
||||
synchronizationsOperations: sdk.z
|
||||
.array(
|
||||
sdk.z
|
||||
.object({
|
||||
integrationInstanceAlias: sdk.z
|
||||
.string()
|
||||
.title('Integration Instance Alias')
|
||||
.describe('The alias of the integration instance being used for synchronization'),
|
||||
startedAt: sdk.z
|
||||
.string()
|
||||
.title('Started At')
|
||||
.describe('The timestamp when the synchronization process started'),
|
||||
syncJobId: sdk.z.string().title('Sync Job ID').describe('The unique ID of the synchronization job'),
|
||||
})
|
||||
.title('Synchronization Status')
|
||||
.describe('Details about the individual synchronization operation')
|
||||
)
|
||||
.title('Synchronizations In Progress')
|
||||
.describe('List of ongoing synchronization operations'),
|
||||
}),
|
||||
},
|
||||
},
|
||||
checkSynchronizationStatus: {
|
||||
title: 'Check synchronization progress',
|
||||
description: 'Get the current status of a synchronization operation',
|
||||
input: {
|
||||
schema: sdk.z.object({
|
||||
syncJobId: sdk.z.string().title('Sync Job ID').describe('The unique ID of the synchronization job to check'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({
|
||||
currentStatus: sdk.z
|
||||
.enum(['inProgress', 'completed', 'failed'])
|
||||
.title('Current Status')
|
||||
.describe('The current status of the synchronization operation'),
|
||||
failureReason: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Failure Reason')
|
||||
.describe('The reason for the failure, if the synchronization has failed'),
|
||||
integrationInstanceAlias: sdk.z
|
||||
.string()
|
||||
.title('Integration Instance Alias')
|
||||
.describe('The alias of the integration instance being used for synchronization'),
|
||||
startedAt: sdk.z
|
||||
.string()
|
||||
.title('Started At')
|
||||
.describe('The timestamp when the synchronization process started'),
|
||||
endedAt: sdk.z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Ended At')
|
||||
.describe(
|
||||
'The timestamp when the synchronization process ended. Present only if the sync is completed or failed.'
|
||||
),
|
||||
totalFiles: sdk.z.number().title('Total Files').describe('The total number of files to synchronize'),
|
||||
processedFiles: sdk.z
|
||||
.number()
|
||||
.title('Processed Files')
|
||||
.describe('The number of files that have been processed so far'),
|
||||
skippedFiles: sdk.z
|
||||
.number()
|
||||
.title('Skipped Files')
|
||||
.describe('The number of files that were skipped during synchronization'),
|
||||
failedFiles: sdk.z.number().title('Failed Files').describe('The number of files that failed to synchronize'),
|
||||
}),
|
||||
},
|
||||
},
|
||||
syncFilesToBotpress: {
|
||||
title: 'Sync files to Botpress',
|
||||
description: 'Start synchronization of files from the external service to Botpress',
|
||||
input: {
|
||||
schema: sdk.z.object({
|
||||
includeFiles: sdk.z
|
||||
.array(
|
||||
sdk.z.object({
|
||||
id: sdk.z.string().title('File ID').describe('The unique identifier of the file to include.'),
|
||||
name: sdk.z.string().title('File Name').describe('The name of the file to include.'),
|
||||
absolutePath: sdk.z
|
||||
.string()
|
||||
.placeholder('Example: /path/to/file')
|
||||
.describe('The absolute path of the file to include.'),
|
||||
sizeInBytes: sdk.z
|
||||
.number()
|
||||
.title('Size in Bytes')
|
||||
.describe('The size of the file in bytes.')
|
||||
.optional(),
|
||||
contentHash: sdk.z
|
||||
.string()
|
||||
.title('Content Hash')
|
||||
.describe('A content hash provided by the external service for deduplication.')
|
||||
.optional(),
|
||||
})
|
||||
)
|
||||
.title('Files to include')
|
||||
.describe('Files to include during synchronization.'),
|
||||
addToKbId: sdk.z
|
||||
.string()
|
||||
.placeholder('Example: kb-2f0a7ea639')
|
||||
.optional()
|
||||
.title('Knowledge Base ID')
|
||||
.describe(
|
||||
'The ID of the knowledge base to add the files to. Note that files added to knowledge bases will count towards both the Vector DB Storage quota and the File Storage quota of the workspace.'
|
||||
),
|
||||
integrationInstanceAlias: sdk.z
|
||||
.string()
|
||||
.title('files-readonly Integration Alias')
|
||||
.describe(
|
||||
'The alias of the integration to use for synchronization. Must implement the files-readonly interface.'
|
||||
),
|
||||
integrationDefinitionName: sdk.z
|
||||
.string()
|
||||
.title('files-readonly Integration Definition Name')
|
||||
.describe(
|
||||
'The name of the integration to use for synchronization. Must implement the files-readonly interface.'
|
||||
),
|
||||
transferFileToBotpressAlias: sdk.z
|
||||
.string()
|
||||
.title('Integration Interface Action Alias')
|
||||
.describe(
|
||||
'The alias of the transferFileToBotpress of the files-readonly interface within the integration.'
|
||||
),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({
|
||||
syncJobId: sdk.z.string().title('Sync Job ID').describe('The unique ID of the new sync job.'),
|
||||
}),
|
||||
},
|
||||
},
|
||||
},
|
||||
interfaces: {
|
||||
'files-readonly': sdk.version.allWithinMajorOf(filesReadonly),
|
||||
},
|
||||
workflows: {
|
||||
processQueue: {
|
||||
title: 'Process file sync queue',
|
||||
description: 'Process the file sync queue and synchronize files to Botpress',
|
||||
input: {
|
||||
schema: sdk.z.object({
|
||||
jobFileId: sdk.z.string().title('Job File').describe("The ID of the job's queue file"),
|
||||
transferFileToBotpressAlias: sdk.z
|
||||
.string()
|
||||
.title('Transfer File to Botpress Action Alias')
|
||||
.describe('The alias of the action used to transfer files to Botpress'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
tags: {
|
||||
syncJobId: {
|
||||
title: 'Sync job ID',
|
||||
description: 'The unique ID of the sync job',
|
||||
},
|
||||
syncInitiatedAt: {
|
||||
title: 'Created at',
|
||||
description: 'The date and time when the sync job was created',
|
||||
},
|
||||
integrationDefinitionName: {
|
||||
title: 'Integration Definition Name',
|
||||
description: 'The name of the integration used for synchronization',
|
||||
},
|
||||
integrationInstanceAlias: {
|
||||
title: 'Integration Instance Alias',
|
||||
description: 'The alias of the integration instance used for synchronization',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,142 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { WORKFLOW_ACTIVE_STATUSES } from '../consts'
|
||||
import { QUEUE_ITEM_SCHEMA, type Workflow } from '../types'
|
||||
import * as utils from '../utils'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type WorkflowStatusResult = {
|
||||
currentStatus: 'inProgress' | 'completed' | 'failed'
|
||||
failureReason?: string
|
||||
endedAt?: string
|
||||
}
|
||||
|
||||
type FileStats = {
|
||||
totalFiles: number
|
||||
processedFiles: number
|
||||
skippedFiles: number
|
||||
failedFiles: number
|
||||
}
|
||||
|
||||
export const callAction: bp.PluginHandlers['actionHandlers']['checkSynchronizationStatus'] = async (props) => {
|
||||
const { syncJobId } = props.input
|
||||
|
||||
props.logger.debug(`Checking synchronization status for sync job with ID "${syncJobId}"`)
|
||||
|
||||
const allWorkflows = await props.workflows.processQueue.listInstances({ tags: { syncJobId } }).take(1)
|
||||
|
||||
if (allWorkflows.length === 0) {
|
||||
props.logger.warn(`No workflow found for sync job with ID "${syncJobId}"`)
|
||||
return _createNotFoundResponse(syncJobId)
|
||||
}
|
||||
|
||||
const workflow = allWorkflows[0]!
|
||||
const integrationInstanceAlias = workflow.tags.integrationInstanceAlias || ''
|
||||
const startedAt = workflow.tags.syncInitiatedAt || workflow.createdAt
|
||||
|
||||
const statusResult = _getWorkflowStatusResult(workflow)
|
||||
|
||||
let fileStats: FileStats = {
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
skippedFiles: 0,
|
||||
failedFiles: 0,
|
||||
}
|
||||
|
||||
if (workflow.input.jobFileId) {
|
||||
try {
|
||||
const { file: jobFile } = await props.client.getFile({ id: workflow.input.jobFileId })
|
||||
const res = await fetch(jobFile.url, { signal: AbortSignal.timeout(30000) })
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to download sync queue file: HTTP ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const jobFileContent = await res.text()
|
||||
fileStats = _computeFileStats(jobFileContent, props.logger)
|
||||
} catch (error) {
|
||||
props.logger.error(`Failed to retrieve sync queue file: ${error}. File stats will be unavailable.`)
|
||||
fileStats = {
|
||||
totalFiles: -1,
|
||||
processedFiles: -1,
|
||||
skippedFiles: -1,
|
||||
failedFiles: -1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...statusResult,
|
||||
integrationInstanceAlias,
|
||||
startedAt,
|
||||
...fileStats,
|
||||
}
|
||||
}
|
||||
|
||||
const _getWorkflowStatusResult = (workflow: Workflow): WorkflowStatusResult => {
|
||||
if (WORKFLOW_ACTIVE_STATUSES.includes(workflow.status as (typeof WORKFLOW_ACTIVE_STATUSES)[number])) {
|
||||
return { currentStatus: 'inProgress' }
|
||||
}
|
||||
|
||||
if (workflow.status === 'completed') {
|
||||
return { currentStatus: 'completed', endedAt: workflow.updatedAt }
|
||||
}
|
||||
|
||||
if (workflow.status === 'failed') {
|
||||
return {
|
||||
currentStatus: 'failed',
|
||||
failureReason: workflow.failureReason || 'Synchronization failed',
|
||||
endedAt: workflow.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
currentStatus: 'failed',
|
||||
failureReason: `Workflow ended with status: ${workflow.status}`,
|
||||
endedAt: workflow.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
const _computeFileStats = (jobFileContent: string, logger: sdk.BotLogger): FileStats => {
|
||||
const stats: FileStats = {
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
skippedFiles: 0,
|
||||
failedFiles: 0,
|
||||
}
|
||||
|
||||
const syncQueueGenerator = utils.jsonl.parseJsonLines(jobFileContent, QUEUE_ITEM_SCHEMA)
|
||||
|
||||
for (const item of syncQueueGenerator) {
|
||||
if ('error' in item) {
|
||||
logger.error('Error while parsing line in job file. This line will be ignored.', item)
|
||||
continue
|
||||
}
|
||||
|
||||
stats.totalFiles++
|
||||
|
||||
switch (item.value.status) {
|
||||
case 'newly-synced':
|
||||
stats.processedFiles++
|
||||
break
|
||||
case 'already-synced':
|
||||
stats.processedFiles++
|
||||
stats.skippedFiles++
|
||||
break
|
||||
case 'errored':
|
||||
stats.failedFiles++
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
const _createNotFoundResponse = (syncJobId: string) => ({
|
||||
currentStatus: 'failed' as const,
|
||||
failureReason: `No synchronization job found with ID "${syncJobId}"`,
|
||||
integrationInstanceAlias: '',
|
||||
startedAt: '',
|
||||
endedAt: undefined,
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
skippedFiles: 0,
|
||||
failedFiles: 0,
|
||||
})
|
||||
@@ -0,0 +1,3 @@
|
||||
export * as syncFilesToBotpress from './sync-files-to-botpress'
|
||||
export * as listSynchronizationOperations from './list-synchronization-operations'
|
||||
export * as checkSynchronizationStatus from './check-synchronization-status'
|
||||
@@ -0,0 +1,28 @@
|
||||
import { WORKFLOW_ACTIVE_STATUSES } from '../consts'
|
||||
import type { Workflow } from '../types'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type SyncOperation = {
|
||||
integrationInstanceAlias: string
|
||||
startedAt: string
|
||||
syncJobId: string
|
||||
}
|
||||
|
||||
export const callAction: bp.PluginHandlers['actionHandlers']['listSynchronizationOperations'] = async (props) => {
|
||||
const activeWorkflows = await props.workflows.processQueue
|
||||
.listInstances({ statuses: [...WORKFLOW_ACTIVE_STATUSES] })
|
||||
.take(100)
|
||||
|
||||
const synchronizationsOperations = activeWorkflows.map(
|
||||
(workflow: Workflow): SyncOperation => ({
|
||||
integrationInstanceAlias: workflow.tags.integrationInstanceAlias || '',
|
||||
startedAt: workflow.tags.syncInitiatedAt || workflow.createdAt,
|
||||
syncJobId: workflow.tags.syncJobId || '',
|
||||
})
|
||||
)
|
||||
|
||||
return {
|
||||
isBusy: synchronizationsOperations.length > 0,
|
||||
synchronizationsOperations,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { WORKFLOW_ACTIVE_STATUSES } from '../consts'
|
||||
import * as SyncQueue from '../sync-queue'
|
||||
import { randomUUID } from '../utils/crypto'
|
||||
import { getAliasedName } from '../utils/get-aliased-name'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const callAction: bp.PluginHandlers['actionHandlers']['syncFilesToBotpress'] = async (props) => {
|
||||
const { includeFiles, integrationInstanceAlias, integrationDefinitionName, transferFileToBotpressAlias } = props.input
|
||||
|
||||
await _assertSyncNotAlreadyInProgress(props)
|
||||
_assertAtLeastOneFileToSync(includeFiles)
|
||||
|
||||
const syncJobId = await randomUUID()
|
||||
const fileKey = _generateFileKey({
|
||||
integrationInstanceAlias,
|
||||
pluginInstanceAlias: props.alias,
|
||||
syncJobId,
|
||||
})
|
||||
|
||||
props.logger.info(
|
||||
`Starting synchronization workflow for integration instance with alias "${integrationInstanceAlias}"`,
|
||||
{ syncJobId, fileKey, filesCount: includeFiles.length }
|
||||
)
|
||||
|
||||
const syncInitiatedAt = new Date().toISOString()
|
||||
|
||||
let jobFileId: string | undefined
|
||||
try {
|
||||
jobFileId = await SyncQueue.jobFileManager.updateSyncQueue(
|
||||
props,
|
||||
fileKey,
|
||||
includeFiles.map((file) => {
|
||||
const contentHash = 'contentHash' in file && typeof file.contentHash === 'string' ? file.contentHash : undefined
|
||||
|
||||
return {
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
absolutePath: file.absolutePath,
|
||||
type: 'file',
|
||||
status: 'pending',
|
||||
sizeInBytes: file.sizeInBytes,
|
||||
contentHash,
|
||||
addToKbId: props.input.addToKbId,
|
||||
}
|
||||
}),
|
||||
{
|
||||
syncJobId,
|
||||
integrationName: integrationDefinitionName,
|
||||
integrationInstanceAlias,
|
||||
syncInitiatedAt,
|
||||
}
|
||||
)
|
||||
|
||||
await props.workflows.processQueue.startNewInstance({
|
||||
input: {
|
||||
jobFileId,
|
||||
transferFileToBotpressAlias,
|
||||
},
|
||||
tags: {
|
||||
integrationDefinitionName,
|
||||
integrationInstanceAlias,
|
||||
syncJobId,
|
||||
syncInitiatedAt,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (jobFileId) {
|
||||
await props.client.deleteFile({ id: jobFileId }).catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
return { syncJobId }
|
||||
}
|
||||
|
||||
const _assertSyncNotAlreadyInProgress = async (
|
||||
props: Extract<bp.ActionHandlerProps, { type?: 'syncFilesToBotpress' }>
|
||||
) => {
|
||||
const { integrationInstanceAlias } = props.input
|
||||
|
||||
const workflowsInProgress = await props.workflows.processQueue
|
||||
.listInstances({
|
||||
statuses: [...WORKFLOW_ACTIVE_STATUSES],
|
||||
tags: {
|
||||
integrationInstanceAlias,
|
||||
},
|
||||
})
|
||||
.take(1)
|
||||
|
||||
if (workflowsInProgress.length > 0) {
|
||||
throw new sdk.RuntimeError(
|
||||
`A synchronization is already in progress for integration instance with alias "${integrationInstanceAlias}". Please wait for it to complete before starting a new one.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const _assertAtLeastOneFileToSync = (includeFiles: unknown[]) => {
|
||||
if (includeFiles.length === 0) {
|
||||
throw new sdk.RuntimeError('No files specified for synchronization.')
|
||||
}
|
||||
}
|
||||
|
||||
const _generateFileKey = (props: {
|
||||
integrationInstanceAlias: string
|
||||
pluginInstanceAlias: string
|
||||
syncJobId: string
|
||||
}) =>
|
||||
`${getAliasedName('knowledge-connector', props.pluginInstanceAlias)}:${props.integrationInstanceAlias}:/${props.syncJobId}.jsonl`
|
||||
@@ -0,0 +1,3 @@
|
||||
export const MAX_BATCH_SIZE_BYTES = 104857600 // 100MB
|
||||
|
||||
export const WORKFLOW_ACTIVE_STATUSES = ['in_progress', 'listening', 'paused', 'pending'] as const
|
||||
@@ -0,0 +1,4 @@
|
||||
export * as onWorkflowStart from './workflow-started'
|
||||
export * as onWorkflowContinue from './workflow-continued'
|
||||
export * as onWorkflowTimeout from './workflow-timed-out'
|
||||
export * as onEvent from './on-event'
|
||||
@@ -0,0 +1,100 @@
|
||||
import { extractIntegrationAlias } from '../../utils/extract-integration-alias'
|
||||
import { handleEvent as handleFileCreated } from './file-created'
|
||||
import { handleEvent as handleFileDeleted } from './file-deleted'
|
||||
import { handleEvent as handleFileUpdated } from './file-updated'
|
||||
import { handleEvent as handleFolderDeletedRecursive } from './folder-deleted-recursive'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const _getErrorMessage = (reason: unknown): string => (reason instanceof Error ? reason.message : String(reason))
|
||||
|
||||
export const handleEvent: bp.EventHandlers['files-readonly:aggregateFileChanges'] = async (props) => {
|
||||
const modifiedItems = props.event.payload.modifiedItems
|
||||
const integrationAlias = extractIntegrationAlias(props.event.type, props.logger)
|
||||
if (!integrationAlias) {
|
||||
return
|
||||
}
|
||||
|
||||
// Process deletes first to avoid conflicts with upserts
|
||||
const deletePromises = modifiedItems.deleted.map((deletedItem) => {
|
||||
if (deletedItem.type === 'file') {
|
||||
return handleFileDeleted({
|
||||
...props,
|
||||
event: {
|
||||
...props.event,
|
||||
type: `${integrationAlias}:fileDeleted` as 'files-readonly:fileDeleted',
|
||||
payload: { file: deletedItem },
|
||||
},
|
||||
})
|
||||
} else {
|
||||
return handleFolderDeletedRecursive({
|
||||
...props,
|
||||
event: {
|
||||
...props.event,
|
||||
type: `${integrationAlias}:folderDeletedRecursive` as 'files-readonly:folderDeletedRecursive',
|
||||
payload: { folder: deletedItem },
|
||||
},
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const deleteResults = await Promise.allSettled(deletePromises)
|
||||
deleteResults.forEach((result, index) => {
|
||||
if (result.status !== 'rejected') {
|
||||
return
|
||||
}
|
||||
|
||||
const deletedItem = modifiedItems.deleted[index]
|
||||
const itemPath = deletedItem?.absolutePath ?? deletedItem?.id ?? 'unknown'
|
||||
props.logger.error(
|
||||
`Failed to process deleted item ${itemPath} during aggregateFileChanges: ${_getErrorMessage(result.reason)}`
|
||||
)
|
||||
})
|
||||
|
||||
const upsertPromises: Promise<void>[] = []
|
||||
const createdFiles = modifiedItems.created.filter(
|
||||
(item): item is (typeof modifiedItems.created)[number] & { type: 'file' } => item.type === 'file'
|
||||
)
|
||||
const updatedFiles = modifiedItems.updated.filter(
|
||||
(item): item is (typeof modifiedItems.updated)[number] & { type: 'file' } => item.type === 'file'
|
||||
)
|
||||
|
||||
for (const createdItem of createdFiles) {
|
||||
upsertPromises.push(
|
||||
handleFileCreated({
|
||||
...props,
|
||||
event: {
|
||||
...props.event,
|
||||
type: `${integrationAlias}:fileCreated` as 'files-readonly:fileCreated',
|
||||
payload: { file: createdItem },
|
||||
},
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
for (const updatedItem of updatedFiles) {
|
||||
upsertPromises.push(
|
||||
handleFileUpdated({
|
||||
...props,
|
||||
event: {
|
||||
...props.event,
|
||||
type: `${integrationAlias}:fileUpdated` as 'files-readonly:fileUpdated',
|
||||
payload: { file: updatedItem },
|
||||
},
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
const upsertResults = await Promise.allSettled(upsertPromises)
|
||||
upsertResults.forEach((result, index) => {
|
||||
if (result.status !== 'rejected') {
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: Add retry/requeue handling for failed aggregate upserts.
|
||||
const sourceItem = index < createdFiles.length ? createdFiles[index] : updatedFiles[index - createdFiles.length]
|
||||
const itemPath = sourceItem?.absolutePath ?? sourceItem?.id ?? 'unknown'
|
||||
props.logger.error(
|
||||
`Failed to process upserted file ${itemPath} during aggregateFileChanges: ${_getErrorMessage(result.reason)}`
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { handleFileUpsert } from './file-upserted'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.EventHandlers['files-readonly:fileCreated'] = async (props) => {
|
||||
await handleFileUpsert(props)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { extractIntegrationAlias } from '../../utils/extract-integration-alias'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.EventHandlers['files-readonly:fileDeleted'] = async (props) => {
|
||||
const deletedFile = props.event.payload.file
|
||||
|
||||
const integrationAlias = extractIntegrationAlias(props.event.type, props.logger)
|
||||
if (!integrationAlias) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const { files } = await props.client.listFiles({
|
||||
tags: {
|
||||
externalId: deletedFile.id,
|
||||
integrationAlias,
|
||||
},
|
||||
})
|
||||
|
||||
for (const filesApiFile of files) {
|
||||
await props.client.deleteFile({ id: filesApiFile.id })
|
||||
}
|
||||
} catch (error) {
|
||||
props.logger.error(`Error deleting file ${deletedFile.absolutePath}:`, error)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { handleFileUpsert } from './file-upserted'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.EventHandlers['files-readonly:fileUpdated'] = async (props) => {
|
||||
await handleFileUpsert(props)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import * as SyncQueue from '../../sync-queue'
|
||||
import { createIntegrationTransferHandler } from '../../utils/create-integration-transfer-handler'
|
||||
import { extractIntegrationAlias } from '../../utils/extract-integration-alias'
|
||||
import { findFolderByPath } from '../../utils/find-folder-by-path'
|
||||
import type * as bp from '.botpress'
|
||||
|
||||
type FileUpsertProps = bp.EventHandlerProps & {
|
||||
event: {
|
||||
payload: {
|
||||
file: { id: string; name: string; absolutePath: string; type: 'file'; sizeInBytes?: number; contentHash?: string }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const handleFileUpsert = async (props: FileUpsertProps) => {
|
||||
const file = props.event.payload.file
|
||||
|
||||
const integrationAlias = extractIntegrationAlias(props.event.type, props.logger)
|
||||
if (!integrationAlias) {
|
||||
return
|
||||
}
|
||||
|
||||
let settings
|
||||
try {
|
||||
settings = await props.states.bot.folderSyncSettings.get(props.ctx.botId)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
if (!settings?.settings) {
|
||||
return
|
||||
}
|
||||
|
||||
const folderMatch = findFolderByPath(settings.settings, file.absolutePath, integrationAlias)
|
||||
|
||||
if (!folderMatch || !folderMatch.syncNewFiles) {
|
||||
return
|
||||
}
|
||||
|
||||
const { kbId, integrationDefinitionName, transferFileToBotpressAlias } = folderMatch
|
||||
|
||||
const result = await SyncQueue.fileProcessor.processQueueFile({
|
||||
fileRepository: props.client,
|
||||
fileToSync: {
|
||||
...file,
|
||||
status: 'pending',
|
||||
addToKbId: kbId,
|
||||
},
|
||||
integration: createIntegrationTransferHandler({
|
||||
integrationName: integrationDefinitionName ?? integrationAlias,
|
||||
integrationAlias,
|
||||
client: props.client,
|
||||
transferFileToBotpressAlias,
|
||||
shouldIndex: kbId !== undefined,
|
||||
}),
|
||||
logger: props.logger,
|
||||
})
|
||||
|
||||
if (result.status === 'errored') {
|
||||
props.logger.error(`File ${file.absolutePath} failed to synchronize: ${result.errorMessage}`)
|
||||
} else {
|
||||
props.logger.info(`File ${file.absolutePath} has been synchronized`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { extractIntegrationAlias } from '../../utils/extract-integration-alias'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.EventHandlers['files-readonly:folderDeletedRecursive'] = async (props) => {
|
||||
const deletedFolder = props.event.payload.folder
|
||||
|
||||
const integrationAlias = extractIntegrationAlias(props.event.type, props.logger)
|
||||
if (!integrationAlias) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (!deletedFolder.absolutePath) {
|
||||
props.logger.warn(`Skipping folder deletion: folder has no absolutePath (id: ${deletedFolder.id})`)
|
||||
return
|
||||
}
|
||||
|
||||
const folderPath = deletedFolder.absolutePath?.endsWith('/')
|
||||
? deletedFolder.absolutePath
|
||||
: `${deletedFolder.absolutePath}/`
|
||||
|
||||
const filesToDelete: Array<{ id: string }> = []
|
||||
let nextToken: string | undefined
|
||||
|
||||
do {
|
||||
const response = await props.client.listFiles({
|
||||
tags: { integrationAlias },
|
||||
nextToken,
|
||||
})
|
||||
|
||||
for (const file of response.files) {
|
||||
const externalPath = file.tags?.['externalPath']
|
||||
if (externalPath && externalPath.startsWith(folderPath)) {
|
||||
filesToDelete.push({ id: file.id })
|
||||
}
|
||||
}
|
||||
|
||||
nextToken = response.meta?.nextToken
|
||||
} while (nextToken)
|
||||
|
||||
await Promise.allSettled(filesToDelete.map((file) => props.client.deleteFile({ id: file.id })))
|
||||
} catch (error) {
|
||||
props.logger.error(`Error deleting files in folder ${deletedFolder.absolutePath}:`, error)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export * as fileCreated from './file-created'
|
||||
export * as fileDeleted from './file-deleted'
|
||||
export * as fileUpdated from './file-updated'
|
||||
export * as folderDeletedRecursive from './folder-deleted-recursive'
|
||||
export * as aggregateFileChanges from './aggregate-file-changes'
|
||||
@@ -0,0 +1,269 @@
|
||||
import { WORKFLOW_ACTIVE_STATUSES } from '../../consts'
|
||||
import * as SyncQueue from '../../sync-queue'
|
||||
import { randomUUID } from '../../utils/crypto'
|
||||
import { enumerateFilesInFolder } from '../../utils/enumerate-files'
|
||||
import { getAliasedName } from '../../utils/get-aliased-name'
|
||||
import type * as bp from '.botpress'
|
||||
|
||||
type ScheduledSyncProps = bp.EventHandlerProps
|
||||
|
||||
type FolderConfig = {
|
||||
syncNewFiles?: boolean
|
||||
path?: string
|
||||
integrationInstanceAlias?: string
|
||||
integrationDefinitionName?: string
|
||||
transferFileToBotpressAlias?: string
|
||||
}
|
||||
|
||||
type FolderSyncSettings = Record<string, Record<string, FolderConfig>>
|
||||
|
||||
type IntegrationGroup = {
|
||||
integrationInstanceAlias: string
|
||||
integrationDefinitionName: string
|
||||
transferFileToBotpressAlias: string
|
||||
folders: Array<{ folderId: string; path: string; kbId: string }>
|
||||
}
|
||||
|
||||
export const handleEvent = async (props: ScheduledSyncProps) => {
|
||||
const { logger, ctx } = props
|
||||
|
||||
if ((props.configuration?.scheduledSyncIntervalHours ?? 0) <= 0) {
|
||||
logger.debug('Scheduled sync disabled (interval is 0)')
|
||||
return
|
||||
}
|
||||
|
||||
const shouldSync = await _hasEnoughTimeElapsed(props, props.configuration?.scheduledSyncIntervalHours ?? 0)
|
||||
if (!shouldSync) {
|
||||
logger.debug('Scheduled sync skipped: not enough time elapsed since last sync')
|
||||
return
|
||||
}
|
||||
|
||||
const settings = await _getFolderSyncSettings(props)
|
||||
if (!settings) {
|
||||
logger.debug('Scheduled sync skipped: no folder sync settings found')
|
||||
return
|
||||
}
|
||||
|
||||
const integrationGroups = _groupFoldersByIntegration(settings, logger)
|
||||
if (integrationGroups.length === 0) {
|
||||
logger.debug('Scheduled sync skipped: no folders with integration info configured')
|
||||
return
|
||||
}
|
||||
|
||||
logger.info(`Scheduled sync starting for ${integrationGroups.length} integration(s)`)
|
||||
|
||||
// Set lastSyncAt immediately to prevent concurrent cron invocations from
|
||||
// starting a duplicate sync while this one is still running.
|
||||
const syncStartedAt = new Date().toISOString()
|
||||
await props.states.bot.lastScheduledSync.set(ctx.botId, {
|
||||
lastSyncAt: syncStartedAt,
|
||||
})
|
||||
|
||||
let hasSucceeded = false
|
||||
try {
|
||||
for (const group of integrationGroups) {
|
||||
try {
|
||||
await _syncIntegrationGroup(props, group)
|
||||
hasSucceeded = true
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Scheduled sync failed for integration "${group.integrationInstanceAlias}": ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (!hasSucceeded) {
|
||||
// Reset lastSyncAt so the next cron trigger retries immediately
|
||||
await props.states.bot.lastScheduledSync.set(ctx.botId, {
|
||||
lastSyncAt: undefined,
|
||||
})
|
||||
logger.warn('Scheduled sync: all integration groups failed, will retry on next trigger')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const _hasEnoughTimeElapsed = async (props: ScheduledSyncProps, intervalHours: number): Promise<boolean> => {
|
||||
try {
|
||||
const { lastSyncAt } = await props.states.bot.lastScheduledSync.get(props.ctx.botId)
|
||||
if (!lastSyncAt) {
|
||||
return true
|
||||
}
|
||||
const elapsed = Date.now() - new Date(lastSyncAt).getTime()
|
||||
return elapsed >= intervalHours * 60 * 60 * 1000
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
const _getFolderSyncSettings = async (props: ScheduledSyncProps) => {
|
||||
try {
|
||||
const state = await props.states.bot.folderSyncSettings.get(props.ctx.botId)
|
||||
return state?.settings
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const _groupFoldersByIntegration = (
|
||||
settings: FolderSyncSettings,
|
||||
logger?: { warn: (...args: unknown[]) => void }
|
||||
): IntegrationGroup[] => {
|
||||
const groupMap = new Map<string, IntegrationGroup>()
|
||||
|
||||
for (const [kbId, folders] of Object.entries(settings)) {
|
||||
for (const [folderId, config] of Object.entries(folders)) {
|
||||
if (!config.integrationInstanceAlias || !config.integrationDefinitionName) {
|
||||
continue
|
||||
}
|
||||
|
||||
const alias = config.integrationInstanceAlias
|
||||
let group = groupMap.get(alias)
|
||||
if (!group) {
|
||||
group = {
|
||||
integrationInstanceAlias: alias,
|
||||
integrationDefinitionName: config.integrationDefinitionName,
|
||||
transferFileToBotpressAlias: config.transferFileToBotpressAlias ?? 'filesReadonlyTransferFileToBotpress',
|
||||
folders: [],
|
||||
}
|
||||
groupMap.set(alias, group)
|
||||
}
|
||||
|
||||
const configuredAlias = config.transferFileToBotpressAlias ?? 'filesReadonlyTransferFileToBotpress'
|
||||
if (configuredAlias !== group.transferFileToBotpressAlias) {
|
||||
logger?.warn(
|
||||
`Integration "${alias}" has conflicting transferFileToBotpressAlias values: ` +
|
||||
`"${group.transferFileToBotpressAlias}" (used) vs "${configuredAlias}" (ignored, from KB "${kbId}")`
|
||||
)
|
||||
}
|
||||
|
||||
group.folders.push({
|
||||
folderId,
|
||||
path: config.path ?? '/',
|
||||
kbId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(groupMap.values())
|
||||
}
|
||||
|
||||
const _syncIntegrationGroup = async (props: ScheduledSyncProps, group: IntegrationGroup) => {
|
||||
const { logger } = props
|
||||
|
||||
const isBusy = await _isSyncInProgress(props, group.integrationInstanceAlias)
|
||||
if (isBusy) {
|
||||
logger.info(`Scheduled sync skipped for "${group.integrationInstanceAlias}": sync already in progress`)
|
||||
return
|
||||
}
|
||||
|
||||
const allFiles: Array<{
|
||||
id: string
|
||||
name: string
|
||||
absolutePath: string
|
||||
sizeInBytes?: number
|
||||
contentHash?: string
|
||||
kbId: string
|
||||
}> = []
|
||||
|
||||
for (const folder of group.folders) {
|
||||
try {
|
||||
const files = await enumerateFilesInFolder({
|
||||
listItemsInFolder: (input) =>
|
||||
props.client
|
||||
.callAction({
|
||||
type: `${group.integrationInstanceAlias}:filesReadonlyListItemsInFolder`,
|
||||
input,
|
||||
})
|
||||
.then((r) => r.output),
|
||||
folderId: folder.folderId,
|
||||
folderPath: folder.path,
|
||||
logger,
|
||||
})
|
||||
|
||||
for (const file of files) {
|
||||
allFiles.push({ ...file, kbId: folder.kbId })
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to enumerate files in folder "${folder.path}" (${folder.folderId}): ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (allFiles.length === 0) {
|
||||
logger.info(`Scheduled sync: no files found for integration "${group.integrationInstanceAlias}"`)
|
||||
return
|
||||
}
|
||||
|
||||
// Group files by kbId to create separate sync jobs per knowledge base
|
||||
const filesByKb = new Map<string, typeof allFiles>()
|
||||
for (const file of allFiles) {
|
||||
const existing = filesByKb.get(file.kbId) ?? []
|
||||
existing.push(file)
|
||||
filesByKb.set(file.kbId, existing)
|
||||
}
|
||||
|
||||
for (const [kbId, files] of filesByKb) {
|
||||
const syncJobId = await randomUUID()
|
||||
const fileKey = `${getAliasedName('knowledge-connector', props.alias)}:${group.integrationInstanceAlias}:/${syncJobId}.jsonl`
|
||||
const syncInitiatedAt = new Date().toISOString()
|
||||
|
||||
logger.info(
|
||||
`Scheduled sync: starting job for "${group.integrationInstanceAlias}" KB "${kbId}" with ${files.length} files`,
|
||||
{ syncJobId }
|
||||
)
|
||||
|
||||
let jobFileId: string | undefined
|
||||
try {
|
||||
jobFileId = await SyncQueue.jobFileManager.updateSyncQueue(
|
||||
props,
|
||||
fileKey,
|
||||
files.map((file) => ({
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
absolutePath: file.absolutePath,
|
||||
type: 'file' as const,
|
||||
status: 'pending' as const,
|
||||
sizeInBytes: file.sizeInBytes,
|
||||
contentHash: file.contentHash,
|
||||
addToKbId: kbId,
|
||||
})),
|
||||
{
|
||||
syncJobId,
|
||||
integrationName: group.integrationDefinitionName,
|
||||
integrationInstanceAlias: group.integrationInstanceAlias,
|
||||
syncInitiatedAt,
|
||||
}
|
||||
)
|
||||
|
||||
await props.workflows.processQueue.startNewInstance({
|
||||
input: {
|
||||
jobFileId,
|
||||
transferFileToBotpressAlias: group.transferFileToBotpressAlias,
|
||||
},
|
||||
tags: {
|
||||
integrationDefinitionName: group.integrationDefinitionName,
|
||||
integrationInstanceAlias: group.integrationInstanceAlias,
|
||||
syncJobId,
|
||||
syncInitiatedAt,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (jobFileId) {
|
||||
await props.client.deleteFile({ id: jobFileId }).catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const _isSyncInProgress = async (props: ScheduledSyncProps, integrationInstanceAlias: string): Promise<boolean> => {
|
||||
const workflowsInProgress = await props.workflows.processQueue
|
||||
.listInstances({
|
||||
statuses: [...WORKFLOW_ACTIVE_STATUSES],
|
||||
tags: { integrationInstanceAlias },
|
||||
})
|
||||
.take(1)
|
||||
|
||||
return workflowsInProgress.length > 0
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * as processQueue from './process-queue'
|
||||
@@ -0,0 +1,59 @@
|
||||
import * as SyncQueue from '../../sync-queue'
|
||||
import { createIntegrationTransferHandler } from '../../utils/create-integration-transfer-handler'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
|
||||
const logger = props.logger.withWorkflowId(props.workflow.id)
|
||||
|
||||
try {
|
||||
const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props)
|
||||
const hasKbTarget = syncQueue.some((item) => item.addToKbId !== undefined)
|
||||
|
||||
if (
|
||||
!props.workflow.tags.integrationInstanceAlias ||
|
||||
!props.workflow.tags.syncJobId ||
|
||||
!props.workflow.tags.integrationDefinitionName ||
|
||||
!props.workflow.tags.syncInitiatedAt
|
||||
) {
|
||||
throw new Error(
|
||||
'Missing required workflow tags: integrationInstanceAlias, integrationDefinitionName, syncJobId, and syncInitiatedAt'
|
||||
)
|
||||
}
|
||||
|
||||
await props.workflow.acknowledgeStartOfProcessing()
|
||||
|
||||
const processResult = await SyncQueue.queueProcessor.processQueue({
|
||||
logger,
|
||||
syncQueue,
|
||||
fileRepository: props.client,
|
||||
integration: createIntegrationTransferHandler({
|
||||
integrationName: props.workflow.tags.integrationDefinitionName,
|
||||
integrationAlias: props.workflow.tags.integrationInstanceAlias,
|
||||
client: props.client,
|
||||
transferFileToBotpressAlias: props.workflow.input.transferFileToBotpressAlias,
|
||||
shouldIndex: hasKbTarget,
|
||||
}),
|
||||
updateSyncQueue: (params) =>
|
||||
SyncQueue.jobFileManager.updateSyncQueue(props, key, params.syncQueue, {
|
||||
syncJobId: props.workflow.tags.syncJobId!,
|
||||
integrationName: props.workflow.tags.integrationDefinitionName!,
|
||||
integrationInstanceAlias: props.workflow.tags.integrationInstanceAlias!,
|
||||
syncInitiatedAt: props.workflow.tags.syncInitiatedAt!,
|
||||
}),
|
||||
})
|
||||
|
||||
if (processResult.finished === 'batch') {
|
||||
logger.info('Batch sync success. Continuing to next batch...')
|
||||
const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
|
||||
await props.workflow.update({ timeoutAt: timeIn5Minutes })
|
||||
return
|
||||
}
|
||||
|
||||
logger.info('Sync completed successfully')
|
||||
await props.workflow.setCompleted()
|
||||
} catch (error) {
|
||||
logger.error('Error processing queue:', error)
|
||||
await props.workflow.setFailed({ failureReason: 'Error processing queue' })
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * as processQueue from './process-queue'
|
||||
@@ -0,0 +1,59 @@
|
||||
import * as SyncQueue from '../../sync-queue'
|
||||
import { createIntegrationTransferHandler } from '../../utils/create-integration-transfer-handler'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
|
||||
const logger = props.logger.withWorkflowId(props.workflow.id)
|
||||
|
||||
try {
|
||||
const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props)
|
||||
const hasKbTarget = syncQueue.some((item) => item.addToKbId !== undefined)
|
||||
|
||||
if (
|
||||
!props.workflow.tags.integrationInstanceAlias ||
|
||||
!props.workflow.tags.syncJobId ||
|
||||
!props.workflow.tags.integrationDefinitionName ||
|
||||
!props.workflow.tags.syncInitiatedAt
|
||||
) {
|
||||
throw new Error(
|
||||
'Missing required workflow tags: integrationInstanceAlias, integrationDefinitionName, syncJobId, and syncInitiatedAt'
|
||||
)
|
||||
}
|
||||
|
||||
await props.workflow.acknowledgeStartOfProcessing()
|
||||
|
||||
const processResult = await SyncQueue.queueProcessor.processQueue({
|
||||
logger,
|
||||
syncQueue,
|
||||
fileRepository: props.client,
|
||||
integration: createIntegrationTransferHandler({
|
||||
integrationName: props.workflow.tags.integrationDefinitionName,
|
||||
integrationAlias: props.workflow.tags.integrationInstanceAlias,
|
||||
client: props.client,
|
||||
transferFileToBotpressAlias: props.workflow.input.transferFileToBotpressAlias,
|
||||
shouldIndex: hasKbTarget,
|
||||
}),
|
||||
updateSyncQueue: (params) =>
|
||||
SyncQueue.jobFileManager.updateSyncQueue(props, key, params.syncQueue, {
|
||||
syncJobId: props.workflow.tags.syncJobId!,
|
||||
integrationName: props.workflow.tags.integrationDefinitionName!,
|
||||
integrationInstanceAlias: props.workflow.tags.integrationInstanceAlias!,
|
||||
syncInitiatedAt: props.workflow.tags.syncInitiatedAt!,
|
||||
}),
|
||||
})
|
||||
|
||||
if (processResult.finished === 'batch') {
|
||||
logger.info('Batch sync success. Continuing to next batch...')
|
||||
const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
|
||||
await props.workflow.update({ timeoutAt: timeIn5Minutes })
|
||||
return
|
||||
}
|
||||
|
||||
logger.info('Sync completed successfully')
|
||||
await props.workflow.setCompleted()
|
||||
} catch (error) {
|
||||
logger.error('Error processing queue:', error)
|
||||
await props.workflow.setFailed({ failureReason: 'Error processing queue' })
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * as processQueue from './process-queue'
|
||||
@@ -0,0 +1,5 @@
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
|
||||
await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import * as actions from './actions'
|
||||
import * as hooks from './hooks'
|
||||
import * as onEventAggregate from './hooks/on-event/aggregate-file-changes'
|
||||
import * as onEventFileCreated from './hooks/on-event/file-created'
|
||||
import * as onEventFileDeleted from './hooks/on-event/file-deleted'
|
||||
import * as onEventFileUpdated from './hooks/on-event/file-updated'
|
||||
import * as onEventFolderDeleted from './hooks/on-event/folder-deleted-recursive'
|
||||
import * as onEventScheduledSync from './hooks/on-event/scheduled-sync'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const plugin = new bp.Plugin({
|
||||
actions: {
|
||||
syncFilesToBotpress: (props) => actions.syncFilesToBotpress.callAction(props),
|
||||
listSynchronizationOperations: (props) => actions.listSynchronizationOperations.callAction(props),
|
||||
checkSynchronizationStatus: (props) => actions.checkSynchronizationStatus.callAction(props),
|
||||
},
|
||||
})
|
||||
|
||||
plugin.on.event('files-readonly:fileDeleted', async (props) => {
|
||||
try {
|
||||
await onEventFileDeleted.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`fileDeleted error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.event('files-readonly:folderDeletedRecursive', async (props) => {
|
||||
try {
|
||||
await onEventFolderDeleted.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`folderDeletedRecursive error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.event('files-readonly:aggregateFileChanges', async (props) => {
|
||||
try {
|
||||
await onEventAggregate.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`aggregateFileChanges error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.event('files-readonly:fileCreated', async (props) => {
|
||||
try {
|
||||
await onEventFileCreated.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`fileCreated error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.event('files-readonly:fileUpdated', async (props) => {
|
||||
try {
|
||||
await onEventFileUpdated.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`fileUpdated error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.event('scheduledSync', async (props) => {
|
||||
try {
|
||||
await onEventScheduledSync.handleEvent(props)
|
||||
} catch (error) {
|
||||
props.logger.error(`scheduledSync error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
|
||||
plugin.on.workflowStart('processQueue', async (props) => {
|
||||
props.logger.info('processQueue workflow started', props.workflow.tags)
|
||||
await hooks.onWorkflowStart.processQueue.handleEvent(props)
|
||||
})
|
||||
|
||||
plugin.on.workflowContinue('processQueue', async (props) => {
|
||||
props.logger.info('processQueue workflow continued', props.workflow.tags)
|
||||
await hooks.onWorkflowContinue.processQueue.handleEvent(props)
|
||||
})
|
||||
|
||||
plugin.on.workflowTimeout('processQueue', async (props) => {
|
||||
props.logger.info('processQueue workflow timed out', props.workflow.tags)
|
||||
await hooks.onWorkflowTimeout.processQueue.handleEvent(props)
|
||||
})
|
||||
|
||||
export default plugin
|
||||
@@ -0,0 +1,129 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { describe, it, expect, vi, type Mocked } from 'vitest'
|
||||
import { processQueueFile, type ProcessFileProps } from './file-processor'
|
||||
import type * as types from '../types'
|
||||
|
||||
const FILE_1 = {
|
||||
id: 'file1',
|
||||
type: 'file',
|
||||
name: 'file1.txt',
|
||||
absolutePath: '/path/to/file1.txt',
|
||||
sizeInBytes: 100,
|
||||
status: 'pending',
|
||||
} as const satisfies types.SyncQueueItem
|
||||
|
||||
describe.concurrent('processQueue', () => {
|
||||
const getMocks = () => ({
|
||||
fileRepository: {
|
||||
listFiles: vi.fn(),
|
||||
deleteFile: vi.fn(),
|
||||
updateFileMetadata: vi.fn(),
|
||||
} as const satisfies Mocked<ProcessFileProps['fileRepository']>,
|
||||
integration: {
|
||||
name: 'test-integration',
|
||||
alias: 'test-integration-alias',
|
||||
transferFileToBotpress: vi.fn(),
|
||||
} as const satisfies Mocked<ProcessFileProps['integration']>,
|
||||
logger: new sdk.BotLogger(),
|
||||
})
|
||||
|
||||
it('should handle errors by modifying the queue item', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
|
||||
// First transfer fails, second succeeds:
|
||||
mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
|
||||
|
||||
// Act
|
||||
const result = processQueueFile({
|
||||
fileToSync: FILE_1,
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toMatchObject({ status: 'errored', errorMessage: 'Transfer failed' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
|
||||
expect(mocks.fileRepository.updateFileMetadata).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should handle listFiles failure by marking the queue item as errored', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockRejectedValueOnce(new Error('Files API unavailable'))
|
||||
|
||||
// Act
|
||||
const result = processQueueFile({
|
||||
fileToSync: FILE_1,
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toMatchObject({ status: 'errored', errorMessage: 'Files API unavailable' })
|
||||
expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
|
||||
expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
|
||||
expect(mocks.fileRepository.updateFileMetadata).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should properly set metadata tags after transferring file', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
|
||||
|
||||
// Act
|
||||
const result = processQueueFile({
|
||||
fileToSync: FILE_1,
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
|
||||
expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
|
||||
id: FILE_1.id,
|
||||
tags: {
|
||||
integrationName: 'test-integration',
|
||||
integrationAlias: 'test-integration-alias',
|
||||
externalId: FILE_1.id,
|
||||
externalSize: FILE_1.sizeInBytes.toString(),
|
||||
externalContentHash: null,
|
||||
externalPath: FILE_1.absolutePath,
|
||||
kbId: null,
|
||||
modalities: null,
|
||||
source: null,
|
||||
title: null,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('should assign to kb if needed after transferring file', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
|
||||
|
||||
// Act
|
||||
const result = processQueueFile({
|
||||
fileToSync: { ...FILE_1, addToKbId: 'kb1' },
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
|
||||
expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
|
||||
id: FILE_1.id,
|
||||
tags: {
|
||||
integrationName: 'test-integration',
|
||||
integrationAlias: 'test-integration-alias',
|
||||
externalId: FILE_1.id,
|
||||
externalSize: FILE_1.sizeInBytes.toString(),
|
||||
externalContentHash: null,
|
||||
externalPath: FILE_1.absolutePath,
|
||||
kbId: 'kb1',
|
||||
modalities: '["text"]',
|
||||
source: 'knowledge-base',
|
||||
title: FILE_1.name,
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,151 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import type * as models from '../../definitions/models'
|
||||
import type * as types from '../types'
|
||||
|
||||
export type ProcessFileProps = {
|
||||
fileToSync: Readonly<types.SyncQueueItem>
|
||||
logger: sdk.BotLogger
|
||||
fileRepository: {
|
||||
listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
|
||||
deleteFile: (params: { id: string }) => Promise<unknown>
|
||||
updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
|
||||
}
|
||||
integration: {
|
||||
name: string
|
||||
alias: string
|
||||
transferFileToBotpress: (params: {
|
||||
file: models.FileWithPath
|
||||
fileKey: string
|
||||
}) => Promise<{ botpressFileId: string }>
|
||||
}
|
||||
}
|
||||
|
||||
export const processQueueFile = async (props: ProcessFileProps): Promise<types.SyncQueueItem> => {
|
||||
const fileToSync = structuredClone(props.fileToSync) as types.SyncQueueItem
|
||||
let existingFile: types.FilesApiFile | undefined
|
||||
try {
|
||||
existingFile = await _getExistingFileFromFilesApi(props, fileToSync)
|
||||
} catch (thrown: unknown) {
|
||||
const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
fileToSync.status = 'errored'
|
||||
fileToSync.errorMessage = err.message
|
||||
props.logger.error(`Error while checking existing file ${fileToSync.absolutePath}: ${err.message}`)
|
||||
return fileToSync
|
||||
}
|
||||
const shouldUploadFile = _shouldUploadFile(fileToSync, existingFile)
|
||||
|
||||
if (!shouldUploadFile) {
|
||||
fileToSync.status = 'already-synced'
|
||||
return fileToSync
|
||||
}
|
||||
|
||||
const existingTags = existingFile?.tags ?? {}
|
||||
await _deleteExistingFileFromFilesApi(props, existingFile)
|
||||
await _transferFileToBotpress(props, fileToSync, existingTags)
|
||||
|
||||
if (fileToSync.status !== 'errored') {
|
||||
fileToSync.status = 'newly-synced'
|
||||
}
|
||||
|
||||
return fileToSync
|
||||
}
|
||||
|
||||
const _getExistingFileFromFilesApi = async (
|
||||
props: ProcessFileProps,
|
||||
fileToSync: models.FileWithPath
|
||||
): Promise<types.FilesApiFile | undefined> => {
|
||||
const { files: existingFiles } = await props.fileRepository.listFiles({
|
||||
tags: {
|
||||
externalId: fileToSync.id,
|
||||
integrationName: props.integration.name,
|
||||
integrationAlias: props.integration.alias,
|
||||
},
|
||||
})
|
||||
|
||||
if (existingFiles.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
return existingFiles[0]!
|
||||
}
|
||||
|
||||
const _shouldUploadFile = (fileToSync: models.FileWithPath, existingFile?: types.FilesApiFile) => {
|
||||
if (!existingFile) {
|
||||
return true
|
||||
}
|
||||
|
||||
const existingHash = existingFile.tags['externalContentHash']
|
||||
const newHash = fileToSync.contentHash
|
||||
|
||||
if (existingHash && newHash) {
|
||||
return existingHash !== newHash
|
||||
}
|
||||
|
||||
const existingSize = existingFile.tags['externalSize']
|
||||
const newSize = fileToSync.sizeInBytes?.toString()
|
||||
|
||||
if (existingSize && newSize) {
|
||||
return existingSize !== newSize
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const _deleteExistingFileFromFilesApi = async (props: ProcessFileProps, existingFile?: types.FilesApiFile) => {
|
||||
if (!existingFile) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
await props.fileRepository.deleteFile({ id: existingFile.id })
|
||||
} catch (error) {
|
||||
props.logger.warn(`Failed to delete existing file: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
const _transferFileToBotpress = async (
|
||||
props: ProcessFileProps,
|
||||
fileToSync: types.SyncQueueItem,
|
||||
existingTags: Record<string, string> = {}
|
||||
) => {
|
||||
let botpressFileId: string
|
||||
try {
|
||||
const result = await props.integration.transferFileToBotpress({
|
||||
file: fileToSync,
|
||||
fileKey: `${props.integration.alias}:${fileToSync.absolutePath}`,
|
||||
})
|
||||
botpressFileId = result.botpressFileId
|
||||
} catch (thrown: unknown) {
|
||||
const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
fileToSync.status = 'errored'
|
||||
fileToSync.errorMessage = err.message
|
||||
props.logger.error(`Error while uploading file ${fileToSync.absolutePath}: ${err.message}`)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
await props.fileRepository.updateFileMetadata({
|
||||
id: botpressFileId,
|
||||
tags: {
|
||||
...existingTags,
|
||||
integrationName: props.integration.name,
|
||||
integrationAlias: props.integration.alias,
|
||||
externalId: fileToSync.id,
|
||||
externalSize: fileToSync.sizeInBytes?.toString() ?? null,
|
||||
externalContentHash: fileToSync.contentHash ?? null,
|
||||
externalPath: fileToSync.absolutePath,
|
||||
kbId: fileToSync.addToKbId ?? null,
|
||||
source: fileToSync.addToKbId ? 'knowledge-base' : null,
|
||||
title: fileToSync.addToKbId ? fileToSync.name : null,
|
||||
modalities: fileToSync.addToKbId ? '["text"]' : null,
|
||||
},
|
||||
})
|
||||
} catch (thrown: unknown) {
|
||||
const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
fileToSync.status = 'errored'
|
||||
fileToSync.errorMessage = err.message
|
||||
props.logger.error(
|
||||
`File ${fileToSync.absolutePath} was uploaded (id: ${botpressFileId}) but metadata update failed: ${err.message}. This file may be orphaned and should be cleaned up manually if the next sync does not overwrite it.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * as queueProcessor from './queue-processor'
|
||||
export * as fileProcessor from './file-processor'
|
||||
export * as jobFileManager from './job-file-manager'
|
||||
@@ -0,0 +1,57 @@
|
||||
import { QUEUE_ITEM_SCHEMA } from '../types'
|
||||
import type * as types from '../types'
|
||||
import * as utils from '../utils'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const getSyncQueue = async (
|
||||
props: bp.WorkflowHandlerProps['processQueue']
|
||||
): Promise<{ syncQueue: types.SyncQueue; key: string }> => {
|
||||
const { jobFileContent, key } = await _retrieveJobFile(props).catch((thrown: unknown) => {
|
||||
const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
throw new Error(`Failed to retrieve job file: ${err.message}`)
|
||||
})
|
||||
|
||||
const syncQueue: types.SyncQueue = []
|
||||
const syncQueueGenerator = utils.jsonl.parseJsonLines(jobFileContent, QUEUE_ITEM_SCHEMA)
|
||||
|
||||
for (const item of syncQueueGenerator) {
|
||||
if ('error' in item) {
|
||||
props.logger.error('Error while parsing line in job file. This line will be ignored.', item)
|
||||
continue
|
||||
}
|
||||
|
||||
syncQueue.push(item.value)
|
||||
}
|
||||
|
||||
return { syncQueue, key }
|
||||
}
|
||||
|
||||
const _retrieveJobFile = async (
|
||||
props: bp.WorkflowHandlerProps['processQueue']
|
||||
): Promise<{ jobFileContent: string; key: string }> => {
|
||||
const { file: jobFile } = await props.client.getFile({
|
||||
id: props.workflow.input.jobFileId,
|
||||
})
|
||||
const res = await fetch(jobFile.url, { signal: AbortSignal.timeout(30000) })
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to download sync queue file: HTTP ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const jobFileContent = await res.text()
|
||||
|
||||
return { jobFileContent, key: jobFile.key }
|
||||
}
|
||||
|
||||
export const updateSyncQueue = async (
|
||||
props: types.CommonProps,
|
||||
key: string,
|
||||
syncQueue: types.SyncQueue,
|
||||
tags?: Record<string, string>
|
||||
): Promise<string> => {
|
||||
const { file: jobFile } = await props.client.uploadFile({
|
||||
key,
|
||||
content: syncQueue.map((item) => JSON.stringify(item)).join('\n') + '\n',
|
||||
tags,
|
||||
})
|
||||
|
||||
return jobFile.id
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { findBatchEndCursor } from './queue-batching'
|
||||
import { MAX_BATCH_SIZE_BYTES as maxBatchSize } from '../consts'
|
||||
|
||||
describe.concurrent('findBatchEndCursor', () => {
|
||||
it.each([
|
||||
{ startCursor: 0, files: [{ sizeInBytes: 100 }, { sizeInBytes: 200 }], expectedEndCursors: [2] },
|
||||
{ startCursor: 0, files: [{ sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
|
||||
{ startCursor: 0, files: [{ sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
|
||||
{
|
||||
startCursor: 0,
|
||||
files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }],
|
||||
expectedEndCursors: [2, 3, 4],
|
||||
},
|
||||
{
|
||||
startCursor: 0,
|
||||
files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }],
|
||||
expectedEndCursors: [2, 3, 4],
|
||||
},
|
||||
{
|
||||
startCursor: 0,
|
||||
files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize - 200 }, { sizeInBytes: 200 }],
|
||||
expectedEndCursors: [2, 4],
|
||||
},
|
||||
{
|
||||
startCursor: 2,
|
||||
files: [
|
||||
{ sizeInBytes: 200 },
|
||||
{ sizeInBytes: 200 },
|
||||
{ sizeInBytes: 200 },
|
||||
{ sizeInBytes: 200 },
|
||||
{ sizeInBytes: maxBatchSize - 200 },
|
||||
{ sizeInBytes: 200 },
|
||||
],
|
||||
expectedEndCursors: [4, 6],
|
||||
},
|
||||
{
|
||||
startCursor: 0,
|
||||
files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }],
|
||||
expectedEndCursors: [2, 3],
|
||||
},
|
||||
])(
|
||||
'should correctly calculate the end cursors for a batch of files',
|
||||
({ startCursor, files, expectedEndCursors }) => {
|
||||
// Act
|
||||
const endCursors: number[] = []
|
||||
let currentCursor = startCursor
|
||||
|
||||
while (currentCursor < files.length) {
|
||||
currentCursor = findBatchEndCursor({ startCursor: currentCursor, files }).endCursor
|
||||
endCursors.push(currentCursor)
|
||||
}
|
||||
|
||||
// Assert
|
||||
expect(endCursors).toStrictEqual(expectedEndCursors)
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
import { MAX_BATCH_SIZE_BYTES } from '../consts'
|
||||
import type * as types from '../types'
|
||||
|
||||
type SimpleFile = Pick<types.SyncQueueItem, 'sizeInBytes'>
|
||||
|
||||
export const findBatchEndCursor = ({
|
||||
startCursor: unsafeStartCursor,
|
||||
files,
|
||||
}: {
|
||||
startCursor: number
|
||||
files: SimpleFile[]
|
||||
}): { endCursor: number } => {
|
||||
const startCursor = Math.min(Math.max(unsafeStartCursor, 0), files.length - 1)
|
||||
let currentBatchSize = 0
|
||||
let endCursor = files.length
|
||||
|
||||
for (let newCursor = startCursor; newCursor < files.length; newCursor++) {
|
||||
const fileToSync = files[newCursor]
|
||||
const fileSizeInBytes = _getFileSize(fileToSync!)
|
||||
currentBatchSize += fileSizeInBytes
|
||||
|
||||
// First file is always included even if it exceeds the batch size:
|
||||
if (newCursor > startCursor && currentBatchSize > MAX_BATCH_SIZE_BYTES) {
|
||||
endCursor = newCursor
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return { endCursor }
|
||||
}
|
||||
|
||||
const _getFileSize = (file: SimpleFile) => {
|
||||
if (file.sizeInBytes === undefined) {
|
||||
// If a file has no size, we assume it takes up half the batch size limit.
|
||||
// This should be relatively safe, since we'd process at most two files of
|
||||
// unknown size in a batch, and we don't want to just skip them.
|
||||
return MAX_BATCH_SIZE_BYTES / 2
|
||||
}
|
||||
|
||||
return file.sizeInBytes
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { describe, it, expect, vi, type Mocked, type Mock } from 'vitest'
|
||||
import { processQueue, type ProcessQueueProps } from './queue-processor'
|
||||
import type * as types from '../types'
|
||||
import { MAX_BATCH_SIZE_BYTES } from '../consts'
|
||||
|
||||
const FILE_1 = {
|
||||
id: 'file1',
|
||||
type: 'file',
|
||||
name: 'file1.txt',
|
||||
absolutePath: '/path/to/file1.txt',
|
||||
sizeInBytes: 100,
|
||||
status: 'pending',
|
||||
} as const satisfies types.SyncQueueItem
|
||||
|
||||
const FILE_2 = {
|
||||
id: 'file2',
|
||||
type: 'file',
|
||||
name: 'file2.txt',
|
||||
absolutePath: '/path/to/file2.txt',
|
||||
sizeInBytes: 200,
|
||||
status: 'pending',
|
||||
} as const satisfies types.SyncQueueItem
|
||||
|
||||
const LARGE_FILE = {
|
||||
id: 'largefile',
|
||||
type: 'file',
|
||||
name: 'largefile.txt',
|
||||
absolutePath: '/path/to/largefile.txt',
|
||||
sizeInBytes: MAX_BATCH_SIZE_BYTES - 200,
|
||||
status: 'pending',
|
||||
} as const satisfies types.SyncQueueItem
|
||||
|
||||
describe.concurrent('processQueue', () => {
|
||||
const getMocks = () => ({
|
||||
fileRepository: {
|
||||
listFiles: vi.fn(),
|
||||
deleteFile: vi.fn(),
|
||||
updateFileMetadata: vi.fn(),
|
||||
} as const satisfies Mocked<ProcessQueueProps['fileRepository']>,
|
||||
integration: {
|
||||
name: 'test-integration',
|
||||
alias: 'test-integration-alias',
|
||||
transferFileToBotpress: vi.fn(),
|
||||
} as const satisfies Mocked<ProcessQueueProps['integration']>,
|
||||
updateSyncQueue: vi.fn() as Mock<ProcessQueueProps['updateSyncQueue']>,
|
||||
logger: new sdk.BotLogger(),
|
||||
})
|
||||
|
||||
it('should process all files in queue when size is within batch limit', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
|
||||
|
||||
// Act
|
||||
const result = processQueue({
|
||||
syncQueue: [FILE_1, FILE_2],
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toEqual({ finished: 'all' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
|
||||
|
||||
const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
|
||||
expect(updatedQueue?.[0]?.status).toBe('newly-synced')
|
||||
expect(updatedQueue?.[1]?.status).toBe('newly-synced')
|
||||
})
|
||||
|
||||
it('should process files in batches when size exceeds batch limit', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
|
||||
|
||||
// Act
|
||||
const result = processQueue({
|
||||
syncQueue: [FILE_1, LARGE_FILE, FILE_2],
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toEqual({ finished: 'batch' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
|
||||
|
||||
const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
|
||||
expect(updatedQueue?.[0]?.status).toBe('newly-synced')
|
||||
expect(updatedQueue?.[1]?.status).toBe('newly-synced')
|
||||
expect(updatedQueue?.[2]?.status).toBe('pending')
|
||||
})
|
||||
|
||||
it('should handle errors by continuing to the next file', async () => {
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
|
||||
// First transfer fails, second succeeds:
|
||||
mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
|
||||
|
||||
// Act
|
||||
const result = processQueue({
|
||||
syncQueue: [FILE_1, FILE_2],
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
await expect(result).resolves.toEqual({ finished: 'all' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(1)
|
||||
|
||||
const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
|
||||
expect(updatedQueue?.[0]?.status).toBe('errored')
|
||||
expect(updatedQueue?.[0]?.errorMessage).toContain('Transfer failed')
|
||||
expect(updatedQueue?.[1]?.status).toBe('newly-synced')
|
||||
})
|
||||
|
||||
it('should handle complex batching', async () => {
|
||||
// >>>>>>>>>>> FIRST BATCH <<<<<<<<<<<<<
|
||||
|
||||
// Arrange
|
||||
const mocks = getMocks()
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
|
||||
|
||||
// Act
|
||||
const firstRun = await processQueue({
|
||||
syncQueue: [FILE_2, FILE_1, LARGE_FILE],
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
expect(firstRun).toEqual({ finished: 'batch' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
|
||||
expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
|
||||
|
||||
const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue!
|
||||
|
||||
expect(updatedQueue[0]?.status).toBe('newly-synced')
|
||||
expect(updatedQueue[1]?.status).toBe('newly-synced')
|
||||
expect(updatedQueue[2]?.status).toBe('pending')
|
||||
|
||||
// >>>>>>>>>>> SECOND BATCH <<<<<<<<<<<<<
|
||||
|
||||
// Arrange
|
||||
mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
|
||||
mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
|
||||
|
||||
// Act
|
||||
const secondRun = await processQueue({
|
||||
syncQueue: updatedQueue!,
|
||||
...mocks,
|
||||
})
|
||||
|
||||
// Assert
|
||||
expect(secondRun).toEqual({ finished: 'all' })
|
||||
expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(3)
|
||||
expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(2)
|
||||
|
||||
const finalQueue = mocks.updateSyncQueue.mock.calls[1]?.[0].syncQueue!
|
||||
|
||||
expect(finalQueue[0]?.status).toBe('newly-synced')
|
||||
expect(finalQueue[1]?.status).toBe('newly-synced')
|
||||
expect(finalQueue[2]?.status).toBe('newly-synced')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,57 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import type * as models from '../../definitions/models'
|
||||
import type * as types from '../types'
|
||||
import { processQueueFile } from './file-processor'
|
||||
import { findBatchEndCursor } from './queue-batching'
|
||||
|
||||
export type ProcessQueueProps = {
|
||||
syncQueue: Readonly<types.SyncQueue>
|
||||
logger: sdk.BotLogger
|
||||
fileRepository: {
|
||||
listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
|
||||
deleteFile: (params: { id: string }) => Promise<unknown>
|
||||
updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
|
||||
}
|
||||
integration: {
|
||||
name: string
|
||||
alias: string
|
||||
transferFileToBotpress: (params: {
|
||||
file: models.FileWithPath
|
||||
fileKey: string
|
||||
}) => Promise<{ botpressFileId: string }>
|
||||
}
|
||||
updateSyncQueue: (props: { syncQueue: types.SyncQueue }) => Promise<unknown>
|
||||
}
|
||||
|
||||
export const processQueue = async (props: ProcessQueueProps) => {
|
||||
// Short-circuit if the sync queue is empty:
|
||||
if (!props.syncQueue?.length) {
|
||||
props.logger.info('Sync queue is empty. Nothing to process.')
|
||||
return { finished: 'all' } as const
|
||||
}
|
||||
|
||||
const syncQueue = structuredClone(props.syncQueue) as types.SyncQueue
|
||||
const startCursor = syncQueue.findIndex((file) => file.status === 'pending')
|
||||
|
||||
if (startCursor < 0) {
|
||||
props.logger.info('No files left to process in the sync queue.')
|
||||
return { finished: 'all' } as const
|
||||
}
|
||||
|
||||
const { endCursor } = findBatchEndCursor({ startCursor, files: syncQueue })
|
||||
const filesInBatch = syncQueue.slice(startCursor, endCursor)
|
||||
|
||||
for (const fileToSync of filesInBatch) {
|
||||
const processedFile = await processQueueFile({ ...props, fileToSync })
|
||||
Object.assign(fileToSync, processedFile)
|
||||
}
|
||||
|
||||
await props.updateSyncQueue({ syncQueue })
|
||||
|
||||
const hasPendingBeyondBatch = syncQueue.slice(endCursor).some((f) => f.status === 'pending')
|
||||
if (hasPendingBeyondBatch) {
|
||||
return { finished: 'batch' } as const
|
||||
}
|
||||
|
||||
return { finished: 'all' } as const
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { CommonHandlerProps } from '@botpress/sdk/dist/plugin'
|
||||
import * as models from '../definitions/models'
|
||||
import type * as bp from '.botpress'
|
||||
|
||||
type TPlugin = sdk.DefaultPlugin<bp.TPlugin>
|
||||
|
||||
export type SyncQueueItem = models.FileWithPath & {
|
||||
status: 'pending' | 'newly-synced' | 'already-synced' | 'errored'
|
||||
errorMessage?: string
|
||||
addToKbId?: string
|
||||
}
|
||||
export type SyncQueue = SyncQueueItem[]
|
||||
|
||||
export type CommonProps = CommonHandlerProps<TPlugin>
|
||||
|
||||
export type FilesApiFile = {
|
||||
id: string
|
||||
tags: Record<string, string>
|
||||
}
|
||||
|
||||
export type Workflow = Awaited<ReturnType<bp.Client['listWorkflows']>>['workflows'][number]
|
||||
|
||||
export const QUEUE_ITEM_SCHEMA = models.FILE_WITH_PATH.extend({
|
||||
status: sdk.z.enum(['pending', 'newly-synced', 'already-synced', 'errored']),
|
||||
errorMessage: sdk.z.string().optional(),
|
||||
addToKbId: sdk.z.string().optional(),
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
import type * as models from '../../definitions/models'
|
||||
|
||||
const TRANSFER_FILE_ACTION_NAME = 'filesReadonlyTransferFileToBotpress'
|
||||
|
||||
export type CreateIntegrationTransferHandlerProps = {
|
||||
integrationName: string
|
||||
integrationAlias: string
|
||||
client: {
|
||||
callAction: (params: {
|
||||
type: string
|
||||
input: {
|
||||
file: models.FileWithPath
|
||||
fileKey: string
|
||||
shouldIndex: boolean
|
||||
}
|
||||
}) => Promise<{ output: { botpressFileId: string } }>
|
||||
}
|
||||
transferFileToBotpressAlias?: string
|
||||
shouldIndex?: boolean
|
||||
}
|
||||
|
||||
export function createIntegrationTransferHandler(props: CreateIntegrationTransferHandlerProps): {
|
||||
name: string
|
||||
alias: string
|
||||
transferFileToBotpress: (params: {
|
||||
file: models.FileWithPath
|
||||
fileKey: string
|
||||
}) => Promise<{ botpressFileId: string }>
|
||||
} {
|
||||
const actionName = props.transferFileToBotpressAlias ?? TRANSFER_FILE_ACTION_NAME
|
||||
const shouldIndex = props.shouldIndex ?? true
|
||||
|
||||
return {
|
||||
name: props.integrationName,
|
||||
alias: props.integrationAlias,
|
||||
async transferFileToBotpress({ file, fileKey }) {
|
||||
const { output } = await props.client.callAction({
|
||||
type: `${props.integrationAlias}:${actionName}`,
|
||||
input: {
|
||||
file,
|
||||
fileKey,
|
||||
shouldIndex,
|
||||
},
|
||||
})
|
||||
|
||||
return { botpressFileId: output.botpressFileId }
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
export const randomUUID = async (): Promise<string> => {
|
||||
const crypto = await _getWebCrypto()
|
||||
return crypto.randomUUID()
|
||||
}
|
||||
|
||||
type _WebCrypto = { randomUUID: () => string }
|
||||
let _webCrypto: _WebCrypto | undefined
|
||||
|
||||
const _getWebCrypto = async (): Promise<_WebCrypto> => {
|
||||
if (!_webCrypto) {
|
||||
if (typeof (globalThis as any).crypto?.randomUUID === 'function') {
|
||||
_webCrypto = (globalThis as any).crypto
|
||||
} else if (typeof process !== 'undefined' && process.versions?.node) {
|
||||
try {
|
||||
const nodeCrypto = await import('crypto')
|
||||
_webCrypto = nodeCrypto.webcrypto
|
||||
} catch (thrown: unknown) {
|
||||
const error: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
throw new Error(`Failed to import 'crypto' module: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (!_webCrypto || typeof _webCrypto.randomUUID !== 'function') {
|
||||
throw new Error('No suitable web crypto implementation available')
|
||||
}
|
||||
}
|
||||
return _webCrypto
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type * as sdk from '@botpress/sdk'
|
||||
|
||||
type ListItemsInFolder = (input: {
|
||||
folderId?: string
|
||||
nextToken?: string
|
||||
filters?: { itemType?: 'file' | 'folder' }
|
||||
}) => Promise<{
|
||||
items: Array<
|
||||
| {
|
||||
id: string
|
||||
type: 'file'
|
||||
name: string
|
||||
parentId?: string
|
||||
absolutePath?: string
|
||||
sizeInBytes?: number
|
||||
lastModifiedDate?: string
|
||||
contentHash?: string
|
||||
}
|
||||
| {
|
||||
id: string
|
||||
type: 'folder'
|
||||
name: string
|
||||
parentId?: string
|
||||
absolutePath?: string
|
||||
}
|
||||
>
|
||||
meta: { nextToken?: string }
|
||||
}>
|
||||
|
||||
export type EnumeratedFile = {
|
||||
id: string
|
||||
name: string
|
||||
absolutePath: string
|
||||
sizeInBytes?: number
|
||||
contentHash?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively enumerates all files within a given folder by traversing subfolders
|
||||
* and handling pagination via nextToken.
|
||||
*/
|
||||
export const enumerateFilesInFolder = async (props: {
|
||||
listItemsInFolder: ListItemsInFolder
|
||||
folderId: string
|
||||
folderPath: string
|
||||
logger: sdk.BotLogger
|
||||
}): Promise<EnumeratedFile[]> => {
|
||||
const { listItemsInFolder, folderId, folderPath, logger } = props
|
||||
|
||||
const files: EnumeratedFile[] = []
|
||||
const pendingFolders: Array<{ folderId?: string; absolutePath: string }> = [{ folderId, absolutePath: folderPath }]
|
||||
let folderIndex = 0
|
||||
|
||||
while (folderIndex < pendingFolders.length) {
|
||||
const currentFolder = pendingFolders[folderIndex++]!
|
||||
let nextToken: string | undefined
|
||||
|
||||
do {
|
||||
const response = await listItemsInFolder({
|
||||
folderId: currentFolder.folderId,
|
||||
nextToken,
|
||||
})
|
||||
|
||||
for (const item of response.items) {
|
||||
const basePath = currentFolder.absolutePath.endsWith('/')
|
||||
? currentFolder.absolutePath
|
||||
: `${currentFolder.absolutePath}/`
|
||||
if (item.type === 'folder') {
|
||||
const subfolderPath = item.absolutePath ?? `${basePath}${item.name}/`
|
||||
pendingFolders.push({ folderId: item.id, absolutePath: subfolderPath })
|
||||
} else {
|
||||
const filePath = item.absolutePath ?? `${basePath}${item.name}`
|
||||
files.push({
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
absolutePath: filePath,
|
||||
sizeInBytes: item.sizeInBytes,
|
||||
contentHash: item.contentHash,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
nextToken = response.meta.nextToken
|
||||
} while (nextToken)
|
||||
}
|
||||
|
||||
logger.info(`Enumerated ${files.length} files in folder "${folderPath}"`)
|
||||
return files
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
|
||||
export function extractIntegrationAlias(eventType: string, logger: sdk.BotLogger): string | null {
|
||||
const integrationAlias = eventType.split(':')[0]
|
||||
if (!integrationAlias) {
|
||||
logger.warn(`Could not extract integration alias from event type: ${eventType}`)
|
||||
return null
|
||||
}
|
||||
return integrationAlias
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
type FolderConfig = {
|
||||
syncNewFiles?: boolean
|
||||
path?: string
|
||||
integrationDefinitionName?: string
|
||||
integrationInstanceAlias?: string
|
||||
transferFileToBotpressAlias?: string
|
||||
}
|
||||
|
||||
type FolderMatch = {
|
||||
kbId: string
|
||||
folderId: string
|
||||
syncNewFiles: boolean
|
||||
integrationDefinitionName?: string
|
||||
integrationInstanceAlias?: string
|
||||
transferFileToBotpressAlias?: string
|
||||
}
|
||||
|
||||
export function findFolderByPath(
|
||||
settings: Record<string, Record<string, FolderConfig>>,
|
||||
filePath: string,
|
||||
integrationAlias?: string
|
||||
): FolderMatch | undefined {
|
||||
let bestMatch: FolderMatch | undefined
|
||||
let bestMatchLength = 0
|
||||
|
||||
for (const [kbId, folders] of Object.entries(settings)) {
|
||||
for (const [folderId, folderSettings] of Object.entries(folders)) {
|
||||
const rawPath = folderSettings.path
|
||||
const folderPath = rawPath && !rawPath.endsWith('/') ? `${rawPath}/` : rawPath
|
||||
if (
|
||||
folderPath &&
|
||||
filePath.startsWith(folderPath) &&
|
||||
folderPath.length > bestMatchLength &&
|
||||
(!integrationAlias || folderSettings.integrationInstanceAlias === integrationAlias)
|
||||
) {
|
||||
bestMatchLength = folderPath.length
|
||||
bestMatch = {
|
||||
kbId,
|
||||
folderId,
|
||||
syncNewFiles: folderSettings.syncNewFiles ?? false,
|
||||
integrationDefinitionName: folderSettings.integrationDefinitionName,
|
||||
integrationInstanceAlias: folderSettings.integrationInstanceAlias,
|
||||
transferFileToBotpressAlias: folderSettings.transferFileToBotpressAlias,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export const getAliasedName = (name: string, alias?: string) => (alias && alias !== name ? `${name}(${alias})` : name)
|
||||
@@ -0,0 +1 @@
|
||||
export * as jsonl from './json-lines'
|
||||
@@ -0,0 +1 @@
|
||||
export * from './parser'
|
||||
@@ -0,0 +1,157 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { parseJsonLines } from './parser'
|
||||
|
||||
describe.concurrent('parseJsonLines', () => {
|
||||
const STRING_SCHEMA = sdk.z.string()
|
||||
const NUMBER_SCHEMA = sdk.z.number()
|
||||
const USER_SCHEMA = sdk.z.object({
|
||||
id: sdk.z.number(),
|
||||
name: sdk.z.string(),
|
||||
email: sdk.z.string().email(),
|
||||
})
|
||||
|
||||
it('should parse valid JSON lines with a string schema', () => {
|
||||
// Arrange
|
||||
const input = '"hello"\n"world"\n"test"'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: 'hello' }, { value: 'world' }, { value: 'test' }])
|
||||
})
|
||||
|
||||
it('should parse valid JSON lines with an object schema', () => {
|
||||
// Arrange
|
||||
const input =
|
||||
'{"id": 1, "name": "John", "email": "john@example.com"}\n' +
|
||||
'{"id": 2, "name": "Jane", "email": "jane@example.com"}\n' +
|
||||
'{"id": 3, "name": "Bob", "email": "bob@example.com"}'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, USER_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([
|
||||
{
|
||||
value: { id: 1, name: 'John', email: 'john@example.com' },
|
||||
},
|
||||
{
|
||||
value: { id: 2, name: 'Jane', email: 'jane@example.com' },
|
||||
},
|
||||
{ value: { id: 3, name: 'Bob', email: 'bob@example.com' } },
|
||||
])
|
||||
})
|
||||
|
||||
it('should handle empty input strings', () => {
|
||||
// Arrange
|
||||
const input = ''
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toStrictEqual([])
|
||||
})
|
||||
|
||||
it('should handle input with only whitespace', () => {
|
||||
// Arrange
|
||||
const input = ' \n \n\t '
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toStrictEqual([])
|
||||
})
|
||||
|
||||
it('should skip empty lines', () => {
|
||||
// Arrange
|
||||
const input = '"first"\n\n"second"\n\n\n"third"'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: 'first' }, { value: 'second' }, { value: 'third' }])
|
||||
})
|
||||
|
||||
it('should handle trailing newlines', () => {
|
||||
// Arrange
|
||||
const input = '"line1"\n"line2"\n'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }])
|
||||
})
|
||||
|
||||
it('should handle input without newlines', () => {
|
||||
// Arrange
|
||||
const input = '"single line"'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: 'single line' }])
|
||||
})
|
||||
|
||||
it('should return a Zod error when a line fails Zod validation', () => {
|
||||
// Arrange
|
||||
const input = '{"id": 1, "name": "John", "email": "invalid-email"}'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, USER_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([
|
||||
{ rawLine: '{"id": 1, "name": "John", "email": "invalid-email"}', error: expect.any(Error) },
|
||||
])
|
||||
})
|
||||
|
||||
it('should parse to the end, even with errors', () => {
|
||||
// Arrange
|
||||
const input = '1\n2\n"not a number"\n4'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, NUMBER_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toStrictEqual([
|
||||
{ rawLine: '1', value: 1 },
|
||||
{ rawLine: '2', value: 2 },
|
||||
{ rawLine: '"not a number"', error: expect.any(Error) },
|
||||
{ rawLine: '4', value: 4 },
|
||||
])
|
||||
})
|
||||
|
||||
it('should handle different line endings (CRLF)', () => {
|
||||
// Arrange
|
||||
const input = '"line1"\r\n"line2"\r\n"line3"'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }, { value: 'line3' }])
|
||||
})
|
||||
|
||||
it('should parse properly with schema containing optional fields', () => {
|
||||
// Arrange
|
||||
const optionalSchema = sdk.z.object({
|
||||
id: sdk.z.number(),
|
||||
name: sdk.z.string(),
|
||||
age: sdk.z.number().optional(),
|
||||
})
|
||||
const input = '{"id": 1, "name": "John"}\n{"id": 2, "name": "Jane", "age": 25}'
|
||||
|
||||
// Act
|
||||
const result = Array.from(parseJsonLines(input, optionalSchema))
|
||||
|
||||
// Assert
|
||||
expect(result).toMatchObject([{ value: { id: 1, name: 'John' } }, { value: { id: 2, name: 'Jane', age: 25 } }])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,26 @@
|
||||
import { z } from '@botpress/sdk'
|
||||
|
||||
type _JsonParseResult<T> = { rawLine: string } & ({ value: T } | { error: Error })
|
||||
|
||||
export function* parseJsonLines<TLineSchema extends z.ZodTypeAny>(
|
||||
rawJsonLines: string,
|
||||
zodSchema: TLineSchema
|
||||
): Generator<_JsonParseResult<z.infer<TLineSchema>>, void, undefined> {
|
||||
let startCursor = 0
|
||||
|
||||
for (let endCursor = 0; endCursor <= rawJsonLines.length; endCursor++) {
|
||||
if (rawJsonLines[endCursor] === '\n' || endCursor === rawJsonLines.length) {
|
||||
const line = rawJsonLines.slice(startCursor, endCursor).trim()
|
||||
startCursor = endCursor + 1
|
||||
|
||||
if (line) {
|
||||
try {
|
||||
yield { rawLine: line, value: zodSchema.parse(JSON.parse(line)) }
|
||||
} catch (thrown: unknown) {
|
||||
const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
|
||||
yield { rawLine: line, error: err }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": { "*": ["./*"] },
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import config from '../../vitest.config'
|
||||
export default config
|
||||
Reference in New Issue
Block a user