chore: import upstream snapshot with attribution

plugins/analytics/eslint.config.mjs

@@ -0,0 +1,13 @@
+import rootConfig from '../../eslint.config.mjs'
+
+export default [
+  ...rootConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]

plugins/analytics/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@botpresshub/analytics",
+  "scripts": {
+    "check:type": "tsc --noEmit",
+    "build": "bp add -y && bp build",
+    "add:integrations": "echo 'no integrations to add'",
+    "test": "vitest --run"
+  },
+  "private": true,
+  "dependencies": {
+    "@botpress/client": "workspace:*",
+    "@botpress/sdk": "workspace:*"
+  },
+  "devDependencies": {
+    "@botpress/cli": "workspace:*",
+    "@botpress/common": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "@bpinternal/genenv": "0.0.1"
+  }
+}

plugins/analytics/plugin.definition.ts

@@ -0,0 +1,17 @@
+import * as sdk from '@botpress/sdk'
+
+export default new sdk.PluginDefinition({
+  name: 'analytics',
+  version: '0.0.1',
+  configuration: { schema: sdk.z.object({}) },
+  actions: {
+    track: {
+      input: {
+        schema: sdk.z.object({ name: sdk.z.string(), count: sdk.z.number() }),
+      },
+      output: {
+        schema: sdk.z.object({}),
+      },
+    },
+  },
+})

plugins/analytics/src/index.ts

@@ -0,0 +1,11 @@
+import * as bp from '.botpress'
+
+const plugin = new bp.Plugin({
+  actions: {
+    track: async ({ input, client }) => {
+      return await client.trackAnalytics({ name: input.name, count: input.count })
+    },
+  },
+})
+
+export default plugin

plugins/analytics/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": { "*": ["./*"] },
+    "outDir": "dist"
+  },
+  "include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
+}

plugins/analytics/vitest.config.ts

@@ -0,0 +1,2 @@
+import config from '../../vitest.config'
+export default config

plugins/conversation-insights/eslint.config.mjs

@@ -0,0 +1,13 @@
+import rootConfig from '../../eslint.config.mjs'
+
+export default [
+  ...rootConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]

plugins/conversation-insights/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@botpresshub/conversation-insights",
+  "scripts": {
+    "check:type": "tsc --noEmit",
+    "build": "bp add -y && bp build",
+    "test": "vitest --run"
+  },
+  "private": true,
+  "dependencies": {
+    "@botpress/cognitive": "0.6.1",
+    "@botpress/sdk": "workspace:*",
+    "browser-or-node": "^2.1.1",
+    "jsonrepair": "^3.10.0"
+  },
+  "devDependencies": {
+    "@botpress/cli": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "@botpresshub/llm": "workspace:*"
+  }
+}

plugins/conversation-insights/plugin.definition.ts

@@ -0,0 +1,48 @@
+import { PluginDefinition, z } from '@botpress/sdk'
+
+export default new PluginDefinition({
+  name: 'conversation-insights',
+  version: '0.5.1',
+  configuration: {
+    schema: z.object({
+      aiEnabled: z.boolean().default(true).describe('Set to true to enable title, summary and sentiment ai generation'),
+      aiGenerationInterval: z
+        .number()
+        .optional()
+        .title('AI Generation Interval')
+        .describe('Interval in minutes between AI insight generations'),
+    }),
+  },
+  conversation: {
+    tags: {
+      title: { title: 'Title', description: 'The title of the conversation.' },
+      summary: {
+        title: 'Summary',
+        description: 'A summary of the current conversation.',
+      },
+      message_count: {
+        title: 'Message count',
+        description: 'The count of messages sent in the conversation by both the bot and user(s). Type: int',
+      },
+      participant_count: {
+        title: 'Participant count',
+        description: 'The count of users having participated in the conversation, including the bot. Type: int',
+      },
+      sentiment: {
+        title: 'Sentiment',
+        description: 'The sentiment that best describes the conversation. Type: enum Sentiments',
+      },
+      isDirty: {
+        title: 'Is Dirty',
+        description:
+          "Indicates whether a conversation's AI insight has been updated since the last message. Type: boolean",
+      },
+    },
+  },
+  events: {
+    updateAiInsight: {
+      schema: z.object({}),
+    },
+  },
+  workflows: { updateAllConversations: { input: { schema: z.object({}) }, output: { schema: z.object({}) } } },
+})

plugins/conversation-insights/src/error-handler.ts

@@ -0,0 +1,9 @@
+import { BotLogger } from '@botpress/sdk'
+
+export const handleError =
+  (props: { context: string; logger: BotLogger }) =>
+  async (thrown: unknown): Promise<undefined> => {
+    const error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    const message = `An error occured in the conversation-insights plugin while ${props.context}: ${error.message}`
+    props.logger.error(message)
+  }

plugins/conversation-insights/src/handlers/after-incoming-message.ts

@@ -0,0 +1,23 @@
+import * as onNewMessageHandler from '../onNewMessageHandler'
+import * as bp from '.botpress'
+
+const HOUR_MILLISECONDS = 60 * 60 * 1000
+
+export const handleAfterIncomingMessage: bp.HookHandlers['after_incoming_message']['*'] = async (props) => {
+  const { conversations, configuration, events, data } = props
+  const conversation = await conversations['*']['*'].getById({ id: data.conversationId })
+  await onNewMessageHandler.onNewMessage({ ...props, conversation })
+
+  if (configuration.aiEnabled) {
+    const updateAiEvents = await events.updateAiInsight.list({ status: 'scheduled' }).take(1)
+    if (updateAiEvents.length === 0) {
+      const interval = configuration.aiGenerationInterval
+        ? configuration.aiGenerationInterval * 60 * 1000
+        : HOUR_MILLISECONDS
+      const dateTime = new Date(Date.now() + interval).toISOString()
+      await events.updateAiInsight.schedule({}, { dateTime })
+    }
+  }
+
+  return undefined
+}

plugins/conversation-insights/src/handlers/after-outgoing-message.ts

@@ -0,0 +1,8 @@
+import * as onNewMessageHandler from '../onNewMessageHandler'
+import * as bp from '.botpress'
+
+export const handleAfterOutgoingMessage: bp.HookHandlers['after_outgoing_message']['*'] = async (props) => {
+  const conversation = await props.conversations['*']['*'].getById({ id: props.data.message.conversationId })
+  await onNewMessageHandler.onNewMessage({ ...props, conversation })
+  return undefined
+}

plugins/conversation-insights/src/handlers/index.ts

@@ -0,0 +1,4 @@
+export * from './after-incoming-message'
+export * from './after-outgoing-message'
+export * from './update-ai-insight'
+export * from './update-all-conversations'

plugins/conversation-insights/src/handlers/update-ai-insight.ts

@@ -0,0 +1,11 @@
+import * as bp from '.botpress'
+
+export const handleUpdateAiInsight: bp.EventHandlers['updateAiInsight'] = async (props) => {
+  const workflows = await props.workflows.updateAllConversations
+    .listInstances({ statuses: ['pending', 'cancelled', 'listening', 'paused'] })
+    .take(1)
+
+  if (workflows.length === 0) {
+    await props.workflows.updateAllConversations.startNewInstance({ input: {} })
+  }
+}

plugins/conversation-insights/src/handlers/update-all-conversations.ts

@@ -0,0 +1,18 @@
+import { updateAllConversations } from '../updateAllConversations'
+import * as bp from '.botpress'
+
+export const handleStartUpdateAllConversations: bp.WorkflowHandlers['updateAllConversations'] = async (props) => {
+  props.logger.info('Starting updateAllConversations workflow')
+  await updateAllConversations(props)
+
+  return undefined
+}
+export const handleContinueUpdateAllConversations: bp.WorkflowHandlers['updateAllConversations'] = async (props) => {
+  await updateAllConversations(props)
+
+  return undefined
+}
+
+export const handleTimeoutUpdateAllConversations: bp.WorkflowHandlers['updateAllConversations'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}

plugins/conversation-insights/src/index.ts

@@ -0,0 +1,59 @@
+import { isBrowser } from 'browser-or-node'
+import { handleError } from './error-handler'
+import * as handlers from './handlers'
+import * as bp from '.botpress'
+
+const plugin = new bp.Plugin({
+  actions: {},
+})
+
+plugin.on.afterIncomingMessage('*', async (props) => {
+  if (isBrowser) {
+    return
+  }
+  return await handlers
+    .handleAfterIncomingMessage(props)
+    .catch(handleError({ context: 'trying to process an incoming message', logger: props.logger }))
+})
+
+plugin.on.afterOutgoingMessage('*', async (props) => {
+  if (isBrowser) {
+    return
+  }
+  return await handlers
+    .handleAfterOutgoingMessage(props)
+    .catch(handleError({ context: 'trying to process an outgoing message', logger: props.logger }))
+})
+
+plugin.on.event('updateAiInsight', async (props) => {
+  if (isBrowser) {
+    props.logger.error('This event is not supported by the browser')
+    return
+  }
+  return await handlers
+    .handleUpdateAiInsight(props)
+    .catch(handleError({ context: 'trying to update an AI insight', logger: props.logger }))
+})
+
+plugin.on.workflowStart('updateAllConversations', async (props) => {
+  return await handlers
+    .handleStartUpdateAllConversations(props)
+    .catch(handleError({ context: 'trying to start the updateAllConversations workflow', logger: props.logger }))
+})
+
+plugin.on.workflowContinue('updateAllConversations', async (props) => {
+  return await handlers
+    .handleContinueUpdateAllConversations(props)
+    .catch(handleError({ context: 'trying to continue the updateAllConversations workflow', logger: props.logger }))
+})
+
+plugin.on.workflowTimeout('updateAllConversations', async (props) => {
+  return await handlers.handleTimeoutUpdateAllConversations(props).catch(
+    handleError({
+      context: 'trying to process the timeout of the updateAllConversations workflow',
+      logger: props.logger,
+    })
+  )
+})
+
+export default plugin

plugins/conversation-insights/src/onNewMessageHandler.ts

@@ -0,0 +1,22 @@
+import * as types from './types'
+
+type OnNewMessageProps = types.CommonProps & {
+  conversation: types.ActionableConversation
+}
+export const onNewMessage = async (props: OnNewMessageProps) => {
+  const message_count = props.conversation.tags.message_count ? parseInt(props.conversation.tags.message_count) + 1 : 1
+
+  const participant_count = await props.conversation
+    .listParticipants()
+    .takeAll()
+    .then((participants) => participants.length)
+
+  await props.conversation.update({
+    tags: {
+      message_count: message_count.toString(),
+      participant_count: participant_count.toString(),
+      isDirty: props.configuration.aiEnabled ? 'true' : 'false',
+    },
+  })
+  return
+}

plugins/conversation-insights/src/prompt/parse-content.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect } from 'vitest'
+import { parseLLMOutput } from './parse-content'
+import * as sdk from '@botpress/sdk'
+import { z } from '@botpress/sdk'
+import * as cognitive from '@botpress/cognitive'
+
+const COGNITIVE_OUTPUT = (content: string): cognitive.GenerateContentOutput => ({
+  provider: 'test-provider',
+  model: 'test-model',
+  botpress: { cost: 0 },
+  id: '',
+  usage: {
+    inputCost: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    outputCost: 0,
+  },
+  choices: [{ content, index: 0, role: 'assistant', stopReason: 'other' }],
+})
+
+const CONTENT_PARSE_SCHEMA = z.object({ foo: z.string(), bar: z.number() })
+
+describe('parseLLMOutput', () => {
+  it('valid json parsing is successful', () => {
+    const output = COGNITIVE_OUTPUT(`{"foo": "hello", "bar": 42}`)
+
+    const result = parseLLMOutput<z.infer<typeof CONTENT_PARSE_SCHEMA>>({ schema: CONTENT_PARSE_SCHEMA, ...output })
+
+    expect(result.success).toBe(true)
+  })
+
+  it('invalid json parsing throws an error', () => {
+    const output = COGNITIVE_OUTPUT(`not a json`)
+
+    let thrown: unknown | undefined = undefined
+    try {
+      parseLLMOutput<typeof CONTENT_PARSE_SCHEMA>({ schema: CONTENT_PARSE_SCHEMA, ...output })
+    } catch (e) {
+      thrown = e
+    }
+
+    expect(thrown).toBeDefined()
+    expect(z.is.zuiError(thrown)).toBe(true)
+  })
+
+  it('empty choices parsing throws an error', () => {
+    expect(() => parseLLMOutput<any>({ choices: [] } as any)).toThrow(sdk.RuntimeError)
+  })
+
+  it('valid json with whitespaces parsing is successful', () => {
+    const output = COGNITIVE_OUTPUT(`  { "foo": "bar", "bar": 123 }  `)
+
+    const result = parseLLMOutput<z.infer<typeof CONTENT_PARSE_SCHEMA>>({ schema: CONTENT_PARSE_SCHEMA, ...output })
+
+    expect(result.success).toBe(true)
+  })
+})

plugins/conversation-insights/src/prompt/parse-content.ts

@@ -0,0 +1,29 @@
+import * as cognitive from '@botpress/cognitive'
+import * as sdk from '@botpress/sdk'
+import { jsonrepair } from 'jsonrepair'
+
+export type LLMInput = cognitive.GenerateContentInput
+
+type LLMChoice = cognitive.GenerateContentOutput['choices'][number]
+
+export type PredictResponse<T> = {
+  success: boolean
+  json: T
+}
+
+const parseJson = <T>(expectedSchema: sdk.z.ZodSchema, str: string): T => {
+  const repaired = jsonrepair(str)
+  const parsed = JSON.parse(repaired)
+  return expectedSchema.parse(parsed)
+}
+
+type ParseLLMOutputProps = cognitive.GenerateContentOutput & { schema: sdk.z.ZodSchema }
+export const parseLLMOutput = <T>(props: ParseLLMOutputProps): PredictResponse<T> => {
+  const mappedChoices: LLMChoice['content'][] = props.choices.map((choice) => choice.content)
+  if (!mappedChoices[0]) throw new sdk.RuntimeError('Could not parse LLM output')
+  const firstChoice = mappedChoices[0]
+  return {
+    success: true,
+    json: parseJson<T>(props.schema, firstChoice.toString()),
+  }
+}

plugins/conversation-insights/src/prompt/prompt.ts

@@ -0,0 +1,46 @@
+import type * as client from '@botpress/client'
+import { z } from '@botpress/sdk'
+import { LLMInput } from './parse-content'
+
+export type SentimentAnalysisOutput = z.infer<typeof SentimentAnalysisOutput>
+export const SentimentAnalysisOutput = z.object({
+  sentiment: z.string().describe('The sentiment that best describes the conversation'),
+})
+
+export type InputFormat = z.infer<typeof InputFormat>
+export const InputFormat = z.array(z.string())
+
+const formatMessages = (
+  messages: PromptArgs['messages'],
+  context: PromptArgs['context'],
+  botId: string
+): LLMInput['messages'] => {
+  const contextMessage: LLMInput['messages'][0] = {
+    role: 'assistant',
+    content: `Context: ${JSON.stringify(context)}`,
+  }
+
+  const messagesWithUser: LLMInput['messages'] = []
+  for (const message of messages) {
+    if (message.type !== 'text') continue // only text is supported to analyse messages
+    messagesWithUser.push({
+      role: message.userId === botId ? 'assistant' : 'user',
+      content: message.payload.text,
+    })
+  }
+  return [contextMessage, ...messagesWithUser.reverse()]
+}
+
+export type PromptArgs = {
+  systemPrompt: string
+  messages: client.Message[]
+  context: object
+  botId: string
+}
+export const createPrompt = (args: PromptArgs): LLMInput => ({
+  responseFormat: 'json_object',
+  temperature: 0,
+  systemPrompt: args.systemPrompt.trim(),
+  messages: formatMessages(args.messages, args.context, args.botId),
+  model: 'fast',
+})

plugins/conversation-insights/src/prompt/sentiment-prompt.ts

@@ -0,0 +1,89 @@
+import { z } from '@botpress/sdk'
+import { LLMInput } from './parse-content'
+import * as prompt from './prompt'
+
+export type SentimentAnalysisOutput = z.infer<typeof SentimentAnalysisOutput>
+export const SentimentAnalysisOutput = z.object({
+  sentiment: z
+    .enum(['very_negative', 'negative', 'neutral', 'positive', 'very_positive'])
+    .describe('The sentiment that best describes the conversation'),
+})
+
+export const SENTIMENT_OPTIONS = SentimentAnalysisOutput.shape.sentiment.options.map((opt) => ` "${opt}" `).join('|')
+
+export type PromptArgs = Omit<prompt.PromptArgs, 'systemPrompt'>
+export const createPrompt = (args: PromptArgs): LLMInput =>
+  prompt.createPrompt({
+    ...args,
+    systemPrompt: `
+You are a conversation analyser.
+You will be given:
+- A previous sentiment
+- An array of messages
+
+Your task is to reply the sentiment that best describes the overall conversation.
+
+Return your response only in valid JSON using the following type:
+
+\`\`\`json
+{
+  "sentiment": ${SENTIMENT_OPTIONS},   // The latest sentiment of the conversation
+}
+\`\`\`
+
+Instructions:
+
+- Consider the previous sentiment when choosing the new one — keep it if still relevant, or update it if needed.
+- Focus on the most recent sentiment of the conversation.
+- Only use the available sentiments
+- Do not include extra commentary, formatting, or explanation outside the JSON output.
+- The messages are in order, which means the most recent ones are at the end of the list.
+- Keep in mind that your own messages are included in the messages, but have the 'assistant' role
+
+The available sentiments are: ${SENTIMENT_OPTIONS}
+
+Examples:
+
+Input:
+
+\`\`\`json
+{
+  "messages": [
+    "Context: {'previousSentiment': 'negative'}",
+    "User: I hate your service. I want to unsubscribe right now!",
+    "Bot: I understand your frustation, but there is nothing we can do",
+    "User: I want a refund.",
+  ]
+}
+\`\`\`
+
+Output:
+
+\`\`\`json
+{
+  "sentiment": "very_negative"
+}
+\`\`\`
+
+Input:
+
+\`\`\`json
+{
+  "messages": [
+    "previousSentiment: neutral",
+    "User: Hi, how could I get a premium subscription?",
+    "Bot: You can get it by clicking on the link I just sent you.",
+    "User: Thank you so much, your help has changed my life",
+  ]
+}
+\`\`\`
+
+Output:
+
+\`\`\`json
+{
+  "sentiment": "very_positive"
+}
+\`\`\`
+`,
+  })

plugins/conversation-insights/src/prompt/summary-prompt.ts

@@ -0,0 +1,66 @@
+import { z } from '@botpress/sdk'
+import { LLMInput } from './parse-content'
+import * as prompt from './prompt'
+
+export type SummaryOutput = z.infer<typeof SummaryOutput>
+export const SummaryOutput = z.object({
+  title: z.string().describe('A fitting title for the conversation'),
+  summary: z.string().describe('A short summary of the conversation'),
+})
+
+export type PromptArgs = Omit<prompt.PromptArgs, 'systemPrompt'>
+export const createPrompt = (args: PromptArgs): LLMInput =>
+  prompt.createPrompt({
+    ...args,
+    systemPrompt: `
+You are a conversation summarizer.
+You will be given:
+- A previous title and summary
+- An array of USER MESSAGES
+
+Your task is to produce a title and summary that best describe the overall conversation.
+
+Return your response only in valid JSON using the following type:
+
+\`\`\`json
+{
+  "title": "string",   // A concise, fitting title for the conversation
+  "summary": "string"  // A short summary capturing the main topic or request
+}
+\`\`\`
+
+Instructions:
+
+- Consider the previous title when creating the new one — keep it if still relevant, or update it if needed.
+- Focus on the main subject of the conversation.
+- Make the title short and descriptive (few words).
+- Keep the summary concise (one or two sentences).
+- Do not include extra commentary, formatting, or explanation outside the JSON output.
+- The messages are in order, which means the most recent ones are at the end of the list.
+
+Example:
+
+Input:
+
+\`\`\`json
+{
+  "messages": [
+    "Context: {'previousTitle': 'Used cars', 'previousSummary': 'The user is talking abous a used Toyota Matrix'}",
+    "User: What mileage should I expect from a car that was made two years ago?",
+    "User: What price should I expect from a car manufactured in 2011?",
+    "User: What should I look out for when buying a secondhand Toyota Matrix?",
+    "User: I am looking to buy a used car, what would you recommend?",
+  ]
+}
+\`\`\`
+
+Output:
+
+\`\`\`json
+{
+  "title": "Used cars",
+  "summary": "The user is seeking advice on purchasing a used car."
+}
+\`\`\`
+`,
+  })

plugins/conversation-insights/src/tagsUpdater.ts

@@ -0,0 +1,84 @@
+import * as cognitive from '@botpress/cognitive'
+import * as sdk from '@botpress/sdk'
+import * as gen from './prompt/parse-content'
+import * as sentiment from './prompt/sentiment-prompt'
+import * as summarizer from './prompt/summary-prompt'
+import * as types from './types'
+
+type CommonProps = types.CommonProps
+
+type UpdateTitleAndSummaryProps = Omit<CommonProps, 'messages'> & {
+  conversation: types.ActionableConversation
+  messages: types.ActionableMessage[]
+  client: cognitive.BotpressClientLike
+}
+export const updateTitleAndSummary = async (props: UpdateTitleAndSummaryProps) => {
+  const summaryPrompt = summarizer.createPrompt({
+    messages: props.messages,
+    botId: props.ctx.botId,
+    context: { previousTitle: props.conversation.tags.title, previousSummary: props.conversation.tags.summary },
+  })
+
+  const parsedSummary = await _generateContentWithRetries<summarizer.SummaryOutput>({
+    actions: props.actions,
+    logger: props.logger,
+    prompt: summaryPrompt,
+    client: props.client,
+    schema: summarizer.SummaryOutput,
+  })
+
+  const sentimentPrompt = sentiment.createPrompt({
+    messages: props.messages,
+    botId: props.ctx.botId,
+    context: { previousSentiment: props.conversation.tags.sentiment },
+  })
+
+  const parsedSentiment = await _generateContentWithRetries<sentiment.SentimentAnalysisOutput>({
+    actions: props.actions,
+    logger: props.logger,
+    prompt: sentimentPrompt,
+    client: props.client,
+    schema: sentiment.SentimentAnalysisOutput,
+  })
+
+  await props.conversation.update({
+    tags: {
+      title: parsedSummary.json.title,
+      summary: parsedSummary.json.summary,
+      sentiment: parsedSentiment.json.sentiment,
+      isDirty: 'false',
+    },
+  })
+  props.logger.info(`The AI insight was updated for conversation ${props.conversation.id}`)
+}
+
+type ParsePromptProps = {
+  actions: UpdateTitleAndSummaryProps['actions']
+  logger: UpdateTitleAndSummaryProps['logger']
+  prompt: gen.LLMInput
+  client: cognitive.BotpressClientLike
+  schema: sdk.z.ZodSchema
+}
+const _generateContentWithRetries = async <T>(props: ParsePromptProps): Promise<gen.PredictResponse<T>> => {
+  let attemptCount = 0
+  const maxRetries = 3
+
+  const cognitiveClient = new cognitive.Cognitive({ client: props.client, __experimental_beta: true })
+  let llmOutput = await cognitiveClient.generateContent(props.prompt)
+  let parsed = gen.parseLLMOutput<T>({ schema: props.schema, ...llmOutput.output })
+
+  while (!parsed.success && attemptCount < maxRetries) {
+    props.logger.debug(
+      `Attempt ${attemptCount + 1}: The LLM output did not respect the schema. It submitted: `,
+      parsed.json
+    )
+    llmOutput = await cognitiveClient.generateContent(props.prompt)
+    parsed = gen.parseLLMOutput<T>({ schema: props.schema, ...llmOutput.output })
+    attemptCount++
+  }
+
+  if (!parsed.success) {
+    props.logger.debug(`The LLM output did not respect the schema after ${attemptCount} retries.`, parsed.json)
+  }
+  return parsed
+}

plugins/conversation-insights/src/types.ts

@@ -0,0 +1,12 @@
+import * as bp from '.botpress'
+
+// TODO: generate a type for CommonProps in the CLI / SDK
+export type CommonProps =
+  | bp.HookHandlerProps['after_incoming_message']
+  | bp.HookHandlerProps['after_outgoing_message']
+  | bp.EventHandlerProps
+
+export type ActionableConversation = NonNullable<
+  Awaited<ReturnType<bp.MessageHandlerProps['conversations']['*']['*']['getById']>>
+>
+export type ActionableMessage = NonNullable<Awaited<ReturnType<bp.MessageHandlerProps['messages']['getById']>>>

plugins/conversation-insights/src/updateAllConversations.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect, vi, beforeEach, MockInstance, Mock } from 'vitest'
+import { updateAllConversations, WorkflowProps } from './updateAllConversations'
+import * as summaryUpdater from './tagsUpdater'
+
+const as = <T>(x: Partial<T>): T => x as T
+
+type WorkflowProxy = WorkflowProps['workflow']
+type Client = WorkflowProps['client']
+type Logger = WorkflowProps['logger']
+type Conversation = Awaited<ReturnType<Client['listConversations']>>['conversations'][number]
+type Message = Awaited<ReturnType<Client['listMessages']>>['messages'][number]
+
+// NOTE: These tests are temporarily skipped because the mocks need to be
+//       adjusted to use plugin proxies instead of direct client calls (the
+//       plugin should not rely on the client at all).
+
+describe.skip('updateAllConversations', () => {
+  let updateTitleAndSummarySpy: MockInstance<typeof summaryUpdater.updateTitleAndSummary>
+  let acknowledgeStartOfProcessingMock: Mock<WorkflowProxy['acknowledgeStartOfProcessing']>
+  let setCompletedMock: Mock<WorkflowProxy['setCompleted']>
+  let listConversationsMock: Mock<Client['listConversations']>
+  let listMessagesMock: Mock<Client['listMessages']>
+  let loggerInfoMock: Mock<Logger['info']>
+
+  const initProps = (): WorkflowProps =>
+    as<WorkflowProps>({
+      workflow: as<WorkflowProxy>({
+        acknowledgeStartOfProcessing: acknowledgeStartOfProcessingMock,
+        setCompleted: setCompletedMock,
+      }),
+      client: as<Client>({
+        listConversations: listConversationsMock,
+        listMessages: listMessagesMock,
+      }),
+      logger: as<Logger>({
+        info: loggerInfoMock,
+      }),
+    })
+
+  beforeEach(() => {
+    updateTitleAndSummarySpy = vi.spyOn(summaryUpdater, 'updateTitleAndSummary').mockResolvedValue(undefined)
+    acknowledgeStartOfProcessingMock = vi.fn().mockResolvedValue(undefined)
+    setCompletedMock = vi.fn().mockResolvedValue(undefined)
+    listConversationsMock = vi.fn()
+    listMessagesMock = vi.fn()
+    loggerInfoMock = vi.fn()
+  })
+
+  it('should acknowledge start, update all dirty conversations, and complete if no nextToken', async () => {
+    listConversationsMock.mockResolvedValue({
+      conversations: [as<Conversation>({ id: 'c1' }), as<Conversation>({ id: 'c2' })],
+      meta: {},
+    })
+    listMessagesMock.mockResolvedValue({
+      messages: [
+        as<Message>({ id: 'm1', conversationId: 'c1', type: 'text', payload: { text: 'Hello1' } }),
+        as<Message>({ id: 'm2', conversationId: 'c1', type: 'text', payload: { text: 'Hello2' } }),
+      ],
+      meta: {},
+    })
+
+    const props = initProps()
+    await updateAllConversations(props)
+
+    expect(props.workflow.acknowledgeStartOfProcessing).toHaveBeenCalled()
+    expect(props.client.listConversations).toHaveBeenCalledWith({ tags: { isDirty: 'true' } })
+    expect(props.client.listMessages).toHaveBeenCalledTimes(2)
+    expect(updateTitleAndSummarySpy).toHaveBeenCalledTimes(2)
+    expect(props.workflow.setCompleted).toHaveBeenCalled()
+  })
+
+  it('should handle no dirty conversations gracefully', async () => {
+    listConversationsMock.mockResolvedValue({
+      conversations: [],
+      meta: { nextToken: undefined },
+    })
+
+    const props = initProps()
+    await updateAllConversations(props)
+
+    expect(props.client.listMessages).not.toHaveBeenCalled()
+    expect(updateTitleAndSummarySpy).not.toHaveBeenCalled()
+    expect(props.workflow.setCompleted).toHaveBeenCalled()
+  })
+})

plugins/conversation-insights/src/updateAllConversations.ts

@@ -0,0 +1,22 @@
+import * as summaryUpdater from './tagsUpdater'
+import * as types from './types'
+import * as bp from '.botpress'
+
+export type WorkflowProps = types.CommonProps & bp.WorkflowHandlerProps['updateAllConversations']
+export const updateAllConversations = async (props: WorkflowProps) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+  const conversations = props.conversations['*']['*'].list({ tags: { isDirty: 'true' } })
+  const dirtyConversations = await conversations.takePage(1)
+
+  const promises: Promise<void>[] = []
+  for (const conversation of dirtyConversations) {
+    const firstMessagePage = await conversation.listMessages().takePage(1)
+    const promise = summaryUpdater.updateTitleAndSummary({ ...props, conversation, messages: firstMessagePage })
+    promises.push(promise)
+  }
+
+  await Promise.all(promises)
+  if (conversations.isExhausted) {
+    await props.workflow.setCompleted()
+  }
+}

plugins/conversation-insights/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": { "*": ["./*"] },
+    "outDir": "dist"
+  },
+  "include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
+}

plugins/conversation-insights/vitest.config.ts

@@ -0,0 +1,2 @@
+import config from '../../vitest.config'
+export default config

plugins/file-synchronizer/definitions/models.ts

@@ -0,0 +1,43 @@
+import * as sdk from '@botpress/sdk'
+
+const _baseItem = (itemType: 'file' | 'folder' | 'item' = 'item') =>
+  sdk.z.object({
+    id: sdk.z
+      .string()
+      .describe(
+        `The ${itemType}'s ID. This could be a unique identifier from the external service, or a relative or absolute path, so long as it's unique.`
+      ),
+    type: sdk.z.union([sdk.z.literal('file'), sdk.z.literal('folder')]).describe('The entity type'),
+    name: sdk.z
+      .string()
+      .describe(
+        `The ${itemType}'s name. This will be displayed in the Botpress UI and be used as the ${itemType}'s name on Files API."`
+      ),
+    parentId: sdk.z
+      .string()
+      .optional()
+      .describe(`The parent folder ID. Leave empty if the ${itemType} is in the root folder.`),
+  })
+
+export const FOLDER = _baseItem('folder').extend({
+  type: sdk.z.literal('folder'),
+})
+
+export const FILE = _baseItem('file').extend({
+  type: sdk.z.literal('file'),
+  sizeInBytes: sdk.z.number().optional().describe('The file size in bytes, if available'),
+  lastModifiedDate: sdk.z.string().datetime().optional().describe('The last modified date of the file, if available'),
+  contentHash: sdk.z
+    .string()
+    .optional()
+    .describe('The hash of the file content, or version/revision number, if available'),
+})
+
+export const FILE_WITH_PATH = FILE.extend({
+  absolutePath: sdk.z.string().describe('The absolute path of the file'),
+})
+
+export type Folder = sdk.z.infer<typeof FOLDER>
+export type File = sdk.z.infer<typeof FILE>
+export type FileWithPath = sdk.z.infer<typeof FILE_WITH_PATH>
+export type FolderItem = Folder | File

plugins/file-synchronizer/eslint.config.mjs

@@ -0,0 +1,13 @@
+import rootConfig from '../../eslint.config.mjs'
+
+export default [
+  ...rootConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]

plugins/file-synchronizer/hub.md

@@ -0,0 +1 @@
+# File Synchronizer Plugin

plugins/file-synchronizer/icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="241" height="241" fill="none"><path fill="purple" d="M34 0A34 34 0 0 0 0 34v173a34 34 0 0 0 34 34h173a34 34 0 0 0 34-34V34a34 34 0 0 0-34-34Zm51.8 33.6h62a8.8 8.7 0 0 1 3.2.6 8.8 8.7 0 0 1 .2.1 8.8 8.7 0 0 1 2.8 1.9l44.2 43.4a8.8 8.7 0 0 1 1.9 2.8 8.8 8.7 0 0 1 0 .2 8.8 8.7 0 0 1 .7 3.2v95.7c0 14.3-12 26.1-26.5 26.1h-48.7a8.8 8.7 0 0 1-8.8-8.7 8.8 8.7 0 0 1 8.8-8.7h48.7c5 0 8.8-3.8 8.8-8.7v-87h-26.5c-9.7 0-17.7-7.9-17.7-17.4v-26H85.8c-5 0-8.9 3.7-8.9 8.6v69.6a8.8 8.7 0 0 1-8.8 8.7 8.8 8.7 0 0 1-8.9-8.7V59.7c0-14.3 12-26.1 26.6-26.1zm70.8 29.7v13.8h14zm-79.7 83.4a8.8 8.7 0 0 1 6.3 2.6l26.5 26a8.8 8.7 0 0 1 2.6 6.2 8.8 8.7 0 0 1-2.6 6.2l-26.5 26a8.8 8.7 0 0 1-12.5 0 8.8 8.7 0 0 1 0-12.2L82 190.2H41.6a8.8 8.7 0 0 1-8.9-8.7 8.8 8.7 0 0 1 8.9-8.7H82l-11.4-11.2a8.8 8.7 0 0 1 0-12.3 8.8 8.7 0 0 1 6.2-2.6z"/></svg>

plugins/file-synchronizer/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@botpresshub/file-synchronizer",
+  "scripts": {
+    "check:type": "tsc --noEmit",
+    "build": "bp add -y && bp build",
+    "test": "vitest --run"
+  },
+  "private": true,
+  "dependencies": {
+    "@botpress/client": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "picomatch": "^4.0.2"
+  },
+  "devDependencies": {
+    "@botpress/cli": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "@botpresshub/hitl": "workspace:*",
+    "@types/picomatch": "^3.0.2",
+    "@types/semver": "^7.3.11",
+    "semver": "^7.3.8"
+  },
+  "bpDependencies": {
+    "files-readonly": "../../interfaces/files-readonly"
+  }
+}

plugins/file-synchronizer/plugin.definition.ts

@@ -0,0 +1,202 @@
+import * as sdk from '@botpress/sdk'
+import filesReadonly from './bp_modules/files-readonly'
+
+const FILE_FILTER_PROPS = sdk.z.object({
+  includeFiles: sdk.z.array(
+    sdk.z
+      .object({
+        pathGlobPattern: sdk.z
+          .string()
+          .placeholder('Example: /path/to/folder/**')
+          .describe(
+            'A glob pattern to match against the file path. Only files that match the pattern will be synchronized. Any pattern supported by picomatch is supported. For example, use rule "**" to match all files, or enter a path like "/path/to/folder/**" to match all files in a specific folder.'
+          ),
+        maxSizeInBytes: sdk.z
+          .number()
+          .optional()
+          .describe(
+            'Filter by maximum size (in bytes). Only files smaller than the specified size will be synchronized.'
+          ),
+        modifiedAfter: sdk.z
+          .string()
+          .datetime()
+          .optional()
+          .describe(
+            'Filter the items by modified date. Only files modified after the specified date will be synchronized.'
+          ),
+        applyOptionsToMatchedFiles: sdk.z
+          .object({
+            addToKbId: sdk.z
+              .string()
+              .placeholder('Example: kb-2f0a7ea639')
+              .optional()
+              .title('Knowledge Base ID')
+              .describe(
+                'The ID of the knowledge base to add the file to. Note that files added to knowledge bases will count towards both the Vector DB Storage quota and the File Storage quota of the workspace.'
+              ),
+          })
+          .optional()
+          .title('Apply to Matched Files')
+          .describe('Options to apply to the matched files.'),
+      })
+      .title('Include Criteria')
+      .describe('A file must match all criteria to be synchronized.')
+  ),
+  excludeFiles: sdk.z.array(
+    sdk.z
+      .object({
+        pathGlobPattern: sdk.z
+          .string()
+          .placeholder('Example: /path/to/folder/**')
+          .describe(
+            'A glob pattern to match against the file path. Files that match the pattern will be ignored, even if they match the includeFiles configuration.'
+          ),
+      })
+      .title('Exclude Criteria')
+      .describe('A file must match all exclude criteria to be ignored.')
+  ),
+})
+
+export default new sdk.PluginDefinition({
+  name: 'file-synchronizer',
+  version: '1.1.2',
+  title: 'File Synchronizer',
+  description: 'Synchronize files from external services to Botpress',
+  icon: 'icon.svg',
+  readme: 'hub.md',
+  configuration: {
+    schema: sdk.z.object({
+      enableRealTimeSync: sdk.z
+        .boolean()
+        .default(true)
+        .describe(
+          'Enable real-time synchronization. Whenever a file is created, updated, or deleted, synchronize it to Botpress immediately. This does not work with every integration.'
+        ),
+      includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+        .title('Include Rules')
+        .describe('A list of rules to include files. Only files that match one or more rules will be synchronized.'),
+      excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+        .title('Exclude Rules')
+        .describe(
+          'A list of rules to exclude files. Files that match one or more rules will be ignored. This takes precedence over Include Rules.'
+        ),
+    }),
+  },
+  actions: {
+    syncFilesToBotpess: {
+      title: 'Sync files to Botpress',
+      description: 'Start synchronization of files from the external service to Botpress',
+      input: {
+        schema: sdk.z.object({
+          includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+            .title('Include Rules Override')
+            .describe('If omitted, the global Include Rules will be used.')
+            .optional(),
+          excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+            .title('Exclude Rules Override')
+            .describe('If omitted, the global Exclude Rules will be used')
+            .optional(),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({
+          status: sdk.z.enum(['queued', 'already-running', 'error']),
+        }),
+      },
+    },
+    listItemsInFolder: {
+      // Re-export the action from the files-readonly interface:
+      ...filesReadonly.definition.actions.listItemsInFolder,
+    },
+  },
+  workflows: {
+    buildQueue: {
+      title: 'Build file sync queue',
+      description: 'Build the file sync queue and synchronize files to Botpress',
+      input: {
+        schema: sdk.z.object({
+          includeFiles: FILE_FILTER_PROPS.shape.includeFiles
+            .title('Include Rules Override')
+            .describe('If omitted, the global Include Rules will be used.'),
+          excludeFiles: FILE_FILTER_PROPS.shape.excludeFiles
+            .title('Exclude Rules Override')
+            .describe('If omitted, the global Exclude Rules will be used'),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({}),
+      },
+      tags: {
+        syncJobId: {
+          title: 'Sync job ID',
+          description: 'The unique ID of the sync job',
+        },
+        syncType: {
+          title: 'Sync type',
+          description: 'The type of sync job',
+        },
+        syncInitiatedAt: {
+          title: 'Created at',
+          description: 'The date and time when the sync job was created',
+        },
+      },
+    },
+    processQueue: {
+      title: 'Process file sync queue',
+      description: 'Process the file sync queue and synchronize files to Botpress',
+      input: {
+        schema: sdk.z.object({
+          jobFileId: sdk.z.string().title('Job File').describe("The ID of the job's queue file"),
+        }),
+      },
+      output: {
+        schema: sdk.z.object({}),
+      },
+      tags: {
+        syncJobId: {
+          title: 'Sync job ID',
+          description: 'The unique ID of the sync job',
+        },
+        syncType: {
+          title: 'Sync type',
+          description: 'The type of sync job',
+        },
+        syncInitiatedAt: {
+          title: 'Created at',
+          description: 'The date and time when the sync job was created',
+        },
+      },
+    },
+  },
+  states: {
+    buildQueueRuntimeState: {
+      type: 'workflow',
+      schema: sdk.z.object({
+        jobFileId: sdk.z.string().title('Job File').describe("The ID of the job's queue file"),
+        enumerationState: sdk.z
+          .object({
+            pendingFolders: sdk.z
+              .array(
+                sdk.z.object({
+                  folderId: sdk.z.string().optional().title('Folder ID').describe('The ID of the folder'),
+                  absolutePath: sdk.z.string().title('Absolute Path').describe('The absolute path of the folder'),
+                })
+              )
+              .title('Pending Folders')
+              .describe('Folders awaiting enumeration'),
+            currentFolderNextToken: sdk.z
+              .string()
+              .optional()
+              .title('Current Folder Paging Token')
+              .describe('The next token to use for pagination'),
+          })
+          .optional()
+          .title('Enumeration State')
+          .describe('The current state of the enumeration process'),
+      }),
+    },
+  },
+  interfaces: {
+    'files-readonly': sdk.version.allWithinMajorOf(filesReadonly),
+  },
+})

plugins/file-synchronizer/src/actions/index.ts

@@ -0,0 +1 @@
+export * as syncFilesToBotpess from './sync-files-to-botpress'

plugins/file-synchronizer/src/actions/sync-files-to-botpress.ts

@@ -0,0 +1,52 @@
+import * as sdk from '@botpress/sdk'
+import { randomUUID } from '../crypto'
+import * as bp from '.botpress'
+
+export const callAction: bp.PluginHandlers['actionHandlers']['syncFilesToBotpess'] = async (props) => {
+  if (await _isSyncAlreadyInProgress(props)) {
+    props.logger.info('Sync is already in progress. Ignoring sync event...')
+    return { status: 'already-running' }
+  }
+
+  const includeFiles = props.input.includeFiles ?? props.configuration.includeFiles
+  const excludeFiles = props.input.excludeFiles ?? props.configuration.excludeFiles
+
+  if (includeFiles.length === 0) {
+    throw new sdk.RuntimeError(
+      'No include rules defined. Please define at least one include rule. For example, create a rule with glob pattern "**" to include all files.'
+    )
+  }
+
+  props.logger.info('Syncing files to Botpress...', {
+    includeFiles,
+    excludeFiles,
+  })
+
+  props.logger.info('Enumerating files...')
+  await props.workflows.buildQueue.startNewInstance({
+    input: { includeFiles, excludeFiles },
+    tags: {
+      syncJobId: await randomUUID(),
+      syncType: 'manual',
+      syncInitiatedAt: new Date().toISOString(),
+    },
+  })
+
+  return { status: 'queued' }
+}
+
+const _isSyncAlreadyInProgress = async (props: bp.ActionHandlerProps) => {
+  const runningBuildQueueWorkflows = await props.workflows.buildQueue
+    .listInstances({ statuses: ['pending', 'in_progress', 'listening', 'paused'] })
+    .take(1)
+
+  if (runningBuildQueueWorkflows.length > 0) {
+    return true
+  }
+
+  const runningProcessQueueWorkflows = await props.workflows.processQueue
+    .listInstances({ statuses: ['pending', 'in_progress', 'listening', 'paused'] })
+    .take(1)
+
+  return runningProcessQueueWorkflows.length > 0
+}

plugins/file-synchronizer/src/consts.ts

@@ -0,0 +1 @@
+export const MAX_BATCH_SIZE_BYTES = 104857600 // 100MB

plugins/file-synchronizer/src/crypto.ts

@@ -0,0 +1,28 @@
+export const randomUUID = async (): Promise<string> => {
+  const crypto = await _getWebCrypto()
+  return crypto.randomUUID()
+}
+
+type _WebCrypto = { randomUUID: () => string }
+let _webCrypto: _WebCrypto | undefined
+
+export const _getWebCrypto = async (): Promise<_WebCrypto> => {
+  if (!_webCrypto) {
+    if (typeof (globalThis as any).crypto?.randomUUID === 'function') {
+      _webCrypto = (globalThis as any).crypto
+    } else if (typeof process !== 'undefined' && process.versions?.node) {
+      try {
+        const nodeCrypto = await import('crypto')
+        _webCrypto = nodeCrypto.webcrypto
+      } catch (thrown: unknown) {
+        const error: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+        throw new Error(`Failed to import 'crypto' module: ${error.message}`)
+      }
+    }
+
+    if (!_webCrypto || typeof _webCrypto.randomUUID !== 'function') {
+      throw new Error('No suitable web crypto implementation available')
+    }
+  }
+  return _webCrypto
+}

plugins/file-synchronizer/src/hooks/index.ts

@@ -0,0 +1,4 @@
+export * as onEvent from './on-event'
+export * as onWorkflowStart from './workflow-started'
+export * as onWorkflowContinue from './workflow-continued'
+export * as onWorkflowTimeout from './workflow-timed-out'

plugins/file-synchronizer/src/hooks/on-event/aggregate-file-changes.ts

@@ -0,0 +1,45 @@
+import { handleEvent as handleFileCreated } from './file-created'
+import { handleEvent as handleFileDeleted } from './file-deleted'
+import { handleEvent as handleFileUpdated } from './file-updated'
+import { handleEvent as handleFolderDeletedRecursive } from './folder-deleted-recursive'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:aggregateFileChanges'] = async (props) => {
+  const modifiedItems = props.event.payload.modifiedItems
+
+  for (const deletedItem of modifiedItems.deleted) {
+    if (deletedItem.type === 'file') {
+      await handleFileDeleted({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:fileDeleted', payload: { file: deletedItem } },
+      })
+    } else {
+      await handleFolderDeletedRecursive({
+        ...props,
+        event: { ...props.event, type: 'files-readonly:folderDeletedRecursive', payload: { folder: deletedItem } },
+      })
+    }
+  }
+
+  for (const createdItem of modifiedItems.created) {
+    if (createdItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileCreated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileCreated', payload: { file: createdItem } },
+    })
+  }
+
+  for (const updatedItem of modifiedItems.updated) {
+    if (updatedItem.type !== 'file') {
+      continue
+    }
+
+    await handleFileUpdated({
+      ...props,
+      event: { ...props.event, type: 'files-readonly:fileUpdated', payload: { file: updatedItem } },
+    })
+  }
+}

plugins/file-synchronizer/src/hooks/on-event/file-created.ts

@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileCreated'] = async (props) => {
+  const createdFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: createdFile,
+    itemPath: createdFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${createdFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...createdFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${createdFile.absolutePath} has been synchronized`)
+}

plugins/file-synchronizer/src/hooks/on-event/file-deleted.ts

@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileDeleted'] = async (props) => {
+  const deletedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFile,
+    itemPath: deletedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${deletedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalId: deletedFile.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFile.absolutePath} has been deleted`)
+    }
+  } catch {}
+}

plugins/file-synchronizer/src/hooks/on-event/file-updated.ts

@@ -0,0 +1,34 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:fileUpdated'] = async (props) => {
+  const updatedFile = props.event.payload.file
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: updatedFile,
+    itemPath: updatedFile.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring file ${updatedFile.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  await SyncQueue.fileProcessor.processQueueFile({
+    fileRepository: props.client,
+    fileToSync: {
+      ...updatedFile,
+      status: 'pending',
+      shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+      addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+    },
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    logger: props.logger,
+  })
+
+  props.logger.info(`File ${updatedFile.absolutePath} has been synchronized`)
+}

plugins/file-synchronizer/src/hooks/on-event/folder-deleted-recursive.ts

@@ -0,0 +1,26 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.EventHandlers['files-readonly:folderDeletedRecursive'] = async (props) => {
+  const deletedFolder = props.event.payload.folder
+  const globMatchResult = SyncQueue.globMatcher.matchItem({
+    configuration: props.configuration,
+    item: deletedFolder,
+    itemPath: deletedFolder.absolutePath,
+  })
+
+  if (globMatchResult.shouldBeIgnored) {
+    props.logger.debug(`Ignoring folder ${deletedFolder.absolutePath}. Reason: ${globMatchResult.reason}`)
+    return
+  }
+
+  try {
+    const { files } = await props.client.listFiles({ tags: { externalParentId: deletedFolder.id } })
+
+    for (const filesApiFile of files) {
+      await props.client.deleteFile({ id: filesApiFile.id })
+
+      props.logger.info(`File ${deletedFolder.absolutePath} has been deleted`)
+    }
+  } catch {}
+}

plugins/file-synchronizer/src/hooks/on-event/index.ts

@@ -0,0 +1,5 @@
+export * as fileCreated from './file-created'
+export * as fileDeleted from './file-deleted'
+export * as fileUpdated from './file-updated'
+export * as folderDeletedRecursive from './folder-deleted-recursive'
+export * as aggregateFileChanges from './aggregate-file-changes'

plugins/file-synchronizer/src/hooks/workflow-continued/build-queue.ts

@@ -0,0 +1,77 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  props.logger.info('Retrieving job file...')
+  const workflowState = await props.states.workflow.buildQueueRuntimeState.get(props.workflow.id)
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Enumerating files...')
+  const enumerationState = await SyncQueue.directoryTraversalWithBatching.enumerateAllFilesRecursive(
+    _getEnumerateAllFilesRecursiveProps(props, syncQueue, key, workflowState)
+  )
+
+  if (enumerationState !== undefined) {
+    // Enumeration is still in progress
+    props.logger.debug('Enumeration partially completed')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { ...workflowState, enumerationState })
+    return
+  }
+
+  const { syncQueue: finalSyncQueue } = await SyncQueue.jobFileManager.getSyncQueue(props, workflowState.jobFileId)
+
+  if (finalSyncQueue.length === 0) {
+    props.logger.info(
+      'Enumeration matched no files. Nothing to sync. Please check your include and exclude rules. ' +
+        `There could also be an issue with your configuration in the "${props.interfaces['files-readonly'].name}" integration. ` +
+        'For example, your access token might be missing some permissions or the integration might not be set up correctly.'
+    )
+    await props.workflow.setCompleted()
+    return
+  }
+
+  props.logger.info('Enumeration completed. Starting sync job...')
+  await props.workflow.setCompleted()
+  await props.workflows.processQueue.startNewInstance({
+    input: { jobFileId: workflowState.jobFileId },
+    tags: {
+      syncInitiatedAt: props.workflow.tags.syncInitiatedAt!,
+      syncJobId: props.workflow.tags.syncJobId!,
+      syncType: props.workflow.tags.syncType!,
+    },
+  })
+}
+
+const _getEnumerateAllFilesRecursiveProps = (
+  props: bp.WorkflowHandlerProps['buildQueue'],
+  syncQueue: SyncQueue.queueProcessor.ProcessQueueProps['syncQueue'],
+  syncFileKey: string,
+  workflowState: bp.states.States['buildQueueRuntimeState']['payload']
+): SyncQueue.directoryTraversalWithBatching.EnumerateAllFilesRecursiveProps => ({
+  ...props,
+
+  configuration: props.workflow.input,
+
+  currentEnumerationState: workflowState.enumerationState,
+  integration: { listItemsInFolder: props.actions['files-readonly'].listItemsInFolder },
+
+  async pushFilesToQueue(files) {
+    if (!files.length) {
+      return
+    }
+
+    const dedupedQueue = new Map(syncQueue.map((file) => [file.id, file]))
+    files.forEach((file) => dedupedQueue.set(file.id, { ...file, status: 'pending' } as const))
+
+    await SyncQueue.jobFileManager.updateSyncQueue(
+      props,
+      syncFileKey,
+      Array.from(dedupedQueue.values()),
+      props.workflow.tags
+    )
+  },
+})

plugins/file-synchronizer/src/hooks/workflow-continued/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-continued/process-queue.ts

@@ -0,0 +1,31 @@
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  const { syncQueue, key } = await SyncQueue.jobFileManager.getSyncQueue(props)
+  const logger = props.logger.withWorkflowId(props.workflow.id)
+
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  const { finished } = await SyncQueue.queueProcessor.processQueue({
+    logger,
+    syncQueue,
+    fileRepository: props.client,
+    integration: {
+      ...props.interfaces['files-readonly'],
+      alias: props.interfaces['files-readonly'].integrationAlias,
+      transferFileToBotpress: props.actions['files-readonly'].transferFileToBotpress,
+    },
+    updateSyncQueue: (params) => SyncQueue.jobFileManager.updateSyncQueue(props, key, params.syncQueue),
+  })
+
+  if (finished === 'batch') {
+    logger.info('Batch sync success. Continuing to next batch...')
+    const timeIn5Minutes = new Date(Date.now() + 300_000).toISOString()
+    await props.workflow.update({ timeoutAt: timeIn5Minutes })
+    return
+  }
+
+  logger.info('Sync completed successfully')
+  await props.workflow.setCompleted()
+}

plugins/file-synchronizer/src/hooks/workflow-started/build-queue.ts

@@ -0,0 +1,24 @@
+import * as sdk from '@botpress/sdk'
+import * as SyncQueue from '../../sync-queue'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+
+  props.logger.info('Creating job file...')
+  const syncFileKey = _generateFileKey(props)
+  const jobFileId = await SyncQueue.jobFileManager.updateSyncQueue(props, syncFileKey, [], props.workflow.tags)
+
+  await props.states.workflow.buildQueueRuntimeState.set(props.workflow.id, { jobFileId })
+}
+
+const _generateFileKey = (props: bp.WorkflowHandlerProps['buildQueue']) => {
+  const integrationName = props.interfaces['files-readonly'].name
+  const syncJobId = props.workflow.tags.syncJobId
+
+  if (!syncJobId) {
+    throw new sdk.RuntimeError('Sync job ID is not defined in workflow tags')
+  }
+
+  return `file-synchronizer:${integrationName}:/${syncJobId}.jsonl`
+}

plugins/file-synchronizer/src/hooks/workflow-started/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-started/process-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.acknowledgeStartOfProcessing()
+}

plugins/file-synchronizer/src/hooks/workflow-timed-out/build-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['buildQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}

plugins/file-synchronizer/src/hooks/workflow-timed-out/index.ts

@@ -0,0 +1,2 @@
+export * as buildQueue from './build-queue'
+export * as processQueue from './process-queue'

plugins/file-synchronizer/src/hooks/workflow-timed-out/process-queue.ts

@@ -0,0 +1,5 @@
+import * as bp from '.botpress'
+
+export const handleEvent: bp.WorkflowHandlers['processQueue'] = async (props) => {
+  await props.workflow.setFailed({ failureReason: 'Workflow timed out' })
+}

plugins/file-synchronizer/src/index.ts

@@ -0,0 +1,93 @@
+import * as actions from './actions'
+import * as hooks from './hooks'
+import * as bp from '.botpress'
+
+const plugin = new bp.Plugin({
+  actions: {
+    async syncFilesToBotpess(props) {
+      props.logger.info('Called action syncFilesToBotpess')
+      return await actions.syncFilesToBotpess.callAction(props)
+    },
+    async listItemsInFolder(props) {
+      props.logger.info('Called action listItemsInFolder. Redirecting to integration...')
+      return await props.actions['files-readonly'].listItemsInFolder(props.input)
+    },
+  },
+})
+
+plugin.on.event('files-readonly:fileCreated', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File created event triggered', props.event.payload.file)
+  await hooks.onEvent.fileCreated.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:fileDeleted', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File deleted event triggered', props.event.payload.file)
+  await hooks.onEvent.fileDeleted.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:fileUpdated', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('File updated event triggered', props.event.payload.file)
+  await hooks.onEvent.fileUpdated.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:folderDeletedRecursive', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('Folder deleted event triggered', props.event.payload.folder)
+  await hooks.onEvent.folderDeletedRecursive.handleEvent(props)
+})
+
+plugin.on.event('files-readonly:aggregateFileChanges', async (props) => {
+  if (!props.configuration.enableRealTimeSync) {
+    return
+  }
+
+  props.logger.info('Aggregate file changes event triggered', props.event.payload.modifiedItems)
+  await hooks.onEvent.aggregateFileChanges.handleEvent(props)
+})
+
+plugin.on.workflowStart('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow started', props.workflow.tags)
+  await hooks.onWorkflowStart.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowContinue('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow continued', props.workflow.tags)
+  await hooks.onWorkflowContinue.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowTimeout('buildQueue', async (props) => {
+  props.logger.info('buildQueue workflow timed out', props.workflow.tags)
+  await hooks.onWorkflowTimeout.buildQueue.handleEvent(props)
+})
+
+plugin.on.workflowStart('processQueue', async (props) => {
+  props.logger.info('processQueue workflow started', props.workflow.tags)
+  await hooks.onWorkflowStart.processQueue.handleEvent(props)
+})
+
+plugin.on.workflowContinue('processQueue', async (props) => {
+  props.logger.info('processQueue workflow continued', props.workflow.tags)
+  await hooks.onWorkflowContinue.processQueue.handleEvent(props)
+})
+
+plugin.on.workflowTimeout('processQueue', async (props) => {
+  props.logger.info('processQueue workflow timed out', props.workflow.tags)
+  await hooks.onWorkflowTimeout.processQueue.handleEvent(props)
+})
+
+export default plugin

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts

@@ -0,0 +1,326 @@
+import type * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { enumerateAllFilesRecursive, type EnumerationState } from './directory-traversal-with-batching'
+
+const _getMocks = () => ({
+  logger: {
+    debug: vi.fn(),
+  } as unknown as sdk.BotLogger,
+  integration: {
+    listItemsInFolder: vi.fn().mockResolvedValue({ items: [], meta: {} }),
+  },
+  globMatcher: {
+    matchItem: vi.fn().mockReturnValue({ shouldBeIgnored: false, shouldApplyOptions: {} }),
+  },
+  configuration: {
+    includeFiles: [{ pathGlobPattern: '**' }],
+    excludeFiles: [],
+  },
+  pushFilesToQueue: vi.fn(),
+})
+
+describe.concurrent('enumerateAllFilesRecursive', () => {
+  describe('Basic enumeration', () => {
+    it('should enumerate an empty root folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.integration.listItemsInFolder).toHaveBeenNthCalledWith(1, {
+        folderId: undefined,
+        nextToken: undefined,
+      })
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should enumerate a single file in root folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+      const testFile: models.FolderItem = {
+        id: 'file1',
+        type: 'file',
+        name: 'test.txt',
+      }
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [testFile],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          ...testFile,
+          absolutePath: '/test.txt',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Recursive folder traversal', () => {
+    it('should recursively enumerate files in nested folders', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // Root folder items:
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [
+          { id: 'folder1', type: 'folder', name: 'folder1' },
+          { id: 'file1', type: 'file', name: 'root-file.txt' },
+        ],
+        meta: { nextToken: undefined },
+      })
+
+      // folder1 items:
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file2', type: 'file', name: 'nested-file.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: '/root-file.txt',
+        }),
+        expect.objectContaining({
+          absolutePath: '/folder1/nested-file.txt',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should handle deeply nested folder structures', async () => {
+      // Arrange
+      const mocks = _getMocks()
+      const folderDepth = 100
+
+      // Create a chain of nested folders
+      for (let i = 1; i <= folderDepth; i++) {
+        mocks.integration.listItemsInFolder.mockResolvedValueOnce(
+          i === folderDepth
+            ? {
+                items: [{ id: 'deepFile', type: 'file', name: 'deepFile.txt' }],
+                meta: { nextToken: undefined },
+              }
+            : {
+                items: [{ id: `folder${i}`, type: 'folder', name: `folder${i}` }],
+                meta: { nextToken: undefined },
+              }
+        )
+      }
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(folderDepth)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: `/${Array.from({ length: folderDepth - 1 }, (_, i) => `folder${i + 1}`).join('/')}/deepFile.txt`,
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Pagination handling', () => {
+    it('should support pagination within a folder', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      // First page of root folder
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file1', type: 'file', name: 'file1.txt' }],
+        meta: { nextToken: 'page2' },
+      })
+
+      // Second page of root folder
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file2', type: 'file', name: 'file2.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const enumerationState = await enumerateAllFilesRecursive(mocks)
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
+        expect.objectContaining({
+          id: 'file1',
+        }),
+        expect.objectContaining({
+          id: 'file2',
+        }),
+      ])
+      expect(enumerationState).toBeUndefined()
+    })
+
+    it('should resume using an enumeration state', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      const previousEnumerationState: EnumerationState = {
+        pendingFolders: [{ absolutePath: '/folder1/', folderId: 'folder1' }],
+      }
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [{ id: 'file3', type: 'file', name: 'file3.txt' }],
+        meta: { nextToken: undefined },
+      })
+
+      // Act
+      const newEnumerationState = await enumerateAllFilesRecursive({
+        ...mocks,
+        currentEnumerationState: previousEnumerationState,
+      })
+
+      // Assert
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledWith({
+        folderId: 'folder1',
+        nextToken: undefined,
+      })
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+        expect.objectContaining({
+          absolutePath: '/folder1/file3.txt',
+        }),
+      ])
+      expect(newEnumerationState).toBeUndefined()
+    })
+  })
+
+  describe('Glob file filtering', () => {
+    it('should properly apply file filtering', async () => {
+      // Arrange
+      const mocks = _getMocks()
+
+      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+        items: [
+          { id: 'file1', type: 'file', name: 'included.txt' },
+          { id: 'file2', type: 'file', name: 'excluded.txt' },
+        ],
+        meta: { nextToken: undefined },
+      })
+
+      // First file is included, second is excluded
+      mocks.globMatcher.matchItem
+        .mockReturnValueOnce({
+          shouldBeIgnored: false,
+          shouldApplyOptions: { addToKbId: 'kb1' },
+        })
+        .mockReturnValueOnce({
+          shouldBeIgnored: true,
+          reason: 'matches-exclude-pattern',
+        })
+
+      // Act
+      void (await enumerateAllFilesRecursive(mocks))
+
+      // Assert
+      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
+        expect.objectContaining({
+          id: 'file1',
+        }),
+      ])
+      expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object), expect.any(String))
+    })
+  })
+})
+
+describe.sequential('enumerateAllFilesRecursive duration handling', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    vi.useRealTimers()
+  })
+
+  it('should not proceed to the next batch in case of a timeout', async () => {
+    // Arrange
+    const mocks = _getMocks()
+    const maximumExecutionTimeMs = 5_000 // 5 seconds
+
+    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+      items: [
+        { id: 'file1', type: 'file', name: 'file1.txt' },
+        { id: 'file2', type: 'file', name: 'file2.txt' },
+        { id: 'file3', type: 'file', name: 'file3.txt' },
+      ],
+      meta: { nextToken: 'abcd' },
+    })
+
+    const resultPromise = enumerateAllFilesRecursive({
+      ...mocks,
+      maximumExecutionTimeMs,
+    })
+
+    // Advance time after the first batch is processed:
+    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)
+
+    void (await resultPromise)
+
+    // Assert
+    expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
+  })
+
+  it('should process the entire current batch in case of a timeout', async () => {
+    // Arrange
+    const mocks = _getMocks()
+    const maximumExecutionTimeMs = 5_000 // 5 seconds
+
+    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
+      items: [
+        { id: 'file1', type: 'file', name: 'file1.txt' },
+        { id: 'file2', type: 'file', name: 'file2.txt' },
+        { id: 'file3', type: 'file', name: 'file3.txt' },
+      ],
+      meta: { nextToken: 'abcd' },
+    })
+
+    const resultPromise = enumerateAllFilesRecursive({
+      ...mocks,
+      maximumExecutionTimeMs,
+    })
+
+    // Advance time after the first batch is processed:
+    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)
+
+    void (await resultPromise)
+
+    // Assert
+    expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
+    expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
+      expect.objectContaining({
+        id: 'file1',
+      }),
+      expect.objectContaining({
+        id: 'file2',
+      }),
+      expect.objectContaining({
+        id: 'file3',
+      }),
+    ])
+  })
+})

plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.ts

@@ -0,0 +1,132 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import * as syncQueueGlobMatcher from './glob-matcher'
+import * as bp from '.botpress'
+
+type FileWithOptions = models.FileWithPath & { shouldIndex: boolean; addToKbId?: string }
+
+export type EnumerationState = NonNullable<bp.states.States['buildQueueRuntimeState']['payload']['enumerationState']>
+
+type IntegrationActionProxy = Pick<
+  bp.WorkflowHandlerProps['processQueue']['actions']['files-readonly'],
+  'listItemsInFolder'
+>
+
+export type EnumerateAllFilesRecursiveProps = {
+  logger: sdk.BotLogger
+  integration: IntegrationActionProxy
+  configuration: Pick<bp.configuration.Configuration, 'includeFiles' | 'excludeFiles'>
+  currentEnumerationState?: EnumerationState
+  maximumExecutionTimeMs?: number
+  globMatcher?: {
+    matchItem: (props: syncQueueGlobMatcher.GlobMatcherProps) => syncQueueGlobMatcher.GlobMatchResult
+  }
+  pushFilesToQueue: (files: FileWithOptions[]) => Promise<void>
+}
+
+// 30 seconds should be a safe default for the max execution time:
+const DEFAULT_MAXIMUM_EXECUTION_TIME_MS = 30_000
+
+export const enumerateAllFilesRecursive = async ({
+  logger,
+  integration,
+  configuration,
+  currentEnumerationState,
+  maximumExecutionTimeMs = DEFAULT_MAXIMUM_EXECUTION_TIME_MS,
+  globMatcher = syncQueueGlobMatcher,
+  pushFilesToQueue: pushFilesToSyncQueue,
+}: EnumerateAllFilesRecursiveProps): Promise<EnumerationState | undefined> => {
+  const enumerationState: EnumerationState = currentEnumerationState ?? {
+    pendingFolders: [{ absolutePath: '/' }],
+  }
+
+  const includedFiles: FileWithOptions[] = []
+  const startTime = Date.now()
+
+  while (enumerationState.pendingFolders.length > 0) {
+    const { items: folderItems, nextToken } = await _listFolderContents(integration, enumerationState)
+
+    for (const folderItem of folderItems) {
+      const currentFolder = enumerationState.pendingFolders[0]!
+      const itemPath = `${currentFolder.absolutePath}${folderItem.name}`
+      const globMatchResult = globMatcher.matchItem({ configuration, item: folderItem, itemPath })
+
+      if (globMatchResult.shouldBeIgnored) {
+        logger.debug(
+          'Ignoring item',
+          { itemPath, reason: globMatchResult.reason },
+          JSON.stringify({ item: folderItem, configuration })
+        )
+        continue
+      }
+
+      if (folderItem.type === 'folder') {
+        _addFolderToEnumerationQueue(enumerationState, folderItem.id, itemPath)
+      } else {
+        _addFileToIncludedFiles(includedFiles, folderItem, itemPath, globMatchResult, logger)
+      }
+    }
+
+    if (nextToken) {
+      enumerationState.currentFolderNextToken = nextToken
+
+      if (_isTimeoutReached(startTime, maximumExecutionTimeMs)) {
+        await pushFilesToSyncQueue(includedFiles)
+        return enumerationState
+      }
+    } else {
+      enumerationState.pendingFolders.shift()
+      enumerationState.currentFolderNextToken = undefined
+    }
+  }
+
+  await pushFilesToSyncQueue(includedFiles)
+  return undefined
+}
+
+const _isTimeoutReached = (startTime: number, maximumExecutionTimeMs: number): boolean =>
+  Date.now() - startTime >= maximumExecutionTimeMs
+
+const _listFolderContents = async (integration: IntegrationActionProxy, enumerationState: EnumerationState) => {
+  const folder = enumerationState.pendingFolders[0]!
+  const response = await integration.listItemsInFolder({
+    folderId: folder.folderId,
+    nextToken: enumerationState.currentFolderNextToken,
+  })
+
+  return {
+    items: response.items,
+    nextToken: response.meta.nextToken,
+  }
+}
+
+const _addFolderToEnumerationQueue = (
+  enumerationState: EnumerationState,
+  folderId: string | undefined,
+  itemPath: string
+): void => {
+  enumerationState.pendingFolders.push({
+    folderId,
+    absolutePath: `${itemPath}/`,
+  })
+}
+
+const _addFileToIncludedFiles = (
+  includedFiles: FileWithOptions[],
+  folderItem: models.File,
+  itemPath: string,
+  globMatchResult: syncQueueGlobMatcher.GlobMatchResult,
+  logger: sdk.BotLogger
+): void => {
+  if (globMatchResult.shouldBeIgnored) {
+    return
+  }
+
+  logger.debug('Including file', itemPath)
+  includedFiles.push({
+    ...folderItem,
+    absolutePath: itemPath,
+    shouldIndex: (globMatchResult.shouldApplyOptions.addToKbId?.length ?? 0) > 0,
+    addToKbId: globMatchResult.shouldApplyOptions.addToKbId,
+  })
+}

plugins/file-synchronizer/src/sync-queue/file-processor.test.ts

@@ -0,0 +1,276 @@
+import * as sdk from '@botpress/sdk'
+import { describe, it, expect, vi, type Mocked } from 'vitest'
+import { processQueueFile, type ProcessFileProps } from './file-processor'
+import type * as types from '../types'
+
+const FILE_1 = {
+  id: 'file1',
+  type: 'file',
+  name: 'file1.txt',
+  absolutePath: '/path/to/file1.txt',
+  sizeInBytes: 100,
+  lastModifiedDate: '2025-01-01T00:00:00Z',
+  contentHash: 'hash1',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const FILE_2 = {
+  id: 'file2',
+  type: 'file',
+  name: 'file2.txt',
+  absolutePath: '/path/to/file2.txt',
+  sizeInBytes: 200,
+  lastModifiedDate: '2025-01-02T00:00:00Z',
+  contentHash: 'hash2',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const EXISTING_FILE = 'dummy-id'
+const FUTURE_DATE = '9999-01-01T00:00:00Z'
+const PAST_DATE = '0000-01-01T00:00:00Z'
+const DIFFERENT_HASH = 'different-hash'
+
+describe.concurrent('processQueue', () => {
+  const getMocks = () => ({
+    fileRepository: {
+      listFiles: vi.fn(),
+      deleteFile: vi.fn(),
+      updateFileMetadata: vi.fn(),
+    } as const satisfies Mocked<ProcessFileProps['fileRepository']>,
+    integration: {
+      name: 'test-integration',
+      alias: 'test-integration-alias',
+      transferFileToBotpress: vi.fn(),
+    } as const satisfies Mocked<ProcessFileProps['integration']>,
+    logger: new sdk.BotLogger(),
+  })
+
+  it('should skip files that are already synced with identical content hash', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalContentHash: FILE_1.contentHash,
+          },
+        },
+      ],
+    })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+  })
+
+  it('should upload file when more recent and content hash is different', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: PAST_DATE,
+          },
+        },
+      ],
+    })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.deleteFile).toHaveBeenCalledWith({ id: EXISTING_FILE })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+  })
+
+  it('should not upload file when existing file is same age', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: FILE_1.lastModifiedDate,
+          },
+        },
+      ],
+    })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+  })
+
+  it('should not upload file when existing file is more recent', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: FUTURE_DATE,
+          },
+        },
+      ],
+    })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'already-synced' })
+    expect(mocks.fileRepository.deleteFile).not.toHaveBeenCalled()
+    expect(mocks.integration.transferFileToBotpress).not.toHaveBeenCalled()
+  })
+
+  it('should upload file when different hash and missing last modified date', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({
+      files: [
+        {
+          id: EXISTING_FILE,
+          tags: {
+            externalId: FILE_1.id,
+            externalContentHash: DIFFERENT_HASH,
+            externalModifiedDate: PAST_DATE,
+          },
+        },
+      ],
+    })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.deleteFile).toHaveBeenCalledWith({ id: EXISTING_FILE })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+  })
+
+  it('should handle errors by modifying the queue item', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+
+    // First transfer fails, second succeeds:
+    mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'errored', errorMessage: 'Transfer failed' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(1)
+    expect(mocks.fileRepository.updateFileMetadata).not.toHaveBeenCalled()
+  })
+
+  it('should properly set metadata tags after transferring file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: FILE_1,
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
+      id: FILE_1.id,
+      tags: {
+        integrationName: 'test-integration',
+        integrationAlias: 'test-integration-alias',
+        externalId: FILE_1.id,
+        externalModifiedDate: FILE_1.lastModifiedDate,
+        externalSize: FILE_1.sizeInBytes.toString(),
+        externalContentHash: FILE_1.contentHash,
+        externalPath: FILE_1.absolutePath,
+        externalParentId: FILE_1.parentId,
+      },
+    })
+  })
+
+  it('should assign to kb if needed after transferring file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const result = processQueueFile({
+      fileToSync: { ...FILE_1, addToKbId: 'kb1' },
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toMatchObject({ status: 'newly-synced' })
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledWith({
+      id: FILE_1.id,
+      tags: {
+        integrationName: 'test-integration',
+        integrationAlias: 'test-integration-alias',
+        externalId: FILE_1.id,
+        externalModifiedDate: FILE_1.lastModifiedDate,
+        externalSize: FILE_1.sizeInBytes.toString(),
+        externalContentHash: FILE_1.contentHash,
+        externalPath: FILE_1.absolutePath,
+        externalParentId: FILE_1.parentId,
+        kbId: 'kb1',
+        modalities: '["text"]',
+        source: 'knowledge-base',
+        title: FILE_1.name,
+      },
+    })
+  })
+})

plugins/file-synchronizer/src/sync-queue/file-processor.ts

@@ -0,0 +1,149 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import type * as types from '../types'
+
+export type ProcessFileProps = {
+  fileToSync: Readonly<types.SyncQueueItem>
+  logger: sdk.BotLogger
+  fileRepository: {
+    listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
+    deleteFile: (params: { id: string }) => Promise<unknown>
+    updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
+  }
+  integration: {
+    name: string
+    alias: string
+    transferFileToBotpress: (params: {
+      file: models.FileWithPath
+      fileKey: string
+      shouldIndex: boolean
+    }) => Promise<{ botpressFileId: string }>
+  }
+}
+
+export const processQueueFile = async (props: ProcessFileProps): Promise<types.SyncQueueItem> => {
+  const fileToSync = structuredClone(props.fileToSync) as types.SyncQueueItem
+  const existingFile = await _getExistingFileFromFilesApi(props, fileToSync)
+  const shouldUploadFile = await _shouldUploadFile(props, fileToSync, existingFile)
+
+  if (!shouldUploadFile) {
+    fileToSync.status = 'already-synced'
+    return fileToSync
+  }
+
+  fileToSync.status = 'newly-synced'
+
+  await _deleteExistingFileFromFilesApi(props, existingFile)
+  await _transferFileToBotpress(props, fileToSync)
+
+  return fileToSync
+}
+
+const _getExistingFileFromFilesApi = async (
+  props: ProcessFileProps,
+  fileToSync: models.FileWithPath
+): Promise<types.FilesApiFile | undefined> => {
+  const { files: existingFiles } = await props.fileRepository.listFiles({
+    tags: {
+      externalId: fileToSync.id,
+      integrationName: props.integration.name,
+      integrationAlias: props.integration.alias,
+    },
+  })
+
+  if (existingFiles.length === 0) {
+    return
+  }
+
+  // Unfortunately, we cannot assume that there is only one file on Files API
+  // with the same externalId, because there is no unique constraint on the tag.
+
+  // However, the implementation would be more complicated if we take this into
+  // account, so we will naively assume that we have full control over the
+  // externalId tag and that it is unique. If users go out of their way to use
+  // the API to create files with the same externalId, we will not be able to
+  // handle this case correctly.
+
+  return existingFiles[0]!
+}
+
+const _shouldUploadFile = async (
+  props: ProcessFileProps,
+  fileToSync: models.FileWithPath,
+  existingFile?: types.FilesApiFile
+) => {
+  if (!existingFile) {
+    props.logger.debug(`No existing file found. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  const newFileHasIdenticalContentHash =
+    fileToSync.contentHash &&
+    existingFile.tags.externalContentHash &&
+    fileToSync.contentHash === existingFile.tags.externalContentHash
+
+  if (newFileHasIdenticalContentHash) {
+    props.logger.debug(`An identical file already exists in Botpress. Ignoring ${fileToSync.absolutePath} ...`)
+    return false
+  }
+
+  const bothFilesHaveModifiedDate = fileToSync.lastModifiedDate && existingFile.tags.externalModifiedDate
+
+  if (!bothFilesHaveModifiedDate) {
+    // Not enough information to compare the files, so we always overwrite:
+    props.logger.debug(`Not enough information to compare files. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  const newFileIsMoreRecent = new Date(fileToSync.lastModifiedDate!) > new Date(existingFile.tags.externalModifiedDate!)
+
+  if (newFileIsMoreRecent) {
+    props.logger.debug(`New file is more recent. Uploading ${fileToSync.absolutePath} ...`)
+    return true
+  }
+
+  props.logger.debug(`Existing file is more recent or same date. Ignoring ${fileToSync.absolutePath} ...`)
+  return false
+}
+
+const _deleteExistingFileFromFilesApi = async (props: ProcessFileProps, existingFile?: types.FilesApiFile) => {
+  if (!existingFile) {
+    return
+  }
+
+  try {
+    await props.fileRepository.deleteFile({ id: existingFile.id })
+  } catch {}
+}
+
+const _transferFileToBotpress = async (props: ProcessFileProps, fileToSync: types.SyncQueueItem) => {
+  try {
+    const { botpressFileId } = await props.integration.transferFileToBotpress({
+      file: fileToSync,
+      fileKey: `${props.integration.alias}:${fileToSync.absolutePath}`,
+      shouldIndex: fileToSync.shouldIndex,
+    })
+
+    await props.fileRepository.updateFileMetadata({
+      id: botpressFileId,
+      tags: {
+        integrationName: props.integration.name,
+        integrationAlias: props.integration.alias,
+        externalId: fileToSync.id,
+        externalModifiedDate: fileToSync.lastModifiedDate ?? null,
+        externalSize: fileToSync.sizeInBytes?.toString() ?? null,
+        externalContentHash: fileToSync.contentHash ?? null,
+        externalPath: fileToSync.absolutePath,
+        externalParentId: fileToSync.parentId ?? null,
+        ...(fileToSync.addToKbId !== undefined
+          ? { kbId: fileToSync.addToKbId, source: 'knowledge-base', title: fileToSync.name, modalities: '["text"]' }
+          : {}),
+      },
+    })
+  } catch (thrown: unknown) {
+    const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    fileToSync.status = 'errored'
+    fileToSync.errorMessage = err.message
+    props.logger.error(`Error while uploading file ${fileToSync.absolutePath}: ${err.message}`)
+  }
+}

plugins/file-synchronizer/src/sync-queue/glob-matcher.test.ts

@@ -0,0 +1,397 @@
+import { describe, it, expect } from 'vitest'
+import { matchItem } from './glob-matcher'
+import type * as models from '../../definitions/models'
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+import type * as bp from '.botpress'
+
+describe.concurrent('matchItem', () => {
+  const createConfiguration = ({
+    includeFiles,
+    excludeFiles,
+  }: {
+    includeFiles?: bp.configuration.Configuration['includeFiles']
+    excludeFiles?: bp.configuration.Configuration['excludeFiles']
+  }) =>
+    ({
+      includeFiles: includeFiles ?? [],
+      excludeFiles: excludeFiles ?? [],
+    }) as const
+
+  describe.concurrent('with file items', () => {
+    const createFileItem = (overrides: Readonly<Partial<models.File>> = {}) =>
+      ({
+        id: 'file-1',
+        type: 'file',
+        name: 'test-file.txt',
+        sizeInBytes: 1000,
+        lastModifiedDate: '1965-01-01T00:00:00Z',
+        ...overrides,
+      }) as const satisfies models.File
+
+    it('should exclude when path matches explicit exclude pattern', () => {
+      // Arrange
+      const itemPath = 'src/data/excluded-file.txt'
+      const configuration = createConfiguration({
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*.txt' }],
+      })
+      const item = createFileItem({ name: 'excluded-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should include when path matches include pattern with no restrictions', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*.txt', applyOptionsToMatchedFiles: { addToKbId: 'kbId' } }],
+      })
+      const item = createFileItem({ name: 'included-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+        shouldApplyOptions: { addToKbId: 'kbId' },
+      })
+    })
+
+    it('should exclude when path does not match any patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/unknown-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*.txt' }],
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*.txt' }],
+      })
+      const item = createFileItem({ name: 'unknown-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+
+    it('should exclude when file size exceeds maxSizeInBytes', () => {
+      // Arrange
+      const itemPath = 'src/data/large-file.txt'
+      const maxSizeInBytes = 1000
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/large-*.txt', maxSizeInBytes }],
+      })
+      const item = createFileItem({
+        name: 'large-file.txt',
+        sizeInBytes: maxSizeInBytes + 1,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'unmet-include-requirements',
+      })
+    })
+
+    it('should ignore maxSizeInBytes when set to 0', () => {
+      // Arrange
+      const itemPath = 'src/data/valid-file.txt'
+      const maxSizeInBytes = 0
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/valid-*.txt',
+            maxSizeInBytes,
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'valid-file.txt',
+        sizeInBytes: 100,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should allow files larger than MAX_BATCH_SIZE_BYTES', () => {
+      // Arrange
+      const itemPath = 'src/data/large-file.txt'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/large-*.txt' }] })
+      const item = createFileItem({
+        name: 'large-file.txt',
+        sizeInBytes: MAX_BATCH_SIZE_BYTES + 1,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should exclude when modified before modifiedAfter date', () => {
+      // Arrange
+      const itemPath = 'src/data/old-file.txt'
+      const modifiedAfter = '1965-02-01T00:00:00Z'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/old-*.txt', modifiedAfter }] })
+      const item = createFileItem({
+        name: 'old-file.txt',
+        lastModifiedDate: '1965-01-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'unmet-include-requirements',
+      })
+    })
+
+    it('should include when modified after modifiedAfter date', () => {
+      // Arrange
+      const itemPath = 'src/data/new-file.txt'
+      const modifiedAfter = '1965-01-01T00:00:00Z'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/new-*.txt', modifiedAfter }] })
+      const item = createFileItem({
+        name: 'new-file.txt',
+        lastModifiedDate: '1965-02-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should include when file meets all requirements', () => {
+      // Arrange
+      const itemPath = 'src/data/valid-file.txt'
+      const maxSizeInBytes = 2000
+      const modifiedAfter = '1965-01-01T00:00:00Z'
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/valid-*.txt',
+            maxSizeInBytes,
+            modifiedAfter,
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'valid-file.txt',
+        sizeInBytes: maxSizeInBytes - 100,
+        lastModifiedDate: '1965-02-01T00:00:00Z',
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle missing optional properties', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [
+          {
+            pathGlobPattern: '**/included-*.txt',
+            maxSizeInBytes: 1000,
+            modifiedAfter: '1965-01-01T00:00:00Z',
+          },
+        ],
+      })
+      const item = createFileItem({
+        name: 'included-file.txt',
+        sizeInBytes: undefined,
+        lastModifiedDate: undefined,
+      })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle multiple include patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/not-matching-*.txt' }, { pathGlobPattern: '**/included-*.txt' }],
+      })
+      const item = createFileItem({ name: 'included-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should handle multiple include patterns with different restrictions', () => {
+      // Arrange
+      const itemPath = 'src/data/included-file.txt'
+      const configuration = createConfiguration({
+        includeFiles: [
+          { pathGlobPattern: '**/included-*.txt', maxSizeInBytes: 1 },
+          { pathGlobPattern: '**/included-*.txt', maxSizeInBytes: 100 },
+        ],
+      })
+      const item = createFileItem({ name: 'included-file.txt', sizeInBytes: 100 })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should prioritize excludeFiles over includeFiles', () => {
+      // Arrange
+      const itemPath = 'src/data/both-match.txt'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/both-*.txt' }],
+        excludeFiles: [{ pathGlobPattern: '**/both-*.txt' }],
+      })
+      const item = createFileItem({ name: 'both-match.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should exclude by default when there are no defined include patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/any-file.txt'
+      const configuration = createConfiguration({})
+      const item = createFileItem({ name: 'any-file.txt' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+  })
+
+  describe.concurrent('with folder items', () => {
+    const createFolderItem = (overrides: Readonly<Partial<models.Folder>> = {}) =>
+      ({
+        id: 'folder-1',
+        type: 'folder',
+        name: 'test-folder',
+        ...overrides,
+      }) as const satisfies models.Folder
+
+    it('should exclude when path matches explicit exclude pattern', () => {
+      // Arrange
+      const itemPath = 'src/data/__ignored'
+      const configuration = createConfiguration({ excludeFiles: [{ pathGlobPattern: '**/__ignored' }] })
+      const item = createFolderItem({ name: '__ignored' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      })
+    })
+
+    it('should include when path matches include pattern', () => {
+      // Arrange
+      const itemPath = 'src/data'
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern: '**/data' }] })
+      const item = createFolderItem({ name: 'data' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it.each([
+      { path: '/foo/bar', glob: '/foo/bar/baz/**' },
+      { path: '/abc', glob: '/abc/[def]/[ghi]/**' },
+      { path: '/abc/[def]', glob: '/abc/[def]/[ghi]/**' },
+    ])('should include when path matches part of include pattern', ({ path: itemPath, glob: pathGlobPattern }) => {
+      // Arrange
+      const configuration = createConfiguration({ includeFiles: [{ pathGlobPattern }] })
+      const item = createFolderItem({ name: 'bar' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: false,
+      })
+    })
+
+    it('should exclude when path does not match any patterns', () => {
+      // Arrange
+      const itemPath = 'src/data/unknown-folder'
+      const configuration = createConfiguration({
+        includeFiles: [{ pathGlobPattern: '**/included-*' }],
+        excludeFiles: [{ pathGlobPattern: '**/excluded-*' }],
+      })
+      const item = createFolderItem({ name: 'unknown-folder' })
+
+      // Act
+      const result = matchItem({ configuration, item, itemPath })
+
+      // Assert
+      expect(result).toMatchObject({
+        shouldBeIgnored: true,
+        reason: 'does-not-match-any-pattern',
+      })
+    })
+  })
+})

plugins/file-synchronizer/src/sync-queue/glob-matcher.ts

@@ -0,0 +1,129 @@
+import * as picomatch from 'picomatch'
+import type * as models from '../../definitions/models'
+import * as bp from '.botpress'
+
+export type GlobMatcherProps = {
+  configuration: Pick<bp.configuration.Configuration, 'includeFiles' | 'excludeFiles'>
+  item: models.FolderItem
+  itemPath: string
+}
+
+export type GlobMatchResult =
+  | {
+      shouldBeIgnored: false
+      shouldApplyOptions: Exclude<
+        GlobMatcherProps['configuration']['includeFiles'][number]['applyOptionsToMatchedFiles'],
+        undefined
+      >
+    }
+  | {
+      shouldBeIgnored: true
+      reason: 'matches-exclude-pattern' | 'unmet-include-requirements' | 'does-not-match-any-pattern'
+    }
+
+export const matchItem = ({ configuration, item, itemPath }: GlobMatcherProps): GlobMatchResult => {
+  for (const { pathGlobPattern } of configuration.excludeFiles) {
+    if (!pathGlobPattern) {
+      continue
+    }
+
+    if (picomatch.isMatch(itemPath, pathGlobPattern)) {
+      return {
+        shouldBeIgnored: true,
+        reason: 'matches-exclude-pattern',
+      }
+    }
+  }
+
+  let matchesButHasUnmetRequirements = false
+
+  for (const { pathGlobPattern, applyOptionsToMatchedFiles, ...requirements } of configuration.includeFiles) {
+    const isAncestorFolder = item.type === 'folder' && _isAncestorOfGlob(itemPath, pathGlobPattern)
+
+    if (!pathGlobPattern || (!_isMatch(itemPath, pathGlobPattern) && !isAncestorFolder)) {
+      continue
+    }
+
+    if (_isFileWithUnmetRequirements(item, requirements)) {
+      matchesButHasUnmetRequirements = true
+      continue
+    }
+
+    return {
+      shouldBeIgnored: false,
+      shouldApplyOptions: applyOptionsToMatchedFiles ?? {},
+    }
+  }
+
+  return {
+    shouldBeIgnored: true,
+    reason: matchesButHasUnmetRequirements ? 'unmet-include-requirements' : 'does-not-match-any-pattern',
+  }
+}
+
+const _isMatch = (itemPath: string, globPattern: string) =>
+  picomatch.isMatch(itemPath, globPattern, {
+    // allow dotfiles to match:
+    dot: true,
+    // escape brackets in the glob pattern so that only literal brackets are matched:
+    literalBrackets: true,
+    nobracket: true,
+  })
+
+type FileRequirements = Omit<
+  GlobMatcherProps['configuration']['includeFiles'][number],
+  'pathGlobPattern' | 'applyOptionsToMatchedFiles'
+>
+
+const _isFileWithUnmetRequirements = (
+  item: models.FolderItem,
+  { maxSizeInBytes, modifiedAfter }: FileRequirements
+): boolean => {
+  if (item.type !== 'file') {
+    return false
+  }
+
+  const exceedsUserDefinedMaxSize =
+    maxSizeInBytes !== undefined &&
+    maxSizeInBytes > 0 &&
+    item.sizeInBytes !== undefined &&
+    item.sizeInBytes > maxSizeInBytes
+
+  const isItemOlderThanGivenDate =
+    modifiedAfter !== undefined &&
+    modifiedAfter.length > 0 &&
+    item.lastModifiedDate !== undefined &&
+    new Date(item.lastModifiedDate) < new Date(modifiedAfter)
+
+  return exceedsUserDefinedMaxSize || isItemOlderThanGivenDate
+}
+
+const _isAncestorOfGlob = (candidatePath: string, globPattern: string): boolean =>
+  _isAncestorPath(candidatePath, _extractStaticPrefix(globPattern))
+
+const _extractStaticPrefix = (globPattern: string): string => {
+  const wildcardIndex = globPattern.search(/[*?{}]/)
+  if (wildcardIndex === -1) {
+    return globPattern
+  }
+
+  const prefix = globPattern.substring(0, wildcardIndex)
+  const lastSlashIndex = prefix.lastIndexOf('/')
+
+  return lastSlashIndex === -1 ? '' : prefix.substring(0, lastSlashIndex)
+}
+
+const _isAncestorPath = (ancestor: string, descendant: string): boolean => {
+  if (ancestor === descendant) {
+    return true
+  }
+
+  const ancestorParts = ancestor.split('/').filter((part) => part !== '')
+  const descendantParts = descendant.split('/').filter((part) => part !== '')
+
+  if (ancestorParts.length >= descendantParts.length) {
+    return false
+  }
+
+  return ancestorParts.every((part, index) => part === descendantParts[index])
+}

plugins/file-synchronizer/src/sync-queue/index.ts

@@ -0,0 +1,5 @@
+export * as queueProcessor from './queue-processor'
+export * as fileProcessor from './file-processor'
+export * as jobFileManager from './job-file-manager'
+export * as globMatcher from './glob-matcher'
+export * as directoryTraversalWithBatching from './directory-traversal-with-batching'

plugins/file-synchronizer/src/sync-queue/job-file-manager.ts

@@ -0,0 +1,64 @@
+import * as sdk from '@botpress/sdk'
+import * as models from '../../definitions/models'
+import type * as types from '../types'
+import * as utils from '../utils'
+import * as bp from '.botpress'
+
+const QUEUE_ITEM = models.FILE_WITH_PATH.extend({
+  status: sdk.z.enum(['pending', 'newly-synced', 'already-synced', 'errored']),
+  errorMessage: sdk.z.string().optional(),
+  shouldIndex: sdk.z.boolean(),
+  addToKbId: sdk.z.string().optional(),
+})
+
+export const getSyncQueue = async (
+  props: bp.WorkflowHandlerProps['processQueue'] | bp.WorkflowHandlerProps['buildQueue'],
+  jobFileId?: string
+): Promise<{ syncQueue: types.SyncQueue; key: string }> => {
+  const { jobFileContent, key } = await _retrieveJobFile(props, jobFileId).catch(async (thrown: unknown) => {
+    const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    await props.workflow.setFailed({ failureReason: `Failed to retrieve job file: ${err.message}` })
+    throw new Error(`Failed to retrieve job file: ${thrown}`)
+  })
+
+  const syncQueue: types.SyncQueue = []
+  const syncQueueGenerator = utils.jsonl.parseJsonLines(jobFileContent, QUEUE_ITEM)
+
+  for (const item of syncQueueGenerator) {
+    if ('error' in item) {
+      props.logger.error('Error while parsing line in job file. This line will be ignored.', item)
+      continue
+    }
+
+    syncQueue.push(item.value)
+  }
+
+  return { syncQueue, key }
+}
+
+const _retrieveJobFile = async (
+  props: bp.WorkflowHandlerProps['processQueue'] | bp.WorkflowHandlerProps['buildQueue'],
+  jobFileId?: string
+): Promise<{ jobFileContent: string; key: string }> => {
+  const { file: jobFile } = await props.client.getFile({
+    id: props.workflow.input.jobFileId ?? jobFileId,
+  })
+  const jobFileContent = await fetch(jobFile.url).then((res) => res.text())
+
+  return { jobFileContent, key: jobFile.key }
+}
+
+export const updateSyncQueue = async (
+  props: types.CommonProps,
+  key: string,
+  syncQueue: types.SyncQueue,
+  tags?: Record<string, string>
+): Promise<string> => {
+  const { file: jobFile } = await props.client.uploadFile({
+    key,
+    content: syncQueue.map((item) => JSON.stringify(item)).join('\n') + '\n',
+    tags,
+  })
+
+  return jobFile.id
+}

plugins/file-synchronizer/src/sync-queue/queue-batching.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from 'vitest'
+import { findBatchEndCursor } from './queue-batching'
+import { MAX_BATCH_SIZE_BYTES as maxBatchSize } from '../consts'
+
+describe.concurrent('findBatchEndCursor', () => {
+  it.each([
+    { startCursor: 0, files: [{ sizeInBytes: 100 }, { sizeInBytes: 200 }], expectedEndCursors: [2] },
+    { startCursor: 0, files: [{ sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
+    { startCursor: 0, files: [{ sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }], expectedEndCursors: [1, 2] },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 3, 4],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 3, 4],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize - 200 }, { sizeInBytes: 200 }],
+      expectedEndCursors: [2, 4],
+    },
+    {
+      startCursor: 2,
+      files: [
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: 200 },
+        { sizeInBytes: maxBatchSize - 200 },
+        { sizeInBytes: 200 },
+      ],
+      expectedEndCursors: [4, 6],
+    },
+    {
+      startCursor: 0,
+      files: [{ sizeInBytes: 200 }, { sizeInBytes: 200 }, { sizeInBytes: maxBatchSize + 100 }],
+      expectedEndCursors: [2, 3],
+    },
+  ])(
+    'should correctly calculate the end cursors for a batch of files',
+    ({ startCursor, files, expectedEndCursors }) => {
+      // Act
+      const endCursors: number[] = []
+      let currentCursor = startCursor
+
+      while (currentCursor < files.length) {
+        currentCursor = findBatchEndCursor({ startCursor: currentCursor, files }).endCursor
+        endCursors.push(currentCursor)
+      }
+
+      // Assert
+      expect(endCursors).toStrictEqual(expectedEndCursors)
+    }
+  )
+})

plugins/file-synchronizer/src/sync-queue/queue-batching.ts

@@ -0,0 +1,41 @@
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+import type * as types from '../types'
+
+type SimpleFile = Pick<types.SyncQueueItem, 'sizeInBytes'>
+
+export const findBatchEndCursor = ({
+  startCursor: unsafeStartCursor,
+  files,
+}: {
+  startCursor: number
+  files: SimpleFile[]
+}): { endCursor: number } => {
+  const startCursor = Math.min(Math.max(unsafeStartCursor, 0), files.length - 1)
+  let currentBatchSize = 0
+  let endCursor = files.length
+
+  for (let newCursor = startCursor; newCursor < files.length; newCursor++) {
+    const fileToSync = files[newCursor]
+    const fileSizeInBytes = _getFileSize(fileToSync!)
+    currentBatchSize += fileSizeInBytes
+
+    // First file is always included even if it exceeds the batch size:
+    if (newCursor > startCursor && currentBatchSize > MAX_BATCH_SIZE_BYTES) {
+      endCursor = newCursor
+      break
+    }
+  }
+
+  return { endCursor }
+}
+
+const _getFileSize = (file: SimpleFile) => {
+  if (file.sizeInBytes === undefined) {
+    // If a file has no size, we assume it takes up half the batch size limit.
+    // This should be relatively safe, since we'd process at most two files of
+    // unknown size in a batch, and we don't want to just skip them.
+    return MAX_BATCH_SIZE_BYTES / 2
+  }
+
+  return file.sizeInBytes
+}

plugins/file-synchronizer/src/sync-queue/queue-processor.test.ts

@@ -0,0 +1,187 @@
+import * as sdk from '@botpress/sdk'
+import { describe, it, expect, vi, type Mocked, type Mock } from 'vitest'
+import { processQueue, type ProcessQueueProps } from './queue-processor'
+import type * as types from '../types'
+import { MAX_BATCH_SIZE_BYTES } from '../consts'
+
+const FILE_1 = {
+  id: 'file1',
+  type: 'file',
+  name: 'file1.txt',
+  absolutePath: '/path/to/file1.txt',
+  sizeInBytes: 100,
+  lastModifiedDate: '2025-01-01T00:00:00Z',
+  contentHash: 'hash1',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const FILE_2 = {
+  id: 'file2',
+  type: 'file',
+  name: 'file2.txt',
+  absolutePath: '/path/to/file2.txt',
+  sizeInBytes: 200,
+  lastModifiedDate: '2025-01-02T00:00:00Z',
+  contentHash: 'hash2',
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+const LARGE_FILE = {
+  id: 'largefile',
+  type: 'file',
+  name: 'largefile.txt',
+  absolutePath: '/path/to/largefile.txt',
+  sizeInBytes: MAX_BATCH_SIZE_BYTES - 200,
+  status: 'pending',
+  parentId: 'abcde',
+  shouldIndex: false,
+} as const satisfies types.SyncQueueItem
+
+describe.concurrent('processQueue', () => {
+  const getMocks = () => ({
+    fileRepository: {
+      listFiles: vi.fn(),
+      deleteFile: vi.fn(),
+      updateFileMetadata: vi.fn(),
+    } as const satisfies Mocked<ProcessQueueProps['fileRepository']>,
+    integration: {
+      name: 'test-integration',
+      alias: 'test-integration-alias',
+      transferFileToBotpress: vi.fn(),
+    } as const satisfies Mocked<ProcessQueueProps['integration']>,
+    updateSyncQueue: vi.fn() as Mock<ProcessQueueProps['updateSyncQueue']>,
+    logger: new sdk.BotLogger(),
+  })
+
+  it('should process all files in queue when size is within batch limit', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+  })
+
+  it('should process files in batches when size exceeds batch limit', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, LARGE_FILE, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'batch' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+    expect(updatedQueue?.[2]?.status).toBe('pending')
+  })
+
+  it('should handle errors by continuing to the next file', async () => {
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+
+    // First transfer fails, second succeeds:
+    mocks.integration.transferFileToBotpress.mockRejectedValueOnce(new Error('Transfer failed'))
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+
+    // Act
+    const result = processQueue({
+      syncQueue: [FILE_1, FILE_2],
+      ...mocks,
+    })
+
+    // Assert
+    await expect(result).resolves.toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.fileRepository.updateFileMetadata).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue
+    expect(updatedQueue?.[0]?.status).toBe('errored')
+    expect(updatedQueue?.[0]?.errorMessage).toContain('Transfer failed')
+    expect(updatedQueue?.[1]?.status).toBe('newly-synced')
+  })
+
+  it('should handle complex batching', async () => {
+    // >>>>>>>>>>> FIRST BATCH <<<<<<<<<<<<<
+
+    // Arrange
+    const mocks = getMocks()
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_2.id })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: FILE_1.id })
+
+    // Act
+    const firstRun = await processQueue({
+      syncQueue: [FILE_2, FILE_1, LARGE_FILE],
+      ...mocks,
+    })
+
+    // Assert
+    expect(firstRun).toEqual({ finished: 'batch' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(2)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(1)
+
+    const updatedQueue = mocks.updateSyncQueue.mock.calls[0]?.[0].syncQueue!
+
+    expect(updatedQueue[0]?.status).toBe('newly-synced')
+    expect(updatedQueue[1]?.status).toBe('newly-synced')
+    expect(updatedQueue[2]?.status).toBe('pending')
+
+    // >>>>>>>>>>> SECOND BATCH <<<<<<<<<<<<<
+
+    // Arrange
+    mocks.fileRepository.listFiles.mockResolvedValueOnce({ files: [] })
+    mocks.integration.transferFileToBotpress.mockResolvedValueOnce({ botpressFileId: LARGE_FILE.id })
+
+    // Act
+    const secondRun = await processQueue({
+      syncQueue: updatedQueue!,
+      ...mocks,
+    })
+
+    // Assert
+    expect(secondRun).toEqual({ finished: 'all' })
+    expect(mocks.integration.transferFileToBotpress).toHaveBeenCalledTimes(3)
+    expect(mocks.updateSyncQueue).toHaveBeenCalledTimes(2)
+
+    const finalQueue = mocks.updateSyncQueue.mock.calls[1]?.[0].syncQueue!
+
+    expect(finalQueue[0]?.status).toBe('newly-synced')
+    expect(finalQueue[1]?.status).toBe('newly-synced')
+    expect(finalQueue[2]?.status).toBe('newly-synced')
+  })
+})

plugins/file-synchronizer/src/sync-queue/queue-processor.ts

@@ -0,0 +1,57 @@
+import * as sdk from '@botpress/sdk'
+import type * as models from '../../definitions/models'
+import type * as types from '../types'
+import { processQueueFile } from './file-processor'
+import { findBatchEndCursor } from './queue-batching'
+
+export type ProcessQueueProps = {
+  syncQueue: Readonly<types.SyncQueue>
+  logger: sdk.BotLogger
+  fileRepository: {
+    listFiles: (params: { tags: Record<string, string> }) => Promise<{ files: types.FilesApiFile[] }>
+    deleteFile: (params: { id: string }) => Promise<unknown>
+    updateFileMetadata: (params: { id: string; tags: Record<string, string | null> }) => Promise<unknown>
+  }
+  integration: {
+    name: string
+    alias: string
+    transferFileToBotpress: (params: {
+      file: models.FileWithPath
+      fileKey: string
+      shouldIndex: boolean
+    }) => Promise<{ botpressFileId: string }>
+  }
+  updateSyncQueue: (props: { syncQueue: types.SyncQueue }) => Promise<unknown>
+}
+
+export const processQueue = async (props: ProcessQueueProps) => {
+  // Short-circuit if the sync queue is empty:
+  if (!props.syncQueue?.length) {
+    props.logger.info('Sync queue is empty. Nothing to process.')
+    return { finished: 'all' } as const
+  }
+
+  const syncQueue = structuredClone(props.syncQueue) as types.SyncQueue
+  const startCursor = syncQueue.findIndex((file) => file.status === 'pending') ?? syncQueue.length - 1
+
+  if (startCursor < 0) {
+    props.logger.info('No files left to process in the sync queue.')
+    return { finished: 'all' } as const
+  }
+
+  const { endCursor } = findBatchEndCursor({ startCursor, files: syncQueue })
+  const filesInBatch = syncQueue.slice(startCursor, endCursor)
+
+  for (const fileToSync of filesInBatch) {
+    const processedFile = await processQueueFile({ ...props, fileToSync })
+    Object.assign(fileToSync, processedFile)
+  }
+
+  await props.updateSyncQueue({ syncQueue })
+
+  if (endCursor < syncQueue.length) {
+    return { finished: 'batch' } as const
+  }
+
+  return { finished: 'all' } as const
+}

plugins/file-synchronizer/src/types.ts

@@ -0,0 +1,21 @@
+import * as sdk from '@botpress/sdk'
+import { CommonHandlerProps } from '@botpress/sdk/dist/plugin'
+import type * as models from '../definitions/models'
+import type * as bp from '.botpress'
+
+type TPlugin = sdk.DefaultPlugin<bp.TPlugin>
+
+export type SyncQueueItem = models.FileWithPath & {
+  status: 'pending' | 'newly-synced' | 'already-synced' | 'errored'
+  errorMessage?: string
+  shouldIndex: boolean
+  addToKbId?: string
+}
+export type SyncQueue = SyncQueueItem[]
+
+export type CommonProps = CommonHandlerProps<TPlugin>
+
+export type FilesApiFile = {
+  id: string
+  tags: Record<string, string>
+}

plugins/file-synchronizer/src/utils/index.ts

@@ -0,0 +1 @@
+export * as jsonl from './json-lines'

plugins/file-synchronizer/src/utils/json-lines/index.ts

@@ -0,0 +1 @@
+export * from './parser'

plugins/file-synchronizer/src/utils/json-lines/parser.test.ts

@@ -0,0 +1,160 @@
+import { describe, it, expect } from 'vitest'
+import * as sdk from '@botpress/sdk'
+import { parseJsonLines } from './parser'
+
+describe.concurrent('parseJsonLines', () => {
+  const STRING_SCHEMA = sdk.z.string()
+  const NUMBER_SCHEMA = sdk.z.number()
+  const USER_SCHEMA = sdk.z.object({
+    id: sdk.z.number(),
+    name: sdk.z.string(),
+    email: sdk.z.string().email(),
+  })
+
+  it('should parse valid JSON lines with a string schema', () => {
+    // Arrange
+    const input = '"hello"\n"world"\n"test"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'hello' }, { value: 'world' }, { value: 'test' }])
+  })
+
+  it('should parse valid JSON lines with an object schema', () => {
+    // Arrange
+    const input =
+      '{"id": 1, "name": "John", "email": "john@example.com"}\n' +
+      '{"id": 2, "name": "Jane", "email": "jane@example.com"}\n' +
+      '{"id": 3, "name": "Bob", "email": "bob@example.com"}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, USER_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([
+      {
+        value: { id: 1, name: 'John', email: 'john@example.com' },
+      },
+      {
+        value: { id: 2, name: 'Jane', email: 'jane@example.com' },
+      },
+      { value: { id: 3, name: 'Bob', email: 'bob@example.com' } },
+    ])
+  })
+
+  it('should handle empty input strings', () => {
+    // Arrange
+    const input = ''
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([])
+  })
+
+  it('should handle input with only whitespace', () => {
+    // Arrange
+    const input = '   \n  \n\t  '
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([])
+  })
+
+  it('should skip empty lines', () => {
+    // Arrange
+    const input = '"first"\n\n"second"\n\n\n"third"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'first' }, { value: 'second' }, { value: 'third' }])
+  })
+
+  it('should handle trailing newlines', () => {
+    // Arrange
+    const input = '"line1"\n"line2"\n'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }])
+  })
+
+  it('should handle input without newlines', () => {
+    // Arrange
+    const input = '"single line"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'single line' }])
+  })
+
+  it('should return a Zod error when a line fails Zod validation', () => {
+    // Arrange
+    const input = '{"id": 1, "name": "John", "email": "invalid-email"}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, USER_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([
+      {
+        rawLine: '{"id": 1, "name": "John", "email": "invalid-email"}',
+        error: expect.objectContaining({ __type__: 'ZuiError' }),
+      },
+    ])
+  })
+
+  it('should parse to the end, even with errors', () => {
+    // Arrange
+    const input = '1\n2\n"not a number"\n4'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, NUMBER_SCHEMA))
+
+    // Assert
+    expect(result).toStrictEqual([
+      { rawLine: '1', value: 1 },
+      { rawLine: '2', value: 2 },
+      { rawLine: '"not a number"', error: expect.objectContaining({ __type__: 'ZuiError' }) },
+      { rawLine: '4', value: 4 },
+    ])
+  })
+
+  it('should handle different line endings (CRLF)', () => {
+    // Arrange
+    const input = '"line1"\r\n"line2"\r\n"line3"'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, STRING_SCHEMA))
+
+    // Assert
+    expect(result).toMatchObject([{ value: 'line1' }, { value: 'line2' }, { value: 'line3' }])
+  })
+
+  it('should parse properly with schema containing optional fields', () => {
+    // Arrange
+    const optionalSchema = sdk.z.object({
+      id: sdk.z.number(),
+      name: sdk.z.string(),
+      age: sdk.z.number().optional(),
+    })
+    const input = '{"id": 1, "name": "John"}\n{"id": 2, "name": "Jane", "age": 25}'
+
+    // Act
+    const result = Array.from(parseJsonLines(input, optionalSchema))
+
+    // Assert
+    expect(result).toMatchObject([{ value: { id: 1, name: 'John' } }, { value: { id: 2, name: 'Jane', age: 25 } }])
+  })
+})

plugins/file-synchronizer/src/utils/json-lines/parser.ts

@@ -0,0 +1,26 @@
+import * as sdk from '@botpress/sdk'
+
+type _JsonParseResult<T> = { rawLine: string } & ({ value: T } | { error: Error })
+
+export function* parseJsonLines<TLineSchema extends sdk.z.ZodTypeAny>(
+  rawJsonLines: string,
+  zodSchema: TLineSchema
+): Generator<_JsonParseResult<sdk.z.infer<TLineSchema>>, void, undefined> {
+  let startCursor = 0
+
+  for (let endCursor = 0; endCursor <= rawJsonLines.length; endCursor++) {
+    if (rawJsonLines[endCursor] === '\n' || endCursor === rawJsonLines.length) {
+      const line = rawJsonLines.slice(startCursor, endCursor).trim()
+      startCursor = endCursor + 1
+
+      if (line) {
+        try {
+          yield { rawLine: line, value: zodSchema.parse(JSON.parse(line)) }
+        } catch (thrown: unknown) {
+          const err: Error = thrown instanceof Error ? thrown : new Error(String(thrown))
+          yield { rawLine: line, error: err }
+        }
+      }
+    }
+  }
+}

plugins/file-synchronizer/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": { "*": ["./*"] },
+    "outDir": "dist"
+  },
+  "include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
+}

plugins/file-synchronizer/vitest.config.ts

@@ -0,0 +1,2 @@
+import config from '../../vitest.config'
+export default config

plugins/hitl/eslint.config.mjs

@@ -0,0 +1,13 @@
+import rootConfig from '../../eslint.config.mjs'
+
+export default [
+  ...rootConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]

plugins/hitl/hub.md

@@ -0,0 +1,16 @@
+# Human In The Loop Plugin
+
+This plugin allows your bot to ask for human help when it is not able to answer a user query.
+
+To use this plugin, you must first configure a compatible HITL integration:
+
+- Zendesk
+- Freshchat
+- Hitl (official Botpress HITL integration)
+
+## Using the plugin in the Studio
+
+Place the card "Start HITL" anywhere in your workflow and fill in the required fields.
+
+In the conversation ID field, you can use the `{{event.conversationId}}` variable to get the current conversation ID.
+Likewise, you can use the `{{event.userId}}` variable to get the ID of the current user.

plugins/hitl/icon.svg

@@ -0,0 +1,251 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="300px" height="299px"
+  style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"
+  xmlns:xlink="http://www.w3.org/1999/xlink">
+  <circle cx="147" cy="130" r="120" fill="#f9ded3" />
+  <g>
+    <path style="opacity:1" fill="#1b1e2f"
+      d="M 73.5,67.5 C 75.2959,68.7927 76.9625,70.2927 78.5,72C 76.981,73.3513 75.6477,74.8513 74.5,76.5C 69.6341,73.9265 69.3008,70.9265 73.5,67.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#f9f8f7"
+      d="M 185.5,78.5 C 185.5,80.8333 185.5,83.1667 185.5,85.5C 179.752,95.5759 171.085,102.076 159.5,105C 157.93,106.397 156.596,107.897 155.5,109.5C 149.704,104.583 143.537,104.25 137,108.5C 132.819,115.197 134.319,120.364 141.5,124C 146.949,125.393 151.616,124.226 155.5,120.5C 167.929,121.369 177.763,116.702 185,106.5C 185.464,107.094 185.631,107.761 185.5,108.5C 183.957,117.88 178.624,124.047 169.5,127C 166.238,128.469 162.905,129.636 159.5,130.5C 140.327,135.165 123.494,130.832 109,117.5C 106.004,112.23 104.838,106.563 105.5,100.5C 108.073,94.5924 110.239,88.2591 112,81.5C 112.75,80.8742 113.584,80.3742 114.5,80C 135.266,79.7939 154.599,74.6273 172.5,64.5C 176.12,69.7885 180.454,74.4552 185.5,78.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#23232f"
+      d="M 221.5,82.5 C 222.833,82.5 224.167,82.5 225.5,82.5C 225.778,84.3464 225.111,85.6798 223.5,86.5C 221.889,85.6798 221.222,84.3464 221.5,82.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#262124"
+      d="M 62.5,91.5 C 64.288,91.2148 65.9547,91.5481 67.5,92.5C 66.6667,92.8333 65.8333,93.1667 65,93.5C 63.6244,93.3158 62.7911,92.6491 62.5,91.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#e9c9b7"
+      d="M 185.5,85.5 C 185.408,92.4499 182.742,98.2832 177.5,103C 171.407,108.219 164.407,110.886 156.5,111C 155.944,110.617 155.611,110.117 155.5,109.5C 156.596,107.897 157.93,106.397 159.5,105C 171.085,102.076 179.752,95.5759 185.5,85.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#21212f"
+      d="M 212.5,102.5 C 216.555,102.176 220.555,102.509 224.5,103.5C 222.5,103.833 220.5,104.167 218.5,104.5C 216.048,104.461 214.048,103.794 212.5,102.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#1d1b25"
+      d="M 73.5,107.5 C 76.5,107.167 77.8333,108.5 77.5,111.5C 73.56,112.234 72.2266,110.901 73.5,107.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#f08d77"
+      d="M 108.5,109.5 C 107.167,106.833 107.167,104.167 108.5,101.5C 110.681,101.716 112.347,100.883 113.5,99C 119.011,97.7514 122.011,99.918 122.5,105.5C 120.794,111.02 117.127,112.853 111.5,111C 110.756,109.961 109.756,109.461 108.5,109.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#e09785"
+      d="M 108.5,109.5 C 109.756,109.461 110.756,109.961 111.5,111C 117.127,112.853 120.794,111.02 122.5,105.5C 122.011,99.918 119.011,97.7514 113.5,99C 112.347,100.883 110.681,101.716 108.5,101.5C 110.359,97.0744 113.692,95.5744 118.5,97C 126.644,103.002 126.311,108.668 117.5,114C 113.305,114.987 110.305,113.487 108.5,109.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#1e1822"
+      d="M 237.5,110.5 C 238.777,111.694 239.444,113.36 239.5,115.5C 239.167,117.167 238.833,118.833 238.5,120.5C 237.513,117.232 237.18,113.898 237.5,110.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#2c2426"
+      d="M 221.5,116.5 C 223.346,116.222 224.68,116.889 225.5,118.5C 224.68,120.111 223.346,120.778 221.5,120.5C 221.5,119.167 221.5,117.833 221.5,116.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#12111b"
+      d="M 75.5,127.5 C 79.226,127.177 82.8927,127.511 86.5,128.5C 84.6667,128.833 82.8333,129.167 81,129.5C 78.7047,129.453 76.8714,128.787 75.5,127.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#181b2a"
+      d="M 57.5,122.5 C 63.3753,125.167 63.7086,128.167 58.5,131.5C 53.6341,128.927 53.3008,125.927 57.5,122.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#403b43"
+      d="M 54.5,103.5 C 55.7647,104.514 56.4313,106.014 56.5,108C 56.1667,109.5 55.8333,111 55.5,112.5C 54.5159,109.572 54.1826,106.572 54.5,103.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#0f1528"
+      d="M 235.5,130.5 C 241.532,132.544 241.865,135.21 236.5,138.5C 235.059,137.135 233.726,135.635 232.5,134C 233.71,132.961 234.71,131.794 235.5,130.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#20172b"
+      d="M 209.5,132.5 C 213.226,132.177 216.893,132.511 220.5,133.5C 218.667,133.833 216.833,134.167 215,134.5C 212.705,134.453 210.871,133.787 209.5,132.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#4f474a"
+      d="M 68.5,141.5 C 69.5,141.5 70.5,141.5 71.5,141.5C 71.5,144.5 71.5,147.5 71.5,150.5C 70.5,150.5 69.5,150.5 68.5,150.5C 68.5,147.5 68.5,144.5 68.5,141.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7db6c0"
+      d="M 166.5,134.5 C 168.943,135.499 170.609,137.332 171.5,140C 170.586,141.406 169.92,142.906 169.5,144.5C 167.108,147.598 164.942,150.931 163,154.5C 160.021,152.356 157.188,150.022 154.5,147.5C 159.932,144.405 163.932,140.072 166.5,134.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#1d1a20"
+      d="M 219.5,142.5 C 220.748,143.334 221.415,144.667 221.5,146.5C 221.167,147.833 220.833,149.167 220.5,150.5C 219.52,147.914 219.187,145.247 219.5,142.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#c9d2d7"
+      d="M 175.5,143.5 C 182.468,146.314 189.468,149.147 196.5,152C 199.898,153.357 202.898,155.19 205.5,157.5C 194.806,154.758 184.139,151.758 173.5,148.5C 174.441,146.95 175.107,145.284 175.5,143.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#f9f8f6"
+      d="M 173.5,148.5 C 184.139,151.758 194.806,154.758 205.5,157.5C 207.47,159.647 209.303,161.98 211,164.5C 216.667,181.5 222.333,198.5 228,215.5C 228.943,221.222 228.11,226.556 225.5,231.5C 222.173,233.396 218.84,234.396 215.5,234.5C 205.233,235.22 201.566,230.553 204.5,220.5C 209.989,220.496 211.822,217.829 210,212.5C 206.115,203.411 202.282,194.411 198.5,185.5C 196.099,183.938 194.099,184.271 192.5,186.5C 191.816,192.901 191.15,199.401 190.5,206C 181.011,213.001 170.345,215.501 158.5,213.5C 156.745,200.02 155.412,186.353 154.5,172.5C 154.02,168.694 155.02,165.361 157.5,162.5C 161.31,165.338 164.81,165.005 168,161.5C 168.847,159.146 169.68,156.812 170.5,154.5C 171.471,152.517 172.471,150.517 173.5,148.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#bcc6cb"
+      d="M 170.5,154.5 C 169.68,156.812 168.847,159.146 168,161.5C 164.81,165.005 161.31,165.338 157.5,162.5C 155.02,165.361 154.02,168.694 154.5,172.5C 153.08,166.901 153.247,161.235 155,155.5C 157.719,157.729 160.553,159.729 163.5,161.5C 164.167,161.333 164.833,161.167 165.5,161C 167.066,158.674 168.732,156.508 170.5,154.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cdd5db"
+      d="M 198.5,185.5 C 202.282,194.411 206.115,203.411 210,212.5C 211.822,217.829 209.989,220.496 204.5,220.5C 204.5,220.167 204.5,219.833 204.5,219.5C 203.205,215.577 201.372,211.91 199,208.5C 198.5,200.841 198.334,193.174 198.5,185.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#09162c"
+      d="M 189.5,37.5 C 196.262,47.5131 198.262,58.5131 195.5,70.5C 196.552,70.3505 197.552,70.5172 198.5,71C 204.167,76.3189 206.001,82.8189 204,90.5C 203.313,97.6842 199.48,101.684 192.5,102.5C 192.168,116.154 185.835,125.821 173.5,131.5C 174.803,133.937 176.47,136.104 178.5,138C 186.868,141.675 195.201,145.341 203.5,149C 207.241,150.738 210.408,153.238 213,156.5C 220.401,174.538 226.901,192.872 232.5,211.5C 233.638,214.719 234.471,218.053 235,221.5C 237.292,222.79 239.292,224.457 241,226.5C 244.015,234.118 241.848,239.452 234.5,242.5C 231.15,242.335 227.817,242.502 224.5,243C 212.545,251.953 200.878,261.287 189.5,271C 186.833,271.667 184.167,271.667 181.5,271C 180.619,270.292 179.953,269.458 179.5,268.5C 176.294,266.936 174.96,264.269 175.5,260.5C 174.5,260.5 173.5,260.5 172.5,260.5C 172.5,258.833 172.5,257.167 172.5,255.5C 173.5,255.5 174.5,255.5 175.5,255.5C 176.488,256.795 177.488,258.128 178.5,259.5C 179.794,261.907 181.461,264.074 183.5,266C 185.074,266.699 186.407,266.365 187.5,265C 185.267,262.004 182.934,259.171 180.5,256.5C 178.86,253.851 177.193,251.184 175.5,248.5C 175.034,246.799 175.034,245.299 175.5,244C 174.181,242.372 172.848,242.539 171.5,244.5C 171.167,243.667 170.833,242.833 170.5,242C 171.378,240.2 172.045,238.367 172.5,236.5C 175.681,234.408 178.681,232.075 181.5,229.5C 183.741,228.7 184.741,227.034 184.5,224.5C 186.92,224.708 189.253,224.374 191.5,223.5C 192.308,223.808 192.975,224.308 193.5,225C 190.833,227.526 187.833,229.359 184.5,230.5C 182.5,233.5 180.5,236.5 178.5,239.5C 178.082,240.222 177.416,240.722 176.5,241C 180.066,245.991 183.732,250.825 187.5,255.5C 188.748,257.649 190.248,259.649 192,261.5C 201.833,253.667 211.667,245.833 221.5,238C 223.734,236.362 225.067,234.195 225.5,231.5C 228.11,226.556 228.943,221.222 228,215.5C 222.333,198.5 216.667,181.5 211,164.5C 209.303,161.98 207.47,159.647 205.5,157.5C 202.898,155.19 199.898,153.357 196.5,152C 189.468,149.147 182.468,146.314 175.5,143.5C 173.379,143.325 171.379,143.659 169.5,144.5C 169.92,142.906 170.586,141.406 171.5,140C 170.609,137.332 168.943,135.499 166.5,134.5C 165.85,134.196 165.183,133.863 164.5,133.5C 162.038,136.429 159.038,137.096 155.5,135.5C 154.479,134.998 154.312,134.332 155,133.5C 156.409,132.219 157.909,131.219 159.5,130.5C 162.905,129.636 166.238,128.469 169.5,127C 178.624,124.047 183.957,117.88 185.5,108.5C 186.793,106.636 186.793,104.636 185.5,102.5C 177.79,111.192 168.124,116.525 156.5,118.5C 156.137,119.183 155.804,119.85 155.5,120.5C 151.616,124.226 146.949,125.393 141.5,124C 134.319,120.364 132.819,115.197 137,108.5C 143.537,104.25 149.704,104.583 155.5,109.5C 155.611,110.117 155.944,110.617 156.5,111C 164.407,110.886 171.407,108.219 177.5,103C 182.742,98.2832 185.408,92.4499 185.5,85.5C 185.5,83.1667 185.5,80.8333 185.5,78.5C 185.657,77.1266 185.49,75.7932 185,74.5C 179.365,69.8898 175.198,64.2231 172.5,57.5C 160.885,65.836 147.885,71.0026 133.5,73C 124.22,74.4935 114.887,75.3268 105.5,75.5C 104.67,84.9913 104.17,94.658 104,104.5C 101.479,101.648 98.3127,100.148 94.5,100C 93.8333,99.3333 93.1667,98.6667 92.5,98C 93.9504,97.8829 95.4504,98.0495 97,98.5C 98.7106,98.5607 100.044,97.8941 101,96.5C 100.17,88.8391 100.336,81.1724 101.5,73.5C 99.4471,73.4021 97.7804,74.0687 96.5,75.5C 92.0395,78.5808 90.3728,82.9141 91.5,88.5C 91.5386,89.7562 91.0386,90.7562 90,91.5C 87.2372,87.1058 86.9038,82.4392 89,77.5C 89.3333,76.5 89.6667,75.5 90,74.5C 92.3743,73.3685 94.2076,71.7019 95.5,69.5C 94.4886,60.5936 96.1552,52.2602 100.5,44.5C 107.719,32.0492 118.386,24.2159 132.5,21C 148.078,18.4761 161.745,22.3094 173.5,32.5C 179.411,32.6076 184.744,34.2742 189.5,37.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#d2b0aa"
+      d="M 142.5,110.5 C 145.32,110.248 147.986,110.748 150.5,112C 152.129,118.694 149.463,121.027 142.5,119C 139.899,116.17 139.899,113.337 142.5,110.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#eda386"
+      d="M 143.5,112.5 C 146.38,112.033 148.713,112.866 150.5,115C 150.095,115.945 149.428,116.612 148.5,117C 146.527,117.495 144.527,117.662 142.5,117.5C 142.366,115.708 142.699,114.042 143.5,112.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#e1c2b4"
+      d="M 185.5,108.5 C 185.631,107.761 185.464,107.094 185,106.5C 177.763,116.702 167.929,121.369 155.5,120.5C 155.804,119.85 156.137,119.183 156.5,118.5C 168.124,116.525 177.79,111.192 185.5,102.5C 186.793,104.636 186.793,106.636 185.5,108.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#111b2f"
+      d="M 159.5,85.5 C 163.949,85.2556 165.615,87.4223 164.5,92C 165.745,97.2539 163.745,99.4206 158.5,98.5C 158.231,94.0685 158.564,89.7352 159.5,85.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#121d32"
+      d="M 126.5,85.5 C 130.949,85.2556 132.615,87.4223 131.5,92C 132.745,97.2539 130.745,99.4206 125.5,98.5C 125.231,94.0685 125.564,89.7352 126.5,85.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#becbd1"
+      d="M 195.5,76.5 C 196.79,77.0577 197.623,78.0577 198,79.5C 199.223,85.3792 198.389,90.8792 195.5,96C 194.552,96.4828 193.552,96.6495 192.5,96.5C 192.5,89.8333 192.5,83.1667 192.5,76.5C 193.5,76.5 194.5,76.5 195.5,76.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#47596c"
+      d="M 195.5,76.5 C 194.5,76.5 193.5,76.5 192.5,76.5C 192.5,83.1667 192.5,89.8333 192.5,96.5C 191.509,89.6866 191.175,82.6866 191.5,75.5C 193.099,75.2322 194.432,75.5655 195.5,76.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#e8cdc0"
+      d="M 185.5,78.5 C 180.454,74.4552 176.12,69.7885 172.5,64.5C 154.599,74.6273 135.266,79.7939 114.5,80C 113.584,80.3742 112.75,80.8742 112,81.5C 110.239,88.2591 108.073,94.5924 105.5,100.5C 105.5,92.1667 105.5,83.8333 105.5,75.5C 114.887,75.3268 124.22,74.4935 133.5,73C 147.885,71.0026 160.885,65.836 172.5,57.5C 175.198,64.2231 179.365,69.8898 185,74.5C 185.49,75.7932 185.657,77.1266 185.5,78.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#beccd1"
+      d="M 189.5,51.5 C 190.917,57.4549 190.584,63.4549 188.5,69.5C 183.183,64.8744 179.35,59.2078 177,52.5C 175.961,51.2899 174.794,50.2899 173.5,49.5C 175.182,45.5507 177.848,44.7174 181.5,47C 184.629,50.7522 187.129,54.9188 189,59.5C 189.497,56.854 189.664,54.1873 189.5,51.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#c5d0d6"
+      d="M 160.5,30.5 C 142.375,28.0346 127.875,34.0346 117,48.5C 114.258,53.8909 112.425,59.5575 111.5,65.5C 130.647,66.0079 147.147,59.6746 161,46.5C 161.667,47.5 161.667,48.5 161,49.5C 157.317,55.4126 152.483,60.0793 146.5,63.5C 131.982,68.0564 116.982,70.0564 101.5,69.5C 100.822,48.3753 110.489,34.2087 130.5,27C 141.05,24.4311 151.05,25.5978 160.5,30.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#f8f9fa"
+      d="M 160.5,30.5 C 164.861,32.3524 168.528,35.1858 171.5,39C 181.959,36.6136 187.959,40.7803 189.5,51.5C 189.664,54.1873 189.497,56.854 189,59.5C 187.129,54.9188 184.629,50.7522 181.5,47C 177.848,44.7174 175.182,45.5507 173.5,49.5C 170.062,51.7589 166.729,54.2589 163.5,57C 158.227,60.2566 152.561,62.4232 146.5,63.5C 152.483,60.0793 157.317,55.4126 161,49.5C 161.667,48.5 161.667,47.5 161,46.5C 147.147,59.6746 130.647,66.0079 111.5,65.5C 112.425,59.5575 114.258,53.8909 117,48.5C 127.875,34.0346 142.375,28.0346 160.5,30.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7d7c7b"
+      d="M 204.5,219.5 C 204.5,219.833 204.5,220.167 204.5,220.5C 201.566,230.553 205.233,235.22 215.5,234.5C 215.586,235.496 215.252,236.329 214.5,237C 206.5,241.667 198.5,246.333 190.5,251C 189.117,252.271 188.117,253.771 187.5,255.5C 183.732,250.825 180.066,245.991 176.5,241C 177.416,240.722 178.082,240.222 178.5,239.5C 186.507,233.483 194.174,226.983 201.5,220C 202.448,219.517 203.448,219.351 204.5,219.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#c8d3dc"
+      d="M 232.5,227.5 C 236.146,228.462 237.313,230.795 236,234.5C 234.556,236.155 232.722,236.822 230.5,236.5C 231.461,233.574 232.127,230.574 232.5,227.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#94a0a9"
+      d="M 60.5,242.5 C 94.1667,242.5 127.833,242.5 161.5,242.5C 161.5,244.167 161.5,245.833 161.5,247.5C 159.585,247.784 157.919,247.451 156.5,246.5C 157.833,246.5 159.167,246.5 160.5,246.5C 160.5,245.5 160.5,244.5 160.5,243.5C 127.662,243.168 94.9957,243.502 62.5,244.5C 61.5,244.167 60.8333,243.5 60.5,242.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cdd7dd"
+      d="M 62.5,244.5 C 94.9957,243.502 127.662,243.168 160.5,243.5C 160.5,244.5 160.5,245.5 160.5,246.5C 159.167,246.5 157.833,246.5 156.5,246.5C 153.813,246.336 151.146,246.503 148.5,247C 146.297,249.372 143.963,251.539 141.5,253.5C 126.468,253.257 111.468,252.424 96.5,251C 84.8773,250.08 73.5439,247.913 62.5,244.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#5f6567"
+      d="M 161.5,242.5 C 166.451,241.601 167.451,242.935 164.5,246.5C 163.675,247.386 162.675,247.719 161.5,247.5C 161.5,245.833 161.5,244.167 161.5,242.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#08162a"
+      d="M 95.5,69.5 C 94.2076,71.7019 92.3743,73.3685 90,74.5C 89.6667,75.5 89.3333,76.5 89,77.5C 86.9038,82.4392 87.2372,87.1058 90,91.5C 91.0386,90.7562 91.5386,89.7562 91.5,88.5C 92.322,90.3885 93.1553,92.3885 94,94.5C 95.4443,96.1554 97.2776,96.822 99.5,96.5C 99.5,89.5 99.5,82.5 99.5,75.5C 98.5,75.5 97.5,75.5 96.5,75.5C 97.7804,74.0687 99.4471,73.4021 101.5,73.5C 100.336,81.1724 100.17,88.8391 101,96.5C 100.044,97.8941 98.7106,98.5607 97,98.5C 95.4504,98.0495 93.9504,97.8829 92.5,98C 93.1667,98.6667 93.8333,99.3333 94.5,100C 98.3127,100.148 101.479,101.648 104,104.5C 104.17,94.658 104.67,84.9913 105.5,75.5C 105.5,83.8333 105.5,92.1667 105.5,100.5C 104.838,106.563 106.004,112.23 109,117.5C 123.494,130.832 140.327,135.165 159.5,130.5C 157.909,131.219 156.409,132.219 155,133.5C 154.312,134.332 154.479,134.998 155.5,135.5C 159.038,137.096 162.038,136.429 164.5,133.5C 165.183,133.863 165.85,134.196 166.5,134.5C 163.932,140.072 159.932,144.405 154.5,147.5C 157.188,150.022 160.021,152.356 163,154.5C 164.942,150.931 167.108,147.598 169.5,144.5C 171.379,143.659 173.379,143.325 175.5,143.5C 175.107,145.284 174.441,146.95 173.5,148.5C 172.471,150.517 171.471,152.517 170.5,154.5C 168.732,156.508 167.066,158.674 165.5,161C 164.833,161.167 164.167,161.333 163.5,161.5C 160.553,159.729 157.719,157.729 155,155.5C 153.247,161.235 153.08,166.901 154.5,172.5C 155.412,186.353 156.745,200.02 158.5,213.5C 158.5,216.167 158.5,218.833 158.5,221.5C 169.833,221.5 181.167,221.5 192.5,221.5C 192.5,209.833 192.5,198.167 192.5,186.5C 194.099,184.271 196.099,183.938 198.5,185.5C 198.334,193.174 198.5,200.841 199,208.5C 201.372,211.91 203.205,215.577 204.5,219.5C 203.448,219.351 202.448,219.517 201.5,220C 194.174,226.983 186.507,233.483 178.5,239.5C 180.5,236.5 182.5,233.5 184.5,230.5C 187.833,229.359 190.833,227.526 193.5,225C 192.975,224.308 192.308,223.808 191.5,223.5C 189.253,224.374 186.92,224.708 184.5,224.5C 184.741,227.034 183.741,228.7 181.5,229.5C 181.611,228.883 181.944,228.383 182.5,228C 141.5,227.333 100.5,227.333 59.5,228C 55.5,230.667 55.5,233.333 59.5,236C 97.1652,236.5 134.832,236.667 172.5,236.5C 172.045,238.367 171.378,240.2 170.5,242C 170.833,242.833 171.167,243.667 171.5,244.5C 172.848,242.539 174.181,242.372 175.5,244C 175.034,245.299 175.034,246.799 175.5,248.5C 174.288,247.77 173.288,246.77 172.5,245.5C 171.5,245.833 170.833,246.5 170.5,247.5C 172.178,250.195 173.844,252.861 175.5,255.5C 174.5,255.5 173.5,255.5 172.5,255.5C 172.5,257.167 172.5,258.833 172.5,260.5C 173.5,260.5 174.5,260.5 175.5,260.5C 174.96,264.269 176.294,266.936 179.5,268.5C 172.728,273.428 166.061,278.594 159.5,284C 154.716,284.614 150.383,283.447 146.5,280.5C 144.402,279.065 142.069,277.898 139.5,277C 138.874,276.25 138.374,275.416 138,274.5C 137.265,269.162 136.932,263.828 137,258.5C 138.193,256.503 139.693,254.836 141.5,253.5C 143.963,251.539 146.297,249.372 148.5,247C 151.146,246.503 153.813,246.336 156.5,246.5C 157.919,247.451 159.585,247.784 161.5,247.5C 162.675,247.719 163.675,247.386 164.5,246.5C 167.451,242.935 166.451,241.601 161.5,242.5C 127.833,242.5 94.1667,242.5 60.5,242.5C 57.383,242.358 54.883,241.024 53,238.5C 49.5064,230.75 51.5064,225.083 59,221.5C 59.498,218.518 59.6646,215.518 59.5,212.5C 65.2988,196.656 70.7988,180.656 76,164.5C 77.8137,159.036 80.9804,154.536 85.5,151C 95.1667,146.5 104.833,142 114.5,137.5C 115.48,136.019 116.647,134.685 118,133.5C 118.749,132.365 118.583,131.365 117.5,130.5C 110.226,127.221 104.56,122.221 100.5,115.5C 99.8005,111.202 99.4672,106.869 99.5,102.5C 96.1066,102.938 93.1066,102.105 90.5,100C 86.3098,92.2015 85.4764,84.0349 88,75.5C 89.4802,72.1828 91.9802,70.1828 95.5,69.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7c7b79"
+      d="M 181.5,229.5 C 178.681,232.075 175.681,234.408 172.5,236.5C 134.832,236.667 97.1652,236.5 59.5,236C 55.5,233.333 55.5,230.667 59.5,228C 100.5,227.333 141.5,227.333 182.5,228C 181.944,228.383 181.611,228.883 181.5,229.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cbd7dd"
+      d="M 95.5,188.5 C 97.0722,199.912 95.0722,210.578 89.5,220.5C 82.1968,221.497 74.8635,221.831 67.5,221.5C 73.0162,217.483 78.8495,213.816 85,210.5C 88.2246,202.97 91.7246,195.636 95.5,188.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cdd7db"
+      d="M 192.5,186.5 C 192.5,198.167 192.5,209.833 192.5,221.5C 181.167,221.5 169.833,221.5 158.5,221.5C 158.5,218.833 158.5,216.167 158.5,213.5C 170.345,215.501 181.011,213.001 190.5,206C 191.15,199.401 191.816,192.901 192.5,186.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cbd7dc"
+      d="M 101.5,205.5 C 105.217,209.047 109.55,211.547 114.5,213C 121.158,213.5 127.825,213.666 134.5,213.5C 134.5,216.167 134.5,218.833 134.5,221.5C 123.5,221.5 112.5,221.5 101.5,221.5C 101.5,216.167 101.5,210.833 101.5,205.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#131e30"
+      d="M 165.5,188.5 C 171.176,188.334 176.843,188.501 182.5,189C 183.833,190.667 183.833,192.333 182.5,194C 176.833,194.667 171.167,194.667 165.5,194C 164.248,192.172 164.248,190.339 165.5,188.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7db5bd"
+      d="M 144.5,164.5 C 145.5,164.5 146.5,164.5 147.5,164.5C 149.637,183.46 151.304,202.46 152.5,221.5C 148.5,221.5 144.5,221.5 140.5,221.5C 141.37,202.468 142.703,183.468 144.5,164.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#bfc8ce"
+      d="M 136.5,190.5 C 136.314,183.972 136.648,177.472 137.5,171C 137.238,168.453 136.738,165.953 136,163.5C 134.129,162.332 132.629,162.832 131.5,165C 127.477,164.815 125.144,162.649 124.5,158.5C 125.626,159.714 126.959,160.714 128.5,161.5C 131.557,159.882 134.39,157.882 137,155.5C 137.667,156.833 138.333,158.167 139,159.5C 139.287,170.052 138.453,180.385 136.5,190.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#f6f5f2"
+      d="M 119.5,149.5 C 121.432,152.366 123.099,155.366 124.5,158.5C 125.144,162.649 127.477,164.815 131.5,165C 132.629,162.832 134.129,162.332 136,163.5C 136.738,165.953 137.238,168.453 137.5,171C 136.648,177.472 136.314,183.972 136.5,190.5C 136.295,198.217 135.628,205.884 134.5,213.5C 127.825,213.666 121.158,213.5 114.5,213C 109.55,211.547 105.217,209.047 101.5,205.5C 101.666,198.825 101.5,192.158 101,185.5C 97.5341,183.65 95.7007,184.65 95.5,188.5C 91.7246,195.636 88.2246,202.97 85,210.5C 78.8495,213.816 73.0162,217.483 67.5,221.5C 66.2353,220.486 65.5687,218.986 65.5,217C 70.7833,199.481 76.2833,181.981 82,164.5C 83.2775,162.253 84.7775,160.253 86.5,158.5C 93.6173,157.627 100.284,155.461 106.5,152C 111.014,150.737 115.347,149.904 119.5,149.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#76acb4"
+      d="M 145.5,148.5 C 150.229,151.321 150.395,154.654 146,158.5C 142.102,155.107 141.935,151.774 145.5,148.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#cad1d7"
+      d="M 119.5,149.5 C 115.347,149.904 111.014,150.737 106.5,152C 100.284,155.461 93.6173,157.627 86.5,158.5C 86.9528,157.542 87.6195,156.708 88.5,156C 97.5,151.833 106.5,147.667 115.5,143.5C 117.326,145.147 118.66,147.147 119.5,149.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7eb3bd"
+      d="M 123.5,134.5 C 127.419,139.412 132.086,143.745 137.5,147.5C 134.859,149.804 132.192,152.138 129.5,154.5C 126.368,149.734 123.368,144.901 120.5,140C 121.741,138.269 122.741,136.436 123.5,134.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#d5c3bb"
+      d="M 135.5,138.5 C 142.337,138.632 149.337,138.466 156.5,138C 153.278,140.888 149.778,143.388 146,145.5C 142.346,143.337 138.846,141.004 135.5,138.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#bdc4cc"
+      d="M 96.5,75.5 C 97.5,75.5 98.5,75.5 99.5,75.5C 99.5,82.5 99.5,89.5 99.5,96.5C 97.2776,96.822 95.4443,96.1554 94,94.5C 93.1553,92.3885 92.322,90.3885 91.5,88.5C 90.3728,82.9141 92.0395,78.5808 96.5,75.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#c6d2d6"
+      d="M 225.5,231.5 C 225.067,234.195 223.734,236.362 221.5,238C 211.667,245.833 201.833,253.667 192,261.5C 190.248,259.649 188.748,257.649 187.5,255.5C 188.117,253.771 189.117,252.271 190.5,251C 198.5,246.333 206.5,241.667 214.5,237C 215.252,236.329 215.586,235.496 215.5,234.5C 218.84,234.396 222.173,233.396 225.5,231.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#7aacb6"
+      d="M 175.5,248.5 C 177.193,251.184 178.86,253.851 180.5,256.5C 180.715,258.179 180.048,259.179 178.5,259.5C 177.488,258.128 176.488,256.795 175.5,255.5C 173.844,252.861 172.178,250.195 170.5,247.5C 170.833,246.5 171.5,245.833 172.5,245.5C 173.288,246.77 174.288,247.77 175.5,248.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#438a9a"
+      d="M 180.5,256.5 C 182.934,259.171 185.267,262.004 187.5,265C 186.407,266.365 185.074,266.699 183.5,266C 181.461,264.074 179.794,261.907 178.5,259.5C 180.048,259.179 180.715,258.179 180.5,256.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#081328"
+      d="M 97.5,256.5 C 100.271,256.421 102.937,256.921 105.5,258C 109.203,259.851 112.87,261.685 116.5,263.5C 120.213,261.96 123.547,259.793 126.5,257C 130.367,256.692 131.367,258.359 129.5,262C 125.419,265.578 120.752,268.078 115.5,269.5C 110.653,266.494 105.487,264.161 100,262.5C 96.0078,263.413 92.5078,265.246 89.5,268C 86.594,269.318 83.594,269.652 80.5,269C 76.7216,266.888 73.2216,264.388 70,261.5C 69.51,260.207 69.3433,258.873 69.5,257.5C 71.1992,257.34 72.8659,257.506 74.5,258C 77.1879,260.694 80.3545,262.527 84,263.5C 88.7686,261.534 93.2686,259.2 97.5,256.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#e6c3a9"
+      d="M 172.5,260.5 C 173.028,261.721 173.695,262.887 174.5,264C 169.167,268.667 163.833,273.333 158.5,278C 155.076,277.639 151.742,276.639 148.5,275C 146.508,274.426 145.008,273.259 144,271.5C 142.723,267.903 142.39,264.236 143,260.5C 145.5,258 148,255.5 150.5,253C 156.519,253.134 162.519,253.8 168.5,255C 169.697,257.015 171.03,258.849 172.5,260.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#ffffff"
+      d="M 151.5,255.5 C 155.555,255.176 159.555,255.509 163.5,256.5C 165.652,265.868 161.819,270.868 152,271.5C 150.395,270.899 149.062,269.899 148,268.5C 147.713,266.109 147.213,263.776 146.5,261.5C 148.19,259.481 149.856,257.481 151.5,255.5 Z" />
+  </g>
+  <g>
+    <path style="opacity:1" fill="#ccd8de"
+      d="M 234.5,242.5 C 231.206,259.878 221.206,271.711 204.5,278C 196.745,280.083 189.078,282.417 181.5,285C 177.443,288.355 173.11,291.188 168.5,293.5C 159.472,295.867 151.972,293.533 146,286.5C 145.211,284.288 145.378,282.288 146.5,280.5C 150.383,283.447 154.716,284.614 159.5,284C 166.061,278.594 172.728,273.428 179.5,268.5C 179.953,269.458 180.619,270.292 181.5,271C 184.167,271.667 186.833,271.667 189.5,271C 200.878,261.287 212.545,251.953 224.5,243C 227.817,242.502 231.15,242.335 234.5,242.5 Z" />
+  </g>
+</svg>

plugins/hitl/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@botpresshub/hitl-plugin",
+  "scripts": {
+    "check:type": "tsc --noEmit",
+    "build": "bp add -y && bp build",
+    "test": "vitest --run"
+  },
+  "private": true,
+  "dependencies": {
+    "@botpress/client": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "browser-or-node": "^2.1.1"
+  },
+  "devDependencies": {
+    "@botpress/cli": "workspace:*",
+    "@botpress/sdk": "workspace:*",
+    "@botpresshub/hitl": "workspace:*",
+    "@types/lodash": "^4.14.191",
+    "@types/semver": "^7.3.11",
+    "semver": "^7.3.8"
+  },
+  "bpDependencies": {
+    "hitl": "../../interfaces/hitl"
+  }
+}

plugins/hitl/plugin.definition.ts

@@ -0,0 +1,274 @@
+import * as sdk from '@botpress/sdk'
+import hitl from './bp_modules/hitl'
+
+export const NULL_MESSAGE_CODE = 'NULL'
+export const DEFAULT_HITL_HANDOFF_MESSAGE =
+  'I have escalated this conversation to a human agent. They should be with you shortly.'
+export const DEFAULT_HUMAN_AGENT_ASSIGNED_MESSAGE = 'A human agent has joined the conversation.'
+export const DEFAULT_HITL_STOPPED_MESSAGE = 'The human agent closed the conversation. I will continue assisting you.'
+export const DEFAULT_USER_HITL_CANCELLED_MESSAGE = '( The user has ended the session. )'
+export const DEFAULT_INCOMPATIBLE_MSGTYPE_MESSAGE =
+  'Sorry, the user can not receive this type of message. Please resend your message as a text message.'
+export const DEFAULT_USER_HITL_CLOSE_COMMAND = '/end'
+export const DEFAULT_USER_HITL_COMMAND_MESSAGE =
+  'You have ended the session with the human agent. I will continue assisting you.'
+export const DEFAULT_AGENT_ASSIGNED_TIMEOUT_MESSAGE =
+  'No human agent is available at the moment. Please try again later. I will continue assisting you for the time being.'
+
+const _nullCodeDescription = `Use "${NULL_MESSAGE_CODE}" to indicate that no message should be sent.`
+const PLUGIN_CONFIG_SCHEMA = sdk.z.object({
+  onHitlHandoffMessage: sdk.z
+    .string()
+    .title('Escalation Started Message')
+    .describe(`The message to send to the user when transferring to a human agent. ${_nullCodeDescription}`)
+    .optional()
+    .placeholder(DEFAULT_HITL_HANDOFF_MESSAGE),
+  onHumanAgentAssignedMessage: sdk.z
+    .string()
+    .title('Human Agent Assigned Message')
+    .describe(`The message to send to the user when a human agent is assigned. ${_nullCodeDescription}`)
+    .optional()
+    .placeholder(DEFAULT_HUMAN_AGENT_ASSIGNED_MESSAGE),
+  onHitlStoppedMessage: sdk.z
+    .string()
+    .title('Escalation Terminated Message')
+    .describe(
+      `The message to send to the user when the HITL session stops and control is transferred back to bot. ${_nullCodeDescription}`
+    )
+    .optional()
+    .placeholder(DEFAULT_HITL_STOPPED_MESSAGE),
+  onUserHitlCancelledMessage: sdk.z
+    .string()
+    .title('Escalation Aborted Message')
+    .describe(
+      `The message to send to the human agent when the user abruptly ends the HITL session. ${_nullCodeDescription}`
+    )
+    .optional()
+    .placeholder(DEFAULT_USER_HITL_CANCELLED_MESSAGE),
+  onIncompatibleMsgTypeMessage: sdk.z
+    .string()
+    .title('Incompatible Message Type Warning')
+    .describe(
+      `The warning to send to the human agent when they send a message that is not supported by the hitl session. ${_nullCodeDescription}`
+    )
+    .optional()
+    .placeholder(DEFAULT_INCOMPATIBLE_MSGTYPE_MESSAGE),
+  onUserHitlCloseMessage: sdk.z
+    .string()
+    .title('Termination Command Message')
+    .describe(
+      `The message to send to the user when they end the HITL session using the termination command. ${_nullCodeDescription}`
+    )
+    .optional()
+    .placeholder(DEFAULT_USER_HITL_COMMAND_MESSAGE),
+  onAgentAssignedTimeoutMessage: sdk.z
+    .string()
+    .title('Agent Assigned Timeout Message')
+    .describe(
+      `The message to send to the user when no human agent is assigned within the timeout period. ${_nullCodeDescription}`
+    )
+    .optional()
+    .placeholder(DEFAULT_AGENT_ASSIGNED_TIMEOUT_MESSAGE),
+  userHitlCloseCommand: sdk.z
+    .string()
+    .title('Termination Command')
+    .describe(
+      'Users may send this command to end the HITL session at any time. It is case-insensitive, so it works regardless of letter casing.'
+    )
+    .optional()
+    .placeholder(DEFAULT_USER_HITL_CLOSE_COMMAND),
+  agentAssignedTimeoutSeconds: sdk.z
+    .number()
+    .title('Agent Assigned Timeout')
+    .describe(
+      'The time in seconds to wait before giving up and telling the user that no human agent is available. Set to 0 to disable'
+    )
+    .nonnegative()
+    .optional()
+    .placeholder('0'),
+  useHumanAgentInfo: sdk.z
+    .boolean()
+    .default(true)
+    .title('Use Human Agent Info')
+    .describe(
+      '(Works only with webchat) Enable this to use the human agent name and photo (if available) as the bot name and photo while in HITL session.'
+    ),
+  flowOnHitlStopped: sdk.z
+    .boolean()
+    .default(true)
+    .title('Continue Flow on Session End?')
+    .describe('Enable this to continue the flow when the HITL session ends. Otherwise, the flow waits for user input.'),
+})
+
+export default new sdk.PluginDefinition({
+  name: 'hitl',
+  version: '1.4.0',
+  title: 'Human In The Loop',
+  description: 'Seamlessly transfer conversations to human agents',
+  icon: 'icon.svg',
+  readme: 'hub.md',
+  configuration: {
+    schema: PLUGIN_CONFIG_SCHEMA,
+  },
+  actions: {
+    startHitl: {
+      title: 'Start HITL',
+      description: 'Starts the HITL session',
+      input: {
+        schema: ({ entities }) =>
+          sdk.z
+            .object({
+              title: sdk.z.string().title('Ticket Title').describe('Title of the HITL ticket'),
+              description: sdk.z
+                .string()
+                .title('Ticket Description')
+                .optional()
+                .describe('Description of the HITL ticket'),
+              hitlSession: entities.hitl.hitlSession
+                .optional()
+                .title('Extra configuration')
+                .describe('Configuration of the HITL session'),
+              userId: sdk.z
+                .string()
+                .title('User ID')
+                .describe('ID of the user that starts the HITL session')
+                .placeholder('{{ event.userId }}'),
+              userEmail: sdk.z
+                .string()
+                .title('User Email')
+                .optional()
+                .describe(
+                  'Email of the user that starts the HITL session. If this value is unset, the agent will try to use the email provided by the channel.'
+                ),
+              conversationId: sdk.z
+                .string()
+                .title('Conversation ID') // this is the upstream conversation
+                .describe('ID of the conversation on which to start the HITL session')
+                .placeholder('{{ event.conversationId }}'),
+              configurationOverrides: PLUGIN_CONFIG_SCHEMA.partial()
+                .optional()
+                .title('Configuration Overrides')
+                .describe('Use this to override the global configuration for this specific HITL session'),
+            })
+            .passthrough(),
+      },
+      output: { schema: sdk.z.object({}) },
+    },
+    stopHitl: {
+      title: 'Stop HITL',
+      description: 'Stop the HITL session',
+      input: {
+        schema: sdk.z.object({
+          conversationId: sdk.z
+            .string()
+            .describe('ID of the conversation on which to stop the HITL session')
+            .placeholder('{{ event.conversationId }}'),
+        }),
+      },
+      output: { schema: sdk.z.object({}) },
+    },
+  },
+  states: {
+    hitl: {
+      type: 'conversation',
+      schema: sdk.z.object({
+        hitlActive: sdk.z.boolean().title('Is HITL Enabled?').describe('Whether the conversation is in HITL session'),
+      }),
+    },
+    initiatingUser: {
+      type: 'conversation',
+      schema: sdk.z.object({
+        upstreamUserId: sdk.z
+          .string()
+          .title('Upstream User ID')
+          .describe('The ID of the user that triggered the HITL session (set on the upstream conversation)'),
+      }),
+    },
+    effectiveSessionConfig: {
+      type: 'conversation',
+      schema: PLUGIN_CONFIG_SCHEMA,
+    },
+  },
+  user: {
+    tags: {
+      downstream: {
+        title: 'Downstream User ID',
+        description: 'ID of the downstream user bound to the upstream one',
+      },
+      upstream: {
+        title: 'Upstream User ID',
+        description: 'ID of the upstream user bound to the downstream one',
+      },
+      integrationName: {
+        title: 'HITL Integration Name',
+        description: 'Name of the integration which created the downstream user',
+      },
+    },
+  },
+  conversation: {
+    tags: {
+      downstream: {
+        title: 'Downstream Conversation ID',
+        description: 'ID of the downstream conversation bound to the upstream one',
+      },
+      upstream: {
+        title: 'Upstream Conversation ID',
+        description: 'ID of the upstream conversation bound to the downstream one',
+      },
+      humanAgentId: {
+        title: 'Human Agent ID',
+        description: 'ID of the human agent assigned to the ticket',
+      },
+      humanAgentName: {
+        title: 'Human Agent Name',
+        description: 'Name of the human agent assigned to the ticket',
+      },
+      hitlEndReason: {
+        title: 'HITL End Reason',
+        description: 'Reason why the HITL session ended',
+      },
+      startMessageId: {
+        title: 'Start Message ID',
+        description: 'ID of the user message that initiated the HITL session',
+      },
+    },
+  },
+  message: {
+    tags: {
+      downstream: {
+        title: 'Downstream Message ID',
+        description: 'ID of the downstream message bound to the upstream one',
+      },
+      upstream: {
+        title: 'Upstream Message ID',
+        description: 'ID of the upstream message bound to the downstream one',
+      },
+      additionalData: {
+        title: 'Additional Data',
+        description: 'Additional data optionally sent with the message by the downstream integration',
+      },
+    },
+  },
+  interfaces: {
+    hitl: sdk.version.allWithinMajorOf(hitl),
+  },
+  events: {
+    humanAgentAssignedTimeout: {
+      schema: sdk.z.object({
+        sessionStartedAt: sdk.z
+          .string()
+          .title('Session Started At')
+          .describe('Timestamp of when the HITL session started'),
+        downstreamConversationId: sdk.z
+          .string()
+          .title('Downstream Conversation ID')
+          .describe('ID of the downstream conversation'),
+      }),
+    },
+    continueWorkflow: {
+      schema: sdk.z.object({
+        conversationId: sdk.z.string().title('Upstream Conversation ID').describe('ID of the upstream conversation'),
+      }),
+    },
+  },
+})

plugins/hitl/src/actions/index.ts

@@ -0,0 +1,2 @@
+export * from './start-hitl'
+export * from './stop-hitl'

plugins/hitl/src/actions/start-hitl.ts

@@ -0,0 +1,204 @@
+import * as sdk from '@botpress/sdk'
+import { DEFAULT_HITL_HANDOFF_MESSAGE } from '../../plugin.definition'
+import * as configuration from '../configuration'
+import * as conv from '../conv-manager'
+import type * as types from '../types'
+import * as user from '../user-linker'
+import * as bp from '.botpress'
+
+type StartHitlInput = bp.actions.startHitl.input.Input
+type MessageHistoryElement = NonNullable<bp.interfaces.hitl.actions.startHitl.input.Input['messageHistory']>[number]
+type Props = Parameters<bp.PluginProps['actions']['startHitl']>[0]
+
+export const startHitl: bp.PluginProps['actions']['startHitl'] = async (props) => {
+  const { conversationId: upstreamConversationId, userId: upstreamUserId, userEmail: upstreamUserEmail } = props.input
+  if (!upstreamConversationId.length) {
+    throw new sdk.RuntimeError('conversationId is required to start HITL')
+  }
+  if (!upstreamUserId.length) {
+    throw new sdk.RuntimeError('userId is required to start HITL')
+  }
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({ id: upstreamConversationId })
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+
+  if (upstreamConversation.tags.upstream || upstreamConversation.integration === props.interfaces.hitl.name) {
+    // Without this check, closing the downstream conversation (the ticket) can
+    // result in the bot calling startHitl a second time, but using the
+    // downstream conversation as if it was the upstream conversation. Human
+    // support agents would thus see "I have escalated this to a human" inside
+    // the ticket after closing it.
+    return {}
+  }
+
+  if (await upstreamCm.isHitlActive()) {
+    return {}
+  }
+
+  const lastMessageByUser = await _getLastUserMessageId({ upstreamConversation })
+
+  if (upstreamConversation.tags.startMessageId && upstreamConversation.tags.startMessageId === lastMessageByUser) {
+    // This is a hack because of a bug in the studio or webchat that causes it
+    // to call startHitl by replaying the same message.
+    return {}
+  }
+
+  const sessionConfig = await configuration.configureNewHitlSession({
+    ...props,
+    upstreamConversationId,
+    configurationOverrides: props.input.configurationOverrides,
+  })
+  await _sendHandoffMessage(upstreamCm, sessionConfig)
+
+  const users = new user.UserLinker(props)
+  const downstreamUserId = await users.getDownstreamUserId(upstreamUserId, { email: upstreamUserEmail })
+
+  const messageHistory = await _buildMessageHistory(upstreamConversation, users)
+
+  const downstreamConversation = await _createDownstreamConversation(
+    props,
+    downstreamUserId,
+    props.input,
+    messageHistory
+  )
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  await upstreamCm.setUserId(upstreamUserId)
+
+  await _linkConversations(upstreamConversation, downstreamConversation)
+  await _saveStartMessageId({ upstreamConversation, startMessageId: lastMessageByUser })
+  await _activateHitl(upstreamCm, downstreamCm)
+  await _startHitlTimeout(props, upstreamCm, downstreamCm, upstreamUserId, sessionConfig)
+
+  return {}
+}
+
+const _sendHandoffMessage = (
+  upstreamCm: conv.ConversationManager,
+  sessionConfig: bp.configuration.Configuration
+): Promise<void> => upstreamCm.maybeRespondText(sessionConfig.onHitlHandoffMessage, DEFAULT_HITL_HANDOFF_MESSAGE)
+
+const _buildMessageHistory = async (
+  upstreamConversation: types.ActionableConversation,
+  users: user.UserLinker
+): Promise<MessageHistoryElement[]> => {
+  const upstreamMessages = await upstreamConversation.listMessages().takePage(1)
+
+  // Sort messages by creation date, with the oldest first:
+  upstreamMessages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime())
+
+  const messageHistory: MessageHistoryElement[] = await Promise.all(
+    upstreamMessages.map(
+      async (message) =>
+        ({
+          source:
+            message.direction === 'outgoing'
+              ? { type: 'bot' }
+              : {
+                  type: 'user',
+                  userId: await users.getDownstreamUserId(message.userId),
+                },
+          type: message.type,
+          payload: message.payload,
+        }) as MessageHistoryElement
+    )
+  )
+
+  return messageHistory
+}
+
+const _createDownstreamConversation = async (
+  props: Props,
+  downstreamUserId: string,
+  input: StartHitlInput,
+  messageHistory: MessageHistoryElement[]
+): Promise<types.ActionableConversation> => {
+  // Call startHitl in the hitl integration (zendesk, etc.):
+  const { conversationId: downstreamConversationId } = await props.actions.hitl.startHitl({
+    title: input.title,
+    description: input.description,
+    hitlSession: input.hitlSession,
+    userId: downstreamUserId,
+    messageHistory,
+  })
+
+  return await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+}
+
+const _linkConversations = (
+  upstreamConversation: types.ActionableConversation,
+  downstreamConversation: types.ActionableConversation
+) =>
+  Promise.all([
+    upstreamConversation.update({
+      tags: {
+        downstream: downstreamConversation.id,
+      },
+    }),
+    downstreamConversation.update({
+      tags: {
+        upstream: upstreamConversation.id,
+      },
+    }),
+  ])
+
+const _activateHitl = (upstreamCm: conv.ConversationManager, downstreamCm: conv.ConversationManager) =>
+  Promise.all([upstreamCm.setHitlActive(), downstreamCm.setHitlActive()])
+
+const _startHitlTimeout = async (
+  props: Props,
+  upstreamCm: conv.ConversationManager,
+  downstreamCm: conv.ConversationManager,
+  upstreamUserId: string,
+  sessionConfig: bp.configuration.Configuration
+) => {
+  const { agentAssignedTimeoutSeconds } = sessionConfig
+
+  if (!agentAssignedTimeoutSeconds) {
+    return
+  }
+
+  await props.events.humanAgentAssignedTimeout
+    .withConversationId(upstreamCm.conversationId)
+    .withUserId(upstreamUserId)
+    .schedule(
+      {
+        sessionStartedAt: new Date().toISOString(),
+        downstreamConversationId: downstreamCm.conversationId,
+      },
+      { delay: agentAssignedTimeoutSeconds * 1000 }
+    )
+}
+
+/**
+ * Gets the id of the last message sent by the user in the conversation.
+ *
+ * This is effectively a hack to ensure that the studio/webchat does not call
+ * startHitl twice for the same user message.
+ */
+const _getLastUserMessageId = async (props: {
+  upstreamConversation: types.ActionableConversation
+}): Promise<string | undefined> => {
+  for await (const message of props.upstreamConversation.listMessages()) {
+    if (message.direction === 'incoming') {
+      return message.id
+    }
+  }
+
+  return
+}
+
+const _saveStartMessageId = async (props: {
+  upstreamConversation: types.ActionableConversation
+  startMessageId: string | undefined
+}) => {
+  if (!props.startMessageId) {
+    return
+  }
+
+  await props.upstreamConversation.update({
+    tags: {
+      startMessageId: props.startMessageId,
+    },
+  })
+}

plugins/hitl/src/actions/stop-hitl.ts

@@ -0,0 +1,46 @@
+import { DEFAULT_USER_HITL_CANCELLED_MESSAGE } from 'plugin.definition'
+import * as configuration from '../configuration'
+import * as conv from '../conv-manager'
+import * as bp from '.botpress'
+
+export const stopHitl: bp.PluginProps['actions']['stopHitl'] = async (props) => {
+  const { conversationId: upstreamConversationId } = props.input
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({ id: upstreamConversationId })
+
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+  const isHitlActive = await upstreamCm.isHitlActive()
+  if (!isHitlActive) {
+    return {}
+  }
+
+  const downstreamConversationId = upstreamConversation.tags.downstream
+  if (!downstreamConversationId) {
+    props.logger
+      .withConversationId(upstreamConversationId)
+      .error('Upstream conversation is not bound to a downstream conversation')
+    return {}
+  }
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId,
+  })
+
+  await downstreamCm.maybeRespondText(sessionConfig.onUserHitlCancelledMessage, DEFAULT_USER_HITL_CANCELLED_MESSAGE)
+
+  await Promise.allSettled([
+    upstreamCm.setHitlInactive(conv.HITL_END_REASON.CLOSE_ACTION_CALLED),
+    downstreamCm.setHitlInactive(conv.HITL_END_REASON.CLOSE_ACTION_CALLED),
+  ])
+
+  // Call stopHitl in the hitl integration (zendesk, etc.):
+  await props.actions.hitl.stopHitl({ conversationId: downstreamConversationId })
+
+  // TODO: possibly send the workflowContinue event here
+
+  return {}
+}

plugins/hitl/src/configuration.ts

@@ -0,0 +1,29 @@
+import * as bp from '.botpress'
+
+export const configureNewHitlSession = async (props: {
+  states: bp.ActionHandlerProps['states']
+  configuration: bp.configuration.Configuration
+  configurationOverrides?: Partial<bp.configuration.Configuration>
+  upstreamConversationId: string
+}): Promise<bp.configuration.Configuration> => {
+  const effectiveSessionConfiguration = {
+    ...props.configuration,
+    ...props.configurationOverrides,
+  }
+
+  await props.states.conversation.effectiveSessionConfig.set(
+    props.upstreamConversationId,
+    effectiveSessionConfiguration
+  )
+
+  return effectiveSessionConfiguration
+}
+
+export const retrieveSessionConfig = async (props: {
+  states: bp.ActionHandlerProps['states']
+  configuration: bp.configuration.Configuration
+  upstreamConversationId: string
+}): Promise<bp.configuration.Configuration> =>
+  await props.states.conversation.effectiveSessionConfig
+    .get(props.upstreamConversationId)
+    .catch(() => props.configuration)

plugins/hitl/src/consts.ts

@@ -0,0 +1,10 @@
+import * as sdk from '@botpress/sdk'
+
+export const SUPPORTED_MESSAGE_TYPES = [
+  'text',
+  'image',
+  'video',
+  'audio',
+  'file',
+  'bloc',
+] as const satisfies (keyof typeof sdk.messages.defaults)[]

plugins/hitl/src/conv-manager.ts

@@ -0,0 +1,188 @@
+import { NULL_MESSAGE_CODE } from 'plugin.definition'
+import * as types from './types'
+import * as bp from '.botpress'
+
+// this state is hardcoded in the Studio/DM
+type TypingIndicatorState = { enabled: boolean }
+const TYPING_INDICATOR_STATE_NAME = 'typingIndicatorEnabled'
+
+type HitlState = bp.states.hitl.Hitl['payload']
+
+const DEFAULT_STATE: HitlState = { hitlActive: false }
+
+export const HITL_END_REASON = {
+  // PATIENT_LEFT: 'patient-left',
+  PATIENT_USED_TERMINATION_COMMAND: 'patient-used-termination-command',
+  AGENT_ASSIGNMENT_TIMEOUT: 'agent-assignment-timeout',
+  // AGENT_RESPONSE_TIMEOUT: 'agent-response-timeout',
+  AGENT_CLOSED_TICKET: 'agent-closed-ticket',
+  CLOSE_ACTION_CALLED: 'close-action-called',
+  INTERNAL_ERROR: 'internal-error',
+} as const
+type HitlEndReason = (typeof HITL_END_REASON)[keyof typeof HITL_END_REASON]
+
+export class ConversationManager {
+  public static from(props: types.AnyHandlerProps, conversation: types.ActionableConversation): ConversationManager {
+    return new ConversationManager(props, conversation)
+  }
+
+  private constructor(
+    private _props: types.AnyHandlerProps,
+    private _conversation: types.ActionableConversation
+  ) {}
+
+  public get conversationId(): string {
+    return this._conversation.id
+  }
+
+  public async setHumanAgent(humanAgentId: string, humanAgentName: string) {
+    await this._patchConversationTags({ humanAgentId, humanAgentName })
+  }
+
+  public isHumanAgentAssigned(): boolean {
+    return !!this._conversation.tags.humanAgentId?.length
+  }
+
+  public async isHitlActive(): Promise<boolean> {
+    const hitlState = await this._getHitlState()
+    return hitlState.hitlActive
+  }
+
+  public async setHitlActive(): Promise<void> {
+    await this._setHitlState({ hitlActive: true })
+    await this._toggleTypingIndicator({ enabled: false })
+  }
+
+  public async setHitlInactive(reason: HitlEndReason): Promise<void> {
+    await Promise.all([
+      this._setHitlState({ hitlActive: false }),
+      this._patchConversationTags({ hitlEndReason: reason }),
+      this._toggleTypingIndicator({ enabled: true }),
+    ])
+  }
+
+  public async continueWorkflow(): Promise<void> {
+    const initiatingUserId = await this._props.states.conversation.initiatingUser
+      .get(this._conversation.id)
+      .then((state) => state.upstreamUserId)
+
+    let eventEmitter = this._props.events.continueWorkflow.withConversationId(this._conversation.id)
+
+    if (initiatingUserId) {
+      eventEmitter = eventEmitter.withUserId(initiatingUserId)
+    }
+
+    await eventEmitter.emit({
+      conversationId: this._conversation.id,
+    })
+  }
+
+  public async maybeRespondText(message: string | undefined, defaultMsg: string): Promise<void> {
+    if (message === NULL_MESSAGE_CODE) {
+      return
+    }
+    const text = message || defaultMsg
+    await this.respond({ type: 'text', text })
+  }
+
+  public async respond(messagePayload: types.MessagePayload, tags: types.MessageTags = {}): Promise<void> {
+    // FIXME: in the future, we should use the provided UserId so that messages
+    //        on Botpress appear to come from the agent/user instead of the
+    //        bot user. For now, this is not possible because of checks in the
+    //        backend.
+
+    // FIXME: typescript has trouble narrowing the type here, so we use a switch
+    //        statement as a workaround.
+
+    switch (messagePayload.type) {
+      case 'text':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      case 'image':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      case 'audio':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      case 'file':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      case 'video':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      case 'bloc':
+        await this._conversation.createMessage({
+          type: messagePayload.type,
+          userId: this._props.ctx.botId,
+          payload: messagePayload,
+          tags,
+        })
+        break
+      default:
+        messagePayload satisfies never
+    }
+  }
+
+  public async abortHitlSession(errorMessage: string): Promise<void> {
+    await this.setHitlInactive(HITL_END_REASON.INTERNAL_ERROR)
+    await this.respond({ type: 'text', text: errorMessage })
+  }
+
+  public async setUserId(userId: string): Promise<void> {
+    return await this._props.states.conversation.initiatingUser.set(this._conversation.id, { upstreamUserId: userId })
+  }
+
+  private async _getHitlState(): Promise<bp.states.hitl.Hitl['payload']> {
+    return await this._props.states.conversation.hitl.getOrSet(this._conversation.id, DEFAULT_STATE)
+  }
+
+  private async _setHitlState(state: HitlState): Promise<void> {
+    return await this._props.states.conversation.hitl.set(this._conversation.id, state)
+  }
+
+  private async _patchConversationTags(tags: Record<string, string>): Promise<void> {
+    await this._conversation.update({ tags })
+  }
+
+  private async _toggleTypingIndicator(payload: TypingIndicatorState): Promise<void> {
+    try {
+      await this._props.client.setState({
+        id: this._conversation.id,
+        type: 'conversation',
+        name: TYPING_INDICATOR_STATE_NAME,
+        payload,
+      })
+    } catch (thrown) {
+      // because this state is hardcoded in the Studio / DM, it might not exist in some bot-as-code or ADK bots
+      const errorMsg = thrown instanceof Error ? thrown.message : String(thrown)
+      this._props.logger
+        .withConversationId(this._conversation.id)
+        .debug(`Could not set typing indicator state: ${errorMsg}`)
+    }
+  }
+}

plugins/hitl/src/hooks/before-incoming-event/all.ts

@@ -0,0 +1,34 @@
+import * as conv from '../../conv-manager'
+import * as consts from '../consts'
+import * as bp from '.botpress'
+
+const getConversationId = (props: bp.HookHandlerProps['before_incoming_event']): string | undefined => {
+  const { data: event } = props
+  if (event.conversationId) {
+    return event.conversationId
+  }
+  if ('conversationId' in event.payload && typeof event.payload.conversationId === 'string') {
+    return event.payload.conversationId
+  }
+  return undefined
+}
+
+export const handleEvent: bp.HookHandlers['before_incoming_event']['*'] = async (props) => {
+  const conversationId = getConversationId(props)
+  if (!conversationId) {
+    return
+  }
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: conversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+  const isHitlActive = await downstreamCm.isHitlActive()
+  if (isHitlActive) {
+    /**
+     * if conversation is downstream; we prevent the bot from answering in the ticket
+     * if conversation is upstream; we prevent the bot from answering in the chat
+     */
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  return
+}

plugins/hitl/src/hooks/before-incoming-event/hitl-assigned.ts

@@ -0,0 +1,24 @@
+import * as conv from '../../conv-manager'
+import * as consts from '../consts'
+import { assignAgent } from '../operations'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.HookHandlers['before_incoming_event']['hitl:hitlAssigned'] = async (props) => {
+  const { conversationId: downstreamConversationId, userId: humanAgentUserId } = props.data.payload
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  const isHitlActive = await downstreamCm.isHitlActive()
+  if (!isHitlActive) {
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  await assignAgent({
+    props,
+    downstreamConversation,
+    humanAgentUserId,
+  })
+
+  return consts.STOP_EVENT_HANDLING
+}

plugins/hitl/src/hooks/before-incoming-event/hitl-stopped.ts

@@ -0,0 +1,46 @@
+import { DEFAULT_HITL_STOPPED_MESSAGE } from '../../../plugin.definition'
+import * as configuration from '../../configuration'
+import * as conv from '../../conv-manager'
+import * as consts from '../consts'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.HookHandlers['before_incoming_event']['hitl:hitlStopped'] = async (props) => {
+  const { conversationId: downstreamConversationId } = props.data.payload
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  const isHitlActive = await downstreamCm.isHitlActive()
+  if (!isHitlActive) {
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const upstreamConversationId = downstreamConversation.tags.upstream
+  if (!upstreamConversationId) {
+    props.logger
+      .withConversationId(downstreamConversationId)
+      .error('Downstream conversation was not binded to upstream conversation')
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({ id: upstreamConversationId })
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId,
+  })
+
+  await Promise.allSettled([
+    upstreamCm.maybeRespondText(sessionConfig.onHitlStoppedMessage, DEFAULT_HITL_STOPPED_MESSAGE),
+    downstreamCm.setHitlInactive(conv.HITL_END_REASON.AGENT_CLOSED_TICKET),
+    upstreamCm.setHitlInactive(conv.HITL_END_REASON.AGENT_CLOSED_TICKET),
+  ])
+
+  if (sessionConfig.flowOnHitlStopped) {
+    // the bot will continue the conversation without the patient having to send another message
+    await upstreamCm.continueWorkflow()
+  }
+
+  return consts.STOP_EVENT_HANDLING
+}

plugins/hitl/src/hooks/before-incoming-event/human-agent-assigned-timeout.ts

@@ -0,0 +1,98 @@
+import { DEFAULT_AGENT_ASSIGNED_TIMEOUT_MESSAGE, DEFAULT_USER_HITL_CANCELLED_MESSAGE } from '../../../plugin.definition'
+import * as configuration from '../../configuration'
+import * as conv from '../../conv-manager'
+import * as consts from '../consts'
+import * as bp from '.botpress'
+
+export const handleEvent: bp.HookHandlers['before_incoming_event']['humanAgentAssignedTimeout'] = async (props) => {
+  const {
+    conversationId: upstreamConversationId,
+    payload: { downstreamConversationId },
+  } = props.data
+
+  if (!upstreamConversationId || !downstreamConversationId) {
+    props.logger.error('Missing conversationId in event payload')
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({ id: upstreamConversationId })
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+
+  if (upstreamCm.isHumanAgentAssigned()) {
+    props.logger.info('Human agent assigned timeout event ignored because the agent is already assigned')
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  const isHitlActive = (await upstreamCm.isHitlActive()) && (await downstreamCm.isHitlActive())
+
+  if (!isHitlActive) {
+    props.logger.info('Human agent assigned timeout event ignored because hitl is inactive')
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId,
+  })
+
+  if (!_isTimeoutElapsed(props, sessionConfig)) {
+    props.logger.info('Human agent assigned timeout event ignored because the timeout has not lapsed yet')
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  await _handleTimeout(props, upstreamCm, downstreamCm, sessionConfig)
+
+  if (sessionConfig.flowOnHitlStopped) {
+    // the bot will continue the conversation without the patient having to send another message
+    await upstreamCm.continueWorkflow()
+  }
+
+  return consts.STOP_EVENT_HANDLING
+}
+
+const _isTimeoutElapsed = (
+  props: bp.HookHandlerProps['before_incoming_event'],
+  sessionConfig: bp.configuration.Configuration
+): boolean => {
+  if (!_isTimeoutEnabled(sessionConfig)) {
+    props.logger.info('Human agent assigned timeout is not enabled')
+    return false
+  }
+
+  const now = Date.now()
+  const timeWhenCreated = new Date(props.data.payload.sessionStartedAt).getTime()
+  const elapsedSeconds = (now - timeWhenCreated) / 1000
+  const timeoutSeconds = sessionConfig.agentAssignedTimeoutSeconds ?? 0
+
+  return elapsedSeconds >= timeoutSeconds
+}
+
+const _isTimeoutEnabled = (sessionConfig: bp.configuration.Configuration): boolean =>
+  !!sessionConfig.agentAssignedTimeoutSeconds
+
+const _handleTimeout = async (
+  props: bp.HookHandlerProps['before_incoming_event'],
+  upstreamCm: conv.ConversationManager,
+  downstreamCm: conv.ConversationManager,
+  sessionConfig: bp.configuration.Configuration
+) => {
+  await downstreamCm.maybeRespondText(sessionConfig.onUserHitlCancelledMessage, DEFAULT_USER_HITL_CANCELLED_MESSAGE)
+
+  await Promise.allSettled([
+    upstreamCm.setHitlInactive(conv.HITL_END_REASON.AGENT_ASSIGNMENT_TIMEOUT),
+    downstreamCm.setHitlInactive(conv.HITL_END_REASON.AGENT_ASSIGNMENT_TIMEOUT),
+  ])
+
+  props.logger.withConversationId(upstreamCm.conversationId).info('HITL session ended due to agent assignment timeout')
+  props.logger
+    .withConversationId(downstreamCm.conversationId)
+    .info('HITL session ended due to agent assignment timeout')
+
+  // Call stopHitl in the hitl integration (zendesk, etc.):
+  await props.actions.hitl.stopHitl({ conversationId: downstreamCm.conversationId })
+
+  await upstreamCm.maybeRespondText(sessionConfig.onAgentAssignedTimeoutMessage, DEFAULT_AGENT_ASSIGNED_TIMEOUT_MESSAGE)
+}

plugins/hitl/src/hooks/before-incoming-event/index.ts

@@ -0,0 +1,4 @@
+export * as hitlAssigned from './hitl-assigned'
+export * as hitlStopped from './hitl-stopped'
+export * as humanAgentAssignedTimeout from './human-agent-assigned-timeout'
+export * as all from './all'

plugins/hitl/src/hooks/before-incoming-message/all.ts

@@ -0,0 +1,235 @@
+import * as client from '@botpress/client'
+import {
+  DEFAULT_INCOMPATIBLE_MSGTYPE_MESSAGE,
+  DEFAULT_USER_HITL_CANCELLED_MESSAGE,
+  DEFAULT_USER_HITL_CLOSE_COMMAND,
+  DEFAULT_USER_HITL_COMMAND_MESSAGE,
+} from 'plugin.definition'
+import { assignAgent } from 'src/hooks/operations'
+import { tryLinkWebchatUser } from 'src/webchat'
+import * as configuration from '../../configuration'
+import * as conv from '../../conv-manager'
+import type * as types from '../../types'
+import * as consts from '../consts'
+import * as bp from '.botpress'
+
+export const handleMessage: bp.HookHandlers['before_incoming_message']['*'] = async (props) => {
+  const conversation = await props.conversations.hitl.hitl.getById({
+    id: props.data.conversationId,
+  })
+
+  const { integration } = conversation
+  if (integration === props.interfaces.hitl.name) {
+    return await _handleDownstreamMessage(props, conversation)
+  }
+  return await _handleUpstreamMessage(props, conversation)
+}
+
+const _handleDownstreamMessage = async (
+  props: bp.HookHandlerProps['before_incoming_message'],
+  downstreamConversation: types.ActionableConversation
+) => {
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+  const isHitlActive = await downstreamCm.isHitlActive()
+  if (!isHitlActive) {
+    return consts.STOP_EVENT_HANDLING // we don't want the bot to chat with the human agent in a closed ticket
+  }
+
+  const downstreamUserId = props.data.userId
+  const downstreamUser = await props.users.getById({ id: downstreamUserId })
+
+  const upstreamConversationId = downstreamConversation.tags['upstream']
+
+  if (!upstreamConversationId) {
+    return await _abortHitlSession({
+      cm: downstreamCm,
+      internalReason: 'Downstream conversation was not bound to upstream conversation',
+      reasonShownToUser: 'Something went wrong, you are not connected to a human agent...',
+      props,
+    })
+  }
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId,
+  })
+
+  const messagePayload = _getMessagePayloadIfSupported(props.data)
+
+  if (!messagePayload) {
+    props.logger.with(props.data).error('Downstream conversation received a non-text message')
+    await downstreamCm.maybeRespondText(
+      sessionConfig.onIncompatibleMsgTypeMessage,
+      DEFAULT_INCOMPATIBLE_MSGTYPE_MESSAGE
+    )
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({ id: upstreamConversationId })
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+
+  props.logger.withConversationId(downstreamConversation.id).info('Sending message to upstream')
+
+  const upstreamUserId = await tryLinkWebchatUser(props, { downstreamUser, upstreamConversation })
+
+  if (!downstreamConversation.tags.humanAgentId?.length) {
+    // Try to assing here, if there is no human agent assigned to the downstream conversation
+    await assignAgent({
+      props,
+      downstreamConversation,
+      humanAgentUserId: downstreamUser.id,
+    })
+  }
+
+  await upstreamCm.respond(
+    { ...messagePayload, userId: upstreamUserId },
+    { downstream: props.data.id, additionalData: _getMessageAdditionalData(props.data) }
+  )
+  return consts.STOP_EVENT_HANDLING
+}
+
+const _getMessagePayloadIfSupported = (msg: client.Message): types.MessagePayload | undefined =>
+  consts.SUPPORTED_MESSAGE_TYPES.includes(msg.type as types.SupportedMessageTypes)
+    ? ({ type: msg.type, ...msg.payload } as types.MessagePayload)
+    : undefined
+
+const _getMessageAdditionalData = (msg: client.Message): string | undefined => {
+  // should be typed by the SDK because it's part of the hitl interface
+  const propName = 'additionalData'
+  if (propName in msg.payload && typeof msg.payload[propName] === 'string') {
+    return msg.payload[propName]
+  }
+  return
+}
+
+const _handleUpstreamMessage = async (
+  props: bp.HookHandlerProps['before_incoming_message'],
+  upstreamConversation: types.ActionableConversation
+) => {
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+  const isHitlActive = await upstreamCm.isHitlActive()
+  if (!isHitlActive) {
+    return consts.LET_BOT_HANDLE_EVENT
+  }
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId: upstreamCm.conversationId,
+  })
+
+  const messagePayload = _getMessagePayloadIfSupported(props.data)
+
+  if (!messagePayload) {
+    props.logger.with(props.data).error('Upstream conversation received a non-text message')
+
+    const supportedMessageTypes = consts.SUPPORTED_MESSAGE_TYPES.join(', ')
+    await upstreamCm.respond({
+      type: 'text',
+      text: `Sorry, I can only handle one of the following message types: ${supportedMessageTypes}`,
+    })
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  const downstreamConversationId = upstreamConversation.tags['downstream']
+  if (!downstreamConversationId) {
+    return await _abortHitlSession({
+      cm: upstreamCm,
+      internalReason: 'Upstream conversation was not bound to downstream conversation',
+      reasonShownToUser: 'Something went wrong, you are not connected to a human agent...',
+      props,
+    })
+  }
+
+  const patientUpstreamUser = await props.users.getById({ id: props.data.userId })
+
+  const patientDownstreamUserId = patientUpstreamUser.tags['downstream']
+  if (!patientDownstreamUserId) {
+    return await _abortHitlSession({
+      cm: upstreamCm,
+      internalReason: 'Upstream user was not bound to downstream user',
+      reasonShownToUser: 'Something went wrong, you are not connected to a human agent...',
+      props,
+    })
+  }
+
+  const downstreamConversation = await props.conversations.hitl.hitl.getById({ id: downstreamConversationId })
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+
+  if (_isHitlCloseCommand(props, sessionConfig)) {
+    await _handleHitlCloseCommand(props, { downstreamCm, upstreamCm, sessionConfig })
+
+    if (sessionConfig.flowOnHitlStopped) {
+      // the bot will continue the conversation without the patient having to send another message
+      await upstreamCm.continueWorkflow()
+    }
+
+    return consts.STOP_EVENT_HANDLING
+  }
+
+  props.logger.withConversationId(upstreamConversation.id).info('Sending message to downstream')
+  await downstreamCm.respond({ ...messagePayload, userId: patientDownstreamUserId }, { upstream: props.data.id })
+
+  return consts.STOP_EVENT_HANDLING
+}
+
+const _abortHitlSession = async ({
+  cm,
+  internalReason,
+  reasonShownToUser,
+  props,
+}: {
+  cm: conv.ConversationManager
+  internalReason: string
+  reasonShownToUser: string
+  props: bp.HookHandlerProps['before_incoming_message']
+}) => {
+  props.logger.withConversationId(cm.conversationId).error(internalReason)
+
+  await cm.abortHitlSession(reasonShownToUser)
+
+  return consts.STOP_EVENT_HANDLING
+}
+
+const _isHitlCloseCommand = (
+  props: bp.HookHandlerProps['before_incoming_message'],
+  sessionConfig: bp.configuration.Configuration
+) => {
+  const closeCommand = sessionConfig.userHitlCloseCommand?.length
+    ? sessionConfig.userHitlCloseCommand
+    : DEFAULT_USER_HITL_CLOSE_COMMAND
+
+  const inputText: string | undefined = props.data.payload.text
+  return inputText && inputText.trim().toLowerCase() === closeCommand.trim().toLowerCase()
+}
+
+const _handleHitlCloseCommand = async (
+  props: bp.HookHandlerProps['before_incoming_message'],
+  {
+    downstreamCm,
+    upstreamCm,
+    sessionConfig,
+  }: {
+    downstreamCm: conv.ConversationManager
+    upstreamCm: conv.ConversationManager
+    sessionConfig: bp.configuration.Configuration
+  }
+) => {
+  await downstreamCm.maybeRespondText(sessionConfig.onUserHitlCancelledMessage, DEFAULT_USER_HITL_CANCELLED_MESSAGE)
+
+  await Promise.allSettled([
+    upstreamCm.setHitlInactive(conv.HITL_END_REASON.PATIENT_USED_TERMINATION_COMMAND),
+    downstreamCm.setHitlInactive(conv.HITL_END_REASON.PATIENT_USED_TERMINATION_COMMAND),
+  ])
+
+  props.logger
+    .withConversationId(upstreamCm.conversationId)
+    .info('User ended the HITL session using the termination command')
+  props.logger
+    .withConversationId(downstreamCm.conversationId)
+    .info('User ended the HITL session using the termination command')
+
+  // Call stopHitl in the hitl integration (zendesk, etc.):
+  await props.actions.hitl.stopHitl({ conversationId: downstreamCm.conversationId })
+
+  await upstreamCm.maybeRespondText(sessionConfig.onUserHitlCloseMessage, DEFAULT_USER_HITL_COMMAND_MESSAGE)
+}

plugins/hitl/src/hooks/before-incoming-message/index.ts

@@ -0,0 +1 @@
+export * as all from './all'

plugins/hitl/src/hooks/consts.ts

@@ -0,0 +1,4 @@
+export * from '../consts'
+
+export const LET_BOT_HANDLE_EVENT = { stop: false } as const // let the event / message propagate to the bot
+export const STOP_EVENT_HANDLING = { stop: true } as const // prevent the event / message from propagating to the bot

plugins/hitl/src/hooks/index.ts

@@ -0,0 +1,2 @@
+export * as beforeIncomingMessage from './before-incoming-message'
+export * as beforeIncomingEvent from './before-incoming-event'

plugins/hitl/src/hooks/operations.ts

@@ -0,0 +1,58 @@
+import { DEFAULT_HUMAN_AGENT_ASSIGNED_MESSAGE } from '../../plugin.definition'
+import * as configuration from '../configuration'
+import * as conv from '../conv-manager'
+import * as types from '../types'
+import { tryLinkWebchatUser } from '../webchat'
+import * as bp from '.botpress'
+
+type AssignAgentProps = bp.HookHandlerProps['before_incoming_message'] | bp.HookHandlerProps['before_incoming_event']
+
+export type AssignAgentOptions = {
+  props: AssignAgentProps
+  downstreamConversation: types.ActionableConversation
+  humanAgentUserId: string
+  forceLinkWebchatUser?: boolean
+}
+
+/**
+ * Assigns a human agent to both downstream and upstream conversations.
+ * Also links the webchat user if applicable and sends the agent assigned message.
+ */
+export const assignAgent = async (options: AssignAgentOptions): Promise<boolean> => {
+  const { props, downstreamConversation, humanAgentUserId, forceLinkWebchatUser = true } = options
+
+  const upstreamConversationId = downstreamConversation.tags.upstream
+  if (!upstreamConversationId?.length) {
+    props.logger
+      .withConversationId(downstreamConversation.id)
+      .error('Downstream conversation was not binded to upstream conversation')
+    return false
+  }
+
+  const upstreamConversation = await props.conversations.hitl.hitl.getById({
+    id: upstreamConversationId,
+  })
+
+  const upstreamCm = conv.ConversationManager.from(props, upstreamConversation)
+  const downstreamCm = conv.ConversationManager.from(props, downstreamConversation)
+  const humanAgentUser = await props.users.getById({ id: humanAgentUserId })
+  const humanAgentName = humanAgentUser?.name?.length ? humanAgentUser.name : 'A Human Agent'
+
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId: upstreamConversation.id,
+  })
+
+  await Promise.all([
+    downstreamCm.setHumanAgent(humanAgentUserId, humanAgentName),
+    upstreamCm.setHumanAgent(humanAgentUserId, humanAgentName),
+    upstreamCm.maybeRespondText(sessionConfig.onHumanAgentAssignedMessage, DEFAULT_HUMAN_AGENT_ASSIGNED_MESSAGE),
+    tryLinkWebchatUser(props, {
+      downstreamUser: humanAgentUser,
+      upstreamConversation,
+      forceLink: forceLinkWebchatUser,
+    }),
+  ])
+
+  return true
+}

plugins/hitl/src/index.ts

@@ -0,0 +1,68 @@
+import * as sdk from '@botpress/sdk'
+import { isBrowser } from 'browser-or-node'
+import * as actions from './actions'
+import * as hooks from './hooks'
+import * as bp from '.botpress'
+
+const plugin = new bp.Plugin({
+  actions: {
+    startHitl: async (props) => {
+      if (isBrowser) {
+        throw new sdk.RuntimeError('HITL is not supported in the browser')
+      }
+      return await actions.startHitl(props)
+    },
+    stopHitl: async (props) => {
+      if (isBrowser) {
+        throw new sdk.RuntimeError('HITL is not supported in the browser')
+      }
+      return await actions.stopHitl(props)
+    },
+  },
+})
+
+plugin.on.beforeIncomingMessage('*', async (props) => {
+  if (isBrowser) {
+    props.logger.warn('HITL is not supported in the browser')
+    return
+  }
+  props.logger.info('Before incoming message', props.data.payload)
+  return await hooks.beforeIncomingMessage.all.handleMessage(props)
+})
+
+plugin.on.beforeIncomingEvent('hitl:hitlAssigned', async (props) => {
+  if (isBrowser) {
+    props.logger.warn('HITL is not supported in the browser')
+    return
+  }
+  props.logger.info('HITL assigned', props.data.payload)
+  return await hooks.beforeIncomingEvent.hitlAssigned.handleEvent(props)
+})
+
+plugin.on.beforeIncomingEvent('hitl:hitlStopped', async (props) => {
+  if (isBrowser) {
+    props.logger.warn('HITL is not supported in the browser')
+    return
+  }
+  props.logger.info('HITL stopped', props.data.payload)
+  return await hooks.beforeIncomingEvent.hitlStopped.handleEvent(props)
+})
+
+plugin.on.beforeIncomingEvent('humanAgentAssignedTimeout', async (props) => {
+  if (isBrowser) {
+    props.logger.warn('HITL is not supported in the browser')
+    return
+  }
+  props.logger.info('HITL agent assigned timeout', props.data.payload)
+  return await hooks.beforeIncomingEvent.humanAgentAssignedTimeout.handleEvent(props)
+})
+
+plugin.on.beforeIncomingEvent('*', async (props) => {
+  if (isBrowser) {
+    return
+  }
+  props.logger.info('Before incoming event', props.data.payload)
+  return await hooks.beforeIncomingEvent.all.handleEvent(props)
+})
+
+export default plugin

plugins/hitl/src/types.ts

@@ -0,0 +1,32 @@
+import type * as sdk from '@botpress/sdk'
+import * as consts from './consts'
+import * as bp from '.botpress'
+
+export type AnyHandlerProps =
+  | bp.MessageHandlerProps
+  | bp.EventHandlerProps
+  | bp.ActionHandlerProps
+  | bp.HookHandlerProps['before_incoming_message']
+  | bp.HookHandlerProps['before_incoming_event']
+
+export type ValueOf<T> = T[Extract<keyof T, string>]
+type ArrayToUnion<T> = T extends Array<infer U> ? U : never
+
+export type SupportedMessageTypes = ArrayToUnion<typeof consts.SUPPORTED_MESSAGE_TYPES>
+type BaseMessagePayloads = Pick<typeof sdk.messages.defaults, SupportedMessageTypes>
+export type MessagePayload = {
+  [TMsgType in keyof BaseMessagePayloads]: {
+    type: TMsgType
+    userId?: string
+  } & sdk.z.infer<BaseMessagePayloads[TMsgType]['schema']>
+}[keyof BaseMessagePayloads]
+
+export type MessageTags = {
+  [T in keyof bp.message.Message['tags']]?: string
+}
+
+export type ActionableConversation = NonNullable<
+  Awaited<ReturnType<bp.ActionHandlerProps['conversations']['hitl']['hitl']['getById']>>
+>
+export type ActionableMessage = NonNullable<Awaited<ReturnType<bp.ActionHandlerProps['messages']['getById']>>>
+export type ActionableUser = NonNullable<Awaited<ReturnType<bp.ActionHandlerProps['users']['getById']>>>

plugins/hitl/src/user-linker.ts

@@ -0,0 +1,88 @@
+import * as types from './types'
+
+export type UserOverrides = { name?: string; email?: string; pictureUrl?: string }
+
+export class UserLinker {
+  public constructor(
+    private _props: types.AnyHandlerProps,
+    private _users: Record<string, types.ActionableUser> = {}
+  ) {}
+
+  public async getDownstreamUserId(upstreamUserId: string, upstreamUserOverrides?: UserOverrides): Promise<string> {
+    let upstreamUser = this._users[upstreamUserId]
+    if (!upstreamUser) {
+      const fetchedUser = await this._props.users.getById({ id: upstreamUserId })
+      this._users[upstreamUserId] = fetchedUser
+      upstreamUser = fetchedUser
+    }
+
+    const existingDownstreamUserId = await this._getExistingDownstreamUserId(upstreamUser)
+
+    if (existingDownstreamUserId !== null) {
+      return existingDownstreamUserId
+    }
+
+    const {
+      downstreamUser: { id: downstreamUserId },
+      upstreamUser: updatedUpstreamUser,
+    } = await this._linkUser(upstreamUser, upstreamUserOverrides)
+
+    this._users[upstreamUserId] = updatedUpstreamUser
+
+    return downstreamUserId
+  }
+
+  private async _getExistingDownstreamUserId(upstreamUser: types.ActionableUser) {
+    const downstreamUserId = upstreamUser?.tags?.downstream
+
+    if (!downstreamUserId || upstreamUser?.tags?.integrationName !== this._props.interfaces.hitl.name) {
+      return null
+    }
+
+    try {
+      await this._props.users.getById({ id: downstreamUserId })
+    } catch {
+      return null
+    }
+
+    return downstreamUserId
+  }
+
+  private async _linkUser(upstreamUser: types.ActionableUser, upstreamUserOverrides?: UserOverrides) {
+    // To access bot-level tags:
+    const untypedUserTags: Record<string, string> = upstreamUser.tags
+
+    // Call createUser in the hitl integration (zendesk, etc.):
+    const { userId: downstreamUserId } = await this._props.actions.hitl.createUser({
+      name: upstreamUserOverrides?.name ?? untypedUserTags['name'] ?? upstreamUser.name ?? 'Unknown User',
+      pictureUrl: upstreamUserOverrides?.pictureUrl ?? untypedUserTags['pictureUrl'] ?? upstreamUser.pictureUrl,
+      email: upstreamUserOverrides?.email ?? untypedUserTags['email'] ?? this._generateFakeEmail(upstreamUser),
+    })
+    const downstreamUser = await this._props.users.getById({ id: downstreamUserId })
+
+    const [updatedUpstreamUser, updatedDownstreamUser] = await Promise.all([
+      upstreamUser.update({
+        tags: {
+          downstream: downstreamUserId,
+          integrationName: this._props.interfaces.hitl.name,
+        },
+      }),
+      downstreamUser.update({
+        tags: {
+          upstream: upstreamUser.id,
+          integrationName: this._props.interfaces.hitl.name,
+        },
+      }),
+    ])
+
+    return {
+      upstreamUser: updatedUpstreamUser,
+      downstreamUser: updatedDownstreamUser,
+    }
+  }
+
+  private _generateFakeEmail(user: types.ActionableUser) {
+    const botId = this._props.ctx.botId.replaceAll('_', '-')
+    return `${user.id}@no-reply.${botId}.botpress.com`
+  }
+}

plugins/hitl/src/webchat.ts

@@ -0,0 +1,93 @@
+import * as configuration from './configuration'
+import type * as types from './types'
+import * as bp from '.botpress'
+
+type WebchatGetOrCreateUserInput = {
+  name?: string
+  pictureUrl?: string
+  email?: string
+  user: {
+    id: string
+    conversationId: string
+  }
+}
+
+export type TryLinkWebchatUserOptions = {
+  downstreamUser: types.ActionableUser
+  upstreamConversation: types.ActionableConversation
+  forceLink?: boolean
+}
+
+/**
+ * This functions tries to create a copy of the downstream user in the upstream system.
+ * The goal is for the patient to have the illusion that they receive messages from a human agent instead of the bot.
+ *
+ * This only works when the hitl frontend is webchat.
+ * Currently, this dependency of hitl plugin on webchat integration is not declared.
+ * If it fails, the plugin still works, but patients will see messages as sent by the bot.
+ *
+ * This workaround is extremely flaky. If we need this feature in another integration, we should do it properly.
+ *
+ * @param props hook handler props
+ * @param upstreamConversationId the upstream conversation id
+ * @returns the fake upstream user id that the bot pretends to be when sending messages
+ */
+export const tryLinkWebchatUser = async (
+  props: bp.HookHandlerProps['before_incoming_message'] | bp.HookHandlerProps['before_incoming_event'],
+  { forceLink, downstreamUser, upstreamConversation }: TryLinkWebchatUserOptions
+): Promise<string | undefined> => {
+  const sessionConfig = await configuration.retrieveSessionConfig({
+    ...props,
+    upstreamConversationId: upstreamConversation.id,
+  })
+
+  const { integration: upstreamIntegration } = upstreamConversation
+
+  if (upstreamIntegration !== 'webchat' || !sessionConfig.useHumanAgentInfo) {
+    // this only works when the hitl frontend is webchat
+    return undefined
+  }
+
+  try {
+    props.logger.info(
+      `Trying to link downstream user ${downstreamUser.id} to upstream conversation ${upstreamConversation.id}`
+    )
+
+    const upstreamUserId = downstreamUser.tags['upstream']
+    if (upstreamUserId && !forceLink) {
+      // the user is already linked
+      return upstreamUserId
+    }
+
+    // To access bot-level tags:
+    const untypedDownstreamUserTags: Record<string, string> = downstreamUser.tags
+
+    const { output } = await props.client.callAction({
+      type: 'webchat:getOrCreateUser',
+      input: {
+        name: downstreamUser.name,
+        pictureUrl: downstreamUser.pictureUrl,
+        email: untypedDownstreamUserTags['email'],
+        user: {
+          id: downstreamUser.id,
+          conversationId: upstreamConversation.id,
+        },
+      } as WebchatGetOrCreateUserInput,
+    })
+
+    const { userId } = output as Record<string, unknown>
+    if (typeof userId !== 'string') {
+      return undefined
+    }
+
+    await downstreamUser.update({
+      tags: {
+        upstream: userId,
+      },
+    })
+
+    return userId
+  } catch {
+    return undefined
+  }
+}

plugins/hitl/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "paths": { "*": ["./*"] },
+    "outDir": "dist"
+  },
+  "include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
+}