simstudioai--sim/apps/sim/background/knowledge-processing.ts

import { createLogger } from '@sim/logger'
import { task } from '@trigger.dev/sdk'
import { env, envNumber } from '@/lib/core/config/env'
import { processDocumentAsync } from '@/lib/knowledge/documents/service'

const logger = createLogger('TriggerKnowledgeProcessing')

export type DocumentProcessingPayload = {
  knowledgeBaseId: string
  documentId: string
  docData: {
    filename: string
    fileUrl: string
    fileSize: number
    mimeType: string
  }
  processingOptions: {
    recipe?: string
    lang?: string
  }
  requestId: string
}

export const processDocument = task({
  id: 'knowledge-process-document',
  maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
  machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
  retry: {
    maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
    factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
    minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
    maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
  },
  queue: {
    concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
    name: 'document-processing-queue',
  },
  run: async (payload: DocumentProcessingPayload) => {
    const { knowledgeBaseId, documentId, docData, processingOptions, requestId } = payload

    logger.info(`[${requestId}] Starting Trigger.dev processing for document: ${docData.filename}`)

    try {
      await processDocumentAsync(knowledgeBaseId, documentId, docData, processingOptions)

      logger.info(`[${requestId}] Successfully processed document: ${docData.filename}`)

      return {
        success: true,
        documentId,
        filename: docData.filename,
        processingTime: Date.now(),
      }
    } catch (error) {
      logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
      throw error
    }
  },
})