chore: import upstream snapshot with attribution

apps/sim/lib/knowledge/documents/document-processor.ts

@@ -0,0 +1,859 @@
+import { randomBytes } from 'crypto'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { PDFDocument } from 'pdf-lib'
+import { getBYOKKey } from '@/lib/api-key/byok'
+import {
+  type Chunk,
+  JsonYamlChunker,
+  RecursiveChunker,
+  RegexChunker,
+  SentenceChunker,
+  StructuredDataChunker,
+  TextChunker,
+  TokenChunker,
+} from '@/lib/chunkers'
+import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
+import { env, envNumber } from '@/lib/core/config/env'
+import { parseBuffer } from '@/lib/file-parsers'
+import type { FileParseMetadata } from '@/lib/file-parsers/types'
+import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
+import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
+import { StorageService } from '@/lib/uploads'
+import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
+import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
+import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
+import { mistralParserTool } from '@/tools/mistral/parser'
+
+const logger = createLogger('DocumentProcessor')
+
+const TIMEOUTS = {
+  FILE_DOWNLOAD: 600000,
+  MISTRAL_OCR_API: 120000,
+} as const
+
+const MAX_CONCURRENT_CHUNKS = envNumber(env.KB_CONFIG_CHUNK_CONCURRENCY, 10)
+
+type OCRResult = {
+  success: boolean
+  error?: string
+  output?: {
+    content?: string
+  }
+}
+
+type OCRPage = {
+  markdown?: string
+}
+
+type OCRRequestBody = {
+  model: string
+  document: {
+    type: string
+    document_url: string
+  }
+  include_image_base64: boolean
+}
+
+const MISTRAL_MAX_PAGES = 1000
+
+async function getPdfPageCount(buffer: Buffer): Promise<number> {
+  try {
+    const { getDocumentProxy } = await import('unpdf')
+    const uint8Array = new Uint8Array(buffer)
+    const pdf = await getDocumentProxy(uint8Array)
+    return pdf.numPages
+  } catch (error) {
+    logger.warn('Failed to get PDF page count:', error)
+    return 0
+  }
+}
+
+async function splitPdfIntoChunks(
+  pdfBuffer: Buffer,
+  maxPages: number
+): Promise<{ buffer: Buffer; startPage: number; endPage: number }[]> {
+  const sourcePdf = await PDFDocument.load(pdfBuffer)
+  const totalPages = sourcePdf.getPageCount()
+
+  if (totalPages <= maxPages) {
+    return [{ buffer: pdfBuffer, startPage: 0, endPage: totalPages - 1 }]
+  }
+
+  const chunks: { buffer: Buffer; startPage: number; endPage: number }[] = []
+
+  for (let startPage = 0; startPage < totalPages; startPage += maxPages) {
+    const endPage = Math.min(startPage + maxPages - 1, totalPages - 1)
+    const pageCount = endPage - startPage + 1
+
+    const newPdf = await PDFDocument.create()
+    const pageIndices = Array.from({ length: pageCount }, (_, i) => startPage + i)
+    const copiedPages = await newPdf.copyPages(sourcePdf, pageIndices)
+
+    copiedPages.forEach((page) => newPdf.addPage(page))
+
+    const pdfBytes = await newPdf.save()
+    chunks.push({
+      buffer: Buffer.from(pdfBytes),
+      startPage,
+      endPage,
+    })
+  }
+
+  return chunks
+}
+
+type AzureOCRResponse = {
+  pages?: OCRPage[]
+  [key: string]: unknown
+}
+
+class APIError extends Error {
+  public status: number
+
+  constructor(message: string, status: number) {
+    super(message)
+    this.name = 'APIError'
+    this.status = status
+  }
+}
+
+async function applyStrategy(
+  strategy: ChunkingStrategy,
+  content: string,
+  chunkSize: number,
+  chunkOverlap: number,
+  minCharactersPerChunk: number,
+  strategyOptions?: StrategyOptions
+): Promise<Chunk[]> {
+  const baseOptions = { chunkSize, chunkOverlap, minCharactersPerChunk }
+
+  switch (strategy) {
+    case 'token': {
+      const chunker = new TokenChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+    case 'sentence': {
+      const chunker = new SentenceChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+    case 'recursive': {
+      const chunker = new RecursiveChunker({
+        ...baseOptions,
+        separators: strategyOptions?.separators,
+        recipe: strategyOptions?.recipe,
+      })
+      return chunker.chunk(content)
+    }
+    case 'regex': {
+      if (!strategyOptions?.pattern) {
+        logger.warn(
+          'Regex strategy requested but no pattern provided, falling back to text chunker'
+        )
+        const chunker = new TextChunker(baseOptions)
+        return chunker.chunk(content)
+      }
+      const chunker = new RegexChunker({
+        ...baseOptions,
+        pattern: strategyOptions.pattern,
+        strictBoundaries: strategyOptions.strictBoundaries,
+      })
+      return chunker.chunk(content)
+    }
+    default: {
+      const chunker = new TextChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+  }
+}
+
+export async function processDocument(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  chunkSize = 1024,
+  chunkOverlap = 200,
+  minCharactersPerChunk = 100,
+  userId?: string,
+  workspaceId?: string | null,
+  strategy?: ChunkingStrategy,
+  strategyOptions?: StrategyOptions
+): Promise<{
+  chunks: Chunk[]
+  metadata: {
+    filename: string
+    fileSize: number
+    mimeType: string
+    chunkCount: number
+    tokenCount: number
+    characterCount: number
+    processingMethod: 'file-parser' | 'mistral-ocr'
+    cloudUrl?: string
+  }
+}> {
+  logger.info(`Processing document: ${filename}`)
+
+  try {
+    const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId)
+    const { content, processingMethod } = parseResult
+    const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined
+
+    let chunks: Chunk[]
+    const metadata: FileParseMetadata = parseResult.metadata ?? {}
+
+    if (strategy && strategy !== 'auto') {
+      logger.info(`Using explicit chunking strategy: ${strategy}`)
+      chunks = await applyStrategy(
+        strategy,
+        content,
+        chunkSize,
+        chunkOverlap,
+        minCharactersPerChunk,
+        strategyOptions
+      )
+    } else {
+      const isJsonYaml =
+        metadata.type === 'json' ||
+        metadata.type === 'yaml' ||
+        mimeType.includes('json') ||
+        mimeType.includes('yaml')
+
+      if (isJsonYaml && JsonYamlChunker.isStructuredData(content)) {
+        logger.info('Using JSON/YAML chunker for structured data')
+        chunks = await JsonYamlChunker.chunkJsonYaml(content, {
+          chunkSize,
+          minCharactersPerChunk,
+        })
+      } else if (StructuredDataChunker.isStructuredData(content, mimeType)) {
+        logger.info('Using structured data chunker for spreadsheet/CSV content')
+        const rowCount = metadata.totalRows ?? metadata.rowCount
+        chunks = await StructuredDataChunker.chunkStructuredData(content, {
+          chunkSize,
+          headers: metadata.headers,
+          totalRows: typeof rowCount === 'number' ? rowCount : undefined,
+          sheetName: metadata.sheetNames?.[0],
+        })
+      } else {
+        const chunker = new TextChunker({ chunkSize, chunkOverlap, minCharactersPerChunk })
+        chunks = await chunker.chunk(content)
+      }
+    }
+
+    const characterCount = content.length
+    const tokenCount = chunks.reduce((sum, chunk) => sum + chunk.tokenCount, 0)
+
+    logger.info(`Document processed: ${chunks.length} chunks, ${tokenCount} tokens`)
+
+    return {
+      chunks,
+      metadata: {
+        filename,
+        fileSize: characterCount,
+        mimeType,
+        chunkCount: chunks.length,
+        tokenCount,
+        characterCount,
+        processingMethod,
+        cloudUrl,
+      },
+    }
+  } catch (error) {
+    logger.error(`Error processing document ${filename}:`, error)
+    throw error
+  }
+}
+
+async function getMistralApiKey(workspaceId?: string | null): Promise<string | null> {
+  if (workspaceId) {
+    const byokResult = await getBYOKKey(workspaceId, 'mistral')
+    if (byokResult) {
+      logger.info('Using workspace BYOK key for Mistral OCR')
+      return byokResult.apiKey
+    }
+  }
+  return env.MISTRAL_API_KEY || null
+}
+
+async function parseDocument(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null
+): Promise<{
+  content: string
+  processingMethod: 'file-parser' | 'mistral-ocr'
+  cloudUrl?: string
+  metadata?: FileParseMetadata
+}> {
+  const isPDF = mimeType === 'application/pdf'
+  const hasAzureMistralOCR =
+    env.OCR_AZURE_API_KEY && env.OCR_AZURE_ENDPOINT && env.OCR_AZURE_MODEL_NAME
+
+  const mistralApiKey = await getMistralApiKey(workspaceId)
+  const hasMistralOCR = !!mistralApiKey
+
+  if (isPDF && (hasAzureMistralOCR || hasMistralOCR)) {
+    if (hasAzureMistralOCR) {
+      logger.info(`Using Azure Mistral OCR: ${filename}`)
+      return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId)
+    }
+
+    if (hasMistralOCR) {
+      logger.info(`Using Mistral OCR: ${filename}`)
+      return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey)
+    }
+  }
+
+  logger.info(`Using file parser: ${filename}`)
+  return parseWithFileParser(fileUrl, filename, mimeType, userId)
+}
+
+async function handleFileForOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null
+) {
+  const isExternalHttps = /^https:\/\//i.test(fileUrl) && !isInternalFileUrl(fileUrl)
+
+  if (isExternalHttps) {
+    if (mimeType === 'application/pdf') {
+      logger.info(`handleFileForOCR: Downloading external PDF to check page count`)
+      try {
+        const buffer = await downloadFileWithTimeout(fileUrl, userId)
+        logger.info(`handleFileForOCR: Downloaded external PDF: ${buffer.length} bytes`)
+        return { httpsUrl: fileUrl, buffer }
+      } catch (error) {
+        logger.warn(
+          `handleFileForOCR: Failed to download external PDF for page count check, proceeding without batching`,
+          {
+            error: toError(error).message,
+          }
+        )
+        return { httpsUrl: fileUrl, buffer: undefined }
+      }
+    }
+    logger.info(`handleFileForOCR: Using external URL directly`)
+    return { httpsUrl: fileUrl, buffer: undefined }
+  }
+
+  logger.info(`Uploading "${filename}" to cloud storage for OCR`)
+
+  const buffer = await downloadFileWithTimeout(fileUrl, userId)
+
+  logger.info(`Downloaded ${filename}: ${buffer.length} bytes`)
+
+  try {
+    const metadata: Record<string, string> = {
+      originalName: filename,
+      uploadedAt: new Date().toISOString(),
+      purpose: 'knowledge-base',
+      ...(userId && { userId }),
+      ...(workspaceId && { workspaceId }),
+    }
+
+    const timestamp = Date.now()
+    const uniqueId = randomBytes(8).toString('hex')
+    const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
+    const customKey = `kb/${timestamp}-${uniqueId}-${safeFileName}`
+
+    const cloudResult = await StorageService.uploadFile({
+      file: buffer,
+      fileName: filename,
+      contentType: mimeType,
+      context: 'knowledge-base',
+      customKey,
+      metadata,
+    })
+
+    const httpsUrl = await StorageService.generatePresignedDownloadUrl(
+      cloudResult.key,
+      'knowledge-base',
+      900 // 15 minutes
+    )
+
+    return { httpsUrl, cloudUrl: httpsUrl, buffer }
+  } catch (uploadError) {
+    const message = getErrorMessage(uploadError, 'Unknown error')
+    throw new Error(`Cloud upload failed: ${message}. Cloud upload is required for OCR.`)
+  }
+}
+
+/**
+ * Downloads an ingestion source file, enforcing the {@link MAX_FILE_SIZE} document
+ * limit. `maxBytes` aborts the streaming read once the cap is exceeded (and rejects
+ * up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl`
+ * pointing at an unbounded body cannot exhaust the processing worker's memory.
+ */
+async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise<Buffer> {
+  return downloadFileFromUrl(fileUrl, {
+    timeoutMs: TIMEOUTS.FILE_DOWNLOAD,
+    maxBytes: MAX_FILE_SIZE,
+    userId,
+  })
+}
+
+async function downloadFileForBase64(fileUrl: string, userId?: string): Promise<Buffer> {
+  if (/^data:/i.test(fileUrl)) {
+    const [, base64Data] = fileUrl.split(',')
+    if (!base64Data) {
+      throw new Error('Invalid data URI format')
+    }
+    return Buffer.from(base64Data, 'base64')
+  }
+  if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) {
+    return downloadFileWithTimeout(fileUrl, userId)
+  }
+  throw new Error(
+    'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed'
+  )
+}
+
+function processOCRContent(result: OCRResult, filename: string): string {
+  if (!result.success) {
+    throw new Error(`OCR processing failed: ${result.error || 'Unknown error'}`)
+  }
+
+  const content = result.output?.content || ''
+  if (!content.trim()) {
+    throw new Error('OCR returned empty content')
+  }
+
+  logger.info(`OCR completed: ${filename}`)
+  return content
+}
+
+function validateOCRConfig(
+  apiKey?: string,
+  endpoint?: string,
+  modelName?: string,
+  service = 'OCR'
+) {
+  if (!apiKey) throw new Error(`${service} API key required`)
+  if (!endpoint) throw new Error(`${service} endpoint required`)
+  if (!modelName) throw new Error(`${service} model name required`)
+}
+
+function extractPageContent(pages: OCRPage[]): string {
+  if (!pages?.length) return ''
+
+  return pages
+    .map((page) => page?.markdown || '')
+    .filter(Boolean)
+    .join('\n\n')
+}
+
+async function makeOCRRequest(
+  endpoint: string,
+  headers: Record<string, string>,
+  body: OCRRequestBody
+): Promise<Response> {
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), TIMEOUTS.MISTRAL_OCR_API)
+
+  try {
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    })
+
+    clearTimeout(timeoutId)
+
+    if (!response.ok) {
+      const errorText = await response.text()
+      throw new APIError(
+        `OCR failed: ${response.status} ${response.statusText} - ${errorText}`,
+        response.status
+      )
+    }
+
+    return response
+  } catch (error) {
+    clearTimeout(timeoutId)
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error('OCR API request timed out')
+    }
+    throw error
+  }
+}
+
+async function parseWithAzureMistralOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string
+) {
+  validateOCRConfig(
+    env.OCR_AZURE_API_KEY,
+    env.OCR_AZURE_ENDPOINT,
+    env.OCR_AZURE_MODEL_NAME,
+    'Azure Mistral OCR'
+  )
+
+  const fileBuffer = await downloadFileForBase64(fileUrl, userId)
+
+  if (mimeType === 'application/pdf') {
+    const pageCount = await getPdfPageCount(fileBuffer)
+    if (pageCount > MISTRAL_MAX_PAGES) {
+      logger.info(
+        `PDF has ${pageCount} pages, exceeds Azure OCR limit of ${MISTRAL_MAX_PAGES}. ` +
+          `Falling back to file parser.`
+      )
+      return parseWithFileParser(fileUrl, filename, mimeType, userId)
+    }
+    logger.info(`Azure Mistral OCR: PDF page count for ${filename}: ${pageCount}`)
+  }
+
+  const base64Data = fileBuffer.toString('base64')
+  const dataUri = `data:${mimeType};base64,${base64Data}`
+
+  try {
+    const response = await retryWithExponentialBackoff(
+      () =>
+        makeOCRRequest(
+          env.OCR_AZURE_ENDPOINT!,
+          {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`,
+          },
+          {
+            model: env.OCR_AZURE_MODEL_NAME!,
+            document: {
+              type: 'document_url',
+              document_url: dataUri,
+            },
+            include_image_base64: false,
+          }
+        ),
+      { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
+    )
+
+    const ocrResult = (await response.json()) as AzureOCRResponse
+    const content = extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2)
+
+    if (!content.trim()) {
+      throw new Error('Azure Mistral OCR returned empty content')
+    }
+
+    logger.info(`Azure Mistral OCR completed: ${filename}`)
+    return { content, processingMethod: 'mistral-ocr' as const, cloudUrl: undefined }
+  } catch (error) {
+    logger.error(`Azure Mistral OCR failed for ${filename}:`, {
+      message: toError(error).message,
+    })
+
+    logger.info(`Falling back to file parser: ${filename}`)
+    return parseWithFileParser(fileUrl, filename, mimeType, userId)
+  }
+}
+
+async function parseWithMistralOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null,
+  mistralApiKey?: string | null
+) {
+  const apiKey = mistralApiKey || env.MISTRAL_API_KEY
+  if (!apiKey) {
+    throw new Error('Mistral API key required')
+  }
+
+  if (!mistralParserTool.request?.body) {
+    throw new Error('Mistral parser tool not configured')
+  }
+
+  const { httpsUrl, cloudUrl, buffer } = await handleFileForOCR(
+    fileUrl,
+    filename,
+    mimeType,
+    userId,
+    workspaceId
+  )
+
+  logger.info(`Mistral OCR: Using presigned URL for ${filename}: ${httpsUrl}`)
+
+  let pageCount = 0
+  if (mimeType === 'application/pdf' && buffer) {
+    pageCount = await getPdfPageCount(buffer)
+    logger.info(`PDF page count for ${filename}: ${pageCount}`)
+  }
+
+  const needsBatching = pageCount > MISTRAL_MAX_PAGES
+
+  if (needsBatching && buffer) {
+    logger.info(
+      `PDF has ${pageCount} pages, exceeds limit of ${MISTRAL_MAX_PAGES}. Splitting and processing in chunks.`
+    )
+    return processMistralOCRInBatches(filename, apiKey, buffer, userId, cloudUrl)
+  }
+
+  const params = { filePath: httpsUrl, apiKey, resultType: 'text' as const }
+
+  try {
+    const response = await executeMistralOCRRequest(params, userId)
+    const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult
+    const content = processOCRContent(result, filename)
+
+    return { content, processingMethod: 'mistral-ocr' as const, cloudUrl }
+  } catch (error) {
+    logger.error(`Mistral OCR failed for ${filename}:`, {
+      message: toError(error).message,
+    })
+
+    logger.info(`Falling back to file parser: ${filename}`)
+    return parseWithFileParser(fileUrl, filename, mimeType, userId)
+  }
+}
+
+async function executeMistralOCRRequest(
+  params: { filePath: string; apiKey: string; resultType: 'text' },
+  userId?: string
+): Promise<Response> {
+  return retryWithExponentialBackoff(
+    async () => {
+      let url =
+        typeof mistralParserTool.request!.url === 'function'
+          ? mistralParserTool.request!.url(params)
+          : mistralParserTool.request!.url
+
+      const isInternalRoute = url.startsWith('/')
+
+      if (isInternalRoute) {
+        const { getInternalApiBaseUrl } = await import('@/lib/core/utils/urls')
+        url = `${getInternalApiBaseUrl()}${url}`
+      }
+
+      let headers =
+        typeof mistralParserTool.request!.headers === 'function'
+          ? mistralParserTool.request!.headers(params)
+          : mistralParserTool.request!.headers
+
+      if (isInternalRoute) {
+        const { generateInternalToken } = await import('@/lib/auth/internal')
+        const internalToken = await generateInternalToken(userId)
+        headers = {
+          ...headers,
+          Authorization: `Bearer ${internalToken}`,
+        }
+      }
+
+      const requestBody = mistralParserTool.request!.body!(params) as OCRRequestBody
+      return makeOCRRequest(url, headers as Record<string, string>, requestBody)
+    },
+    { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
+  )
+}
+
+async function processChunk(
+  chunk: { buffer: Buffer; startPage: number; endPage: number },
+  chunkIndex: number,
+  totalChunks: number,
+  filename: string,
+  apiKey: string,
+  userId?: string
+): Promise<{ index: number; content: string | null }> {
+  const chunkPageCount = chunk.endPage - chunk.startPage + 1
+
+  logger.info(
+    `Processing chunk ${chunkIndex + 1}/${totalChunks} (pages ${chunk.startPage + 1}-${chunk.endPage + 1}, ${chunkPageCount} pages)`
+  )
+
+  let uploadedKey: string | null = null
+
+  try {
+    const timestamp = Date.now()
+    const uniqueId = randomBytes(8).toString('hex')
+    const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
+    const chunkKey = `kb/${timestamp}-${uniqueId}-chunk${chunkIndex + 1}-${safeFileName}`
+
+    // No metadata: these chunks are ephemeral OCR artifacts (deleted in the
+    // finally below) that are fetched via a direct presigned URL, never through
+    // verifyKBFileAccess. Omitting metadata avoids writing an orphan ownership
+    // binding row per chunk.
+    const uploadResult = await StorageService.uploadFile({
+      file: chunk.buffer,
+      fileName: `${filename}_chunk${chunkIndex + 1}`,
+      contentType: 'application/pdf',
+      context: 'knowledge-base',
+      customKey: chunkKey,
+    })
+
+    uploadedKey = uploadResult.key
+
+    const chunkUrl = await StorageService.generatePresignedDownloadUrl(
+      uploadResult.key,
+      'knowledge-base',
+      900 // 15 minutes
+    )
+
+    logger.info(`Uploaded chunk ${chunkIndex + 1} to S3: ${chunkKey}`)
+
+    const params = {
+      filePath: chunkUrl,
+      apiKey,
+      resultType: 'text' as const,
+    }
+
+    const response = await executeMistralOCRRequest(params, userId)
+    const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult
+
+    if (result.success && result.output?.content) {
+      logger.info(`Chunk ${chunkIndex + 1}/${totalChunks} completed successfully`)
+      return { index: chunkIndex, content: result.output.content }
+    }
+    logger.warn(`Chunk ${chunkIndex + 1}/${totalChunks} returned no content`)
+    return { index: chunkIndex, content: null }
+  } catch (error) {
+    logger.error(`Chunk ${chunkIndex + 1}/${totalChunks} failed:`, {
+      message: toError(error).message,
+    })
+    return { index: chunkIndex, content: null }
+  } finally {
+    if (uploadedKey) {
+      try {
+        await StorageService.deleteFile({ key: uploadedKey, context: 'knowledge-base' })
+        logger.info(`Cleaned up chunk ${chunkIndex + 1} from S3`)
+      } catch (deleteError) {
+        logger.warn(`Failed to clean up chunk ${chunkIndex + 1} from S3:`, {
+          message: toError(deleteError).message,
+        })
+      }
+    }
+  }
+}
+
+async function processMistralOCRInBatches(
+  filename: string,
+  apiKey: string,
+  pdfBuffer: Buffer,
+  userId?: string,
+  cloudUrl?: string
+): Promise<{
+  content: string
+  processingMethod: 'mistral-ocr'
+  cloudUrl?: string
+}> {
+  const totalPages = await getPdfPageCount(pdfBuffer)
+  logger.info(
+    `Splitting ${filename} (${totalPages} pages) into chunks of ${MISTRAL_MAX_PAGES} pages`
+  )
+
+  const pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES)
+  logger.info(
+    `Split into ${pdfChunks.length} chunks, processing with concurrency ${MAX_CONCURRENT_CHUNKS}`
+  )
+
+  const results: { index: number; content: string | null }[] = []
+
+  for (let i = 0; i < pdfChunks.length; i += MAX_CONCURRENT_CHUNKS) {
+    const batch = pdfChunks.slice(i, i + MAX_CONCURRENT_CHUNKS)
+    const batchPromises = batch.map((chunk, batchIndex) =>
+      processChunk(chunk, i + batchIndex, pdfChunks.length, filename, apiKey, userId)
+    )
+
+    const batchResults = await Promise.all(batchPromises)
+    for (const result of batchResults) {
+      results.push(result)
+    }
+
+    logger.info(
+      `Completed batch ${Math.floor(i / MAX_CONCURRENT_CHUNKS) + 1}/${Math.ceil(pdfChunks.length / MAX_CONCURRENT_CHUNKS)}`
+    )
+  }
+
+  const sortedResults = results
+    .sort((a, b) => a.index - b.index)
+    .filter((r) => r.content !== null)
+    .map((r) => r.content as string)
+
+  if (sortedResults.length === 0) {
+    throw new Error(
+      `OCR failed for all ${pdfChunks.length} chunks of ${filename}. ` +
+        `Large PDFs require OCR - file parser fallback would produce poor results.`
+    )
+  }
+
+  const combinedContent = sortedResults.join('\n\n')
+  logger.info(
+    `Successfully processed ${sortedResults.length}/${pdfChunks.length} chunks for ${filename}`
+  )
+
+  return {
+    content: combinedContent,
+    processingMethod: 'mistral-ocr',
+    cloudUrl,
+  }
+}
+
+async function parseWithFileParser(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string
+) {
+  try {
+    let content: string
+    let metadata: FileParseMetadata = {}
+
+    if (/^data:/i.test(fileUrl)) {
+      content = await parseDataURI(fileUrl, filename, mimeType)
+    } else if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) {
+      // Internal URLs may arrive as an app-relative `/api/files/serve/...` path
+      // (some ingestion callers store the relative path); downloadFileFromUrl
+      // resolves it directly against storage without an absolute origin.
+      const result = await parseHttpFile(fileUrl, filename, mimeType, userId)
+      content = result.content
+      metadata = result.metadata || {}
+    } else {
+      throw new Error(
+        'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed'
+      )
+    }
+
+    if (!content.trim()) {
+      throw new Error('File parser returned empty content')
+    }
+
+    return { content, processingMethod: 'file-parser' as const, cloudUrl: undefined, metadata }
+  } catch (error) {
+    logger.error(`File parser failed for ${filename}:`, error)
+    throw error
+  }
+}
+
+async function parseDataURI(fileUrl: string, filename: string, mimeType: string): Promise<string> {
+  const [header, base64Data] = fileUrl.split(',')
+  if (!base64Data) {
+    throw new Error('Invalid data URI format')
+  }
+
+  if (mimeType === 'text/plain') {
+    return header.includes('base64')
+      ? Buffer.from(base64Data, 'base64').toString('utf8')
+      : decodeURIComponent(base64Data)
+  }
+
+  const extension = resolveParserExtension(filename, mimeType, 'txt')
+  const buffer = Buffer.from(base64Data, 'base64')
+  const result = await parseBuffer(buffer, extension)
+  return result.content
+}
+
+async function parseHttpFile(
+  fileUrl: string,
+  filename: string,
+  mimeType?: string,
+  userId?: string
+): Promise<{ content: string; metadata?: FileParseMetadata }> {
+  const buffer = await downloadFileWithTimeout(fileUrl, userId)
+
+  const extension = resolveParserExtension(filename, mimeType)
+  const result = await parseBuffer(buffer, extension)
+  return result
+}

apps/sim/lib/knowledge/documents/lock-order.test.ts

@@ -0,0 +1,44 @@
+/**
+ * @vitest-environment node
+ *
+ * Lock-order regression guard: `updateDocument` must lock the document's
+ * embedding rows BEFORE the document row when cascading tag updates, matching
+ * the embedding → document order every chunk-mutation path uses
+ * (chunks/service.ts). The opposite order deadlocks a document tag edit against
+ * a concurrent chunk edit of the same document.
+ */
+import { document, embedding } from '@sim/db/schema'
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { updateDocument } from '@/lib/knowledge/documents/service'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+/** invocationCallOrder of the first `tx.update(table)` call. */
+function updateOrderForTable(table: unknown): number {
+  const { calls, invocationCallOrder } = dbChainMockFns.update.mock
+  for (let i = 0; i < calls.length; i++) {
+    if (calls[i][0] === table) return invocationCallOrder[i]
+  }
+  return -1
+}
+
+describe('updateDocument lock ordering', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    // Post-transaction re-read of the updated document must return a row.
+    dbChainMockFns.limit.mockResolvedValue([{ id: 'doc-1', knowledgeBaseId: 'kb-1' }])
+  })
+
+  it('updates embeddings before the document row when cascading tag changes', async () => {
+    await updateDocument('doc-1', { tag1: 'priority' }, 'req-1')
+
+    const embeddingWriteOrder = updateOrderForTable(embedding)
+    const documentWriteOrder = updateOrderForTable(document)
+
+    expect(embeddingWriteOrder).toBeGreaterThan(0)
+    expect(documentWriteOrder).toBeGreaterThan(0)
+    expect(embeddingWriteOrder).toBeLessThan(documentWriteOrder)
+  })
+})

apps/sim/lib/knowledge/documents/parser-extension.test.ts

@@ -0,0 +1,27 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
+
+describe('resolveParserExtension', () => {
+  it('uses a supported filename extension when present', () => {
+    expect(resolveParserExtension('report.pdf', 'application/pdf')).toBe('pdf')
+  })
+
+  it('falls back to mime type when filename has no extension', () => {
+    expect(
+      resolveParserExtension('[Business] Your Thursday morning trip with Uber', 'text/plain')
+    ).toBe('txt')
+  })
+
+  it('falls back to mime type when filename extension is unsupported', () => {
+    expect(resolveParserExtension('uber-message.business', 'text/plain')).toBe('txt')
+  })
+
+  it('throws when neither filename nor mime type resolves to a supported parser', () => {
+    expect(() =>
+      resolveParserExtension('uber-message.unknown', 'application/octet-stream')
+    ).toThrow('Unsupported file type')
+  })
+})

apps/sim/lib/knowledge/documents/parser-extension.ts

@@ -0,0 +1,38 @@
+import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils'
+import {
+  isAlphanumericExtension,
+  isSupportedExtension,
+  SUPPORTED_DOCUMENT_EXTENSIONS,
+} from '@/lib/uploads/utils/validation'
+
+const SUPPORTED_EXTENSIONS_TEXT = SUPPORTED_DOCUMENT_EXTENSIONS.join(', ')
+
+export function resolveParserExtension(
+  filename: string,
+  mimeType?: string,
+  fallback?: string
+): string {
+  const raw = filename.includes('.') ? filename.split('.').pop()?.toLowerCase() : undefined
+  const filenameExtension = raw && isAlphanumericExtension(raw) ? raw : undefined
+
+  if (filenameExtension && isSupportedExtension(filenameExtension)) {
+    return filenameExtension
+  }
+
+  const mimeExtension = mimeType ? getExtensionFromMimeType(mimeType) : undefined
+  if (mimeExtension && isSupportedExtension(mimeExtension)) {
+    return mimeExtension
+  }
+
+  if (fallback) {
+    return fallback
+  }
+
+  if (filenameExtension) {
+    throw new Error(
+      `Unsupported file type: ${filenameExtension}. Supported types are: ${SUPPORTED_EXTENSIONS_TEXT}`
+    )
+  }
+
+  throw new Error(`Could not determine file type for ${filename || 'document'}`)
+}

apps/sim/lib/knowledge/documents/secure-fetch.server.ts

@@ -0,0 +1,71 @@
+import {
+  type SecureFetchOptions,
+  type SecureFetchResponse,
+  secureFetchWithValidation,
+} from '@/lib/core/security/input-validation.server'
+import {
+  type HTTPError,
+  isRetryableError,
+  type RetryOptions,
+  retryWithExponentialBackoff,
+} from '@/lib/knowledge/documents/utils'
+
+export interface SecureFetchRetryOptions extends RetryOptions {
+  allowHttp?: boolean
+  timeout?: number
+  maxResponseBytes?: number
+}
+
+/**
+ * SSRF-safe counterpart to {@link fetchWithRetry} for connector requests to
+ * user-controlled hosts. Every attempt re-runs {@link secureFetchWithValidation}
+ * (DNS resolution, private/loopback/reserved-IP rejection, IP-pinned connection,
+ * redirect re-validation); retry/backoff semantics mirror {@link fetchWithRetry}.
+ *
+ * Lives in a `.server.ts` module because it pulls in Node-only `dns/promises`
+ * via {@link secureFetchWithValidation}; importing it from the shared
+ * `documents/utils` barrel would drag that into client bundles.
+ */
+export async function secureFetchWithRetry(
+  url: string,
+  options: SecureFetchOptions = {},
+  retryOptions: SecureFetchRetryOptions = {}
+): Promise<SecureFetchResponse> {
+  const { allowHttp, timeout, maxResponseBytes, ...retry } = retryOptions
+
+  return retryWithExponentialBackoff(async () => {
+    const response = await secureFetchWithValidation(
+      url,
+      {
+        ...options,
+        ...(allowHttp !== undefined ? { allowHttp } : {}),
+        ...(timeout !== undefined ? { timeout } : {}),
+        ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}),
+      },
+      'url'
+    )
+
+    if (!response.ok && isRetryableError({ status: response.status })) {
+      const errorText = await response.text()
+      const error: HTTPError = new Error(
+        `HTTP ${response.status}: ${response.statusText} - ${errorText}`
+      )
+      error.status = response.status
+      error.statusText = response.statusText
+
+      const retryAfter = response.headers.get('retry-after')
+      if (retryAfter) {
+        const waitMs = Number.isNaN(Number(retryAfter))
+          ? Math.max(0, new Date(retryAfter).getTime() - Date.now())
+          : Number(retryAfter) * 1000
+        if (waitMs > 0) {
+          error.retryAfterMs = waitMs
+        }
+      }
+
+      throw error
+    }
+
+    return response
+  }, retry)
+}

apps/sim/lib/knowledge/documents/tag-filter.test.ts

@@ -0,0 +1,177 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { buildTagFilterCondition } from '@/lib/knowledge/documents/tag-filter'
+
+/**
+ * The global `drizzle-orm` mock renders `sql` fragments to a `?`-placeholder
+ * string via `toSQL()` and returns plain `{ type, left, right }` objects for the
+ * comparison operators, so we can assert the exact predicate each filter builds.
+ */
+function rendered(condition: ReturnType<typeof buildTagFilterCondition>) {
+  return (condition as unknown as { toSQL: () => { sql: string; params: unknown[] } }).toSQL()
+}
+
+describe('buildTagFilterCondition', () => {
+  it('ignores unknown tag slots', () => {
+    expect(
+      buildTagFilterCondition({
+        tagSlot: 'not_a_real_slot',
+        fieldType: 'text',
+        operator: 'eq',
+        value: 'x',
+      })
+    ).toBeUndefined()
+  })
+
+  describe('text', () => {
+    it('matches eq case-insensitively', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'eq',
+          value: 'Ada Lovelace',
+        })
+      )
+      expect(sql).toBe('LOWER(?) = LOWER(?)')
+      expect(params).toEqual(['tag1', 'Ada Lovelace'])
+    })
+
+    it('matches neq case-insensitively', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag2',
+          fieldType: 'text',
+          operator: 'neq',
+          value: 'Spreadsheet',
+        })
+      )
+      expect(sql).toBe('LOWER(?) != LOWER(?)')
+      expect(params).toEqual(['tag2', 'Spreadsheet'])
+    })
+
+    it('escapes LIKE wildcards in contains', () => {
+      const { params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'contains',
+          value: '50%_off',
+        })
+      )
+      expect(params).toContain('%50\\%\\_off%')
+    })
+
+    it('returns undefined for an unsupported operator', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'gt',
+          value: 'x',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('date', () => {
+    it('compares eq on the calendar day', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'eq',
+          value: '2026-04-21',
+        })
+      )
+      expect(sql).toBe('?::date = ?::date')
+      expect(params).toEqual(['date1', '2026-04-21'])
+    })
+
+    it('compares range bounds on the calendar day', () => {
+      const condition = buildTagFilterCondition({
+        tagSlot: 'date1',
+        fieldType: 'date',
+        operator: 'between',
+        value: '2026-04-01',
+        valueTo: '2026-04-30',
+      }) as unknown as { type: string; conditions: unknown[] }
+      expect(condition.type).toBe('and')
+      expect(condition.conditions).toHaveLength(2)
+      expect(rendered(condition.conditions[0] as never).sql).toBe('?::date >= ?::date')
+      expect(rendered(condition.conditions[1] as never).sql).toBe('?::date <= ?::date')
+    })
+
+    it('ignores values that are not YYYY-MM-DD', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'eq',
+          value: 'not-a-date',
+        })
+      ).toBeUndefined()
+    })
+
+    it('ignores a between filter missing its upper bound', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'between',
+          value: '2026-04-01',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('number', () => {
+    it('builds an equality comparison', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'number1',
+          fieldType: 'number',
+          operator: 'eq',
+          value: '42',
+        })
+      ).toEqual({ type: 'eq', left: 'number1', right: 42 })
+    })
+
+    it('ignores non-numeric values', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'number1',
+          fieldType: 'number',
+          operator: 'eq',
+          value: 'abc',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('boolean', () => {
+    it('parses string values', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'boolean1',
+          fieldType: 'boolean',
+          operator: 'eq',
+          value: 'true',
+        })
+      ).toEqual({ type: 'eq', left: 'boolean1', right: true })
+    })
+
+    it('ignores values that are not boolean-like', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'boolean1',
+          fieldType: 'boolean',
+          operator: 'eq',
+          value: 'maybe',
+        })
+      ).toBeUndefined()
+    })
+  })
+})

apps/sim/lib/knowledge/documents/tag-filter.ts

@@ -0,0 +1,153 @@
+import { document } from '@sim/db/schema'
+import { and, eq, gt, gte, lt, lte, ne, type SQL, sql } from 'drizzle-orm'
+import { parseBooleanValue } from '@/lib/knowledge/tags/utils'
+
+/**
+ * A single tag filter applied to a document list query.
+ */
+export interface TagFilterCondition {
+  tagSlot: string
+  fieldType: 'text' | 'number' | 'date' | 'boolean'
+  operator: string
+  value: unknown
+  valueTo?: unknown
+}
+
+const ALLOWED_TAG_SLOTS = new Set([
+  'tag1',
+  'tag2',
+  'tag3',
+  'tag4',
+  'tag5',
+  'tag6',
+  'tag7',
+  'number1',
+  'number2',
+  'number3',
+  'number4',
+  'number5',
+  'date1',
+  'date2',
+  'boolean1',
+  'boolean2',
+  'boolean3',
+])
+
+const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/
+
+function escapeLikePattern(s: string): string {
+  return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')
+}
+
+/**
+ * Builds a SQL predicate for a single tag filter against the document table.
+ *
+ * Text comparisons are case-insensitive and date comparisons are evaluated on
+ * the calendar day, matching the semantics of the knowledge base search filter
+ * (`app/api/knowledge/search/utils.ts`). Returns `undefined` when the slot,
+ * operator, or value is not usable so the caller can skip the condition.
+ */
+export function buildTagFilterCondition(filter: TagFilterCondition): SQL | undefined {
+  if (!ALLOWED_TAG_SLOTS.has(filter.tagSlot)) return undefined
+
+  const col = document[filter.tagSlot as keyof typeof document]
+
+  if (filter.fieldType === 'text') {
+    const v = String(filter.value ?? '')
+    switch (filter.operator) {
+      case 'eq':
+        return sql`LOWER(${col}) = LOWER(${v})`
+      case 'neq':
+        return sql`LOWER(${col}) != LOWER(${v})`
+      case 'contains': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'not_contains': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) NOT LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'starts_with': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'ends_with': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}`}) ESCAPE '\\'`
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'number') {
+    const num = Number(filter.value)
+    if (Number.isNaN(num)) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return eq(col as typeof document.number1, num)
+      case 'neq':
+        return ne(col as typeof document.number1, num)
+      case 'gt':
+        return gt(col as typeof document.number1, num)
+      case 'gte':
+        return gte(col as typeof document.number1, num)
+      case 'lt':
+        return lt(col as typeof document.number1, num)
+      case 'lte':
+        return lte(col as typeof document.number1, num)
+      case 'between': {
+        const numTo = Number(filter.valueTo)
+        if (Number.isNaN(numTo)) return undefined
+        return and(
+          gte(col as typeof document.number1, num),
+          lte(col as typeof document.number1, numTo)
+        )
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'date') {
+    const v = String(filter.value ?? '')
+    if (!DATE_ONLY_PATTERN.test(v)) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return sql`${col}::date = ${v}::date`
+      case 'neq':
+        return sql`${col}::date != ${v}::date`
+      case 'gt':
+        return sql`${col}::date > ${v}::date`
+      case 'gte':
+        return sql`${col}::date >= ${v}::date`
+      case 'lt':
+        return sql`${col}::date < ${v}::date`
+      case 'lte':
+        return sql`${col}::date <= ${v}::date`
+      case 'between': {
+        const valueTo = String(filter.valueTo ?? '')
+        if (!DATE_ONLY_PATTERN.test(valueTo)) return undefined
+        return and(sql`${col}::date >= ${v}::date`, sql`${col}::date <= ${valueTo}::date`)
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'boolean') {
+    const boolVal =
+      typeof filter.value === 'boolean' ? filter.value : parseBooleanValue(String(filter.value))
+    if (boolVal === null) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return eq(col as typeof document.boolean1, boolVal)
+      case 'neq':
+        return ne(col as typeof document.boolean1, boolVal)
+      default:
+        return undefined
+    }
+  }
+
+  return undefined
+}

apps/sim/lib/knowledge/documents/types.ts

@@ -0,0 +1,26 @@
+// Document sorting options
+export type DocumentSortField =
+  | 'filename'
+  | 'fileSize'
+  | 'tokenCount'
+  | 'chunkCount'
+  | 'uploadedAt'
+  | 'processingStatus'
+  | 'enabled'
+export type SortOrder = 'asc' | 'desc'
+
+interface DocumentSortOptions {
+  sortBy?: DocumentSortField
+  sortOrder?: SortOrder
+}
+
+interface HeaderInfo {
+  /** Header text */
+  text: string
+  /** Header level (1-6) */
+  level: number
+  /** Anchor link */
+  anchor: string
+  /** Position in document */
+  position: number
+}

apps/sim/lib/knowledge/documents/utils.test.ts

@@ -0,0 +1,280 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockSecureFetchWithValidation } = vi.hoisted(() => ({
+  mockSecureFetchWithValidation: vi.fn(),
+}))
+
+vi.mock('@/lib/core/security/input-validation.server', () => ({
+  secureFetchWithValidation: mockSecureFetchWithValidation,
+}))
+
+import { secureFetchWithRetry } from './secure-fetch.server'
+import { isRetryableError } from './utils'
+
+/** Builds a minimal SecureFetchResponse-shaped object for tests. */
+function fakeResponse(
+  status: number,
+  options: { headers?: Record<string, string>; body?: string } = {}
+) {
+  const headers = options.headers ?? {}
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    statusText: `status-${status}`,
+    headers: { get: (name: string) => headers[name.toLowerCase()] ?? null },
+    body: null,
+    text: async () => options.body ?? '',
+    json: async () => JSON.parse(options.body ?? '{}'),
+    arrayBuffer: async () => new ArrayBuffer(0),
+  }
+}
+
+const FAST_RETRY = { initialDelayMs: 1, maxDelayMs: 2, maxRetries: 3 }
+
+describe('isRetryableError', () => {
+  describe('retryable status codes', () => {
+    it.concurrent('returns true for 429 on Error with status', () => {
+      const error = Object.assign(new Error('Too Many Requests'), { status: 429 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 502 on Error with status', () => {
+      const error = Object.assign(new Error('Bad Gateway'), { status: 502 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 503 on Error with status', () => {
+      const error = Object.assign(new Error('Service Unavailable'), { status: 503 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 504 on Error with status', () => {
+      const error = Object.assign(new Error('Gateway Timeout'), { status: 504 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 429', () => {
+      expect(isRetryableError({ status: 429 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 502', () => {
+      expect(isRetryableError({ status: 502 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 503', () => {
+      expect(isRetryableError({ status: 503 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 504', () => {
+      expect(isRetryableError({ status: 504 })).toBe(true)
+    })
+  })
+
+  describe('non-retryable status codes', () => {
+    it.concurrent('returns false for 400', () => {
+      const error = Object.assign(new Error('Bad Request'), { status: 400 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 401', () => {
+      const error = Object.assign(new Error('Unauthorized'), { status: 401 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 403', () => {
+      const error = Object.assign(new Error('Forbidden'), { status: 403 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 404', () => {
+      const error = Object.assign(new Error('Not Found'), { status: 404 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 500', () => {
+      const error = Object.assign(new Error('Internal Server Error'), { status: 500 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+  })
+
+  describe('retryable error messages', () => {
+    it.concurrent('returns true for "rate limit" in message', () => {
+      expect(isRetryableError(new Error('You have hit the rate limit'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "rate_limit" in message', () => {
+      expect(isRetryableError(new Error('rate_limit_exceeded'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "too many requests" in message', () => {
+      expect(isRetryableError(new Error('too many requests, slow down'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "quota exceeded" in message', () => {
+      expect(isRetryableError(new Error('API quota exceeded'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "throttled" in message', () => {
+      expect(isRetryableError(new Error('Request was throttled'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "retry after" in message', () => {
+      expect(isRetryableError(new Error('Please retry after 60 seconds'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "temporarily unavailable" in message', () => {
+      expect(isRetryableError(new Error('Service is temporarily unavailable'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "service unavailable" in message', () => {
+      expect(isRetryableError(new Error('The service unavailable right now'))).toBe(true)
+    })
+
+    it.concurrent('returns true for a transient DNS resolution failure', () => {
+      expect(isRetryableError(new Error('url hostname could not be resolved'))).toBe(true)
+    })
+  })
+
+  describe('case insensitivity', () => {
+    it.concurrent('matches "Rate Limit" with mixed case', () => {
+      expect(isRetryableError(new Error('Rate Limit Exceeded'))).toBe(true)
+    })
+
+    it.concurrent('matches "THROTTLED" in uppercase', () => {
+      expect(isRetryableError(new Error('REQUEST THROTTLED'))).toBe(true)
+    })
+
+    it.concurrent('matches "Too Many Requests" in title case', () => {
+      expect(isRetryableError(new Error('Too Many Requests'))).toBe(true)
+    })
+  })
+
+  describe('null, undefined, and non-error inputs', () => {
+    it.concurrent('returns false for null', () => {
+      expect(isRetryableError(null)).toBe(false)
+    })
+
+    it.concurrent('returns false for undefined', () => {
+      expect(isRetryableError(undefined)).toBe(false)
+    })
+
+    it.concurrent('returns false for empty string', () => {
+      expect(isRetryableError('')).toBe(false)
+    })
+
+    it.concurrent('returns false for a number', () => {
+      expect(isRetryableError(42)).toBe(false)
+    })
+  })
+
+  describe('non-retryable errors', () => {
+    it.concurrent('returns false for Error with no status and unrelated message', () => {
+      expect(isRetryableError(new Error('Something went wrong'))).toBe(false)
+    })
+
+    it.concurrent('returns false for plain object with only non-retryable status', () => {
+      expect(isRetryableError({ status: 404 })).toBe(false)
+    })
+
+    it.concurrent('returns false for plain object with non-retryable status and no message', () => {
+      expect(isRetryableError({ status: 500 })).toBe(false)
+    })
+
+    it.concurrent('returns false for the deterministic blocked-IP SSRF rejection', () => {
+      expect(isRetryableError(new Error('url resolves to a blocked IP address'))).toBe(false)
+    })
+  })
+})
+
+describe('secureFetchWithRetry', () => {
+  beforeEach(() => {
+    mockSecureFetchWithValidation.mockReset()
+  })
+
+  it('routes the request through secureFetchWithValidation and returns the response', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200, { body: 'ok' }))
+
+    const response = await secureFetchWithRetry('https://example.com/api', {
+      method: 'GET',
+      headers: { Accept: 'application/json' },
+    })
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+    const [url, options, paramName] = mockSecureFetchWithValidation.mock.calls[0]
+    expect(url).toBe('https://example.com/api')
+    expect(options).toMatchObject({ method: 'GET', headers: { Accept: 'application/json' } })
+    expect(paramName).toBe('url')
+  })
+
+  it('propagates SSRF validation failures without retrying', async () => {
+    mockSecureFetchWithValidation.mockRejectedValue(
+      new Error('url resolves to a blocked IP address')
+    )
+
+    await expect(
+      secureFetchWithRetry('https://attacker.test', { method: 'GET' }, FAST_RETRY)
+    ).rejects.toThrow('blocked IP address')
+
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries on a retryable status (503) and succeeds', async () => {
+    mockSecureFetchWithValidation
+      .mockResolvedValueOnce(fakeResponse(503, { body: 'try later' }))
+      .mockResolvedValueOnce(fakeResponse(200, { body: 'ok' }))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not retry a non-retryable status (404) and returns it to the caller', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(404, { body: 'missing' }))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(404)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+  })
+
+  it('forwards allowHttp / timeout / maxResponseBytes to the pinned fetch', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200))
+
+    await secureFetchWithRetry(
+      'http://localhost:9000',
+      { method: 'GET' },
+      { allowHttp: true, timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY }
+    )
+
+    const [, options] = mockSecureFetchWithValidation.mock.calls[0]
+    expect(options).toMatchObject({ allowHttp: true, timeout: 5000, maxResponseBytes: 1024 })
+  })
+
+  it('honors Retry-After (seconds) on a 429 before retrying', async () => {
+    mockSecureFetchWithValidation
+      .mockResolvedValueOnce(fakeResponse(429, { headers: { 'retry-after': '0' } }))
+      .mockResolvedValueOnce(fakeResponse(200))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(2)
+  })
+})

apps/sim/lib/knowledge/documents/utils.ts

@@ -0,0 +1,215 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { randomFloat } from '@sim/utils/random'
+
+const logger = createLogger('RetryUtils')
+
+export interface HTTPError extends Error {
+  status?: number
+  statusText?: string
+  retryAfterMs?: number
+}
+
+type RetryableError = HTTPError | Error | { status?: number; message?: string }
+
+export interface RetryOptions {
+  maxRetries?: number
+  initialDelayMs?: number
+  maxDelayMs?: number
+  backoffMultiplier?: number
+  retryCondition?: (error: unknown) => boolean
+}
+
+interface RetryResult<T> {
+  success: boolean
+  data?: T
+  error?: Error
+  attemptCount: number
+}
+
+function hasStatus(
+  error: RetryableError
+): error is HTTPError | { status?: number; message?: string } {
+  return typeof error === 'object' && error !== null && 'status' in error
+}
+
+function isRetryableErrorType(error: unknown): error is RetryableError {
+  if (!error) return false
+  if (error instanceof Error) return true
+  if (typeof error === 'object' && ('status' in error || 'message' in error)) return true
+  return false
+}
+
+/**
+ * Default retry condition for rate limiting errors
+ */
+export function isRetryableError(error: unknown): boolean {
+  if (!isRetryableErrorType(error)) return false
+
+  // Check for rate limiting status codes
+  if (
+    hasStatus(error) &&
+    (error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504)
+  ) {
+    return true
+  }
+
+  // Check for network-level errors (DNS, connection, timeout)
+  const errorMessage = toError(error).message
+  const lowerMessage = errorMessage.toLowerCase()
+
+  const networkKeywords = [
+    'fetch failed',
+    'econnreset',
+    'econnrefused',
+    'etimedout',
+    'enetunreach',
+    'socket hang up',
+    'network error',
+    // Transient DNS resolution failure surfaced by secureFetchWithValidation
+    // before the request is made. The deterministic "resolves to a blocked IP
+    // address" security rejection is a distinct message and stays non-retryable.
+    'could not be resolved',
+  ]
+
+  if (networkKeywords.some((keyword) => lowerMessage.includes(keyword))) {
+    return true
+  }
+
+  // Check for rate limiting in error messages
+  const rateLimitKeywords = [
+    'rate limit',
+    'rate_limit',
+    'too many requests',
+    'quota exceeded',
+    'throttled',
+    'retry after',
+    'temporarily unavailable',
+    'service unavailable',
+  ]
+
+  return rateLimitKeywords.some((keyword) => lowerMessage.includes(keyword))
+}
+
+/**
+ * Executes a function with exponential backoff retry logic
+ */
+export async function retryWithExponentialBackoff<T>(
+  operation: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const {
+    maxRetries = 5,
+    initialDelayMs = 1000,
+    maxDelayMs = 30000,
+    backoffMultiplier = 2,
+    retryCondition = isRetryableError,
+  } = options
+
+  let lastError: Error | undefined
+  let delay = initialDelayMs
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      logger.debug(`Executing operation attempt ${attempt + 1}/${maxRetries + 1}`)
+      const result = await operation()
+
+      if (attempt > 0) {
+        logger.info(`Operation succeeded after ${attempt + 1} attempts`)
+      }
+
+      return result
+    } catch (error) {
+      lastError = toError(error)
+      logger.warn(`Operation failed on attempt ${attempt + 1}`, { error })
+
+      // If this is the last attempt, throw the error
+      if (attempt === maxRetries) {
+        logger.error(`Operation failed after ${maxRetries + 1} attempts`, { error })
+        throw lastError
+      }
+
+      // Check if error is retryable
+      if (!retryCondition(error as RetryableError)) {
+        logger.warn('Error is not retryable, throwing immediately', { error })
+        throw lastError
+      }
+
+      // Use Retry-After if the server told us how long to wait, otherwise exponential backoff.
+      // Cap Retry-After at maxDelayMs to bound total retry duration (matches Google Cloud SDK behavior).
+      const retryAfterMs = (lastError as HTTPError)?.retryAfterMs
+      const cappedRetryAfter = retryAfterMs ? Math.min(retryAfterMs, maxDelayMs) : undefined
+
+      if (retryAfterMs && retryAfterMs > maxDelayMs) {
+        logger.warn(
+          `Retry-After ${retryAfterMs}ms exceeds maxDelayMs ${maxDelayMs}ms — capping to ${maxDelayMs}ms`
+        )
+      }
+
+      const jitter = randomFloat() * 0.1 * delay
+      const actualDelay = cappedRetryAfter ?? Math.min(delay + jitter, maxDelayMs)
+
+      logger.info(
+        `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${cappedRetryAfter ? ' (Retry-After)' : ''}`
+      )
+
+      await sleep(actualDelay)
+
+      // Exponential backoff (skip if we used Retry-After)
+      if (!cappedRetryAfter) {
+        delay = Math.min(delay * backoffMultiplier, maxDelayMs)
+      }
+    }
+  }
+
+  throw lastError || new Error('Retry operation failed')
+}
+
+/**
+ * Tighter retry options for user-facing operations (e.g. validateConfig).
+ * Caps total wait at ~7s instead of ~31s to avoid API route timeouts.
+ */
+export const VALIDATE_RETRY_OPTIONS: RetryOptions = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 10000,
+}
+
+/**
+ * Wrapper for fetch requests with retry logic
+ */
+export async function fetchWithRetry(
+  url: string,
+  options: RequestInit = {},
+  retryOptions: RetryOptions = {}
+): Promise<Response> {
+  return retryWithExponentialBackoff(async () => {
+    const response = await fetch(url, options)
+
+    // If response is not ok and status indicates rate limiting, throw an error
+    if (!response.ok && isRetryableError({ status: response.status })) {
+      const errorText = await response.text()
+      const error: HTTPError = new Error(
+        `HTTP ${response.status}: ${response.statusText} - ${errorText}`
+      )
+      error.status = response.status
+      error.statusText = response.statusText
+
+      // Pass Retry-After to the retry loop so it replaces exponential backoff
+      const retryAfter = response.headers.get('Retry-After')
+      if (retryAfter) {
+        const waitMs = Number.isNaN(Number(retryAfter))
+          ? Math.max(0, new Date(retryAfter).getTime() - Date.now())
+          : Number(retryAfter) * 1000
+        if (waitMs > 0) {
+          error.retryAfterMs = waitMs
+        }
+      }
+
+      throw error
+    }
+
+    return response
+  }, retryOptions)
+}