chore: import upstream snapshot with attribution

apps/sim/lib/knowledge/chunks/service.ts

@@ -0,0 +1,583 @@
+import { db } from '@sim/db'
+import { document, embedding, knowledgeBase } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { sha256Hex } from '@sim/security/hash'
+import { generateId } from '@sim/utils/id'
+import { and, asc, desc, eq, ilike, inArray, isNull, sql } from 'drizzle-orm'
+import type {
+  BatchOperationResult,
+  ChunkData,
+  ChunkFilters,
+  ChunkQueryResult,
+  CreateChunkData,
+} from '@/lib/knowledge/chunks/types'
+import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models'
+import { generateEmbeddings } from '@/lib/knowledge/embeddings'
+import { estimateTokenCount } from '@/lib/tokenization/estimators'
+
+const logger = createLogger('ChunksService')
+
+const KB_CHUNK_LOCK_TIMEOUT_MS = 5_000
+
+/**
+ * Query chunks for a document with filtering and pagination
+ */
+export async function queryChunks(
+  documentId: string,
+  filters: ChunkFilters,
+  requestId: string
+): Promise<ChunkQueryResult> {
+  const {
+    search,
+    enabled = 'all',
+    limit = 50,
+    offset = 0,
+    sortBy = 'chunkIndex',
+    sortOrder = 'asc',
+  } = filters
+
+  const conditions = [eq(embedding.documentId, documentId)]
+
+  if (enabled === 'true') {
+    conditions.push(eq(embedding.enabled, true))
+  } else if (enabled === 'false') {
+    conditions.push(eq(embedding.enabled, false))
+  }
+
+  if (search) {
+    conditions.push(ilike(embedding.content, `%${search}%`))
+  }
+
+  const chunks = await db
+    .select({
+      id: embedding.id,
+      chunkIndex: embedding.chunkIndex,
+      content: embedding.content,
+      contentLength: embedding.contentLength,
+      tokenCount: embedding.tokenCount,
+      enabled: embedding.enabled,
+      startOffset: embedding.startOffset,
+      endOffset: embedding.endOffset,
+      tag1: embedding.tag1,
+      tag2: embedding.tag2,
+      tag3: embedding.tag3,
+      tag4: embedding.tag4,
+      tag5: embedding.tag5,
+      tag6: embedding.tag6,
+      tag7: embedding.tag7,
+      createdAt: embedding.createdAt,
+      updatedAt: embedding.updatedAt,
+    })
+    .from(embedding)
+    .where(and(...conditions))
+    .orderBy(
+      (() => {
+        const col =
+          sortBy === 'tokenCount'
+            ? embedding.tokenCount
+            : sortBy === 'enabled'
+              ? embedding.enabled
+              : embedding.chunkIndex
+        return sortOrder === 'desc' ? desc(col) : asc(col)
+      })()
+    )
+    .limit(limit)
+    .offset(offset)
+
+  const totalCount = await db
+    .select({ count: sql`count(*)` })
+    .from(embedding)
+    .where(and(...conditions))
+
+  logger.info(`[${requestId}] Retrieved ${chunks.length} chunks for document ${documentId}`)
+
+  return {
+    chunks: chunks as ChunkData[],
+    pagination: {
+      total: Number(totalCount[0]?.count || 0),
+      limit,
+      offset,
+      hasMore: chunks.length === limit,
+    },
+  }
+}
+
+/**
+ * Create a new chunk for a document.
+ *
+ * Assigns `chunkIndex` as `max(chunkIndex) + 1` under a transactional
+ * `pg_advisory_xact_lock` keyed on the document, so concurrent calls for the
+ * same document serialize instead of computing the same index and colliding
+ * on the `(document_id, chunk_index)` unique constraint. A `SELECT ... FOR
+ * UPDATE` on the current max row doesn't prevent that collision, since the
+ * row it would lock is unrelated to the not-yet-inserted next row. A row
+ * lock on `document` instead of an advisory lock would also work, but would
+ * invert the embedding-before-document lock order every other chunk
+ * mutation path uses (see lock-order.test.ts) — the advisory lock is a
+ * separate namespace, so it can't deadlock against that convention.
+ *
+ * `pg_advisory_xact_lock` auto-releases at transaction end, so there's no
+ * session lock to leak onto a pooled connection, and `lock_timeout` bounds
+ * the wait (it raises SQLSTATE 55P03 instead of hanging a pooled connection)
+ * if a same-document holder is stuck.
+ */
+export async function createChunk(
+  knowledgeBaseId: string,
+  documentId: string,
+  docTags: Record<string, string | number | boolean | Date | null>,
+  chunkData: CreateChunkData,
+  requestId: string,
+  workspaceId?: string | null
+): Promise<ChunkData> {
+  logger.info(`[${requestId}] Generating embedding for manual chunk`)
+  const kbRow = await db
+    .select({ embeddingModel: knowledgeBase.embeddingModel })
+    .from(knowledgeBase)
+    .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+    .limit(1)
+  if (kbRow.length === 0) {
+    throw new Error('Knowledge base not found')
+  }
+  const kbEmbeddingModel = kbRow[0].embeddingModel
+  const { embeddings } = await generateEmbeddings(
+    [chunkData.content],
+    kbEmbeddingModel,
+    workspaceId
+  )
+
+  const tokenCount = estimateTokenCount(
+    chunkData.content,
+    getEmbeddingModelInfo(kbEmbeddingModel).tokenizerProvider
+  )
+
+  const chunkId = generateId()
+  const now = new Date()
+
+  const newChunk = await db.transaction(async (tx) => {
+    await tx.execute(
+      sql`select set_config('lock_timeout', ${`${KB_CHUNK_LOCK_TIMEOUT_MS}ms`}, true)`
+    )
+    await tx.execute(
+      sql`select pg_advisory_xact_lock(hashtextextended(${`kb_chunk_seq:${documentId}`}, 0))`
+    )
+
+    const activeDocument = await tx
+      .select({ id: document.id })
+      .from(document)
+      .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
+      .where(
+        and(
+          eq(document.id, documentId),
+          eq(document.knowledgeBaseId, knowledgeBaseId),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt),
+          isNull(knowledgeBase.deletedAt)
+        )
+      )
+      .limit(1)
+
+    if (activeDocument.length === 0) {
+      throw new Error('Document not found')
+    }
+
+    const lastChunk = await tx
+      .select({ chunkIndex: embedding.chunkIndex })
+      .from(embedding)
+      .where(eq(embedding.documentId, documentId))
+      .orderBy(sql`${embedding.chunkIndex} DESC`)
+      .limit(1)
+
+    const nextChunkIndex = lastChunk.length > 0 ? lastChunk[0].chunkIndex + 1 : 0
+
+    const chunkDBData = {
+      id: chunkId,
+      knowledgeBaseId,
+      documentId,
+      chunkIndex: nextChunkIndex,
+      chunkHash: sha256Hex(chunkData.content),
+      content: chunkData.content,
+      contentLength: chunkData.content.length,
+      tokenCount: tokenCount.count,
+      embedding: embeddings[0],
+      embeddingModel: kbEmbeddingModel,
+      startOffset: 0, // Manual chunks don't have document offsets
+      endOffset: chunkData.content.length,
+      // Inherit text tags from parent document
+      tag1: docTags.tag1 as string | null,
+      tag2: docTags.tag2 as string | null,
+      tag3: docTags.tag3 as string | null,
+      tag4: docTags.tag4 as string | null,
+      tag5: docTags.tag5 as string | null,
+      tag6: docTags.tag6 as string | null,
+      tag7: docTags.tag7 as string | null,
+      // Inherit number tags from parent document (5 slots)
+      number1: docTags.number1 as number | null,
+      number2: docTags.number2 as number | null,
+      number3: docTags.number3 as number | null,
+      number4: docTags.number4 as number | null,
+      number5: docTags.number5 as number | null,
+      // Inherit date tags from parent document (2 slots)
+      date1: docTags.date1 as Date | null,
+      date2: docTags.date2 as Date | null,
+      // Inherit boolean tags from parent document (3 slots)
+      boolean1: docTags.boolean1 as boolean | null,
+      boolean2: docTags.boolean2 as boolean | null,
+      boolean3: docTags.boolean3 as boolean | null,
+      enabled: chunkData.enabled ?? true,
+      createdAt: now,
+      updatedAt: now,
+    }
+
+    await tx.insert(embedding).values(chunkDBData)
+
+    // Update document statistics
+    await tx
+      .update(document)
+      .set({
+        chunkCount: sql`${document.chunkCount} + 1`,
+        tokenCount: sql`${document.tokenCount} + ${tokenCount.count}`,
+        characterCount: sql`${document.characterCount} + ${chunkData.content.length}`,
+      })
+      .where(eq(document.id, documentId))
+
+    return {
+      id: chunkId,
+      chunkIndex: nextChunkIndex,
+      content: chunkData.content,
+      contentLength: chunkData.content.length,
+      tokenCount: tokenCount.count,
+      enabled: chunkData.enabled ?? true,
+      startOffset: 0,
+      endOffset: chunkData.content.length,
+      tag1: docTags.tag1,
+      tag2: docTags.tag2,
+      tag3: docTags.tag3,
+      tag4: docTags.tag4,
+      tag5: docTags.tag5,
+      tag6: docTags.tag6,
+      tag7: docTags.tag7,
+      createdAt: now,
+      updatedAt: now,
+    } as ChunkData
+  })
+
+  logger.info(`[${requestId}] Created chunk ${chunkId} in document ${documentId}`)
+
+  return newChunk
+}
+
+/**
+ * Perform batch operations on chunks
+ */
+export async function batchChunkOperation(
+  documentId: string,
+  operation: 'enable' | 'disable' | 'delete',
+  chunkIds: string[],
+  requestId: string
+): Promise<BatchOperationResult> {
+  logger.info(
+    `[${requestId}] Starting batch ${operation} operation on ${chunkIds.length} chunks for document ${documentId}`
+  )
+
+  const errors: string[] = []
+  let successCount = 0
+
+  if (operation === 'delete') {
+    // Handle batch delete with transaction for consistency
+    await db.transaction(async (tx) => {
+      // Get chunks to delete for statistics update
+      const chunksToDelete = await tx
+        .select({
+          id: embedding.id,
+          tokenCount: embedding.tokenCount,
+          contentLength: embedding.contentLength,
+        })
+        .from(embedding)
+        .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
+
+      if (chunksToDelete.length === 0) {
+        errors.push('No matching chunks found to delete')
+        return
+      }
+
+      const totalTokensToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.tokenCount, 0)
+      const totalCharsToRemove = chunksToDelete.reduce((sum, chunk) => sum + chunk.contentLength, 0)
+
+      // Delete chunks
+      const deleteResult = await tx
+        .delete(embedding)
+        .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
+
+      // Update document statistics
+      await tx
+        .update(document)
+        .set({
+          chunkCount: sql`${document.chunkCount} - ${chunksToDelete.length}`,
+          tokenCount: sql`${document.tokenCount} - ${totalTokensToRemove}`,
+          characterCount: sql`${document.characterCount} - ${totalCharsToRemove}`,
+        })
+        .where(eq(document.id, documentId))
+
+      successCount = chunksToDelete.length
+    })
+  } else {
+    // Handle enable/disable operations
+    const enabled = operation === 'enable'
+
+    await db
+      .update(embedding)
+      .set({
+        enabled,
+        updatedAt: new Date(),
+      })
+      .where(and(eq(embedding.documentId, documentId), inArray(embedding.id, chunkIds)))
+
+    // For enable/disable, we assume all chunks were processed successfully
+    successCount = chunkIds.length
+  }
+
+  logger.info(
+    `[${requestId}] Batch ${operation} completed: ${successCount} chunks processed, ${errors.length} errors`
+  )
+
+  return {
+    success: errors.length === 0,
+    processed: successCount,
+    errors,
+  }
+}
+
+/**
+ * Update a single chunk
+ */
+export async function updateChunk(
+  chunkId: string,
+  updateData: {
+    content?: string
+    enabled?: boolean
+  },
+  requestId: string,
+  workspaceId?: string | null
+): Promise<ChunkData> {
+  // Content updates run in a transaction to keep document statistics
+  // consistent. The embedding API call happens BEFORE the transaction opens so
+  // a held pooled connection never waits on external I/O; the transaction then
+  // re-reads the chunk under a row lock and retries the whole flow in the rare
+  // case a concurrent edit invalidated the regeneration decision.
+  if (updateData.content !== undefined && typeof updateData.content === 'string') {
+    const content = updateData.content
+    const MAX_UPDATE_ATTEMPTS = 3
+
+    for (let attempt = 1; attempt <= MAX_UPDATE_ATTEMPTS; attempt++) {
+      const [preRead] = await db
+        .select({ documentId: embedding.documentId, content: embedding.content })
+        .from(embedding)
+        .where(eq(embedding.id, chunkId))
+        .limit(1)
+
+      if (!preRead) {
+        throw new Error(`Chunk ${chunkId} not found`)
+      }
+
+      // The embedding is a function of the new content alone, so generating it
+      // outside the transaction is always valid.
+      let regenerated: { embedding: number[]; tokenCount: number } | null = null
+      if (content !== preRead.content) {
+        const kbRow = await db
+          .select({ embeddingModel: knowledgeBase.embeddingModel })
+          .from(knowledgeBase)
+          .innerJoin(document, eq(document.knowledgeBaseId, knowledgeBase.id))
+          .where(eq(document.id, preRead.documentId))
+          .limit(1)
+        const chunkEmbeddingModel = kbRow[0]?.embeddingModel
+        if (!chunkEmbeddingModel) {
+          throw new Error('Knowledge base for chunk not found')
+        }
+
+        logger.info(`[${requestId}] Content changed, regenerating embedding for chunk ${chunkId}`)
+        const { embeddings } = await generateEmbeddings([content], chunkEmbeddingModel, workspaceId)
+        regenerated = {
+          embedding: embeddings[0],
+          tokenCount: estimateTokenCount(
+            content,
+            getEmbeddingModelInfo(chunkEmbeddingModel).tokenizerProvider
+          ).count,
+        }
+      }
+
+      const result = await db.transaction(async (tx) => {
+        const currentChunk = await tx
+          .select({
+            documentId: embedding.documentId,
+            content: embedding.content,
+            contentLength: embedding.contentLength,
+            tokenCount: embedding.tokenCount,
+          })
+          .from(embedding)
+          .where(eq(embedding.id, chunkId))
+          .limit(1)
+          .for('update')
+
+        if (currentChunk.length === 0) {
+          throw new Error(`Chunk ${chunkId} not found`)
+        }
+
+        // A concurrent edit landed between the pre-read and this row lock and
+        // we skipped regeneration based on stale content; retry so the
+        // decision is re-made against the committed content.
+        if (!regenerated && currentChunk[0].content !== content) {
+          return null
+        }
+
+        const oldContentLength = currentChunk[0].contentLength
+        const oldTokenCount = currentChunk[0].tokenCount
+        const newContentLength = content.length
+
+        const chunkUpdate = {
+          updatedAt: new Date(),
+          content,
+          contentLength: newContentLength,
+          chunkHash: sha256Hex(content),
+          tokenCount: regenerated ? regenerated.tokenCount : oldTokenCount,
+          ...(regenerated ? { embedding: regenerated.embedding } : {}),
+          ...(updateData.enabled !== undefined ? { enabled: updateData.enabled } : {}),
+        }
+
+        await tx.update(embedding).set(chunkUpdate).where(eq(embedding.id, chunkId))
+
+        const charDiff = newContentLength - oldContentLength
+        const tokenDiff = chunkUpdate.tokenCount - oldTokenCount
+
+        await tx
+          .update(document)
+          .set({
+            characterCount: sql`${document.characterCount} + ${charDiff}`,
+            tokenCount: sql`${document.tokenCount} + ${tokenDiff}`,
+          })
+          .where(eq(document.id, currentChunk[0].documentId))
+
+        const updatedChunk = await tx
+          .select({
+            id: embedding.id,
+            chunkIndex: embedding.chunkIndex,
+            content: embedding.content,
+            contentLength: embedding.contentLength,
+            tokenCount: embedding.tokenCount,
+            enabled: embedding.enabled,
+            startOffset: embedding.startOffset,
+            endOffset: embedding.endOffset,
+            tag1: embedding.tag1,
+            tag2: embedding.tag2,
+            tag3: embedding.tag3,
+            tag4: embedding.tag4,
+            tag5: embedding.tag5,
+            tag6: embedding.tag6,
+            tag7: embedding.tag7,
+            createdAt: embedding.createdAt,
+            updatedAt: embedding.updatedAt,
+          })
+          .from(embedding)
+          .where(eq(embedding.id, chunkId))
+          .limit(1)
+
+        logger.info(
+          `[${requestId}] Updated chunk: ${chunkId}${regenerated ? ' (regenerated embedding)' : ''}`
+        )
+
+        return updatedChunk[0] as ChunkData
+      })
+
+      if (result) {
+        return result
+      }
+    }
+
+    throw new Error(
+      `Chunk ${chunkId} was concurrently modified ${MAX_UPDATE_ATTEMPTS} times; retry the update`
+    )
+  }
+
+  // If only enabled status is being updated, no need for transaction
+  await db
+    .update(embedding)
+    .set({
+      updatedAt: new Date(),
+      ...(updateData.enabled !== undefined ? { enabled: updateData.enabled } : {}),
+    })
+    .where(eq(embedding.id, chunkId))
+
+  // Fetch the updated chunk
+  const updatedChunk = await db
+    .select({
+      id: embedding.id,
+      chunkIndex: embedding.chunkIndex,
+      content: embedding.content,
+      contentLength: embedding.contentLength,
+      tokenCount: embedding.tokenCount,
+      enabled: embedding.enabled,
+      startOffset: embedding.startOffset,
+      endOffset: embedding.endOffset,
+      tag1: embedding.tag1,
+      tag2: embedding.tag2,
+      tag3: embedding.tag3,
+      tag4: embedding.tag4,
+      tag5: embedding.tag5,
+      tag6: embedding.tag6,
+      tag7: embedding.tag7,
+      createdAt: embedding.createdAt,
+      updatedAt: embedding.updatedAt,
+    })
+    .from(embedding)
+    .where(eq(embedding.id, chunkId))
+    .limit(1)
+
+  if (updatedChunk.length === 0) {
+    throw new Error(`Chunk ${chunkId} not found`)
+  }
+
+  logger.info(`[${requestId}] Updated chunk: ${chunkId}`)
+
+  return updatedChunk[0] as ChunkData
+}
+
+/**
+ * Delete a single chunk with document statistics updates
+ */
+export async function deleteChunk(
+  chunkId: string,
+  documentId: string,
+  requestId: string
+): Promise<void> {
+  await db.transaction(async (tx) => {
+    // Get chunk data before deletion for statistics update
+    const chunkToDelete = await tx
+      .select({
+        tokenCount: embedding.tokenCount,
+        contentLength: embedding.contentLength,
+      })
+      .from(embedding)
+      .where(eq(embedding.id, chunkId))
+      .limit(1)
+
+    if (chunkToDelete.length === 0) {
+      throw new Error('Chunk not found')
+    }
+
+    const chunk = chunkToDelete[0]
+
+    // Delete the chunk
+    await tx.delete(embedding).where(eq(embedding.id, chunkId))
+
+    // Update document statistics
+    await tx
+      .update(document)
+      .set({
+        chunkCount: sql`${document.chunkCount} - 1`,
+        tokenCount: sql`${document.tokenCount} - ${chunk.tokenCount}`,
+        characterCount: sql`${document.characterCount} - ${chunk.contentLength}`,
+      })
+      .where(eq(document.id, documentId))
+  })
+
+  logger.info(`[${requestId}] Deleted chunk: ${chunkId}`)
+}

apps/sim/lib/knowledge/chunks/types.ts

@@ -0,0 +1,49 @@
+export interface ChunkFilters {
+  search?: string
+  enabled?: 'true' | 'false' | 'all'
+  limit?: number
+  offset?: number
+  sortBy?: 'chunkIndex' | 'tokenCount' | 'enabled'
+  sortOrder?: 'asc' | 'desc'
+}
+
+export interface ChunkData {
+  id: string
+  chunkIndex: number
+  content: string
+  contentLength: number
+  tokenCount: number
+  enabled: boolean
+  startOffset: number
+  endOffset: number
+  tag1?: string | null
+  tag2?: string | null
+  tag3?: string | null
+  tag4?: string | null
+  tag5?: string | null
+  tag6?: string | null
+  tag7?: string | null
+  createdAt: Date
+  updatedAt: Date
+}
+
+export interface ChunkQueryResult {
+  chunks: ChunkData[]
+  pagination: {
+    total: number
+    limit: number
+    offset: number
+    hasMore: boolean
+  }
+}
+
+export interface CreateChunkData {
+  content: string
+  enabled?: boolean
+}
+
+export interface BatchOperationResult {
+  success: boolean
+  processed: number
+  errors: string[]
+}

apps/sim/lib/knowledge/connectors/sync-engine.test.ts

@@ -0,0 +1,290 @@
+/**
+ * @vitest-environment node
+ */
+import { authOAuthUtilsMock, urlsMock } from '@sim/testing'
+import { generateShortId } from '@sim/utils/id'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => ({ db: {} }))
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn(),
+  eq: vi.fn(),
+  inArray: vi.fn(),
+  isNull: vi.fn(),
+  ne: vi.fn(),
+}))
+vi.mock('@/lib/core/utils/urls', () => urlsMock)
+vi.mock('@/lib/knowledge/documents/service', () => ({
+  hardDeleteDocuments: vi.fn(),
+  isTriggerAvailable: vi.fn(),
+  processDocumentAsync: vi.fn(),
+}))
+vi.mock('@/lib/uploads', () => ({ StorageService: {} }))
+vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
+vi.mock('@/background/knowledge-connector-sync', () => ({
+  knowledgeConnectorSync: { trigger: vi.fn() },
+}))
+
+const mockMapTags = vi.fn()
+
+vi.mock('@/connectors/registry.server', () => ({
+  CONNECTOR_REGISTRY: {
+    jira: {
+      mapTags: mockMapTags,
+    },
+    'no-tags': {
+      name: 'No Tags',
+    },
+  },
+}))
+
+describe('shouldReconcileDeletions', () => {
+  it('runs on a clean full listing', async () => {
+    const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(shouldReconcileDeletions(false, {}, undefined)).toBe(true)
+    expect(shouldReconcileDeletions(false, undefined, undefined)).toBe(true)
+  })
+
+  it('never runs on incremental syncs', async () => {
+    const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(shouldReconcileDeletions(true, {}, undefined)).toBe(false)
+    expect(shouldReconcileDeletions(true, {}, true)).toBe(false)
+    expect(shouldReconcileDeletions(true, { listingCapped: true }, true)).toBe(false)
+  })
+
+  it('skips when a connector capped the listing', async () => {
+    const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(shouldReconcileDeletions(false, { listingCapped: true }, undefined)).toBe(false)
+    expect(shouldReconcileDeletions(false, { listingCapped: true }, false)).toBe(false)
+  })
+
+  it('lets a forced fullSync override a connector cap', async () => {
+    const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(shouldReconcileDeletions(false, { listingCapped: true }, true)).toBe(true)
+  })
+
+  it('never runs when the engine truncated pagination, even on a forced fullSync', async () => {
+    const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(shouldReconcileDeletions(false, { listingTruncated: true }, undefined)).toBe(false)
+    expect(shouldReconcileDeletions(false, { listingTruncated: true }, true)).toBe(false)
+    expect(
+      shouldReconcileDeletions(false, { listingCapped: true, listingTruncated: true }, true)
+    ).toBe(false)
+  })
+})
+
+describe('resolveTagMapping', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('maps semantic keys to DB slots', async () => {
+    mockMapTags.mockReturnValue({
+      issueType: 'Bug',
+      status: 'Open',
+      priority: 'High',
+    })
+
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping(
+      'jira',
+      { issueType: 'Bug', status: 'Open', priority: 'High' },
+      {
+        tagSlotMapping: {
+          issueType: 'tag1',
+          status: 'tag2',
+          priority: 'tag3',
+        },
+      }
+    )
+
+    expect(result).toEqual({
+      tag1: 'Bug',
+      tag2: 'Open',
+      tag3: 'High',
+    })
+  })
+
+  it('returns undefined when connector has no mapTags', async () => {
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping(
+      'no-tags',
+      { key: 'value' },
+      {
+        tagSlotMapping: { key: 'tag1' },
+      }
+    )
+
+    expect(result).toBeUndefined()
+  })
+
+  it('returns undefined when connector type is unknown', async () => {
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping('unknown', { key: 'value' }, {})
+
+    expect(result).toBeUndefined()
+  })
+
+  it('returns undefined when no tagSlotMapping in sourceConfig', async () => {
+    mockMapTags.mockReturnValue({ issueType: 'Bug' })
+
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping('jira', { issueType: 'Bug' }, {})
+
+    expect(result).toBeUndefined()
+  })
+
+  it('sets null for missing metadata keys', async () => {
+    mockMapTags.mockReturnValue({
+      issueType: 'Bug',
+      status: undefined,
+    })
+
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping(
+      'jira',
+      { issueType: 'Bug' },
+      {
+        tagSlotMapping: {
+          issueType: 'tag1',
+          status: 'tag2',
+          missing: 'tag3',
+        },
+      }
+    )
+
+    expect(result).toEqual({
+      tag1: 'Bug',
+      tag2: null,
+      tag3: null,
+    })
+  })
+
+  it('returns undefined when sourceConfig is undefined', async () => {
+    mockMapTags.mockReturnValue({ issueType: 'Bug' })
+
+    const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    const result = resolveTagMapping('jira', { issueType: 'Bug' }, undefined)
+
+    expect(result).toBeUndefined()
+  })
+})
+
+describe('classifyExternalDoc', () => {
+  const base = { content: 'hello', contentDeferred: false, contentHash: 'h1' }
+
+  it('records a new skipped file as a failed row', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    expect(
+      classifyExternalDoc({ ...base, content: '', skippedReason: 'too big' }, undefined)
+    ).toEqual({ type: 'skip' })
+  })
+
+  it('keeps an already-indexed file as-is when it becomes skipped (last-known-good)', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    expect(
+      classifyExternalDoc(
+        { ...base, content: '', skippedReason: 'too big' },
+        {
+          id: 'doc-1',
+          contentHash: 'old',
+        }
+      )
+    ).toEqual({ type: 'unchanged' })
+  })
+
+  it('drops empty non-deferred content', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    expect(classifyExternalDoc({ ...base, content: '   ' }, undefined)).toEqual({ type: 'drop' })
+  })
+
+  it('adds new content and deferred stubs', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    expect(classifyExternalDoc(base, undefined)).toEqual({ type: 'add' })
+    expect(classifyExternalDoc({ ...base, content: '', contentDeferred: true }, undefined)).toEqual(
+      { type: 'add' }
+    )
+  })
+
+  it('updates when the content hash changed and is unchanged otherwise', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+    expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'old' })).toEqual({
+      type: 'update',
+      existingId: 'doc-1',
+    })
+    expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'h1' })).toEqual({
+      type: 'unchanged',
+    })
+  })
+})
+
+describe('chunkOpsByByteBudget', () => {
+  const MB = 1024 * 1024
+  const addOp = (sizeBytes?: number) => ({
+    type: 'add' as const,
+    extDoc: {
+      externalId: `e-${generateShortId()}`,
+      title: 'f',
+      content: 'x',
+      contentHash: 'h',
+      mimeType: 'text/plain',
+      ...(sizeBytes != null ? { metadata: { fileSize: sizeBytes } } : {}),
+    },
+  })
+  const skipOp = (sizeBytes: number) => ({
+    type: 'skip' as const,
+    extDoc: {
+      externalId: `s-${generateShortId()}`,
+      title: 'f',
+      content: '',
+      contentHash: 'h',
+      mimeType: 'text/plain',
+      skippedReason: 'too big',
+      metadata: { fileSize: sizeBytes },
+    },
+  })
+
+  it('batches small ops up to the count cap', async () => {
+    const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine')
+    const chunks = chunkOpsByByteBudget(
+      Array.from({ length: 7 }, () => addOp(1024)),
+      64 * MB,
+      5
+    )
+    expect(chunks.map((c) => c.length)).toEqual([5, 2])
+  })
+
+  it('isolates a file larger than the budget into its own chunk', async () => {
+    const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine')
+    const chunks = chunkOpsByByteBudget([addOp(100 * MB), addOp(1024)], 64 * MB, 5)
+    expect(chunks.map((c) => c.length)).toEqual([1, 1])
+  })
+
+  it('caps summed bytes per chunk for medium files', async () => {
+    const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine')
+    // 40 + 40 = 80 MB exceeds the 64 MB budget, so they split.
+    const chunks = chunkOpsByByteBudget([addOp(40 * MB), addOp(40 * MB)], 64 * MB, 5)
+    expect(chunks.map((c) => c.length)).toEqual([1, 1])
+  })
+
+  it('treats skip ops as zero bytes so they do not consume the budget', async () => {
+    const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine')
+    const chunks = chunkOpsByByteBudget(
+      [skipOp(100 * MB), skipOp(100 * MB), addOp(1024)],
+      64 * MB,
+      5
+    )
+    expect(chunks).toHaveLength(1)
+  })
+})

apps/sim/lib/knowledge/constants.test.ts

@@ -0,0 +1,113 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { allocateTagSlots, getSlotsForFieldType } from '@/lib/knowledge/constants'
+
+describe('allocateTagSlots', () => {
+  it.concurrent('assigns unique slots for multiple text tags', () => {
+    const defs = [
+      { id: 'issueType', displayName: 'Issue Type', fieldType: 'text' },
+      { id: 'status', displayName: 'Status', fieldType: 'text' },
+      { id: 'priority', displayName: 'Priority', fieldType: 'text' },
+    ]
+
+    const { mapping, skipped } = allocateTagSlots(defs, new Set())
+
+    expect(mapping).toEqual({
+      issueType: 'tag1',
+      status: 'tag2',
+      priority: 'tag3',
+    })
+    expect(skipped).toEqual([])
+  })
+
+  it.concurrent('assigns slots across different field types', () => {
+    const defs = [
+      { id: 'label', displayName: 'Label', fieldType: 'text' },
+      { id: 'count', displayName: 'Count', fieldType: 'number' },
+      { id: 'updated', displayName: 'Updated', fieldType: 'date' },
+      { id: 'active', displayName: 'Active', fieldType: 'boolean' },
+    ]
+
+    const { mapping, skipped } = allocateTagSlots(defs, new Set())
+
+    expect(mapping).toEqual({
+      label: 'tag1',
+      count: 'number1',
+      updated: 'date1',
+      active: 'boolean1',
+    })
+    expect(skipped).toEqual([])
+  })
+
+  it.concurrent('skips already-used slots', () => {
+    const defs = [
+      { id: 'a', displayName: 'A', fieldType: 'text' },
+      { id: 'b', displayName: 'B', fieldType: 'text' },
+    ]
+
+    const usedSlots = new Set(['tag1', 'tag3'])
+    const { mapping, skipped } = allocateTagSlots(defs, usedSlots)
+
+    expect(mapping).toEqual({
+      a: 'tag2',
+      b: 'tag4',
+    })
+    expect(skipped).toEqual([])
+  })
+
+  it.concurrent('skips tags when all slots of that type are used', () => {
+    const defs = [
+      { id: 'a', displayName: 'Date A', fieldType: 'date' },
+      { id: 'b', displayName: 'Date B', fieldType: 'date' },
+      { id: 'c', displayName: 'Date C', fieldType: 'date' },
+    ]
+
+    const { mapping, skipped } = allocateTagSlots(defs, new Set())
+
+    expect(mapping).toEqual({
+      a: 'date1',
+      b: 'date2',
+    })
+    expect(skipped).toEqual(['Date C'])
+  })
+
+  it.concurrent('returns empty mapping when all slots are used', () => {
+    const allTextSlots = getSlotsForFieldType('text')
+    const usedSlots = new Set<string>(allTextSlots)
+
+    const defs = [{ id: 'label', displayName: 'Label', fieldType: 'text' }]
+    const { mapping, skipped } = allocateTagSlots(defs, usedSlots)
+
+    expect(mapping).toEqual({})
+    expect(skipped).toEqual(['Label'])
+  })
+
+  it.concurrent('handles empty definitions list', () => {
+    const { mapping, skipped } = allocateTagSlots([], new Set())
+
+    expect(mapping).toEqual({})
+    expect(skipped).toEqual([])
+  })
+
+  it.concurrent('handles unknown field type gracefully', () => {
+    const defs = [{ id: 'x', displayName: 'Unknown', fieldType: 'unknown' }]
+    const { mapping, skipped } = allocateTagSlots(defs, new Set())
+
+    expect(mapping).toEqual({})
+    expect(skipped).toEqual(['Unknown'])
+  })
+
+  it.concurrent('does not mutate the input usedSlots set', () => {
+    const defs = [
+      { id: 'a', displayName: 'A', fieldType: 'text' },
+      { id: 'b', displayName: 'B', fieldType: 'text' },
+    ]
+
+    const usedSlots = new Set<string>()
+    allocateTagSlots(defs, usedSlots)
+
+    expect(usedSlots.size).toBe(0)
+  })
+})

apps/sim/lib/knowledge/constants.ts

@@ -0,0 +1,140 @@
+/** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */
+export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000
+
+export const TAG_SLOT_CONFIG = {
+  text: {
+    slots: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const,
+    maxSlots: 7,
+  },
+  number: {
+    slots: ['number1', 'number2', 'number3', 'number4', 'number5'] as const,
+    maxSlots: 5,
+  },
+  date: {
+    slots: ['date1', 'date2'] as const,
+    maxSlots: 2,
+  },
+  boolean: {
+    slots: ['boolean1', 'boolean2', 'boolean3'] as const,
+    maxSlots: 3,
+  },
+} as const
+
+export const SUPPORTED_FIELD_TYPES = Object.keys(TAG_SLOT_CONFIG) as Array<
+  keyof typeof TAG_SLOT_CONFIG
+>
+
+/** Text tag slots (for backwards compatibility) */
+export const TAG_SLOTS = TAG_SLOT_CONFIG.text.slots
+
+/** All tag slots across all field types */
+export const ALL_TAG_SLOTS = [
+  ...TAG_SLOT_CONFIG.text.slots,
+  ...TAG_SLOT_CONFIG.number.slots,
+  ...TAG_SLOT_CONFIG.date.slots,
+  ...TAG_SLOT_CONFIG.boolean.slots,
+] as const
+
+export const MAX_TAG_SLOTS = TAG_SLOT_CONFIG.text.maxSlots
+
+/** Type for text tag slots (for backwards compatibility) */
+export type TagSlot = (typeof TAG_SLOTS)[number]
+
+/** Type for all tag slots */
+export type AllTagSlot = (typeof ALL_TAG_SLOTS)[number]
+
+/** Type for number tag slots */
+export type NumberTagSlot = (typeof TAG_SLOT_CONFIG.number.slots)[number]
+
+/** Type for date tag slots */
+export type DateTagSlot = (typeof TAG_SLOT_CONFIG.date.slots)[number]
+
+/** Type for boolean tag slots */
+export type BooleanTagSlot = (typeof TAG_SLOT_CONFIG.boolean.slots)[number]
+
+/**
+ * Get the available slots for a field type
+ */
+export function getSlotsForFieldType(fieldType: string): readonly string[] {
+  const config = TAG_SLOT_CONFIG[fieldType as keyof typeof TAG_SLOT_CONFIG]
+  if (!config) {
+    return []
+  }
+  return config.slots
+}
+
+/**
+ * Get the field type for a tag slot
+ */
+export function getFieldTypeForSlot(tagSlot: string): keyof typeof TAG_SLOT_CONFIG | null {
+  for (const [fieldType, config] of Object.entries(TAG_SLOT_CONFIG)) {
+    if ((config.slots as readonly string[]).includes(tagSlot)) {
+      return fieldType as keyof typeof TAG_SLOT_CONFIG
+    }
+  }
+  return null
+}
+
+/**
+ * Check if a slot is valid for a given field type
+ */
+export function isValidSlotForFieldType(tagSlot: string, fieldType: string): boolean {
+  const config = TAG_SLOT_CONFIG[fieldType as keyof typeof TAG_SLOT_CONFIG]
+  if (!config) {
+    return false
+  }
+  return (config.slots as readonly string[]).includes(tagSlot)
+}
+
+/**
+ * Display labels for field types
+ */
+export const FIELD_TYPE_LABELS: Record<string, string> = {
+  text: 'Text',
+  number: 'Number',
+  date: 'Date',
+  boolean: 'Boolean',
+}
+
+/**
+ * Allocate tag slots for a set of tag definitions, avoiding already-used slots.
+ * Returns a mapping of semantic IDs to slot names and a list of skipped tag names.
+ */
+export function allocateTagSlots(
+  tagDefinitions: Array<{ id: string; displayName: string; fieldType: string }>,
+  usedSlots: Set<string>
+): { mapping: Record<string, string>; skipped: string[] } {
+  const mapping: Record<string, string> = {}
+  const skipped: string[] = []
+  const claimed = new Set(usedSlots)
+
+  for (const td of tagDefinitions) {
+    const slots = getSlotsForFieldType(td.fieldType)
+    const available = slots.find((s) => !claimed.has(s))
+
+    if (!available) {
+      skipped.push(td.displayName)
+      continue
+    }
+    claimed.add(available)
+    mapping[td.id] = available
+  }
+
+  return { mapping, skipped }
+}
+
+/**
+ * Get placeholder text for value input based on field type
+ */
+export function getPlaceholderForFieldType(fieldType: string): string {
+  switch (fieldType) {
+    case 'boolean':
+      return 'true or false'
+    case 'number':
+      return 'Enter number'
+    case 'date':
+      return 'YYYY-MM-DD'
+    default:
+      return 'Enter value'
+  }
+}

apps/sim/lib/knowledge/documents/document-processor.ts

@@ -0,0 +1,859 @@
+import { randomBytes } from 'crypto'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { PDFDocument } from 'pdf-lib'
+import { getBYOKKey } from '@/lib/api-key/byok'
+import {
+  type Chunk,
+  JsonYamlChunker,
+  RecursiveChunker,
+  RegexChunker,
+  SentenceChunker,
+  StructuredDataChunker,
+  TextChunker,
+  TokenChunker,
+} from '@/lib/chunkers'
+import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
+import { env, envNumber } from '@/lib/core/config/env'
+import { parseBuffer } from '@/lib/file-parsers'
+import type { FileParseMetadata } from '@/lib/file-parsers/types'
+import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
+import { retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
+import { StorageService } from '@/lib/uploads'
+import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
+import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
+import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
+import { mistralParserTool } from '@/tools/mistral/parser'
+
+const logger = createLogger('DocumentProcessor')
+
+const TIMEOUTS = {
+  FILE_DOWNLOAD: 600000,
+  MISTRAL_OCR_API: 120000,
+} as const
+
+const MAX_CONCURRENT_CHUNKS = envNumber(env.KB_CONFIG_CHUNK_CONCURRENCY, 10)
+
+type OCRResult = {
+  success: boolean
+  error?: string
+  output?: {
+    content?: string
+  }
+}
+
+type OCRPage = {
+  markdown?: string
+}
+
+type OCRRequestBody = {
+  model: string
+  document: {
+    type: string
+    document_url: string
+  }
+  include_image_base64: boolean
+}
+
+const MISTRAL_MAX_PAGES = 1000
+
+async function getPdfPageCount(buffer: Buffer): Promise<number> {
+  try {
+    const { getDocumentProxy } = await import('unpdf')
+    const uint8Array = new Uint8Array(buffer)
+    const pdf = await getDocumentProxy(uint8Array)
+    return pdf.numPages
+  } catch (error) {
+    logger.warn('Failed to get PDF page count:', error)
+    return 0
+  }
+}
+
+async function splitPdfIntoChunks(
+  pdfBuffer: Buffer,
+  maxPages: number
+): Promise<{ buffer: Buffer; startPage: number; endPage: number }[]> {
+  const sourcePdf = await PDFDocument.load(pdfBuffer)
+  const totalPages = sourcePdf.getPageCount()
+
+  if (totalPages <= maxPages) {
+    return [{ buffer: pdfBuffer, startPage: 0, endPage: totalPages - 1 }]
+  }
+
+  const chunks: { buffer: Buffer; startPage: number; endPage: number }[] = []
+
+  for (let startPage = 0; startPage < totalPages; startPage += maxPages) {
+    const endPage = Math.min(startPage + maxPages - 1, totalPages - 1)
+    const pageCount = endPage - startPage + 1
+
+    const newPdf = await PDFDocument.create()
+    const pageIndices = Array.from({ length: pageCount }, (_, i) => startPage + i)
+    const copiedPages = await newPdf.copyPages(sourcePdf, pageIndices)
+
+    copiedPages.forEach((page) => newPdf.addPage(page))
+
+    const pdfBytes = await newPdf.save()
+    chunks.push({
+      buffer: Buffer.from(pdfBytes),
+      startPage,
+      endPage,
+    })
+  }
+
+  return chunks
+}
+
+type AzureOCRResponse = {
+  pages?: OCRPage[]
+  [key: string]: unknown
+}
+
+class APIError extends Error {
+  public status: number
+
+  constructor(message: string, status: number) {
+    super(message)
+    this.name = 'APIError'
+    this.status = status
+  }
+}
+
+async function applyStrategy(
+  strategy: ChunkingStrategy,
+  content: string,
+  chunkSize: number,
+  chunkOverlap: number,
+  minCharactersPerChunk: number,
+  strategyOptions?: StrategyOptions
+): Promise<Chunk[]> {
+  const baseOptions = { chunkSize, chunkOverlap, minCharactersPerChunk }
+
+  switch (strategy) {
+    case 'token': {
+      const chunker = new TokenChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+    case 'sentence': {
+      const chunker = new SentenceChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+    case 'recursive': {
+      const chunker = new RecursiveChunker({
+        ...baseOptions,
+        separators: strategyOptions?.separators,
+        recipe: strategyOptions?.recipe,
+      })
+      return chunker.chunk(content)
+    }
+    case 'regex': {
+      if (!strategyOptions?.pattern) {
+        logger.warn(
+          'Regex strategy requested but no pattern provided, falling back to text chunker'
+        )
+        const chunker = new TextChunker(baseOptions)
+        return chunker.chunk(content)
+      }
+      const chunker = new RegexChunker({
+        ...baseOptions,
+        pattern: strategyOptions.pattern,
+        strictBoundaries: strategyOptions.strictBoundaries,
+      })
+      return chunker.chunk(content)
+    }
+    default: {
+      const chunker = new TextChunker(baseOptions)
+      return chunker.chunk(content)
+    }
+  }
+}
+
+export async function processDocument(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  chunkSize = 1024,
+  chunkOverlap = 200,
+  minCharactersPerChunk = 100,
+  userId?: string,
+  workspaceId?: string | null,
+  strategy?: ChunkingStrategy,
+  strategyOptions?: StrategyOptions
+): Promise<{
+  chunks: Chunk[]
+  metadata: {
+    filename: string
+    fileSize: number
+    mimeType: string
+    chunkCount: number
+    tokenCount: number
+    characterCount: number
+    processingMethod: 'file-parser' | 'mistral-ocr'
+    cloudUrl?: string
+  }
+}> {
+  logger.info(`Processing document: ${filename}`)
+
+  try {
+    const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId)
+    const { content, processingMethod } = parseResult
+    const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined
+
+    let chunks: Chunk[]
+    const metadata: FileParseMetadata = parseResult.metadata ?? {}
+
+    if (strategy && strategy !== 'auto') {
+      logger.info(`Using explicit chunking strategy: ${strategy}`)
+      chunks = await applyStrategy(
+        strategy,
+        content,
+        chunkSize,
+        chunkOverlap,
+        minCharactersPerChunk,
+        strategyOptions
+      )
+    } else {
+      const isJsonYaml =
+        metadata.type === 'json' ||
+        metadata.type === 'yaml' ||
+        mimeType.includes('json') ||
+        mimeType.includes('yaml')
+
+      if (isJsonYaml && JsonYamlChunker.isStructuredData(content)) {
+        logger.info('Using JSON/YAML chunker for structured data')
+        chunks = await JsonYamlChunker.chunkJsonYaml(content, {
+          chunkSize,
+          minCharactersPerChunk,
+        })
+      } else if (StructuredDataChunker.isStructuredData(content, mimeType)) {
+        logger.info('Using structured data chunker for spreadsheet/CSV content')
+        const rowCount = metadata.totalRows ?? metadata.rowCount
+        chunks = await StructuredDataChunker.chunkStructuredData(content, {
+          chunkSize,
+          headers: metadata.headers,
+          totalRows: typeof rowCount === 'number' ? rowCount : undefined,
+          sheetName: metadata.sheetNames?.[0],
+        })
+      } else {
+        const chunker = new TextChunker({ chunkSize, chunkOverlap, minCharactersPerChunk })
+        chunks = await chunker.chunk(content)
+      }
+    }
+
+    const characterCount = content.length
+    const tokenCount = chunks.reduce((sum, chunk) => sum + chunk.tokenCount, 0)
+
+    logger.info(`Document processed: ${chunks.length} chunks, ${tokenCount} tokens`)
+
+    return {
+      chunks,
+      metadata: {
+        filename,
+        fileSize: characterCount,
+        mimeType,
+        chunkCount: chunks.length,
+        tokenCount,
+        characterCount,
+        processingMethod,
+        cloudUrl,
+      },
+    }
+  } catch (error) {
+    logger.error(`Error processing document ${filename}:`, error)
+    throw error
+  }
+}
+
+async function getMistralApiKey(workspaceId?: string | null): Promise<string | null> {
+  if (workspaceId) {
+    const byokResult = await getBYOKKey(workspaceId, 'mistral')
+    if (byokResult) {
+      logger.info('Using workspace BYOK key for Mistral OCR')
+      return byokResult.apiKey
+    }
+  }
+  return env.MISTRAL_API_KEY || null
+}
+
+async function parseDocument(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null
+): Promise<{
+  content: string
+  processingMethod: 'file-parser' | 'mistral-ocr'
+  cloudUrl?: string
+  metadata?: FileParseMetadata
+}> {
+  const isPDF = mimeType === 'application/pdf'
+  const hasAzureMistralOCR =
+    env.OCR_AZURE_API_KEY && env.OCR_AZURE_ENDPOINT && env.OCR_AZURE_MODEL_NAME
+
+  const mistralApiKey = await getMistralApiKey(workspaceId)
+  const hasMistralOCR = !!mistralApiKey
+
+  if (isPDF && (hasAzureMistralOCR || hasMistralOCR)) {
+    if (hasAzureMistralOCR) {
+      logger.info(`Using Azure Mistral OCR: ${filename}`)
+      return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId)
+    }
+
+    if (hasMistralOCR) {
+      logger.info(`Using Mistral OCR: ${filename}`)
+      return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey)
+    }
+  }
+
+  logger.info(`Using file parser: ${filename}`)
+  return parseWithFileParser(fileUrl, filename, mimeType, userId)
+}
+
+async function handleFileForOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null
+) {
+  const isExternalHttps = /^https:\/\//i.test(fileUrl) && !isInternalFileUrl(fileUrl)
+
+  if (isExternalHttps) {
+    if (mimeType === 'application/pdf') {
+      logger.info(`handleFileForOCR: Downloading external PDF to check page count`)
+      try {
+        const buffer = await downloadFileWithTimeout(fileUrl, userId)
+        logger.info(`handleFileForOCR: Downloaded external PDF: ${buffer.length} bytes`)
+        return { httpsUrl: fileUrl, buffer }
+      } catch (error) {
+        logger.warn(
+          `handleFileForOCR: Failed to download external PDF for page count check, proceeding without batching`,
+          {
+            error: toError(error).message,
+          }
+        )
+        return { httpsUrl: fileUrl, buffer: undefined }
+      }
+    }
+    logger.info(`handleFileForOCR: Using external URL directly`)
+    return { httpsUrl: fileUrl, buffer: undefined }
+  }
+
+  logger.info(`Uploading "${filename}" to cloud storage for OCR`)
+
+  const buffer = await downloadFileWithTimeout(fileUrl, userId)
+
+  logger.info(`Downloaded ${filename}: ${buffer.length} bytes`)
+
+  try {
+    const metadata: Record<string, string> = {
+      originalName: filename,
+      uploadedAt: new Date().toISOString(),
+      purpose: 'knowledge-base',
+      ...(userId && { userId }),
+      ...(workspaceId && { workspaceId }),
+    }
+
+    const timestamp = Date.now()
+    const uniqueId = randomBytes(8).toString('hex')
+    const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
+    const customKey = `kb/${timestamp}-${uniqueId}-${safeFileName}`
+
+    const cloudResult = await StorageService.uploadFile({
+      file: buffer,
+      fileName: filename,
+      contentType: mimeType,
+      context: 'knowledge-base',
+      customKey,
+      metadata,
+    })
+
+    const httpsUrl = await StorageService.generatePresignedDownloadUrl(
+      cloudResult.key,
+      'knowledge-base',
+      900 // 15 minutes
+    )
+
+    return { httpsUrl, cloudUrl: httpsUrl, buffer }
+  } catch (uploadError) {
+    const message = getErrorMessage(uploadError, 'Unknown error')
+    throw new Error(`Cloud upload failed: ${message}. Cloud upload is required for OCR.`)
+  }
+}
+
+/**
+ * Downloads an ingestion source file, enforcing the {@link MAX_FILE_SIZE} document
+ * limit. `maxBytes` aborts the streaming read once the cap is exceeded (and rejects
+ * up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl`
+ * pointing at an unbounded body cannot exhaust the processing worker's memory.
+ */
+async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise<Buffer> {
+  return downloadFileFromUrl(fileUrl, {
+    timeoutMs: TIMEOUTS.FILE_DOWNLOAD,
+    maxBytes: MAX_FILE_SIZE,
+    userId,
+  })
+}
+
+async function downloadFileForBase64(fileUrl: string, userId?: string): Promise<Buffer> {
+  if (/^data:/i.test(fileUrl)) {
+    const [, base64Data] = fileUrl.split(',')
+    if (!base64Data) {
+      throw new Error('Invalid data URI format')
+    }
+    return Buffer.from(base64Data, 'base64')
+  }
+  if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) {
+    return downloadFileWithTimeout(fileUrl, userId)
+  }
+  throw new Error(
+    'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed'
+  )
+}
+
+function processOCRContent(result: OCRResult, filename: string): string {
+  if (!result.success) {
+    throw new Error(`OCR processing failed: ${result.error || 'Unknown error'}`)
+  }
+
+  const content = result.output?.content || ''
+  if (!content.trim()) {
+    throw new Error('OCR returned empty content')
+  }
+
+  logger.info(`OCR completed: ${filename}`)
+  return content
+}
+
+function validateOCRConfig(
+  apiKey?: string,
+  endpoint?: string,
+  modelName?: string,
+  service = 'OCR'
+) {
+  if (!apiKey) throw new Error(`${service} API key required`)
+  if (!endpoint) throw new Error(`${service} endpoint required`)
+  if (!modelName) throw new Error(`${service} model name required`)
+}
+
+function extractPageContent(pages: OCRPage[]): string {
+  if (!pages?.length) return ''
+
+  return pages
+    .map((page) => page?.markdown || '')
+    .filter(Boolean)
+    .join('\n\n')
+}
+
+async function makeOCRRequest(
+  endpoint: string,
+  headers: Record<string, string>,
+  body: OCRRequestBody
+): Promise<Response> {
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), TIMEOUTS.MISTRAL_OCR_API)
+
+  try {
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    })
+
+    clearTimeout(timeoutId)
+
+    if (!response.ok) {
+      const errorText = await response.text()
+      throw new APIError(
+        `OCR failed: ${response.status} ${response.statusText} - ${errorText}`,
+        response.status
+      )
+    }
+
+    return response
+  } catch (error) {
+    clearTimeout(timeoutId)
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error('OCR API request timed out')
+    }
+    throw error
+  }
+}
+
+async function parseWithAzureMistralOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string
+) {
+  validateOCRConfig(
+    env.OCR_AZURE_API_KEY,
+    env.OCR_AZURE_ENDPOINT,
+    env.OCR_AZURE_MODEL_NAME,
+    'Azure Mistral OCR'
+  )
+
+  const fileBuffer = await downloadFileForBase64(fileUrl, userId)
+
+  if (mimeType === 'application/pdf') {
+    const pageCount = await getPdfPageCount(fileBuffer)
+    if (pageCount > MISTRAL_MAX_PAGES) {
+      logger.info(
+        `PDF has ${pageCount} pages, exceeds Azure OCR limit of ${MISTRAL_MAX_PAGES}. ` +
+          `Falling back to file parser.`
+      )
+      return parseWithFileParser(fileUrl, filename, mimeType, userId)
+    }
+    logger.info(`Azure Mistral OCR: PDF page count for ${filename}: ${pageCount}`)
+  }
+
+  const base64Data = fileBuffer.toString('base64')
+  const dataUri = `data:${mimeType};base64,${base64Data}`
+
+  try {
+    const response = await retryWithExponentialBackoff(
+      () =>
+        makeOCRRequest(
+          env.OCR_AZURE_ENDPOINT!,
+          {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${env.OCR_AZURE_API_KEY}`,
+          },
+          {
+            model: env.OCR_AZURE_MODEL_NAME!,
+            document: {
+              type: 'document_url',
+              document_url: dataUri,
+            },
+            include_image_base64: false,
+          }
+        ),
+      { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
+    )
+
+    const ocrResult = (await response.json()) as AzureOCRResponse
+    const content = extractPageContent(ocrResult.pages || []) || JSON.stringify(ocrResult, null, 2)
+
+    if (!content.trim()) {
+      throw new Error('Azure Mistral OCR returned empty content')
+    }
+
+    logger.info(`Azure Mistral OCR completed: ${filename}`)
+    return { content, processingMethod: 'mistral-ocr' as const, cloudUrl: undefined }
+  } catch (error) {
+    logger.error(`Azure Mistral OCR failed for ${filename}:`, {
+      message: toError(error).message,
+    })
+
+    logger.info(`Falling back to file parser: ${filename}`)
+    return parseWithFileParser(fileUrl, filename, mimeType, userId)
+  }
+}
+
+async function parseWithMistralOCR(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string,
+  workspaceId?: string | null,
+  mistralApiKey?: string | null
+) {
+  const apiKey = mistralApiKey || env.MISTRAL_API_KEY
+  if (!apiKey) {
+    throw new Error('Mistral API key required')
+  }
+
+  if (!mistralParserTool.request?.body) {
+    throw new Error('Mistral parser tool not configured')
+  }
+
+  const { httpsUrl, cloudUrl, buffer } = await handleFileForOCR(
+    fileUrl,
+    filename,
+    mimeType,
+    userId,
+    workspaceId
+  )
+
+  logger.info(`Mistral OCR: Using presigned URL for ${filename}: ${httpsUrl}`)
+
+  let pageCount = 0
+  if (mimeType === 'application/pdf' && buffer) {
+    pageCount = await getPdfPageCount(buffer)
+    logger.info(`PDF page count for ${filename}: ${pageCount}`)
+  }
+
+  const needsBatching = pageCount > MISTRAL_MAX_PAGES
+
+  if (needsBatching && buffer) {
+    logger.info(
+      `PDF has ${pageCount} pages, exceeds limit of ${MISTRAL_MAX_PAGES}. Splitting and processing in chunks.`
+    )
+    return processMistralOCRInBatches(filename, apiKey, buffer, userId, cloudUrl)
+  }
+
+  const params = { filePath: httpsUrl, apiKey, resultType: 'text' as const }
+
+  try {
+    const response = await executeMistralOCRRequest(params, userId)
+    const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult
+    const content = processOCRContent(result, filename)
+
+    return { content, processingMethod: 'mistral-ocr' as const, cloudUrl }
+  } catch (error) {
+    logger.error(`Mistral OCR failed for ${filename}:`, {
+      message: toError(error).message,
+    })
+
+    logger.info(`Falling back to file parser: ${filename}`)
+    return parseWithFileParser(fileUrl, filename, mimeType, userId)
+  }
+}
+
+async function executeMistralOCRRequest(
+  params: { filePath: string; apiKey: string; resultType: 'text' },
+  userId?: string
+): Promise<Response> {
+  return retryWithExponentialBackoff(
+    async () => {
+      let url =
+        typeof mistralParserTool.request!.url === 'function'
+          ? mistralParserTool.request!.url(params)
+          : mistralParserTool.request!.url
+
+      const isInternalRoute = url.startsWith('/')
+
+      if (isInternalRoute) {
+        const { getInternalApiBaseUrl } = await import('@/lib/core/utils/urls')
+        url = `${getInternalApiBaseUrl()}${url}`
+      }
+
+      let headers =
+        typeof mistralParserTool.request!.headers === 'function'
+          ? mistralParserTool.request!.headers(params)
+          : mistralParserTool.request!.headers
+
+      if (isInternalRoute) {
+        const { generateInternalToken } = await import('@/lib/auth/internal')
+        const internalToken = await generateInternalToken(userId)
+        headers = {
+          ...headers,
+          Authorization: `Bearer ${internalToken}`,
+        }
+      }
+
+      const requestBody = mistralParserTool.request!.body!(params) as OCRRequestBody
+      return makeOCRRequest(url, headers as Record<string, string>, requestBody)
+    },
+    { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 }
+  )
+}
+
+async function processChunk(
+  chunk: { buffer: Buffer; startPage: number; endPage: number },
+  chunkIndex: number,
+  totalChunks: number,
+  filename: string,
+  apiKey: string,
+  userId?: string
+): Promise<{ index: number; content: string | null }> {
+  const chunkPageCount = chunk.endPage - chunk.startPage + 1
+
+  logger.info(
+    `Processing chunk ${chunkIndex + 1}/${totalChunks} (pages ${chunk.startPage + 1}-${chunk.endPage + 1}, ${chunkPageCount} pages)`
+  )
+
+  let uploadedKey: string | null = null
+
+  try {
+    const timestamp = Date.now()
+    const uniqueId = randomBytes(8).toString('hex')
+    const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
+    const chunkKey = `kb/${timestamp}-${uniqueId}-chunk${chunkIndex + 1}-${safeFileName}`
+
+    // No metadata: these chunks are ephemeral OCR artifacts (deleted in the
+    // finally below) that are fetched via a direct presigned URL, never through
+    // verifyKBFileAccess. Omitting metadata avoids writing an orphan ownership
+    // binding row per chunk.
+    const uploadResult = await StorageService.uploadFile({
+      file: chunk.buffer,
+      fileName: `${filename}_chunk${chunkIndex + 1}`,
+      contentType: 'application/pdf',
+      context: 'knowledge-base',
+      customKey: chunkKey,
+    })
+
+    uploadedKey = uploadResult.key
+
+    const chunkUrl = await StorageService.generatePresignedDownloadUrl(
+      uploadResult.key,
+      'knowledge-base',
+      900 // 15 minutes
+    )
+
+    logger.info(`Uploaded chunk ${chunkIndex + 1} to S3: ${chunkKey}`)
+
+    const params = {
+      filePath: chunkUrl,
+      apiKey,
+      resultType: 'text' as const,
+    }
+
+    const response = await executeMistralOCRRequest(params, userId)
+    const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult
+
+    if (result.success && result.output?.content) {
+      logger.info(`Chunk ${chunkIndex + 1}/${totalChunks} completed successfully`)
+      return { index: chunkIndex, content: result.output.content }
+    }
+    logger.warn(`Chunk ${chunkIndex + 1}/${totalChunks} returned no content`)
+    return { index: chunkIndex, content: null }
+  } catch (error) {
+    logger.error(`Chunk ${chunkIndex + 1}/${totalChunks} failed:`, {
+      message: toError(error).message,
+    })
+    return { index: chunkIndex, content: null }
+  } finally {
+    if (uploadedKey) {
+      try {
+        await StorageService.deleteFile({ key: uploadedKey, context: 'knowledge-base' })
+        logger.info(`Cleaned up chunk ${chunkIndex + 1} from S3`)
+      } catch (deleteError) {
+        logger.warn(`Failed to clean up chunk ${chunkIndex + 1} from S3:`, {
+          message: toError(deleteError).message,
+        })
+      }
+    }
+  }
+}
+
+async function processMistralOCRInBatches(
+  filename: string,
+  apiKey: string,
+  pdfBuffer: Buffer,
+  userId?: string,
+  cloudUrl?: string
+): Promise<{
+  content: string
+  processingMethod: 'mistral-ocr'
+  cloudUrl?: string
+}> {
+  const totalPages = await getPdfPageCount(pdfBuffer)
+  logger.info(
+    `Splitting ${filename} (${totalPages} pages) into chunks of ${MISTRAL_MAX_PAGES} pages`
+  )
+
+  const pdfChunks = await splitPdfIntoChunks(pdfBuffer, MISTRAL_MAX_PAGES)
+  logger.info(
+    `Split into ${pdfChunks.length} chunks, processing with concurrency ${MAX_CONCURRENT_CHUNKS}`
+  )
+
+  const results: { index: number; content: string | null }[] = []
+
+  for (let i = 0; i < pdfChunks.length; i += MAX_CONCURRENT_CHUNKS) {
+    const batch = pdfChunks.slice(i, i + MAX_CONCURRENT_CHUNKS)
+    const batchPromises = batch.map((chunk, batchIndex) =>
+      processChunk(chunk, i + batchIndex, pdfChunks.length, filename, apiKey, userId)
+    )
+
+    const batchResults = await Promise.all(batchPromises)
+    for (const result of batchResults) {
+      results.push(result)
+    }
+
+    logger.info(
+      `Completed batch ${Math.floor(i / MAX_CONCURRENT_CHUNKS) + 1}/${Math.ceil(pdfChunks.length / MAX_CONCURRENT_CHUNKS)}`
+    )
+  }
+
+  const sortedResults = results
+    .sort((a, b) => a.index - b.index)
+    .filter((r) => r.content !== null)
+    .map((r) => r.content as string)
+
+  if (sortedResults.length === 0) {
+    throw new Error(
+      `OCR failed for all ${pdfChunks.length} chunks of ${filename}. ` +
+        `Large PDFs require OCR - file parser fallback would produce poor results.`
+    )
+  }
+
+  const combinedContent = sortedResults.join('\n\n')
+  logger.info(
+    `Successfully processed ${sortedResults.length}/${pdfChunks.length} chunks for ${filename}`
+  )
+
+  return {
+    content: combinedContent,
+    processingMethod: 'mistral-ocr',
+    cloudUrl,
+  }
+}
+
+async function parseWithFileParser(
+  fileUrl: string,
+  filename: string,
+  mimeType: string,
+  userId?: string
+) {
+  try {
+    let content: string
+    let metadata: FileParseMetadata = {}
+
+    if (/^data:/i.test(fileUrl)) {
+      content = await parseDataURI(fileUrl, filename, mimeType)
+    } else if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) {
+      // Internal URLs may arrive as an app-relative `/api/files/serve/...` path
+      // (some ingestion callers store the relative path); downloadFileFromUrl
+      // resolves it directly against storage without an absolute origin.
+      const result = await parseHttpFile(fileUrl, filename, mimeType, userId)
+      content = result.content
+      metadata = result.metadata || {}
+    } else {
+      throw new Error(
+        'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed'
+      )
+    }
+
+    if (!content.trim()) {
+      throw new Error('File parser returned empty content')
+    }
+
+    return { content, processingMethod: 'file-parser' as const, cloudUrl: undefined, metadata }
+  } catch (error) {
+    logger.error(`File parser failed for ${filename}:`, error)
+    throw error
+  }
+}
+
+async function parseDataURI(fileUrl: string, filename: string, mimeType: string): Promise<string> {
+  const [header, base64Data] = fileUrl.split(',')
+  if (!base64Data) {
+    throw new Error('Invalid data URI format')
+  }
+
+  if (mimeType === 'text/plain') {
+    return header.includes('base64')
+      ? Buffer.from(base64Data, 'base64').toString('utf8')
+      : decodeURIComponent(base64Data)
+  }
+
+  const extension = resolveParserExtension(filename, mimeType, 'txt')
+  const buffer = Buffer.from(base64Data, 'base64')
+  const result = await parseBuffer(buffer, extension)
+  return result.content
+}
+
+async function parseHttpFile(
+  fileUrl: string,
+  filename: string,
+  mimeType?: string,
+  userId?: string
+): Promise<{ content: string; metadata?: FileParseMetadata }> {
+  const buffer = await downloadFileWithTimeout(fileUrl, userId)
+
+  const extension = resolveParserExtension(filename, mimeType)
+  const result = await parseBuffer(buffer, extension)
+  return result
+}

apps/sim/lib/knowledge/documents/lock-order.test.ts

@@ -0,0 +1,44 @@
+/**
+ * @vitest-environment node
+ *
+ * Lock-order regression guard: `updateDocument` must lock the document's
+ * embedding rows BEFORE the document row when cascading tag updates, matching
+ * the embedding → document order every chunk-mutation path uses
+ * (chunks/service.ts). The opposite order deadlocks a document tag edit against
+ * a concurrent chunk edit of the same document.
+ */
+import { document, embedding } from '@sim/db/schema'
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { updateDocument } from '@/lib/knowledge/documents/service'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+/** invocationCallOrder of the first `tx.update(table)` call. */
+function updateOrderForTable(table: unknown): number {
+  const { calls, invocationCallOrder } = dbChainMockFns.update.mock
+  for (let i = 0; i < calls.length; i++) {
+    if (calls[i][0] === table) return invocationCallOrder[i]
+  }
+  return -1
+}
+
+describe('updateDocument lock ordering', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    // Post-transaction re-read of the updated document must return a row.
+    dbChainMockFns.limit.mockResolvedValue([{ id: 'doc-1', knowledgeBaseId: 'kb-1' }])
+  })
+
+  it('updates embeddings before the document row when cascading tag changes', async () => {
+    await updateDocument('doc-1', { tag1: 'priority' }, 'req-1')
+
+    const embeddingWriteOrder = updateOrderForTable(embedding)
+    const documentWriteOrder = updateOrderForTable(document)
+
+    expect(embeddingWriteOrder).toBeGreaterThan(0)
+    expect(documentWriteOrder).toBeGreaterThan(0)
+    expect(embeddingWriteOrder).toBeLessThan(documentWriteOrder)
+  })
+})

apps/sim/lib/knowledge/documents/parser-extension.test.ts

@@ -0,0 +1,27 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
+
+describe('resolveParserExtension', () => {
+  it('uses a supported filename extension when present', () => {
+    expect(resolveParserExtension('report.pdf', 'application/pdf')).toBe('pdf')
+  })
+
+  it('falls back to mime type when filename has no extension', () => {
+    expect(
+      resolveParserExtension('[Business] Your Thursday morning trip with Uber', 'text/plain')
+    ).toBe('txt')
+  })
+
+  it('falls back to mime type when filename extension is unsupported', () => {
+    expect(resolveParserExtension('uber-message.business', 'text/plain')).toBe('txt')
+  })
+
+  it('throws when neither filename nor mime type resolves to a supported parser', () => {
+    expect(() =>
+      resolveParserExtension('uber-message.unknown', 'application/octet-stream')
+    ).toThrow('Unsupported file type')
+  })
+})

apps/sim/lib/knowledge/documents/parser-extension.ts

@@ -0,0 +1,38 @@
+import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils'
+import {
+  isAlphanumericExtension,
+  isSupportedExtension,
+  SUPPORTED_DOCUMENT_EXTENSIONS,
+} from '@/lib/uploads/utils/validation'
+
+const SUPPORTED_EXTENSIONS_TEXT = SUPPORTED_DOCUMENT_EXTENSIONS.join(', ')
+
+export function resolveParserExtension(
+  filename: string,
+  mimeType?: string,
+  fallback?: string
+): string {
+  const raw = filename.includes('.') ? filename.split('.').pop()?.toLowerCase() : undefined
+  const filenameExtension = raw && isAlphanumericExtension(raw) ? raw : undefined
+
+  if (filenameExtension && isSupportedExtension(filenameExtension)) {
+    return filenameExtension
+  }
+
+  const mimeExtension = mimeType ? getExtensionFromMimeType(mimeType) : undefined
+  if (mimeExtension && isSupportedExtension(mimeExtension)) {
+    return mimeExtension
+  }
+
+  if (fallback) {
+    return fallback
+  }
+
+  if (filenameExtension) {
+    throw new Error(
+      `Unsupported file type: ${filenameExtension}. Supported types are: ${SUPPORTED_EXTENSIONS_TEXT}`
+    )
+  }
+
+  throw new Error(`Could not determine file type for ${filename || 'document'}`)
+}

apps/sim/lib/knowledge/documents/secure-fetch.server.ts

@@ -0,0 +1,71 @@
+import {
+  type SecureFetchOptions,
+  type SecureFetchResponse,
+  secureFetchWithValidation,
+} from '@/lib/core/security/input-validation.server'
+import {
+  type HTTPError,
+  isRetryableError,
+  type RetryOptions,
+  retryWithExponentialBackoff,
+} from '@/lib/knowledge/documents/utils'
+
+export interface SecureFetchRetryOptions extends RetryOptions {
+  allowHttp?: boolean
+  timeout?: number
+  maxResponseBytes?: number
+}
+
+/**
+ * SSRF-safe counterpart to {@link fetchWithRetry} for connector requests to
+ * user-controlled hosts. Every attempt re-runs {@link secureFetchWithValidation}
+ * (DNS resolution, private/loopback/reserved-IP rejection, IP-pinned connection,
+ * redirect re-validation); retry/backoff semantics mirror {@link fetchWithRetry}.
+ *
+ * Lives in a `.server.ts` module because it pulls in Node-only `dns/promises`
+ * via {@link secureFetchWithValidation}; importing it from the shared
+ * `documents/utils` barrel would drag that into client bundles.
+ */
+export async function secureFetchWithRetry(
+  url: string,
+  options: SecureFetchOptions = {},
+  retryOptions: SecureFetchRetryOptions = {}
+): Promise<SecureFetchResponse> {
+  const { allowHttp, timeout, maxResponseBytes, ...retry } = retryOptions
+
+  return retryWithExponentialBackoff(async () => {
+    const response = await secureFetchWithValidation(
+      url,
+      {
+        ...options,
+        ...(allowHttp !== undefined ? { allowHttp } : {}),
+        ...(timeout !== undefined ? { timeout } : {}),
+        ...(maxResponseBytes !== undefined ? { maxResponseBytes } : {}),
+      },
+      'url'
+    )
+
+    if (!response.ok && isRetryableError({ status: response.status })) {
+      const errorText = await response.text()
+      const error: HTTPError = new Error(
+        `HTTP ${response.status}: ${response.statusText} - ${errorText}`
+      )
+      error.status = response.status
+      error.statusText = response.statusText
+
+      const retryAfter = response.headers.get('retry-after')
+      if (retryAfter) {
+        const waitMs = Number.isNaN(Number(retryAfter))
+          ? Math.max(0, new Date(retryAfter).getTime() - Date.now())
+          : Number(retryAfter) * 1000
+        if (waitMs > 0) {
+          error.retryAfterMs = waitMs
+        }
+      }
+
+      throw error
+    }
+
+    return response
+  }, retry)
+}

apps/sim/lib/knowledge/documents/tag-filter.test.ts

@@ -0,0 +1,177 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { buildTagFilterCondition } from '@/lib/knowledge/documents/tag-filter'
+
+/**
+ * The global `drizzle-orm` mock renders `sql` fragments to a `?`-placeholder
+ * string via `toSQL()` and returns plain `{ type, left, right }` objects for the
+ * comparison operators, so we can assert the exact predicate each filter builds.
+ */
+function rendered(condition: ReturnType<typeof buildTagFilterCondition>) {
+  return (condition as unknown as { toSQL: () => { sql: string; params: unknown[] } }).toSQL()
+}
+
+describe('buildTagFilterCondition', () => {
+  it('ignores unknown tag slots', () => {
+    expect(
+      buildTagFilterCondition({
+        tagSlot: 'not_a_real_slot',
+        fieldType: 'text',
+        operator: 'eq',
+        value: 'x',
+      })
+    ).toBeUndefined()
+  })
+
+  describe('text', () => {
+    it('matches eq case-insensitively', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'eq',
+          value: 'Ada Lovelace',
+        })
+      )
+      expect(sql).toBe('LOWER(?) = LOWER(?)')
+      expect(params).toEqual(['tag1', 'Ada Lovelace'])
+    })
+
+    it('matches neq case-insensitively', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag2',
+          fieldType: 'text',
+          operator: 'neq',
+          value: 'Spreadsheet',
+        })
+      )
+      expect(sql).toBe('LOWER(?) != LOWER(?)')
+      expect(params).toEqual(['tag2', 'Spreadsheet'])
+    })
+
+    it('escapes LIKE wildcards in contains', () => {
+      const { params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'contains',
+          value: '50%_off',
+        })
+      )
+      expect(params).toContain('%50\\%\\_off%')
+    })
+
+    it('returns undefined for an unsupported operator', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'tag1',
+          fieldType: 'text',
+          operator: 'gt',
+          value: 'x',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('date', () => {
+    it('compares eq on the calendar day', () => {
+      const { sql, params } = rendered(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'eq',
+          value: '2026-04-21',
+        })
+      )
+      expect(sql).toBe('?::date = ?::date')
+      expect(params).toEqual(['date1', '2026-04-21'])
+    })
+
+    it('compares range bounds on the calendar day', () => {
+      const condition = buildTagFilterCondition({
+        tagSlot: 'date1',
+        fieldType: 'date',
+        operator: 'between',
+        value: '2026-04-01',
+        valueTo: '2026-04-30',
+      }) as unknown as { type: string; conditions: unknown[] }
+      expect(condition.type).toBe('and')
+      expect(condition.conditions).toHaveLength(2)
+      expect(rendered(condition.conditions[0] as never).sql).toBe('?::date >= ?::date')
+      expect(rendered(condition.conditions[1] as never).sql).toBe('?::date <= ?::date')
+    })
+
+    it('ignores values that are not YYYY-MM-DD', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'eq',
+          value: 'not-a-date',
+        })
+      ).toBeUndefined()
+    })
+
+    it('ignores a between filter missing its upper bound', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'date1',
+          fieldType: 'date',
+          operator: 'between',
+          value: '2026-04-01',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('number', () => {
+    it('builds an equality comparison', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'number1',
+          fieldType: 'number',
+          operator: 'eq',
+          value: '42',
+        })
+      ).toEqual({ type: 'eq', left: 'number1', right: 42 })
+    })
+
+    it('ignores non-numeric values', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'number1',
+          fieldType: 'number',
+          operator: 'eq',
+          value: 'abc',
+        })
+      ).toBeUndefined()
+    })
+  })
+
+  describe('boolean', () => {
+    it('parses string values', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'boolean1',
+          fieldType: 'boolean',
+          operator: 'eq',
+          value: 'true',
+        })
+      ).toEqual({ type: 'eq', left: 'boolean1', right: true })
+    })
+
+    it('ignores values that are not boolean-like', () => {
+      expect(
+        buildTagFilterCondition({
+          tagSlot: 'boolean1',
+          fieldType: 'boolean',
+          operator: 'eq',
+          value: 'maybe',
+        })
+      ).toBeUndefined()
+    })
+  })
+})

apps/sim/lib/knowledge/documents/tag-filter.ts

@@ -0,0 +1,153 @@
+import { document } from '@sim/db/schema'
+import { and, eq, gt, gte, lt, lte, ne, type SQL, sql } from 'drizzle-orm'
+import { parseBooleanValue } from '@/lib/knowledge/tags/utils'
+
+/**
+ * A single tag filter applied to a document list query.
+ */
+export interface TagFilterCondition {
+  tagSlot: string
+  fieldType: 'text' | 'number' | 'date' | 'boolean'
+  operator: string
+  value: unknown
+  valueTo?: unknown
+}
+
+const ALLOWED_TAG_SLOTS = new Set([
+  'tag1',
+  'tag2',
+  'tag3',
+  'tag4',
+  'tag5',
+  'tag6',
+  'tag7',
+  'number1',
+  'number2',
+  'number3',
+  'number4',
+  'number5',
+  'date1',
+  'date2',
+  'boolean1',
+  'boolean2',
+  'boolean3',
+])
+
+const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/
+
+function escapeLikePattern(s: string): string {
+  return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')
+}
+
+/**
+ * Builds a SQL predicate for a single tag filter against the document table.
+ *
+ * Text comparisons are case-insensitive and date comparisons are evaluated on
+ * the calendar day, matching the semantics of the knowledge base search filter
+ * (`app/api/knowledge/search/utils.ts`). Returns `undefined` when the slot,
+ * operator, or value is not usable so the caller can skip the condition.
+ */
+export function buildTagFilterCondition(filter: TagFilterCondition): SQL | undefined {
+  if (!ALLOWED_TAG_SLOTS.has(filter.tagSlot)) return undefined
+
+  const col = document[filter.tagSlot as keyof typeof document]
+
+  if (filter.fieldType === 'text') {
+    const v = String(filter.value ?? '')
+    switch (filter.operator) {
+      case 'eq':
+        return sql`LOWER(${col}) = LOWER(${v})`
+      case 'neq':
+        return sql`LOWER(${col}) != LOWER(${v})`
+      case 'contains': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'not_contains': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) NOT LIKE LOWER(${`%${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'starts_with': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`${escaped}%`}) ESCAPE '\\'`
+      }
+      case 'ends_with': {
+        const escaped = escapeLikePattern(v)
+        return sql`LOWER(${col}) LIKE LOWER(${`%${escaped}`}) ESCAPE '\\'`
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'number') {
+    const num = Number(filter.value)
+    if (Number.isNaN(num)) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return eq(col as typeof document.number1, num)
+      case 'neq':
+        return ne(col as typeof document.number1, num)
+      case 'gt':
+        return gt(col as typeof document.number1, num)
+      case 'gte':
+        return gte(col as typeof document.number1, num)
+      case 'lt':
+        return lt(col as typeof document.number1, num)
+      case 'lte':
+        return lte(col as typeof document.number1, num)
+      case 'between': {
+        const numTo = Number(filter.valueTo)
+        if (Number.isNaN(numTo)) return undefined
+        return and(
+          gte(col as typeof document.number1, num),
+          lte(col as typeof document.number1, numTo)
+        )
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'date') {
+    const v = String(filter.value ?? '')
+    if (!DATE_ONLY_PATTERN.test(v)) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return sql`${col}::date = ${v}::date`
+      case 'neq':
+        return sql`${col}::date != ${v}::date`
+      case 'gt':
+        return sql`${col}::date > ${v}::date`
+      case 'gte':
+        return sql`${col}::date >= ${v}::date`
+      case 'lt':
+        return sql`${col}::date < ${v}::date`
+      case 'lte':
+        return sql`${col}::date <= ${v}::date`
+      case 'between': {
+        const valueTo = String(filter.valueTo ?? '')
+        if (!DATE_ONLY_PATTERN.test(valueTo)) return undefined
+        return and(sql`${col}::date >= ${v}::date`, sql`${col}::date <= ${valueTo}::date`)
+      }
+      default:
+        return undefined
+    }
+  }
+
+  if (filter.fieldType === 'boolean') {
+    const boolVal =
+      typeof filter.value === 'boolean' ? filter.value : parseBooleanValue(String(filter.value))
+    if (boolVal === null) return undefined
+    switch (filter.operator) {
+      case 'eq':
+        return eq(col as typeof document.boolean1, boolVal)
+      case 'neq':
+        return ne(col as typeof document.boolean1, boolVal)
+      default:
+        return undefined
+    }
+  }
+
+  return undefined
+}

apps/sim/lib/knowledge/documents/types.ts

@@ -0,0 +1,26 @@
+// Document sorting options
+export type DocumentSortField =
+  | 'filename'
+  | 'fileSize'
+  | 'tokenCount'
+  | 'chunkCount'
+  | 'uploadedAt'
+  | 'processingStatus'
+  | 'enabled'
+export type SortOrder = 'asc' | 'desc'
+
+interface DocumentSortOptions {
+  sortBy?: DocumentSortField
+  sortOrder?: SortOrder
+}
+
+interface HeaderInfo {
+  /** Header text */
+  text: string
+  /** Header level (1-6) */
+  level: number
+  /** Anchor link */
+  anchor: string
+  /** Position in document */
+  position: number
+}

apps/sim/lib/knowledge/documents/utils.test.ts

@@ -0,0 +1,280 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockSecureFetchWithValidation } = vi.hoisted(() => ({
+  mockSecureFetchWithValidation: vi.fn(),
+}))
+
+vi.mock('@/lib/core/security/input-validation.server', () => ({
+  secureFetchWithValidation: mockSecureFetchWithValidation,
+}))
+
+import { secureFetchWithRetry } from './secure-fetch.server'
+import { isRetryableError } from './utils'
+
+/** Builds a minimal SecureFetchResponse-shaped object for tests. */
+function fakeResponse(
+  status: number,
+  options: { headers?: Record<string, string>; body?: string } = {}
+) {
+  const headers = options.headers ?? {}
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    statusText: `status-${status}`,
+    headers: { get: (name: string) => headers[name.toLowerCase()] ?? null },
+    body: null,
+    text: async () => options.body ?? '',
+    json: async () => JSON.parse(options.body ?? '{}'),
+    arrayBuffer: async () => new ArrayBuffer(0),
+  }
+}
+
+const FAST_RETRY = { initialDelayMs: 1, maxDelayMs: 2, maxRetries: 3 }
+
+describe('isRetryableError', () => {
+  describe('retryable status codes', () => {
+    it.concurrent('returns true for 429 on Error with status', () => {
+      const error = Object.assign(new Error('Too Many Requests'), { status: 429 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 502 on Error with status', () => {
+      const error = Object.assign(new Error('Bad Gateway'), { status: 502 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 503 on Error with status', () => {
+      const error = Object.assign(new Error('Service Unavailable'), { status: 503 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for 504 on Error with status', () => {
+      const error = Object.assign(new Error('Gateway Timeout'), { status: 504 })
+      expect(isRetryableError(error)).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 429', () => {
+      expect(isRetryableError({ status: 429 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 502', () => {
+      expect(isRetryableError({ status: 502 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 503', () => {
+      expect(isRetryableError({ status: 503 })).toBe(true)
+    })
+
+    it.concurrent('returns true for plain object with status 504', () => {
+      expect(isRetryableError({ status: 504 })).toBe(true)
+    })
+  })
+
+  describe('non-retryable status codes', () => {
+    it.concurrent('returns false for 400', () => {
+      const error = Object.assign(new Error('Bad Request'), { status: 400 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 401', () => {
+      const error = Object.assign(new Error('Unauthorized'), { status: 401 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 403', () => {
+      const error = Object.assign(new Error('Forbidden'), { status: 403 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 404', () => {
+      const error = Object.assign(new Error('Not Found'), { status: 404 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+
+    it.concurrent('returns false for 500', () => {
+      const error = Object.assign(new Error('Internal Server Error'), { status: 500 })
+      expect(isRetryableError(error)).toBe(false)
+    })
+  })
+
+  describe('retryable error messages', () => {
+    it.concurrent('returns true for "rate limit" in message', () => {
+      expect(isRetryableError(new Error('You have hit the rate limit'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "rate_limit" in message', () => {
+      expect(isRetryableError(new Error('rate_limit_exceeded'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "too many requests" in message', () => {
+      expect(isRetryableError(new Error('too many requests, slow down'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "quota exceeded" in message', () => {
+      expect(isRetryableError(new Error('API quota exceeded'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "throttled" in message', () => {
+      expect(isRetryableError(new Error('Request was throttled'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "retry after" in message', () => {
+      expect(isRetryableError(new Error('Please retry after 60 seconds'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "temporarily unavailable" in message', () => {
+      expect(isRetryableError(new Error('Service is temporarily unavailable'))).toBe(true)
+    })
+
+    it.concurrent('returns true for "service unavailable" in message', () => {
+      expect(isRetryableError(new Error('The service unavailable right now'))).toBe(true)
+    })
+
+    it.concurrent('returns true for a transient DNS resolution failure', () => {
+      expect(isRetryableError(new Error('url hostname could not be resolved'))).toBe(true)
+    })
+  })
+
+  describe('case insensitivity', () => {
+    it.concurrent('matches "Rate Limit" with mixed case', () => {
+      expect(isRetryableError(new Error('Rate Limit Exceeded'))).toBe(true)
+    })
+
+    it.concurrent('matches "THROTTLED" in uppercase', () => {
+      expect(isRetryableError(new Error('REQUEST THROTTLED'))).toBe(true)
+    })
+
+    it.concurrent('matches "Too Many Requests" in title case', () => {
+      expect(isRetryableError(new Error('Too Many Requests'))).toBe(true)
+    })
+  })
+
+  describe('null, undefined, and non-error inputs', () => {
+    it.concurrent('returns false for null', () => {
+      expect(isRetryableError(null)).toBe(false)
+    })
+
+    it.concurrent('returns false for undefined', () => {
+      expect(isRetryableError(undefined)).toBe(false)
+    })
+
+    it.concurrent('returns false for empty string', () => {
+      expect(isRetryableError('')).toBe(false)
+    })
+
+    it.concurrent('returns false for a number', () => {
+      expect(isRetryableError(42)).toBe(false)
+    })
+  })
+
+  describe('non-retryable errors', () => {
+    it.concurrent('returns false for Error with no status and unrelated message', () => {
+      expect(isRetryableError(new Error('Something went wrong'))).toBe(false)
+    })
+
+    it.concurrent('returns false for plain object with only non-retryable status', () => {
+      expect(isRetryableError({ status: 404 })).toBe(false)
+    })
+
+    it.concurrent('returns false for plain object with non-retryable status and no message', () => {
+      expect(isRetryableError({ status: 500 })).toBe(false)
+    })
+
+    it.concurrent('returns false for the deterministic blocked-IP SSRF rejection', () => {
+      expect(isRetryableError(new Error('url resolves to a blocked IP address'))).toBe(false)
+    })
+  })
+})
+
+describe('secureFetchWithRetry', () => {
+  beforeEach(() => {
+    mockSecureFetchWithValidation.mockReset()
+  })
+
+  it('routes the request through secureFetchWithValidation and returns the response', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200, { body: 'ok' }))
+
+    const response = await secureFetchWithRetry('https://example.com/api', {
+      method: 'GET',
+      headers: { Accept: 'application/json' },
+    })
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+    const [url, options, paramName] = mockSecureFetchWithValidation.mock.calls[0]
+    expect(url).toBe('https://example.com/api')
+    expect(options).toMatchObject({ method: 'GET', headers: { Accept: 'application/json' } })
+    expect(paramName).toBe('url')
+  })
+
+  it('propagates SSRF validation failures without retrying', async () => {
+    mockSecureFetchWithValidation.mockRejectedValue(
+      new Error('url resolves to a blocked IP address')
+    )
+
+    await expect(
+      secureFetchWithRetry('https://attacker.test', { method: 'GET' }, FAST_RETRY)
+    ).rejects.toThrow('blocked IP address')
+
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries on a retryable status (503) and succeeds', async () => {
+    mockSecureFetchWithValidation
+      .mockResolvedValueOnce(fakeResponse(503, { body: 'try later' }))
+      .mockResolvedValueOnce(fakeResponse(200, { body: 'ok' }))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not retry a non-retryable status (404) and returns it to the caller', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(404, { body: 'missing' }))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(404)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(1)
+  })
+
+  it('forwards allowHttp / timeout / maxResponseBytes to the pinned fetch', async () => {
+    mockSecureFetchWithValidation.mockResolvedValue(fakeResponse(200))
+
+    await secureFetchWithRetry(
+      'http://localhost:9000',
+      { method: 'GET' },
+      { allowHttp: true, timeout: 5000, maxResponseBytes: 1024, ...FAST_RETRY }
+    )
+
+    const [, options] = mockSecureFetchWithValidation.mock.calls[0]
+    expect(options).toMatchObject({ allowHttp: true, timeout: 5000, maxResponseBytes: 1024 })
+  })
+
+  it('honors Retry-After (seconds) on a 429 before retrying', async () => {
+    mockSecureFetchWithValidation
+      .mockResolvedValueOnce(fakeResponse(429, { headers: { 'retry-after': '0' } }))
+      .mockResolvedValueOnce(fakeResponse(200))
+
+    const response = await secureFetchWithRetry(
+      'https://example.com/api',
+      { method: 'GET' },
+      FAST_RETRY
+    )
+
+    expect(response.status).toBe(200)
+    expect(mockSecureFetchWithValidation).toHaveBeenCalledTimes(2)
+  })
+})

apps/sim/lib/knowledge/documents/utils.ts

@@ -0,0 +1,215 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { randomFloat } from '@sim/utils/random'
+
+const logger = createLogger('RetryUtils')
+
+export interface HTTPError extends Error {
+  status?: number
+  statusText?: string
+  retryAfterMs?: number
+}
+
+type RetryableError = HTTPError | Error | { status?: number; message?: string }
+
+export interface RetryOptions {
+  maxRetries?: number
+  initialDelayMs?: number
+  maxDelayMs?: number
+  backoffMultiplier?: number
+  retryCondition?: (error: unknown) => boolean
+}
+
+interface RetryResult<T> {
+  success: boolean
+  data?: T
+  error?: Error
+  attemptCount: number
+}
+
+function hasStatus(
+  error: RetryableError
+): error is HTTPError | { status?: number; message?: string } {
+  return typeof error === 'object' && error !== null && 'status' in error
+}
+
+function isRetryableErrorType(error: unknown): error is RetryableError {
+  if (!error) return false
+  if (error instanceof Error) return true
+  if (typeof error === 'object' && ('status' in error || 'message' in error)) return true
+  return false
+}
+
+/**
+ * Default retry condition for rate limiting errors
+ */
+export function isRetryableError(error: unknown): boolean {
+  if (!isRetryableErrorType(error)) return false
+
+  // Check for rate limiting status codes
+  if (
+    hasStatus(error) &&
+    (error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504)
+  ) {
+    return true
+  }
+
+  // Check for network-level errors (DNS, connection, timeout)
+  const errorMessage = toError(error).message
+  const lowerMessage = errorMessage.toLowerCase()
+
+  const networkKeywords = [
+    'fetch failed',
+    'econnreset',
+    'econnrefused',
+    'etimedout',
+    'enetunreach',
+    'socket hang up',
+    'network error',
+    // Transient DNS resolution failure surfaced by secureFetchWithValidation
+    // before the request is made. The deterministic "resolves to a blocked IP
+    // address" security rejection is a distinct message and stays non-retryable.
+    'could not be resolved',
+  ]
+
+  if (networkKeywords.some((keyword) => lowerMessage.includes(keyword))) {
+    return true
+  }
+
+  // Check for rate limiting in error messages
+  const rateLimitKeywords = [
+    'rate limit',
+    'rate_limit',
+    'too many requests',
+    'quota exceeded',
+    'throttled',
+    'retry after',
+    'temporarily unavailable',
+    'service unavailable',
+  ]
+
+  return rateLimitKeywords.some((keyword) => lowerMessage.includes(keyword))
+}
+
+/**
+ * Executes a function with exponential backoff retry logic
+ */
+export async function retryWithExponentialBackoff<T>(
+  operation: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const {
+    maxRetries = 5,
+    initialDelayMs = 1000,
+    maxDelayMs = 30000,
+    backoffMultiplier = 2,
+    retryCondition = isRetryableError,
+  } = options
+
+  let lastError: Error | undefined
+  let delay = initialDelayMs
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      logger.debug(`Executing operation attempt ${attempt + 1}/${maxRetries + 1}`)
+      const result = await operation()
+
+      if (attempt > 0) {
+        logger.info(`Operation succeeded after ${attempt + 1} attempts`)
+      }
+
+      return result
+    } catch (error) {
+      lastError = toError(error)
+      logger.warn(`Operation failed on attempt ${attempt + 1}`, { error })
+
+      // If this is the last attempt, throw the error
+      if (attempt === maxRetries) {
+        logger.error(`Operation failed after ${maxRetries + 1} attempts`, { error })
+        throw lastError
+      }
+
+      // Check if error is retryable
+      if (!retryCondition(error as RetryableError)) {
+        logger.warn('Error is not retryable, throwing immediately', { error })
+        throw lastError
+      }
+
+      // Use Retry-After if the server told us how long to wait, otherwise exponential backoff.
+      // Cap Retry-After at maxDelayMs to bound total retry duration (matches Google Cloud SDK behavior).
+      const retryAfterMs = (lastError as HTTPError)?.retryAfterMs
+      const cappedRetryAfter = retryAfterMs ? Math.min(retryAfterMs, maxDelayMs) : undefined
+
+      if (retryAfterMs && retryAfterMs > maxDelayMs) {
+        logger.warn(
+          `Retry-After ${retryAfterMs}ms exceeds maxDelayMs ${maxDelayMs}ms — capping to ${maxDelayMs}ms`
+        )
+      }
+
+      const jitter = randomFloat() * 0.1 * delay
+      const actualDelay = cappedRetryAfter ?? Math.min(delay + jitter, maxDelayMs)
+
+      logger.info(
+        `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${cappedRetryAfter ? ' (Retry-After)' : ''}`
+      )
+
+      await sleep(actualDelay)
+
+      // Exponential backoff (skip if we used Retry-After)
+      if (!cappedRetryAfter) {
+        delay = Math.min(delay * backoffMultiplier, maxDelayMs)
+      }
+    }
+  }
+
+  throw lastError || new Error('Retry operation failed')
+}
+
+/**
+ * Tighter retry options for user-facing operations (e.g. validateConfig).
+ * Caps total wait at ~7s instead of ~31s to avoid API route timeouts.
+ */
+export const VALIDATE_RETRY_OPTIONS: RetryOptions = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 10000,
+}
+
+/**
+ * Wrapper for fetch requests with retry logic
+ */
+export async function fetchWithRetry(
+  url: string,
+  options: RequestInit = {},
+  retryOptions: RetryOptions = {}
+): Promise<Response> {
+  return retryWithExponentialBackoff(async () => {
+    const response = await fetch(url, options)
+
+    // If response is not ok and status indicates rate limiting, throw an error
+    if (!response.ok && isRetryableError({ status: response.status })) {
+      const errorText = await response.text()
+      const error: HTTPError = new Error(
+        `HTTP ${response.status}: ${response.statusText} - ${errorText}`
+      )
+      error.status = response.status
+      error.statusText = response.statusText
+
+      // Pass Retry-After to the retry loop so it replaces exponential backoff
+      const retryAfter = response.headers.get('Retry-After')
+      if (retryAfter) {
+        const waitMs = Number.isNaN(Number(retryAfter))
+          ? Math.max(0, new Date(retryAfter).getTime() - Date.now())
+          : Number(retryAfter) * 1000
+        if (waitMs > 0) {
+          error.retryAfterMs = waitMs
+        }
+      }
+
+      throw error
+    }
+
+    return response
+  }, retryOptions)
+}

apps/sim/lib/knowledge/embedding-models.ts

@@ -0,0 +1,48 @@
+/**
+ * Registry of embedding models supported by the platform.
+ * Selection happens server-side via the `KB_EMBEDDING_MODEL` env var; this
+ * registry exists to resolve provider, tokenizer, and pricing metadata at
+ * runtime for any model recorded on a knowledge base row.
+ */
+
+export const EMBEDDING_DIMENSIONS = 1536 as const
+
+export const DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small'
+
+export type EmbeddingProviderKind = 'openai' | 'azure-openai' | 'gemini'
+
+export type TokenizerProviderId = 'openai' | 'google'
+
+export interface EmbeddingModelInfo {
+  provider: EmbeddingProviderKind
+  /** Pricing/billing label — must match an entry in EMBEDDING_MODEL_PRICING when billed. */
+  pricingId: string
+  /** Provider id for `estimateTokenCount` so token counts match the embedding provider's tokenization. */
+  tokenizerProvider: TokenizerProviderId
+}
+
+export const SUPPORTED_EMBEDDING_MODELS: Partial<Record<string, EmbeddingModelInfo>> = {
+  'text-embedding-3-small': {
+    provider: 'openai',
+    pricingId: 'text-embedding-3-small',
+    tokenizerProvider: 'openai',
+  },
+  'text-embedding-3-large': {
+    provider: 'openai',
+    pricingId: 'text-embedding-3-large',
+    tokenizerProvider: 'openai',
+  },
+  'gemini-embedding-001': {
+    provider: 'gemini',
+    pricingId: 'gemini-embedding-001',
+    tokenizerProvider: 'google',
+  },
+}
+
+export function getEmbeddingModelInfo(model: string): EmbeddingModelInfo {
+  const info = SUPPORTED_EMBEDDING_MODELS[model]
+  if (!info) {
+    throw new Error(`Unsupported embedding model: ${model}`)
+  }
+  return info
+}

apps/sim/lib/knowledge/embeddings.ts

@@ -0,0 +1,451 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { getBYOKKey } from '@/lib/api-key/byok'
+import { recordUsage } from '@/lib/billing/core/usage-log'
+import { getRotatingApiKey } from '@/lib/core/config/api-keys'
+import { env, envNumber } from '@/lib/core/config/env'
+import { isRetryableError, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
+import {
+  DEFAULT_EMBEDDING_MODEL,
+  EMBEDDING_DIMENSIONS,
+  getEmbeddingModelInfo,
+  SUPPORTED_EMBEDDING_MODELS,
+  type TokenizerProviderId,
+} from '@/lib/knowledge/embedding-models'
+import { batchByTokenLimit, estimateTokenCount } from '@/lib/tokenization'
+import { calculateCost } from '@/providers/utils'
+
+const logger = createLogger('EmbeddingUtils')
+
+const MAX_TOKENS_PER_REQUEST = 8000
+const MAX_CONCURRENT_BATCHES = envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 50)
+const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000
+
+export { EMBEDDING_DIMENSIONS } from '@/lib/knowledge/embedding-models'
+
+class EmbeddingAPIError extends Error {
+  public status: number
+
+  constructor(message: string, status: number) {
+    super(message)
+    this.name = 'EmbeddingAPIError'
+    this.status = status
+  }
+}
+
+export type EmbeddingInputType = 'document' | 'query'
+
+interface ProviderRequest {
+  apiUrl: string
+  headers: Record<string, string>
+  body: unknown
+  parse: (json: unknown) => number[][]
+}
+
+interface ResolvedProvider {
+  modelName: string
+  pricingId: string
+  isBYOK: boolean
+  /** Tokenizer used to estimate tokens when the API does not return a usage field. */
+  tokenizerProvider: TokenizerProviderId
+  /** Hard per-request item cap enforced by the provider (e.g. Gemini caps at 100). */
+  maxItemsPerRequest?: number
+  buildRequest: (inputs: string[], inputType: EmbeddingInputType) => ProviderRequest
+}
+
+/** Gemini's `batchEmbedContents` rejects requests with more than 100 items. */
+const GEMINI_MAX_ITEMS_PER_REQUEST = 100
+
+async function resolveOpenAIKey(workspaceId?: string | null): Promise<{
+  apiKey: string
+  isBYOK: boolean
+}> {
+  if (workspaceId) {
+    const byokResult = await getBYOKKey(workspaceId, 'openai')
+    if (byokResult) {
+      logger.info('Using workspace BYOK key for OpenAI embeddings')
+      return { apiKey: byokResult.apiKey, isBYOK: true }
+    }
+  }
+  if (env.OPENAI_API_KEY) {
+    return { apiKey: env.OPENAI_API_KEY, isBYOK: false }
+  }
+  try {
+    return { apiKey: getRotatingApiKey('openai'), isBYOK: false }
+  } catch {
+    throw new Error('OPENAI_API_KEY is not configured')
+  }
+}
+
+async function resolveGeminiKey(workspaceId?: string | null): Promise<{
+  apiKey: string
+  isBYOK: boolean
+}> {
+  if (workspaceId) {
+    const byokResult = await getBYOKKey(workspaceId, 'google')
+    if (byokResult) {
+      logger.info('Using workspace BYOK key for Gemini embeddings')
+      return { apiKey: byokResult.apiKey, isBYOK: true }
+    }
+  }
+  if (env.GEMINI_API_KEY) {
+    return { apiKey: env.GEMINI_API_KEY, isBYOK: false }
+  }
+  try {
+    return { apiKey: getRotatingApiKey('gemini'), isBYOK: false }
+  } catch {
+    throw new Error(
+      'GEMINI_API_KEY (or GEMINI_API_KEY_1/2/3 for rotation) must be configured for Gemini embeddings'
+    )
+  }
+}
+
+function buildOpenAIProvider(modelName: string, apiKey: string): ResolvedProvider['buildRequest'] {
+  return (inputs) => ({
+    apiUrl: 'https://api.openai.com/v1/embeddings',
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: {
+      input: inputs,
+      model: modelName,
+      encoding_format: 'float',
+      dimensions: EMBEDDING_DIMENSIONS,
+    },
+    parse: (json) => {
+      const data = json as { data: Array<{ embedding: number[] }> }
+      return data.data.map((item) => item.embedding)
+    },
+  })
+}
+
+function buildAzureOpenAIProvider(
+  deployment: string,
+  apiKey: string,
+  endpoint: string,
+  apiVersion: string
+): ResolvedProvider['buildRequest'] {
+  return (inputs) => ({
+    apiUrl: `${endpoint}/openai/deployments/${deployment}/embeddings?api-version=${apiVersion}`,
+    headers: {
+      'api-key': apiKey,
+      'Content-Type': 'application/json',
+    },
+    body: {
+      input: inputs,
+      encoding_format: 'float',
+      dimensions: EMBEDDING_DIMENSIONS,
+    },
+    parse: (json) => {
+      const data = json as { data: Array<{ embedding: number[] }> }
+      return data.data.map((item) => item.embedding)
+    },
+  })
+}
+
+/**
+ * Gemini does NOT auto-normalize embeddings when `outputDimensionality` is set below the
+ * native 3072 dimension on `gemini-embedding-001`. Manually L2-normalize so cosine and
+ * inner-product similarity work correctly.
+ */
+function l2Normalize(vector: number[]): number[] {
+  let sumSquares = 0
+  for (const v of vector) sumSquares += v * v
+  const norm = Math.sqrt(sumSquares)
+  if (norm === 0) return vector
+  return vector.map((v) => v / norm)
+}
+
+function buildGeminiProvider(modelName: string, apiKey: string): ResolvedProvider['buildRequest'] {
+  return (inputs, inputType) => ({
+    apiUrl: `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:batchEmbedContents`,
+    headers: {
+      'Content-Type': 'application/json',
+      'x-goog-api-key': apiKey,
+    },
+    body: {
+      requests: inputs.map((text) => ({
+        model: `models/${modelName}`,
+        content: { parts: [{ text }] },
+        taskType: inputType === 'query' ? 'RETRIEVAL_QUERY' : 'RETRIEVAL_DOCUMENT',
+        outputDimensionality: EMBEDDING_DIMENSIONS,
+      })),
+    },
+    parse: (json) => {
+      const data = json as { embeddings: Array<{ values: number[] }> }
+      return data.embeddings.map((item) => l2Normalize(item.values))
+    },
+  })
+}
+
+/**
+ * Returns the embedding model to use for new knowledge bases.
+ * Sourced from the `KB_EMBEDDING_MODEL` env var; falls back to the default if
+ * unset or set to an unsupported model.
+ */
+export function getConfiguredEmbeddingModel(): string {
+  const configured = env.KB_EMBEDDING_MODEL
+  if (configured && SUPPORTED_EMBEDDING_MODELS[configured]) {
+    return configured
+  }
+  if (configured) {
+    logger.warn(
+      `KB_EMBEDDING_MODEL="${configured}" is not a supported embedding model — falling back to ${DEFAULT_EMBEDDING_MODEL}`
+    )
+  }
+  return DEFAULT_EMBEDDING_MODEL
+}
+
+async function resolveProvider(
+  embeddingModel: string,
+  workspaceId?: string | null
+): Promise<ResolvedProvider> {
+  const azureApiKey = env.AZURE_OPENAI_API_KEY
+  const azureEndpoint = env.AZURE_OPENAI_ENDPOINT
+  const azureApiVersion = env.AZURE_OPENAI_API_VERSION
+  const isOpenAIModel = SUPPORTED_EMBEDDING_MODELS[embeddingModel]?.provider === 'openai'
+  /**
+   * Azure deployment names default to the embedding model name when
+   * `KB_OPENAI_MODEL_NAME` is unset — this matches the pre-existing
+   * convention where deployments are named after the model they host.
+   */
+  const azureDeploymentName = env.KB_OPENAI_MODEL_NAME || embeddingModel
+  const useAzure = Boolean(isOpenAIModel && azureApiKey && azureEndpoint && azureApiVersion)
+
+  const info = getEmbeddingModelInfo(embeddingModel)
+
+  if (useAzure) {
+    return {
+      modelName: azureDeploymentName,
+      pricingId: info.pricingId,
+      isBYOK: false,
+      tokenizerProvider: info.tokenizerProvider,
+      buildRequest: buildAzureOpenAIProvider(
+        azureDeploymentName,
+        azureApiKey!,
+        azureEndpoint!,
+        azureApiVersion!
+      ),
+    }
+  }
+
+  if (info.provider === 'openai') {
+    const { apiKey, isBYOK } = await resolveOpenAIKey(workspaceId)
+    return {
+      modelName: embeddingModel,
+      pricingId: info.pricingId,
+      isBYOK,
+      tokenizerProvider: info.tokenizerProvider,
+      buildRequest: buildOpenAIProvider(embeddingModel, apiKey),
+    }
+  }
+
+  if (info.provider === 'gemini') {
+    const { apiKey, isBYOK } = await resolveGeminiKey(workspaceId)
+    return {
+      modelName: embeddingModel,
+      pricingId: info.pricingId,
+      isBYOK,
+      tokenizerProvider: info.tokenizerProvider,
+      maxItemsPerRequest: GEMINI_MAX_ITEMS_PER_REQUEST,
+      buildRequest: buildGeminiProvider(embeddingModel, apiKey),
+    }
+  }
+
+  throw new Error(`Unknown embedding provider for model ${embeddingModel}`)
+}
+
+async function callEmbeddingAPI(
+  inputs: string[],
+  provider: ResolvedProvider,
+  inputType: EmbeddingInputType
+): Promise<{ embeddings: number[][]; totalTokens: number }> {
+  return retryWithExponentialBackoff(
+    async () => {
+      const request = provider.buildRequest(inputs, inputType)
+
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS)
+
+      const response = await fetch(request.apiUrl, {
+        method: 'POST',
+        headers: request.headers,
+        body: JSON.stringify(request.body),
+        signal: controller.signal,
+      }).finally(() => clearTimeout(timeout))
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new EmbeddingAPIError(
+          `Embedding API failed: ${response.status} ${response.statusText} - ${errorText}`,
+          response.status
+        )
+      }
+
+      const json = await response.json()
+      const embeddings = request.parse(json)
+      const usage = (json as { usage?: { total_tokens?: number } }).usage
+      const totalTokens =
+        usage?.total_tokens ??
+        // Gemini does not return usage.total_tokens — estimate with the provider's tokenizer
+        inputs.reduce(
+          (sum, text) => sum + estimateTokenCount(text, provider.tokenizerProvider).count,
+          0
+        )
+
+      return { embeddings, totalTokens }
+    },
+    {
+      maxRetries: 3,
+      initialDelayMs: 1000,
+      maxDelayMs: 10000,
+      retryCondition: (error: unknown) => {
+        if (error instanceof EmbeddingAPIError) {
+          return error.status === 429 || error.status >= 500
+        }
+        return isRetryableError(error)
+      },
+    }
+  )
+}
+
+function splitByItemLimit<T>(items: T[], limit: number): T[][] {
+  if (items.length <= limit) return [items]
+  const result: T[][] = []
+  for (let i = 0; i < items.length; i += limit) {
+    result.push(items.slice(i, i + limit))
+  }
+  return result
+}
+
+async function processWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  processor: (item: T, index: number) => Promise<R>
+): Promise<R[]> {
+  const results: R[] = new Array(items.length)
+  let currentIndex = 0
+
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (currentIndex < items.length) {
+      const index = currentIndex++
+      results[index] = await processor(items[index], index)
+    }
+  })
+
+  await Promise.all(workers)
+  return results
+}
+
+export interface GenerateEmbeddingsResult {
+  embeddings: number[][]
+  totalTokens: number
+  isBYOK: boolean
+  modelName: string
+  /** Pricing identifier for use with calculateCost / EMBEDDING_MODEL_PRICING. */
+  pricingId: string
+}
+
+/**
+ * Generate embeddings for multiple texts with token-aware batching and parallel processing.
+ */
+export async function generateEmbeddings(
+  texts: string[],
+  embeddingModel: string = DEFAULT_EMBEDDING_MODEL,
+  workspaceId?: string | null
+): Promise<GenerateEmbeddingsResult> {
+  const provider = await resolveProvider(embeddingModel, workspaceId)
+
+  const tokenBatches = batchByTokenLimit(texts, MAX_TOKENS_PER_REQUEST, embeddingModel)
+  const batches = provider.maxItemsPerRequest
+    ? tokenBatches.flatMap((batch) => splitByItemLimit(batch, provider.maxItemsPerRequest!))
+    : tokenBatches
+
+  const batchResults = await processWithConcurrency(
+    batches,
+    MAX_CONCURRENT_BATCHES,
+    async (batch, i) => {
+      try {
+        return await callEmbeddingAPI(batch, provider, 'document')
+      } catch (error) {
+        logger.error(`Failed to generate embeddings for batch ${i + 1}/${batches.length}:`, error)
+        throw error
+      }
+    }
+  )
+
+  const allEmbeddings: number[][] = []
+  let totalTokens = 0
+  for (const batch of batchResults) {
+    for (const emb of batch.embeddings) {
+      allEmbeddings.push(emb)
+    }
+    totalTokens += batch.totalTokens
+  }
+
+  return {
+    embeddings: allEmbeddings,
+    totalTokens,
+    isBYOK: provider.isBYOK,
+    modelName: provider.modelName,
+    pricingId: provider.pricingId,
+  }
+}
+
+/**
+ * Generate embedding for a single search query.
+ */
+export async function generateSearchEmbedding(
+  query: string,
+  embeddingModel: string = DEFAULT_EMBEDDING_MODEL,
+  workspaceId?: string | null
+): Promise<{ embedding: number[]; isBYOK: boolean }> {
+  const provider = await resolveProvider(embeddingModel, workspaceId)
+
+  logger.info(`Using ${provider.modelName} for search embedding generation`)
+
+  const { embeddings } = await callEmbeddingAPI([query], provider, 'query')
+  return { embedding: embeddings[0], isBYOK: provider.isBYOK }
+}
+
+/**
+ * Records a query embedding's hosted-key cost for callers that generate a search
+ * embedding directly, outside the metered `/api/knowledge/search` route (e.g. the
+ * v1 search API and copilot KB search). No-ops for BYOK (no Sim cost) or when
+ * there is no workspace to attribute to. Best-effort: never throws.
+ */
+export async function recordSearchEmbeddingUsage(params: {
+  userId: string
+  workspaceId?: string | null
+  embeddingModel: string
+  query: string
+  isBYOK: boolean
+  sourceReference: string
+}): Promise<void> {
+  const { userId, workspaceId, embeddingModel, query, isBYOK, sourceReference } = params
+  if (isBYOK || !workspaceId) return
+  try {
+    const { count } = estimateTokenCount(
+      query,
+      getEmbeddingModelInfo(embeddingModel).tokenizerProvider
+    )
+    const cost = calculateCost(embeddingModel, count, 0, false)
+    if (!cost || cost.total <= 0) return
+    await recordUsage({
+      userId,
+      workspaceId,
+      entries: [
+        {
+          category: 'model',
+          source: 'knowledge-base',
+          description: embeddingModel,
+          cost: cost.total,
+          sourceReference,
+        },
+      ],
+    })
+  } catch (error) {
+    logger.warn('Failed to record search embedding usage', { error: getErrorMessage(error) })
+  }
+}

apps/sim/lib/knowledge/filters/types.ts

@@ -0,0 +1,233 @@
+/**
+ * Filter operators for different field types
+ */
+
+/**
+ * Text filter operators
+ */
+export type TextOperator = 'eq' | 'neq' | 'contains' | 'not_contains' | 'starts_with' | 'ends_with'
+
+/**
+ * Number filter operators
+ */
+export type NumberOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'between'
+
+/**
+ * Date filter operators
+ */
+export type DateOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'between'
+
+/**
+ * Boolean filter operators
+ */
+export type BooleanOperator = 'eq' | 'neq'
+
+/**
+ * All filter operators union
+ */
+export type FilterOperator = TextOperator | NumberOperator | DateOperator | BooleanOperator
+
+/**
+ * Field types supported for filtering
+ */
+export type FilterFieldType = 'text' | 'number' | 'date' | 'boolean'
+
+/**
+ * Logical operators for combining filters
+ */
+export type LogicalOperator = 'AND' | 'OR'
+
+/**
+ * Base filter condition interface
+ */
+interface BaseFilterCondition {
+  tagSlot: string
+  fieldType: FilterFieldType
+}
+
+/**
+ * Text filter condition
+ */
+interface TextFilterCondition extends BaseFilterCondition {
+  fieldType: 'text'
+  operator: TextOperator
+  value: string
+}
+
+/**
+ * Number filter condition
+ */
+interface NumberFilterCondition extends BaseFilterCondition {
+  fieldType: 'number'
+  operator: NumberOperator
+  value: number
+  valueTo?: number // For 'between' operator
+}
+
+/**
+ * Date filter condition
+ */
+interface DateFilterCondition extends BaseFilterCondition {
+  fieldType: 'date'
+  operator: DateOperator
+  value: string // ISO date string
+  valueTo?: string // For 'between' operator (ISO date string)
+}
+
+/**
+ * Boolean filter condition
+ */
+interface BooleanFilterCondition extends BaseFilterCondition {
+  fieldType: 'boolean'
+  operator: BooleanOperator
+  value: boolean
+}
+
+/**
+ * Union of all filter conditions
+ */
+export type FilterCondition =
+  | TextFilterCondition
+  | NumberFilterCondition
+  | DateFilterCondition
+  | BooleanFilterCondition
+
+/**
+ * Filter group with logical operator
+ */
+interface FilterGroup {
+  operator: LogicalOperator
+  conditions: FilterCondition[]
+}
+
+/**
+ * Complete filter query structure
+ * Supports nested groups with AND/OR logic
+ */
+interface TagFilter {
+  rootOperator: LogicalOperator
+  groups: FilterGroup[]
+}
+
+/**
+ * Simplified flat filter structure for simple use cases
+ */
+interface SimpleTagFilter {
+  operator: LogicalOperator
+  conditions: FilterCondition[]
+}
+
+/**
+ * Operator metadata for UI display
+ */
+export interface OperatorInfo {
+  value: string
+  label: string
+  requiresSecondValue?: boolean
+}
+
+/**
+ * Text operators metadata
+ */
+export const TEXT_OPERATORS: OperatorInfo[] = [
+  { value: 'eq', label: 'equals' },
+  { value: 'neq', label: 'not equals' },
+  { value: 'contains', label: 'contains' },
+  { value: 'not_contains', label: 'does not contain' },
+  { value: 'starts_with', label: 'starts with' },
+  { value: 'ends_with', label: 'ends with' },
+]
+
+/**
+ * Number operators metadata
+ */
+export const NUMBER_OPERATORS: OperatorInfo[] = [
+  { value: 'eq', label: 'equals' },
+  { value: 'neq', label: 'not equals' },
+  { value: 'gt', label: 'greater than' },
+  { value: 'gte', label: 'greater than or equal' },
+  { value: 'lt', label: 'less than' },
+  { value: 'lte', label: 'less than or equal' },
+  { value: 'between', label: 'between', requiresSecondValue: true },
+]
+
+/**
+ * Date operators metadata
+ */
+export const DATE_OPERATORS: OperatorInfo[] = [
+  { value: 'eq', label: 'equals' },
+  { value: 'neq', label: 'not equals' },
+  { value: 'gt', label: 'after' },
+  { value: 'gte', label: 'on or after' },
+  { value: 'lt', label: 'before' },
+  { value: 'lte', label: 'on or before' },
+  { value: 'between', label: 'between', requiresSecondValue: true },
+]
+
+/**
+ * Boolean operators metadata
+ */
+export const BOOLEAN_OPERATORS: OperatorInfo[] = [
+  { value: 'eq', label: 'is' },
+  { value: 'neq', label: 'is not' },
+]
+
+/**
+ * Get operators for a field type
+ */
+export function getOperatorsForFieldType(fieldType: FilterFieldType): OperatorInfo[] {
+  switch (fieldType) {
+    case 'text':
+      return TEXT_OPERATORS
+    case 'number':
+      return NUMBER_OPERATORS
+    case 'date':
+      return DATE_OPERATORS
+    case 'boolean':
+      return BOOLEAN_OPERATORS
+    default:
+      return []
+  }
+}
+
+/** Wire format for a date filter value (`YYYY-MM-DD`). */
+const DATE_ONLY_VALUE = /^\d{4}-\d{2}-\d{2}$/
+
+/**
+ * Whether a `YYYY-MM-DD` string is a real calendar date. The format regex alone
+ * still admits impossible dates (`2026-02-30`, `2026-99-99`) that pass the
+ * boundary and then make the document query's `::date` cast throw a 500; this
+ * round-trips the parsed parts to reject them.
+ */
+function isRealCalendarDate(value: string): boolean {
+  if (!DATE_ONLY_VALUE.test(value)) return false
+  const [year, month, day] = value.split('-').map(Number)
+  const date = new Date(year, month - 1, day)
+  return date.getFullYear() === year && date.getMonth() === month - 1 && date.getDate() === day
+}
+
+/**
+ * Whether a raw filter value is usable for the given field type. Shared source
+ * of truth so the API boundary can reject unusable values (e.g. `"abc"` for a
+ * number, `"not-a-date"` for a date) instead of letting them be silently
+ * dropped further down. Values arrive as strings from the filter UI.
+ */
+export function isValidFilterValue(fieldType: FilterFieldType, value: unknown): boolean {
+  if (value === undefined || value === null) return false
+  switch (fieldType) {
+    case 'text':
+      return typeof value === 'string' && value.length > 0
+    case 'number':
+      if (typeof value === 'number') return Number.isFinite(value)
+      return typeof value === 'string' && value.trim() !== '' && Number.isFinite(Number(value))
+    case 'date':
+      return typeof value === 'string' && isRealCalendarDate(value)
+    case 'boolean':
+      return (
+        typeof value === 'boolean' ||
+        (typeof value === 'string' && ['true', 'false'].includes(value.trim().toLowerCase()))
+      )
+    default:
+      return false
+  }
+}

apps/sim/lib/knowledge/orchestration/index.ts

@@ -0,0 +1,88 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { db } from '@sim/db'
+import { knowledgeBase } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { eq } from 'drizzle-orm'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { KnowledgeBaseConflictError, restoreKnowledgeBase } from '@/lib/knowledge/service'
+
+const logger = createLogger('KnowledgeBaseOrchestration')
+
+export type KnowledgeOrchestrationErrorCode = 'not_found' | 'conflict' | 'internal'
+
+export interface RestorableKnowledgeBase {
+  id: string
+  name: string
+  workspaceId: string | null
+  userId: string
+}
+
+export interface PerformRestoreKnowledgeBaseParams {
+  knowledgeBaseId: string
+  userId: string
+  requestId?: string
+}
+
+export interface PerformRestoreKnowledgeBaseResult {
+  success: boolean
+  error?: string
+  errorCode?: KnowledgeOrchestrationErrorCode
+  knowledgeBase?: RestorableKnowledgeBase
+}
+
+export async function getRestorableKnowledgeBase(
+  knowledgeBaseId: string
+): Promise<RestorableKnowledgeBase | null> {
+  const [kb] = await db
+    .select({
+      id: knowledgeBase.id,
+      name: knowledgeBase.name,
+      workspaceId: knowledgeBase.workspaceId,
+      userId: knowledgeBase.userId,
+    })
+    .from(knowledgeBase)
+    .where(eq(knowledgeBase.id, knowledgeBaseId))
+    .limit(1)
+
+  return kb ?? null
+}
+
+export async function performRestoreKnowledgeBase(
+  params: PerformRestoreKnowledgeBaseParams
+): Promise<PerformRestoreKnowledgeBaseResult> {
+  const { knowledgeBaseId, userId } = params
+  const requestId = params.requestId ?? generateRequestId()
+
+  const kb = await getRestorableKnowledgeBase(knowledgeBaseId)
+  if (!kb) {
+    return { success: false, error: 'Knowledge base not found', errorCode: 'not_found' }
+  }
+
+  try {
+    await restoreKnowledgeBase(knowledgeBaseId, requestId)
+
+    logger.info(`[${requestId}] Restored knowledge base ${knowledgeBaseId}`)
+
+    recordAudit({
+      workspaceId: kb.workspaceId,
+      actorId: userId,
+      action: AuditAction.KNOWLEDGE_BASE_RESTORED,
+      resourceType: AuditResourceType.KNOWLEDGE_BASE,
+      resourceId: knowledgeBaseId,
+      resourceName: kb.name,
+      description: `Restored knowledge base "${kb.name}"`,
+      metadata: {
+        knowledgeBaseName: kb.name,
+      },
+    })
+
+    return { success: true, knowledgeBase: kb }
+  } catch (error) {
+    logger.error(`[${requestId}] Failed to restore knowledge base ${knowledgeBaseId}`, { error })
+    if (error instanceof KnowledgeBaseConflictError) {
+      return { success: false, error: error.message, errorCode: 'conflict' }
+    }
+    return { success: false, error: toError(error).message, errorCode: 'internal' }
+  }
+}

apps/sim/lib/knowledge/reranker-models.ts

@@ -0,0 +1,18 @@
+/**
+ * Client-safe registry of Cohere rerank models supported by the platform.
+ * Kept free of server imports so it can be imported into UI / block code.
+ */
+
+import { z } from 'zod'
+
+/** Cohere rerank model identifiers we accept. Must match Cohere's model ids exactly. */
+export const rerankerModelSchema = z.enum(['rerank-v4.0-pro', 'rerank-v4.0-fast', 'rerank-v3.5'])
+export type RerankerModelId = z.output<typeof rerankerModelSchema>
+
+export const SUPPORTED_RERANKER_MODELS = rerankerModelSchema.options
+
+export const DEFAULT_RERANKER_MODEL: RerankerModelId = 'rerank-v4.0-fast'
+
+export function isSupportedRerankerModel(model: string): model is RerankerModelId {
+  return rerankerModelSchema.safeParse(model).success
+}

apps/sim/lib/knowledge/reranker.ts

@@ -0,0 +1,182 @@
+import { createLogger } from '@sim/logger'
+import { getBYOKKey } from '@/lib/api-key/byok'
+import { getRotatingApiKey } from '@/lib/core/config/api-keys'
+import { env } from '@/lib/core/config/env'
+import { isHosted } from '@/lib/core/config/env-flags'
+import { isRetryableError, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
+import { isSupportedRerankerModel } from '@/lib/knowledge/reranker-models'
+
+const logger = createLogger('Reranker')
+
+const RERANK_REQUEST_TIMEOUT_MS = 30_000
+
+/**
+ * Cohere bills per "search unit" = one query with up to 100 documents.
+ * We cap at 100 so each rerank call costs exactly 1 unit and matches
+ * `RERANK_MODEL_PRICING` in `providers/models.ts`. The search route also
+ * caps `candidateTopK` at 100, so this is a defensive ceiling.
+ */
+const MAX_DOCUMENTS_PER_RERANK = 100
+
+export interface RerankItem {
+  /** Stable identifier so callers can correlate ranked results back to source rows. */
+  id: string
+  text: string
+}
+
+interface RerankedResult<T extends RerankItem> {
+  item: T
+  relevanceScore: number
+}
+
+export interface RerankResponse<T extends RerankItem> {
+  results: RerankedResult<T>[]
+  /** True when a workspace-supplied (BYOK) Cohere key was used. Callers should skip platform billing in that case. */
+  isBYOK: boolean
+}
+
+class RerankAPIError extends Error {
+  public status: number
+  constructor(message: string, status: number) {
+    super(message)
+    this.name = 'RerankAPIError'
+    this.status = status
+  }
+}
+
+async function resolveCohereKey(
+  workspaceId?: string | null,
+  userApiKey?: string
+): Promise<{ apiKey: string; isBYOK: boolean }> {
+  /**
+   * Mirrors the agent block hosted-key pattern (`injectHostedKeyIfNeeded`):
+   * on self-hosted the user-supplied key from the block field flows through
+   * unchanged; on hosted Sim we always source the key from workspace BYOK or
+   * platform env, so any user-supplied value is ignored.
+   */
+  if (!isHosted && userApiKey) {
+    return { apiKey: userApiKey, isBYOK: false }
+  }
+  if (workspaceId) {
+    const byokResult = await getBYOKKey(workspaceId, 'cohere')
+    if (byokResult) {
+      logger.info('Using workspace BYOK key for Cohere reranker')
+      return { apiKey: byokResult.apiKey, isBYOK: true }
+    }
+  }
+  if (env.COHERE_API_KEY) {
+    return { apiKey: env.COHERE_API_KEY, isBYOK: false }
+  }
+  try {
+    return { apiKey: getRotatingApiKey('cohere'), isBYOK: false }
+  } catch {
+    throw new Error(
+      'No Cohere API key configured. Set COHERE_API_KEY_1/2/3 (rotation) or COHERE_API_KEY.'
+    )
+  }
+}
+
+/**
+ * Subset of Cohere v2/rerank response fields we read.
+ * Reference: https://docs.cohere.com/v2/reference/rerank
+ * - `results[].index` maps back to the position in the documents we sent.
+ * - `results[].relevance_score` is normalized 0–1.
+ * - `meta.warnings` is documented as an array of strings; we surface them in logs
+ *   so issues like document truncation don't disappear silently.
+ */
+interface CohereRerankResponse {
+  results: Array<{ index: number; relevance_score: number }>
+  meta?: {
+    warnings?: string[]
+  }
+}
+
+/**
+ * Rerank documents against a query using Cohere's `/v2/rerank` endpoint.
+ * Returns the items in descending order of relevance, capped at `topN`.
+ */
+export async function rerank<T extends RerankItem>(
+  query: string,
+  items: T[],
+  options: {
+    model: string
+    topN?: number
+    workspaceId?: string | null
+    /** User-supplied Cohere key from the Knowledge block field. Honored only on self-hosted. */
+    apiKey?: string
+  }
+): Promise<RerankResponse<T>> {
+  if (items.length === 0) return { results: [], isBYOK: false }
+
+  if (!isSupportedRerankerModel(options.model)) {
+    throw new Error(`Unsupported reranker model: ${options.model}`)
+  }
+
+  const { apiKey, isBYOK } = await resolveCohereKey(options.workspaceId, options.apiKey)
+  const cappedItems =
+    items.length > MAX_DOCUMENTS_PER_RERANK ? items.slice(0, MAX_DOCUMENTS_PER_RERANK) : items
+  if (items.length > MAX_DOCUMENTS_PER_RERANK) {
+    logger.warn(`Rerank input capped from ${items.length} to ${MAX_DOCUMENTS_PER_RERANK} documents`)
+  }
+  const documents = cappedItems.map((it) => it.text)
+
+  const response = await retryWithExponentialBackoff(
+    async () => {
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), RERANK_REQUEST_TIMEOUT_MS)
+
+      const res = await fetch('https://api.cohere.com/v2/rerank', {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          model: options.model,
+          query,
+          documents,
+          top_n: options.topN ?? cappedItems.length,
+        }),
+        signal: controller.signal,
+      }).finally(() => clearTimeout(timeout))
+
+      if (!res.ok) {
+        const errorText = await res.text()
+        throw new RerankAPIError(
+          `Cohere rerank failed: ${res.status} ${res.statusText} - ${errorText}`,
+          res.status
+        )
+      }
+
+      return (await res.json()) as CohereRerankResponse
+    },
+    {
+      maxRetries: 3,
+      initialDelayMs: 500,
+      maxDelayMs: 5000,
+      retryCondition: (error: unknown) => {
+        if (error instanceof RerankAPIError) {
+          return error.status === 429 || error.status >= 500
+        }
+        return isRetryableError(error)
+      },
+    }
+  )
+
+  if (response.meta?.warnings && response.meta.warnings.length > 0) {
+    logger.warn('Cohere rerank returned warnings', {
+      model: options.model,
+      warnings: response.meta.warnings,
+    })
+  }
+
+  return {
+    results: response.results
+      .filter((r) => r.index >= 0 && r.index < cappedItems.length)
+      .map((r) => ({
+        item: cappedItems[r.index],
+        relevanceScore: r.relevance_score,
+      })),
+    isBYOK,
+  }
+}

apps/sim/lib/knowledge/service.test.ts

@@ -0,0 +1,202 @@
+/**
+ * @vitest-environment node
+ */
+import {
+  dbChainMock,
+  dbChainMockFns,
+  permissionsMock,
+  permissionsMockFns,
+  resetDbChainMock,
+} from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
+
+import { KnowledgeBasePermissionError, updateKnowledgeBase } from '@/lib/knowledge/service'
+
+/**
+ * These tests guard the workspace mass-assignment fix:
+ * a user with write/admin on the *source* workspace must not be able to move a
+ * knowledge base into a workspace where they have no permission, and must not
+ * be able to clear `workspaceId` (which would orphan the KB to its original
+ * `userId`, who may not be the caller).
+ */
+describe('updateKnowledgeBase — workspace transfer authorization', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    dbChainMockFns.limit.mockReset()
+    resetDbChainMock()
+  })
+
+  it('rejects workspaceId change without actorUserId', async () => {
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1')
+    ).rejects.toBeInstanceOf(KnowledgeBasePermissionError)
+    expect(permissionsMockFns.mockGetUserEntityPermissions).not.toHaveBeenCalled()
+  })
+
+  it('rejects clearing workspaceId to null when actor is not the KB owner', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'owner' }])
+
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'attacker' })
+    ).rejects.toMatchObject({
+      code: 'KNOWLEDGE_BASE_FORBIDDEN',
+      message: 'Only the knowledge base owner can remove it from a workspace',
+    })
+    expect(permissionsMockFns.mockGetUserEntityPermissions).not.toHaveBeenCalled()
+  })
+
+  it('allows the KB owner to clear workspaceId to null (gate passes; target permission not checked)', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'owner' }])
+
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'owner' })
+    ).rejects.not.toBeInstanceOf(KnowledgeBasePermissionError)
+    expect(permissionsMockFns.mockGetUserEntityPermissions).not.toHaveBeenCalled()
+  })
+
+  it('rejects transfer when actor has no permission on target workspace', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce(null)
+
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', {
+        actorUserId: 'attacker',
+      })
+    ).rejects.toMatchObject({
+      code: 'KNOWLEDGE_BASE_FORBIDDEN',
+      message: 'User does not have permission on the target workspace',
+    })
+    expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith(
+      'attacker',
+      'workspace',
+      'ws-target'
+    )
+  })
+
+  it('rejects transfer when actor only has read permission on target workspace', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('read')
+
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', {
+        actorUserId: 'reader',
+      })
+    ).rejects.toBeInstanceOf(KnowledgeBasePermissionError)
+  })
+
+  it('throws when knowledge base does not exist during transfer', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+
+    await expect(
+      updateKnowledgeBase('kb-missing', { workspaceId: 'ws-target' }, 'req-1', {
+        actorUserId: 'u-1',
+      })
+    ).rejects.toThrow('Knowledge base kb-missing not found')
+    // The target-workspace permission is resolved before the transaction
+    // opens (pool safety), so the lookup runs even when the KB is missing.
+    expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith(
+      'u-1',
+      'workspace',
+      'ws-target'
+    )
+  })
+
+  it('locks the knowledge base row (SELECT … FOR UPDATE) and enforces the pre-resolved permission', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce(null)
+
+    await expect(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', {
+        actorUserId: 'attacker',
+      })
+    ).rejects.toBeInstanceOf(KnowledgeBasePermissionError)
+
+    expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1)
+    expect(dbChainMockFns.for).toHaveBeenCalledWith('update')
+  })
+})
+
+/**
+ * These tests guard the file-authorization follow-through: KB file ownership is
+ * resolved from the trusted `workspace_files` binding, so when a KB moves to a
+ * new workspace the bindings for its stored files must move with it. Otherwise
+ * the bindings stay frozen at the upload-time workspace and the KB's files
+ * become unreadable after a move.
+ */
+describe('updateKnowledgeBase — file ownership binding re-point on workspace change', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    dbChainMockFns.limit.mockReset()
+    resetDbChainMock()
+  })
+
+  // The mocked `@sim/db` cannot satisfy the post-transaction read-back select, so
+  // the call rejects after the transaction body commits. These tests assert the
+  // in-transaction binding statements, then swallow that read-back rejection.
+  const runIgnoringReadBack = (promise: Promise<unknown>) => promise.catch(() => undefined)
+
+  it('re-points file ownership bindings to the new workspace on a move', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('admin')
+
+    await runIgnoringReadBack(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', { actorUserId: 'u-1' })
+    )
+
+    // Two updates inside the txn: the KB row, then the file bindings.
+    expect(dbChainMockFns.update).toHaveBeenCalledTimes(2)
+    expect(dbChainMockFns.set).toHaveBeenCalledWith({ workspaceId: 'ws-target' })
+  })
+
+  it('clears file ownership bindings when the KB is removed from its workspace', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'owner' }])
+
+    await runIgnoringReadBack(
+      updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'owner' })
+    )
+
+    expect(dbChainMockFns.update).toHaveBeenCalledTimes(2)
+    expect(dbChainMockFns.set).toHaveBeenCalledWith({ workspaceId: null })
+  })
+
+  it('does not re-point bindings when promoting a personal (null-workspace) KB into a workspace', async () => {
+    // A null current workspace owns no bindings, so the move must not rewrite
+    // any binding — this prevents a key planted in a personal KB from being
+    // laundered into the destination workspace on move.
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: null, userId: 'owner' }])
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('admin')
+
+    await runIgnoringReadBack(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', { actorUserId: 'owner' })
+    )
+
+    // Only the KB row is updated; the binding re-point is skipped entirely.
+    expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
+    expect(dbChainMockFns.set).not.toHaveBeenCalledWith({ workspaceId: 'ws-target' })
+  })
+
+  it('does not touch bindings when the workspace is unchanged', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
+
+    await runIgnoringReadBack(
+      updateKnowledgeBase('kb-1', { workspaceId: 'ws-current' }, 'req-1', { actorUserId: 'u-1' })
+    )
+
+    // Only the KB row is updated; no binding re-point statement runs.
+    expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
+    expect(dbChainMockFns.set).not.toHaveBeenCalledWith({ workspaceId: 'ws-current' })
+  })
+
+  it('does not touch bindings when no workspace change is requested', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }]) // currentKb lock
+      .mockResolvedValueOnce([]) // duplicate-name check: none
+
+    await runIgnoringReadBack(updateKnowledgeBase('kb-1', { name: 'Renamed' }, 'req-1'))
+
+    expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
+  })
+})

apps/sim/lib/knowledge/service.ts

@@ -0,0 +1,649 @@
+import { db } from '@sim/db'
+import {
+  document,
+  knowledgeBase,
+  knowledgeConnector,
+  permissions,
+  workspace,
+  workspaceFiles,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getPostgresErrorCode } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, count, eq, exists, inArray, isNotNull, isNull, ne, or, sql } from 'drizzle-orm'
+import { generateRestoreName } from '@/lib/core/utils/restore-name'
+import type {
+  ChunkingConfig,
+  CreateKnowledgeBaseData,
+  KnowledgeBaseWithCounts,
+} from '@/lib/knowledge/types'
+import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
+
+const logger = createLogger('KnowledgeBaseService')
+
+export class KnowledgeBaseConflictError extends Error {
+  readonly code = 'KNOWLEDGE_BASE_EXISTS' as const
+  constructor(name: string) {
+    super(`A knowledge base named "${name}" already exists in this workspace`)
+  }
+}
+
+export class KnowledgeBasePermissionError extends Error {
+  readonly code = 'KNOWLEDGE_BASE_FORBIDDEN' as const
+}
+
+export type KnowledgeBaseScope = 'active' | 'archived' | 'all'
+
+/**
+ * Get knowledge bases that a user can access
+ */
+export async function getKnowledgeBases(
+  userId: string,
+  workspaceId?: string | null,
+  scope: KnowledgeBaseScope = 'active'
+): Promise<KnowledgeBaseWithCounts[]> {
+  const scopeCondition =
+    scope === 'all'
+      ? undefined
+      : scope === 'archived'
+        ? sql`${knowledgeBase.deletedAt} IS NOT NULL`
+        : isNull(knowledgeBase.deletedAt)
+
+  const knowledgeBasesWithCounts = await db
+    .select({
+      id: knowledgeBase.id,
+      userId: knowledgeBase.userId,
+      name: knowledgeBase.name,
+      description: knowledgeBase.description,
+      tokenCount: sql<number>`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number),
+      embeddingModel: knowledgeBase.embeddingModel,
+      embeddingDimension: knowledgeBase.embeddingDimension,
+      chunkingConfig: knowledgeBase.chunkingConfig,
+      createdAt: knowledgeBase.createdAt,
+      updatedAt: knowledgeBase.updatedAt,
+      deletedAt: knowledgeBase.deletedAt,
+      workspaceId: knowledgeBase.workspaceId,
+      docCount: count(document.id),
+    })
+    .from(knowledgeBase)
+    .leftJoin(
+      document,
+      and(
+        eq(document.knowledgeBaseId, knowledgeBase.id),
+        eq(document.userExcluded, false),
+        isNull(document.archivedAt),
+        isNull(document.deletedAt)
+      )
+    )
+    .leftJoin(
+      permissions,
+      and(
+        eq(permissions.entityType, 'workspace'),
+        eq(permissions.entityId, knowledgeBase.workspaceId),
+        eq(permissions.userId, userId)
+      )
+    )
+    .leftJoin(workspace, eq(knowledgeBase.workspaceId, workspace.id))
+    .where(
+      and(
+        scopeCondition,
+        workspaceId
+          ? // When filtering by workspace
+            or(
+              // Knowledge bases belonging to the specified workspace (user must have workspace permissions)
+              and(
+                eq(knowledgeBase.workspaceId, workspaceId),
+                isNotNull(permissions.userId),
+                isNull(workspace.archivedAt)
+              ),
+              // Fallback: User-owned knowledge bases without workspace (legacy)
+              and(eq(knowledgeBase.userId, userId), isNull(knowledgeBase.workspaceId))
+            )
+          : // When not filtering by workspace, use original logic
+            or(
+              // User owns the knowledge base directly
+              eq(knowledgeBase.userId, userId),
+              // User has permissions on the knowledge base's workspace
+              and(isNotNull(permissions.userId), isNull(workspace.archivedAt))
+            )
+      )
+    )
+    .groupBy(knowledgeBase.id)
+    .orderBy(knowledgeBase.createdAt)
+
+  const kbIds = knowledgeBasesWithCounts.map((kb) => kb.id)
+
+  const connectorRows =
+    kbIds.length > 0
+      ? await db
+          .select({
+            knowledgeBaseId: knowledgeConnector.knowledgeBaseId,
+            connectorType: knowledgeConnector.connectorType,
+          })
+          .from(knowledgeConnector)
+          .where(
+            and(
+              inArray(knowledgeConnector.knowledgeBaseId, kbIds),
+              isNull(knowledgeConnector.archivedAt),
+              isNull(knowledgeConnector.deletedAt)
+            )
+          )
+      : []
+
+  const connectorTypesByKb = new Map<string, string[]>()
+  for (const row of connectorRows) {
+    const types = connectorTypesByKb.get(row.knowledgeBaseId) ?? []
+    if (!types.includes(row.connectorType)) {
+      types.push(row.connectorType)
+    }
+    connectorTypesByKb.set(row.knowledgeBaseId, types)
+  }
+
+  return knowledgeBasesWithCounts.map((kb) => ({
+    ...kb,
+    chunkingConfig: kb.chunkingConfig as ChunkingConfig,
+    docCount: Number(kb.docCount),
+    connectorTypes: connectorTypesByKb.get(kb.id) ?? [],
+  }))
+}
+
+/**
+ * Create a new knowledge base
+ */
+export async function createKnowledgeBase(
+  data: CreateKnowledgeBaseData,
+  requestId: string
+): Promise<KnowledgeBaseWithCounts> {
+  const kbId = generateId()
+  const now = new Date()
+
+  const hasPermission = await getUserEntityPermissions(data.userId, 'workspace', data.workspaceId)
+  if (hasPermission !== 'admin' && hasPermission !== 'write') {
+    throw new KnowledgeBasePermissionError(
+      'User does not have permission to create knowledge bases in this workspace'
+    )
+  }
+
+  const newKnowledgeBase = {
+    id: kbId,
+    name: data.name,
+    description: data.description ?? null,
+    workspaceId: data.workspaceId,
+    userId: data.userId,
+    tokenCount: 0,
+    embeddingModel: data.embeddingModel,
+    embeddingDimension: data.embeddingDimension,
+    chunkingConfig: data.chunkingConfig,
+    createdAt: now,
+    updatedAt: now,
+    deletedAt: null,
+  }
+
+  const duplicate = await db
+    .select({ id: knowledgeBase.id })
+    .from(knowledgeBase)
+    .where(
+      and(
+        eq(knowledgeBase.workspaceId, data.workspaceId),
+        eq(knowledgeBase.name, data.name),
+        isNull(knowledgeBase.deletedAt)
+      )
+    )
+    .limit(1)
+
+  if (duplicate.length > 0) {
+    throw new KnowledgeBaseConflictError(data.name)
+  }
+
+  try {
+    await db.insert(knowledgeBase).values(newKnowledgeBase)
+  } catch (error: unknown) {
+    if (getPostgresErrorCode(error) === '23505') {
+      throw new KnowledgeBaseConflictError(data.name)
+    }
+    throw error
+  }
+
+  logger.info(`[${requestId}] Created knowledge base: ${data.name} (${kbId})`)
+
+  return {
+    id: kbId,
+    userId: data.userId,
+    name: data.name,
+    description: data.description ?? null,
+    tokenCount: 0,
+    embeddingModel: data.embeddingModel,
+    embeddingDimension: data.embeddingDimension,
+    chunkingConfig: data.chunkingConfig,
+    createdAt: now,
+    updatedAt: now,
+    deletedAt: null,
+    workspaceId: data.workspaceId,
+    docCount: 0,
+    connectorTypes: [],
+  }
+}
+
+/**
+ * Update a knowledge base
+ */
+export async function updateKnowledgeBase(
+  knowledgeBaseId: string,
+  updates: {
+    name?: string
+    description?: string
+    workspaceId?: string | null
+    chunkingConfig?: {
+      maxSize: number
+      minSize: number
+      overlap: number
+    }
+  },
+  requestId: string,
+  options?: { actorUserId?: string }
+): Promise<KnowledgeBaseWithCounts> {
+  const now = new Date()
+  const updateData: {
+    updatedAt: Date
+    name?: string
+    description?: string | null
+    workspaceId?: string | null
+    chunkingConfig?: {
+      maxSize: number
+      minSize: number
+      overlap: number
+    }
+    embeddingModel?: string
+    embeddingDimension?: number
+  } = {
+    updatedAt: now,
+  }
+
+  if (updates.name !== undefined) updateData.name = updates.name
+  if (updates.description !== undefined) updateData.description = updates.description
+  if (updates.workspaceId !== undefined) updateData.workspaceId = updates.workspaceId
+  if (updates.chunkingConfig !== undefined) {
+    updateData.chunkingConfig = updates.chunkingConfig
+  }
+
+  if (updates.workspaceId !== undefined && !options?.actorUserId) {
+    throw new KnowledgeBasePermissionError(
+      'actorUserId is required to change a knowledge base workspace'
+    )
+  }
+
+  // Resolved before the transaction: the target workspace comes from the
+  // request input, so checking it inside the FOR UPDATE tx would only issue a
+  // second pooled-connection checkout while the first is held.
+  const targetWorkspacePermission = updates.workspaceId
+    ? await getUserEntityPermissions(
+        options?.actorUserId as string,
+        'workspace',
+        updates.workspaceId
+      )
+    : null
+
+  try {
+    await db.transaction(async (tx) => {
+      const [currentKb] = await tx
+        .select({ workspaceId: knowledgeBase.workspaceId, userId: knowledgeBase.userId })
+        .from(knowledgeBase)
+        .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+        .for('update')
+        .limit(1)
+
+      if (!currentKb) {
+        throw new Error(`Knowledge base ${knowledgeBaseId} not found`)
+      }
+
+      if (updates.workspaceId !== undefined) {
+        const actorUserId = options?.actorUserId as string
+        const currentWorkspaceId = currentKb.workspaceId ?? null
+        const targetWorkspaceId = updates.workspaceId ?? null
+
+        if (targetWorkspaceId !== currentWorkspaceId) {
+          if (!targetWorkspaceId) {
+            if (actorUserId !== currentKb.userId) {
+              throw new KnowledgeBasePermissionError(
+                'Only the knowledge base owner can remove it from a workspace'
+              )
+            }
+          } else if (
+            targetWorkspacePermission !== 'write' &&
+            targetWorkspacePermission !== 'admin'
+          ) {
+            throw new KnowledgeBasePermissionError(
+              'User does not have permission on the target workspace'
+            )
+          }
+        }
+      }
+
+      if (updates.name !== undefined) {
+        const effectiveWorkspaceId =
+          updates.workspaceId !== undefined ? updates.workspaceId : currentKb.workspaceId
+
+        if (effectiveWorkspaceId) {
+          const duplicate = await tx
+            .select({ id: knowledgeBase.id })
+            .from(knowledgeBase)
+            .where(
+              and(
+                eq(knowledgeBase.workspaceId, effectiveWorkspaceId),
+                eq(knowledgeBase.name, updates.name),
+                isNull(knowledgeBase.deletedAt),
+                ne(knowledgeBase.id, knowledgeBaseId)
+              )
+            )
+            .limit(1)
+
+          if (duplicate.length > 0) {
+            throw new KnowledgeBaseConflictError(updates.name)
+          }
+        }
+      }
+
+      await tx
+        .update(knowledgeBase)
+        .set(updateData)
+        .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+
+      // When a KB changes workspace, re-point the ownership bindings for its
+      // stored files so file authorization (which resolves the owning workspace
+      // from the trusted binding, not from document.fileUrl) follows the KB to
+      // its new workspace. Only bindings the KB's *current* workspace already
+      // owns are moved: this scopes the update to this KB's own files and
+      // prevents a document referencing another tenant's key (e.g. one planted
+      // while the KB had no workspace) from hijacking that key's binding on
+      // move. A null current workspace owns no bindings, so nothing is moved.
+      if (updates.workspaceId !== undefined) {
+        const currentWorkspaceId = currentKb.workspaceId ?? null
+        const targetWorkspaceId = updates.workspaceId ?? null
+
+        if (currentWorkspaceId && targetWorkspaceId !== currentWorkspaceId) {
+          await tx
+            .update(workspaceFiles)
+            .set({ workspaceId: targetWorkspaceId })
+            .where(
+              and(
+                eq(workspaceFiles.context, 'knowledge-base'),
+                eq(workspaceFiles.workspaceId, currentWorkspaceId),
+                isNull(workspaceFiles.deletedAt),
+                exists(
+                  tx
+                    .select({ one: sql`1` })
+                    .from(document)
+                    .where(
+                      and(
+                        eq(document.knowledgeBaseId, knowledgeBaseId),
+                        isNotNull(document.storageKey),
+                        eq(document.storageKey, workspaceFiles.key)
+                      )
+                    )
+                )
+              )
+            )
+        }
+      }
+    })
+  } catch (error: unknown) {
+    if (getPostgresErrorCode(error) === '23505' && updates.name !== undefined) {
+      throw new KnowledgeBaseConflictError(updates.name)
+    }
+    throw error
+  }
+
+  const updatedKb = await db
+    .select({
+      id: knowledgeBase.id,
+      userId: knowledgeBase.userId,
+      name: knowledgeBase.name,
+      description: knowledgeBase.description,
+      tokenCount: sql<number>`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number),
+      embeddingModel: knowledgeBase.embeddingModel,
+      embeddingDimension: knowledgeBase.embeddingDimension,
+      chunkingConfig: knowledgeBase.chunkingConfig,
+      createdAt: knowledgeBase.createdAt,
+      updatedAt: knowledgeBase.updatedAt,
+      deletedAt: knowledgeBase.deletedAt,
+      workspaceId: knowledgeBase.workspaceId,
+      docCount: count(document.id),
+    })
+    .from(knowledgeBase)
+    .leftJoin(
+      document,
+      and(
+        eq(document.knowledgeBaseId, knowledgeBase.id),
+        eq(document.userExcluded, false),
+        isNull(document.archivedAt),
+        isNull(document.deletedAt)
+      )
+    )
+    .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+    .groupBy(knowledgeBase.id)
+    .limit(1)
+
+  if (updatedKb.length === 0) {
+    throw new Error(`Knowledge base ${knowledgeBaseId} not found`)
+  }
+
+  logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`)
+
+  return {
+    ...updatedKb[0],
+    chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig,
+    docCount: Number(updatedKb[0].docCount),
+    connectorTypes: [],
+  }
+}
+
+/**
+ * Get a single knowledge base by ID
+ */
+export async function getKnowledgeBaseById(
+  knowledgeBaseId: string
+): Promise<KnowledgeBaseWithCounts | null> {
+  const result = await db
+    .select({
+      id: knowledgeBase.id,
+      userId: knowledgeBase.userId,
+      name: knowledgeBase.name,
+      description: knowledgeBase.description,
+      tokenCount: sql<number>`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number),
+      embeddingModel: knowledgeBase.embeddingModel,
+      embeddingDimension: knowledgeBase.embeddingDimension,
+      chunkingConfig: knowledgeBase.chunkingConfig,
+      createdAt: knowledgeBase.createdAt,
+      updatedAt: knowledgeBase.updatedAt,
+      deletedAt: knowledgeBase.deletedAt,
+      workspaceId: knowledgeBase.workspaceId,
+      docCount: count(document.id),
+    })
+    .from(knowledgeBase)
+    .leftJoin(
+      document,
+      and(
+        eq(document.knowledgeBaseId, knowledgeBase.id),
+        eq(document.userExcluded, false),
+        isNull(document.archivedAt),
+        isNull(document.deletedAt)
+      )
+    )
+    .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+    .groupBy(knowledgeBase.id)
+    .limit(1)
+
+  if (result.length === 0) {
+    return null
+  }
+
+  return {
+    ...result[0],
+    chunkingConfig: result[0].chunkingConfig as ChunkingConfig,
+    docCount: Number(result[0].docCount),
+    connectorTypes: [],
+  }
+}
+
+/**
+ * Delete a knowledge base (soft delete)
+ */
+export async function deleteKnowledgeBase(
+  knowledgeBaseId: string,
+  requestId: string
+): Promise<void> {
+  const now = new Date()
+
+  await db.transaction(async (tx) => {
+    await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`)
+
+    await tx
+      .update(knowledgeBase)
+      .set({
+        deletedAt: now,
+        updatedAt: now,
+      })
+      .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
+
+    await tx
+      .update(document)
+      .set({
+        archivedAt: now,
+      })
+      .where(
+        and(
+          eq(document.knowledgeBaseId, knowledgeBaseId),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt)
+        )
+      )
+
+    await tx
+      .update(knowledgeConnector)
+      .set({
+        archivedAt: now,
+        status: 'paused',
+        updatedAt: now,
+      })
+      .where(
+        and(
+          eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId),
+          isNull(knowledgeConnector.archivedAt),
+          isNull(knowledgeConnector.deletedAt)
+        )
+      )
+  })
+
+  logger.info(`[${requestId}] Soft deleted knowledge base: ${knowledgeBaseId}`)
+}
+
+/**
+ * Restore a soft-deleted knowledge base and its graph children.
+ * Clears archivedAt on children that were archived as part of the KB snapshot.
+ * Does NOT revive children that were directly deleted (deletedAt set).
+ */
+export async function restoreKnowledgeBase(
+  knowledgeBaseId: string,
+  requestId: string
+): Promise<void> {
+  const [kb] = await db
+    .select({
+      id: knowledgeBase.id,
+      name: knowledgeBase.name,
+      deletedAt: knowledgeBase.deletedAt,
+      workspaceId: knowledgeBase.workspaceId,
+    })
+    .from(knowledgeBase)
+    .where(eq(knowledgeBase.id, knowledgeBaseId))
+    .limit(1)
+
+  if (!kb) {
+    throw new Error('Knowledge base not found')
+  }
+
+  if (!kb.deletedAt) {
+    throw new Error('Knowledge base is not archived')
+  }
+
+  if (kb.workspaceId) {
+    const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils')
+    const ws = await getWorkspaceWithOwner(kb.workspaceId)
+    if (!ws || ws.archivedAt) {
+      throw new Error('Cannot restore knowledge base into an archived workspace')
+    }
+  }
+
+  /**
+   * A concurrent create/rename can commit the same active name after `generateRestoreName`'s check
+   * (MVCC) and before this transaction commits. Retries pick a new random suffix; 23505 is still
+   * mapped to {@link KnowledgeBaseConflictError} if exhaustion occurs.
+   */
+  const maxUniqueViolationRetries = 8
+  let attemptedRestoreName = ''
+
+  for (let attempt = 0; attempt < maxUniqueViolationRetries; attempt++) {
+    attemptedRestoreName = ''
+    try {
+      await db.transaction(async (tx) => {
+        await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`)
+
+        attemptedRestoreName = await generateRestoreName(kb.name, async (candidate) => {
+          if (!kb.workspaceId) return false
+          const [match] = await tx
+            .select({ id: knowledgeBase.id })
+            .from(knowledgeBase)
+            .where(
+              and(
+                eq(knowledgeBase.workspaceId, kb.workspaceId),
+                eq(knowledgeBase.name, candidate),
+                isNull(knowledgeBase.deletedAt)
+              )
+            )
+            .limit(1)
+          return !!match
+        })
+
+        const now = new Date()
+
+        await tx
+          .update(knowledgeBase)
+          .set({ deletedAt: null, updatedAt: now, name: attemptedRestoreName })
+          .where(eq(knowledgeBase.id, knowledgeBaseId))
+
+        await tx
+          .update(document)
+          .set({ archivedAt: null })
+          .where(
+            and(
+              eq(document.knowledgeBaseId, knowledgeBaseId),
+              isNotNull(document.archivedAt),
+              isNull(document.deletedAt)
+            )
+          )
+
+        await tx
+          .update(knowledgeConnector)
+          .set({ archivedAt: null, status: 'active', updatedAt: now })
+          .where(
+            and(
+              eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId),
+              isNotNull(knowledgeConnector.archivedAt),
+              isNull(knowledgeConnector.deletedAt)
+            )
+          )
+      })
+      break
+    } catch (error: unknown) {
+      if (getPostgresErrorCode(error) !== '23505') {
+        throw error
+      }
+      if (attempt === maxUniqueViolationRetries - 1) {
+        throw new KnowledgeBaseConflictError(attemptedRestoreName || kb.name)
+      }
+    }
+  }
+
+  logger.info(
+    `[${requestId}] Restored knowledge base: ${knowledgeBaseId} as "${attemptedRestoreName}"`
+  )
+}

apps/sim/lib/knowledge/tags/service.ts

@@ -0,0 +1,745 @@
+import { db } from '@sim/db'
+import { document, embedding, knowledgeBaseTagDefinitions } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
+import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { getSlotsForFieldType, SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants'
+import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types'
+import type {
+  CreateTagDefinitionData,
+  TagDefinition,
+  UpdateTagDefinitionData,
+} from '@/lib/knowledge/types'
+
+const logger = createLogger('TagsService')
+
+/** Text tag slots */
+const VALID_TEXT_SLOTS = ['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7'] as const
+
+const VALID_NUMBER_SLOTS = ['number1', 'number2', 'number3', 'number4', 'number5'] as const
+/** Date tag slots (reduced to 2 for write performance) */
+const VALID_DATE_SLOTS = ['date1', 'date2'] as const
+/** Boolean tag slots */
+const VALID_BOOLEAN_SLOTS = ['boolean1', 'boolean2', 'boolean3'] as const
+
+/** All valid tag slots combined */
+const VALID_TAG_SLOTS = [
+  ...VALID_TEXT_SLOTS,
+  ...VALID_NUMBER_SLOTS,
+  ...VALID_DATE_SLOTS,
+  ...VALID_BOOLEAN_SLOTS,
+] as const
+
+type ValidTagSlot = (typeof VALID_TAG_SLOTS)[number]
+
+/**
+ * Validates that a tag slot is a valid slot name
+ */
+function validateTagSlot(tagSlot: string): asserts tagSlot is ValidTagSlot {
+  if (!VALID_TAG_SLOTS.includes(tagSlot as ValidTagSlot)) {
+    throw new Error(`Invalid tag slot: ${tagSlot}. Must be one of: ${VALID_TAG_SLOTS.join(', ')}`)
+  }
+}
+
+/**
+ * Get the field type for a tag slot
+ */
+function getFieldTypeForSlot(tagSlot: string): string | null {
+  if ((VALID_TEXT_SLOTS as readonly string[]).includes(tagSlot)) return 'text'
+  if ((VALID_NUMBER_SLOTS as readonly string[]).includes(tagSlot)) return 'number'
+  if ((VALID_DATE_SLOTS as readonly string[]).includes(tagSlot)) return 'date'
+  if ((VALID_BOOLEAN_SLOTS as readonly string[]).includes(tagSlot)) return 'boolean'
+  return null
+}
+
+/**
+ * Get the next available slot for a knowledge base and field type
+ */
+export async function getNextAvailableSlot(
+  knowledgeBaseId: string,
+  fieldType: string,
+  existingBySlot?: Map<string, any>
+): Promise<string | null> {
+  const availableSlots = getSlotsForFieldType(fieldType)
+  let usedSlots: Set<string>
+
+  if (existingBySlot) {
+    usedSlots = new Set(
+      Array.from(existingBySlot.entries())
+        .filter(([_, def]) => def.fieldType === fieldType)
+        .map(([slot, _]) => slot)
+    )
+  } else {
+    const existingDefinitions = await db
+      .select({ tagSlot: knowledgeBaseTagDefinitions.tagSlot })
+      .from(knowledgeBaseTagDefinitions)
+      .where(
+        and(
+          eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
+          eq(knowledgeBaseTagDefinitions.fieldType, fieldType)
+        )
+      )
+
+    usedSlots = new Set(existingDefinitions.map((def) => def.tagSlot))
+  }
+
+  for (const slot of availableSlots) {
+    if (!usedSlots.has(slot)) {
+      return slot
+    }
+  }
+
+  return null // All slots for this field type are used
+}
+
+/**
+ * Get all tag definitions for a knowledge base
+ */
+export async function getDocumentTagDefinitions(
+  knowledgeBaseId: string
+): Promise<DocumentTagDefinition[]> {
+  const definitions = await db
+    .select({
+      id: knowledgeBaseTagDefinitions.id,
+      knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId,
+      tagSlot: knowledgeBaseTagDefinitions.tagSlot,
+      displayName: knowledgeBaseTagDefinitions.displayName,
+      fieldType: knowledgeBaseTagDefinitions.fieldType,
+      createdAt: knowledgeBaseTagDefinitions.createdAt,
+      updatedAt: knowledgeBaseTagDefinitions.updatedAt,
+    })
+    .from(knowledgeBaseTagDefinitions)
+    .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+    .orderBy(knowledgeBaseTagDefinitions.tagSlot)
+
+  return definitions.map((def) => ({
+    ...def,
+    tagSlot: def.tagSlot as string,
+  }))
+}
+
+/**
+ * Get all tag definitions for a knowledge base (alias for compatibility)
+ */
+export async function getTagDefinitions(knowledgeBaseId: string): Promise<TagDefinition[]> {
+  const tagDefinitions = await db
+    .select({
+      id: knowledgeBaseTagDefinitions.id,
+      tagSlot: knowledgeBaseTagDefinitions.tagSlot,
+      displayName: knowledgeBaseTagDefinitions.displayName,
+      fieldType: knowledgeBaseTagDefinitions.fieldType,
+      createdAt: knowledgeBaseTagDefinitions.createdAt,
+      updatedAt: knowledgeBaseTagDefinitions.updatedAt,
+    })
+    .from(knowledgeBaseTagDefinitions)
+    .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+    .orderBy(knowledgeBaseTagDefinitions.tagSlot)
+
+  return tagDefinitions.map((def) => ({
+    ...def,
+    tagSlot: def.tagSlot as string,
+  }))
+}
+
+/**
+ * Create or update tag definitions in bulk
+ */
+export async function createOrUpdateTagDefinitionsBulk(
+  knowledgeBaseId: string,
+  bulkData: BulkTagDefinitionsData,
+  requestId: string
+): Promise<{
+  created: DocumentTagDefinition[]
+  updated: DocumentTagDefinition[]
+  errors: string[]
+}> {
+  const { definitions } = bulkData
+  const created: DocumentTagDefinition[] = []
+  const updated: DocumentTagDefinition[] = []
+  const errors: string[] = []
+
+  // Get existing definitions to check for conflicts and determine operations
+  const existingDefinitions = await getDocumentTagDefinitions(knowledgeBaseId)
+  const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot, def]))
+  const existingByDisplayName = new Map(existingDefinitions.map((def) => [def.displayName, def]))
+
+  // Process each definition
+  for (const defData of definitions) {
+    try {
+      const { tagSlot, displayName, fieldType, originalDisplayName } = defData
+
+      // Validate field type
+      if (!SUPPORTED_FIELD_TYPES.includes(fieldType as (typeof SUPPORTED_FIELD_TYPES)[number])) {
+        errors.push(`Invalid field type: ${fieldType}`)
+        continue
+      }
+
+      // Check if this is an update (has originalDisplayName) or create
+      const isUpdate = !!originalDisplayName
+
+      if (isUpdate) {
+        // Update existing definition
+        const existingDef = existingByDisplayName.get(originalDisplayName!)
+        if (!existingDef) {
+          errors.push(`Tag definition with display name "${originalDisplayName}" not found`)
+          continue
+        }
+
+        // Check if new display name conflicts with another definition
+        if (displayName !== originalDisplayName && existingByDisplayName.has(displayName)) {
+          errors.push(`Display name "${displayName}" already exists`)
+          continue
+        }
+
+        const now = new Date()
+        await db
+          .update(knowledgeBaseTagDefinitions)
+          .set({
+            displayName,
+            fieldType,
+            updatedAt: now,
+          })
+          .where(eq(knowledgeBaseTagDefinitions.id, existingDef.id))
+
+        updated.push({
+          id: existingDef.id,
+          knowledgeBaseId,
+          tagSlot: existingDef.tagSlot,
+          displayName,
+          fieldType,
+          createdAt: existingDef.createdAt,
+          updatedAt: now,
+        })
+      } else {
+        // Create new definition
+        let finalTagSlot = tagSlot
+
+        // If no slot provided or slot is taken, find next available
+        if (!finalTagSlot || existingBySlot.has(finalTagSlot)) {
+          const nextSlot = await getNextAvailableSlot(knowledgeBaseId, fieldType, existingBySlot)
+          if (!nextSlot) {
+            errors.push(`No available slots for field type "${fieldType}"`)
+            continue
+          }
+          finalTagSlot = nextSlot
+        }
+
+        // Check slot conflicts
+        if (existingBySlot.has(finalTagSlot)) {
+          errors.push(`Tag slot "${finalTagSlot}" is already in use`)
+          continue
+        }
+
+        // Check display name conflicts
+        if (existingByDisplayName.has(displayName)) {
+          errors.push(`Display name "${displayName}" already exists`)
+          continue
+        }
+
+        const id = generateId()
+        const now = new Date()
+
+        const newDefinition = {
+          id,
+          knowledgeBaseId,
+          tagSlot: finalTagSlot as ValidTagSlot,
+          displayName,
+          fieldType,
+          createdAt: now,
+          updatedAt: now,
+        }
+
+        await db.insert(knowledgeBaseTagDefinitions).values(newDefinition)
+
+        // Add to maps to track for subsequent definitions in this batch
+        existingBySlot.set(finalTagSlot, newDefinition)
+        existingByDisplayName.set(displayName, newDefinition)
+
+        created.push(newDefinition as DocumentTagDefinition)
+      }
+    } catch (error) {
+      errors.push(`Error processing definition "${defData.displayName}": ${error}`)
+    }
+  }
+
+  logger.info(
+    `[${requestId}] Bulk tag definitions processed: ${created.length} created, ${updated.length} updated, ${errors.length} errors`
+  )
+
+  return { created, updated, errors }
+}
+
+/**
+ * Get a single tag definition by ID
+ */
+export async function getTagDefinitionById(
+  tagDefinitionId: string
+): Promise<DocumentTagDefinition | null> {
+  const result = await db
+    .select({
+      id: knowledgeBaseTagDefinitions.id,
+      knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId,
+      tagSlot: knowledgeBaseTagDefinitions.tagSlot,
+      displayName: knowledgeBaseTagDefinitions.displayName,
+      fieldType: knowledgeBaseTagDefinitions.fieldType,
+      createdAt: knowledgeBaseTagDefinitions.createdAt,
+      updatedAt: knowledgeBaseTagDefinitions.updatedAt,
+    })
+    .from(knowledgeBaseTagDefinitions)
+    .where(eq(knowledgeBaseTagDefinitions.id, tagDefinitionId))
+    .limit(1)
+
+  if (result.length === 0) {
+    return null
+  }
+
+  const def = result[0]
+  return {
+    ...def,
+    tagSlot: def.tagSlot as string,
+  }
+}
+
+/**
+ * Update tags on all documents and chunks when a tag value is changed
+ */
+async function updateTagValuesInDocumentsAndChunks(
+  knowledgeBaseId: string,
+  tagSlot: string,
+  oldValue: string | null,
+  newValue: string | null,
+  requestId: string
+): Promise<{ documentsUpdated: number; chunksUpdated: number }> {
+  validateTagSlot(tagSlot)
+
+  let documentsUpdated = 0
+  let chunksUpdated = 0
+
+  await db.transaction(async (tx) => {
+    if (oldValue) {
+      await tx
+        .update(document)
+        .set({
+          [tagSlot]: newValue,
+        })
+        .where(
+          and(
+            eq(document.knowledgeBaseId, knowledgeBaseId),
+            eq(sql.raw(`${document}.${tagSlot}`), oldValue)
+          )
+        )
+      documentsUpdated = 1
+    }
+
+    if (oldValue) {
+      await tx
+        .update(embedding)
+        .set({
+          [tagSlot]: newValue,
+        })
+        .where(
+          and(
+            eq(embedding.knowledgeBaseId, knowledgeBaseId),
+            eq(sql.raw(`${embedding}.${tagSlot}`), oldValue)
+          )
+        )
+      chunksUpdated = 1
+    }
+  })
+
+  logger.info(
+    `[${requestId}] Updated tag values: ${documentsUpdated} documents, ${chunksUpdated} chunks`
+  )
+
+  return { documentsUpdated, chunksUpdated }
+}
+
+/**
+ * Cleanup unused tag definitions for a knowledge base
+ */
+export async function cleanupUnusedTagDefinitions(
+  knowledgeBaseId: string,
+  requestId: string
+): Promise<number> {
+  const definitions = await getDocumentTagDefinitions(knowledgeBaseId)
+  let cleanedUp = 0
+
+  for (const def of definitions) {
+    const tagSlot = def.tagSlot
+    validateTagSlot(tagSlot)
+
+    const docCountResult = await db
+      .select({ count: sql<number>`count(*)` })
+      .from(document)
+      .where(
+        and(
+          eq(document.knowledgeBaseId, knowledgeBaseId),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt),
+          sql`${sql.raw(tagSlot)} IS NOT NULL`
+        )
+      )
+
+    const chunkCountResult = await db
+      .select({ count: sql<number>`count(*)` })
+      .from(embedding)
+      .innerJoin(document, eq(embedding.documentId, document.id))
+      .where(
+        and(
+          eq(embedding.knowledgeBaseId, knowledgeBaseId),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt),
+          sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL`
+        )
+      )
+
+    const docCount = Number(docCountResult[0]?.count || 0)
+    const chunkCount = Number(chunkCountResult[0]?.count || 0)
+
+    if (docCount === 0 && chunkCount === 0) {
+      await db.delete(knowledgeBaseTagDefinitions).where(eq(knowledgeBaseTagDefinitions.id, def.id))
+
+      cleanedUp++
+      logger.info(
+        `[${requestId}] Cleaned up unused tag definition: ${def.displayName} (${def.tagSlot})`
+      )
+    }
+  }
+
+  logger.info(`[${requestId}] Cleanup completed: ${cleanedUp} unused tag definitions removed`)
+  return cleanedUp
+}
+
+/**
+ * Delete all tag definitions for a knowledge base
+ */
+export async function deleteAllTagDefinitions(
+  knowledgeBaseId: string,
+  requestId: string
+): Promise<number> {
+  const definitions = await db
+    .select({ id: knowledgeBaseTagDefinitions.id, tagSlot: knowledgeBaseTagDefinitions.tagSlot })
+    .from(knowledgeBaseTagDefinitions)
+    .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+
+  await db.transaction(async (tx) => {
+    for (const definition of definitions) {
+      const tagSlot = definition.tagSlot as string
+      validateTagSlot(tagSlot)
+
+      await tx
+        .update(document)
+        .set({ [tagSlot]: null })
+        .where(
+          and(eq(document.knowledgeBaseId, knowledgeBaseId), isNotNull(sql`${sql.raw(tagSlot)}`))
+        )
+
+      await tx
+        .update(embedding)
+        .set({ [tagSlot]: null })
+        .where(
+          and(eq(embedding.knowledgeBaseId, knowledgeBaseId), isNotNull(sql`${sql.raw(tagSlot)}`))
+        )
+    }
+
+    await tx
+      .delete(knowledgeBaseTagDefinitions)
+      .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
+  })
+
+  const deletedCount = definitions.length
+  logger.info(`[${requestId}] Deleted ${deletedCount} tag definitions for KB: ${knowledgeBaseId}`)
+
+  return deletedCount
+}
+
+/**
+ * Delete a tag definition with comprehensive cleanup
+ * This removes the definition and clears all document/chunk references
+ */
+export async function deleteTagDefinition(
+  knowledgeBaseId: string,
+  tagDefinitionId: string,
+  requestId: string
+): Promise<{ tagSlot: string; displayName: string }> {
+  const tagDef = await db
+    .select({
+      id: knowledgeBaseTagDefinitions.id,
+      knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId,
+      tagSlot: knowledgeBaseTagDefinitions.tagSlot,
+      displayName: knowledgeBaseTagDefinitions.displayName,
+    })
+    .from(knowledgeBaseTagDefinitions)
+    .where(
+      and(
+        eq(knowledgeBaseTagDefinitions.id, tagDefinitionId),
+        eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)
+      )
+    )
+    .limit(1)
+
+  if (tagDef.length === 0) {
+    throw new Error(`Tag definition ${tagDefinitionId} not found`)
+  }
+
+  const definition = tagDef[0]
+  const definitionKnowledgeBaseId = definition.knowledgeBaseId
+  const tagSlot = definition.tagSlot as string
+
+  validateTagSlot(tagSlot)
+
+  await db.transaction(async (tx) => {
+    await tx
+      .update(document)
+      .set({ [tagSlot]: null })
+      .where(
+        and(
+          eq(document.knowledgeBaseId, definitionKnowledgeBaseId),
+          isNotNull(sql`${sql.raw(tagSlot)}`)
+        )
+      )
+
+    await tx
+      .update(embedding)
+      .set({ [tagSlot]: null })
+      .where(
+        and(
+          eq(embedding.knowledgeBaseId, definitionKnowledgeBaseId),
+          isNotNull(sql`${sql.raw(tagSlot)}`)
+        )
+      )
+
+    await tx
+      .delete(knowledgeBaseTagDefinitions)
+      .where(eq(knowledgeBaseTagDefinitions.id, tagDefinitionId))
+  })
+
+  logger.info(
+    `[${requestId}] Deleted tag definition with cleanup: ${definition.displayName} (${tagSlot})`
+  )
+
+  return {
+    tagSlot,
+    displayName: definition.displayName,
+  }
+}
+
+/**
+ * Create a new tag definition
+ */
+export async function createTagDefinition(
+  data: CreateTagDefinitionData,
+  requestId: string,
+  txDb?: DbOrTx
+): Promise<TagDefinition> {
+  const dbInstance = txDb ?? db
+  const tagDefinitionId = generateId()
+  const now = new Date()
+
+  const newDefinition = {
+    id: tagDefinitionId,
+    knowledgeBaseId: data.knowledgeBaseId,
+    tagSlot: data.tagSlot as ValidTagSlot,
+    displayName: data.displayName,
+    fieldType: data.fieldType,
+    createdAt: now,
+    updatedAt: now,
+  }
+
+  await dbInstance.insert(knowledgeBaseTagDefinitions).values(newDefinition)
+
+  logger.info(
+    `[${requestId}] Created tag definition: ${data.displayName} -> ${data.tagSlot} in KB ${data.knowledgeBaseId}`
+  )
+
+  return {
+    id: tagDefinitionId,
+    tagSlot: data.tagSlot,
+    displayName: data.displayName,
+    fieldType: data.fieldType,
+    createdAt: now,
+    updatedAt: now,
+  }
+}
+
+/**
+ * Update an existing tag definition
+ */
+export async function updateTagDefinition(
+  tagDefinitionId: string,
+  data: UpdateTagDefinitionData,
+  requestId: string
+): Promise<TagDefinition> {
+  const now = new Date()
+
+  const updateData: {
+    updatedAt: Date
+    displayName?: string
+    fieldType?: string
+  } = {
+    updatedAt: now,
+  }
+
+  if (data.displayName !== undefined) {
+    updateData.displayName = data.displayName
+  }
+
+  if (data.fieldType !== undefined) {
+    updateData.fieldType = data.fieldType
+  }
+
+  const updatedRows = await db
+    .update(knowledgeBaseTagDefinitions)
+    .set(updateData)
+    .where(eq(knowledgeBaseTagDefinitions.id, tagDefinitionId))
+    .returning({
+      id: knowledgeBaseTagDefinitions.id,
+      tagSlot: knowledgeBaseTagDefinitions.tagSlot,
+      displayName: knowledgeBaseTagDefinitions.displayName,
+      fieldType: knowledgeBaseTagDefinitions.fieldType,
+      createdAt: knowledgeBaseTagDefinitions.createdAt,
+      updatedAt: knowledgeBaseTagDefinitions.updatedAt,
+    })
+
+  if (updatedRows.length === 0) {
+    throw new Error(`Tag definition ${tagDefinitionId} not found`)
+  }
+
+  const updated = updatedRows[0]
+  logger.info(`[${requestId}] Updated tag definition: ${tagDefinitionId}`)
+
+  return {
+    ...updated,
+    tagSlot: updated.tagSlot as string,
+  }
+}
+
+/**
+ * Get tag usage with detailed document information (original format)
+ */
+export async function getTagUsage(
+  knowledgeBaseId: string,
+  requestId = 'api'
+): Promise<
+  Array<{
+    tagName: string
+    tagSlot: string
+    documentCount: number
+    documents: Array<{ id: string; name: string; tagValue: string }>
+  }>
+> {
+  const definitions = await getDocumentTagDefinitions(knowledgeBaseId)
+  const usage = []
+
+  for (const def of definitions) {
+    const tagSlot = def.tagSlot
+    validateTagSlot(tagSlot)
+
+    // Build WHERE conditions based on field type
+    // Text columns need both IS NOT NULL and != '' checks
+    // Numeric/date/boolean columns only need IS NOT NULL
+    const fieldType = getFieldTypeForSlot(tagSlot)
+    const isTextColumn = fieldType === 'text'
+
+    const whereConditions = [
+      eq(document.knowledgeBaseId, knowledgeBaseId),
+      eq(document.userExcluded, false),
+      isNull(document.archivedAt),
+      isNull(document.deletedAt),
+      isNotNull(sql`${sql.raw(tagSlot)}`),
+    ]
+
+    // Only add empty string check for text columns
+    if (isTextColumn) {
+      whereConditions.push(sql`${sql.raw(tagSlot)} != ''`)
+    }
+
+    const documentsWithTag = await db
+      .select({
+        id: document.id,
+        filename: document.filename,
+        tagValue: sql<string>`${sql.raw(tagSlot)}::text`,
+      })
+      .from(document)
+      .where(and(...whereConditions))
+
+    usage.push({
+      tagName: def.displayName,
+      tagSlot: def.tagSlot,
+      documentCount: documentsWithTag.length,
+      documents: documentsWithTag.map((doc) => ({
+        id: doc.id,
+        name: doc.filename,
+        tagValue: doc.tagValue || '',
+      })),
+    })
+  }
+
+  logger.info(`[${requestId}] Retrieved detailed tag usage for ${usage.length} definitions`)
+
+  return usage
+}
+
+/**
+ * Get tag usage statistics
+ */
+export async function getTagUsageStats(
+  knowledgeBaseId: string,
+  requestId: string
+): Promise<
+  Array<{
+    tagSlot: string
+    displayName: string
+    fieldType: string
+    documentCount: number
+    chunkCount: number
+  }>
+> {
+  const definitions = await getDocumentTagDefinitions(knowledgeBaseId)
+  const stats = []
+
+  for (const def of definitions) {
+    const tagSlot = def.tagSlot
+    validateTagSlot(tagSlot)
+
+    const docCountResult = await db
+      .select({ count: sql<number>`count(*)` })
+      .from(document)
+      .where(
+        and(
+          eq(document.knowledgeBaseId, knowledgeBaseId),
+          eq(document.userExcluded, false),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt),
+          sql`${sql.raw(tagSlot)} IS NOT NULL`
+        )
+      )
+
+    const chunkCountResult = await db
+      .select({ count: sql<number>`count(*)` })
+      .from(embedding)
+      .innerJoin(document, eq(embedding.documentId, document.id))
+      .where(
+        and(
+          eq(embedding.knowledgeBaseId, knowledgeBaseId),
+          eq(document.userExcluded, false),
+          isNull(document.archivedAt),
+          isNull(document.deletedAt),
+          sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL`
+        )
+      )
+
+    stats.push({
+      tagSlot: def.tagSlot,
+      displayName: def.displayName,
+      fieldType: def.fieldType,
+      documentCount: Number(docCountResult[0]?.count || 0),
+      chunkCount: Number(chunkCountResult[0]?.count || 0),
+    })
+  }
+
+  logger.info(`[${requestId}] Retrieved tag usage stats for ${stats.length} definitions`)
+
+  return stats
+}

apps/sim/lib/knowledge/tags/types.ts

@@ -0,0 +1,30 @@
+export interface DocumentTagDefinition {
+  id: string
+  knowledgeBaseId: string
+  tagSlot: string
+  displayName: string
+  fieldType: string
+  createdAt: Date
+  updatedAt: Date
+}
+
+/**
+ * Represents a tag assigned to a document with its slot, display name, type, and value
+ */
+export interface DocumentTag {
+  slot: string
+  displayName: string
+  fieldType: string
+  value: string
+}
+
+interface CreateTagDefinitionData {
+  tagSlot: string
+  displayName: string
+  fieldType: string
+  originalDisplayName?: string
+}
+
+export interface BulkTagDefinitionsData {
+  definitions: CreateTagDefinitionData[]
+}

apps/sim/lib/knowledge/tags/utils.ts

@@ -0,0 +1,89 @@
+/**
+ * Validate a tag value against its expected field type
+ * Returns an error message if invalid, or null if valid
+ */
+export function validateTagValue(tagName: string, value: string, fieldType: string): string | null {
+  const stringValue = String(value).trim()
+
+  switch (fieldType) {
+    case 'boolean': {
+      const lowerValue = stringValue.toLowerCase()
+      if (lowerValue !== 'true' && lowerValue !== 'false') {
+        return `Tag "${tagName}" expects a boolean value (true/false), but received "${value}"`
+      }
+      return null
+    }
+    case 'number': {
+      const numValue = Number(stringValue)
+      if (Number.isNaN(numValue)) {
+        return `Tag "${tagName}" expects a number value, but received "${value}"`
+      }
+      return null
+    }
+    case 'date': {
+      // Check format first
+      if (!/^\d{4}-\d{2}-\d{2}$/.test(stringValue)) {
+        return `Tag "${tagName}" expects a date in YYYY-MM-DD format, but received "${value}"`
+      }
+      // Validate the date is actually valid (e.g., reject 2024-02-31)
+      const [year, month, day] = stringValue.split('-').map(Number)
+      const date = new Date(year, month - 1, day)
+      if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
+        return `Tag "${tagName}" has an invalid date: "${value}"`
+      }
+      return null
+    }
+    default:
+      return null
+  }
+}
+
+/**
+ * Build error message for undefined tags
+ */
+export function buildUndefinedTagsError(undefinedTags: string[]): string {
+  const tagList = undefinedTags.map((t) => `"${t}"`).join(', ')
+  return `The following tags are not defined in this knowledge base: ${tagList}. Please define them at the knowledge base level first.`
+}
+
+/**
+ * Parse a string to number with strict validation
+ * Returns null if invalid
+ */
+export function parseNumberValue(value: string): number | null {
+  const num = Number(value)
+  return Number.isNaN(num) ? null : num
+}
+
+/**
+ * Parse a string to Date with strict YYYY-MM-DD validation
+ * Returns null if invalid format or invalid date
+ */
+export function parseDateValue(value: string): Date | null {
+  const stringValue = String(value).trim()
+
+  // Must be YYYY-MM-DD format
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(stringValue)) {
+    return null
+  }
+
+  // Validate the date is actually valid (e.g., reject 2024-02-31)
+  const [year, month, day] = stringValue.split('-').map(Number)
+  const date = new Date(year, month - 1, day)
+  if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
+    return null
+  }
+
+  return date
+}
+
+/**
+ * Parse a string to boolean with strict validation
+ * Returns null if not 'true' or 'false'
+ */
+export function parseBooleanValue(value: string): boolean | null {
+  const lowerValue = String(value).trim().toLowerCase()
+  if (lowerValue === 'true') return true
+  if (lowerValue === 'false') return false
+  return null
+}

apps/sim/lib/knowledge/types.ts

@@ -0,0 +1,201 @@
+import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
+
+/**
+ * Units:
+ * - maxSize/overlap: TOKENS (1 token ≈ 4 characters)
+ * - minSize: CHARACTERS
+ */
+export interface ChunkingConfig {
+  maxSize: number
+  minSize: number
+  overlap: number
+  strategy?: ChunkingStrategy
+  strategyOptions?: StrategyOptions
+}
+
+export interface KnowledgeBaseWithCounts {
+  id: string
+  userId: string
+  name: string
+  description: string | null
+  tokenCount: number
+  embeddingModel: string
+  embeddingDimension: number
+  chunkingConfig: ChunkingConfig
+  createdAt: Date
+  updatedAt: Date
+  deletedAt: Date | null
+  workspaceId: string | null
+  docCount: number
+  connectorTypes: string[]
+}
+
+export interface CreateKnowledgeBaseData {
+  name: string
+  description?: string
+  workspaceId: string
+  embeddingModel: string
+  embeddingDimension: 1536
+  chunkingConfig: ChunkingConfig
+  userId: string
+}
+
+export interface TagDefinition {
+  id: string
+  tagSlot: string
+  displayName: string
+  fieldType: string
+  createdAt: Date
+  updatedAt: Date
+}
+
+export interface CreateTagDefinitionData {
+  knowledgeBaseId: string
+  tagSlot: string
+  displayName: string
+  fieldType: string
+}
+
+export interface UpdateTagDefinitionData {
+  displayName?: string
+  fieldType?: string
+}
+
+export interface StructuredFilter {
+  tagName?: string
+  tagSlot: string
+  fieldType: string
+  operator: string
+  value: string | number | boolean
+  valueTo?: string | number
+}
+
+export interface ProcessedDocumentTags {
+  tag1: string | null
+  tag2: string | null
+  tag3: string | null
+  tag4: string | null
+  tag5: string | null
+  tag6: string | null
+  tag7: string | null
+  number1: number | null
+  number2: number | null
+  number3: number | null
+  number4: number | null
+  number5: number | null
+  date1: Date | null
+  date2: Date | null
+  boolean1: boolean | null
+  boolean2: boolean | null
+  boolean3: boolean | null
+  [key: string]: string | number | Date | boolean | null
+}
+
+/** These types use string dates for JSON serialization */
+
+interface ExtendedChunkingConfig extends ChunkingConfig {
+  chunkSize?: number
+  minCharactersPerChunk?: number
+  recipe?: string
+  lang?: string
+  [key: string]: unknown
+}
+
+export interface KnowledgeBaseData {
+  id: string
+  userId: string
+  name: string
+  description: string | null
+  tokenCount: number
+  embeddingModel: string
+  embeddingDimension: number
+  chunkingConfig: ExtendedChunkingConfig
+  createdAt: string
+  updatedAt: string
+  deletedAt: string | null
+  workspaceId: string | null
+  docCount?: number
+  connectorTypes?: string[]
+}
+
+export interface DocumentData {
+  id: string
+  knowledgeBaseId: string
+  filename: string
+  fileUrl: string
+  fileSize: number
+  mimeType: string
+  chunkCount: number
+  tokenCount: number
+  characterCount: number
+  processingStatus: 'pending' | 'processing' | 'completed' | 'failed'
+  processingStartedAt?: string | null
+  processingCompletedAt?: string | null
+  processingError?: string | null
+  enabled: boolean
+  uploadedAt: string
+  tag1?: string | null
+  tag2?: string | null
+  tag3?: string | null
+  tag4?: string | null
+  tag5?: string | null
+  tag6?: string | null
+  tag7?: string | null
+  number1?: number | null
+  number2?: number | null
+  number3?: number | null
+  number4?: number | null
+  number5?: number | null
+  date1?: string | null
+  date2?: string | null
+  boolean1?: boolean | null
+  boolean2?: boolean | null
+  boolean3?: boolean | null
+  connectorId?: string | null
+  connectorType?: string | null
+  sourceUrl?: string | null
+}
+
+export interface ChunkData {
+  id: string
+  chunkIndex: number
+  content: string
+  contentLength: number
+  tokenCount: number
+  enabled: boolean
+  startOffset: number
+  endOffset: number
+  tag1?: string | null
+  tag2?: string | null
+  tag3?: string | null
+  tag4?: string | null
+  tag5?: string | null
+  tag6?: string | null
+  tag7?: string | null
+  number1?: number | null
+  number2?: number | null
+  number3?: number | null
+  number4?: number | null
+  number5?: number | null
+  date1?: string | null
+  date2?: string | null
+  boolean1?: boolean | null
+  boolean2?: boolean | null
+  boolean3?: boolean | null
+  createdAt: string
+  updatedAt: string
+}
+
+interface ChunksPagination {
+  total: number
+  limit: number
+  offset: number
+  hasMore: boolean
+}
+
+interface DocumentsPagination {
+  total: number
+  limit: number
+  offset: number
+  hasMore: boolean
+}