chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,175 @@
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
import { db } from '@sim/db'
import { document, knowledgeConnector } from '@sim/db/schema'
import { and, eq, isNull } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import {
v1DeleteKnowledgeDocumentContract,
v1GetKnowledgeDocumentContract,
} from '@/lib/api/contracts/v1/knowledge'
import { parseRequest } from '@/lib/api/server'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { deleteDocument } from '@/lib/knowledge/documents/service'
import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils'
import { authenticateRequest } from '@/app/api/v1/middleware'
export const dynamic = 'force-dynamic'
export const revalidate = 0
interface DocumentDetailRouteParams {
params: Promise<{ id: string; documentId: string }>
}
/** GET /api/v1/knowledge/[id]/documents/[documentId] — Get document details. */
export const GET = withRouteHandler(
async (request: NextRequest, context: DocumentDetailRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1GetKnowledgeDocumentContract, request, context)
if (!parsed.success) return parsed.response
const { id: knowledgeBaseId, documentId } = parsed.data.params
const result = await resolveKnowledgeBase(
knowledgeBaseId,
parsed.data.query.workspaceId,
userId,
rateLimit
)
if (result instanceof NextResponse) return result
const docs = await db
.select({
id: document.id,
knowledgeBaseId: document.knowledgeBaseId,
filename: document.filename,
fileSize: document.fileSize,
mimeType: document.mimeType,
processingStatus: document.processingStatus,
processingError: document.processingError,
processingStartedAt: document.processingStartedAt,
processingCompletedAt: document.processingCompletedAt,
chunkCount: document.chunkCount,
tokenCount: document.tokenCount,
characterCount: document.characterCount,
enabled: document.enabled,
uploadedAt: document.uploadedAt,
connectorId: document.connectorId,
connectorType: knowledgeConnector.connectorType,
sourceUrl: document.sourceUrl,
})
.from(document)
.leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id))
.where(
and(
eq(document.id, documentId),
eq(document.knowledgeBaseId, knowledgeBaseId),
eq(document.userExcluded, false),
isNull(document.archivedAt),
isNull(document.deletedAt)
)
)
.limit(1)
if (docs.length === 0) {
return NextResponse.json({ error: 'Document not found' }, { status: 404 })
}
const doc = docs[0]
return NextResponse.json({
success: true,
data: {
document: {
id: doc.id,
knowledgeBaseId: doc.knowledgeBaseId,
filename: doc.filename,
fileSize: doc.fileSize,
mimeType: doc.mimeType,
processingStatus: doc.processingStatus,
processingError: doc.processingError,
processingStartedAt: serializeDate(doc.processingStartedAt),
processingCompletedAt: serializeDate(doc.processingCompletedAt),
chunkCount: doc.chunkCount,
tokenCount: doc.tokenCount,
characterCount: doc.characterCount,
enabled: doc.enabled,
connectorId: doc.connectorId,
connectorType: doc.connectorType,
sourceUrl: doc.sourceUrl,
createdAt: serializeDate(doc.uploadedAt),
},
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to get document')
}
}
)
/** DELETE /api/v1/knowledge/[id]/documents/[documentId] — Delete a document. */
export const DELETE = withRouteHandler(
async (request: NextRequest, context: DocumentDetailRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1DeleteKnowledgeDocumentContract, request, context)
if (!parsed.success) return parsed.response
const { id: knowledgeBaseId, documentId } = parsed.data.params
const result = await resolveKnowledgeBase(
knowledgeBaseId,
parsed.data.query.workspaceId,
userId,
rateLimit,
'write'
)
if (result instanceof NextResponse) return result
const docs = await db
.select({ id: document.id, filename: document.filename })
.from(document)
.where(
and(
eq(document.id, documentId),
eq(document.knowledgeBaseId, knowledgeBaseId),
eq(document.userExcluded, false),
isNull(document.archivedAt),
isNull(document.deletedAt)
)
)
.limit(1)
if (docs.length === 0) {
return NextResponse.json({ error: 'Document not found' }, { status: 404 })
}
await deleteDocument(documentId, requestId)
recordAudit({
workspaceId: parsed.data.query.workspaceId,
actorId: userId,
action: AuditAction.DOCUMENT_DELETED,
resourceType: AuditResourceType.DOCUMENT,
resourceId: documentId,
resourceName: docs[0].filename,
description: `Deleted document "${docs[0].filename}" from knowledge base via API`,
metadata: { knowledgeBaseId },
request,
})
return NextResponse.json({
success: true,
data: {
message: 'Document deleted successfully',
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to delete document')
}
}
)
@@ -0,0 +1,168 @@
/**
* Tests for the v1 knowledge document upload route's bounded multipart read.
*
* @vitest-environment node
*/
import { getErrorMessage } from '@sim/utils/errors'
import { NextRequest } from 'next/server'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockAuthenticateRequest,
mockResolveKnowledgeBase,
mockCheckActorUsageLimits,
mockUploadWorkspaceFile,
mockCreateSingleDocument,
mockProcessDocumentsWithQueue,
mockValidateFileType,
} = vi.hoisted(() => ({
mockAuthenticateRequest: vi.fn(),
mockResolveKnowledgeBase: vi.fn(),
mockCheckActorUsageLimits: vi.fn(),
mockUploadWorkspaceFile: vi.fn(),
mockCreateSingleDocument: vi.fn(),
mockProcessDocumentsWithQueue: vi.fn(),
mockValidateFileType: vi.fn(),
}))
vi.mock('@/app/api/v1/middleware', () => ({
authenticateRequest: mockAuthenticateRequest,
}))
vi.mock('@/app/api/v1/knowledge/utils', () => ({
resolveKnowledgeBase: mockResolveKnowledgeBase,
serializeDate: (date: unknown) => (date instanceof Date ? date.toISOString() : date),
handleError: (_requestId: string, error: unknown) =>
new Response(JSON.stringify({ error: getErrorMessage(error, 'error') }), {
status: 500,
}),
}))
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
checkActorUsageLimits: mockCheckActorUsageLimits,
}))
vi.mock('@/lib/uploads/contexts/workspace', () => ({
uploadWorkspaceFile: mockUploadWorkspaceFile,
}))
vi.mock('@/lib/uploads/utils/validation', () => ({
validateFileType: mockValidateFileType,
}))
vi.mock('@/lib/knowledge/documents/service', () => ({
createSingleDocument: mockCreateSingleDocument,
getDocuments: vi.fn(),
processDocumentsWithQueue: mockProcessDocumentsWithQueue,
}))
import { POST } from '@/app/api/v1/knowledge/[id]/documents/route'
const routeContext = { params: Promise.resolve({ id: 'kb-1' }) }
function buildFormData(file: File, workspaceId = 'ws-1'): FormData {
const formData = new FormData()
formData.append('workspaceId', workspaceId)
formData.append('file', file)
return formData
}
/**
* Builds a pull-based stream that emits fixed-size chunks on demand, so the
* size-capped reader's `reader.cancel()` simply stops future `pull` calls
* instead of racing an external (e.g. undici FormData) chunk producer.
*/
function makeChunkedOverLimitBody(
chunkBytes: number,
chunkCount: number
): ReadableStream<Uint8Array> {
let emitted = 0
return new ReadableStream<Uint8Array>({
pull(controller) {
if (emitted >= chunkCount) {
controller.close()
return
}
emitted++
controller.enqueue(new Uint8Array(chunkBytes))
},
})
}
describe('v1 knowledge document upload route', () => {
beforeEach(() => {
vi.clearAllMocks()
mockAuthenticateRequest.mockResolvedValue({
requestId: 'req-1',
userId: 'user-1',
rateLimit: {},
})
mockResolveKnowledgeBase.mockResolvedValue({ kb: { id: 'kb-1', workspaceId: 'ws-1' } })
mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false })
mockValidateFileType.mockReturnValue(null)
mockUploadWorkspaceFile.mockResolvedValue({
url: 'https://example.com/file.txt',
})
mockCreateSingleDocument.mockResolvedValue({
id: 'doc-1',
filename: 'file.txt',
fileSize: 100,
mimeType: 'text/plain',
enabled: true,
uploadedAt: new Date('2026-01-01T00:00:00.000Z'),
})
mockProcessDocumentsWithQueue.mockResolvedValue(undefined)
})
it('rejects a declared content-length above the limit before reading the body', async () => {
const formData = buildFormData(new File(['x'.repeat(10)], 'file.txt', { type: 'text/plain' }))
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
method: 'POST',
headers: { 'content-length': String(200 * 1024 * 1024) },
body: formData,
})
const response = await POST(req, routeContext)
const data = await response.json()
expect(response.status).toBe(413)
expect(data.error).toContain('exceeds maximum size')
expect(mockUploadWorkspaceFile).not.toHaveBeenCalled()
})
it('rejects a chunked body without content-length once the streamed size trips the cap', async () => {
const body = makeChunkedOverLimitBody(1024 * 1024, 200)
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
method: 'POST',
body,
// @ts-expect-error - duplex is required by undici for streamed bodies but missing from NextRequestInit types
duplex: 'half',
})
expect(req.headers.get('content-length')).toBeNull()
const response = await POST(req, routeContext)
const data = await response.json()
expect(response.status).toBe(413)
expect(data.error).toContain('exceeds maximum size')
expect(mockUploadWorkspaceFile).not.toHaveBeenCalled()
})
it('uploads a normal, well-under-limit document successfully', async () => {
const file = new File(['hello world'], 'file.txt', { type: 'text/plain' })
const formData = buildFormData(file)
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
method: 'POST',
headers: { 'content-length': '1024' },
body: formData,
})
const response = await POST(req, routeContext)
const data = await response.json()
expect(response.status).toBe(200)
expect(data.success).toBe(true)
expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(1)
expect(mockCreateSingleDocument).toHaveBeenCalledTimes(1)
})
})
@@ -0,0 +1,236 @@
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
import { type NextRequest, NextResponse } from 'next/server'
import {
v1ListKnowledgeDocumentsContract,
v1UploadKnowledgeDocumentContract,
} from '@/lib/api/contracts/v1/knowledge'
import { parseRequest } from '@/lib/api/server'
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
import {
isPayloadSizeLimitError,
MAX_MULTIPART_OVERHEAD_BYTES,
readFormDataWithLimit,
} from '@/lib/core/utils/stream-limits'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import {
createSingleDocument,
type DocumentData,
getDocuments,
processDocumentsWithQueue,
} from '@/lib/knowledge/documents/service'
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace'
import { validateFileType } from '@/lib/uploads/utils/validation'
import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils'
import { authenticateRequest } from '@/app/api/v1/middleware'
export const dynamic = 'force-dynamic'
export const revalidate = 0
const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100MB
interface DocumentsRouteParams {
params: Promise<{ id: string }>
}
/** GET /api/v1/knowledge/[id]/documents — List documents in a knowledge base. */
export const GET = withRouteHandler(async (request: NextRequest, context: DocumentsRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1ListKnowledgeDocumentsContract, request, context)
if (!parsed.success) return parsed.response
const { workspaceId, limit, offset, search, enabledFilter, sortBy, sortOrder } =
parsed.data.query
const { id: knowledgeBaseId } = parsed.data.params
const result = await resolveKnowledgeBase(knowledgeBaseId, workspaceId, userId, rateLimit)
if (result instanceof NextResponse) return result
const documentsResult = await getDocuments(
knowledgeBaseId,
{
enabledFilter: enabledFilter === 'all' ? undefined : enabledFilter,
search,
limit,
offset,
sortBy: sortBy as DocumentSortField,
sortOrder: sortOrder as SortOrder,
},
requestId
)
return NextResponse.json({
success: true,
data: {
documents: documentsResult.documents.map((doc) => ({
id: doc.id,
knowledgeBaseId,
filename: doc.filename,
fileSize: doc.fileSize,
mimeType: doc.mimeType,
processingStatus: doc.processingStatus,
chunkCount: doc.chunkCount,
tokenCount: doc.tokenCount,
characterCount: doc.characterCount,
enabled: doc.enabled,
createdAt: serializeDate(doc.uploadedAt),
})),
pagination: documentsResult.pagination,
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to list documents')
}
})
/** POST /api/v1/knowledge/[id]/documents — Upload a document to a knowledge base. */
export const POST = withRouteHandler(
async (request: NextRequest, context: DocumentsRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1UploadKnowledgeDocumentContract, request, context)
if (!parsed.success) return parsed.response
const { id: knowledgeBaseId } = parsed.data.params
let formData: FormData
try {
formData = await readFormDataWithLimit(request, {
maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES,
label: 'knowledge document upload body',
})
} catch (error) {
if (isPayloadSizeLimitError(error)) {
return NextResponse.json({ error: error.message }, { status: 413 })
}
return NextResponse.json(
{ error: 'Request body must be valid multipart form data' },
{ status: 400 }
)
}
const rawFile = formData.get('file')
const file = rawFile instanceof File ? rawFile : null
const rawWorkspaceId = formData.get('workspaceId')
const workspaceId = typeof rawWorkspaceId === 'string' ? rawWorkspaceId : null
if (!workspaceId) {
return NextResponse.json({ error: 'workspaceId form field is required' }, { status: 400 })
}
if (!file) {
return NextResponse.json({ error: 'file form field is required' }, { status: 400 })
}
if (file.size > MAX_FILE_SIZE) {
return NextResponse.json(
{
error: `File size exceeds 100MB limit (${(file.size / (1024 * 1024)).toFixed(2)}MB)`,
},
{ status: 413 }
)
}
const fileTypeError = validateFileType(file.name, file.type || '')
if (fileTypeError) {
return NextResponse.json({ error: fileTypeError.message }, { status: 415 })
}
const result = await resolveKnowledgeBase(
knowledgeBaseId,
workspaceId,
userId,
rateLimit,
'write'
)
if (result instanceof NextResponse) return result
// Fast usage gate before the storage write + indexing (the async backstop
// in processDocumentAsync still covers non-HTTP paths).
const usage = await checkActorUsageLimits(userId, workspaceId)
if (usage.isExceeded) {
return NextResponse.json(
{
error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.',
},
{ status: 402 }
)
}
const buffer = Buffer.from(await file.arrayBuffer())
const contentType = file.type || 'application/octet-stream'
const uploadedFile = await uploadWorkspaceFile(
workspaceId,
userId,
buffer,
file.name,
contentType
)
const newDocument = await createSingleDocument(
{
filename: file.name,
fileUrl: uploadedFile.url,
fileSize: file.size,
mimeType: contentType,
},
knowledgeBaseId,
requestId,
userId
)
const documentData: DocumentData = {
documentId: newDocument.id,
filename: file.name,
fileUrl: uploadedFile.url,
fileSize: file.size,
mimeType: contentType,
}
processDocumentsWithQueue([documentData], knowledgeBaseId, {}, requestId).catch(() => {
// Processing errors are logged internally
})
recordAudit({
workspaceId,
actorId: userId,
action: AuditAction.DOCUMENT_UPLOADED,
resourceType: AuditResourceType.DOCUMENT,
resourceId: newDocument.id,
resourceName: file.name,
description: `Uploaded document "${file.name}" to knowledge base via API`,
metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType },
request,
})
return NextResponse.json({
success: true,
data: {
document: {
id: newDocument.id,
knowledgeBaseId,
filename: newDocument.filename,
fileSize: newDocument.fileSize,
mimeType: newDocument.mimeType,
processingStatus: 'pending',
chunkCount: 0,
tokenCount: 0,
characterCount: 0,
enabled: newDocument.enabled,
createdAt: serializeDate(newDocument.uploadedAt),
},
message: 'Document uploaded successfully. Processing will begin shortly.',
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to upload document')
}
}
)
+145
View File
@@ -0,0 +1,145 @@
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
import { type NextRequest, NextResponse } from 'next/server'
import {
v1DeleteKnowledgeBaseContract,
v1GetKnowledgeBaseContract,
v1UpdateKnowledgeBaseContract,
} from '@/lib/api/contracts/v1/knowledge'
import { parseRequest } from '@/lib/api/server'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service'
import {
formatKnowledgeBase,
handleError,
resolveKnowledgeBase,
} from '@/app/api/v1/knowledge/utils'
import { authenticateRequest } from '@/app/api/v1/middleware'
export const dynamic = 'force-dynamic'
export const revalidate = 0
interface KnowledgeRouteParams {
params: Promise<{ id: string }>
}
/** GET /api/v1/knowledge/[id] — Get knowledge base details. */
export const GET = withRouteHandler(async (request: NextRequest, context: KnowledgeRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1GetKnowledgeBaseContract, request, context)
if (!parsed.success) return parsed.response
const { id } = parsed.data.params
const result = await resolveKnowledgeBase(id, parsed.data.query.workspaceId, userId, rateLimit)
if (result instanceof NextResponse) return result
return NextResponse.json({
success: true,
data: {
knowledgeBase: formatKnowledgeBase(result.kb),
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to get knowledge base')
}
})
/** PUT /api/v1/knowledge/[id] — Update a knowledge base. */
export const PUT = withRouteHandler(async (request: NextRequest, context: KnowledgeRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1UpdateKnowledgeBaseContract, request, context)
if (!parsed.success) return parsed.response
const { id } = parsed.data.params
const { workspaceId, name, description, chunkingConfig } = parsed.data.body
const result = await resolveKnowledgeBase(id, workspaceId, userId, rateLimit, 'write')
if (result instanceof NextResponse) return result
const updates: {
name?: string
description?: string
chunkingConfig?: { maxSize: number; minSize: number; overlap: number }
} = {}
if (name !== undefined) updates.name = name
if (description !== undefined) updates.description = description
if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig
const updatedKb = await updateKnowledgeBase(id, updates, requestId)
recordAudit({
workspaceId,
actorId: userId,
action: AuditAction.KNOWLEDGE_BASE_UPDATED,
resourceType: AuditResourceType.KNOWLEDGE_BASE,
resourceId: id,
resourceName: updatedKb.name,
description: `Updated knowledge base "${updatedKb.name}" via API`,
metadata: { updatedFields: Object.keys(updates) },
request,
})
return NextResponse.json({
success: true,
data: {
knowledgeBase: formatKnowledgeBase(updatedKb),
message: 'Knowledge base updated successfully',
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to update knowledge base')
}
})
/** DELETE /api/v1/knowledge/[id] — Delete a knowledge base. */
export const DELETE = withRouteHandler(
async (request: NextRequest, context: KnowledgeRouteParams) => {
const auth = await authenticateRequest(request, 'knowledge-detail')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1DeleteKnowledgeBaseContract, request, context)
if (!parsed.success) return parsed.response
const { id } = parsed.data.params
const result = await resolveKnowledgeBase(
id,
parsed.data.query.workspaceId,
userId,
rateLimit,
'write'
)
if (result instanceof NextResponse) return result
await deleteKnowledgeBase(id, requestId)
recordAudit({
workspaceId: parsed.data.query.workspaceId,
actorId: userId,
action: AuditAction.KNOWLEDGE_BASE_DELETED,
resourceType: AuditResourceType.KNOWLEDGE_BASE,
resourceId: id,
resourceName: result.kb.name,
description: `Deleted knowledge base "${result.kb.name}" via API`,
request,
})
return NextResponse.json({
success: true,
data: {
message: 'Knowledge base deleted successfully',
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to delete knowledge base')
}
}
)
+96
View File
@@ -0,0 +1,96 @@
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
import { type NextRequest, NextResponse } from 'next/server'
import {
v1CreateKnowledgeBaseContract,
v1ListKnowledgeBasesContract,
} from '@/lib/api/contracts/v1/knowledge'
import { parseRequest } from '@/lib/api/server'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings'
import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service'
import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils'
import { authenticateRequest, validateWorkspaceAccess } from '@/app/api/v1/middleware'
export const dynamic = 'force-dynamic'
export const revalidate = 0
/** GET /api/v1/knowledge — List knowledge bases in a workspace. */
export const GET = withRouteHandler(async (request: NextRequest) => {
const auth = await authenticateRequest(request, 'knowledge')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1ListKnowledgeBasesContract, request, {})
if (!parsed.success) return parsed.response
const { workspaceId } = parsed.data.query
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId)
if (accessError) return accessError
const knowledgeBases = await getKnowledgeBases(userId, workspaceId)
return NextResponse.json({
success: true,
data: {
knowledgeBases: knowledgeBases.map(formatKnowledgeBase),
totalCount: knowledgeBases.length,
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to list knowledge bases')
}
})
/** POST /api/v1/knowledge — Create a new knowledge base. */
export const POST = withRouteHandler(async (request: NextRequest) => {
const auth = await authenticateRequest(request, 'knowledge')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1CreateKnowledgeBaseContract, request, {})
if (!parsed.success) return parsed.response
const { workspaceId, name, description, chunkingConfig } = parsed.data.body
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, 'write')
if (accessError) return accessError
const kb = await createKnowledgeBase(
{
name,
description,
workspaceId,
userId,
embeddingModel: getConfiguredEmbeddingModel(),
embeddingDimension: EMBEDDING_DIMENSIONS,
chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 },
},
requestId
)
recordAudit({
workspaceId,
actorId: userId,
action: AuditAction.KNOWLEDGE_BASE_CREATED,
resourceType: AuditResourceType.KNOWLEDGE_BASE,
resourceId: kb.id,
resourceName: kb.name,
description: `Created knowledge base "${kb.name}" via API`,
metadata: { chunkingConfig },
request,
})
return NextResponse.json({
success: true,
data: {
knowledgeBase: formatKnowledgeBase(kb),
message: 'Knowledge base created successfully',
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to create knowledge base')
}
})
@@ -0,0 +1,192 @@
/**
* Tests for v1 knowledge search API route.
* Specifically guards the per-KB embedding model resolution and the
* multi-model rejection so the v1 endpoint stays in lockstep with the
* internal route.
*
* @vitest-environment node
*/
import { createMockRequest, knowledgeApiUtilsMock, knowledgeApiUtilsMockFns } from '@sim/testing'
import { getErrorMessage } from '@sim/utils/errors'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockHandleVectorOnlySearch,
mockHandleTagOnlySearch,
mockHandleTagAndVectorSearch,
mockGetQueryStrategy,
mockGenerateSearchEmbedding,
mockGetDocumentMetadataByIds,
mockAuthenticateRequest,
mockValidateWorkspaceAccess,
} = vi.hoisted(() => ({
mockHandleVectorOnlySearch: vi.fn(),
mockHandleTagOnlySearch: vi.fn(),
mockHandleTagAndVectorSearch: vi.fn(),
mockGetQueryStrategy: vi.fn(),
mockGenerateSearchEmbedding: vi.fn(),
mockGetDocumentMetadataByIds: vi.fn(),
mockAuthenticateRequest: vi.fn(),
mockValidateWorkspaceAccess: vi.fn(),
}))
vi.mock('@/app/api/knowledge/search/utils', () => ({
handleVectorOnlySearch: mockHandleVectorOnlySearch,
handleTagOnlySearch: mockHandleTagOnlySearch,
handleTagAndVectorSearch: mockHandleTagAndVectorSearch,
getQueryStrategy: mockGetQueryStrategy,
generateSearchEmbedding: mockGenerateSearchEmbedding,
getDocumentMetadataByIds: mockGetDocumentMetadataByIds,
}))
vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock)
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
checkActorUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }),
}))
vi.mock('@/app/api/v1/middleware', () => ({
authenticateRequest: mockAuthenticateRequest,
validateWorkspaceAccess: mockValidateWorkspaceAccess,
}))
vi.mock('@/app/api/v1/knowledge/utils', () => ({
handleError: (e: unknown) =>
new Response(JSON.stringify({ error: getErrorMessage(e, 'error') }), {
status: 500,
}),
}))
vi.mock('@/lib/knowledge/tags/service', () => ({
getDocumentTagDefinitions: vi.fn().mockResolvedValue([]),
}))
import { POST } from '@/app/api/v1/knowledge/search/route'
const mockCheckKnowledgeBaseAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseAccess
const baseKb = (id: string, embeddingModel: string) => ({
id,
userId: 'user-1',
name: `KB ${id}`,
workspaceId: 'ws-1',
embeddingModel,
deletedAt: null,
})
describe('v1 knowledge search route — per-KB embedding model', () => {
beforeEach(() => {
vi.clearAllMocks()
mockAuthenticateRequest.mockResolvedValue({
requestId: 'req-1',
userId: 'user-1',
rateLimit: {},
})
mockValidateWorkspaceAccess.mockResolvedValue(null)
mockGetQueryStrategy.mockReturnValue({ distanceThreshold: 0.5 })
mockGenerateSearchEmbedding.mockResolvedValue([0.1, 0.2, 0.3])
mockHandleVectorOnlySearch.mockResolvedValue([])
mockGetDocumentMetadataByIds.mockResolvedValue({})
})
it('passes the KB embedding model into generateSearchEmbedding', async () => {
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
hasAccess: true,
knowledgeBase: baseKb('kb-gemini', 'gemini-embedding-001'),
})
const req = createMockRequest('POST', {
workspaceId: 'ws-1',
knowledgeBaseIds: 'kb-gemini',
query: 'hello',
})
const res = await POST(req)
expect(res.status).toBe(200)
expect(mockGenerateSearchEmbedding).toHaveBeenCalledWith(
'hello',
'gemini-embedding-001',
'ws-1'
)
})
it('rejects cross-KB queries with mixed embedding models', async () => {
mockCheckKnowledgeBaseAccess
.mockResolvedValueOnce({
hasAccess: true,
knowledgeBase: baseKb('kb-openai', 'text-embedding-3-small'),
})
.mockResolvedValueOnce({
hasAccess: true,
knowledgeBase: baseKb('kb-gemini', 'gemini-embedding-001'),
})
const req = createMockRequest('POST', {
workspaceId: 'ws-1',
knowledgeBaseIds: ['kb-openai', 'kb-gemini'],
query: 'hello',
})
const res = await POST(req)
expect(res.status).toBe(400)
expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
})
it('surfaces sourceUrl from document metadata in search results', async () => {
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
hasAccess: true,
knowledgeBase: baseKb('kb-confluence', 'text-embedding-3-small'),
})
mockHandleVectorOnlySearch.mockResolvedValue([
{
documentId: 'doc-confluence',
knowledgeBaseId: 'kb-confluence',
content: 'page content',
chunkIndex: 0,
distance: 0.1,
},
])
mockGetDocumentMetadataByIds.mockResolvedValue({
'doc-confluence': {
filename: 'Runbook.md',
sourceUrl: 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345',
},
})
const req = createMockRequest('POST', {
workspaceId: 'ws-1',
knowledgeBaseIds: 'kb-confluence',
query: 'runbook',
})
const res = await POST(req)
const body = await res.json()
expect(res.status).toBe(200)
expect(body.data.results[0].sourceUrl).toBe(
'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345'
)
expect(body.data.results[0].documentName).toBe('Runbook.md')
})
it('allows tag-only search across mixed embedding models', async () => {
mockHandleTagOnlySearch.mockResolvedValue([])
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
hasAccess: true,
knowledgeBase: baseKb('kb-mixed', 'text-embedding-3-small'),
})
const req = createMockRequest('POST', {
workspaceId: 'ws-1',
knowledgeBaseIds: 'kb-mixed',
tagFilters: [{ tagName: 'category', operator: 'eq', value: 'docs' }],
})
const res = await POST(req)
expect(res.status).toBe(400)
// tagName "category" is undefined in our empty getDocumentTagDefinitions mock,
// so the route returns 400 before reaching the search handlers — but crucially
// it never tries to generate an embedding.
expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,279 @@
import { type NextRequest, NextResponse } from 'next/server'
import { v1KnowledgeSearchContract } from '@/lib/api/contracts/v1/knowledge'
import { parseRequest } from '@/lib/api/server'
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants'
import { recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings'
import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils'
import type { StructuredFilter } from '@/lib/knowledge/types'
import {
generateSearchEmbedding,
getDocumentMetadataByIds,
getQueryStrategy,
handleTagAndVectorSearch,
handleTagOnlySearch,
handleVectorOnlySearch,
type SearchResult,
} from '@/app/api/knowledge/search/utils'
import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils'
import { handleError } from '@/app/api/v1/knowledge/utils'
import { authenticateRequest, validateWorkspaceAccess } from '@/app/api/v1/middleware'
export const dynamic = 'force-dynamic'
export const revalidate = 0
/** POST /api/v1/knowledge/search — Vector search across knowledge bases. */
export const POST = withRouteHandler(async (request: NextRequest) => {
const auth = await authenticateRequest(request, 'knowledge-search')
if (auth instanceof NextResponse) return auth
const { requestId, userId, rateLimit } = auth
try {
const parsed = await parseRequest(v1KnowledgeSearchContract, request, {})
if (!parsed.success) return parsed.response
const { workspaceId, topK, query, tagFilters } = parsed.data.body
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId)
if (accessError) return accessError
// A query incurs hosted embedding (+ optional rerank) cost — gate the actor's
// usage and frozen status before spending. Tag-only search is free, so skip it.
if (query && query.trim().length > 0) {
const usage = await checkActorUsageLimits(userId, workspaceId)
if (usage.isExceeded) {
return NextResponse.json(
{ error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' },
{ status: 402 }
)
}
}
const knowledgeBaseIds = Array.isArray(parsed.data.body.knowledgeBaseIds)
? parsed.data.body.knowledgeBaseIds
: [parsed.data.body.knowledgeBaseIds]
const accessChecks = await Promise.all(
knowledgeBaseIds.map((kbId) => checkKnowledgeBaseAccess(kbId, userId))
)
const accessibleKbs = accessChecks
.filter(
(ac): ac is KnowledgeBaseAccessResult =>
ac.hasAccess === true && ac.knowledgeBase.workspaceId === workspaceId
)
.map((ac) => ac.knowledgeBase)
const accessibleKbIds = accessibleKbs.map((kb) => kb.id)
if (accessibleKbIds.length === 0) {
return NextResponse.json(
{ error: 'Knowledge base not found or access denied' },
{ status: 404 }
)
}
const inaccessibleKbIds = knowledgeBaseIds.filter((id) => !accessibleKbIds.includes(id))
if (inaccessibleKbIds.length > 0) {
return NextResponse.json(
{ error: `Knowledge bases not found or access denied: ${inaccessibleKbIds.join(', ')}` },
{ status: 404 }
)
}
let structuredFilters: StructuredFilter[] = []
const tagDefsCache = new Map<string, Awaited<ReturnType<typeof getDocumentTagDefinitions>>>()
if (tagFilters && tagFilters.length > 0 && accessibleKbIds.length > 1) {
return NextResponse.json(
{ error: 'Tag filters are only supported when searching a single knowledge base' },
{ status: 400 }
)
}
if (tagFilters && tagFilters.length > 0 && accessibleKbIds.length > 0) {
const kbId = accessibleKbIds[0]
const tagDefs = await getDocumentTagDefinitions(kbId)
tagDefsCache.set(kbId, tagDefs)
const displayNameToTagDef: Record<string, { tagSlot: string; fieldType: string }> = {}
tagDefs.forEach((def) => {
displayNameToTagDef[def.displayName] = {
tagSlot: def.tagSlot,
fieldType: def.fieldType,
}
})
const undefinedTags: string[] = []
const typeErrors: string[] = []
for (const filter of tagFilters) {
const tagDef = displayNameToTagDef[filter.tagName]
if (!tagDef) {
undefinedTags.push(filter.tagName)
continue
}
const validationError = validateTagValue(
filter.tagName,
String(filter.value),
tagDef.fieldType
)
if (validationError) {
typeErrors.push(validationError)
}
}
if (undefinedTags.length > 0 || typeErrors.length > 0) {
const errorParts: string[] = []
if (undefinedTags.length > 0) {
errorParts.push(buildUndefinedTagsError(undefinedTags))
}
if (typeErrors.length > 0) {
errorParts.push(...typeErrors)
}
return NextResponse.json({ error: errorParts.join('\n') }, { status: 400 })
}
structuredFilters = tagFilters.map((filter) => {
const tagDef = displayNameToTagDef[filter.tagName]!
return {
tagSlot: tagDef.tagSlot,
fieldType: tagDef.fieldType,
operator: filter.operator,
value: filter.value,
valueTo: filter.valueTo,
}
})
}
const hasQuery = query && query.trim().length > 0
const hasFilters = structuredFilters.length > 0
const embeddingModels = Array.from(new Set(accessibleKbs.map((kb) => kb.embeddingModel)))
if (hasQuery && embeddingModels.length > 1) {
return NextResponse.json(
{
error:
'Selected knowledge bases use different embedding models and cannot be searched together. Search them separately.',
},
{ status: 400 }
)
}
const queryEmbeddingModel = embeddingModels[0]
let results: SearchResult[]
let queryEmbeddingIsBYOK: boolean | null = null
if (!hasQuery && hasFilters) {
results = await handleTagOnlySearch({
knowledgeBaseIds: accessibleKbIds,
topK,
structuredFilters,
})
} else if (hasQuery && hasFilters) {
const strategy = getQueryStrategy(accessibleKbIds.length, topK)
const queryEmbeddingResult = await generateSearchEmbedding(
query!,
queryEmbeddingModel,
workspaceId
)
queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK
const queryVector = JSON.stringify(queryEmbeddingResult.embedding)
results = await handleTagAndVectorSearch({
knowledgeBaseIds: accessibleKbIds,
topK,
structuredFilters,
queryVector,
distanceThreshold: strategy.distanceThreshold,
})
} else if (hasQuery) {
const strategy = getQueryStrategy(accessibleKbIds.length, topK)
const queryEmbeddingResult = await generateSearchEmbedding(
query!,
queryEmbeddingModel,
workspaceId
)
queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK
const queryVector = JSON.stringify(queryEmbeddingResult.embedding)
results = await handleVectorOnlySearch({
knowledgeBaseIds: accessibleKbIds,
topK,
queryVector,
distanceThreshold: strategy.distanceThreshold,
})
} else {
return NextResponse.json(
{ error: 'Either query or tagFilters must be provided' },
{ status: 400 }
)
}
if (queryEmbeddingIsBYOK !== null) {
await recordSearchEmbeddingUsage({
userId,
workspaceId,
embeddingModel: queryEmbeddingModel,
query: query!,
isBYOK: queryEmbeddingIsBYOK,
sourceReference: `v1-kb-search:${requestId}`,
})
}
const tagDefsResults = await Promise.all(
accessibleKbIds.map(async (kbId) => {
try {
const tagDefs = tagDefsCache.get(kbId) ?? (await getDocumentTagDefinitions(kbId))
const map: Record<string, string> = {}
tagDefs.forEach((def) => {
map[def.tagSlot] = def.displayName
})
return { kbId, map }
} catch {
return { kbId, map: {} as Record<string, string> }
}
})
)
const tagDefinitionsMap: Record<string, Record<string, string>> = {}
tagDefsResults.forEach(({ kbId, map }) => {
tagDefinitionsMap[kbId] = map
})
const documentIds = results.map((r) => r.documentId)
const documentMetadataMap = await getDocumentMetadataByIds(documentIds)
return NextResponse.json({
success: true,
data: {
results: results.map((result) => {
const kbTagMap = tagDefinitionsMap[result.knowledgeBaseId] || {}
const tags: Record<string, string | number | boolean | Date | null> = {}
ALL_TAG_SLOTS.forEach((slot) => {
const tagValue = result[slot as keyof SearchResult]
if (tagValue !== null && tagValue !== undefined) {
const displayName = kbTagMap[slot] || slot
tags[displayName] = tagValue as string | number | boolean | Date | null
}
})
const docMeta = documentMetadataMap[result.documentId]
return {
documentId: result.documentId,
documentName: docMeta?.filename || undefined,
sourceUrl: docMeta?.sourceUrl ?? null,
content: result.content,
chunkIndex: result.chunkIndex,
metadata: tags,
similarity: hasQuery ? 1 - result.distance : 1,
}
}),
query: query || '',
knowledgeBaseIds: accessibleKbIds,
topK,
totalResults: results.length,
},
})
} catch (error) {
return handleError(requestId, error, 'Failed to perform search')
}
})
+92
View File
@@ -0,0 +1,92 @@
import { createLogger } from '@sim/logger'
import { NextResponse } from 'next/server'
import { validationErrorResponseFromError } from '@/lib/api/server'
import { getKnowledgeBaseById } from '@/lib/knowledge/service'
import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types'
import { type RateLimitResult, validateWorkspaceAccess } from '@/app/api/v1/middleware'
const logger = createLogger('V1KnowledgeAPI')
/**
* Fetches a KB by ID, validates it exists, belongs to the workspace,
* and the user has permission. Returns the KB or a NextResponse error.
*/
export async function resolveKnowledgeBase(
id: string,
workspaceId: string,
userId: string,
rateLimit: RateLimitResult,
level: 'read' | 'write' = 'read'
): Promise<{ kb: KnowledgeBaseWithCounts } | NextResponse> {
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, level)
if (accessError) return accessError
const kb = await getKnowledgeBaseById(id)
if (!kb) {
return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 })
}
if (kb.workspaceId !== workspaceId) {
return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 })
}
return { kb }
}
/**
* Serializes a date value for JSON responses.
*/
export function serializeDate(date: Date | string | null | undefined): string | null {
if (date === null || date === undefined) return null
if (date instanceof Date) return date.toISOString()
return String(date)
}
/**
* Formats a KnowledgeBaseWithCounts into the API response shape.
*/
export function formatKnowledgeBase(kb: KnowledgeBaseWithCounts) {
return {
id: kb.id,
name: kb.name,
description: kb.description,
tokenCount: kb.tokenCount,
embeddingModel: kb.embeddingModel,
embeddingDimension: kb.embeddingDimension,
chunkingConfig: kb.chunkingConfig,
docCount: kb.docCount,
connectorTypes: kb.connectorTypes,
createdAt: serializeDate(kb.createdAt),
updatedAt: serializeDate(kb.updatedAt),
}
}
/**
* Handles unexpected errors with consistent logging and response.
*/
export function handleError(
requestId: string,
error: unknown,
defaultMessage: string
): NextResponse {
const validationResponse = validationErrorResponseFromError(error)
if (validationResponse) return validationResponse
if (error instanceof Error) {
if (error.message.includes('does not have permission')) {
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
}
const isStorageLimitError =
error.message.includes('Storage limit exceeded') || error.message.includes('storage limit')
if (isStorageLimitError) {
return NextResponse.json({ error: 'Storage limit exceeded' }, { status: 413 })
}
const isDuplicate = error.message.includes('already exists')
if (isDuplicate) {
return NextResponse.json({ error: 'Resource already exists' }, { status: 409 })
}
}
logger.error(`[${requestId}] ${defaultMessage}:`, error)
return NextResponse.json({ error: defaultMessage }, { status: 500 })
}