chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { db } from '@sim/db'
|
||||
import { document, knowledgeConnector } from '@sim/db/schema'
|
||||
import { and, eq, isNull } from 'drizzle-orm'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
v1DeleteKnowledgeDocumentContract,
|
||||
v1GetKnowledgeDocumentContract,
|
||||
} from '@/lib/api/contracts/v1/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { deleteDocument } from '@/lib/knowledge/documents/service'
|
||||
import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils'
|
||||
import { authenticateRequest } from '@/app/api/v1/middleware'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const revalidate = 0
|
||||
|
||||
interface DocumentDetailRouteParams {
|
||||
params: Promise<{ id: string; documentId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/v1/knowledge/[id]/documents/[documentId] — Get document details. */
|
||||
export const GET = withRouteHandler(
|
||||
async (request: NextRequest, context: DocumentDetailRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1GetKnowledgeDocumentContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
const { id: knowledgeBaseId, documentId } = parsed.data.params
|
||||
|
||||
const result = await resolveKnowledgeBase(
|
||||
knowledgeBaseId,
|
||||
parsed.data.query.workspaceId,
|
||||
userId,
|
||||
rateLimit
|
||||
)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
const docs = await db
|
||||
.select({
|
||||
id: document.id,
|
||||
knowledgeBaseId: document.knowledgeBaseId,
|
||||
filename: document.filename,
|
||||
fileSize: document.fileSize,
|
||||
mimeType: document.mimeType,
|
||||
processingStatus: document.processingStatus,
|
||||
processingError: document.processingError,
|
||||
processingStartedAt: document.processingStartedAt,
|
||||
processingCompletedAt: document.processingCompletedAt,
|
||||
chunkCount: document.chunkCount,
|
||||
tokenCount: document.tokenCount,
|
||||
characterCount: document.characterCount,
|
||||
enabled: document.enabled,
|
||||
uploadedAt: document.uploadedAt,
|
||||
connectorId: document.connectorId,
|
||||
connectorType: knowledgeConnector.connectorType,
|
||||
sourceUrl: document.sourceUrl,
|
||||
})
|
||||
.from(document)
|
||||
.leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id))
|
||||
.where(
|
||||
and(
|
||||
eq(document.id, documentId),
|
||||
eq(document.knowledgeBaseId, knowledgeBaseId),
|
||||
eq(document.userExcluded, false),
|
||||
isNull(document.archivedAt),
|
||||
isNull(document.deletedAt)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
|
||||
if (docs.length === 0) {
|
||||
return NextResponse.json({ error: 'Document not found' }, { status: 404 })
|
||||
}
|
||||
|
||||
const doc = docs[0]
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
document: {
|
||||
id: doc.id,
|
||||
knowledgeBaseId: doc.knowledgeBaseId,
|
||||
filename: doc.filename,
|
||||
fileSize: doc.fileSize,
|
||||
mimeType: doc.mimeType,
|
||||
processingStatus: doc.processingStatus,
|
||||
processingError: doc.processingError,
|
||||
processingStartedAt: serializeDate(doc.processingStartedAt),
|
||||
processingCompletedAt: serializeDate(doc.processingCompletedAt),
|
||||
chunkCount: doc.chunkCount,
|
||||
tokenCount: doc.tokenCount,
|
||||
characterCount: doc.characterCount,
|
||||
enabled: doc.enabled,
|
||||
connectorId: doc.connectorId,
|
||||
connectorType: doc.connectorType,
|
||||
sourceUrl: doc.sourceUrl,
|
||||
createdAt: serializeDate(doc.uploadedAt),
|
||||
},
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to get document')
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** DELETE /api/v1/knowledge/[id]/documents/[documentId] — Delete a document. */
|
||||
export const DELETE = withRouteHandler(
|
||||
async (request: NextRequest, context: DocumentDetailRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1DeleteKnowledgeDocumentContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
const { id: knowledgeBaseId, documentId } = parsed.data.params
|
||||
|
||||
const result = await resolveKnowledgeBase(
|
||||
knowledgeBaseId,
|
||||
parsed.data.query.workspaceId,
|
||||
userId,
|
||||
rateLimit,
|
||||
'write'
|
||||
)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
const docs = await db
|
||||
.select({ id: document.id, filename: document.filename })
|
||||
.from(document)
|
||||
.where(
|
||||
and(
|
||||
eq(document.id, documentId),
|
||||
eq(document.knowledgeBaseId, knowledgeBaseId),
|
||||
eq(document.userExcluded, false),
|
||||
isNull(document.archivedAt),
|
||||
isNull(document.deletedAt)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
|
||||
if (docs.length === 0) {
|
||||
return NextResponse.json({ error: 'Document not found' }, { status: 404 })
|
||||
}
|
||||
|
||||
await deleteDocument(documentId, requestId)
|
||||
|
||||
recordAudit({
|
||||
workspaceId: parsed.data.query.workspaceId,
|
||||
actorId: userId,
|
||||
action: AuditAction.DOCUMENT_DELETED,
|
||||
resourceType: AuditResourceType.DOCUMENT,
|
||||
resourceId: documentId,
|
||||
resourceName: docs[0].filename,
|
||||
description: `Deleted document "${docs[0].filename}" from knowledge base via API`,
|
||||
metadata: { knowledgeBaseId },
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Document deleted successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to delete document')
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* Tests for the v1 knowledge document upload route's bounded multipart read.
|
||||
*
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockAuthenticateRequest,
|
||||
mockResolveKnowledgeBase,
|
||||
mockCheckActorUsageLimits,
|
||||
mockUploadWorkspaceFile,
|
||||
mockCreateSingleDocument,
|
||||
mockProcessDocumentsWithQueue,
|
||||
mockValidateFileType,
|
||||
} = vi.hoisted(() => ({
|
||||
mockAuthenticateRequest: vi.fn(),
|
||||
mockResolveKnowledgeBase: vi.fn(),
|
||||
mockCheckActorUsageLimits: vi.fn(),
|
||||
mockUploadWorkspaceFile: vi.fn(),
|
||||
mockCreateSingleDocument: vi.fn(),
|
||||
mockProcessDocumentsWithQueue: vi.fn(),
|
||||
mockValidateFileType: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/v1/middleware', () => ({
|
||||
authenticateRequest: mockAuthenticateRequest,
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/v1/knowledge/utils', () => ({
|
||||
resolveKnowledgeBase: mockResolveKnowledgeBase,
|
||||
serializeDate: (date: unknown) => (date instanceof Date ? date.toISOString() : date),
|
||||
handleError: (_requestId: string, error: unknown) =>
|
||||
new Response(JSON.stringify({ error: getErrorMessage(error, 'error') }), {
|
||||
status: 500,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
|
||||
checkActorUsageLimits: mockCheckActorUsageLimits,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace', () => ({
|
||||
uploadWorkspaceFile: mockUploadWorkspaceFile,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/utils/validation', () => ({
|
||||
validateFileType: mockValidateFileType,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/knowledge/documents/service', () => ({
|
||||
createSingleDocument: mockCreateSingleDocument,
|
||||
getDocuments: vi.fn(),
|
||||
processDocumentsWithQueue: mockProcessDocumentsWithQueue,
|
||||
}))
|
||||
|
||||
import { POST } from '@/app/api/v1/knowledge/[id]/documents/route'
|
||||
|
||||
const routeContext = { params: Promise.resolve({ id: 'kb-1' }) }
|
||||
|
||||
function buildFormData(file: File, workspaceId = 'ws-1'): FormData {
|
||||
const formData = new FormData()
|
||||
formData.append('workspaceId', workspaceId)
|
||||
formData.append('file', file)
|
||||
return formData
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a pull-based stream that emits fixed-size chunks on demand, so the
|
||||
* size-capped reader's `reader.cancel()` simply stops future `pull` calls
|
||||
* instead of racing an external (e.g. undici FormData) chunk producer.
|
||||
*/
|
||||
function makeChunkedOverLimitBody(
|
||||
chunkBytes: number,
|
||||
chunkCount: number
|
||||
): ReadableStream<Uint8Array> {
|
||||
let emitted = 0
|
||||
return new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (emitted >= chunkCount) {
|
||||
controller.close()
|
||||
return
|
||||
}
|
||||
emitted++
|
||||
controller.enqueue(new Uint8Array(chunkBytes))
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('v1 knowledge document upload route', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockAuthenticateRequest.mockResolvedValue({
|
||||
requestId: 'req-1',
|
||||
userId: 'user-1',
|
||||
rateLimit: {},
|
||||
})
|
||||
mockResolveKnowledgeBase.mockResolvedValue({ kb: { id: 'kb-1', workspaceId: 'ws-1' } })
|
||||
mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false })
|
||||
mockValidateFileType.mockReturnValue(null)
|
||||
mockUploadWorkspaceFile.mockResolvedValue({
|
||||
url: 'https://example.com/file.txt',
|
||||
})
|
||||
mockCreateSingleDocument.mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
filename: 'file.txt',
|
||||
fileSize: 100,
|
||||
mimeType: 'text/plain',
|
||||
enabled: true,
|
||||
uploadedAt: new Date('2026-01-01T00:00:00.000Z'),
|
||||
})
|
||||
mockProcessDocumentsWithQueue.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('rejects a declared content-length above the limit before reading the body', async () => {
|
||||
const formData = buildFormData(new File(['x'.repeat(10)], 'file.txt', { type: 'text/plain' }))
|
||||
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
|
||||
method: 'POST',
|
||||
headers: { 'content-length': String(200 * 1024 * 1024) },
|
||||
body: formData,
|
||||
})
|
||||
|
||||
const response = await POST(req, routeContext)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(413)
|
||||
expect(data.error).toContain('exceeds maximum size')
|
||||
expect(mockUploadWorkspaceFile).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a chunked body without content-length once the streamed size trips the cap', async () => {
|
||||
const body = makeChunkedOverLimitBody(1024 * 1024, 200)
|
||||
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
|
||||
method: 'POST',
|
||||
body,
|
||||
// @ts-expect-error - duplex is required by undici for streamed bodies but missing from NextRequestInit types
|
||||
duplex: 'half',
|
||||
})
|
||||
expect(req.headers.get('content-length')).toBeNull()
|
||||
|
||||
const response = await POST(req, routeContext)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(413)
|
||||
expect(data.error).toContain('exceeds maximum size')
|
||||
expect(mockUploadWorkspaceFile).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uploads a normal, well-under-limit document successfully', async () => {
|
||||
const file = new File(['hello world'], 'file.txt', { type: 'text/plain' })
|
||||
const formData = buildFormData(file)
|
||||
const req = new NextRequest('http://localhost:3000/api/v1/knowledge/kb-1/documents', {
|
||||
method: 'POST',
|
||||
headers: { 'content-length': '1024' },
|
||||
body: formData,
|
||||
})
|
||||
|
||||
const response = await POST(req, routeContext)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.success).toBe(true)
|
||||
expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(1)
|
||||
expect(mockCreateSingleDocument).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,236 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
v1ListKnowledgeDocumentsContract,
|
||||
v1UploadKnowledgeDocumentContract,
|
||||
} from '@/lib/api/contracts/v1/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
|
||||
import {
|
||||
isPayloadSizeLimitError,
|
||||
MAX_MULTIPART_OVERHEAD_BYTES,
|
||||
readFormDataWithLimit,
|
||||
} from '@/lib/core/utils/stream-limits'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
createSingleDocument,
|
||||
type DocumentData,
|
||||
getDocuments,
|
||||
processDocumentsWithQueue,
|
||||
} from '@/lib/knowledge/documents/service'
|
||||
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
|
||||
import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace'
|
||||
import { validateFileType } from '@/lib/uploads/utils/validation'
|
||||
import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils'
|
||||
import { authenticateRequest } from '@/app/api/v1/middleware'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const revalidate = 0
|
||||
|
||||
const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100MB
|
||||
|
||||
interface DocumentsRouteParams {
|
||||
params: Promise<{ id: string }>
|
||||
}
|
||||
|
||||
/** GET /api/v1/knowledge/[id]/documents — List documents in a knowledge base. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest, context: DocumentsRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1ListKnowledgeDocumentsContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { workspaceId, limit, offset, search, enabledFilter, sortBy, sortOrder } =
|
||||
parsed.data.query
|
||||
const { id: knowledgeBaseId } = parsed.data.params
|
||||
|
||||
const result = await resolveKnowledgeBase(knowledgeBaseId, workspaceId, userId, rateLimit)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
const documentsResult = await getDocuments(
|
||||
knowledgeBaseId,
|
||||
{
|
||||
enabledFilter: enabledFilter === 'all' ? undefined : enabledFilter,
|
||||
search,
|
||||
limit,
|
||||
offset,
|
||||
sortBy: sortBy as DocumentSortField,
|
||||
sortOrder: sortOrder as SortOrder,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
documents: documentsResult.documents.map((doc) => ({
|
||||
id: doc.id,
|
||||
knowledgeBaseId,
|
||||
filename: doc.filename,
|
||||
fileSize: doc.fileSize,
|
||||
mimeType: doc.mimeType,
|
||||
processingStatus: doc.processingStatus,
|
||||
chunkCount: doc.chunkCount,
|
||||
tokenCount: doc.tokenCount,
|
||||
characterCount: doc.characterCount,
|
||||
enabled: doc.enabled,
|
||||
createdAt: serializeDate(doc.uploadedAt),
|
||||
})),
|
||||
pagination: documentsResult.pagination,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to list documents')
|
||||
}
|
||||
})
|
||||
|
||||
/** POST /api/v1/knowledge/[id]/documents — Upload a document to a knowledge base. */
|
||||
export const POST = withRouteHandler(
|
||||
async (request: NextRequest, context: DocumentsRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1UploadKnowledgeDocumentContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
const { id: knowledgeBaseId } = parsed.data.params
|
||||
|
||||
let formData: FormData
|
||||
try {
|
||||
formData = await readFormDataWithLimit(request, {
|
||||
maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES,
|
||||
label: 'knowledge document upload body',
|
||||
})
|
||||
} catch (error) {
|
||||
if (isPayloadSizeLimitError(error)) {
|
||||
return NextResponse.json({ error: error.message }, { status: 413 })
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: 'Request body must be valid multipart form data' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const rawFile = formData.get('file')
|
||||
const file = rawFile instanceof File ? rawFile : null
|
||||
const rawWorkspaceId = formData.get('workspaceId')
|
||||
const workspaceId = typeof rawWorkspaceId === 'string' ? rawWorkspaceId : null
|
||||
|
||||
if (!workspaceId) {
|
||||
return NextResponse.json({ error: 'workspaceId form field is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (!file) {
|
||||
return NextResponse.json({ error: 'file form field is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `File size exceeds 100MB limit (${(file.size / (1024 * 1024)).toFixed(2)}MB)`,
|
||||
},
|
||||
{ status: 413 }
|
||||
)
|
||||
}
|
||||
|
||||
const fileTypeError = validateFileType(file.name, file.type || '')
|
||||
if (fileTypeError) {
|
||||
return NextResponse.json({ error: fileTypeError.message }, { status: 415 })
|
||||
}
|
||||
|
||||
const result = await resolveKnowledgeBase(
|
||||
knowledgeBaseId,
|
||||
workspaceId,
|
||||
userId,
|
||||
rateLimit,
|
||||
'write'
|
||||
)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
// Fast usage gate before the storage write + indexing (the async backstop
|
||||
// in processDocumentAsync still covers non-HTTP paths).
|
||||
const usage = await checkActorUsageLimits(userId, workspaceId)
|
||||
if (usage.isExceeded) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.',
|
||||
},
|
||||
{ status: 402 }
|
||||
)
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await file.arrayBuffer())
|
||||
const contentType = file.type || 'application/octet-stream'
|
||||
|
||||
const uploadedFile = await uploadWorkspaceFile(
|
||||
workspaceId,
|
||||
userId,
|
||||
buffer,
|
||||
file.name,
|
||||
contentType
|
||||
)
|
||||
|
||||
const newDocument = await createSingleDocument(
|
||||
{
|
||||
filename: file.name,
|
||||
fileUrl: uploadedFile.url,
|
||||
fileSize: file.size,
|
||||
mimeType: contentType,
|
||||
},
|
||||
knowledgeBaseId,
|
||||
requestId,
|
||||
userId
|
||||
)
|
||||
|
||||
const documentData: DocumentData = {
|
||||
documentId: newDocument.id,
|
||||
filename: file.name,
|
||||
fileUrl: uploadedFile.url,
|
||||
fileSize: file.size,
|
||||
mimeType: contentType,
|
||||
}
|
||||
|
||||
processDocumentsWithQueue([documentData], knowledgeBaseId, {}, requestId).catch(() => {
|
||||
// Processing errors are logged internally
|
||||
})
|
||||
|
||||
recordAudit({
|
||||
workspaceId,
|
||||
actorId: userId,
|
||||
action: AuditAction.DOCUMENT_UPLOADED,
|
||||
resourceType: AuditResourceType.DOCUMENT,
|
||||
resourceId: newDocument.id,
|
||||
resourceName: file.name,
|
||||
description: `Uploaded document "${file.name}" to knowledge base via API`,
|
||||
metadata: { knowledgeBaseId, fileSize: file.size, mimeType: contentType },
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
document: {
|
||||
id: newDocument.id,
|
||||
knowledgeBaseId,
|
||||
filename: newDocument.filename,
|
||||
fileSize: newDocument.fileSize,
|
||||
mimeType: newDocument.mimeType,
|
||||
processingStatus: 'pending',
|
||||
chunkCount: 0,
|
||||
tokenCount: 0,
|
||||
characterCount: 0,
|
||||
enabled: newDocument.enabled,
|
||||
createdAt: serializeDate(newDocument.uploadedAt),
|
||||
},
|
||||
message: 'Document uploaded successfully. Processing will begin shortly.',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to upload document')
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
v1DeleteKnowledgeBaseContract,
|
||||
v1GetKnowledgeBaseContract,
|
||||
v1UpdateKnowledgeBaseContract,
|
||||
} from '@/lib/api/contracts/v1/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { deleteKnowledgeBase, updateKnowledgeBase } from '@/lib/knowledge/service'
|
||||
import {
|
||||
formatKnowledgeBase,
|
||||
handleError,
|
||||
resolveKnowledgeBase,
|
||||
} from '@/app/api/v1/knowledge/utils'
|
||||
import { authenticateRequest } from '@/app/api/v1/middleware'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const revalidate = 0
|
||||
|
||||
interface KnowledgeRouteParams {
|
||||
params: Promise<{ id: string }>
|
||||
}
|
||||
|
||||
/** GET /api/v1/knowledge/[id] — Get knowledge base details. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest, context: KnowledgeRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1GetKnowledgeBaseContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { id } = parsed.data.params
|
||||
const result = await resolveKnowledgeBase(id, parsed.data.query.workspaceId, userId, rateLimit)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
knowledgeBase: formatKnowledgeBase(result.kb),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to get knowledge base')
|
||||
}
|
||||
})
|
||||
|
||||
/** PUT /api/v1/knowledge/[id] — Update a knowledge base. */
|
||||
export const PUT = withRouteHandler(async (request: NextRequest, context: KnowledgeRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1UpdateKnowledgeBaseContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { id } = parsed.data.params
|
||||
const { workspaceId, name, description, chunkingConfig } = parsed.data.body
|
||||
|
||||
const result = await resolveKnowledgeBase(id, workspaceId, userId, rateLimit, 'write')
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
const updates: {
|
||||
name?: string
|
||||
description?: string
|
||||
chunkingConfig?: { maxSize: number; minSize: number; overlap: number }
|
||||
} = {}
|
||||
if (name !== undefined) updates.name = name
|
||||
if (description !== undefined) updates.description = description
|
||||
if (chunkingConfig !== undefined) updates.chunkingConfig = chunkingConfig
|
||||
|
||||
const updatedKb = await updateKnowledgeBase(id, updates, requestId)
|
||||
|
||||
recordAudit({
|
||||
workspaceId,
|
||||
actorId: userId,
|
||||
action: AuditAction.KNOWLEDGE_BASE_UPDATED,
|
||||
resourceType: AuditResourceType.KNOWLEDGE_BASE,
|
||||
resourceId: id,
|
||||
resourceName: updatedKb.name,
|
||||
description: `Updated knowledge base "${updatedKb.name}" via API`,
|
||||
metadata: { updatedFields: Object.keys(updates) },
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
knowledgeBase: formatKnowledgeBase(updatedKb),
|
||||
message: 'Knowledge base updated successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to update knowledge base')
|
||||
}
|
||||
})
|
||||
|
||||
/** DELETE /api/v1/knowledge/[id] — Delete a knowledge base. */
|
||||
export const DELETE = withRouteHandler(
|
||||
async (request: NextRequest, context: KnowledgeRouteParams) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-detail')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1DeleteKnowledgeBaseContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { id } = parsed.data.params
|
||||
const result = await resolveKnowledgeBase(
|
||||
id,
|
||||
parsed.data.query.workspaceId,
|
||||
userId,
|
||||
rateLimit,
|
||||
'write'
|
||||
)
|
||||
if (result instanceof NextResponse) return result
|
||||
|
||||
await deleteKnowledgeBase(id, requestId)
|
||||
|
||||
recordAudit({
|
||||
workspaceId: parsed.data.query.workspaceId,
|
||||
actorId: userId,
|
||||
action: AuditAction.KNOWLEDGE_BASE_DELETED,
|
||||
resourceType: AuditResourceType.KNOWLEDGE_BASE,
|
||||
resourceId: id,
|
||||
resourceName: result.kb.name,
|
||||
description: `Deleted knowledge base "${result.kb.name}" via API`,
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Knowledge base deleted successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to delete knowledge base')
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
v1CreateKnowledgeBaseContract,
|
||||
v1ListKnowledgeBasesContract,
|
||||
} from '@/lib/api/contracts/v1/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings'
|
||||
import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service'
|
||||
import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils'
|
||||
import { authenticateRequest, validateWorkspaceAccess } from '@/app/api/v1/middleware'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const revalidate = 0
|
||||
|
||||
/** GET /api/v1/knowledge — List knowledge bases in a workspace. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1ListKnowledgeBasesContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { workspaceId } = parsed.data.query
|
||||
|
||||
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId)
|
||||
if (accessError) return accessError
|
||||
|
||||
const knowledgeBases = await getKnowledgeBases(userId, workspaceId)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
knowledgeBases: knowledgeBases.map(formatKnowledgeBase),
|
||||
totalCount: knowledgeBases.length,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to list knowledge bases')
|
||||
}
|
||||
})
|
||||
|
||||
/** POST /api/v1/knowledge — Create a new knowledge base. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1CreateKnowledgeBaseContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { workspaceId, name, description, chunkingConfig } = parsed.data.body
|
||||
|
||||
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, 'write')
|
||||
if (accessError) return accessError
|
||||
|
||||
const kb = await createKnowledgeBase(
|
||||
{
|
||||
name,
|
||||
description,
|
||||
workspaceId,
|
||||
userId,
|
||||
embeddingModel: getConfiguredEmbeddingModel(),
|
||||
embeddingDimension: EMBEDDING_DIMENSIONS,
|
||||
chunkingConfig: chunkingConfig ?? { maxSize: 1024, minSize: 100, overlap: 200 },
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
recordAudit({
|
||||
workspaceId,
|
||||
actorId: userId,
|
||||
action: AuditAction.KNOWLEDGE_BASE_CREATED,
|
||||
resourceType: AuditResourceType.KNOWLEDGE_BASE,
|
||||
resourceId: kb.id,
|
||||
resourceName: kb.name,
|
||||
description: `Created knowledge base "${kb.name}" via API`,
|
||||
metadata: { chunkingConfig },
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
knowledgeBase: formatKnowledgeBase(kb),
|
||||
message: 'Knowledge base created successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to create knowledge base')
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Tests for v1 knowledge search API route.
|
||||
* Specifically guards the per-KB embedding model resolution and the
|
||||
* multi-model rejection so the v1 endpoint stays in lockstep with the
|
||||
* internal route.
|
||||
*
|
||||
* @vitest-environment node
|
||||
*/
|
||||
|
||||
import { createMockRequest, knowledgeApiUtilsMock, knowledgeApiUtilsMockFns } from '@sim/testing'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockHandleVectorOnlySearch,
|
||||
mockHandleTagOnlySearch,
|
||||
mockHandleTagAndVectorSearch,
|
||||
mockGetQueryStrategy,
|
||||
mockGenerateSearchEmbedding,
|
||||
mockGetDocumentMetadataByIds,
|
||||
mockAuthenticateRequest,
|
||||
mockValidateWorkspaceAccess,
|
||||
} = vi.hoisted(() => ({
|
||||
mockHandleVectorOnlySearch: vi.fn(),
|
||||
mockHandleTagOnlySearch: vi.fn(),
|
||||
mockHandleTagAndVectorSearch: vi.fn(),
|
||||
mockGetQueryStrategy: vi.fn(),
|
||||
mockGenerateSearchEmbedding: vi.fn(),
|
||||
mockGetDocumentMetadataByIds: vi.fn(),
|
||||
mockAuthenticateRequest: vi.fn(),
|
||||
mockValidateWorkspaceAccess: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/knowledge/search/utils', () => ({
|
||||
handleVectorOnlySearch: mockHandleVectorOnlySearch,
|
||||
handleTagOnlySearch: mockHandleTagOnlySearch,
|
||||
handleTagAndVectorSearch: mockHandleTagAndVectorSearch,
|
||||
getQueryStrategy: mockGetQueryStrategy,
|
||||
generateSearchEmbedding: mockGenerateSearchEmbedding,
|
||||
getDocumentMetadataByIds: mockGetDocumentMetadataByIds,
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock)
|
||||
|
||||
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
|
||||
checkActorUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/v1/middleware', () => ({
|
||||
authenticateRequest: mockAuthenticateRequest,
|
||||
validateWorkspaceAccess: mockValidateWorkspaceAccess,
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/v1/knowledge/utils', () => ({
|
||||
handleError: (e: unknown) =>
|
||||
new Response(JSON.stringify({ error: getErrorMessage(e, 'error') }), {
|
||||
status: 500,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/knowledge/tags/service', () => ({
|
||||
getDocumentTagDefinitions: vi.fn().mockResolvedValue([]),
|
||||
}))
|
||||
|
||||
import { POST } from '@/app/api/v1/knowledge/search/route'
|
||||
|
||||
const mockCheckKnowledgeBaseAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseAccess
|
||||
|
||||
const baseKb = (id: string, embeddingModel: string) => ({
|
||||
id,
|
||||
userId: 'user-1',
|
||||
name: `KB ${id}`,
|
||||
workspaceId: 'ws-1',
|
||||
embeddingModel,
|
||||
deletedAt: null,
|
||||
})
|
||||
|
||||
describe('v1 knowledge search route — per-KB embedding model', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockAuthenticateRequest.mockResolvedValue({
|
||||
requestId: 'req-1',
|
||||
userId: 'user-1',
|
||||
rateLimit: {},
|
||||
})
|
||||
mockValidateWorkspaceAccess.mockResolvedValue(null)
|
||||
mockGetQueryStrategy.mockReturnValue({ distanceThreshold: 0.5 })
|
||||
mockGenerateSearchEmbedding.mockResolvedValue([0.1, 0.2, 0.3])
|
||||
mockHandleVectorOnlySearch.mockResolvedValue([])
|
||||
mockGetDocumentMetadataByIds.mockResolvedValue({})
|
||||
})
|
||||
|
||||
it('passes the KB embedding model into generateSearchEmbedding', async () => {
|
||||
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
|
||||
hasAccess: true,
|
||||
knowledgeBase: baseKb('kb-gemini', 'gemini-embedding-001'),
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
workspaceId: 'ws-1',
|
||||
knowledgeBaseIds: 'kb-gemini',
|
||||
query: 'hello',
|
||||
})
|
||||
const res = await POST(req)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockGenerateSearchEmbedding).toHaveBeenCalledWith(
|
||||
'hello',
|
||||
'gemini-embedding-001',
|
||||
'ws-1'
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects cross-KB queries with mixed embedding models', async () => {
|
||||
mockCheckKnowledgeBaseAccess
|
||||
.mockResolvedValueOnce({
|
||||
hasAccess: true,
|
||||
knowledgeBase: baseKb('kb-openai', 'text-embedding-3-small'),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
hasAccess: true,
|
||||
knowledgeBase: baseKb('kb-gemini', 'gemini-embedding-001'),
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
workspaceId: 'ws-1',
|
||||
knowledgeBaseIds: ['kb-openai', 'kb-gemini'],
|
||||
query: 'hello',
|
||||
})
|
||||
const res = await POST(req)
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('surfaces sourceUrl from document metadata in search results', async () => {
|
||||
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
|
||||
hasAccess: true,
|
||||
knowledgeBase: baseKb('kb-confluence', 'text-embedding-3-small'),
|
||||
})
|
||||
mockHandleVectorOnlySearch.mockResolvedValue([
|
||||
{
|
||||
documentId: 'doc-confluence',
|
||||
knowledgeBaseId: 'kb-confluence',
|
||||
content: 'page content',
|
||||
chunkIndex: 0,
|
||||
distance: 0.1,
|
||||
},
|
||||
])
|
||||
mockGetDocumentMetadataByIds.mockResolvedValue({
|
||||
'doc-confluence': {
|
||||
filename: 'Runbook.md',
|
||||
sourceUrl: 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345',
|
||||
},
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
workspaceId: 'ws-1',
|
||||
knowledgeBaseIds: 'kb-confluence',
|
||||
query: 'runbook',
|
||||
})
|
||||
const res = await POST(req)
|
||||
const body = await res.json()
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(body.data.results[0].sourceUrl).toBe(
|
||||
'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345'
|
||||
)
|
||||
expect(body.data.results[0].documentName).toBe('Runbook.md')
|
||||
})
|
||||
|
||||
it('allows tag-only search across mixed embedding models', async () => {
|
||||
mockHandleTagOnlySearch.mockResolvedValue([])
|
||||
mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({
|
||||
hasAccess: true,
|
||||
knowledgeBase: baseKb('kb-mixed', 'text-embedding-3-small'),
|
||||
})
|
||||
|
||||
const req = createMockRequest('POST', {
|
||||
workspaceId: 'ws-1',
|
||||
knowledgeBaseIds: 'kb-mixed',
|
||||
tagFilters: [{ tagName: 'category', operator: 'eq', value: 'docs' }],
|
||||
})
|
||||
const res = await POST(req)
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
// tagName "category" is undefined in our empty getDocumentTagDefinitions mock,
|
||||
// so the route returns 400 before reaching the search handlers — but crucially
|
||||
// it never tries to generate an embedding.
|
||||
expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,279 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { v1KnowledgeSearchContract } from '@/lib/api/contracts/v1/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants'
|
||||
import { recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings'
|
||||
import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service'
|
||||
import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils'
|
||||
import type { StructuredFilter } from '@/lib/knowledge/types'
|
||||
import {
|
||||
generateSearchEmbedding,
|
||||
getDocumentMetadataByIds,
|
||||
getQueryStrategy,
|
||||
handleTagAndVectorSearch,
|
||||
handleTagOnlySearch,
|
||||
handleVectorOnlySearch,
|
||||
type SearchResult,
|
||||
} from '@/app/api/knowledge/search/utils'
|
||||
import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils'
|
||||
import { handleError } from '@/app/api/v1/knowledge/utils'
|
||||
import { authenticateRequest, validateWorkspaceAccess } from '@/app/api/v1/middleware'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const revalidate = 0
|
||||
|
||||
/** POST /api/v1/knowledge/search — Vector search across knowledge bases. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const auth = await authenticateRequest(request, 'knowledge-search')
|
||||
if (auth instanceof NextResponse) return auth
|
||||
const { requestId, userId, rateLimit } = auth
|
||||
|
||||
try {
|
||||
const parsed = await parseRequest(v1KnowledgeSearchContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { workspaceId, topK, query, tagFilters } = parsed.data.body
|
||||
|
||||
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId)
|
||||
if (accessError) return accessError
|
||||
|
||||
// A query incurs hosted embedding (+ optional rerank) cost — gate the actor's
|
||||
// usage and frozen status before spending. Tag-only search is free, so skip it.
|
||||
if (query && query.trim().length > 0) {
|
||||
const usage = await checkActorUsageLimits(userId, workspaceId)
|
||||
if (usage.isExceeded) {
|
||||
return NextResponse.json(
|
||||
{ error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' },
|
||||
{ status: 402 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const knowledgeBaseIds = Array.isArray(parsed.data.body.knowledgeBaseIds)
|
||||
? parsed.data.body.knowledgeBaseIds
|
||||
: [parsed.data.body.knowledgeBaseIds]
|
||||
|
||||
const accessChecks = await Promise.all(
|
||||
knowledgeBaseIds.map((kbId) => checkKnowledgeBaseAccess(kbId, userId))
|
||||
)
|
||||
const accessibleKbs = accessChecks
|
||||
.filter(
|
||||
(ac): ac is KnowledgeBaseAccessResult =>
|
||||
ac.hasAccess === true && ac.knowledgeBase.workspaceId === workspaceId
|
||||
)
|
||||
.map((ac) => ac.knowledgeBase)
|
||||
const accessibleKbIds = accessibleKbs.map((kb) => kb.id)
|
||||
|
||||
if (accessibleKbIds.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Knowledge base not found or access denied' },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
|
||||
const inaccessibleKbIds = knowledgeBaseIds.filter((id) => !accessibleKbIds.includes(id))
|
||||
if (inaccessibleKbIds.length > 0) {
|
||||
return NextResponse.json(
|
||||
{ error: `Knowledge bases not found or access denied: ${inaccessibleKbIds.join(', ')}` },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
|
||||
let structuredFilters: StructuredFilter[] = []
|
||||
const tagDefsCache = new Map<string, Awaited<ReturnType<typeof getDocumentTagDefinitions>>>()
|
||||
|
||||
if (tagFilters && tagFilters.length > 0 && accessibleKbIds.length > 1) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Tag filters are only supported when searching a single knowledge base' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
if (tagFilters && tagFilters.length > 0 && accessibleKbIds.length > 0) {
|
||||
const kbId = accessibleKbIds[0]
|
||||
const tagDefs = await getDocumentTagDefinitions(kbId)
|
||||
tagDefsCache.set(kbId, tagDefs)
|
||||
|
||||
const displayNameToTagDef: Record<string, { tagSlot: string; fieldType: string }> = {}
|
||||
tagDefs.forEach((def) => {
|
||||
displayNameToTagDef[def.displayName] = {
|
||||
tagSlot: def.tagSlot,
|
||||
fieldType: def.fieldType,
|
||||
}
|
||||
})
|
||||
|
||||
const undefinedTags: string[] = []
|
||||
const typeErrors: string[] = []
|
||||
|
||||
for (const filter of tagFilters) {
|
||||
const tagDef = displayNameToTagDef[filter.tagName]
|
||||
if (!tagDef) {
|
||||
undefinedTags.push(filter.tagName)
|
||||
continue
|
||||
}
|
||||
const validationError = validateTagValue(
|
||||
filter.tagName,
|
||||
String(filter.value),
|
||||
tagDef.fieldType
|
||||
)
|
||||
if (validationError) {
|
||||
typeErrors.push(validationError)
|
||||
}
|
||||
}
|
||||
|
||||
if (undefinedTags.length > 0 || typeErrors.length > 0) {
|
||||
const errorParts: string[] = []
|
||||
if (undefinedTags.length > 0) {
|
||||
errorParts.push(buildUndefinedTagsError(undefinedTags))
|
||||
}
|
||||
if (typeErrors.length > 0) {
|
||||
errorParts.push(...typeErrors)
|
||||
}
|
||||
return NextResponse.json({ error: errorParts.join('\n') }, { status: 400 })
|
||||
}
|
||||
|
||||
structuredFilters = tagFilters.map((filter) => {
|
||||
const tagDef = displayNameToTagDef[filter.tagName]!
|
||||
return {
|
||||
tagSlot: tagDef.tagSlot,
|
||||
fieldType: tagDef.fieldType,
|
||||
operator: filter.operator,
|
||||
value: filter.value,
|
||||
valueTo: filter.valueTo,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const hasQuery = query && query.trim().length > 0
|
||||
const hasFilters = structuredFilters.length > 0
|
||||
|
||||
const embeddingModels = Array.from(new Set(accessibleKbs.map((kb) => kb.embeddingModel)))
|
||||
if (hasQuery && embeddingModels.length > 1) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
'Selected knowledge bases use different embedding models and cannot be searched together. Search them separately.',
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const queryEmbeddingModel = embeddingModels[0]
|
||||
|
||||
let results: SearchResult[]
|
||||
let queryEmbeddingIsBYOK: boolean | null = null
|
||||
|
||||
if (!hasQuery && hasFilters) {
|
||||
results = await handleTagOnlySearch({
|
||||
knowledgeBaseIds: accessibleKbIds,
|
||||
topK,
|
||||
structuredFilters,
|
||||
})
|
||||
} else if (hasQuery && hasFilters) {
|
||||
const strategy = getQueryStrategy(accessibleKbIds.length, topK)
|
||||
const queryEmbeddingResult = await generateSearchEmbedding(
|
||||
query!,
|
||||
queryEmbeddingModel,
|
||||
workspaceId
|
||||
)
|
||||
queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK
|
||||
const queryVector = JSON.stringify(queryEmbeddingResult.embedding)
|
||||
results = await handleTagAndVectorSearch({
|
||||
knowledgeBaseIds: accessibleKbIds,
|
||||
topK,
|
||||
structuredFilters,
|
||||
queryVector,
|
||||
distanceThreshold: strategy.distanceThreshold,
|
||||
})
|
||||
} else if (hasQuery) {
|
||||
const strategy = getQueryStrategy(accessibleKbIds.length, topK)
|
||||
const queryEmbeddingResult = await generateSearchEmbedding(
|
||||
query!,
|
||||
queryEmbeddingModel,
|
||||
workspaceId
|
||||
)
|
||||
queryEmbeddingIsBYOK = queryEmbeddingResult.isBYOK
|
||||
const queryVector = JSON.stringify(queryEmbeddingResult.embedding)
|
||||
results = await handleVectorOnlySearch({
|
||||
knowledgeBaseIds: accessibleKbIds,
|
||||
topK,
|
||||
queryVector,
|
||||
distanceThreshold: strategy.distanceThreshold,
|
||||
})
|
||||
} else {
|
||||
return NextResponse.json(
|
||||
{ error: 'Either query or tagFilters must be provided' },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
if (queryEmbeddingIsBYOK !== null) {
|
||||
await recordSearchEmbeddingUsage({
|
||||
userId,
|
||||
workspaceId,
|
||||
embeddingModel: queryEmbeddingModel,
|
||||
query: query!,
|
||||
isBYOK: queryEmbeddingIsBYOK,
|
||||
sourceReference: `v1-kb-search:${requestId}`,
|
||||
})
|
||||
}
|
||||
|
||||
const tagDefsResults = await Promise.all(
|
||||
accessibleKbIds.map(async (kbId) => {
|
||||
try {
|
||||
const tagDefs = tagDefsCache.get(kbId) ?? (await getDocumentTagDefinitions(kbId))
|
||||
const map: Record<string, string> = {}
|
||||
tagDefs.forEach((def) => {
|
||||
map[def.tagSlot] = def.displayName
|
||||
})
|
||||
return { kbId, map }
|
||||
} catch {
|
||||
return { kbId, map: {} as Record<string, string> }
|
||||
}
|
||||
})
|
||||
)
|
||||
const tagDefinitionsMap: Record<string, Record<string, string>> = {}
|
||||
tagDefsResults.forEach(({ kbId, map }) => {
|
||||
tagDefinitionsMap[kbId] = map
|
||||
})
|
||||
|
||||
const documentIds = results.map((r) => r.documentId)
|
||||
const documentMetadataMap = await getDocumentMetadataByIds(documentIds)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
results: results.map((result) => {
|
||||
const kbTagMap = tagDefinitionsMap[result.knowledgeBaseId] || {}
|
||||
const tags: Record<string, string | number | boolean | Date | null> = {}
|
||||
|
||||
ALL_TAG_SLOTS.forEach((slot) => {
|
||||
const tagValue = result[slot as keyof SearchResult]
|
||||
if (tagValue !== null && tagValue !== undefined) {
|
||||
const displayName = kbTagMap[slot] || slot
|
||||
tags[displayName] = tagValue as string | number | boolean | Date | null
|
||||
}
|
||||
})
|
||||
|
||||
const docMeta = documentMetadataMap[result.documentId]
|
||||
return {
|
||||
documentId: result.documentId,
|
||||
documentName: docMeta?.filename || undefined,
|
||||
sourceUrl: docMeta?.sourceUrl ?? null,
|
||||
content: result.content,
|
||||
chunkIndex: result.chunkIndex,
|
||||
metadata: tags,
|
||||
similarity: hasQuery ? 1 - result.distance : 1,
|
||||
}
|
||||
}),
|
||||
query: query || '',
|
||||
knowledgeBaseIds: accessibleKbIds,
|
||||
topK,
|
||||
totalResults: results.length,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return handleError(requestId, error, 'Failed to perform search')
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { NextResponse } from 'next/server'
|
||||
import { validationErrorResponseFromError } from '@/lib/api/server'
|
||||
import { getKnowledgeBaseById } from '@/lib/knowledge/service'
|
||||
import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types'
|
||||
import { type RateLimitResult, validateWorkspaceAccess } from '@/app/api/v1/middleware'
|
||||
|
||||
const logger = createLogger('V1KnowledgeAPI')
|
||||
|
||||
/**
|
||||
* Fetches a KB by ID, validates it exists, belongs to the workspace,
|
||||
* and the user has permission. Returns the KB or a NextResponse error.
|
||||
*/
|
||||
export async function resolveKnowledgeBase(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
rateLimit: RateLimitResult,
|
||||
level: 'read' | 'write' = 'read'
|
||||
): Promise<{ kb: KnowledgeBaseWithCounts } | NextResponse> {
|
||||
const accessError = await validateWorkspaceAccess(rateLimit, userId, workspaceId, level)
|
||||
if (accessError) return accessError
|
||||
|
||||
const kb = await getKnowledgeBaseById(id)
|
||||
if (!kb) {
|
||||
return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 })
|
||||
}
|
||||
if (kb.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 })
|
||||
}
|
||||
return { kb }
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes a date value for JSON responses.
|
||||
*/
|
||||
export function serializeDate(date: Date | string | null | undefined): string | null {
|
||||
if (date === null || date === undefined) return null
|
||||
if (date instanceof Date) return date.toISOString()
|
||||
return String(date)
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a KnowledgeBaseWithCounts into the API response shape.
|
||||
*/
|
||||
export function formatKnowledgeBase(kb: KnowledgeBaseWithCounts) {
|
||||
return {
|
||||
id: kb.id,
|
||||
name: kb.name,
|
||||
description: kb.description,
|
||||
tokenCount: kb.tokenCount,
|
||||
embeddingModel: kb.embeddingModel,
|
||||
embeddingDimension: kb.embeddingDimension,
|
||||
chunkingConfig: kb.chunkingConfig,
|
||||
docCount: kb.docCount,
|
||||
connectorTypes: kb.connectorTypes,
|
||||
createdAt: serializeDate(kb.createdAt),
|
||||
updatedAt: serializeDate(kb.updatedAt),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles unexpected errors with consistent logging and response.
|
||||
*/
|
||||
export function handleError(
|
||||
requestId: string,
|
||||
error: unknown,
|
||||
defaultMessage: string
|
||||
): NextResponse {
|
||||
const validationResponse = validationErrorResponseFromError(error)
|
||||
if (validationResponse) return validationResponse
|
||||
|
||||
if (error instanceof Error) {
|
||||
if (error.message.includes('does not have permission')) {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
|
||||
const isStorageLimitError =
|
||||
error.message.includes('Storage limit exceeded') || error.message.includes('storage limit')
|
||||
if (isStorageLimitError) {
|
||||
return NextResponse.json({ error: 'Storage limit exceeded' }, { status: 413 })
|
||||
}
|
||||
|
||||
const isDuplicate = error.message.includes('already exists')
|
||||
if (isDuplicate) {
|
||||
return NextResponse.json({ error: 'Resource already exists' }, { status: 409 })
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] ${defaultMessage}:`, error)
|
||||
return NextResponse.json({ error: defaultMessage }, { status: 500 })
|
||||
}
|
||||
Reference in New Issue
Block a user