chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,174 @@
import { z } from 'zod'
import {
knowledgeBaseParamsSchema,
nullableWireDateSchema,
successResponseSchema,
wireDateSchema,
} from '@/lib/api/contracts/knowledge/shared'
import { defineRouteContract } from '@/lib/api/contracts/types'
import type { StrategyOptions } from '@/lib/chunkers/types'
import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants'
export const knowledgeScopeSchema = z.enum(['active', 'archived', 'all'])
export type KnowledgeScope = z.output<typeof knowledgeScopeSchema>
export const listKnowledgeBasesQuerySchema = z.object({
workspaceId: z.string().min(1).optional(),
scope: knowledgeScopeSchema.default('active'),
})
export const chunkingStrategyOptionsSchema = z
.object({
pattern: z.string().max(500).optional(),
separators: z.array(z.string()).optional(),
recipe: z.enum(['plain', 'markdown', 'code']).optional(),
strictBoundaries: z.boolean().optional(),
})
.strict() satisfies z.ZodType<StrategyOptions>
export const chunkingConfigSchema = z
.object({
maxSize: z.number().min(100).max(4000),
minSize: z.number().min(1).max(2000),
overlap: z.number().min(0).max(500),
strategy: z.enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token']).optional(),
strategyOptions: chunkingStrategyOptionsSchema.optional(),
})
.refine((data) => data.minSize < data.maxSize * 4, {
message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
})
.refine((data) => data.overlap < data.maxSize, {
message: 'Overlap must be less than max chunk size',
})
.refine(
(data) => data.strategy !== 'regex' || typeof data.strategyOptions?.pattern === 'string',
{
message: 'Regex pattern is required when using the regex chunking strategy',
}
)
.refine((data) => data.strategy === 'regex' || data.strategyOptions?.strictBoundaries !== true, {
message: 'strictBoundaries is only valid for the regex chunking strategy',
})
export const createKnowledgeBaseBodySchema = z.object({
name: z.string().min(1, 'Name is required'),
description: z
.string()
.max(
KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH,
`Description must be ${KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH} characters or less`
)
.optional(),
workspaceId: z.string().min(1, 'Workspace ID is required'),
embeddingModel: z.literal('text-embedding-3-small').default('text-embedding-3-small'),
embeddingDimension: z.literal(1536).default(1536),
chunkingConfig: chunkingConfigSchema.default({
maxSize: 1024,
minSize: 100,
overlap: 200,
}),
})
export const updateKnowledgeBaseBodySchema = createKnowledgeBaseBodySchema
.pick({
name: true,
description: true,
})
.partial()
.extend({
chunkingConfig: chunkingConfigSchema.optional(),
workspaceId: z.string().nullable().optional(),
embeddingModel: z.literal('text-embedding-3-small').optional(),
embeddingDimension: z.literal(1536).optional(),
})
const knowledgeChunkingConfigSchema = z
.object({
maxSize: z.number(),
minSize: z.number(),
overlap: z.number(),
strategy: z.enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token']).optional(),
strategyOptions: chunkingStrategyOptionsSchema.optional(),
})
.passthrough()
export const knowledgeBaseDataSchema = z
.object({
id: z.string(),
userId: z.string(),
name: z.string(),
description: z.string().nullable(),
tokenCount: z.number(),
embeddingModel: z.string(),
embeddingDimension: z.number(),
chunkingConfig: knowledgeChunkingConfigSchema,
createdAt: wireDateSchema,
updatedAt: wireDateSchema,
deletedAt: nullableWireDateSchema,
workspaceId: z.string().nullable(),
docCount: z.number().optional(),
connectorTypes: z.array(z.string()).optional(),
})
.passthrough()
export type KnowledgeBaseData = z.output<typeof knowledgeBaseDataSchema>
export const listKnowledgeBasesContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge',
query: listKnowledgeBasesQuerySchema,
response: {
mode: 'json',
schema: successResponseSchema(z.array(knowledgeBaseDataSchema)),
},
})
export const createKnowledgeBaseContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge',
body: createKnowledgeBaseBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(knowledgeBaseDataSchema),
},
})
export const getKnowledgeBaseContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(knowledgeBaseDataSchema),
},
})
export const updateKnowledgeBaseContract = defineRouteContract({
method: 'PUT',
path: '/api/knowledge/[id]',
params: knowledgeBaseParamsSchema,
body: updateKnowledgeBaseBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(knowledgeBaseDataSchema),
},
})
export const deleteKnowledgeBaseContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.object({ message: z.string() })),
},
})
export const restoreKnowledgeBaseContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/restore',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: z.object({ success: z.literal(true) }).passthrough(),
},
})
@@ -0,0 +1,152 @@
import { z } from 'zod'
import {
documentBooleanFieldSchema,
documentDateFieldSchema,
documentNumberFieldSchema,
documentTagFieldSchema,
knowledgeChunkParamsSchema,
knowledgeDocumentParamsSchema,
paginationSchema,
successResponseSchema,
wireDateSchema,
} from '@/lib/api/contracts/knowledge/shared'
import { defineRouteContract } from '@/lib/api/contracts/types'
export const listKnowledgeChunksQuerySchema = z.object({
search: z.string().optional(),
enabled: z.enum(['true', 'false', 'all']).optional().default('all'),
limit: z.coerce.number().min(1).max(100).optional().default(50),
offset: z.coerce.number().min(0).optional().default(0),
sortBy: z.enum(['chunkIndex', 'tokenCount', 'enabled']).optional().default('chunkIndex'),
sortOrder: z.enum(['asc', 'desc']).optional().default('asc'),
})
export const createChunkBodySchema = z.object({
content: z.string().min(1, 'Content is required').max(10000, 'Content too long'),
enabled: z.boolean().optional().default(true),
})
export const updateChunkBodySchema = createChunkBodySchema.partial()
export const bulkChunkOperationBodySchema = z.object({
operation: z.enum(['enable', 'disable', 'delete']),
chunkIds: z.array(z.string()).min(1, 'At least one chunk ID is required').max(100),
})
export const bulkChunkOperationDataSchema = z.object({
operation: z.string(),
successCount: z.number(),
errorCount: z.number(),
processed: z.number(),
errors: z.array(z.string()),
})
export type BulkChunkOperationData = z.output<typeof bulkChunkOperationDataSchema>
export const chunkDataSchema = z
.object({
id: z.string(),
chunkIndex: z.number(),
content: z.string(),
contentLength: z.number(),
tokenCount: z.number(),
enabled: z.boolean(),
startOffset: z.number(),
endOffset: z.number(),
tag1: documentTagFieldSchema,
tag2: documentTagFieldSchema,
tag3: documentTagFieldSchema,
tag4: documentTagFieldSchema,
tag5: documentTagFieldSchema,
tag6: documentTagFieldSchema,
tag7: documentTagFieldSchema,
number1: documentNumberFieldSchema,
number2: documentNumberFieldSchema,
number3: documentNumberFieldSchema,
number4: documentNumberFieldSchema,
number5: documentNumberFieldSchema,
date1: documentDateFieldSchema,
date2: documentDateFieldSchema,
boolean1: documentBooleanFieldSchema,
boolean2: documentBooleanFieldSchema,
boolean3: documentBooleanFieldSchema,
createdAt: wireDateSchema,
updatedAt: wireDateSchema,
})
.passthrough()
export type ChunkData = z.output<typeof chunkDataSchema>
export const chunksPaginationSchema = paginationSchema
export type ChunksPagination = z.output<typeof chunksPaginationSchema>
export type KnowledgeChunksResponse = {
chunks: ChunkData[]
pagination: ChunksPagination
}
export const listKnowledgeChunksContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/documents/[documentId]/chunks',
params: knowledgeDocumentParamsSchema,
query: listKnowledgeChunksQuerySchema,
response: {
mode: 'json',
schema: z.object({
success: z.literal(true),
data: z.array(chunkDataSchema),
pagination: chunksPaginationSchema,
}),
},
})
export const createKnowledgeChunkContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/documents/[documentId]/chunks',
params: knowledgeDocumentParamsSchema,
body: createChunkBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(chunkDataSchema),
},
})
export const getKnowledgeChunkContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]',
params: knowledgeChunkParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(chunkDataSchema),
},
})
export const updateKnowledgeChunkContract = defineRouteContract({
method: 'PUT',
path: '/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]',
params: knowledgeChunkParamsSchema,
body: updateChunkBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(chunkDataSchema),
},
})
export const deleteKnowledgeChunkContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]',
params: knowledgeChunkParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.object({ message: z.string() })),
},
})
export const bulkKnowledgeChunksContract = defineRouteContract({
method: 'PATCH',
path: '/api/knowledge/[id]/documents/[documentId]/chunks',
params: knowledgeDocumentParamsSchema,
body: bulkChunkOperationBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(bulkChunkOperationDataSchema),
},
})
@@ -0,0 +1,194 @@
import { z } from 'zod'
import {
knowledgeBaseParamsSchema,
knowledgeConnectorParamsSchema,
successResponseSchema,
} from '@/lib/api/contracts/knowledge/shared'
import { defineRouteContract } from '@/lib/api/contracts/types'
export const createConnectorBodySchema = z.object({
connectorType: z.string().min(1),
credentialId: z.string().min(1).optional(),
apiKey: z.string().min(1).optional(),
sourceConfig: z.record(z.string(), z.unknown()),
syncIntervalMinutes: z.number().int().min(0).default(1440),
})
export const updateConnectorBodySchema = z.object({
sourceConfig: z.record(z.string(), z.unknown()).optional(),
syncIntervalMinutes: z.number().int().min(0).optional(),
status: z.enum(['active', 'paused']).optional(),
})
export const deleteConnectorQuerySchema = z.object({
deleteDocuments: z.boolean().optional(),
})
export const connectorDocumentsQuerySchema = z.object({
includeExcluded: z.boolean().optional(),
})
export const connectorDocumentsPatchBodySchema = z.object({
operation: z.enum(['restore', 'exclude']),
documentIds: z.array(z.string()).min(1),
})
export const connectorDataSchema = z
.object({
id: z.string(),
knowledgeBaseId: z.string(),
connectorType: z.string(),
credentialId: z.string().nullable(),
sourceConfig: z.record(z.string(), z.unknown()),
syncMode: z.string().nullable(),
syncIntervalMinutes: z.number(),
status: z.enum(['active', 'paused', 'syncing', 'error', 'disabled']),
lastSyncAt: z.string().nullable(),
lastSyncError: z.string().nullable(),
lastSyncDocCount: z.number().nullable(),
nextSyncAt: z.string().nullable(),
consecutiveFailures: z.number(),
createdAt: z.string(),
updatedAt: z.string(),
})
.passthrough()
export type ConnectorData = z.output<typeof connectorDataSchema>
export const syncLogDataSchema = z
.object({
id: z.string(),
connectorId: z.string(),
status: z.string(),
startedAt: z.string(),
completedAt: z.string().nullable(),
docsAdded: z.number(),
docsUpdated: z.number(),
docsDeleted: z.number(),
docsUnchanged: z.number(),
docsFailed: z.number(),
errorMessage: z.string().nullable(),
})
.passthrough()
export type SyncLogData = z.output<typeof syncLogDataSchema>
export const connectorDetailDataSchema = connectorDataSchema.extend({
syncLogs: z.array(syncLogDataSchema),
})
export type ConnectorDetailData = z.output<typeof connectorDetailDataSchema>
export const connectorDocumentDataSchema = z
.object({
id: z.string(),
filename: z.string(),
externalId: z.string().nullable(),
sourceUrl: z.string().nullable(),
enabled: z.boolean(),
deletedAt: z.string().nullable().default(null),
userExcluded: z.boolean(),
uploadedAt: z.string(),
processingStatus: z.string(),
})
.passthrough()
export type ConnectorDocumentData = z.output<typeof connectorDocumentDataSchema>
export const connectorDocumentsDataSchema = z.object({
documents: z.array(connectorDocumentDataSchema),
counts: z.object({ active: z.number(), excluded: z.number() }),
})
export type ConnectorDocumentsData = z.output<typeof connectorDocumentsDataSchema>
export const listKnowledgeConnectorsContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/connectors',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.array(connectorDataSchema)),
},
})
export const createKnowledgeConnectorContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/connectors',
params: knowledgeBaseParamsSchema,
body: createConnectorBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(connectorDataSchema),
},
})
export const getKnowledgeConnectorContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/connectors/[connectorId]',
params: knowledgeConnectorParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(connectorDetailDataSchema),
},
})
export const updateKnowledgeConnectorContract = defineRouteContract({
method: 'PATCH',
path: '/api/knowledge/[id]/connectors/[connectorId]',
params: knowledgeConnectorParamsSchema,
body: updateConnectorBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(connectorDataSchema),
},
})
export const deleteKnowledgeConnectorContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]/connectors/[connectorId]',
params: knowledgeConnectorParamsSchema,
query: deleteConnectorQuerySchema,
response: {
mode: 'json',
schema: z.object({ success: z.literal(true) }),
},
})
export const triggerKnowledgeConnectorSyncContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/connectors/[connectorId]/sync',
params: knowledgeConnectorParamsSchema,
response: {
mode: 'json',
schema: z.object({
success: z.literal(true),
message: z.string(),
}),
},
})
export const listKnowledgeConnectorDocumentsContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/connectors/[connectorId]/documents',
params: knowledgeConnectorParamsSchema,
query: connectorDocumentsQuerySchema,
response: {
mode: 'json',
schema: successResponseSchema(connectorDocumentsDataSchema),
},
})
export const patchKnowledgeConnectorDocumentsContract = defineRouteContract({
method: 'PATCH',
path: '/api/knowledge/[id]/connectors/[connectorId]/documents',
params: knowledgeConnectorParamsSchema,
body: connectorDocumentsPatchBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(
z
.object({
excludedCount: z.number().optional(),
restoredCount: z.number().optional(),
documentIds: z.array(z.string()).optional(),
})
.passthrough()
),
},
})
@@ -0,0 +1,145 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import {
listKnowledgeDocumentsQuerySchema,
parseDocumentTagFiltersParam,
} from '@/lib/api/contracts/knowledge/documents'
describe('listKnowledgeDocumentsQuerySchema.tagFilters', () => {
it('keeps tagFilters a raw string (must NOT transform to an array)', () => {
// A transform-to-array here breaks requestJson outbound serialization
// (the array serializes as "[object Object]"). The wire type must stay a
// string; decoding happens server-side via parseDocumentTagFiltersParam.
const tagFilters = JSON.stringify([
{ tagSlot: 'tag1', fieldType: 'text', operator: 'contains', value: 'x' },
])
const parsed = listKnowledgeDocumentsQuerySchema.parse({ tagFilters })
expect(parsed.tagFilters).toBe(tagFilters)
expect(typeof parsed.tagFilters).toBe('string')
})
})
describe('parseDocumentTagFiltersParam', () => {
it('returns undefined for an absent param', () => {
expect(parseDocumentTagFiltersParam(undefined)).toBeUndefined()
expect(parseDocumentTagFiltersParam('')).toBeUndefined()
})
it('decodes a valid JSON array of filters', () => {
const filters = [
{ tagSlot: 'tag1', fieldType: 'text', operator: 'contains', value: 'x' },
{ tagSlot: 'date1', fieldType: 'date', operator: 'eq', value: '2026-04-21' },
]
expect(parseDocumentTagFiltersParam(JSON.stringify(filters))).toEqual(filters)
})
it('throws on malformed JSON', () => {
expect(() => parseDocumentTagFiltersParam('[object Object]')).toThrow()
expect(() => parseDocumentTagFiltersParam('{not json')).toThrow()
})
it('throws when the shape is wrong', () => {
expect(() => parseDocumentTagFiltersParam(JSON.stringify([{ tagSlot: '' }]))).toThrow()
})
it('rejects an operator that is not valid for the field type', () => {
// unknown operator
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([{ tagSlot: 'tag1', fieldType: 'text', operator: 'bogus', value: 'x' }])
)
).toThrow()
// valid operator name, wrong field type (contains is text-only)
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([
{ tagSlot: 'number1', fieldType: 'number', operator: 'contains', value: '1' },
])
)
).toThrow()
})
it('rejects a fieldType that does not match the tag slot', () => {
// number1 is a numeric column; claiming it is text must fail
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([
{ tagSlot: 'number1', fieldType: 'text', operator: 'contains', value: 'x' },
])
)
).toThrow()
})
it('rejects an unknown tag slot', () => {
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([{ tagSlot: 'tag99', fieldType: 'text', operator: 'eq', value: 'x' }])
)
).toThrow()
})
it('rejects values that are unusable for the field type', () => {
// non-numeric value on a number field
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([{ tagSlot: 'number1', fieldType: 'number', operator: 'eq', value: 'abc' }])
)
).toThrow()
// non-date value on a date field
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([{ tagSlot: 'date1', fieldType: 'date', operator: 'eq', value: 'nope' }])
)
).toThrow()
// well-formed but impossible calendar dates (would 500 on ::date)
for (const value of ['2026-02-30', '2026-99-99', '2026-13-01', '2026-00-10']) {
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([{ tagSlot: 'date1', fieldType: 'date', operator: 'eq', value }])
)
).toThrow()
}
// non-boolean value on a boolean field
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([
{ tagSlot: 'boolean1', fieldType: 'boolean', operator: 'eq', value: 'maybe' },
])
)
).toThrow()
})
it('rejects a between filter missing a usable upper bound', () => {
expect(() =>
parseDocumentTagFiltersParam(
JSON.stringify([
{
tagSlot: 'number1',
fieldType: 'number',
operator: 'between',
value: '1',
valueTo: 'x',
},
])
)
).toThrow()
})
it('accepts a valid number, date, boolean, and between filter', () => {
const filters = [
{ tagSlot: 'number1', fieldType: 'number', operator: 'gte', value: '42' },
{ tagSlot: 'date1', fieldType: 'date', operator: 'eq', value: '2026-04-21' },
{ tagSlot: 'boolean1', fieldType: 'boolean', operator: 'eq', value: 'true' },
{
tagSlot: 'number2',
fieldType: 'number',
operator: 'between',
value: '1',
valueTo: '10',
},
]
expect(parseDocumentTagFiltersParam(JSON.stringify(filters))).toEqual(filters)
})
})
@@ -0,0 +1,381 @@
import { z } from 'zod'
import {
documentBooleanFieldSchema,
documentDateFieldSchema,
documentNumberFieldSchema,
documentTagFieldSchema,
knowledgeBaseParamsSchema,
knowledgeDocumentFileUrlSchema,
knowledgeDocumentParamsSchema,
nullableWireDateSchema,
paginationSchema,
successResponseSchema,
wireDateSchema,
} from '@/lib/api/contracts/knowledge/shared'
import { defineRouteContract } from '@/lib/api/contracts/types'
import { getFieldTypeForSlot } from '@/lib/knowledge/constants'
import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types'
export const documentTagFilterSchema = z
.object({
tagSlot: z.string().min(1),
fieldType: z.enum(['text', 'number', 'date', 'boolean']),
operator: z.string().min(1),
value: z.unknown(),
valueTo: z.unknown().optional(),
})
.superRefine((filter, ctx) => {
// The tag slot determines the column type, so validate against the slot
// (the source of truth) — not just the client-supplied fieldType. Rejecting
// unknown slots, type mismatches, and bad operators at the boundary returns
// a 400 instead of the query builder silently dropping or mis-handling the
// filter (e.g. a text `contains` aimed at a numeric column).
const slotFieldType = getFieldTypeForSlot(filter.tagSlot)
if (slotFieldType === null) {
ctx.addIssue({
code: 'custom',
path: ['tagSlot'],
message: `Unknown tag slot "${filter.tagSlot}"`,
})
return
}
if (slotFieldType !== filter.fieldType) {
ctx.addIssue({
code: 'custom',
path: ['fieldType'],
message: `fieldType "${filter.fieldType}" does not match tag slot "${filter.tagSlot}" (expected "${slotFieldType}")`,
})
return
}
const validOperators = getOperatorsForFieldType(filter.fieldType).map((op) => op.value)
if (!validOperators.includes(filter.operator)) {
ctx.addIssue({
code: 'custom',
path: ['operator'],
message: `Unsupported operator "${filter.operator}" for a ${filter.fieldType} tag filter`,
})
return
}
if (!isValidFilterValue(filter.fieldType, filter.value)) {
ctx.addIssue({
code: 'custom',
path: ['value'],
message: `Invalid value for a ${filter.fieldType} tag filter`,
})
}
// `between` is only valid for number/date (enforced by the operator check
// above), and needs a usable upper bound.
if (filter.operator === 'between' && !isValidFilterValue(filter.fieldType, filter.valueTo)) {
ctx.addIssue({
code: 'custom',
path: ['valueTo'],
message: `Invalid second value for a ${filter.fieldType} "between" tag filter`,
})
}
})
export type DocumentTagFilter = z.output<typeof documentTagFilterSchema>
export const listKnowledgeDocumentsQuerySchema = z.object({
enabledFilter: z.enum(['all', 'enabled', 'disabled']).optional(),
search: z.string().optional(),
limit: z.coerce.number().int().min(1).max(100).optional().default(50),
offset: z.coerce.number().int().min(0).optional().default(0),
sortBy: z
.enum([
'filename',
'fileSize',
'tokenCount',
'chunkCount',
'uploadedAt',
'processingStatus',
'enabled',
])
.optional(),
sortOrder: z.enum(['asc', 'desc']).optional(),
// A query param is a string on the wire, so `tagFilters` is carried as a JSON
// string and decoded by the route via `parseDocumentTagFiltersParam`. It must
// NOT be a `.transform()` to an array here: the client's `requestJson` parses
// the query before serializing it, so a transform would turn the string into
// an array that serializes to `tagFilters=[object Object]` and 400s the route.
tagFilters: z.string().optional(),
})
/**
* Decodes the `tagFilters` query string (a JSON array) into validated filters.
* Throws on malformed JSON or a shape mismatch; callers map that to a 400.
*/
export function parseDocumentTagFiltersParam(
value: string | undefined
): DocumentTagFilter[] | undefined {
if (!value) return undefined
return z.array(documentTagFilterSchema).parse(JSON.parse(value))
}
export const createDocumentBodySchema = z.object({
filename: z.string().min(1, 'Filename is required'),
fileUrl: knowledgeDocumentFileUrlSchema,
fileSize: z.number().min(1, 'File size must be greater than 0'),
mimeType: z.string().min(1, 'MIME type is required'),
tag1: z.string().optional(),
tag2: z.string().optional(),
tag3: z.string().optional(),
tag4: z.string().optional(),
tag5: z.string().optional(),
tag6: z.string().optional(),
tag7: z.string().optional(),
documentTagsData: z.string().optional(),
})
export const bulkCreateDocumentsBodySchema = z.object({
documents: z.array(createDocumentBodySchema),
processingOptions: z
.object({
recipe: z.string().optional(),
lang: z.string().optional(),
})
.optional(),
bulk: z.literal(true),
workflowId: z.string().optional(),
})
const singleCreateDocumentBodySchema = createDocumentBodySchema.extend({
bulk: z.literal(false),
workflowId: z.string().optional(),
})
const createKnowledgeDocumentsBodyDiscriminatedUnion = z.discriminatedUnion('bulk', [
bulkCreateDocumentsBodySchema,
singleCreateDocumentBodySchema,
])
export const createKnowledgeDocumentsBodySchema = z
.object({ bulk: z.boolean().default(false) })
.passthrough()
.pipe(createKnowledgeDocumentsBodyDiscriminatedUnion)
export type CreateKnowledgeDocumentsBody = z.input<typeof createKnowledgeDocumentsBodySchema>
export type BulkCreateDocumentsBody = z.input<typeof bulkCreateDocumentsBodySchema>
export type SingleCreateDocumentBody = z.input<typeof singleCreateDocumentBodySchema>
export const upsertDocumentBodySchema = z.object({
documentId: z.string().optional(),
filename: z.string().min(1, 'Filename is required'),
fileUrl: knowledgeDocumentFileUrlSchema,
fileSize: z.number().min(1, 'File size must be greater than 0'),
mimeType: z.string().min(1, 'MIME type is required'),
documentTagsData: z.string().optional(),
processingOptions: z
.object({
recipe: z.string().optional(),
lang: z.string().optional(),
})
.optional(),
workflowId: z.string().optional(),
})
export type UpsertDocumentBody = z.output<typeof upsertDocumentBodySchema>
export const bulkCreateDocumentsResponseSchema = z.object({
total: z.number(),
documentsCreated: z.array(
z.object({
documentId: z.string(),
filename: z.string(),
status: z.string(),
})
),
processingMethod: z.string(),
processingConfig: z
.object({
maxConcurrentDocuments: z.number(),
batchSize: z.number(),
totalBatches: z.number(),
})
.passthrough(),
})
export const updateDocumentBodySchema = z.object({
filename: z.string().min(1, 'Filename is required').optional(),
enabled: z.boolean().optional(),
chunkCount: z.number().min(0).optional(),
tokenCount: z.number().min(0).optional(),
characterCount: z.number().min(0).optional(),
processingStatus: z.enum(['pending', 'processing', 'completed', 'failed']).optional(),
processingError: z.string().optional(),
markFailedDueToTimeout: z.boolean().optional(),
retryProcessing: z.boolean().optional(),
tag1: z.string().optional(),
tag2: z.string().optional(),
tag3: z.string().optional(),
tag4: z.string().optional(),
tag5: z.string().optional(),
tag6: z.string().optional(),
tag7: z.string().optional(),
number1: z.string().optional(),
number2: z.string().optional(),
number3: z.string().optional(),
number4: z.string().optional(),
number5: z.string().optional(),
date1: z.string().optional(),
date2: z.string().optional(),
boolean1: z.string().optional(),
boolean2: z.string().optional(),
boolean3: z.string().optional(),
})
export const updateDocumentTagsBodySchema = z.record(z.string(), z.string())
export const bulkDocumentOperationBodySchema = z
.object({
operation: z.enum(['enable', 'disable', 'delete']),
documentIds: z.array(z.string()).min(1).max(100).optional(),
selectAll: z.boolean().optional(),
enabledFilter: z.enum(['all', 'enabled', 'disabled']).optional(),
})
.refine((data) => data.selectAll || (data.documentIds && data.documentIds.length > 0), {
message: 'Either selectAll must be true or documentIds must be provided',
})
export const bulkDocumentOperationDataSchema = z.object({
operation: z.string().optional(),
successCount: z.number(),
failedCount: z.number().optional(),
updatedDocuments: z
.array(z.object({ id: z.string(), enabled: z.boolean().optional() }))
.optional(),
})
export type BulkDocumentOperationData = z.output<typeof bulkDocumentOperationDataSchema>
export const documentDataSchema = z
.object({
id: z.string(),
knowledgeBaseId: z.string(),
filename: z.string(),
fileUrl: z.string(),
fileSize: z.number(),
mimeType: z.string(),
chunkCount: z.number(),
tokenCount: z.number(),
characterCount: z.number(),
processingStatus: z.enum(['pending', 'processing', 'completed', 'failed']),
processingStartedAt: nullableWireDateSchema.optional(),
processingCompletedAt: nullableWireDateSchema.optional(),
processingError: z.string().nullable().optional(),
enabled: z.boolean(),
uploadedAt: wireDateSchema,
tag1: documentTagFieldSchema,
tag2: documentTagFieldSchema,
tag3: documentTagFieldSchema,
tag4: documentTagFieldSchema,
tag5: documentTagFieldSchema,
tag6: documentTagFieldSchema,
tag7: documentTagFieldSchema,
number1: documentNumberFieldSchema,
number2: documentNumberFieldSchema,
number3: documentNumberFieldSchema,
number4: documentNumberFieldSchema,
number5: documentNumberFieldSchema,
date1: documentDateFieldSchema,
date2: documentDateFieldSchema,
boolean1: documentBooleanFieldSchema,
boolean2: documentBooleanFieldSchema,
boolean3: documentBooleanFieldSchema,
connectorId: z.string().nullable().optional(),
connectorType: z.string().nullable().optional(),
sourceUrl: z.string().nullable().optional(),
})
.passthrough()
export type DocumentData = z.output<typeof documentDataSchema>
export const documentsPaginationSchema = paginationSchema
export type DocumentsPagination = z.output<typeof documentsPaginationSchema>
export const knowledgeDocumentsDataSchema = z.object({
documents: z.array(documentDataSchema),
pagination: documentsPaginationSchema,
})
export type KnowledgeDocumentsResponse = z.output<typeof knowledgeDocumentsDataSchema>
export const getKnowledgeDocumentContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/documents/[documentId]',
params: knowledgeDocumentParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(documentDataSchema),
},
})
export const listKnowledgeDocumentsContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/documents',
params: knowledgeBaseParamsSchema,
query: listKnowledgeDocumentsQuerySchema,
response: {
mode: 'json',
schema: successResponseSchema(knowledgeDocumentsDataSchema),
},
})
export const createKnowledgeDocumentsContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/documents',
params: knowledgeBaseParamsSchema,
body: createKnowledgeDocumentsBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(z.union([bulkCreateDocumentsResponseSchema, documentDataSchema])),
},
})
export const updateKnowledgeDocumentContract = defineRouteContract({
method: 'PUT',
path: '/api/knowledge/[id]/documents/[documentId]',
params: knowledgeDocumentParamsSchema,
body: updateDocumentBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(documentDataSchema),
},
})
export const updateKnowledgeDocumentTagsContract = defineRouteContract({
method: 'PUT',
path: '/api/knowledge/[id]/documents/[documentId]',
params: knowledgeDocumentParamsSchema,
body: updateDocumentTagsBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(documentDataSchema),
},
})
export const deleteKnowledgeDocumentContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]/documents/[documentId]',
params: knowledgeDocumentParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.unknown()),
},
})
export const bulkKnowledgeDocumentsContract = defineRouteContract({
method: 'PATCH',
path: '/api/knowledge/[id]/documents',
params: knowledgeBaseParamsSchema,
body: bulkDocumentOperationBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(bulkDocumentOperationDataSchema),
},
})
export const upsertKnowledgeDocumentContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/documents/upsert',
params: knowledgeBaseParamsSchema,
body: upsertDocumentBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(documentDataSchema),
},
})
@@ -0,0 +1,6 @@
export * from '@/lib/api/contracts/knowledge/base'
export * from '@/lib/api/contracts/knowledge/chunks'
export * from '@/lib/api/contracts/knowledge/connectors'
export * from '@/lib/api/contracts/knowledge/documents'
export * from '@/lib/api/contracts/knowledge/search'
export * from '@/lib/api/contracts/knowledge/tags'
@@ -0,0 +1,68 @@
import { z } from 'zod'
import { DEFAULT_RERANKER_MODEL, rerankerModelSchema } from '@/lib/knowledge/reranker-models'
export const knowledgeSearchTagFilterSchema = z.object({
tagName: z.string(),
tagSlot: z.string().optional(),
fieldType: z.enum(['text', 'number', 'date', 'boolean']).optional(),
operator: z.string().default('eq'),
value: z.union([z.string(), z.number(), z.boolean()]),
valueTo: z.union([z.string(), z.number()]).optional(),
})
export const knowledgeSearchBodySchema = z
.object({
knowledgeBaseIds: z.union([
z.string().min(1, 'Knowledge base ID is required'),
z.array(z.string().min(1)).min(1, 'At least one knowledge base ID is required'),
]),
query: z
.string()
.optional()
.nullable()
.transform((val) => val || undefined),
topK: z
.number()
.min(1)
.max(100)
.optional()
.nullable()
.default(10)
.transform((val) => val ?? 10),
tagFilters: z
.array(knowledgeSearchTagFilterSchema)
.optional()
.nullable()
.transform((val) => val || undefined),
rerankerEnabled: z.boolean().optional().default(false),
rerankerModel: rerankerModelSchema.optional().default(DEFAULT_RERANKER_MODEL),
/**
* Number of vector results sent to Cohere as the documents array for reranking. Capped at 100
* so each rerank call stays within a single Cohere search unit (1 query × ≤100 docs); see
* `RERANK_MODEL_PRICING` in `providers/models.ts`.
*/
rerankerInputCount: z
.number()
.int('rerankerInputCount must be an integer')
.min(1, 'rerankerInputCount must be at least 1')
.max(100, 'rerankerInputCount cannot exceed 100')
.optional()
.nullable()
.transform((val) => val ?? undefined),
rerankerApiKey: z
.string()
.optional()
.nullable()
.transform((val) => val || undefined),
})
.refine(
(data) => {
const hasQuery = data.query && data.query.trim().length > 0
const hasTagFilters = data.tagFilters && data.tagFilters.length > 0
return hasQuery || hasTagFilters
},
{
message: 'Please provide either a search query or tag filters to search your knowledge base',
}
)
export type KnowledgeSearchBody = z.output<typeof knowledgeSearchBodySchema>
@@ -0,0 +1,57 @@
/**
* Tests for shared knowledge contract schemas
*
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { knowledgeDocumentFileUrlSchema } from '@/lib/api/contracts/knowledge/shared'
describe('knowledgeDocumentFileUrlSchema', () => {
it('accepts data: URIs', () => {
const result = knowledgeDocumentFileUrlSchema.safeParse(
'data:text/plain;base64,SGVsbG8gd29ybGQ='
)
expect(result.success).toBe(true)
})
it('accepts https URLs', () => {
const result = knowledgeDocumentFileUrlSchema.safeParse('https://example.com/file.pdf')
expect(result.success).toBe(true)
})
it('accepts http URLs', () => {
const result = knowledgeDocumentFileUrlSchema.safeParse(
'http://localhost:3000/api/files/serve/kb/foo.pdf?context=knowledge-base'
)
expect(result.success).toBe(true)
})
it('is case-insensitive on the scheme', () => {
expect(knowledgeDocumentFileUrlSchema.safeParse('HTTPS://example.com/x').success).toBe(true)
expect(knowledgeDocumentFileUrlSchema.safeParse('Http://example.com/x').success).toBe(true)
})
it.each([
['absolute local path', '/etc/passwd'],
['app path', '/app/.env'],
['relative path', './secrets.txt'],
['parent traversal', '../../etc/shadow'],
['file:// scheme', 'file:///etc/passwd'],
['ftp scheme', 'ftp://example.com/x'],
['javascript scheme', 'javascript:alert(1)'],
['gopher scheme', 'gopher://example.com'],
['relative serve path', '/api/files/serve/kb/foo.pdf'],
['windows path', 'C:\\Windows\\System32\\config\\SAM'],
['empty string', ''],
['whitespace prefix', ' https://example.com/x'],
])('rejects %s', (_label, value) => {
const result = knowledgeDocumentFileUrlSchema.safeParse(value)
expect(result.success).toBe(false)
})
it('returns a useful error message for unsupported schemes', () => {
const result = knowledgeDocumentFileUrlSchema.safeParse('/etc/passwd')
if (result.success) throw new Error('expected failure')
expect(result.error.issues[0].message).toMatch(/data: URI or an http\(s\):\/\/ URL/)
})
})
@@ -0,0 +1,59 @@
import { z } from 'zod'
export const wireDateSchema = z.string()
export const nullableWireDateSchema = z.string().nullable()
export const knowledgeBaseParamsSchema = z.object({
id: z.string().min(1),
})
export const knowledgeDocumentParamsSchema = knowledgeBaseParamsSchema.extend({
documentId: z.string().min(1),
})
export const knowledgeChunkParamsSchema = knowledgeDocumentParamsSchema.extend({
chunkId: z.string().min(1),
})
export const knowledgeTagParamsSchema = knowledgeBaseParamsSchema.extend({
tagId: z.string().min(1),
})
export const knowledgeConnectorParamsSchema = knowledgeBaseParamsSchema.extend({
connectorId: z.string().min(1),
})
/**
* A `fileUrl` accepted by knowledge document ingestion endpoints.
*
* Must be a `data:` URI or an `http(s)://` URL. Local paths, `file://`,
* and other schemes are rejected at the boundary to prevent the background
* parser from reading arbitrary files off the Sim server's filesystem.
*/
export const knowledgeDocumentFileUrlSchema = z
.string()
.min(1, 'File URL is required')
.refine(
(value) => /^data:/i.test(value) || /^https?:\/\//i.test(value),
'File URL must be a data: URI or an http(s):// URL'
)
export const documentTagFieldSchema = z.string().nullable().optional()
export const documentNumberFieldSchema = z.number().nullable().optional()
export const documentBooleanFieldSchema = z.boolean().nullable().optional()
export const documentDateFieldSchema = nullableWireDateSchema.optional()
export const paginationSchema = z
.object({
total: z.number(),
limit: z.number(),
offset: z.number(),
hasMore: z.boolean(),
})
.passthrough()
export const successResponseSchema = <T extends z.ZodType>(dataSchema: T) =>
z.object({
success: z.literal(true),
data: dataSchema,
})
@@ -0,0 +1,161 @@
import { z } from 'zod'
import {
knowledgeBaseParamsSchema,
knowledgeDocumentParamsSchema,
knowledgeTagParamsSchema,
successResponseSchema,
} from '@/lib/api/contracts/knowledge/shared'
import { defineRouteContract } from '@/lib/api/contracts/types'
export const nextAvailableSlotQuerySchema = z.object({
fieldType: z.string().min(1),
})
export const createTagDefinitionBodySchema = z.object({
tagSlot: z.string().min(1, 'Tag slot is required'),
displayName: z.string().min(1, 'Display name is required'),
fieldType: z.string().min(1, 'Invalid field type'),
})
export const documentTagDefinitionInputSchema = z.object({
tagSlot: z.string().min(1, 'Tag slot is required'),
displayName: z.string().min(1, 'Display name is required').max(100, 'Display name too long'),
fieldType: z.string().default('text'),
_originalDisplayName: z.string().optional(),
})
export const saveDocumentTagDefinitionsBodySchema = z.object({
definitions: z.array(documentTagDefinitionInputSchema),
})
export const deleteDocumentTagDefinitionsQuerySchema = z.object({
action: z.enum(['cleanup', 'all']).optional(),
})
export const tagDefinitionDataSchema = z.object({
id: z.string(),
tagSlot: z.string(),
displayName: z.string(),
fieldType: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
})
export type TagDefinitionData = z.output<typeof tagDefinitionDataSchema>
export type DocumentTagDefinitionData = TagDefinitionData
export const nextAvailableSlotDataSchema = z.object({
nextAvailableSlot: z.string().nullable(),
fieldType: z.string(),
usedSlots: z.array(z.string()),
totalSlots: z.number(),
availableSlots: z.number(),
})
export type NextAvailableSlotData = z.output<typeof nextAvailableSlotDataSchema>
export const saveDocumentTagDefinitionsDataSchema = z
.object({
created: z.array(tagDefinitionDataSchema).optional(),
updated: z.array(tagDefinitionDataSchema).optional(),
errors: z.array(z.string()).optional(),
})
.or(z.array(tagDefinitionDataSchema))
export type SaveDocumentTagDefinitionsResult = z.output<typeof saveDocumentTagDefinitionsDataSchema>
export const tagUsageDocumentSchema = z.object({
id: z.string(),
name: z.string(),
tagValue: z.string(),
})
export const tagUsageDataSchema = z.object({
tagName: z.string(),
tagSlot: z.string(),
documentCount: z.number(),
documents: z.array(tagUsageDocumentSchema),
})
export type TagUsageData = z.output<typeof tagUsageDataSchema>
export const listTagDefinitionsContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/tag-definitions',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.array(tagDefinitionDataSchema)),
},
})
export const createTagDefinitionContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/tag-definitions',
params: knowledgeBaseParamsSchema,
body: createTagDefinitionBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(tagDefinitionDataSchema),
},
})
export const deleteTagDefinitionContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]/tag-definitions/[tagId]',
params: knowledgeTagParamsSchema,
response: {
mode: 'json',
schema: z.object({ success: z.literal(true) }).passthrough(),
},
})
export const nextAvailableSlotContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/next-available-slot',
params: knowledgeBaseParamsSchema,
query: nextAvailableSlotQuerySchema,
response: {
mode: 'json',
schema: successResponseSchema(nextAvailableSlotDataSchema),
},
})
export const listDocumentTagDefinitionsContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/documents/[documentId]/tag-definitions',
params: knowledgeDocumentParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.array(tagDefinitionDataSchema)),
},
})
export const saveDocumentTagDefinitionsContract = defineRouteContract({
method: 'POST',
path: '/api/knowledge/[id]/documents/[documentId]/tag-definitions',
params: knowledgeDocumentParamsSchema,
body: saveDocumentTagDefinitionsBodySchema,
response: {
mode: 'json',
schema: successResponseSchema(saveDocumentTagDefinitionsDataSchema),
},
})
export const deleteDocumentTagDefinitionsContract = defineRouteContract({
method: 'DELETE',
path: '/api/knowledge/[id]/documents/[documentId]/tag-definitions',
params: knowledgeDocumentParamsSchema,
response: {
mode: 'json',
schema: z.object({ success: z.literal(true) }).passthrough(),
},
})
export const getTagUsageContract = defineRouteContract({
method: 'GET',
path: '/api/knowledge/[id]/tag-usage',
params: knowledgeBaseParamsSchema,
response: {
mode: 'json',
schema: successResponseSchema(z.array(tagUsageDataSchema)),
},
})