chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { cancelTableRunsContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns'
|
||||
import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableCancelRunsAPI')
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/table/[tableId]/cancel-runs
|
||||
*
|
||||
* Cancels in-flight and pending workflow-column runs for this table. Scopes:
|
||||
* `all` (every cell) or `row` (every cell for `rowId`).
|
||||
*/
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(cancelTableRunsContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, scope, rowId, filter, excludeRowIds } = parsed.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const filterError = tableFilterError(filter, table.schema.columns)
|
||||
if (filterError) return filterError
|
||||
|
||||
const cancelled = await cancelWorkflowGroupRuns(tableId, scope === 'row' ? rowId : undefined, {
|
||||
filter,
|
||||
excludeRowIds,
|
||||
})
|
||||
logger.info(
|
||||
`[${requestId}] cancel-runs: tableId=${tableId} scope=${scope}${
|
||||
rowId ? ` rowId=${rowId}` : ''
|
||||
} cancelled=${cancelled}`
|
||||
)
|
||||
|
||||
return NextResponse.json({ success: true, data: { cancelled } })
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] cancel-runs failed:`, error)
|
||||
return NextResponse.json({ error: 'Failed to cancel runs' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,229 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
addTableColumnContract,
|
||||
deleteTableColumnContract,
|
||||
updateTableColumnContract,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { isZodError, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
addTableColumn,
|
||||
deleteColumn,
|
||||
renameColumn,
|
||||
updateColumnConstraints,
|
||||
updateColumnType,
|
||||
} from '@/lib/table'
|
||||
import { accessError, checkAccess, normalizeColumn, rootErrorMessage } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableColumnsAPI')
|
||||
|
||||
interface ColumnsRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** POST /api/table/[tableId]/columns - Adds a column to the table schema. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest, context: ColumnsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await context.params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized column creation attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const validation = await parseRequest(addTableColumnContract, request, context)
|
||||
if (!validation.success) return validation.response
|
||||
const validated = validation.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const updatedTable = await addTableColumn(tableId, validated.column, requestId)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error, 'Invalid request data')
|
||||
}
|
||||
|
||||
const msg = rootErrorMessage(error)
|
||||
if (
|
||||
msg.includes('already exists') ||
|
||||
msg.includes('maximum column') ||
|
||||
msg.includes('Invalid column') ||
|
||||
msg.includes('exceeds maximum')
|
||||
) {
|
||||
return NextResponse.json({ error: msg }, { status: 400 })
|
||||
}
|
||||
if (msg === 'Table not found') {
|
||||
return NextResponse.json({ error: msg }, { status: 404 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error adding column to table ${tableId}:`, error)
|
||||
return NextResponse.json({ error: 'Failed to add column' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** PATCH /api/table/[tableId]/columns - Updates a column (rename, type change, constraints). */
|
||||
export const PATCH = withRouteHandler(async (request: NextRequest, context: ColumnsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await context.params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized column update attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const validation = await parseRequest(updateTableColumnContract, request, context)
|
||||
if (!validation.success) return validation.response
|
||||
const validated = validation.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const { updates } = validated
|
||||
let updatedTable = null
|
||||
|
||||
if (updates.name) {
|
||||
updatedTable = await renameColumn(
|
||||
{ tableId, oldName: validated.columnName, newName: updates.name },
|
||||
requestId
|
||||
)
|
||||
}
|
||||
|
||||
if (updates.type) {
|
||||
updatedTable = await updateColumnType(
|
||||
{ tableId, columnName: updates.name ?? validated.columnName, newType: updates.type },
|
||||
requestId
|
||||
)
|
||||
}
|
||||
|
||||
if (updates.required !== undefined || updates.unique !== undefined) {
|
||||
updatedTable = await updateColumnConstraints(
|
||||
{
|
||||
tableId,
|
||||
columnName: updates.name ?? validated.columnName,
|
||||
...(updates.required !== undefined ? { required: updates.required } : {}),
|
||||
...(updates.unique !== undefined ? { unique: updates.unique } : {}),
|
||||
},
|
||||
requestId
|
||||
)
|
||||
}
|
||||
|
||||
if (!updatedTable) {
|
||||
return NextResponse.json({ error: 'No updates specified' }, { status: 400 })
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error, 'Invalid request data')
|
||||
}
|
||||
|
||||
const msg = rootErrorMessage(error)
|
||||
if (msg.includes('not found') || msg.includes('Table not found')) {
|
||||
return NextResponse.json({ error: msg }, { status: 404 })
|
||||
}
|
||||
if (
|
||||
msg.includes('already exists') ||
|
||||
msg.includes('Cannot delete the last column') ||
|
||||
msg.includes('Cannot set column') ||
|
||||
msg.includes('Cannot set unique column') ||
|
||||
msg.includes('Invalid column') ||
|
||||
msg.includes('exceeds maximum') ||
|
||||
msg.includes('incompatible') ||
|
||||
msg.includes('duplicate')
|
||||
) {
|
||||
return NextResponse.json({ error: msg }, { status: 400 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error updating column in table ${tableId}:`, error)
|
||||
return NextResponse.json({ error: 'Failed to update column' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** DELETE /api/table/[tableId]/columns - Deletes a column from the table schema. */
|
||||
export const DELETE = withRouteHandler(
|
||||
async (request: NextRequest, context: ColumnsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await context.params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized column deletion attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const validation = await parseRequest(deleteTableColumnContract, request, context)
|
||||
if (!validation.success) return validation.response
|
||||
const validated = validation.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const updatedTable = await deleteColumn(
|
||||
{ tableId, columnName: validated.columnName },
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error, 'Invalid request data')
|
||||
}
|
||||
|
||||
const msg = rootErrorMessage(error)
|
||||
if (msg.includes('not found') || msg === 'Table not found') {
|
||||
return NextResponse.json({ error: msg }, { status: 404 })
|
||||
}
|
||||
if (msg.includes('Cannot delete') || msg.includes('last column')) {
|
||||
return NextResponse.json({ error: msg }, { status: 400 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error deleting column from table ${tableId}:`, error)
|
||||
return NextResponse.json({ error: 'Failed to delete column' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,58 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { runColumnContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { runWorkflowColumn } from '@/lib/table/workflow-columns'
|
||||
import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableRunColumnAPI')
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** POST /api/table/[tableId]/columns/run */
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!auth.success || !auth.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const parsed = await parseRequest(runColumnContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, groupIds, runMode, rowIds, filter, excludeRowIds, limit } =
|
||||
parsed.data.body
|
||||
const access = await checkAccess(tableId, auth.userId, 'write')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
|
||||
// Validate the filter up front (the dispatcher reuses it) so a bad field fails fast.
|
||||
const filterError = tableFilterError(filter, access.table.schema.columns)
|
||||
if (filterError) return filterError
|
||||
|
||||
const { dispatchId } = await runWorkflowColumn({
|
||||
tableId,
|
||||
workspaceId,
|
||||
groupIds,
|
||||
mode: runMode,
|
||||
rowIds,
|
||||
filter,
|
||||
excludeRowIds,
|
||||
limit,
|
||||
requestId,
|
||||
triggeredByUserId: auth.userId,
|
||||
})
|
||||
|
||||
return NextResponse.json({ success: true, data: { dispatchId } })
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'Invalid workspace ID') {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
logger.error(`run-column failed:`, error)
|
||||
return NextResponse.json({ error: 'Failed to run columns' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const {
|
||||
mockCheckAccess,
|
||||
mockMarkTableJobRunning,
|
||||
mockReleaseJobClaim,
|
||||
mockRunTableDelete,
|
||||
mockTableFilterError,
|
||||
mockTasksTrigger,
|
||||
flags,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockMarkTableJobRunning: vi.fn(),
|
||||
mockReleaseJobClaim: vi.fn(),
|
||||
mockRunTableDelete: vi.fn(),
|
||||
mockTableFilterError: vi.fn(),
|
||||
mockTasksTrigger: vi.fn(),
|
||||
flags: { triggerDev: false },
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('job-id-xyz'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
vi.mock('@/lib/table/jobs/service', () => ({
|
||||
markTableJobRunning: mockMarkTableJobRunning,
|
||||
releaseJobClaim: mockReleaseJobClaim,
|
||||
}))
|
||||
vi.mock('@/lib/table/delete-runner', () => ({ runTableDelete: mockRunTableDelete }))
|
||||
vi.mock('@/lib/core/config/env-flags', () => ({
|
||||
get isTriggerDevEnabled() {
|
||||
return flags.triggerDev
|
||||
},
|
||||
}))
|
||||
vi.mock('@/background/table-delete', () => ({ tableDeleteTask: { id: 'table-delete' } }))
|
||||
vi.mock('@/lib/core/async-jobs/region', () => ({
|
||||
resolveTriggerRegion: vi.fn().mockResolvedValue('us-east-1'),
|
||||
}))
|
||||
vi.mock('@trigger.dev/sdk', () => ({
|
||||
tasks: { trigger: mockTasksTrigger },
|
||||
task: (config: unknown) => config,
|
||||
}))
|
||||
vi.mock('@/lib/core/utils/background', () => ({
|
||||
runDetached: (_label: string, work: () => Promise<unknown>) => {
|
||||
void work()
|
||||
},
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
tableFilterError: mockTableFilterError,
|
||||
}
|
||||
})
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/delete-async/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [{ name: 'status', type: 'string' }] },
|
||||
metadata: null,
|
||||
rowCount: 1000,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(body: unknown, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/delete-async`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const validBody = {
|
||||
workspaceId: 'workspace-1',
|
||||
filter: { status: 'archived' },
|
||||
excludeRowIds: ['row_keep'],
|
||||
}
|
||||
|
||||
describe('POST /api/table/[tableId]/delete-async', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockMarkTableJobRunning.mockResolvedValue(true)
|
||||
mockRunTableDelete.mockResolvedValue(undefined)
|
||||
mockTableFilterError.mockReturnValue(null)
|
||||
mockTasksTrigger.mockResolvedValue({ id: 'run_1' })
|
||||
flags.triggerDev = false
|
||||
})
|
||||
|
||||
it('claims the job slot and kicks off the delete worker with filter + exclusions', async () => {
|
||||
const response = await makeRequest(validBody)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ tableId: 'tbl_1', jobId: 'job-id-xyz' })
|
||||
expect(mockMarkTableJobRunning).toHaveBeenCalledWith('tbl_1', 'job-id-xyz', 'delete', {
|
||||
filter: { status: 'archived' },
|
||||
excludeRowIds: ['row_keep'],
|
||||
cutoff: expect.any(String),
|
||||
})
|
||||
expect(mockRunTableDelete).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
jobId: 'job-id-xyz',
|
||||
tableId: 'tbl_1',
|
||||
workspaceId: 'workspace-1',
|
||||
filter: { status: 'archived' },
|
||||
excludeRowIds: ['row_keep'],
|
||||
cutoff: expect.any(Date),
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('allows a whole-table delete with no filter', async () => {
|
||||
const response = await makeRequest({ workspaceId: 'workspace-1' })
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockRunTableDelete).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ filter: undefined, cutoff: expect.any(Date) })
|
||||
)
|
||||
})
|
||||
|
||||
it('returns 409 when a job is already in progress (claim lost)', async () => {
|
||||
mockMarkTableJobRunning.mockResolvedValue(false)
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(409)
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on an invalid filter without claiming the slot', async () => {
|
||||
mockTableFilterError.mockReturnValue(NextResponse.json({ error: 'bad field' }, { status: 400 }))
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockMarkTableJobRunning).not.toHaveBeenCalled()
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockMarkTableJobRunning).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns the access error status when access is denied', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: false, status: 403 })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(403)
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when the table is archived', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable({ archivedAt: new Date() }) })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on workspace mismatch', async () => {
|
||||
const response = await makeRequest({ ...validBody, workspaceId: 'other-ws' })
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
|
||||
it('routes through trigger.dev (ISO cutoff, tagged) when the flag is on', async () => {
|
||||
flags.triggerDev = true
|
||||
const response = await makeRequest(validBody)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
expect(mockTasksTrigger).toHaveBeenCalledWith(
|
||||
'table-delete',
|
||||
expect.objectContaining({
|
||||
jobId: 'job-id-xyz',
|
||||
tableId: 'tbl_1',
|
||||
filter: { status: 'archived' },
|
||||
excludeRowIds: ['row_keep'],
|
||||
cutoff: expect.any(String),
|
||||
}),
|
||||
{ tags: ['tableId:tbl_1', 'jobId:job-id-xyz'], region: 'us-east-1' }
|
||||
)
|
||||
})
|
||||
|
||||
it('releases the job claim when the trigger.dev dispatch fails (no ghost running job)', async () => {
|
||||
flags.triggerDev = true
|
||||
mockTasksTrigger.mockRejectedValueOnce(new Error('trigger.dev unreachable'))
|
||||
|
||||
const response = await makeRequest(validBody)
|
||||
|
||||
expect(response.status).toBe(500)
|
||||
expect(mockReleaseJobClaim).toHaveBeenCalledWith('tbl_1', 'job-id-xyz')
|
||||
expect(mockRunTableDelete).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { deleteTableRowsAsyncContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
|
||||
import { runDetached } from '@/lib/core/utils/background'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner'
|
||||
import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service'
|
||||
import type { TableDeleteJobPayload } from '@/lib/table/types'
|
||||
import { accessError, checkAccess, tableFilterError } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableDeleteAsync')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/table/[tableId]/delete-async
|
||||
*
|
||||
* Kicks off a background "select all" delete: the client sends the active filter (and an optional
|
||||
* exclusion set) instead of every row id. Claims the table's single job slot (mutually exclusive
|
||||
* with imports), captures a `created_at` cutoff so rows inserted while the job runs survive, then
|
||||
* runs the paginated delete worker detached.
|
||||
*/
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const userId = authResult.userId
|
||||
|
||||
const parsed = await parseRequest(deleteTableRowsAsyncContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, filter, excludeRowIds, estimatedCount } = parsed.data.body
|
||||
|
||||
const access = await checkAccess(tableId, userId, 'write')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
const { table } = access
|
||||
|
||||
if (table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
if (table.archivedAt) {
|
||||
return NextResponse.json({ error: 'Cannot delete from an archived table' }, { status: 400 })
|
||||
}
|
||||
|
||||
// Validate the filter up front so the caller gets immediate feedback (the worker reuses it).
|
||||
const filterError = tableFilterError(filter, table.schema.columns)
|
||||
if (filterError) return filterError
|
||||
|
||||
// Rows inserted after this instant are spared (created_at <= cutoff in the worker).
|
||||
const cutoff = new Date()
|
||||
|
||||
// Atomically claim the job slot — one background job per table, so this also blocks while an
|
||||
// import is in flight (and vice versa). The scope is persisted to the job's payload so read
|
||||
// paths can mask the doomed rows while the job runs (see `pendingDeleteMask`).
|
||||
const jobId = generateId()
|
||||
const payload: TableDeleteJobPayload = {
|
||||
filter,
|
||||
excludeRowIds,
|
||||
cutoff: cutoff.toISOString(),
|
||||
// Clamp the client's display estimate to reality so a stale/bogus value
|
||||
// can't drive counts negative or hide more than the table holds.
|
||||
...(estimatedCount != null ? { doomedCount: Math.min(estimatedCount, table.rowCount) } : {}),
|
||||
}
|
||||
const claimed = await markTableJobRunning(tableId, jobId, 'delete', payload)
|
||||
if (!claimed) {
|
||||
return NextResponse.json(
|
||||
{ error: 'A job is already in progress for this table' },
|
||||
{ status: 409 }
|
||||
)
|
||||
}
|
||||
|
||||
if (isTriggerDevEnabled) {
|
||||
// Trigger.dev runs the delete outside the web container (survives deploys) and retries —
|
||||
// safe: the keyset + cutoff walk just deletes whatever remains.
|
||||
try {
|
||||
const [{ tableDeleteTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
|
||||
import('@/background/table-delete'),
|
||||
import('@trigger.dev/sdk'),
|
||||
import('@/lib/core/async-jobs/region'),
|
||||
])
|
||||
await tasks.trigger<typeof tableDeleteTask>(
|
||||
'table-delete',
|
||||
{ jobId, tableId, workspaceId, filter, excludeRowIds, cutoff: cutoff.toISOString() },
|
||||
{ tags: [`tableId:${tableId}`, `jobId:${jobId}`], region: await resolveTriggerRegion() }
|
||||
)
|
||||
} catch (error) {
|
||||
// A failed dispatch must not leave a ghost `running` job holding the
|
||||
// table's one-write-job slot until the stale-job janitor fires.
|
||||
await releaseJobClaim(tableId, jobId).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
runDetached('table-delete', () =>
|
||||
runTableDelete({
|
||||
jobId,
|
||||
tableId,
|
||||
workspaceId,
|
||||
filter,
|
||||
excludeRowIds,
|
||||
cutoff,
|
||||
}).catch(async (error) => {
|
||||
// No retry machinery on the detached path — fail the job immediately.
|
||||
await markTableDeleteFailed(tableId, jobId, error)
|
||||
throw error
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Async row delete started`, {
|
||||
tableId,
|
||||
jobId,
|
||||
hasFilter: Boolean(filter),
|
||||
excluded: excludeRowIds?.length ?? 0,
|
||||
})
|
||||
return NextResponse.json({ success: true, data: { tableId, jobId } })
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockListActiveDispatches, mockCountRunningCells } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockListActiveDispatches: vi.fn(),
|
||||
mockCountRunningCells: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/dispatcher', () => ({
|
||||
listActiveDispatches: mockListActiveDispatches,
|
||||
countRunningCells: mockCountRunningCells,
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { GET } from '@/app/api/table/[tableId]/dispatches/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/dispatches`)
|
||||
return GET(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
function buildDispatchRow(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 'dispatch-1',
|
||||
tableId: 'tbl_1',
|
||||
workspaceId: 'workspace-1',
|
||||
requestId: 'req-1',
|
||||
mode: 'all',
|
||||
scope: { groupIds: ['group-1'] },
|
||||
status: 'dispatching',
|
||||
cursor: 4,
|
||||
limit: null,
|
||||
isManualRun: true,
|
||||
processedCount: 5,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('GET /api/table/[tableId]/dispatches', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockListActiveDispatches.mockResolvedValue([])
|
||||
mockCountRunningCells.mockResolvedValue({ byRowId: {}, hasRunning: false })
|
||||
})
|
||||
|
||||
it('returns dispatches and the per-row running map, without a total field', async () => {
|
||||
mockListActiveDispatches.mockResolvedValue([buildDispatchRow()])
|
||||
mockCountRunningCells.mockResolvedValue({
|
||||
byRowId: { 'row-1': 2, 'row-2': 1 },
|
||||
hasRunning: true,
|
||||
})
|
||||
|
||||
const response = await makeRequest()
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data.dispatches).toEqual([
|
||||
{
|
||||
id: 'dispatch-1',
|
||||
status: 'dispatching',
|
||||
mode: 'all',
|
||||
isManualRun: true,
|
||||
cursor: 4,
|
||||
scope: { groupIds: ['group-1'] },
|
||||
},
|
||||
])
|
||||
expect(data.data.runningByRowId).toEqual({ 'row-1': 2, 'row-2': 1 })
|
||||
expect(data.data.hasRunning).toBe(true)
|
||||
expect(data.data).not.toHaveProperty('runningCellCount')
|
||||
})
|
||||
|
||||
it('includes unclaimed pre-stamps only while a dispatch is active', async () => {
|
||||
mockListActiveDispatches.mockResolvedValue([buildDispatchRow()])
|
||||
await makeRequest()
|
||||
expect(mockCountRunningCells).toHaveBeenCalledWith('tbl_1', {
|
||||
includeUnclaimedPreStamps: true,
|
||||
})
|
||||
|
||||
mockListActiveDispatches.mockResolvedValue([])
|
||||
await makeRequest()
|
||||
expect(mockCountRunningCells).toHaveBeenLastCalledWith('tbl_1', {
|
||||
includeUnclaimedPreStamps: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest()
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockListActiveDispatches).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { type ActiveDispatch, listActiveDispatchesContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { countRunningCells, listActiveDispatches } from '@/lib/table/dispatcher'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableDispatchesAPI')
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/table/[tableId]/dispatches
|
||||
*
|
||||
* Returns active (`pending` / `dispatching`) dispatches for the table. Drives
|
||||
* the client's "about to run" overlay so refresh during a long Run-all keeps
|
||||
* the queued indicators on rows the dispatcher hasn't reached yet.
|
||||
*/
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(listActiveDispatchesContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const rows = await listActiveDispatches(tableId)
|
||||
// Unclaimed `pending` pre-stamps are real queued work while a dispatch is
|
||||
// active; with none active they're abandoned orphans that would pin the
|
||||
// "X running" badge above zero forever.
|
||||
const { byRowId: runningByRowId, hasRunning } = await countRunningCells(tableId, {
|
||||
includeUnclaimedPreStamps: rows.length > 0,
|
||||
})
|
||||
const dispatches: ActiveDispatch[] = rows.map((r) => ({
|
||||
id: r.id,
|
||||
status: r.status as 'pending' | 'dispatching',
|
||||
mode: r.mode,
|
||||
isManualRun: r.isManualRun,
|
||||
cursor: r.cursor,
|
||||
scope: r.scope,
|
||||
...(r.limit ? { limit: r.limit } : {}),
|
||||
}))
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
dispatches,
|
||||
runningByRowId,
|
||||
hasRunning,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] list-dispatches failed:`, error)
|
||||
return NextResponse.json({ error: 'Failed to list active dispatches' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,173 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { tableEventStreamContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { SSE_HEADERS } from '@/lib/core/utils/sse'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
getLatestTableEventId,
|
||||
readTableEventsSince,
|
||||
type TableEventEntry,
|
||||
} from '@/lib/table/events'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableEventStreamAPI')
|
||||
|
||||
const POLL_INTERVAL_MS = 500
|
||||
const HEARTBEAT_INTERVAL_MS = 15_000
|
||||
const MAX_STREAM_DURATION_MS = 4 * 60 * 60 * 1000 // 4 hours; client reconnects past this
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteContext {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/table/[tableId]/events/stream?from=<lastEventId>
|
||||
*
|
||||
* SSE stream of cell-state transitions. Replay-on-reconnect via `from`;
|
||||
* absent `from` tails from the latest event id (fresh mount — the client has
|
||||
* just fetched current state, so replaying history would rewind it).
|
||||
* Pruning (buffer cap exceeded or TTL expired) sends a `pruned` event and
|
||||
* closes; the client responds with a full row-query refetch and reconnects
|
||||
* tailing from latest. */
|
||||
export const GET = withRouteHandler(async (req: NextRequest, context: RouteContext) => {
|
||||
const requestId = generateRequestId()
|
||||
const parsed = await parseRequest(tableEventStreamContract, req, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { from: fromEventId } = parsed.data.query
|
||||
|
||||
const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false })
|
||||
if (!auth.success || !auth.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const access = await checkAccess(tableId, auth.userId, 'read')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
|
||||
logger.info(`[${requestId}] Table event stream opened`, { tableId, fromEventId })
|
||||
|
||||
const encoder = new TextEncoder()
|
||||
let closed = false
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
let lastEventId = fromEventId ?? 0
|
||||
const deadline = Date.now() + MAX_STREAM_DURATION_MS
|
||||
let nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS
|
||||
|
||||
const enqueue = (text: string) => {
|
||||
if (closed) return
|
||||
try {
|
||||
controller.enqueue(encoder.encode(text))
|
||||
} catch {
|
||||
closed = true
|
||||
}
|
||||
}
|
||||
|
||||
const sendEvents = (events: TableEventEntry[]) => {
|
||||
for (const entry of events) {
|
||||
if (closed) return
|
||||
enqueue(`data: ${JSON.stringify(entry)}\n\n`)
|
||||
lastEventId = entry.eventId
|
||||
}
|
||||
}
|
||||
|
||||
const sendPrunedAndClose = (earliestEventId: number | undefined) => {
|
||||
enqueue(
|
||||
`event: pruned\ndata: ${JSON.stringify({ earliestEventId: earliestEventId ?? null })}\n\n`
|
||||
)
|
||||
if (!closed) {
|
||||
closed = true
|
||||
try {
|
||||
controller.close()
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const sendHeartbeat = () => {
|
||||
// SSE comment line — keeps proxies (ALB default 60s idle) from closing
|
||||
// the connection during quiet periods.
|
||||
enqueue(`: ping ${Date.now()}\n\n`)
|
||||
}
|
||||
|
||||
try {
|
||||
// No replay cursor → tail from the latest event id. Resolved inside
|
||||
// the try so a Redis failure errors the stream (client reconnects
|
||||
// with backoff) rather than silently replaying the whole buffer.
|
||||
if (fromEventId === undefined) {
|
||||
lastEventId = await getLatestTableEventId(tableId)
|
||||
}
|
||||
// Initial replay from buffer.
|
||||
const initial = await readTableEventsSince(tableId, lastEventId)
|
||||
if (initial.status === 'pruned') {
|
||||
sendPrunedAndClose(initial.earliestEventId)
|
||||
return
|
||||
}
|
||||
if (initial.status === 'unavailable') {
|
||||
throw new Error(`Table event buffer unavailable: ${initial.error}`)
|
||||
}
|
||||
sendEvents(initial.events)
|
||||
|
||||
// Stream loop — poll the buffer and forward new events. Workflow
|
||||
// execution stream uses the same shape; pub/sub wakeups are an
|
||||
// optimization we can add later if 500ms polling becomes a problem.
|
||||
while (!closed && Date.now() < deadline) {
|
||||
await sleep(POLL_INTERVAL_MS)
|
||||
if (closed) return
|
||||
|
||||
const result = await readTableEventsSince(tableId, lastEventId)
|
||||
if (result.status === 'pruned') {
|
||||
sendPrunedAndClose(result.earliestEventId)
|
||||
return
|
||||
}
|
||||
if (result.status === 'unavailable') {
|
||||
throw new Error(`Table event buffer unavailable: ${result.error}`)
|
||||
}
|
||||
if (result.events.length > 0) {
|
||||
sendEvents(result.events)
|
||||
}
|
||||
|
||||
if (Date.now() >= nextHeartbeatAt) {
|
||||
sendHeartbeat()
|
||||
nextHeartbeatAt = Date.now() + HEARTBEAT_INTERVAL_MS
|
||||
}
|
||||
}
|
||||
|
||||
// Reached the defensive duration ceiling — close cleanly so the client
|
||||
// reconnects with the latest lastEventId.
|
||||
if (!closed) {
|
||||
enqueue(`event: rotate\ndata: {}\n\n`)
|
||||
closed = true
|
||||
try {
|
||||
controller.close()
|
||||
} catch {}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Table event stream error`, {
|
||||
tableId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
if (!closed) {
|
||||
try {
|
||||
controller.error(error)
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
closed = true
|
||||
logger.info(`[${requestId}] Client disconnected from table event stream`, { tableId })
|
||||
},
|
||||
})
|
||||
|
||||
return new NextResponse(stream, {
|
||||
headers: { ...SSE_HEADERS, 'X-Table-Id': tableId },
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockMarkTableJobRunning, mockRunTableExport } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockMarkTableJobRunning: vi.fn(),
|
||||
mockRunTableExport: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('job-id-xyz'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
vi.mock('@/lib/table/jobs/service', () => ({ markTableJobRunning: mockMarkTableJobRunning }))
|
||||
vi.mock('@/lib/table/export-runner', () => ({ runTableExport: mockRunTableExport }))
|
||||
vi.mock('@/lib/core/utils/background', () => ({
|
||||
runDetached: (_label: string, work: () => Promise<unknown>) => {
|
||||
void work()
|
||||
},
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/export-async/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [{ name: 'name', type: 'string' }] },
|
||||
metadata: null,
|
||||
rowCount: 50000,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(body: unknown, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/export-async`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const validBody = { workspaceId: 'workspace-1', format: 'csv' }
|
||||
|
||||
describe('POST /api/table/[tableId]/export-async', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockMarkTableJobRunning.mockResolvedValue(true)
|
||||
mockRunTableExport.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('claims an export job and kicks off the worker', async () => {
|
||||
const response = await makeRequest(validBody)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ tableId: 'tbl_1', jobId: 'job-id-xyz' })
|
||||
expect(mockMarkTableJobRunning).toHaveBeenCalledWith('tbl_1', 'job-id-xyz', 'export', {
|
||||
format: 'csv',
|
||||
})
|
||||
expect(mockRunTableExport).toHaveBeenCalledWith({
|
||||
jobId: 'job-id-xyz',
|
||||
tableId: 'tbl_1',
|
||||
workspaceId: 'workspace-1',
|
||||
format: 'csv',
|
||||
})
|
||||
})
|
||||
|
||||
it('defaults the format to csv', async () => {
|
||||
const response = await makeRequest({ workspaceId: 'workspace-1' })
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockRunTableExport).toHaveBeenCalledWith(expect.objectContaining({ format: 'csv' }))
|
||||
})
|
||||
|
||||
it('returns 409 when the claim fails', async () => {
|
||||
mockMarkTableJobRunning.mockResolvedValue(false)
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(409)
|
||||
expect(mockRunTableExport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockMarkTableJobRunning).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns the access error status when access is denied', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: false, status: 403 })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(403)
|
||||
expect(mockRunTableExport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on workspace mismatch', async () => {
|
||||
const response = await makeRequest({ ...validBody, workspaceId: 'other-ws' })
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockMarkTableJobRunning).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { exportTableAsyncContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
|
||||
import { runDetached } from '@/lib/core/utils/background'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { captureServerEvent } from '@/lib/posthog/server'
|
||||
import { runTableExport, type TableExportPayload } from '@/lib/table/export-runner'
|
||||
import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service'
|
||||
import type { TableExportJobPayload } from '@/lib/table/types'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableExportAsync')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/table/[tableId]/export-async
|
||||
*
|
||||
* Kicks off a background export for large tables (small ones stream synchronously via `/export`).
|
||||
* Export jobs are read-only, so they bypass the one-running-job-per-table gate (the partial-unique
|
||||
* index excludes `type = 'export'`) — an export can run alongside an import or delete, and the
|
||||
* delete-mask keeps a mid-delete export consistent with the delete's outcome.
|
||||
*/
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(exportTableAsyncContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, format } = parsed.data.body
|
||||
|
||||
const access = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
if (access.table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const jobId = generateId()
|
||||
const jobPayload: TableExportJobPayload = { format }
|
||||
const claimed = await markTableJobRunning(tableId, jobId, 'export', jobPayload)
|
||||
if (!claimed) {
|
||||
// Only possible against another running *export*-typed insert race losing on the pkey, or a
|
||||
// missing table — the active-job index excludes exports.
|
||||
return NextResponse.json({ error: 'Failed to start export' }, { status: 409 })
|
||||
}
|
||||
|
||||
const payload: TableExportPayload = {
|
||||
jobId,
|
||||
tableId,
|
||||
workspaceId,
|
||||
format,
|
||||
}
|
||||
if (isTriggerDevEnabled) {
|
||||
try {
|
||||
const [{ tableExportTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
|
||||
import('@/background/table-export'),
|
||||
import('@trigger.dev/sdk'),
|
||||
import('@/lib/core/async-jobs/region'),
|
||||
])
|
||||
await tasks.trigger<typeof tableExportTask>('table-export', payload, {
|
||||
tags: [`tableId:${tableId}`, `jobId:${jobId}`],
|
||||
region: await resolveTriggerRegion(),
|
||||
})
|
||||
} catch (error) {
|
||||
// A failed dispatch must not leave a ghost `running` job holding the
|
||||
// table's one-write-job slot until the stale-job janitor fires.
|
||||
await releaseJobClaim(tableId, jobId).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
runDetached('table-export', () => runTableExport(payload))
|
||||
}
|
||||
|
||||
// Audit at authorization (like the sync route) so a failed/abandoned job still records the export.
|
||||
recordAudit({
|
||||
workspaceId,
|
||||
actorId: authResult.userId,
|
||||
action: AuditAction.TABLE_EXPORTED,
|
||||
resourceType: AuditResourceType.TABLE,
|
||||
resourceId: tableId,
|
||||
resourceName: access.table.name,
|
||||
description: `Exported table "${access.table.name}" as ${format.toUpperCase()}`,
|
||||
metadata: { format, rowCount: access.table.rowCount, async: true },
|
||||
request,
|
||||
})
|
||||
if (access.table.workspaceId) {
|
||||
captureServerEvent(
|
||||
authResult.userId,
|
||||
'table_exported',
|
||||
{ table_id: tableId, workspace_id: workspaceId },
|
||||
{ groups: { workspace: workspaceId } }
|
||||
)
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Async export started`, { tableId, jobId, format })
|
||||
return NextResponse.json({ success: true, data: { tableId, jobId } })
|
||||
})
|
||||
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockGetTableJob, mockGeneratePresignedDownloadUrl } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockGetTableJob: vi.fn(),
|
||||
mockGeneratePresignedDownloadUrl: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/jobs/service', () => ({ getTableJob: mockGetTableJob }))
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => ({
|
||||
generatePresignedDownloadUrl: mockGeneratePresignedDownloadUrl,
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { GET } from '@/app/api/table/[tableId]/export/download/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(query: Record<string, string>, tableId = 'tbl_1') {
|
||||
const qs = new URLSearchParams(query).toString()
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/export/download?${qs}`)
|
||||
return GET(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const validQuery = { workspaceId: 'workspace-1', jobId: 'job_1' }
|
||||
|
||||
describe('GET /api/table/[tableId]/export/download', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockGetTableJob.mockResolvedValue({
|
||||
id: 'job_1',
|
||||
type: 'export',
|
||||
status: 'ready',
|
||||
payload: { format: 'csv', resultKey: 'workspace/workspace-1/exports/tbl_1/job_1/people.csv' },
|
||||
})
|
||||
mockGeneratePresignedDownloadUrl.mockResolvedValue('https://storage.example/signed-url')
|
||||
})
|
||||
|
||||
it('resolves a ready export to a presigned URL', async () => {
|
||||
const response = await makeRequest(validQuery)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ url: 'https://storage.example/signed-url', fileName: 'people.csv' })
|
||||
expect(mockGeneratePresignedDownloadUrl).toHaveBeenCalledWith(
|
||||
'workspace/workspace-1/exports/tbl_1/job_1/people.csv',
|
||||
'workspace'
|
||||
)
|
||||
})
|
||||
|
||||
it('404s when the job is missing or not an export', async () => {
|
||||
mockGetTableJob.mockResolvedValue({ id: 'job_1', type: 'delete', status: 'ready', payload: {} })
|
||||
const response = await makeRequest(validQuery)
|
||||
expect(response.status).toBe(404)
|
||||
})
|
||||
|
||||
it('409s when the export is not ready yet', async () => {
|
||||
mockGetTableJob.mockResolvedValue({
|
||||
id: 'job_1',
|
||||
type: 'export',
|
||||
status: 'running',
|
||||
payload: { format: 'csv' },
|
||||
})
|
||||
const response = await makeRequest(validQuery)
|
||||
expect(response.status).toBe(409)
|
||||
})
|
||||
|
||||
it('410s when the result file is gone from the payload', async () => {
|
||||
mockGetTableJob.mockResolvedValue({
|
||||
id: 'job_1',
|
||||
type: 'export',
|
||||
status: 'ready',
|
||||
payload: { format: 'csv' },
|
||||
})
|
||||
const response = await makeRequest(validQuery)
|
||||
expect(response.status).toBe(410)
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest(validQuery)
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('returns 400 on workspace mismatch', async () => {
|
||||
const response = await makeRequest({ ...validQuery, workspaceId: 'other-ws' })
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,64 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { exportDownloadContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { getTableJob } from '@/lib/table/jobs/service'
|
||||
import type { TableExportJobPayload } from '@/lib/table/types'
|
||||
import { generatePresignedDownloadUrl } from '@/lib/uploads/core/storage-service'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableExportDownload')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/table/[tableId]/export/download?jobId=…
|
||||
*
|
||||
* Resolves a completed export job to a short-lived presigned URL for the generated file. The job
|
||||
* must belong to the table, be an export, and be `ready` — the worker stamps `resultKey` onto the
|
||||
* job payload when the upload lands.
|
||||
*/
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(exportDownloadContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, jobId } = parsed.data.query
|
||||
|
||||
const access = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
if (access.table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const job = await getTableJob(tableId, jobId)
|
||||
if (!job || job.type !== 'export') {
|
||||
return NextResponse.json({ error: 'Export job not found' }, { status: 404 })
|
||||
}
|
||||
if (job.status !== 'ready') {
|
||||
return NextResponse.json({ error: 'Export is not ready' }, { status: 409 })
|
||||
}
|
||||
const payload = job.payload as TableExportJobPayload | null
|
||||
if (!payload?.resultKey) {
|
||||
return NextResponse.json({ error: 'Export file is no longer available' }, { status: 410 })
|
||||
}
|
||||
|
||||
const url = await generatePresignedDownloadUrl(payload.resultKey, 'workspace')
|
||||
const fileName = payload.resultKey.split('/').pop() ?? `export.${payload.format}`
|
||||
logger.info(`[${requestId}] Export download URL issued`, { tableId, jobId })
|
||||
return NextResponse.json({ success: true, data: { url, fileName } })
|
||||
})
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockQueryRows } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockQueryRows: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'Access denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/lib/table/rows/service', () => ({
|
||||
queryRows: mockQueryRows,
|
||||
}))
|
||||
|
||||
import { GET } from '@/app/api/table/[tableId]/export/route'
|
||||
|
||||
/** Table with an id-native column whose stable id (`col_email`) differs from its display name. */
|
||||
function buildTable(): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: {
|
||||
columns: [
|
||||
{ id: 'col_email', name: 'email', type: 'string' },
|
||||
{ name: 'legacy', type: 'string' }, // legacy: id == name
|
||||
],
|
||||
},
|
||||
metadata: null,
|
||||
rowCount: 1,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
}
|
||||
}
|
||||
|
||||
function callGet(format: string) {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/tbl_1/export?format=${format}`, {
|
||||
method: 'GET',
|
||||
})
|
||||
return GET(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
|
||||
}
|
||||
|
||||
describe('table export route — id→name translation', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
// Row data is keyed by stable column id (`col_email`), not the display name.
|
||||
mockQueryRows.mockResolvedValue({
|
||||
rows: [{ id: 'r1', data: { col_email: 'a@b.c', legacy: 'x' }, executions: {}, position: 0 }],
|
||||
rowCount: 1,
|
||||
totalCount: 1,
|
||||
limit: 1000,
|
||||
offset: 0,
|
||||
})
|
||||
})
|
||||
|
||||
it('CSV: header uses display names and cell values resolve from id-keyed data', async () => {
|
||||
const res = await callGet('csv')
|
||||
expect(res.status).toBe(200)
|
||||
const body = await res.text()
|
||||
const [header, firstRow] = body.trim().split('\n')
|
||||
expect(header).toBe('email,legacy')
|
||||
// Without id→name resolution the email cell would be blank.
|
||||
expect(firstRow).toBe('a@b.c,x')
|
||||
})
|
||||
|
||||
it('JSON: keys are display names, never the stable column id', async () => {
|
||||
const res = await callGet('json')
|
||||
expect(res.status).toBe(200)
|
||||
const parsed = JSON.parse(await res.text())
|
||||
expect(parsed).toEqual([{ email: 'a@b.c', legacy: 'x' }])
|
||||
expect(JSON.stringify(parsed)).not.toContain('col_email')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,168 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { tableExportFormatSchema, tableIdParamsSchema } from '@/lib/api/contracts/tables'
|
||||
import { getValidationErrorMessage } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { neutralizeCsvFormula } from '@/lib/core/utils/csv'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { captureServerEvent } from '@/lib/posthog/server'
|
||||
import { buildNameById, getColumnId, rowDataIdToName } from '@/lib/table/column-keys'
|
||||
import { queryRows } from '@/lib/table/rows/service'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableExport')
|
||||
|
||||
const EXPORT_BATCH_SIZE = 1000
|
||||
|
||||
type ExportFormat = 'csv' | 'json'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/table/[tableId]/export - Streams the full table contents as CSV or JSON. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = tableIdParamsSchema.parse(await params)
|
||||
|
||||
const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!auth.success || !auth.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const userId = auth.userId
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const formatValidation = tableExportFormatSchema.safeParse(
|
||||
searchParams.get('format') ?? undefined
|
||||
)
|
||||
if (!formatValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(formatValidation.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const format: ExportFormat = formatValidation.data
|
||||
|
||||
const access = await checkAccess(tableId, auth.userId, 'read')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
const { table } = access
|
||||
|
||||
const columns = table.schema.columns
|
||||
// Stored row data is id-keyed; CSV headers and JSON keys are display names, so
|
||||
// translate id → name on the way out (export is a name-friendly boundary).
|
||||
const nameById = buildNameById(table.schema)
|
||||
const safeName = sanitizeFilename(table.name)
|
||||
const filename = `${safeName}.${format}`
|
||||
|
||||
// Audit before streaming: rows leave incrementally, so a mid-stream failure still exfiltrates partial data.
|
||||
recordAudit({
|
||||
workspaceId: table.workspaceId ?? null,
|
||||
actorId: userId,
|
||||
action: AuditAction.TABLE_EXPORTED,
|
||||
resourceType: AuditResourceType.TABLE,
|
||||
resourceId: tableId,
|
||||
resourceName: table.name,
|
||||
description: `Exported table "${table.name}" as ${format.toUpperCase()}`,
|
||||
metadata: { format, rowCount: table.rowCount },
|
||||
request,
|
||||
})
|
||||
if (table.workspaceId) {
|
||||
captureServerEvent(
|
||||
userId,
|
||||
'table_exported',
|
||||
{ table_id: tableId, workspace_id: table.workspaceId },
|
||||
{ groups: { workspace: table.workspaceId } }
|
||||
)
|
||||
}
|
||||
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
const encoder = new TextEncoder()
|
||||
try {
|
||||
if (format === 'csv') {
|
||||
controller.enqueue(
|
||||
encoder.encode(`${toCsvRow(columns.map((c) => neutralizeCsvFormula(c.name)))}\n`)
|
||||
)
|
||||
} else {
|
||||
controller.enqueue(encoder.encode('['))
|
||||
}
|
||||
|
||||
let offset = 0
|
||||
let firstJsonRow = true
|
||||
while (true) {
|
||||
const result = await queryRows(
|
||||
table,
|
||||
{ limit: EXPORT_BATCH_SIZE, offset, includeTotal: false },
|
||||
requestId
|
||||
)
|
||||
|
||||
for (const row of result.rows) {
|
||||
if (format === 'csv') {
|
||||
const values = columns.map((c) => formatCsvValue(row.data[getColumnId(c)]))
|
||||
controller.enqueue(encoder.encode(`${toCsvRow(values)}\n`))
|
||||
} else {
|
||||
const prefix = firstJsonRow ? '' : ','
|
||||
firstJsonRow = false
|
||||
controller.enqueue(
|
||||
encoder.encode(prefix + JSON.stringify(rowDataIdToName(row.data, nameById)))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (result.rows.length < EXPORT_BATCH_SIZE) break
|
||||
offset += result.rows.length
|
||||
}
|
||||
|
||||
if (format === 'json') controller.enqueue(encoder.encode(']'))
|
||||
controller.close()
|
||||
|
||||
logger.info(`[${requestId}] Exported table ${tableId}`, {
|
||||
format,
|
||||
rowCount: table.rowCount,
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error(`[${requestId}] Export failed for table ${tableId}`, err)
|
||||
controller.error(err)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
return new NextResponse(stream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': format === 'csv' ? 'text/csv; charset=utf-8' : 'application/json',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
function sanitizeFilename(name: string): string {
|
||||
const cleaned = name.replace(/[^a-zA-Z0-9_-]+/g, '_').replace(/^_+|_+$/g, '')
|
||||
return cleaned || 'table'
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes a cell for CSV. Only string cells are formula-neutralized; numbers,
|
||||
* booleans, dates, and JSON objects can never form a trigger and pass through verbatim.
|
||||
*/
|
||||
function formatCsvValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return ''
|
||||
if (value instanceof Date) return value.toISOString()
|
||||
if (typeof value === 'object') return JSON.stringify(value)
|
||||
if (typeof value === 'string') return neutralizeCsvFormula(value)
|
||||
return String(value)
|
||||
}
|
||||
|
||||
function toCsvRow(values: string[]): string {
|
||||
return values.map(escapeCsvField).join(',')
|
||||
}
|
||||
|
||||
function escapeCsvField(field: string): string {
|
||||
if (/[",\n\r]/.test(field)) {
|
||||
return `"${field.replace(/"/g, '""')}"`
|
||||
}
|
||||
return field
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns, workflowAuthzMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockAddWorkflowGroup, mockUpdateWorkflowGroup } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockAddWorkflowGroup: vi.fn(),
|
||||
mockUpdateWorkflowGroup: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
checkAccess: mockCheckAccess,
|
||||
normalizeColumn: (column: unknown) => column,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/lib/table/workflow-groups/service', () => ({
|
||||
addWorkflowGroup: mockAddWorkflowGroup,
|
||||
updateWorkflowGroup: mockUpdateWorkflowGroup,
|
||||
deleteWorkflowGroup: vi.fn(),
|
||||
}))
|
||||
|
||||
import { PATCH, POST } from '@/app/api/table/[tableId]/groups/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function callPost(body: Record<string, unknown>, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/groups`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
function callPatch(body: Record<string, unknown>, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/groups`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(body),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
return PATCH(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const baseGroup = {
|
||||
id: 'grp_1',
|
||||
workflowId: 'wf_1',
|
||||
outputs: [{ blockId: 'block_1', path: 'result', columnName: 'result' }],
|
||||
}
|
||||
|
||||
const baseOutputColumns = [{ name: 'result', type: 'string', workflowGroupId: 'grp_1' }]
|
||||
|
||||
describe('POST /api/table/[tableId]/groups', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue({
|
||||
workflow: { id: 'wf_1' },
|
||||
workspaceId: 'workspace-1',
|
||||
workspaceOrganizationId: null,
|
||||
})
|
||||
mockAddWorkflowGroup.mockResolvedValue({
|
||||
schema: { columns: baseOutputColumns, workflowGroups: [baseGroup] },
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a workflowId belonging to a different workspace', async () => {
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue({
|
||||
workflow: { id: 'wf_1' },
|
||||
workspaceId: 'other-workspace',
|
||||
workspaceOrganizationId: null,
|
||||
})
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
group: baseGroup,
|
||||
outputColumns: baseOutputColumns,
|
||||
})
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockAddWorkflowGroup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a nonexistent workflowId', async () => {
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue(null)
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
group: baseGroup,
|
||||
outputColumns: baseOutputColumns,
|
||||
})
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockAddWorkflowGroup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('succeeds when the workflow belongs to the same workspace', async () => {
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
group: baseGroup,
|
||||
outputColumns: baseOutputColumns,
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockAddWorkflowGroup).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('skips the workflow check for enrichment groups without a workflowId', async () => {
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
group: { ...baseGroup, workflowId: '', enrichmentId: 'enrich_1' },
|
||||
outputColumns: baseOutputColumns,
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
expect(workflowAuthzMockFns.mockGetActiveWorkflowContext).not.toHaveBeenCalled()
|
||||
expect(mockAddWorkflowGroup).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('PATCH /api/table/[tableId]/groups', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue({
|
||||
workflow: { id: 'wf_1' },
|
||||
workspaceId: 'workspace-1',
|
||||
workspaceOrganizationId: null,
|
||||
})
|
||||
mockUpdateWorkflowGroup.mockResolvedValue({
|
||||
schema: { columns: baseOutputColumns, workflowGroups: [baseGroup] },
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects changing workflowId to one in a different workspace', async () => {
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue({
|
||||
workflow: { id: 'wf_2' },
|
||||
workspaceId: 'other-workspace',
|
||||
workspaceOrganizationId: null,
|
||||
})
|
||||
const res = await callPatch({
|
||||
workspaceId: 'workspace-1',
|
||||
groupId: 'grp_1',
|
||||
workflowId: 'wf_2',
|
||||
})
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockUpdateWorkflowGroup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a nonexistent workflowId', async () => {
|
||||
workflowAuthzMockFns.mockGetActiveWorkflowContext.mockResolvedValue(null)
|
||||
const res = await callPatch({
|
||||
workspaceId: 'workspace-1',
|
||||
groupId: 'grp_1',
|
||||
workflowId: 'wf_missing',
|
||||
})
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockUpdateWorkflowGroup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('succeeds when changing workflowId to one in the same workspace', async () => {
|
||||
const res = await callPatch({
|
||||
workspaceId: 'workspace-1',
|
||||
groupId: 'grp_1',
|
||||
workflowId: 'wf_1',
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockUpdateWorkflowGroup).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('skips the workflow check when workflowId is not being changed', async () => {
|
||||
const res = await callPatch({
|
||||
workspaceId: 'workspace-1',
|
||||
groupId: 'grp_1',
|
||||
name: 'Renamed group',
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
expect(workflowAuthzMockFns.mockGetActiveWorkflowContext).not.toHaveBeenCalled()
|
||||
expect(mockUpdateWorkflowGroup).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,207 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getActiveWorkflowContext } from '@sim/platform-authz/workflow'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
addWorkflowGroupContract,
|
||||
deleteWorkflowGroupContract,
|
||||
updateWorkflowGroupContract,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
addWorkflowGroup,
|
||||
deleteWorkflowGroup,
|
||||
updateWorkflowGroup,
|
||||
} from '@/lib/table/workflow-groups/service'
|
||||
import { accessError, checkAccess, normalizeColumn } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableWorkflowGroupsAPI')
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* Confirms `workflowId` resolves to an active workflow in `workspaceId` before it is
|
||||
* persisted onto a table's workflow group. Returns a 400 response when the workflow
|
||||
* doesn't exist or belongs to a different workspace, otherwise `null`.
|
||||
*/
|
||||
async function validateWorkflowInWorkspace(
|
||||
workflowId: string,
|
||||
workspaceId: string
|
||||
): Promise<NextResponse | null> {
|
||||
const context = await getActiveWorkflowContext(workflowId)
|
||||
if (!context || context.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workflow ID' }, { status: 400 })
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps known service-layer error messages onto HTTP responses; falls through
|
||||
* to a 500 with a generic message for anything unrecognized. The three
|
||||
* group-route handlers all surface the same error shapes from
|
||||
* `addWorkflowGroup` / `updateWorkflowGroup` / `deleteWorkflowGroup`, so they
|
||||
* share this mapper instead of repeating the if-chain three times.
|
||||
*/
|
||||
function mapWorkflowGroupError(error: unknown, fallbackMessage: string): NextResponse {
|
||||
if (error instanceof Error) {
|
||||
const msg = error.message
|
||||
if (msg === 'Table not found' || msg.includes('not found')) {
|
||||
return NextResponse.json({ error: msg }, { status: 404 })
|
||||
}
|
||||
if (
|
||||
msg.includes('Schema validation') ||
|
||||
msg.includes('Missing column definition') ||
|
||||
msg.includes('already exists') ||
|
||||
msg.includes('exceed')
|
||||
) {
|
||||
return NextResponse.json({ error: msg }, { status: 400 })
|
||||
}
|
||||
}
|
||||
logger.error(fallbackMessage, error)
|
||||
return NextResponse.json({ error: fallbackMessage }, { status: 500 })
|
||||
}
|
||||
|
||||
/** POST /api/table/[tableId]/groups — create a workflow group + its output columns. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const parsed = await parseRequest(addWorkflowGroupContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
if (result.table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
if (validated.group.workflowId) {
|
||||
const workflowError = await validateWorkflowInWorkspace(
|
||||
validated.group.workflowId,
|
||||
result.table.workspaceId
|
||||
)
|
||||
if (workflowError) return workflowError
|
||||
}
|
||||
const updatedTable = await addWorkflowGroup(
|
||||
{
|
||||
tableId,
|
||||
group: validated.group,
|
||||
outputColumns: validated.outputColumns,
|
||||
autoRun: validated.autoRun,
|
||||
actorUserId: authResult.userId,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
workflowGroups: updatedTable.schema.workflowGroups ?? [],
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return mapWorkflowGroupError(error, 'Failed to add workflow group')
|
||||
}
|
||||
})
|
||||
|
||||
/** PATCH /api/table/[tableId]/groups — update a workflow group (deps / outputs). */
|
||||
export const PATCH = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const parsed = await parseRequest(updateWorkflowGroupContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
if (result.table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
if (validated.workflowId !== undefined) {
|
||||
const workflowError = await validateWorkflowInWorkspace(
|
||||
validated.workflowId,
|
||||
result.table.workspaceId
|
||||
)
|
||||
if (workflowError) return workflowError
|
||||
}
|
||||
const updatedTable = await updateWorkflowGroup(
|
||||
{
|
||||
tableId,
|
||||
groupId: validated.groupId,
|
||||
actorUserId: authResult.userId,
|
||||
...(validated.workflowId !== undefined ? { workflowId: validated.workflowId } : {}),
|
||||
...(validated.name !== undefined ? { name: validated.name } : {}),
|
||||
...(validated.dependencies !== undefined ? { dependencies: validated.dependencies } : {}),
|
||||
...(validated.outputs !== undefined ? { outputs: validated.outputs } : {}),
|
||||
...(validated.newOutputColumns !== undefined
|
||||
? { newOutputColumns: validated.newOutputColumns }
|
||||
: {}),
|
||||
...(validated.mappingUpdates !== undefined
|
||||
? { mappingUpdates: validated.mappingUpdates }
|
||||
: {}),
|
||||
...(validated.inputMappings !== undefined
|
||||
? { inputMappings: validated.inputMappings }
|
||||
: {}),
|
||||
...(validated.deploymentMode !== undefined
|
||||
? { deploymentMode: validated.deploymentMode }
|
||||
: {}),
|
||||
...(validated.type !== undefined ? { type: validated.type } : {}),
|
||||
...(validated.autoRun !== undefined ? { autoRun: validated.autoRun } : {}),
|
||||
},
|
||||
requestId
|
||||
)
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
workflowGroups: updatedTable.schema.workflowGroups ?? [],
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return mapWorkflowGroupError(error, 'Failed to update workflow group')
|
||||
}
|
||||
})
|
||||
|
||||
/** DELETE /api/table/[tableId]/groups — remove a workflow group + its columns. */
|
||||
export const DELETE = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const parsed = await parseRequest(deleteWorkflowGroupContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
if (result.table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
const updatedTable = await deleteWorkflowGroup(
|
||||
{ tableId, groupId: validated.groupId },
|
||||
requestId
|
||||
)
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
columns: updatedTable.schema.columns.map(normalizeColumn),
|
||||
workflowGroups: updatedTable.schema.workflowGroups ?? [],
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
return mapWorkflowGroupError(error, 'Failed to delete workflow group')
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockMarkTableImporting, mockRunTableImport } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockMarkTableImporting: vi.fn(),
|
||||
mockRunTableImport: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('import-id-xyz'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
vi.mock('@/lib/table/jobs/service', () => ({ markTableJobRunning: mockMarkTableImporting }))
|
||||
vi.mock('@/lib/table/import-runner', () => ({ runTableImport: mockRunTableImport }))
|
||||
vi.mock('@/lib/core/utils/background', () => ({
|
||||
runDetached: (_label: string, work: () => Promise<unknown>) => {
|
||||
void work()
|
||||
},
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/import-async/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [{ name: 'name', type: 'string' }] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(body: unknown, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import-async`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const validBody = {
|
||||
workspaceId: 'workspace-1',
|
||||
fileKey: 'workspace/workspace-1/123-data.csv',
|
||||
fileName: 'data.csv',
|
||||
mode: 'append',
|
||||
}
|
||||
|
||||
describe('POST /api/table/[tableId]/import-async', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockMarkTableImporting.mockResolvedValue(true)
|
||||
mockRunTableImport.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('marks the table importing and kicks off the worker with mode + mapping', async () => {
|
||||
const response = await makeRequest({
|
||||
...validBody,
|
||||
mode: 'replace',
|
||||
mapping: { Name: 'name' },
|
||||
createColumns: ['Extra'],
|
||||
})
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ tableId: 'tbl_1', importId: 'import-id-xyz' })
|
||||
expect(mockMarkTableImporting).toHaveBeenCalledWith('tbl_1', 'import-id-xyz', 'import')
|
||||
expect(mockRunTableImport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tableId: 'tbl_1',
|
||||
mode: 'replace',
|
||||
delimiter: ',',
|
||||
mapping: { Name: 'name' },
|
||||
createColumns: ['Extra'],
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('returns 409 when the table is already importing (claim lost)', async () => {
|
||||
mockMarkTableImporting.mockResolvedValue(false)
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(409)
|
||||
expect(mockRunTableImport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockMarkTableImporting).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns the access error status when access is denied', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: false, status: 403 })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(403)
|
||||
expect(mockRunTableImport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when the target table is archived', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable({ archivedAt: new Date() }) })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockRunTableImport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on workspace mismatch', async () => {
|
||||
const response = await makeRequest({ ...validBody, workspaceId: 'other-ws' })
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
|
||||
it('returns 400 for an invalid mode', async () => {
|
||||
const response = await makeRequest({ ...validBody, mode: 'bogus' })
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,115 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { importIntoTableAsyncContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
|
||||
import { runDetached } from '@/lib/core/utils/background'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner'
|
||||
import { markTableJobRunning, releaseJobClaim } from '@/lib/table/jobs/service'
|
||||
import { getUserSettings } from '@/lib/users/queries'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableImportIntoAsync')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const userId = authResult.userId
|
||||
|
||||
const parsed = await parseRequest(importIntoTableAsyncContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, fileKey, fileName, mode, mapping, createColumns, timezone } =
|
||||
parsed.data.body
|
||||
|
||||
const access = await checkAccess(tableId, userId, 'write')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
const { table } = access
|
||||
|
||||
if (table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
// The fileKey is client-supplied — ensure it points at this workspace's storage prefix so a
|
||||
// caller can't import another workspace's uploaded object.
|
||||
if (!fileKey.startsWith(`workspace/${workspaceId}/`)) {
|
||||
return NextResponse.json({ error: 'Invalid file key for workspace' }, { status: 400 })
|
||||
}
|
||||
if (table.archivedAt) {
|
||||
return NextResponse.json({ error: 'Cannot import into an archived table' }, { status: 400 })
|
||||
}
|
||||
|
||||
const ext = fileName.split('.').pop()?.toLowerCase()
|
||||
if (ext !== 'csv' && ext !== 'tsv') {
|
||||
return NextResponse.json({ error: 'Only CSV and TSV files are supported' }, { status: 400 })
|
||||
}
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
|
||||
// Atomically claim the table's job slot — the single concurrency gate. If another job (import
|
||||
// or delete) already holds it, this returns false (no overlapping workers).
|
||||
const importId = generateId()
|
||||
const claimed = await markTableJobRunning(tableId, importId, 'import')
|
||||
if (!claimed) {
|
||||
return NextResponse.json(
|
||||
{ error: 'A job is already in progress for this table' },
|
||||
{ status: 409 }
|
||||
)
|
||||
}
|
||||
|
||||
const importPayload: TableImportPayload = {
|
||||
importId,
|
||||
tableId,
|
||||
workspaceId,
|
||||
userId,
|
||||
fileKey,
|
||||
fileName,
|
||||
delimiter,
|
||||
mode,
|
||||
mapping,
|
||||
createColumns,
|
||||
timezone: timezone ?? (await getUserSettings(userId)).timezone ?? 'UTC',
|
||||
}
|
||||
if (isTriggerDevEnabled) {
|
||||
// Trigger.dev runs the import outside the web container, so it survives app deploys.
|
||||
try {
|
||||
const [{ tableImportTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
|
||||
import('@/background/table-import'),
|
||||
import('@trigger.dev/sdk'),
|
||||
import('@/lib/core/async-jobs/region'),
|
||||
])
|
||||
await tasks.trigger<typeof tableImportTask>('table-import', importPayload, {
|
||||
tags: [`tableId:${tableId}`, `jobId:${importId}`],
|
||||
region: await resolveTriggerRegion(),
|
||||
})
|
||||
} catch (error) {
|
||||
// A failed dispatch must not leave a ghost `running` job holding the
|
||||
// table's one-write-job slot until the stale-job janitor fires.
|
||||
await releaseJobClaim(tableId, importId).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
runDetached('table-import', () => runTableImport(importPayload))
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Async CSV import into existing table started`, {
|
||||
tableId,
|
||||
importId,
|
||||
mode,
|
||||
fileName,
|
||||
})
|
||||
return NextResponse.json({ success: true, data: { tableId, importId } })
|
||||
})
|
||||
@@ -0,0 +1,536 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const {
|
||||
mockCheckAccess,
|
||||
mockImportAppendRows,
|
||||
mockImportReplaceRows,
|
||||
mockDispatchAfterBatchInsert,
|
||||
mockMarkTableImporting,
|
||||
mockReleaseImportClaim,
|
||||
mockGetMaxRowsPerTable,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockImportAppendRows: vi.fn(),
|
||||
mockImportReplaceRows: vi.fn(),
|
||||
mockDispatchAfterBatchInsert: vi.fn(),
|
||||
mockMarkTableImporting: vi.fn(),
|
||||
mockReleaseImportClaim: vi.fn(),
|
||||
mockGetMaxRowsPerTable: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) => {
|
||||
const message = result.status === 404 ? 'Table not found' : 'Access denied'
|
||||
return NextResponse.json({ error: message }, { status: result.status })
|
||||
},
|
||||
csvProxyBodyCapResponse: () => null,
|
||||
multipartErrorResponse: (error: { code: string; message: string }) =>
|
||||
NextResponse.json(
|
||||
{ error: error.message },
|
||||
{ status: error.code === 'FILE_TOO_LARGE' ? 413 : 400 }
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* The route imports `importAppendRows` / `importReplaceRows` from
|
||||
* `@/lib/table/import-data`. These functions own the import transaction (column
|
||||
* adds + row writes); mocking that module replaces them without touching the
|
||||
* other real helpers (`coerceRowsForTable`, `createCsvParser`, etc.) exported
|
||||
* through the barrel.
|
||||
*/
|
||||
vi.mock('@/lib/table/import-data', () => ({
|
||||
importAppendRows: mockImportAppendRows,
|
||||
importReplaceRows: mockImportReplaceRows,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/jobs/service', () => ({
|
||||
markTableJobRunning: mockMarkTableImporting,
|
||||
releaseJobClaim: mockReleaseImportClaim,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/rows/service', () => ({
|
||||
dispatchAfterBatchInsert: mockDispatchAfterBatchInsert,
|
||||
}))
|
||||
|
||||
/** The append pre-check reads the workspace's current plan row limit, not the frozen `table.maxRows`. */
|
||||
vi.mock('@/lib/table/billing', () => ({
|
||||
getMaxRowsPerTable: mockGetMaxRowsPerTable,
|
||||
wouldExceedRowLimit: (limit: number, current: number, added: number) =>
|
||||
limit >= 0 && current + added > limit,
|
||||
}))
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/import/route'
|
||||
|
||||
function createCsvFile(contents: string, name = 'data.csv', type = 'text/csv'): File {
|
||||
return new File([contents], name, { type })
|
||||
}
|
||||
|
||||
function createFormData(
|
||||
file: File,
|
||||
options?: {
|
||||
workspaceId?: string | null
|
||||
mode?: string | null
|
||||
mapping?: unknown
|
||||
createColumns?: unknown
|
||||
}
|
||||
): FormData {
|
||||
// Text fields must precede the file part for the streaming parser.
|
||||
const form = new FormData()
|
||||
if (options?.workspaceId !== null) {
|
||||
form.append('workspaceId', options?.workspaceId ?? 'workspace-1')
|
||||
}
|
||||
if (options?.mode !== null) {
|
||||
form.append('mode', options?.mode ?? 'append')
|
||||
}
|
||||
if (options?.mapping !== undefined) {
|
||||
form.append(
|
||||
'mapping',
|
||||
typeof options.mapping === 'string' ? options.mapping : JSON.stringify(options.mapping)
|
||||
)
|
||||
}
|
||||
if (options?.createColumns !== undefined) {
|
||||
form.append(
|
||||
'createColumns',
|
||||
typeof options.createColumns === 'string'
|
||||
? options.createColumns
|
||||
: JSON.stringify(options.createColumns)
|
||||
)
|
||||
}
|
||||
form.append('file', file)
|
||||
return form
|
||||
}
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string', required: true },
|
||||
{ name: 'age', type: 'number' },
|
||||
],
|
||||
},
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/** Additions array the route passed to importAppendRows (2nd positional arg). */
|
||||
function appendAdditions(): { name: string; type: string }[] {
|
||||
return mockImportAppendRows.mock.calls[0][1] as { name: string; type: string }[]
|
||||
}
|
||||
|
||||
/** Rows array the route passed to importAppendRows (3rd positional arg). */
|
||||
function appendRows(): unknown[] {
|
||||
return mockImportAppendRows.mock.calls[0][2] as unknown[]
|
||||
}
|
||||
|
||||
async function callPost(form: FormData, { tableId }: { tableId: string } = { tableId: 'tbl_1' }) {
|
||||
// Building the request from a FormData body gives a real multipart stream and
|
||||
// boundary, exercising the streaming `readMultipart` parser end-to-end.
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import`, {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
describe('POST /api/table/[tableId]/import', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockImportAppendRows.mockImplementation(
|
||||
async (table: TableDefinition, _additions: unknown, rows: unknown[]) => ({
|
||||
inserted: rows.map((_, i) => ({ id: `row_${i}` })),
|
||||
table,
|
||||
})
|
||||
)
|
||||
mockImportReplaceRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 })
|
||||
mockMarkTableImporting.mockResolvedValue(true)
|
||||
mockReleaseImportClaim.mockResolvedValue(undefined)
|
||||
mockGetMaxRowsPerTable.mockResolvedValue(1_000_000)
|
||||
})
|
||||
|
||||
it('returns 401 when the user is not authenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({
|
||||
success: false,
|
||||
error: 'Authentication required',
|
||||
})
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('returns 409 when a background import already holds the table (claim lost)', async () => {
|
||||
mockMarkTableImporting.mockResolvedValueOnce(false)
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(409)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
expect(mockImportReplaceRows).not.toHaveBeenCalled()
|
||||
expect(mockReleaseImportClaim).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('releases the import claim after a successful write', async () => {
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockMarkTableImporting).toHaveBeenCalledWith('tbl_1', 'deadbeefcafef00d', 'import')
|
||||
expect(mockReleaseImportClaim).toHaveBeenCalledWith('tbl_1', 'deadbeefcafef00d')
|
||||
})
|
||||
|
||||
it('returns 400 when the mode is invalid', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'bogus' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/Invalid mode/)
|
||||
})
|
||||
|
||||
it('returns 403 when the user lacks workspace write access', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({ ok: false, status: 403 })
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(403)
|
||||
})
|
||||
|
||||
it('returns 400 when the target table is archived', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
table: buildTable({ archivedAt: new Date('2024-01-02') }),
|
||||
})
|
||||
const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/archived/i)
|
||||
})
|
||||
|
||||
it('returns 400 when the file part precedes the required fields', async () => {
|
||||
// Build a raw multipart body with the file BEFORE workspaceId.
|
||||
const boundary = '----orderboundary'
|
||||
const body = Buffer.concat([
|
||||
Buffer.from(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="data.csv"\r\nContent-Type: text/csv\r\n\r\nname,age\nAlice,30\r\n`
|
||||
),
|
||||
Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="workspaceId"\r\n\r\n`),
|
||||
Buffer.from('workspace-1\r\n'),
|
||||
Buffer.from(`--${boundary}--\r\n`),
|
||||
])
|
||||
const req = {
|
||||
headers: new Headers({ 'content-type': `multipart/form-data; boundary=${boundary}` }),
|
||||
body: new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(body))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
signal: undefined,
|
||||
} as unknown as NextRequest
|
||||
|
||||
const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
expect(mockImportReplaceRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when the CSV is missing a required column', async () => {
|
||||
const response = await callPost(createFormData(createCsvFile('age\n30')))
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/missing required columns/i)
|
||||
expect(data.details?.missingRequired).toEqual(['name'])
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('appends rows via importAppendRows', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mode).toBe('append')
|
||||
expect(data.data.insertedCount).toBe(2)
|
||||
expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
|
||||
expect(appendRows()).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
expect(mockImportReplaceRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('accepts chunked multipart imports without a content-length header', async () => {
|
||||
const form = createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
|
||||
const req = new NextRequest('http://localhost:3000/api/table/tbl_1/import', {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
})
|
||||
|
||||
expect(req.headers.get('content-length')).toBeNull()
|
||||
|
||||
const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rejects append when it would exceed the current plan row limit', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({ ok: true, table: buildTable({ rowCount: 99 }) })
|
||||
mockGetMaxRowsPerTable.mockResolvedValueOnce(100)
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/exceed table row limit/)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('replaces rows via importReplaceRows', async () => {
|
||||
mockImportReplaceRows.mockResolvedValueOnce({ deletedCount: 5, insertedCount: 2 })
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'replace' })
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mode).toBe('replace')
|
||||
expect(data.data.deletedCount).toBe(5)
|
||||
expect(data.data.insertedCount).toBe(2)
|
||||
expect(mockImportReplaceRows).toHaveBeenCalledTimes(1)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses an explicit mapping when provided', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('First Name,Years\nAlice,30\nBob,40', 'people.csv'), {
|
||||
mode: 'append',
|
||||
mapping: { 'First Name': 'name', Years: 'age' },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
const data = await response.json()
|
||||
expect(data.data.mappedColumns).toEqual(['First Name', 'Years'])
|
||||
expect(appendRows()).toEqual([
|
||||
{ name: 'Alice', age: 30 },
|
||||
{ name: 'Bob', age: 40 },
|
||||
])
|
||||
})
|
||||
|
||||
it('returns 400 when the mapping targets a non-existent column', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('a\nAlice'), {
|
||||
mode: 'append',
|
||||
mapping: { a: 'nonexistent' },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/do not exist on the table/)
|
||||
})
|
||||
|
||||
it('returns 400 when a mapping value is not a string or null', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), {
|
||||
mode: 'append',
|
||||
mapping: { name: 42 },
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/Mapping values must be/)
|
||||
})
|
||||
|
||||
it('surfaces unique violations from importAppendRows as 400', async () => {
|
||||
mockImportAppendRows.mockRejectedValueOnce(
|
||||
new Error('Row 1: Column "name" must be unique. Value "Alice" already exists in row row_xxx')
|
||||
)
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/must be unique/)
|
||||
expect(data.data?.insertedCount).toBe(0)
|
||||
})
|
||||
|
||||
it('accepts TSV files', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(
|
||||
createCsvFile('name\tage\nAlice\t30', 'data.tsv', 'text/tab-separated-values'),
|
||||
{ mode: 'append' }
|
||||
)
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns 400 for unsupported file extensions', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age', 'data.json', 'application/json'))
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/CSV and TSV/)
|
||||
})
|
||||
|
||||
describe('createColumns', () => {
|
||||
it('auto-creates columns for unmapped CSV headers', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io\nBob,40,b@x.io'), {
|
||||
mode: 'append',
|
||||
createColumns: ['email'],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
|
||||
expect(appendAdditions()).toEqual([
|
||||
expect.objectContaining({ name: 'email', type: 'string' }),
|
||||
])
|
||||
// Existing columns have no id (legacy) → keyed by name; the new `email`
|
||||
// column was assigned id `col_deadbeefcafef00d` (mocked generateId).
|
||||
expect(appendRows()).toEqual([
|
||||
{ name: 'Alice', age: 30, col_deadbeefcafef00d: 'a@x.io' },
|
||||
{ name: 'Bob', age: 40, col_deadbeefcafef00d: 'b@x.io' },
|
||||
])
|
||||
})
|
||||
|
||||
it('infers column type from CSV row values', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,score\nAlice,42\nBob,17'), {
|
||||
mode: 'append',
|
||||
createColumns: ['score'],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(appendAdditions()).toEqual([
|
||||
expect.objectContaining({ name: 'score', type: 'number' }),
|
||||
])
|
||||
})
|
||||
|
||||
it('dedupes when sanitized name collides with an existing column', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
table: buildTable({
|
||||
schema: {
|
||||
columns: [
|
||||
{ name: 'name', type: 'string', required: true },
|
||||
{ name: 'age', type: 'number' },
|
||||
{ name: 'email', type: 'string' },
|
||||
],
|
||||
},
|
||||
}),
|
||||
})
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age,Email\nAlice,30,a@x.io'), {
|
||||
mode: 'append',
|
||||
createColumns: ['Email'],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(appendAdditions()).toEqual([
|
||||
expect.objectContaining({ name: 'Email_2', type: 'string' }),
|
||||
])
|
||||
})
|
||||
|
||||
it('returns 400 when createColumns references a header not in the CSV', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), {
|
||||
mode: 'append',
|
||||
createColumns: ['nonexistent'],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/unknown CSV headers/)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when createColumns is not an array of strings', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), {
|
||||
mode: 'append',
|
||||
createColumns: [1, 2],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/createColumns must be a JSON array/)
|
||||
expect(mockImportAppendRows).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when createColumns is invalid JSON', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), {
|
||||
mode: 'append',
|
||||
createColumns: '{not-json',
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/createColumns must be valid JSON/)
|
||||
})
|
||||
|
||||
it('surfaces column-creation failures from importAppendRows as 400', async () => {
|
||||
mockImportAppendRows.mockRejectedValueOnce(new Error('Column "email" already exists'))
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io'), {
|
||||
mode: 'append',
|
||||
createColumns: ['email'],
|
||||
})
|
||||
)
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.error).toMatch(/already exists/)
|
||||
})
|
||||
|
||||
it('surfaces row insert failures without success when schema was mutated', async () => {
|
||||
mockImportAppendRows.mockRejectedValueOnce(new Error('must be unique'))
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io'), {
|
||||
mode: 'append',
|
||||
createColumns: ['email'],
|
||||
})
|
||||
)
|
||||
// Route forwarded the column addition into the (now atomic) import op.
|
||||
expect(appendAdditions()).toEqual([
|
||||
expect.objectContaining({ name: 'email', type: 'string' }),
|
||||
])
|
||||
expect(response.status).toBe(400)
|
||||
const data = await response.json()
|
||||
expect(data.success).toBeUndefined()
|
||||
expect(data.error).toMatch(/must be unique/)
|
||||
})
|
||||
|
||||
it('passes no additions when createColumns is omitted', async () => {
|
||||
const response = await callPost(
|
||||
createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
expect(appendAdditions()).toEqual([])
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,422 @@
|
||||
import type { Readable } from 'node:stream'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
csvExtensionSchema,
|
||||
csvImportCreateColumnsSchema,
|
||||
csvImportFormSchema,
|
||||
csvImportMappingSchema,
|
||||
csvImportModeSchema,
|
||||
tableIdParamsSchema,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import { ianaTimezoneSchema } from '@/lib/api/contracts/user'
|
||||
import { getValidationErrorMessage } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
buildAutoMapping,
|
||||
CSV_MAX_FILE_SIZE_BYTES,
|
||||
type CsvHeaderMapping,
|
||||
CsvImportValidationError,
|
||||
coerceRowsForTable,
|
||||
createCsvParser,
|
||||
dispatchAfterBatchInsert,
|
||||
generateColumnId,
|
||||
getMaxRowsPerTable,
|
||||
inferColumnType,
|
||||
markTableJobRunning,
|
||||
releaseJobClaim,
|
||||
sanitizeName,
|
||||
type TableDefinition,
|
||||
type TableSchema,
|
||||
validateMapping,
|
||||
wouldExceedRowLimit,
|
||||
} from '@/lib/table'
|
||||
import { importAppendRows, importReplaceRows } from '@/lib/table/import-data'
|
||||
import { getUserSettings } from '@/lib/users/queries'
|
||||
import {
|
||||
accessError,
|
||||
checkAccess,
|
||||
csvProxyBodyCapResponse,
|
||||
multipartErrorResponse,
|
||||
} from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableImportCSVExisting')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const maxDuration = 300
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = tableIdParamsSchema.parse(await params)
|
||||
let fileStream: Readable | undefined
|
||||
let claimedImportId: string | null = null
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const oversize = csvProxyBodyCapResponse(request)
|
||||
if (oversize) return oversize
|
||||
|
||||
let parsed: Awaited<ReturnType<typeof readMultipart>>
|
||||
try {
|
||||
parsed = await readMultipart(request, {
|
||||
maxFileBytes: CSV_MAX_FILE_SIZE_BYTES,
|
||||
requiredFieldsBeforeFile: ['workspaceId'],
|
||||
signal: request.signal,
|
||||
})
|
||||
} catch (err) {
|
||||
if (isMultipartError(err)) return multipartErrorResponse(err)
|
||||
throw err
|
||||
}
|
||||
|
||||
const { fields, file } = parsed
|
||||
if (!file) {
|
||||
return NextResponse.json({ error: 'CSV file is required' }, { status: 400 })
|
||||
}
|
||||
fileStream = file.stream
|
||||
|
||||
const workspaceIdResult = csvImportFormSchema.shape.workspaceId.safeParse(fields.workspaceId)
|
||||
if (!workspaceIdResult.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(workspaceIdResult.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const workspaceId = workspaceIdResult.data
|
||||
|
||||
const rawMode = fields.mode ?? 'append'
|
||||
const modeValidation = csvImportModeSchema.safeParse(rawMode)
|
||||
if (!modeValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: `Invalid mode "${String(rawMode)}". Must be "append" or "replace".` },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const mode = modeValidation.data
|
||||
|
||||
const ext = file.filename.split('.').pop()?.toLowerCase()
|
||||
const extensionValidation = csvExtensionSchema.safeParse(ext)
|
||||
if (!extensionValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(extensionValidation.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (table.workspaceId !== workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (table.archivedAt) {
|
||||
return NextResponse.json({ error: 'Cannot import into an archived table' }, { status: 400 })
|
||||
}
|
||||
// Don't run a sync import on top of an in-flight background job — concurrent writers
|
||||
// would insert at colliding row positions.
|
||||
if (table.jobStatus === 'running') {
|
||||
return NextResponse.json(
|
||||
{ error: 'A job is already in progress for this table' },
|
||||
{ status: 409 }
|
||||
)
|
||||
}
|
||||
|
||||
let mapping: CsvHeaderMapping | undefined
|
||||
if (fields.mapping) {
|
||||
const mappingValidation = csvImportMappingSchema.safeParse(fields.mapping)
|
||||
if (!mappingValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(mappingValidation.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
mapping = mappingValidation.data
|
||||
}
|
||||
|
||||
let createColumns: string[] | undefined
|
||||
if (fields.createColumns) {
|
||||
const createColumnsValidation = csvImportCreateColumnsSchema.safeParse(fields.createColumns)
|
||||
if (!createColumnsValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(createColumnsValidation.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
createColumns = createColumnsValidation.data
|
||||
}
|
||||
|
||||
let timezone = (await getUserSettings(authResult.userId)).timezone ?? 'UTC'
|
||||
if (fields.timezone) {
|
||||
const timezoneValidation = ianaTimezoneSchema.safeParse(fields.timezone)
|
||||
if (!timezoneValidation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(timezoneValidation.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
timezone = timezoneValidation.data
|
||||
}
|
||||
|
||||
const delimiter = extensionValidation.data === 'tsv' ? '\t' : ','
|
||||
const parser = createCsvParser(delimiter)
|
||||
// `.pipe` doesn't forward source errors; forward them so the iterator throws.
|
||||
file.stream.on('error', (streamErr) => parser.destroy(streamErr))
|
||||
file.stream.pipe(parser)
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
|
||||
rows.push(record)
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
return NextResponse.json({ error: 'CSV file has no data rows' }, { status: 400 })
|
||||
}
|
||||
const headers = Object.keys(rows[0])
|
||||
|
||||
let effectiveMapping = mapping ?? buildAutoMapping(headers, table.schema)
|
||||
let prospectiveTable: TableDefinition = table
|
||||
const additions: { id?: string; name: string; type: string }[] = []
|
||||
|
||||
if (createColumns && createColumns.length > 0) {
|
||||
const headerSet = new Set(headers)
|
||||
const unknownHeaders = createColumns.filter((h) => !headerSet.has(h))
|
||||
if (unknownHeaders.length > 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `createColumns references unknown CSV headers: ${unknownHeaders.join(', ')}`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const usedNames = new Set(table.schema.columns.map((c) => c.name.toLowerCase()))
|
||||
const updatedMapping: CsvHeaderMapping = { ...effectiveMapping }
|
||||
const newColumns: TableSchema['columns'] = []
|
||||
|
||||
for (const header of createColumns) {
|
||||
const base = sanitizeName(header)
|
||||
let columnName = base
|
||||
let suffix = 2
|
||||
while (usedNames.has(columnName.toLowerCase())) {
|
||||
columnName = `${base}_${suffix}`
|
||||
suffix++
|
||||
}
|
||||
usedNames.add(columnName.toLowerCase())
|
||||
const inferredType = inferColumnType(rows.map((r) => r[header]))
|
||||
// Pre-assign the id so the prospective schema (used to coerce rows) and
|
||||
// the persisted column (created in importAppendRows) share the same key.
|
||||
const id = generateColumnId()
|
||||
additions.push({ id, name: columnName, type: inferredType })
|
||||
newColumns.push({
|
||||
id,
|
||||
name: columnName,
|
||||
type: inferredType as TableSchema['columns'][number]['type'],
|
||||
required: false,
|
||||
unique: false,
|
||||
})
|
||||
updatedMapping[header] = columnName
|
||||
}
|
||||
|
||||
prospectiveTable = {
|
||||
...table,
|
||||
schema: { columns: [...table.schema.columns, ...newColumns] },
|
||||
}
|
||||
effectiveMapping = updatedMapping
|
||||
}
|
||||
|
||||
let validation: ReturnType<typeof validateMapping>
|
||||
try {
|
||||
validation = validateMapping({
|
||||
csvHeaders: headers,
|
||||
mapping: effectiveMapping,
|
||||
tableSchema: prospectiveTable.schema,
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof CsvImportValidationError) {
|
||||
return NextResponse.json({ error: err.message, details: err.details }, { status: 400 })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
if (validation.mappedHeaders.length === 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `No CSV headers map to columns on the table. CSV headers: ${headers.join(', ')}. Table columns: ${prospectiveTable.schema.columns.map((c) => c.name).join(', ')}`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const coerced = coerceRowsForTable(rows, prospectiveTable.schema, validation.effectiveMap, {
|
||||
timezone,
|
||||
})
|
||||
|
||||
// Atomically claim the table before writing. The pre-check above reads a checkAccess snapshot
|
||||
// taken before the parse/validation; a background import could claim the table in that window.
|
||||
// markTableJobRunning is the single atomic gate (same one the async kickoff uses) — released in
|
||||
// the finally so a sync import can't write concurrently with a background one (corrupts replace).
|
||||
const syncImportId = generateId()
|
||||
if (!(await markTableJobRunning(tableId, syncImportId, 'import'))) {
|
||||
return NextResponse.json(
|
||||
{ error: 'A job is already in progress for this table' },
|
||||
{ status: 409 }
|
||||
)
|
||||
}
|
||||
claimedImportId = syncImportId
|
||||
|
||||
if (mode === 'append') {
|
||||
const maxRows = await getMaxRowsPerTable(workspaceId)
|
||||
if (wouldExceedRowLimit(maxRows, prospectiveTable.rowCount, coerced.length)) {
|
||||
const deficit = prospectiveTable.rowCount + coerced.length - maxRows
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Append would exceed table row limit (${maxRows}). Currently ${prospectiveTable.rowCount} rows, ${coerced.length} new rows, ${deficit} over.`,
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
const { inserted: insertedRows, table: finalTable } = await importAppendRows(
|
||||
table,
|
||||
additions,
|
||||
coerced,
|
||||
{ workspaceId, userId: authResult.userId, requestId }
|
||||
)
|
||||
const inserted = insertedRows.length
|
||||
// Fire trigger + scheduler AFTER the tx commits — both read through the
|
||||
// global db connection and would otherwise see no rows.
|
||||
dispatchAfterBatchInsert(finalTable, insertedRows, requestId, authResult.userId)
|
||||
|
||||
logger.info(`[${requestId}] Append CSV imported`, {
|
||||
tableId: table.id,
|
||||
fileName: file.filename,
|
||||
mode,
|
||||
inserted,
|
||||
createdColumns: additions.length,
|
||||
mappedColumns: validation.mappedHeaders.length,
|
||||
skippedHeaders: validation.skippedHeaders.length,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
mode,
|
||||
insertedCount: inserted,
|
||||
mappedColumns: validation.mappedHeaders,
|
||||
skippedHeaders: validation.skippedHeaders,
|
||||
unmappedColumns: validation.unmappedColumns,
|
||||
sourceFile: file.filename,
|
||||
},
|
||||
})
|
||||
} catch (err) {
|
||||
const message = toError(err).message
|
||||
logger.warn(`[${requestId}] Append failed for table ${tableId}`, {
|
||||
total: coerced.length,
|
||||
createdColumns: additions.length,
|
||||
error: message,
|
||||
})
|
||||
const isClientError =
|
||||
message.includes('row limit') ||
|
||||
message.includes('Insufficient capacity') ||
|
||||
message.includes('Schema validation') ||
|
||||
message.includes('must be unique') ||
|
||||
message.includes('Row size exceeds') ||
|
||||
message.includes('already exists') ||
|
||||
message.includes('Invalid column name') ||
|
||||
/^Row \d+:/.test(message)
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: isClientError ? message : 'Failed to import CSV',
|
||||
data: { insertedCount: 0 },
|
||||
},
|
||||
{ status: isClientError ? 400 : 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await importReplaceRows(
|
||||
table,
|
||||
additions,
|
||||
{ rows: coerced, workspaceId, userId: authResult.userId },
|
||||
requestId
|
||||
)
|
||||
|
||||
logger.info(`[${requestId}] Replace CSV imported`, {
|
||||
tableId: table.id,
|
||||
fileName: file.filename,
|
||||
mode,
|
||||
deleted: result.deletedCount,
|
||||
inserted: result.insertedCount,
|
||||
createdColumns: additions.length,
|
||||
mappedColumns: validation.mappedHeaders.length,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
tableId: table.id,
|
||||
mode,
|
||||
deletedCount: result.deletedCount,
|
||||
insertedCount: result.insertedCount,
|
||||
mappedColumns: validation.mappedHeaders,
|
||||
skippedHeaders: validation.skippedHeaders,
|
||||
unmappedColumns: validation.unmappedColumns,
|
||||
sourceFile: file.filename,
|
||||
},
|
||||
})
|
||||
} catch (err) {
|
||||
const message = toError(err).message
|
||||
const isClientError =
|
||||
message.includes('row limit') ||
|
||||
message.includes('Schema validation') ||
|
||||
message.includes('must be unique') ||
|
||||
message.includes('Row size exceeds') ||
|
||||
message.includes('already exists') ||
|
||||
message.includes('Invalid column name') ||
|
||||
/^Row \d+:/.test(message)
|
||||
if (isClientError) {
|
||||
return NextResponse.json({ error: message }, { status: 400 })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
} catch (error) {
|
||||
if (isMultipartError(error)) return multipartErrorResponse(error)
|
||||
|
||||
const message = toError(error).message
|
||||
logger.error(`[${requestId}] CSV import into existing table failed:`, error)
|
||||
|
||||
const isClientError =
|
||||
message.includes('CSV file has no') ||
|
||||
message.includes('already exists') ||
|
||||
message.includes('Invalid column name')
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: isClientError ? message : 'Failed to import CSV' },
|
||||
{ status: isClientError ? 400 : 500 }
|
||||
)
|
||||
} finally {
|
||||
fileStream?.destroy()
|
||||
// Release before the response returns, so a client refetch never observes the transient claim.
|
||||
if (claimedImportId) await releaseJobClaim(tableId, claimedImportId).catch(() => {})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockMarkJobCanceled, mockGetTableJob, mockAppendTableEvent } = vi.hoisted(
|
||||
() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockMarkJobCanceled: vi.fn(),
|
||||
mockGetTableJob: vi.fn(),
|
||||
mockAppendTableEvent: vi.fn(),
|
||||
})
|
||||
)
|
||||
|
||||
vi.mock('@/lib/table/jobs/service', () => ({
|
||||
markJobCanceled: mockMarkJobCanceled,
|
||||
getTableJob: mockGetTableJob,
|
||||
}))
|
||||
vi.mock('@/lib/table/events', () => ({ appendTableEvent: mockAppendTableEvent }))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { POST } from '@/app/api/table/[tableId]/job/cancel/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(body: unknown, tableId = 'tbl_1') {
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/job/cancel`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
const validBody = { workspaceId: 'workspace-1', jobId: 'job_1' }
|
||||
|
||||
describe('POST /api/table/[tableId]/job/cancel', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockMarkJobCanceled.mockResolvedValue(true)
|
||||
mockGetTableJob.mockResolvedValue({
|
||||
id: 'job_1',
|
||||
type: 'delete',
|
||||
status: 'running',
|
||||
payload: null,
|
||||
})
|
||||
})
|
||||
|
||||
it('cancels the job and emits a typed cancel event', async () => {
|
||||
const response = await makeRequest(validBody)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ canceled: true })
|
||||
expect(mockMarkJobCanceled).toHaveBeenCalledWith('tbl_1', 'job_1')
|
||||
expect(mockAppendTableEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ kind: 'job', type: 'delete', status: 'canceled', jobId: 'job_1' })
|
||||
)
|
||||
})
|
||||
|
||||
it('does not emit an event when nothing was running', async () => {
|
||||
mockMarkJobCanceled.mockResolvedValue(false)
|
||||
const response = await makeRequest(validBody)
|
||||
const data = await response.json()
|
||||
expect(data.data).toEqual({ canceled: false })
|
||||
expect(mockAppendTableEvent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await makeRequest(validBody)
|
||||
expect(response.status).toBe(401)
|
||||
expect(mockMarkJobCanceled).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on workspace mismatch', async () => {
|
||||
const response = await makeRequest({ ...validBody, workspaceId: 'other' })
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockMarkJobCanceled).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,60 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { cancelTableJobContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { appendTableEvent } from '@/lib/table/events'
|
||||
import { getTableJob, markJobCanceled } from '@/lib/table/jobs/service'
|
||||
import type { TableJobType } from '@/lib/table/types'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableJobCancelAPI')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/table/[tableId]/job/cancel
|
||||
*
|
||||
* Cancels an in-flight async table job (import or delete). Flips the table's job status to
|
||||
* `canceled`, which makes the detached worker's next ownership check fail so it stops. Committed
|
||||
* work (inserted/deleted rows) is left in place (no rollback). No-op if the job already finished.
|
||||
*/
|
||||
export const POST = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(cancelTableJobContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId } = parsed.data.params
|
||||
const { workspaceId, jobId } = parsed.data.body
|
||||
|
||||
const access = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!access.ok) return accessError(access, requestId, tableId)
|
||||
if (access.table.workspaceId !== workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
// Resolve the job's actual type (from its own row — the table-level derivation excludes
|
||||
// exports) so the cancel event carries the right `type`.
|
||||
const job = await getTableJob(tableId, jobId)
|
||||
const type = (job?.type ?? 'import') as TableJobType
|
||||
|
||||
const canceled = await markJobCanceled(tableId, jobId)
|
||||
if (canceled) {
|
||||
void appendTableEvent({ kind: 'job', type, tableId, jobId, status: 'canceled' })
|
||||
}
|
||||
logger.info(`[${requestId}] Job cancel requested`, { tableId, jobId, type, canceled })
|
||||
|
||||
return NextResponse.json({ success: true, data: { canceled } })
|
||||
})
|
||||
@@ -0,0 +1,57 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { updateTableMetadataContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { TableMetadata } from '@/lib/table'
|
||||
import { updateTableMetadata } from '@/lib/table'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableMetadataAPI')
|
||||
|
||||
interface TableRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** PUT /api/table/[tableId]/metadata - Update table UI metadata (column widths, etc.) */
|
||||
export const PUT = withRouteHandler(async (request: NextRequest, context: TableRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized metadata update attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(updateTableMetadataContract, request, context, {
|
||||
validationErrorResponse: (error) => validationErrorResponse(error),
|
||||
})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { tableId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const updated = await updateTableMetadata(
|
||||
tableId,
|
||||
validated.metadata,
|
||||
table.metadata as TableMetadata | null
|
||||
)
|
||||
|
||||
return NextResponse.json({ success: true, data: { metadata: updated } })
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error updating table metadata:`, error)
|
||||
return NextResponse.json({ error: 'Failed to update metadata' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { tableIdParamsSchema } from '@/lib/api/contracts/tables'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { getTableById } from '@/lib/table'
|
||||
import { performRestoreTable } from '@/lib/table/orchestration'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
|
||||
const logger = createLogger('RestoreTableAPI')
|
||||
|
||||
export const POST = withRouteHandler(
|
||||
async (request: NextRequest, { params }: { params: Promise<{ tableId: string }> }) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = tableIdParamsSchema.parse(await params)
|
||||
|
||||
try {
|
||||
const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!auth.success || !auth.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const table = await getTableById(tableId, { includeArchived: true })
|
||||
if (!table) {
|
||||
return NextResponse.json({ error: 'Table not found' }, { status: 404 })
|
||||
}
|
||||
|
||||
const permission = await getUserEntityPermissions(auth.userId, 'workspace', table.workspaceId)
|
||||
if (permission !== 'admin' && permission !== 'write') {
|
||||
return NextResponse.json({ error: 'Insufficient permissions' }, { status: 403 })
|
||||
}
|
||||
|
||||
const result = await performRestoreTable({ tableId, userId: auth.userId, requestId })
|
||||
if (!result.success) {
|
||||
const status =
|
||||
result.errorCode === 'not_found' ? 404 : result.errorCode === 'conflict' ? 409 : 500
|
||||
return NextResponse.json({ error: result.error }, { status })
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Restored table ${tableId}`)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { table: result.table },
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error restoring table ${tableId}`, error)
|
||||
return NextResponse.json(
|
||||
{ error: getErrorMessage(error, 'Internal server error') },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,199 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { getTableQuerySchema, renameTableContract } from '@/lib/api/contracts/tables'
|
||||
import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { captureServerEvent } from '@/lib/posthog/server'
|
||||
import { deleteTable, renameTable, TableConflictError, type TableSchema } from '@/lib/table'
|
||||
import { getWorkspaceTableLimits } from '@/lib/table/billing'
|
||||
import { accessError, checkAccess, normalizeColumn } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableDetailAPI')
|
||||
|
||||
interface TableRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/table/[tableId] - Retrieves a single table's details. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: TableRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized table access attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const validated = getTableQuerySchema.parse({
|
||||
workspaceId: searchParams.get('workspaceId'),
|
||||
})
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Retrieved table ${tableId} for user ${authResult.userId}`)
|
||||
|
||||
const schemaData = table.schema as TableSchema
|
||||
|
||||
// Source the row cap from the workspace's live plan, not the value stored on
|
||||
// the table at creation time (which goes stale when the plan changes).
|
||||
const { maxRowsPerTable } = await getWorkspaceTableLimits(table.workspaceId)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
table: {
|
||||
id: table.id,
|
||||
name: table.name,
|
||||
description: table.description,
|
||||
schema: {
|
||||
columns: schemaData.columns.map(normalizeColumn),
|
||||
...(schemaData.workflowGroups ? { workflowGroups: schemaData.workflowGroups } : {}),
|
||||
},
|
||||
metadata: table.metadata ?? null,
|
||||
rowCount: table.rowCount,
|
||||
maxRows: maxRowsPerTable,
|
||||
createdAt:
|
||||
table.createdAt instanceof Date
|
||||
? table.createdAt.toISOString()
|
||||
: String(table.createdAt),
|
||||
updatedAt:
|
||||
table.updatedAt instanceof Date
|
||||
? table.updatedAt.toISOString()
|
||||
: String(table.updatedAt),
|
||||
jobStatus: table.jobStatus ?? null,
|
||||
jobId: table.jobId ?? null,
|
||||
jobType: table.jobType ?? null,
|
||||
jobError: table.jobError ?? null,
|
||||
jobRowsProcessed: table.jobRowsProcessed ?? 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error getting table:`, error)
|
||||
return NextResponse.json({ error: 'Failed to get table' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** PATCH /api/table/[tableId] - Renames a table. */
|
||||
export const PATCH = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized table rename attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(
|
||||
renameTableContract,
|
||||
request,
|
||||
{ params },
|
||||
{
|
||||
validationErrorResponse: (error) => validationErrorResponse(error),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { tableId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const updated = await renameTable(tableId, validated.name, requestId, authResult.userId)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { table: updated },
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof TableConflictError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 409 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error renaming table:`, error)
|
||||
return NextResponse.json(
|
||||
{ error: getErrorMessage(error, 'Failed to rename table') },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** DELETE /api/table/[tableId] - Archives a table. */
|
||||
export const DELETE = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
logger.warn(`[${requestId}] Unauthorized table delete attempt`)
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const validated = getTableQuerySchema.parse({
|
||||
workspaceId: searchParams.get('workspaceId'),
|
||||
})
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
await deleteTable(tableId, requestId, authResult.userId)
|
||||
|
||||
captureServerEvent(
|
||||
authResult.userId,
|
||||
'table_deleted',
|
||||
{ table_id: tableId, workspace_id: table.workspaceId },
|
||||
{ groups: { workspace: table.workspaceId } }
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Table archived successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error deleting table:`, error)
|
||||
return NextResponse.json({ error: 'Failed to delete table' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { EnrichmentRunDetail, TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockLoadEnrichmentDetail } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockLoadEnrichmentDetail: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/rows/executions', () => ({
|
||||
loadEnrichmentDetail: mockLoadEnrichmentDetail,
|
||||
}))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
import { GET } from '@/app/api/table/[tableId]/rows/[rowId]/enrichment/[groupId]/route'
|
||||
|
||||
function buildTable(): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [] },
|
||||
metadata: null,
|
||||
rowCount: 1,
|
||||
maxRows: 1_000_000,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
function makeRequest(tableId = 'tbl_1', rowId = 'row_1', groupId = 'grp_1') {
|
||||
const req = new NextRequest(
|
||||
`http://localhost:3000/api/table/${tableId}/rows/${rowId}/enrichment/${groupId}`
|
||||
)
|
||||
return GET(req, { params: Promise.resolve({ tableId, rowId, groupId }) })
|
||||
}
|
||||
|
||||
const detail: EnrichmentRunDetail = {
|
||||
startedAt: '2026-06-18T00:00:00.000Z',
|
||||
completedAt: '2026-06-18T00:00:01.000Z',
|
||||
durationMs: 1000,
|
||||
totalCost: 0.05,
|
||||
matchedProvider: 'hunter',
|
||||
aborted: false,
|
||||
providers: [
|
||||
{
|
||||
id: 'hunter',
|
||||
label: 'Hunter',
|
||||
toolId: 'hunter_find_email',
|
||||
status: 'matched',
|
||||
cost: 0.05,
|
||||
durationMs: 1000,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
describe('GET /api/table/[tableId]/rows/[rowId]/enrichment/[groupId]', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
})
|
||||
|
||||
it('returns the enrichment detail', async () => {
|
||||
mockLoadEnrichmentDetail.mockResolvedValue(detail)
|
||||
const res = await makeRequest()
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({ success: true, data: { detail } })
|
||||
expect(mockLoadEnrichmentDetail).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'tbl_1',
|
||||
'row_1',
|
||||
'grp_1'
|
||||
)
|
||||
})
|
||||
|
||||
it('returns null when there is no recorded run', async () => {
|
||||
mockLoadEnrichmentDetail.mockResolvedValue(null)
|
||||
const res = await makeRequest()
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({ success: true, data: { detail: null } })
|
||||
})
|
||||
|
||||
it('401s when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const res = await makeRequest()
|
||||
expect(res.status).toBe(401)
|
||||
expect(mockLoadEnrichmentDetail).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('denies when access check fails', async () => {
|
||||
mockCheckAccess.mockResolvedValue({ ok: false, status: 403 })
|
||||
const res = await makeRequest()
|
||||
expect(res.status).toBe(403)
|
||||
expect(mockLoadEnrichmentDetail).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
import { db } from '@sim/db'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { getEnrichmentDetailContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { loadEnrichmentDetail } from '@/lib/table/rows/executions'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('EnrichmentDetailAPI')
|
||||
|
||||
interface RouteParams {
|
||||
params: Promise<{ tableId: string; rowId: string; groupId: string }>
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/table/[tableId]/rows/[rowId]/enrichment/[groupId]
|
||||
*
|
||||
* Returns the enrichment cascade breakdown (provider outcomes, cost, timing)
|
||||
* for one enrichment cell. Read on demand by the enrichment details panel —
|
||||
* this data is deliberately kept off the hot grid read. Returns `null` for
|
||||
* cells with no recorded run or runs that predate the feature.
|
||||
*/
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(getEnrichmentDetailContract, request, { params })
|
||||
if (!parsed.success) return parsed.response
|
||||
const { tableId, rowId, groupId } = parsed.data.params
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const detail = await loadEnrichmentDetail(db, tableId, rowId, groupId)
|
||||
|
||||
logger.info(`[${requestId}] Loaded enrichment detail`, {
|
||||
tableId,
|
||||
rowId,
|
||||
groupId,
|
||||
hasDetail: detail !== null,
|
||||
})
|
||||
|
||||
return NextResponse.json({ success: true, data: { detail } })
|
||||
})
|
||||
@@ -0,0 +1,233 @@
|
||||
import { db } from '@sim/db'
|
||||
import { userTableRows } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { and, eq } from 'drizzle-orm'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
deleteTableRowContract,
|
||||
getTableQuerySchema,
|
||||
updateTableRowContract,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { RowData, TableSchema } from '@/lib/table'
|
||||
import { deleteRow, updateRow } from '@/lib/table'
|
||||
import { rowWireTranslators } from '@/app/api/table/row-wire'
|
||||
import {
|
||||
accessError,
|
||||
checkAccess,
|
||||
rootErrorMessage,
|
||||
rowWriteErrorResponse,
|
||||
} from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableRowAPI')
|
||||
|
||||
interface RowRouteParams {
|
||||
params: Promise<{ tableId: string; rowId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/table/[tableId]/rows/[rowId] - Retrieves a single row. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest, { params }: RowRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId, rowId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const validated = getTableQuerySchema.parse({
|
||||
workspaceId: searchParams.get('workspaceId'),
|
||||
})
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const [row] = await db
|
||||
.select({
|
||||
id: userTableRows.id,
|
||||
data: userTableRows.data,
|
||||
position: userTableRows.position,
|
||||
createdAt: userTableRows.createdAt,
|
||||
updatedAt: userTableRows.updatedAt,
|
||||
})
|
||||
.from(userTableRows)
|
||||
.where(
|
||||
and(
|
||||
eq(userTableRows.id, rowId),
|
||||
eq(userTableRows.tableId, tableId),
|
||||
eq(userTableRows.workspaceId, validated.workspaceId)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
|
||||
if (!row) {
|
||||
return NextResponse.json({ error: 'Row not found' }, { status: 404 })
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Retrieved row ${rowId} from table ${tableId}`)
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
row: {
|
||||
id: row.id,
|
||||
data: wire.dataOut(row.data as RowData),
|
||||
position: row.position,
|
||||
createdAt:
|
||||
row.createdAt instanceof Date ? row.createdAt.toISOString() : String(row.createdAt),
|
||||
updatedAt:
|
||||
row.updatedAt instanceof Date ? row.updatedAt.toISOString() : String(row.updatedAt),
|
||||
},
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error getting row:`, error)
|
||||
return NextResponse.json({ error: 'Failed to get row' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** PATCH /api/table/[tableId]/rows/[rowId] - Updates a single row (supports partial updates). */
|
||||
export const PATCH = withRouteHandler(async (request: NextRequest, context: RowRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(updateTableRowContract, request, context, {
|
||||
validationErrorResponse: (error) => validationErrorResponse(error),
|
||||
})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { tableId, rowId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
const updatedRow = await updateRow(
|
||||
{
|
||||
tableId,
|
||||
rowId,
|
||||
data: wire.dataIn(validated.data as RowData),
|
||||
workspaceId: validated.workspaceId,
|
||||
actorUserId: authResult.userId,
|
||||
},
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
// Only `null` when a `cancellationGuard` is supplied and the SQL guard
|
||||
// rejects the write — this route doesn't pass one, so reaching null is a bug.
|
||||
if (!updatedRow) throw new Error('updateRow returned null without a cancellationGuard')
|
||||
// Auto-dispatch for user edits is handled inside `updateRow` (mode: 'new').
|
||||
// Firing a second mode: 'incomplete' dispatch here would race with the
|
||||
// `mode: 'new'` one AND bulk-clear sibling-group outputs (the incomplete
|
||||
// bulk-clear wipes ALL targeted columns when any one column on the row
|
||||
// is empty).
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
row: {
|
||||
id: updatedRow.id,
|
||||
data: wire.dataOut(updatedRow.data),
|
||||
position: updatedRow.position,
|
||||
createdAt:
|
||||
updatedRow.createdAt instanceof Date
|
||||
? updatedRow.createdAt.toISOString()
|
||||
: updatedRow.createdAt,
|
||||
updatedAt:
|
||||
updatedRow.updatedAt instanceof Date
|
||||
? updatedRow.updatedAt.toISOString()
|
||||
: updatedRow.updatedAt,
|
||||
},
|
||||
message: 'Row updated successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (rootErrorMessage(error) === 'Row not found') {
|
||||
return NextResponse.json({ error: 'Row not found' }, { status: 404 })
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error updating row:`, error)
|
||||
return NextResponse.json({ error: 'Failed to update row' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** DELETE /api/table/[tableId]/rows/[rowId] - Deletes a single row. */
|
||||
export const DELETE = withRouteHandler(async (request: NextRequest, context: RowRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(deleteTableRowContract, request, context, {
|
||||
validationErrorResponse: (error) => validationErrorResponse(error),
|
||||
})
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { tableId, rowId } = parsed.data.params
|
||||
const validated = parsed.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
await deleteRow(tableId, rowId, validated.workspaceId, requestId)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Row deleted successfully',
|
||||
deletedCount: 1,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
const errorMessage = toError(error).message
|
||||
|
||||
if (errorMessage === 'Row not found') {
|
||||
return NextResponse.json({ error: errorMessage }, { status: 404 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error deleting row:`, error)
|
||||
return NextResponse.json({ error: 'Failed to delete row' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockFindRowMatches } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockFindRowMatches: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json(
|
||||
{ error: result.status === 404 ? 'Table not found' : 'Access denied' },
|
||||
{ status: result.status }
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/lib/table/rows/service', () => ({
|
||||
findRowMatches: mockFindRowMatches,
|
||||
}))
|
||||
|
||||
import { GET } from '@/app/api/table/[tableId]/rows/find/route'
|
||||
|
||||
function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: { columns: [{ name: 'name', type: 'string' }] },
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function callGet(
|
||||
query: Record<string, string>,
|
||||
{ tableId }: { tableId: string } = { tableId: 'tbl_1' }
|
||||
) {
|
||||
const params = new URLSearchParams(query)
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/rows/find?${params}`, {
|
||||
method: 'GET',
|
||||
})
|
||||
return GET(req, { params: Promise.resolve({ tableId }) })
|
||||
}
|
||||
|
||||
describe('GET /api/table/[tableId]/rows/find', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockFindRowMatches.mockResolvedValue({
|
||||
matches: [{ ordinal: 4, rowId: 'row_4', column: 'name' }],
|
||||
truncated: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({
|
||||
success: false,
|
||||
error: 'Authentication required',
|
||||
})
|
||||
const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' })
|
||||
expect(res.status).toBe(401)
|
||||
expect(mockFindRowMatches).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when q is missing', async () => {
|
||||
const res = await callGet({ workspaceId: 'workspace-1' })
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockFindRowMatches).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on a workspace mismatch', async () => {
|
||||
const res = await callGet({ workspaceId: 'other-ws', q: 'foo' })
|
||||
expect(res.status).toBe(400)
|
||||
expect(mockFindRowMatches).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 on invalid filter JSON', async () => {
|
||||
const res = await callGet({ workspaceId: 'workspace-1', q: 'foo', filter: '{not json' })
|
||||
expect(res.status).toBe(400)
|
||||
})
|
||||
|
||||
it('returns matches and forwards q/filter/sort to the service', async () => {
|
||||
const res = await callGet({
|
||||
workspaceId: 'workspace-1',
|
||||
q: 'alice',
|
||||
filter: JSON.stringify({ name: { $contains: 'a' } }),
|
||||
sort: JSON.stringify({ name: 'asc' }),
|
||||
})
|
||||
expect(res.status).toBe(200)
|
||||
const body = await res.json()
|
||||
expect(body).toEqual({
|
||||
success: true,
|
||||
data: { matches: [{ ordinal: 4, rowId: 'row_4', column: 'name' }], truncated: false },
|
||||
})
|
||||
expect(mockFindRowMatches).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'tbl_1' }),
|
||||
{ q: 'alice', filter: { name: { $contains: 'a' } }, sort: { name: 'asc' } },
|
||||
expect.any(String)
|
||||
)
|
||||
})
|
||||
|
||||
it('maps a TableQueryValidationError to 400', async () => {
|
||||
const { TableQueryValidationError } = await import('@/lib/table/sql')
|
||||
mockFindRowMatches.mockRejectedValueOnce(new TableQueryValidationError('Invalid field name'))
|
||||
const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' })
|
||||
expect(res.status).toBe(400)
|
||||
const body = await res.json()
|
||||
expect(body.error).toBe('Invalid field name')
|
||||
})
|
||||
|
||||
it('returns 404 when the table is not accessible', async () => {
|
||||
mockCheckAccess.mockResolvedValueOnce({ ok: false, status: 404 })
|
||||
const res = await callGet({ workspaceId: 'workspace-1', q: 'foo' })
|
||||
expect(res.status).toBe(404)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,81 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { findTableRowsQuerySchema } from '@/lib/api/contracts/tables'
|
||||
import { isZodError, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { Sort } from '@/lib/table'
|
||||
import { findRowMatches } from '@/lib/table/rows/service'
|
||||
import { TableQueryValidationError } from '@/lib/table/sql'
|
||||
import { accessError, checkAccess } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableRowsFindAPI')
|
||||
|
||||
interface TableRowsFindRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** GET /api/table/[tableId]/rows/find - Case-insensitive substring search across all cells. */
|
||||
export const GET = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRowsFindRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const workspaceId = searchParams.get('workspaceId')
|
||||
const q = searchParams.get('q')
|
||||
const filterParam = searchParams.get('filter')
|
||||
const sortParam = searchParams.get('sort')
|
||||
|
||||
let filter: Record<string, unknown> | undefined
|
||||
let sort: Sort | undefined
|
||||
|
||||
try {
|
||||
if (filterParam) filter = JSON.parse(filterParam) as Record<string, unknown>
|
||||
if (sortParam) sort = JSON.parse(sortParam) as Sort
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Invalid filter or sort JSON' }, { status: 400 })
|
||||
}
|
||||
|
||||
const validated = findTableRowsQuerySchema.parse({ workspaceId, q, filter, sort })
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const { matches, truncated } = await findRowMatches(
|
||||
table,
|
||||
{ q: validated.q, filter: validated.filter, sort: validated.sort },
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({ success: true, data: { matches, truncated } })
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
if (error instanceof TableQueryValidationError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error finding rows:`, error)
|
||||
return NextResponse.json({ error: 'Failed to find rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TableDefinition } from '@/lib/table'
|
||||
|
||||
const { mockCheckAccess, mockInsertRow, mockValidateRowData, mockQueryRows } = vi.hoisted(() => ({
|
||||
mockCheckAccess: vi.fn(),
|
||||
mockInsertRow: vi.fn(),
|
||||
mockValidateRowData: vi.fn(),
|
||||
mockQueryRows: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
checkAccess: mockCheckAccess,
|
||||
accessError: (result: { status: number }) =>
|
||||
NextResponse.json({ error: 'Access denied' }, { status: result.status }),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/lib/table', async () => {
|
||||
// Real column-keys translation functions; the row-wire helper under test
|
||||
// imports them from this barrel.
|
||||
const columnKeys = await import('@/lib/table/column-keys')
|
||||
return {
|
||||
...columnKeys,
|
||||
insertRow: mockInsertRow,
|
||||
batchInsertRows: vi.fn(),
|
||||
batchUpdateRows: vi.fn(),
|
||||
deleteRowsByFilter: vi.fn(),
|
||||
deleteRowsByIds: vi.fn(),
|
||||
updateRowsByFilter: vi.fn(),
|
||||
validateBatchRows: vi.fn(),
|
||||
validateRowData: mockValidateRowData,
|
||||
validateRowSize: vi.fn(() => ({ valid: true })),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/lib/table/rows/service', () => ({
|
||||
queryRows: mockQueryRows,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/sql', () => ({
|
||||
TableQueryValidationError: class TableQueryValidationError extends Error {},
|
||||
}))
|
||||
|
||||
import { GET, POST } from '@/app/api/table/[tableId]/rows/route'
|
||||
|
||||
function buildTable(): TableDefinition {
|
||||
return {
|
||||
id: 'tbl_1',
|
||||
name: 'People',
|
||||
description: null,
|
||||
schema: {
|
||||
columns: [
|
||||
{ id: 'col_aaa', name: 'Name', type: 'string' },
|
||||
{ id: 'col_bbb', name: 'Age', type: 'number' },
|
||||
],
|
||||
},
|
||||
metadata: null,
|
||||
rowCount: 0,
|
||||
maxRows: 100,
|
||||
workspaceId: 'workspace-1',
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
}
|
||||
}
|
||||
|
||||
function authAs(authType: 'session' | 'internal_jwt') {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType,
|
||||
})
|
||||
}
|
||||
|
||||
function callPost(body: Record<string, unknown>) {
|
||||
const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
|
||||
}
|
||||
|
||||
function callGet(query: Record<string, string>) {
|
||||
const params = new URLSearchParams(query)
|
||||
const req = new NextRequest(`http://localhost:3000/api/table/tbl_1/rows?${params}`, {
|
||||
method: 'GET',
|
||||
})
|
||||
return GET(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
|
||||
}
|
||||
|
||||
describe('POST /api/table/[tableId]/rows', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockValidateRowData.mockResolvedValue({ valid: true })
|
||||
mockInsertRow.mockResolvedValue({
|
||||
id: 'row_1',
|
||||
data: { col_aaa: 'Ada', col_bbb: 36 },
|
||||
position: 1,
|
||||
orderKey: 'a0',
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
})
|
||||
})
|
||||
|
||||
it('translates name-keyed data to column ids for internal-JWT (workflow tool) callers', async () => {
|
||||
authAs('internal_jwt')
|
||||
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
data: { Name: 'Ada', Age: 36 },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockValidateRowData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ rowData: { col_aaa: 'Ada', col_bbb: 36 } })
|
||||
)
|
||||
expect(mockInsertRow).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }),
|
||||
expect.anything(),
|
||||
expect.any(String)
|
||||
)
|
||||
|
||||
const body = await res.json()
|
||||
expect(body.data.row.data).toEqual({ Name: 'Ada', Age: 36 })
|
||||
})
|
||||
|
||||
it('passes id-keyed data through untouched for session (UI) callers', async () => {
|
||||
authAs('session')
|
||||
|
||||
const res = await callPost({
|
||||
workspaceId: 'workspace-1',
|
||||
data: { col_aaa: 'Ada', col_bbb: 36 },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockInsertRow).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }),
|
||||
expect.anything(),
|
||||
expect.any(String)
|
||||
)
|
||||
|
||||
const body = await res.json()
|
||||
expect(body.data.row.data).toEqual({ col_aaa: 'Ada', col_bbb: 36 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('GET /api/table/[tableId]/rows', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
|
||||
mockQueryRows.mockResolvedValue({
|
||||
rows: [
|
||||
{
|
||||
id: 'row_1',
|
||||
data: { col_aaa: 'Ada', col_bbb: 36 },
|
||||
position: 1,
|
||||
orderKey: 'a0',
|
||||
createdAt: new Date('2024-01-01'),
|
||||
updatedAt: new Date('2024-01-01'),
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
totalCount: 1,
|
||||
limit: 100,
|
||||
offset: 0,
|
||||
})
|
||||
})
|
||||
|
||||
it('translates name-keyed filter/sort and returns name-keyed rows for internal-JWT callers', async () => {
|
||||
authAs('internal_jwt')
|
||||
|
||||
const res = await callGet({
|
||||
workspaceId: 'workspace-1',
|
||||
filter: JSON.stringify({ Name: { $eq: 'Ada' } }),
|
||||
sort: JSON.stringify({ Age: 'desc' }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockQueryRows).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'tbl_1' }),
|
||||
expect.objectContaining({
|
||||
filter: { col_aaa: { $eq: 'Ada' } },
|
||||
sort: { col_bbb: 'desc' },
|
||||
}),
|
||||
expect.any(String)
|
||||
)
|
||||
|
||||
const body = await res.json()
|
||||
expect(body.data.rows[0].data).toEqual({ Name: 'Ada', Age: 36 })
|
||||
})
|
||||
|
||||
it('passes id-keyed filter and rows through untouched for session callers', async () => {
|
||||
authAs('session')
|
||||
|
||||
const res = await callGet({
|
||||
workspaceId: 'workspace-1',
|
||||
filter: JSON.stringify({ col_aaa: { $eq: 'Ada' } }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockQueryRows).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'tbl_1' }),
|
||||
expect.objectContaining({ filter: { col_aaa: { $eq: 'Ada' } } }),
|
||||
expect.any(String)
|
||||
)
|
||||
|
||||
const body = await res.json()
|
||||
expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,558 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import {
|
||||
type BatchInsertTableRowsBodyInput,
|
||||
batchUpdateTableRowsBodySchema,
|
||||
deleteTableRowsBodySchema,
|
||||
insertTableRowsContract,
|
||||
tableRowsQuerySchema,
|
||||
updateRowsByFilterBodySchema,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { isZodError, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { Filter, RowData, Sort, TableRowsCursor, TableSchema } from '@/lib/table'
|
||||
import {
|
||||
batchInsertRows,
|
||||
batchUpdateRows,
|
||||
deleteRowsByFilter,
|
||||
deleteRowsByIds,
|
||||
insertRow,
|
||||
updateRowsByFilter,
|
||||
validateBatchRows,
|
||||
validateRowData,
|
||||
validateRowSize,
|
||||
} from '@/lib/table'
|
||||
import { queryRows } from '@/lib/table/rows/service'
|
||||
import { TableQueryValidationError } from '@/lib/table/sql'
|
||||
import { rowWireTranslators } from '@/app/api/table/row-wire'
|
||||
import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableRowsAPI')
|
||||
|
||||
interface TableRowsRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
async function handleBatchInsert(
|
||||
requestId: string,
|
||||
tableId: string,
|
||||
validated: BatchInsertTableRowsBodyInput,
|
||||
userId: string,
|
||||
authType: AuthTypeValue | undefined
|
||||
): Promise<NextResponse> {
|
||||
const accessResult = await checkAccess(tableId, userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authType, table.schema as TableSchema)
|
||||
const rows = (validated.rows as RowData[]).map((row) => wire.dataIn(row))
|
||||
|
||||
// Validate rows before calling service (service also validates, but route-level
|
||||
// validation returns structured HTTP responses)
|
||||
const validation = await validateBatchRows({
|
||||
rows,
|
||||
schema: table.schema as TableSchema,
|
||||
tableId,
|
||||
})
|
||||
if (!validation.valid) return validation.response
|
||||
|
||||
try {
|
||||
const insertedRows = await batchInsertRows(
|
||||
{
|
||||
tableId,
|
||||
rows,
|
||||
workspaceId: validated.workspaceId,
|
||||
userId,
|
||||
orderKeys: validated.orderKeys,
|
||||
},
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
rows: insertedRows.map((r) => ({
|
||||
id: r.id,
|
||||
data: wire.dataOut(r.data),
|
||||
position: r.position,
|
||||
orderKey: r.orderKey ?? undefined,
|
||||
createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : r.createdAt,
|
||||
updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : r.updatedAt,
|
||||
})),
|
||||
insertedCount: insertedRows.length,
|
||||
message: `Successfully inserted ${insertedRows.length} rows`,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error batch inserting rows:`, error)
|
||||
return NextResponse.json({ error: 'Failed to insert rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
|
||||
/** POST /api/table/[tableId]/rows - Inserts row(s). Supports single or batch insert. */
|
||||
export const POST = withRouteHandler(
|
||||
async (request: NextRequest, context: TableRowsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(insertTableRowsContract, request, context)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { tableId } = parsed.data.params
|
||||
const body = parsed.data.body
|
||||
|
||||
if ('rows' in body) {
|
||||
return handleBatchInsert(requestId, tableId, body, authResult.userId, authResult.authType)
|
||||
}
|
||||
|
||||
const validated = body
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
const rowData = wire.dataIn(validated.data as RowData)
|
||||
|
||||
// Validate at route level for structured HTTP error responses
|
||||
const validation = await validateRowData({
|
||||
rowData,
|
||||
schema: table.schema as TableSchema,
|
||||
tableId,
|
||||
})
|
||||
if (!validation.valid) return validation.response
|
||||
|
||||
// Service handles atomic capacity check + insert in a transaction
|
||||
const row = await insertRow(
|
||||
{
|
||||
tableId,
|
||||
data: rowData,
|
||||
workspaceId: validated.workspaceId,
|
||||
userId: authResult.userId,
|
||||
position: validated.position,
|
||||
afterRowId: validated.afterRowId,
|
||||
beforeRowId: validated.beforeRowId,
|
||||
},
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
row: {
|
||||
id: row.id,
|
||||
data: wire.dataOut(row.data),
|
||||
position: row.position,
|
||||
orderKey: row.orderKey ?? undefined,
|
||||
createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : row.createdAt,
|
||||
updatedAt: row.updatedAt instanceof Date ? row.updatedAt.toISOString() : row.updatedAt,
|
||||
},
|
||||
|
||||
message: 'Row inserted successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error inserting row:`, error)
|
||||
return NextResponse.json({ error: 'Failed to insert row' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** GET /api/table/[tableId]/rows - Queries rows with filtering, sorting, and pagination. */
|
||||
export const GET = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRowsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const workspaceId = searchParams.get('workspaceId')
|
||||
const filterParam = searchParams.get('filter')
|
||||
const sortParam = searchParams.get('sort')
|
||||
const afterParam = searchParams.get('after')
|
||||
const limit = searchParams.get('limit')
|
||||
const offset = searchParams.get('offset')
|
||||
const includeTotalParam = searchParams.get('includeTotal')
|
||||
|
||||
let filter: Record<string, unknown> | undefined
|
||||
let sort: Sort | undefined
|
||||
let after: TableRowsCursor | undefined
|
||||
|
||||
try {
|
||||
if (filterParam) {
|
||||
filter = JSON.parse(filterParam) as Record<string, unknown>
|
||||
}
|
||||
if (sortParam) {
|
||||
sort = JSON.parse(sortParam) as Sort
|
||||
}
|
||||
if (afterParam) {
|
||||
after = JSON.parse(afterParam) as TableRowsCursor
|
||||
}
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Invalid filter, sort, or after JSON' }, { status: 400 })
|
||||
}
|
||||
|
||||
const validated = tableRowsQuerySchema.parse({
|
||||
workspaceId,
|
||||
filter,
|
||||
sort,
|
||||
after,
|
||||
limit,
|
||||
offset,
|
||||
includeTotal: includeTotalParam,
|
||||
})
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'read')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
const result = await queryRows(
|
||||
table,
|
||||
{
|
||||
filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined,
|
||||
sort: validated.sort ? wire.sortIn(validated.sort) : undefined,
|
||||
limit: validated.limit,
|
||||
offset: validated.offset,
|
||||
after: validated.after,
|
||||
includeTotal: validated.includeTotal,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
rows: result.rows.map((r) => ({
|
||||
id: r.id,
|
||||
data: wire.dataOut(r.data),
|
||||
executions: r.executions,
|
||||
position: r.position,
|
||||
orderKey: r.orderKey ?? undefined,
|
||||
createdAt:
|
||||
r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt),
|
||||
updatedAt:
|
||||
r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt),
|
||||
})),
|
||||
rowCount: result.rowCount,
|
||||
totalCount: result.totalCount,
|
||||
limit: result.limit,
|
||||
offset: result.offset,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
if (error instanceof TableQueryValidationError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error querying rows:`, error)
|
||||
return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** PUT /api/table/[tableId]/rows - Updates rows matching filter criteria. */
|
||||
export const PUT = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRowsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
let body: unknown
|
||||
try {
|
||||
body = await request.json()
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 })
|
||||
}
|
||||
|
||||
const validated = updateRowsByFilterBodySchema.parse(body)
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
const patchData = wire.dataIn(validated.data as RowData)
|
||||
|
||||
const sizeValidation = validateRowSize(patchData)
|
||||
if (!sizeValidation.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid row data', details: sizeValidation.errors },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const result = await updateRowsByFilter(
|
||||
table,
|
||||
{
|
||||
filter: wire.filterIn(validated.filter as Filter),
|
||||
data: patchData,
|
||||
limit: validated.limit,
|
||||
actorUserId: authResult.userId,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
if (result.affectedCount === 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
message: 'No rows matched the filter criteria',
|
||||
updatedCount: 0,
|
||||
},
|
||||
},
|
||||
{ status: 200 }
|
||||
)
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Rows updated successfully',
|
||||
updatedCount: result.affectedCount,
|
||||
updatedRowIds: result.affectedRowIds,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
if (error instanceof TableQueryValidationError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error updating rows by filter:`, error)
|
||||
return NextResponse.json({ error: 'Failed to update rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** DELETE /api/table/[tableId]/rows - Deletes rows matching filter criteria or by IDs. */
|
||||
export const DELETE = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRowsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
let body: unknown
|
||||
try {
|
||||
body = await request.json()
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 })
|
||||
}
|
||||
|
||||
const validated = deleteTableRowsBodySchema.parse(body)
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (validated.rowIds) {
|
||||
const result = await deleteRowsByIds(
|
||||
{ tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId },
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message:
|
||||
result.deletedCount === 0
|
||||
? 'No matching rows found for the provided IDs'
|
||||
: 'Rows deleted successfully',
|
||||
deletedCount: result.deletedCount,
|
||||
deletedRowIds: result.deletedRowIds,
|
||||
requestedCount: result.requestedCount,
|
||||
...(result.missingRowIds.length > 0 ? { missingRowIds: result.missingRowIds } : {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
const result = await deleteRowsByFilter(
|
||||
table,
|
||||
{
|
||||
filter: wire.filterIn(validated.filter as Filter),
|
||||
limit: validated.limit,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message:
|
||||
result.affectedCount === 0
|
||||
? 'No rows matched the filter criteria'
|
||||
: 'Rows deleted successfully',
|
||||
deletedCount: result.affectedCount,
|
||||
deletedRowIds: result.affectedRowIds,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
if (error instanceof TableQueryValidationError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error deleting rows:`, error)
|
||||
return NextResponse.json({ error: 'Failed to delete rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/** PATCH /api/table/[tableId]/rows - Batch updates rows by ID. */
|
||||
export const PATCH = withRouteHandler(
|
||||
async (request: NextRequest, { params }: TableRowsRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
let body: unknown
|
||||
try {
|
||||
body = await request.json()
|
||||
} catch {
|
||||
return NextResponse.json({ error: 'Request body must be valid JSON' }, { status: 400 })
|
||||
}
|
||||
|
||||
const validated = batchUpdateTableRowsBodySchema.parse(body)
|
||||
|
||||
const accessResult = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!accessResult.ok) return accessError(accessResult, requestId, tableId)
|
||||
|
||||
const { table } = accessResult
|
||||
|
||||
if (validated.workspaceId !== table.workspaceId) {
|
||||
logger.warn(
|
||||
`[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}`
|
||||
)
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const result = await batchUpdateRows(
|
||||
{
|
||||
tableId,
|
||||
updates: validated.updates as Array<{ rowId: string; data: RowData }>,
|
||||
workspaceId: validated.workspaceId,
|
||||
actorUserId: authResult.userId,
|
||||
},
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: 'Rows updated successfully',
|
||||
updatedCount: result.affectedCount,
|
||||
updatedRowIds: result.affectedRowIds,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error batch updating rows:`, error)
|
||||
return NextResponse.json({ error: 'Failed to update rows' }, { status: 500 })
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { upsertTableRowContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { isZodError, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import type { RowData, TableSchema } from '@/lib/table'
|
||||
import { upsertRow } from '@/lib/table'
|
||||
import { rowWireTranslators } from '@/app/api/table/row-wire'
|
||||
import { accessError, checkAccess, rowWriteErrorResponse } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableUpsertAPI')
|
||||
|
||||
interface UpsertRouteParams {
|
||||
params: Promise<{ tableId: string }>
|
||||
}
|
||||
|
||||
/** POST /api/table/[tableId]/rows/upsert - Inserts or updates based on unique columns. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest, context: UpsertRouteParams) => {
|
||||
const requestId = generateRequestId()
|
||||
const { tableId } = await context.params
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const validation = await parseRequest(upsertTableRowContract, request, context)
|
||||
if (!validation.success) return validation.response
|
||||
const validated = validation.data.body
|
||||
|
||||
const result = await checkAccess(tableId, authResult.userId, 'write')
|
||||
if (!result.ok) return accessError(result, requestId, tableId)
|
||||
|
||||
const { table } = result
|
||||
|
||||
if (table.workspaceId !== validated.workspaceId) {
|
||||
return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 })
|
||||
}
|
||||
|
||||
const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema)
|
||||
// conflictTarget passes through untranslated — upsertRow resolves it id-or-name.
|
||||
const upsertResult = await upsertRow(
|
||||
{
|
||||
tableId,
|
||||
workspaceId: validated.workspaceId,
|
||||
data: wire.dataIn(validated.data as RowData),
|
||||
userId: authResult.userId,
|
||||
conflictTarget: validated.conflictTarget,
|
||||
},
|
||||
table,
|
||||
requestId
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
row: {
|
||||
id: upsertResult.row.id,
|
||||
data: wire.dataOut(upsertResult.row.data),
|
||||
createdAt:
|
||||
upsertResult.row.createdAt instanceof Date
|
||||
? upsertResult.row.createdAt.toISOString()
|
||||
: upsertResult.row.createdAt,
|
||||
updatedAt:
|
||||
upsertResult.row.updatedAt instanceof Date
|
||||
? upsertResult.row.updatedAt.toISOString()
|
||||
: upsertResult.row.updatedAt,
|
||||
},
|
||||
operation: upsertResult.operation,
|
||||
message: `Row ${upsertResult.operation === 'update' ? 'updated' : 'inserted'} successfully`,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
const response = rowWriteErrorResponse(error)
|
||||
if (response) return response
|
||||
|
||||
logger.error(`[${requestId}] Error upserting row:`, error)
|
||||
return NextResponse.json({ error: 'Failed to upsert row' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns, permissionsMock, permissionsMockFns } from '@sim/testing'
|
||||
import { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockCreateTable,
|
||||
mockGetLimits,
|
||||
mockListTables,
|
||||
mockRunTableImport,
|
||||
mockRunDetached,
|
||||
MockTableConflictError,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCreateTable: vi.fn(),
|
||||
mockGetLimits: vi.fn(),
|
||||
mockListTables: vi.fn(),
|
||||
mockRunTableImport: vi.fn(),
|
||||
mockRunDetached: vi.fn(),
|
||||
MockTableConflictError: class extends Error {
|
||||
readonly code = 'TABLE_EXISTS' as const
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('import-id-123'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table', () => ({
|
||||
createTable: mockCreateTable,
|
||||
getWorkspaceTableLimits: mockGetLimits,
|
||||
listTables: mockListTables,
|
||||
sanitizeName: (name: string) => name.replace(/[^a-zA-Z0-9_]/g, '_'),
|
||||
TABLE_LIMITS: { MAX_TABLE_NAME_LENGTH: 128 },
|
||||
TableConflictError: MockTableConflictError,
|
||||
}))
|
||||
vi.mock('@/lib/table/import-runner', () => ({ runTableImport: mockRunTableImport }))
|
||||
vi.mock('@/lib/core/utils/background', () => ({
|
||||
runDetached: mockRunDetached.mockImplementation(
|
||||
(_label: string, work: () => Promise<unknown>) => {
|
||||
void work()
|
||||
}
|
||||
),
|
||||
}))
|
||||
vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
|
||||
|
||||
import { POST } from '@/app/api/table/import-async/route'
|
||||
|
||||
function makeRequest(body: unknown): NextRequest {
|
||||
return new NextRequest('http://localhost:3000/api/table/import-async', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
}
|
||||
|
||||
const validBody = {
|
||||
workspaceId: 'workspace-1',
|
||||
fileKey: 'workspace/workspace-1/123-data.csv',
|
||||
fileName: 'data.csv',
|
||||
}
|
||||
|
||||
describe('POST /api/table/import-async', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write')
|
||||
mockGetLimits.mockResolvedValue({ maxRowsPerTable: 1_000_000, maxTables: 50 })
|
||||
mockListTables.mockResolvedValue([])
|
||||
mockCreateTable.mockResolvedValue({ id: 'tbl_async', name: 'data' })
|
||||
mockRunTableImport.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('creates an importing table and kicks off the background import', async () => {
|
||||
const response = await POST(makeRequest(validBody))
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data).toEqual({ tableId: 'tbl_async', importId: 'import-id-123' })
|
||||
expect(mockCreateTable).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ jobStatus: 'running', jobType: 'import', jobId: 'import-id-123' }),
|
||||
expect.any(String)
|
||||
)
|
||||
expect(mockRunTableImport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ tableId: 'tbl_async', mode: 'create', delimiter: ',' })
|
||||
)
|
||||
})
|
||||
|
||||
it('uses a tab delimiter for .tsv files', async () => {
|
||||
await POST(makeRequest({ ...validBody, fileName: 'data.tsv' }))
|
||||
expect(mockRunTableImport).toHaveBeenCalledWith(expect.objectContaining({ delimiter: '\t' }))
|
||||
})
|
||||
|
||||
it('returns 400 for unsupported extensions', async () => {
|
||||
const response = await POST(makeRequest({ ...validBody, fileName: 'data.json' }))
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockCreateTable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await POST(makeRequest(validBody))
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('returns 403 without write permission', async () => {
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read')
|
||||
const response = await POST(makeRequest(validBody))
|
||||
expect(response.status).toBe(403)
|
||||
expect(mockCreateTable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when the body is missing required fields', async () => {
|
||||
const response = await POST(makeRequest({ workspaceId: 'workspace-1' }))
|
||||
expect(response.status).toBe(400)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,156 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { importTableAsyncContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
|
||||
import { runDetached } from '@/lib/core/utils/background'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { captureServerEvent } from '@/lib/posthog/server'
|
||||
import {
|
||||
createTable,
|
||||
deleteTable,
|
||||
getWorkspaceTableLimits,
|
||||
listTables,
|
||||
releaseJobClaim,
|
||||
sanitizeName,
|
||||
TABLE_LIMITS,
|
||||
TableConflictError,
|
||||
} from '@/lib/table'
|
||||
import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner'
|
||||
import { getUserSettings } from '@/lib/users/queries'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
|
||||
const logger = createLogger('TableImportAsync')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const userId = authResult.userId
|
||||
|
||||
const parsed = await parseRequest(importTableAsyncContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
const { workspaceId, fileKey, fileName, deleteSourceFile, timezone } = parsed.data.body
|
||||
|
||||
const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
|
||||
if (permission !== 'write' && permission !== 'admin') {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
// The fileKey is client-supplied — ensure it points at this workspace's storage prefix so a
|
||||
// caller can't import another workspace's uploaded object.
|
||||
if (!fileKey.startsWith(`workspace/${workspaceId}/`)) {
|
||||
return NextResponse.json({ error: 'Invalid file key for workspace' }, { status: 400 })
|
||||
}
|
||||
|
||||
const ext = fileName.split('.').pop()?.toLowerCase()
|
||||
if (ext !== 'csv' && ext !== 'tsv') {
|
||||
return NextResponse.json({ error: 'Only CSV and TSV files are supported' }, { status: 400 })
|
||||
}
|
||||
const delimiter = ext === 'tsv' ? '\t' : ','
|
||||
|
||||
const planLimits = await getWorkspaceTableLimits(workspaceId)
|
||||
const baseName = sanitizeName(fileName.replace(/\.[^.]+$/, ''), 'imported_table').slice(
|
||||
0,
|
||||
TABLE_LIMITS.MAX_TABLE_NAME_LENGTH
|
||||
)
|
||||
// Re-importing the same file shouldn't fail on a name collision — pick the next free
|
||||
// `name_2`, `name_3`, … (matching how "New table" auto-names), keeping under the cap.
|
||||
const existingNames = new Set(
|
||||
(await listTables(workspaceId, { scope: 'all' })).map((t) => t.name.toLowerCase())
|
||||
)
|
||||
let tableName = baseName
|
||||
for (let n = 2; existingNames.has(tableName.toLowerCase()); n++) {
|
||||
const suffix = `_${n}`
|
||||
tableName = `${baseName.slice(0, TABLE_LIMITS.MAX_TABLE_NAME_LENGTH - suffix.length)}${suffix}`
|
||||
}
|
||||
const importId = generateId()
|
||||
|
||||
// Placeholder schema satisfies createTable's validation; the import worker infers the
|
||||
// real columns from the file and overwrites it before any rows become visible.
|
||||
let table: Awaited<ReturnType<typeof createTable>>
|
||||
try {
|
||||
table = await createTable(
|
||||
{
|
||||
name: tableName,
|
||||
description: `Imported from ${fileName}`,
|
||||
schema: { columns: [{ name: 'column_1', type: 'string' }] },
|
||||
workspaceId,
|
||||
userId,
|
||||
maxTables: planLimits.maxTables,
|
||||
jobStatus: 'running',
|
||||
jobType: 'import',
|
||||
jobId: importId,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
} catch (error) {
|
||||
if (error instanceof TableConflictError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 409 })
|
||||
}
|
||||
if (error instanceof Error && error.message.includes('maximum table limit')) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const importPayload: TableImportPayload = {
|
||||
importId,
|
||||
tableId: table.id,
|
||||
workspaceId,
|
||||
userId,
|
||||
fileKey,
|
||||
fileName,
|
||||
delimiter,
|
||||
mode: 'create',
|
||||
deleteSourceFile,
|
||||
timezone: timezone ?? (await getUserSettings(userId)).timezone ?? 'UTC',
|
||||
}
|
||||
if (isTriggerDevEnabled) {
|
||||
// Trigger.dev runs the import outside the web container, so it survives app deploys.
|
||||
try {
|
||||
const [{ tableImportTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
|
||||
import('@/background/table-import'),
|
||||
import('@trigger.dev/sdk'),
|
||||
import('@/lib/core/async-jobs/region'),
|
||||
])
|
||||
await tasks.trigger<typeof tableImportTask>('table-import', importPayload, {
|
||||
tags: [`tableId:${table.id}`, `jobId:${importId}`],
|
||||
region: await resolveTriggerRegion(),
|
||||
})
|
||||
} catch (error) {
|
||||
// A failed dispatch must not leave a ghost `running` job holding the
|
||||
// table's one-write-job slot — nor, in create mode, the placeholder
|
||||
// table itself: the user never saw it, so archive it back out of the
|
||||
// workspace (no hard-delete surface exists; archived is invisible).
|
||||
await releaseJobClaim(table.id, importId).catch(() => {})
|
||||
await deleteTable(table.id, requestId).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
} else {
|
||||
runDetached('table-import', () => runTableImport(importPayload))
|
||||
}
|
||||
|
||||
captureServerEvent(
|
||||
userId,
|
||||
'table_import_started',
|
||||
{
|
||||
table_id: table.id,
|
||||
workspace_id: workspaceId,
|
||||
import_id: importId,
|
||||
file_type: ext,
|
||||
},
|
||||
{ groups: { workspace: workspaceId } }
|
||||
)
|
||||
|
||||
logger.info(`[${requestId}] Async CSV import started`, { tableId: table.id, importId, fileName })
|
||||
return NextResponse.json({ success: true, data: { tableId: table.id, importId } })
|
||||
})
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { hybridAuthMockFns, permissionsMock, permissionsMockFns } from '@sim/testing'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import type { NextRequest } from 'next/server'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockCreateTable, mockBatchInsertRows, mockDeleteTable, mockGetLimits } = vi.hoisted(() => ({
|
||||
mockCreateTable: vi.fn(),
|
||||
mockBatchInsertRows: vi.fn(),
|
||||
mockDeleteTable: vi.fn(),
|
||||
mockGetLimits: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/utils/id', () => ({
|
||||
generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
|
||||
generateShortId: vi.fn().mockReturnValue('short-id'),
|
||||
}))
|
||||
|
||||
// Mock only the DB-backed service/billing functions; the real `./import` helpers
|
||||
// (createCsvParser, inferSchemaFromCsv, coerceRowsForTable, …) run for real so the
|
||||
// streaming multipart + CSV pipeline is exercised end-to-end.
|
||||
vi.mock('@/lib/table/service', () => ({
|
||||
createTable: mockCreateTable,
|
||||
deleteTable: mockDeleteTable,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/table/rows/service', () => ({
|
||||
batchInsertRows: mockBatchInsertRows,
|
||||
}))
|
||||
vi.mock('@/lib/table/billing', () => ({ getWorkspaceTableLimits: mockGetLimits }))
|
||||
vi.mock('@/app/api/table/utils', async () => {
|
||||
const { NextResponse } = await import('next/server')
|
||||
return {
|
||||
normalizeColumn: (column: unknown) => column,
|
||||
csvProxyBodyCapResponse: () => null,
|
||||
multipartErrorResponse: (error: { code: string; message: string }) =>
|
||||
NextResponse.json(
|
||||
{ error: error.message },
|
||||
{ status: error.code === 'FILE_TOO_LARGE' ? 413 : 400 }
|
||||
),
|
||||
rowWriteErrorResponse: (error: unknown) => {
|
||||
const message = getErrorMessage(error)
|
||||
return message.includes('row limit')
|
||||
? NextResponse.json({ error: message }, { status: 400 })
|
||||
: null
|
||||
},
|
||||
}
|
||||
})
|
||||
vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
|
||||
|
||||
import { POST } from '@/app/api/table/import-csv/route'
|
||||
|
||||
type Part =
|
||||
| { name: string; value: string }
|
||||
| { name: string; filename: string; value: string; contentType?: string }
|
||||
|
||||
const BOUNDARY = '----testboundaryCSV'
|
||||
|
||||
function buildBody(parts: Part[]): Buffer {
|
||||
const segments: Buffer[] = []
|
||||
for (const part of parts) {
|
||||
let header = `--${BOUNDARY}\r\nContent-Disposition: form-data; name="${part.name}"`
|
||||
if ('filename' in part) {
|
||||
header += `; filename="${part.filename}"\r\nContent-Type: ${part.contentType ?? 'text/csv'}`
|
||||
}
|
||||
header += '\r\n\r\n'
|
||||
segments.push(Buffer.from(header, 'utf8'), Buffer.from(part.value, 'utf8'), Buffer.from('\r\n'))
|
||||
}
|
||||
segments.push(Buffer.from(`--${BOUNDARY}--\r\n`, 'utf8'))
|
||||
return Buffer.concat(segments)
|
||||
}
|
||||
|
||||
function makeRequest(parts: Part[], chunkSize?: number): NextRequest {
|
||||
const body = buildBody(parts)
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
if (chunkSize) {
|
||||
for (let i = 0; i < body.length; i += chunkSize) {
|
||||
controller.enqueue(new Uint8Array(body.subarray(i, i + chunkSize)))
|
||||
}
|
||||
} else {
|
||||
controller.enqueue(new Uint8Array(body))
|
||||
}
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
return {
|
||||
headers: new Headers({ 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }),
|
||||
body: stream,
|
||||
signal: undefined,
|
||||
} as unknown as NextRequest
|
||||
}
|
||||
|
||||
function csvWithRows(count: number): string {
|
||||
const lines = ['name,age']
|
||||
for (let i = 0; i < count; i++) lines.push(`Person${i},${20 + (i % 50)}`)
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
|
||||
function uploadParts(csv: string): Part[] {
|
||||
return [
|
||||
{ name: 'workspaceId', value: 'workspace-1' },
|
||||
{ name: 'file', filename: 'data.csv', value: csv },
|
||||
]
|
||||
}
|
||||
|
||||
describe('POST /api/table/import-csv', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
|
||||
success: true,
|
||||
userId: 'user-1',
|
||||
authType: 'session',
|
||||
})
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write')
|
||||
mockGetLimits.mockResolvedValue({ maxRowsPerTable: 1_000_000, maxTables: 50 })
|
||||
mockCreateTable.mockImplementation(async (data) => ({
|
||||
id: 'tbl_1',
|
||||
name: data.name,
|
||||
description: data.description ?? null,
|
||||
schema: data.schema,
|
||||
workspaceId: data.workspaceId,
|
||||
maxRows: data.maxRows,
|
||||
rowCount: 0,
|
||||
createdBy: 'user-1',
|
||||
archivedAt: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}))
|
||||
mockBatchInsertRows.mockImplementation(async ({ rows }: { rows: unknown[] }) =>
|
||||
rows.map((_, i) => ({ id: `row-${i}` }))
|
||||
)
|
||||
mockDeleteTable.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('streams a CSV upload into a new table and reports the row count', async () => {
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(mockCreateTable).toHaveBeenCalledTimes(1)
|
||||
expect(data.data.table.id).toBe('tbl_1')
|
||||
expect(data.data.table.rowCount).toBe(250)
|
||||
// 250 rows = a 100-row schema-sample batch + a 150-row remainder batch.
|
||||
expect(mockBatchInsertRows).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('parses a body delivered in tiny chunks (regression: missing final boundary)', async () => {
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(5)), 7))
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.data.table.rowCount).toBe(5)
|
||||
})
|
||||
|
||||
it('returns 400 for a CSV with no data rows', async () => {
|
||||
const response = await POST(makeRequest(uploadParts('name,age\n')))
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(400)
|
||||
expect(data.error).toMatch(/no data rows/i)
|
||||
expect(mockCreateTable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when the file precedes required fields', async () => {
|
||||
const response = await POST(
|
||||
makeRequest([
|
||||
{ name: 'file', filename: 'data.csv', value: csvWithRows(3) },
|
||||
{ name: 'workspaceId', value: 'workspace-1' },
|
||||
])
|
||||
)
|
||||
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockCreateTable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 when no file part is present', async () => {
|
||||
const response = await POST(makeRequest([{ name: 'workspaceId', value: 'workspace-1' }]))
|
||||
expect(response.status).toBe(400)
|
||||
expect(mockCreateTable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns 400 with the reason when an insert exceeds the plan row limit', async () => {
|
||||
mockBatchInsertRows.mockRejectedValueOnce(
|
||||
new Error('This table has reached its row limit (1,000 rows) on your current plan.')
|
||||
)
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(400)
|
||||
expect(data.error).toMatch(/row limit/)
|
||||
})
|
||||
|
||||
it('rolls back the created table when a batch insert fails mid-stream', async () => {
|
||||
mockBatchInsertRows
|
||||
.mockResolvedValueOnce(Array.from({ length: 100 }, () => ({ id: 'row' })))
|
||||
.mockRejectedValueOnce(new Error('insert boom'))
|
||||
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
|
||||
|
||||
expect(response.status).toBe(500)
|
||||
expect(mockDeleteTable).toHaveBeenCalledWith('tbl_1', expect.any(String))
|
||||
})
|
||||
|
||||
it('returns 401 when unauthenticated', async () => {
|
||||
hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(3))))
|
||||
expect(response.status).toBe(401)
|
||||
})
|
||||
|
||||
it('returns 403 without write permission', async () => {
|
||||
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read')
|
||||
const response = await POST(makeRequest(uploadParts(csvWithRows(3))))
|
||||
expect(response.status).toBe(403)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,244 @@
|
||||
import type { Readable } from 'node:stream'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { csvExtensionSchema, csvImportFormSchema } from '@/lib/api/contracts/tables'
|
||||
import { ianaTimezoneSchema } from '@/lib/api/contracts/user'
|
||||
import { getValidationErrorMessage } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import {
|
||||
batchInsertRows,
|
||||
CSV_MAX_BATCH_SIZE,
|
||||
CSV_MAX_FILE_SIZE_BYTES,
|
||||
CSV_SCHEMA_SAMPLE_SIZE,
|
||||
coerceRowsForTable,
|
||||
createCsvParser,
|
||||
createTable,
|
||||
deleteTable,
|
||||
getWorkspaceTableLimits,
|
||||
inferSchemaFromCsv,
|
||||
sanitizeName,
|
||||
TABLE_LIMITS,
|
||||
type TableDefinition,
|
||||
type TableSchema,
|
||||
} from '@/lib/table'
|
||||
import { getUserSettings } from '@/lib/users/queries'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
import {
|
||||
csvProxyBodyCapResponse,
|
||||
multipartErrorResponse,
|
||||
normalizeColumn,
|
||||
rowWriteErrorResponse,
|
||||
} from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableImportCSV')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
export const maxDuration = 300
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
let fileStream: Readable | undefined
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
const userId = authResult.userId
|
||||
|
||||
const oversize = csvProxyBodyCapResponse(request)
|
||||
if (oversize) return oversize
|
||||
|
||||
let parsed: Awaited<ReturnType<typeof readMultipart>>
|
||||
try {
|
||||
parsed = await readMultipart(request, {
|
||||
maxFileBytes: CSV_MAX_FILE_SIZE_BYTES,
|
||||
requiredFieldsBeforeFile: ['workspaceId'],
|
||||
signal: request.signal,
|
||||
})
|
||||
} catch (err) {
|
||||
if (isMultipartError(err)) return multipartErrorResponse(err)
|
||||
throw err
|
||||
}
|
||||
|
||||
const { fields, file } = parsed
|
||||
if (!file) {
|
||||
return NextResponse.json({ error: 'CSV file is required' }, { status: 400 })
|
||||
}
|
||||
fileStream = file.stream
|
||||
|
||||
const workspaceIdResult = csvImportFormSchema.shape.workspaceId.safeParse(fields.workspaceId)
|
||||
if (!workspaceIdResult.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(workspaceIdResult.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const workspaceId = workspaceIdResult.data
|
||||
|
||||
const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
|
||||
if (permission !== 'write' && permission !== 'admin') {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
|
||||
let timezone = (await getUserSettings(userId)).timezone ?? 'UTC'
|
||||
if (fields.timezone) {
|
||||
const timezoneResult = ianaTimezoneSchema.safeParse(fields.timezone)
|
||||
if (!timezoneResult.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(timezoneResult.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
timezone = timezoneResult.data
|
||||
}
|
||||
|
||||
const ext = file.filename.split('.').pop()?.toLowerCase()
|
||||
const extensionResult = csvExtensionSchema.safeParse(ext)
|
||||
if (!extensionResult.success) {
|
||||
return NextResponse.json(
|
||||
{ error: getValidationErrorMessage(extensionResult.error) },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
const delimiter = extensionResult.data === 'tsv' ? '\t' : ','
|
||||
|
||||
const parser = createCsvParser(delimiter)
|
||||
// `.pipe` doesn't forward source errors; forward them so the iterator throws.
|
||||
file.stream.on('error', (err) => parser.destroy(err))
|
||||
file.stream.pipe(parser)
|
||||
|
||||
interface ImportState {
|
||||
table: TableDefinition
|
||||
schema: TableSchema
|
||||
headerToColumn: Map<string, string>
|
||||
}
|
||||
|
||||
const insertRows = async (
|
||||
rows: Record<string, unknown>[],
|
||||
state: ImportState,
|
||||
currentRowCount: number
|
||||
) => {
|
||||
if (rows.length === 0) return 0
|
||||
const coerced = coerceRowsForTable(rows, state.schema, state.headerToColumn, { timezone })
|
||||
const result = await batchInsertRows(
|
||||
{ tableId: state.table.id, rows: coerced, workspaceId, userId },
|
||||
// The created table's rowCount is frozen at 0; pass the running total so the
|
||||
// per-batch capacity check sees cumulative rows, not an always-empty table.
|
||||
{ ...state.table, rowCount: currentRowCount },
|
||||
generateId().slice(0, 8)
|
||||
)
|
||||
return result.length
|
||||
}
|
||||
|
||||
/** Infer the schema from the buffered sample and create the (empty) table. */
|
||||
const buildTable = async (sampleRows: Record<string, unknown>[]): Promise<ImportState> => {
|
||||
const inferred = inferSchemaFromCsv(Object.keys(sampleRows[0]), sampleRows)
|
||||
const schema: TableSchema = { columns: inferred.columns.map(normalizeColumn) }
|
||||
const planLimits = await getWorkspaceTableLimits(workspaceId)
|
||||
const tableName = sanitizeName(file.filename.replace(/\.[^.]+$/, ''), 'imported_table').slice(
|
||||
0,
|
||||
TABLE_LIMITS.MAX_TABLE_NAME_LENGTH
|
||||
)
|
||||
const table = await createTable(
|
||||
{
|
||||
name: tableName,
|
||||
description: `Imported from ${file.filename}`,
|
||||
schema,
|
||||
workspaceId,
|
||||
userId,
|
||||
maxTables: planLimits.maxTables,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
// Coerce against the *created* schema so rows key by the ids `createTable`
|
||||
// assigned (the local `schema` is the id-less inferred one).
|
||||
return { table, schema: table.schema, headerToColumn: inferred.headerToColumn }
|
||||
}
|
||||
|
||||
let state: ImportState | null = null
|
||||
let inserted = 0
|
||||
const sample: Record<string, unknown>[] = []
|
||||
let batch: Record<string, unknown>[] = []
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
|
||||
if (!state) {
|
||||
sample.push(record)
|
||||
if (sample.length >= CSV_SCHEMA_SAMPLE_SIZE) {
|
||||
state = await buildTable(sample)
|
||||
inserted += await insertRows(sample, state, inserted)
|
||||
}
|
||||
continue
|
||||
}
|
||||
batch.push(record)
|
||||
if (batch.length >= CSV_MAX_BATCH_SIZE) {
|
||||
inserted += await insertRows(batch, state, inserted)
|
||||
batch = []
|
||||
}
|
||||
}
|
||||
|
||||
if (!state) {
|
||||
if (sample.length === 0) {
|
||||
return NextResponse.json({ error: 'CSV file has no data rows' }, { status: 400 })
|
||||
}
|
||||
state = await buildTable(sample)
|
||||
inserted += await insertRows(sample, state, inserted)
|
||||
} else {
|
||||
inserted += await insertRows(batch, state, inserted)
|
||||
}
|
||||
} catch (streamError) {
|
||||
if (state) await deleteTable(state.table.id, requestId).catch(() => {})
|
||||
throw streamError
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] CSV imported`, {
|
||||
tableId: state.table.id,
|
||||
fileName: file.filename,
|
||||
columns: state.schema.columns.length,
|
||||
rows: inserted,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
table: {
|
||||
id: state.table.id,
|
||||
name: state.table.name,
|
||||
description: state.table.description,
|
||||
schema: state.schema,
|
||||
rowCount: inserted,
|
||||
},
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isMultipartError(error)) return multipartErrorResponse(error)
|
||||
|
||||
logger.error(`[${requestId}] CSV import failed:`, error)
|
||||
|
||||
// Row-write failures (e.g. the plan row-limit check) map to a 400 with the real reason.
|
||||
const rowWriteError = rowWriteErrorResponse(error)
|
||||
if (rowWriteError) return rowWriteError
|
||||
|
||||
const message = toError(error).message
|
||||
const isClientError =
|
||||
message.includes('maximum table limit') ||
|
||||
message.includes('CSV file has no') ||
|
||||
message.includes('Invalid table name') ||
|
||||
message.includes('Invalid schema') ||
|
||||
message.includes('already exists')
|
||||
|
||||
return NextResponse.json(
|
||||
{ error: isClientError ? message : 'Failed to import CSV' },
|
||||
{ status: isClientError ? 400 : 500 }
|
||||
)
|
||||
} finally {
|
||||
fileStream?.destroy()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { listTableJobsContract } from '@/lib/api/contracts/tables'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { listWorkspaceExportJobs } from '@/lib/table/jobs/service'
|
||||
import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils'
|
||||
|
||||
const logger = createLogger('TableJobsAPI')
|
||||
|
||||
export const runtime = 'nodejs'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
/**
|
||||
* GET /api/table/jobs?workspaceId=…&type=export
|
||||
*
|
||||
* Lists a workspace's export jobs (running + recently finished) for the header tray. Exports are
|
||||
* excluded from the table-level job derivation, so the tray reads them here.
|
||||
*/
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(listTableJobsContract, request, {})
|
||||
if (!parsed.success) return parsed.response
|
||||
const { workspaceId } = parsed.data.query
|
||||
|
||||
const { hasAccess } = await checkWorkspaceAccess(workspaceId, authResult.userId)
|
||||
if (!hasAccess) {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
|
||||
const jobs = await listWorkspaceExportJobs(workspaceId)
|
||||
logger.info(`[${requestId}] Listed ${jobs.length} export jobs`, { workspaceId })
|
||||
return NextResponse.json({ success: true, data: { jobs } })
|
||||
})
|
||||
@@ -0,0 +1,242 @@
|
||||
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { createTableContract, listTablesQuerySchema } from '@/lib/api/contracts/tables'
|
||||
import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/server/validation'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { captureServerEvent } from '@/lib/posthog/server'
|
||||
import {
|
||||
createTable,
|
||||
getWorkspaceTableLimits,
|
||||
listTables,
|
||||
type TableSchema,
|
||||
type TableScope,
|
||||
} from '@/lib/table'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
import { normalizeColumn } from '@/app/api/table/utils'
|
||||
|
||||
const logger = createLogger('TableAPI')
|
||||
|
||||
interface WorkspaceAccessResult {
|
||||
hasAccess: boolean
|
||||
canWrite: boolean
|
||||
}
|
||||
|
||||
async function checkWorkspaceAccess(
|
||||
workspaceId: string,
|
||||
userId: string
|
||||
): Promise<WorkspaceAccessResult> {
|
||||
const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
|
||||
|
||||
if (permission === null) {
|
||||
return { hasAccess: false, canWrite: false }
|
||||
}
|
||||
|
||||
const canWrite = permission === 'admin' || permission === 'write'
|
||||
return { hasAccess: true, canWrite }
|
||||
}
|
||||
|
||||
/** POST /api/table - Creates a new user-defined table. */
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const parsed = await parseRequest(
|
||||
createTableContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
validationErrorResponse: (error) => validationErrorResponse(error),
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const params = parsed.data.body
|
||||
|
||||
const { hasAccess, canWrite } = await checkWorkspaceAccess(
|
||||
params.workspaceId,
|
||||
authResult.userId
|
||||
)
|
||||
|
||||
if (!hasAccess || !canWrite) {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
|
||||
const planLimits = await getWorkspaceTableLimits(params.workspaceId)
|
||||
|
||||
const normalizedSchema: TableSchema = {
|
||||
columns: params.schema.columns.map(normalizeColumn),
|
||||
}
|
||||
|
||||
const table = await createTable(
|
||||
{
|
||||
name: params.name,
|
||||
description: params.description,
|
||||
schema: normalizedSchema,
|
||||
workspaceId: params.workspaceId,
|
||||
userId: authResult.userId,
|
||||
maxTables: planLimits.maxTables,
|
||||
initialRowCount: params.initialRowCount,
|
||||
},
|
||||
requestId
|
||||
)
|
||||
|
||||
captureServerEvent(
|
||||
authResult.userId,
|
||||
'table_created',
|
||||
{
|
||||
table_id: table.id,
|
||||
workspace_id: params.workspaceId,
|
||||
column_count: params.schema.columns.length,
|
||||
},
|
||||
{
|
||||
groups: { workspace: params.workspaceId },
|
||||
setOnce: { first_table_created_at: new Date().toISOString() },
|
||||
}
|
||||
)
|
||||
|
||||
recordAudit({
|
||||
workspaceId: params.workspaceId,
|
||||
actorId: authResult.userId,
|
||||
actorName: authResult.userName ?? undefined,
|
||||
actorEmail: authResult.userEmail ?? undefined,
|
||||
action: AuditAction.TABLE_CREATED,
|
||||
resourceType: AuditResourceType.TABLE,
|
||||
resourceId: table.id,
|
||||
resourceName: table.name,
|
||||
description: `Created table "${table.name}"`,
|
||||
request,
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
table: {
|
||||
id: table.id,
|
||||
name: table.name,
|
||||
description: table.description,
|
||||
schema: {
|
||||
columns: (table.schema as TableSchema).columns.map(normalizeColumn),
|
||||
},
|
||||
rowCount: table.rowCount,
|
||||
maxRows: table.maxRows,
|
||||
createdAt:
|
||||
table.createdAt instanceof Date
|
||||
? table.createdAt.toISOString()
|
||||
: String(table.createdAt),
|
||||
updatedAt:
|
||||
table.updatedAt instanceof Date
|
||||
? table.updatedAt.toISOString()
|
||||
: String(table.updatedAt),
|
||||
},
|
||||
message: 'Table created successfully',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
if (error.message.includes('maximum table limit')) {
|
||||
return NextResponse.json({ error: error.message }, { status: 403 })
|
||||
}
|
||||
if (
|
||||
error.message.includes('Invalid table name') ||
|
||||
error.message.includes('Invalid schema') ||
|
||||
error.message.includes('already exists')
|
||||
) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error creating table:`, error)
|
||||
return NextResponse.json({ error: 'Failed to create table' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
/** GET /api/table - Lists all tables in a workspace. */
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
try {
|
||||
const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!authResult.success || !authResult.userId) {
|
||||
return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const workspaceId = searchParams.get('workspaceId')
|
||||
const scope = searchParams.get('scope')
|
||||
|
||||
const validation = listTablesQuerySchema.safeParse({
|
||||
workspaceId,
|
||||
scope: scope ?? undefined,
|
||||
})
|
||||
if (!validation.success) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Validation error', details: validation.error.issues },
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const params = validation.data
|
||||
|
||||
const { hasAccess } = await checkWorkspaceAccess(params.workspaceId, authResult.userId)
|
||||
|
||||
if (!hasAccess) {
|
||||
return NextResponse.json({ error: 'Access denied' }, { status: 403 })
|
||||
}
|
||||
|
||||
const tables = await listTables(params.workspaceId, { scope: params.scope as TableScope })
|
||||
|
||||
logger.info(`[${requestId}] Listed ${tables.length} tables in workspace ${params.workspaceId}`)
|
||||
|
||||
const responseTables = tables.map((t) => {
|
||||
const schemaData = t.schema as TableSchema
|
||||
return {
|
||||
id: t.id,
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
schema: {
|
||||
columns: schemaData.columns.map(normalizeColumn),
|
||||
},
|
||||
rowCount: t.rowCount,
|
||||
maxRows: t.maxRows,
|
||||
workspaceId: t.workspaceId,
|
||||
createdBy: t.createdBy,
|
||||
createdAt: t.createdAt instanceof Date ? t.createdAt.toISOString() : String(t.createdAt),
|
||||
updatedAt: t.updatedAt instanceof Date ? t.updatedAt.toISOString() : String(t.updatedAt),
|
||||
archivedAt:
|
||||
t.archivedAt instanceof Date
|
||||
? t.archivedAt.toISOString()
|
||||
: t.archivedAt
|
||||
? String(t.archivedAt)
|
||||
: null,
|
||||
jobStatus: t.jobStatus ?? null,
|
||||
jobId: t.jobId ?? null,
|
||||
jobType: t.jobType ?? null,
|
||||
jobError: t.jobError ?? null,
|
||||
jobRowsProcessed: t.jobRowsProcessed ?? 0,
|
||||
}
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
tables: responseTables,
|
||||
totalCount: tables.length,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
if (isZodError(error)) {
|
||||
return validationErrorResponse(error)
|
||||
}
|
||||
|
||||
logger.error(`[${requestId}] Error listing tables:`, error)
|
||||
return NextResponse.json({ error: 'Failed to list tables' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid'
|
||||
import type { Filter, RowData, Sort, TableSchema } from '@/lib/table'
|
||||
import {
|
||||
buildIdByName,
|
||||
buildNameById,
|
||||
filterNamesToIds,
|
||||
rowDataIdToName,
|
||||
rowDataNameToId,
|
||||
sortNamesToIds,
|
||||
} from '@/lib/table'
|
||||
|
||||
export interface RowWireTranslators {
|
||||
/** Inbound row data: wire keys → storage column ids. */
|
||||
dataIn: (data: RowData) => RowData
|
||||
/** Outbound row data: storage column ids → wire keys. */
|
||||
dataOut: (data: RowData) => RowData
|
||||
/** Inbound filter: wire field refs → storage column ids. */
|
||||
filterIn: (filter: Filter) => Filter
|
||||
/** Inbound sort: wire field refs → storage column ids. */
|
||||
sortIn: (sort: Sort) => Sort
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire-keying translators for the internal table row routes, which serve two
|
||||
* caller kinds: the first-party UI (session auth) speaks stable column ids and
|
||||
* passes through untouched, while workflow tool executions (internal JWT) speak
|
||||
* column names — tool enrichment surfaces names to the LLM — and translate
|
||||
* name↔id at this boundary, mirroring the public v1 routes.
|
||||
*/
|
||||
export function rowWireTranslators(
|
||||
authType: AuthTypeValue | undefined,
|
||||
schema: TableSchema
|
||||
): RowWireTranslators {
|
||||
if (authType !== AuthType.INTERNAL_JWT) {
|
||||
const identity = <T>(value: T): T => value
|
||||
return { dataIn: identity, dataOut: identity, filterIn: identity, sortIn: identity }
|
||||
}
|
||||
const idByName = buildIdByName(schema)
|
||||
const nameById = buildNameById(schema)
|
||||
return {
|
||||
dataIn: (data) => rowDataNameToId(data, idByName),
|
||||
dataOut: (data) => rowDataIdToName(data, nameById),
|
||||
filterIn: (filter) => filterNamesToIds(filter, idByName),
|
||||
sortIn: (sort) => sortNamesToIds(sort, idByName),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { TableRowLimitError } from '@/lib/table/billing'
|
||||
import { rootErrorMessage, rowWriteErrorResponse } from '@/app/api/table/utils'
|
||||
|
||||
/** Mimics drizzle's DrizzleQueryError: message is the failed SQL, real error on `cause`. */
|
||||
function wrapLikeDrizzle(cause: Error): Error {
|
||||
return new Error('Failed query: insert into "user_table_rows" ...', { cause })
|
||||
}
|
||||
|
||||
describe('rootErrorMessage', () => {
|
||||
it('returns the message of a plain error', () => {
|
||||
expect(rootErrorMessage(new Error('Schema validation failed: bad'))).toBe(
|
||||
'Schema validation failed: bad'
|
||||
)
|
||||
})
|
||||
|
||||
it('unwraps the cause chain to the deepest error', () => {
|
||||
const root = new Error('Value for column "email" must be unique')
|
||||
expect(rootErrorMessage(wrapLikeDrizzle(root))).toBe(root.message)
|
||||
})
|
||||
|
||||
it('stringifies non-Error values', () => {
|
||||
expect(rootErrorMessage('boom')).toBe('boom')
|
||||
})
|
||||
})
|
||||
|
||||
describe('rowWriteErrorResponse', () => {
|
||||
it('passes the plan row-limit error through as a 400', async () => {
|
||||
const response = rowWriteErrorResponse(new TableRowLimitError(10000))
|
||||
expect(response?.status).toBe(400)
|
||||
const body = await response?.json()
|
||||
expect(body.error).toBe(
|
||||
'This table has reached its row limit (10,000 rows) on your current plan.'
|
||||
)
|
||||
})
|
||||
|
||||
it('passes known validation messages through as 400', async () => {
|
||||
const response = rowWriteErrorResponse(new Error('Value for column "email" must be unique'))
|
||||
expect(response?.status).toBe(400)
|
||||
const body = await response?.json()
|
||||
expect(body.error).toBe('Value for column "email" must be unique')
|
||||
})
|
||||
|
||||
it('matches per-row batch validation messages', () => {
|
||||
expect(rowWriteErrorResponse(new Error('Row 3: name is required'))?.status).toBe(400)
|
||||
})
|
||||
|
||||
it('returns null for unknown errors so callers keep their generic 500', () => {
|
||||
expect(rowWriteErrorResponse(new Error('connection refused'))).toBeNull()
|
||||
expect(rowWriteErrorResponse(wrapLikeDrizzle(new Error('deadlock detected')))).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,283 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { permissionSatisfies } from '@sim/platform-authz/workspace'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { NextResponse } from 'next/server'
|
||||
import {
|
||||
createTableColumnBodySchema,
|
||||
deleteTableColumnBodySchema,
|
||||
updateTableColumnBodySchema,
|
||||
} from '@/lib/api/contracts/tables'
|
||||
import type { MultipartError } from '@/lib/core/utils/multipart'
|
||||
import type { ColumnDefinition, Filter, TableDefinition } from '@/lib/table'
|
||||
import { buildFilterClause, getTableById, TableQueryValidationError } from '@/lib/table'
|
||||
import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants'
|
||||
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
|
||||
|
||||
/**
|
||||
* Validates a `filter` against the table's column schema, returning a 400 response on a bad field
|
||||
* (or `null` when the filter is valid or absent). Shared by the routes that accept a filter
|
||||
* (`delete-async`, `columns/run`) so a bad field fails fast with a clear message.
|
||||
*/
|
||||
export function tableFilterError(
|
||||
filter: Filter | undefined,
|
||||
columns: ColumnDefinition[]
|
||||
): NextResponse | null {
|
||||
if (!filter) return null
|
||||
try {
|
||||
buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, columns)
|
||||
return null
|
||||
} catch (error) {
|
||||
if (error instanceof TableQueryValidationError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const logger = createLogger('TableUtils')
|
||||
|
||||
/**
|
||||
* Deepest `Error` message in the cause chain. Drizzle wraps DB errors in a
|
||||
* `DrizzleQueryError` whose own message is just the failed SQL — substring
|
||||
* classification must look at the root cause.
|
||||
*/
|
||||
export function rootErrorMessage(error: unknown): string {
|
||||
let current: unknown = error
|
||||
while (current instanceof Error && current.cause instanceof Error) {
|
||||
current = current.cause
|
||||
}
|
||||
return toError(current).message
|
||||
}
|
||||
|
||||
/**
|
||||
* Known user-facing row-write failures (service validation + the best-effort
|
||||
* plan row-limit check). Anything outside this list stays a generic 500 —
|
||||
* unknown errors can carry SQL/internals that don't belong in a toast.
|
||||
*/
|
||||
const ROW_WRITE_ERROR_PATTERNS = [
|
||||
'row limit',
|
||||
'Insufficient capacity',
|
||||
'Schema validation',
|
||||
'must be unique',
|
||||
'must be valid',
|
||||
'must be string',
|
||||
'must be number',
|
||||
'must be boolean',
|
||||
'unique column',
|
||||
'Unique constraint violation',
|
||||
'Row size exceeds',
|
||||
'conflictTarget',
|
||||
'Upsert requires',
|
||||
'Rows not found',
|
||||
'Filter is required',
|
||||
] as const
|
||||
|
||||
/**
|
||||
* Maps a known user-facing row-write failure to a 400 carrying the real message
|
||||
* (so client toasts can show the actual reason); `null` when the error is
|
||||
* unrecognized and the caller should log it and return its generic 500.
|
||||
*/
|
||||
export function rowWriteErrorResponse(error: unknown): NextResponse | null {
|
||||
const message = rootErrorMessage(error)
|
||||
|
||||
if (ROW_WRITE_ERROR_PATTERNS.some((p) => message.includes(p)) || /^Row .+?:/.test(message)) {
|
||||
return NextResponse.json({ error: message }, { status: 400 })
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Next.js buffers the request body for the proxy and silently truncates it past this
|
||||
* size (`experimental.proxyClientMaxBodySize`, default 10MB). The synchronous CSV
|
||||
* import routes reject bodies over the cap up front; larger files use the async
|
||||
* direct-to-storage path instead.
|
||||
*/
|
||||
export const CSV_IMPORT_PROXY_BODY_CAP_BYTES = 10 * 1024 * 1024
|
||||
|
||||
/** 413 response when a synchronous CSV upload would exceed (and be truncated at) the proxy cap; `null` otherwise. */
|
||||
export function csvProxyBodyCapResponse(request: { headers: Headers }): NextResponse | null {
|
||||
const contentLength = Number(request.headers.get('content-length') ?? 0)
|
||||
if (contentLength > CSV_IMPORT_PROXY_BODY_CAP_BYTES) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
'File too large to import through the server. Files over 10MB import in the background.',
|
||||
},
|
||||
{ status: 413 }
|
||||
)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Maps a {@link MultipartError} from the streaming CSV parser to its HTTP response. */
|
||||
export function multipartErrorResponse(error: MultipartError): NextResponse {
|
||||
if (error.code === 'FILE_TOO_LARGE') {
|
||||
return NextResponse.json({ error: 'CSV import file exceeds maximum size' }, { status: 413 })
|
||||
}
|
||||
const message =
|
||||
error.code === 'NO_FILE' ? 'CSV file is required' : `Invalid CSV upload: ${error.message}`
|
||||
return NextResponse.json({ error: message }, { status: 400 })
|
||||
}
|
||||
|
||||
interface TableAccessResult {
|
||||
hasAccess: true
|
||||
table: TableDefinition
|
||||
}
|
||||
|
||||
interface TableAccessDenied {
|
||||
hasAccess: false
|
||||
notFound?: boolean
|
||||
reason?: string
|
||||
}
|
||||
|
||||
export type TableAccessCheck = TableAccessResult | TableAccessDenied
|
||||
|
||||
export type AccessResult = { ok: true; table: TableDefinition } | { ok: false; status: 404 | 403 }
|
||||
|
||||
interface ApiErrorResponse {
|
||||
error: string
|
||||
details?: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a user has read access to a table.
|
||||
* Read access requires any workspace permission (read, write, or admin).
|
||||
*/
|
||||
async function checkTableAccess(tableId: string, userId: string): Promise<TableAccessCheck> {
|
||||
const table = await getTableById(tableId)
|
||||
|
||||
if (!table) {
|
||||
return { hasAccess: false, notFound: true }
|
||||
}
|
||||
|
||||
const userPermission = await getUserEntityPermissions(userId, 'workspace', table.workspaceId)
|
||||
if (userPermission !== null) {
|
||||
return { hasAccess: true, table }
|
||||
}
|
||||
|
||||
return { hasAccess: false, reason: 'User does not have access to this table' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a user has write access to a table.
|
||||
* Write access requires write or admin workspace permission.
|
||||
*/
|
||||
async function checkTableWriteAccess(tableId: string, userId: string): Promise<TableAccessCheck> {
|
||||
const table = await getTableById(tableId)
|
||||
|
||||
if (!table) {
|
||||
return { hasAccess: false, notFound: true }
|
||||
}
|
||||
|
||||
const userPermission = await getUserEntityPermissions(userId, 'workspace', table.workspaceId)
|
||||
if (permissionSatisfies(userPermission, 'write')) {
|
||||
return { hasAccess: true, table }
|
||||
}
|
||||
|
||||
return { hasAccess: false, reason: 'User does not have write access to this table' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Access check returning `{ ok, table }` or `{ ok: false, status }`.
|
||||
* Uses workspace permissions only.
|
||||
*/
|
||||
export async function checkAccess(
|
||||
tableId: string,
|
||||
userId: string,
|
||||
level: 'read' | 'write' | 'admin' = 'read'
|
||||
): Promise<AccessResult> {
|
||||
const table = await getTableById(tableId)
|
||||
|
||||
if (!table) {
|
||||
return { ok: false, status: 404 }
|
||||
}
|
||||
|
||||
const permission = await getUserEntityPermissions(userId, 'workspace', table.workspaceId)
|
||||
const hasAccess = permissionSatisfies(permission, level)
|
||||
|
||||
return hasAccess ? { ok: true, table } : { ok: false, status: 403 }
|
||||
}
|
||||
|
||||
export function accessError(
|
||||
result: { ok: false; status: 404 | 403 },
|
||||
requestId: string,
|
||||
context?: string
|
||||
): NextResponse {
|
||||
const message = result.status === 404 ? 'Table not found' : 'Access denied'
|
||||
logger.warn(`[${requestId}] ${message}${context ? `: ${context}` : ''}`)
|
||||
return NextResponse.json({ error: message }, { status: result.status })
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a TableAccessDenied result to an appropriate HTTP response.
|
||||
* Use with checkTableAccess or checkTableWriteAccess.
|
||||
*/
|
||||
export function tableAccessError(
|
||||
result: TableAccessDenied,
|
||||
requestId: string,
|
||||
context?: string
|
||||
): NextResponse {
|
||||
const status = result.notFound ? 404 : 403
|
||||
const message = result.notFound ? 'Table not found' : (result.reason ?? 'Access denied')
|
||||
logger.warn(`[${requestId}] ${message}${context ? `: ${context}` : ''}`)
|
||||
return NextResponse.json({ error: message }, { status })
|
||||
}
|
||||
|
||||
async function verifyTableWorkspace(tableId: string, workspaceId: string): Promise<boolean> {
|
||||
const table = await getTableById(tableId)
|
||||
return table?.workspaceId === workspaceId
|
||||
}
|
||||
|
||||
export function errorResponse(
|
||||
message: string,
|
||||
status: number,
|
||||
details?: unknown
|
||||
): NextResponse<ApiErrorResponse> {
|
||||
const body: ApiErrorResponse = { error: message }
|
||||
if (details !== undefined) {
|
||||
body.details = details
|
||||
}
|
||||
return NextResponse.json(body, { status })
|
||||
}
|
||||
|
||||
export function badRequestResponse(message: string, details?: unknown) {
|
||||
return errorResponse(message, 400, details)
|
||||
}
|
||||
|
||||
export function unauthorizedResponse(message = 'Authentication required') {
|
||||
return errorResponse(message, 401)
|
||||
}
|
||||
|
||||
export function forbiddenResponse(message = 'Access denied') {
|
||||
return errorResponse(message, 403)
|
||||
}
|
||||
|
||||
export function notFoundResponse(message = 'Resource not found') {
|
||||
return errorResponse(message, 404)
|
||||
}
|
||||
|
||||
export function serverErrorResponse(message = 'Internal server error') {
|
||||
return errorResponse(message, 500)
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-exports from `lib/api/contracts/tables` so existing routes that import
|
||||
* these names keep working while sharing a single source of truth.
|
||||
*/
|
||||
export const CreateColumnSchema = createTableColumnBodySchema
|
||||
export const UpdateColumnSchema = updateTableColumnBodySchema
|
||||
export const DeleteColumnSchema = deleteTableColumnBodySchema
|
||||
|
||||
export function normalizeColumn(col: ColumnDefinition): ColumnDefinition {
|
||||
return {
|
||||
// Preserve the stable column id — it's the row-data storage key, so dropping
|
||||
// it makes clients fall back to `name` and miss id-keyed cell values.
|
||||
...(col.id ? { id: col.id } : {}),
|
||||
name: col.name,
|
||||
type: col.type,
|
||||
required: col.required ?? false,
|
||||
unique: col.unique ?? false,
|
||||
...(col.workflowGroupId ? { workflowGroupId: col.workflowGroupId } : {}),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user