chore: import upstream snapshot with attribution

apps/sim/app/api/table/import-csv/route.test.ts

@@ -0,0 +1,218 @@
+/**
+ * @vitest-environment node
+ */
+import { hybridAuthMockFns, permissionsMock, permissionsMockFns } from '@sim/testing'
+import { getErrorMessage } from '@sim/utils/errors'
+import type { NextRequest } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockCreateTable, mockBatchInsertRows, mockDeleteTable, mockGetLimits } = vi.hoisted(() => ({
+  mockCreateTable: vi.fn(),
+  mockBatchInsertRows: vi.fn(),
+  mockDeleteTable: vi.fn(),
+  mockGetLimits: vi.fn(),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
+  generateShortId: vi.fn().mockReturnValue('short-id'),
+}))
+
+// Mock only the DB-backed service/billing functions; the real `./import` helpers
+// (createCsvParser, inferSchemaFromCsv, coerceRowsForTable, …) run for real so the
+// streaming multipart + CSV pipeline is exercised end-to-end.
+vi.mock('@/lib/table/service', () => ({
+  createTable: mockCreateTable,
+  deleteTable: mockDeleteTable,
+}))
+
+vi.mock('@/lib/table/rows/service', () => ({
+  batchInsertRows: mockBatchInsertRows,
+}))
+vi.mock('@/lib/table/billing', () => ({ getWorkspaceTableLimits: mockGetLimits }))
+vi.mock('@/app/api/table/utils', async () => {
+  const { NextResponse } = await import('next/server')
+  return {
+    normalizeColumn: (column: unknown) => column,
+    csvProxyBodyCapResponse: () => null,
+    multipartErrorResponse: (error: { code: string; message: string }) =>
+      NextResponse.json(
+        { error: error.message },
+        { status: error.code === 'FILE_TOO_LARGE' ? 413 : 400 }
+      ),
+    rowWriteErrorResponse: (error: unknown) => {
+      const message = getErrorMessage(error)
+      return message.includes('row limit')
+        ? NextResponse.json({ error: message }, { status: 400 })
+        : null
+    },
+  }
+})
+vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock)
+
+import { POST } from '@/app/api/table/import-csv/route'
+
+type Part =
+  | { name: string; value: string }
+  | { name: string; filename: string; value: string; contentType?: string }
+
+const BOUNDARY = '----testboundaryCSV'
+
+function buildBody(parts: Part[]): Buffer {
+  const segments: Buffer[] = []
+  for (const part of parts) {
+    let header = `--${BOUNDARY}\r\nContent-Disposition: form-data; name="${part.name}"`
+    if ('filename' in part) {
+      header += `; filename="${part.filename}"\r\nContent-Type: ${part.contentType ?? 'text/csv'}`
+    }
+    header += '\r\n\r\n'
+    segments.push(Buffer.from(header, 'utf8'), Buffer.from(part.value, 'utf8'), Buffer.from('\r\n'))
+  }
+  segments.push(Buffer.from(`--${BOUNDARY}--\r\n`, 'utf8'))
+  return Buffer.concat(segments)
+}
+
+function makeRequest(parts: Part[], chunkSize?: number): NextRequest {
+  const body = buildBody(parts)
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      if (chunkSize) {
+        for (let i = 0; i < body.length; i += chunkSize) {
+          controller.enqueue(new Uint8Array(body.subarray(i, i + chunkSize)))
+        }
+      } else {
+        controller.enqueue(new Uint8Array(body))
+      }
+      controller.close()
+    },
+  })
+  return {
+    headers: new Headers({ 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }),
+    body: stream,
+    signal: undefined,
+  } as unknown as NextRequest
+}
+
+function csvWithRows(count: number): string {
+  const lines = ['name,age']
+  for (let i = 0; i < count; i++) lines.push(`Person${i},${20 + (i % 50)}`)
+  return `${lines.join('\n')}\n`
+}
+
+function uploadParts(csv: string): Part[] {
+  return [
+    { name: 'workspaceId', value: 'workspace-1' },
+    { name: 'file', filename: 'data.csv', value: csv },
+  ]
+}
+
+describe('POST /api/table/import-csv', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
+      success: true,
+      userId: 'user-1',
+      authType: 'session',
+    })
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write')
+    mockGetLimits.mockResolvedValue({ maxRowsPerTable: 1_000_000, maxTables: 50 })
+    mockCreateTable.mockImplementation(async (data) => ({
+      id: 'tbl_1',
+      name: data.name,
+      description: data.description ?? null,
+      schema: data.schema,
+      workspaceId: data.workspaceId,
+      maxRows: data.maxRows,
+      rowCount: 0,
+      createdBy: 'user-1',
+      archivedAt: null,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    }))
+    mockBatchInsertRows.mockImplementation(async ({ rows }: { rows: unknown[] }) =>
+      rows.map((_, i) => ({ id: `row-${i}` }))
+    )
+    mockDeleteTable.mockResolvedValue(undefined)
+  })
+
+  it('streams a CSV upload into a new table and reports the row count', async () => {
+    const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
+    const data = await response.json()
+
+    expect(response.status).toBe(200)
+    expect(mockCreateTable).toHaveBeenCalledTimes(1)
+    expect(data.data.table.id).toBe('tbl_1')
+    expect(data.data.table.rowCount).toBe(250)
+    // 250 rows = a 100-row schema-sample batch + a 150-row remainder batch.
+    expect(mockBatchInsertRows).toHaveBeenCalledTimes(2)
+  })
+
+  it('parses a body delivered in tiny chunks (regression: missing final boundary)', async () => {
+    const response = await POST(makeRequest(uploadParts(csvWithRows(5)), 7))
+    const data = await response.json()
+
+    expect(response.status).toBe(200)
+    expect(data.data.table.rowCount).toBe(5)
+  })
+
+  it('returns 400 for a CSV with no data rows', async () => {
+    const response = await POST(makeRequest(uploadParts('name,age\n')))
+    const data = await response.json()
+
+    expect(response.status).toBe(400)
+    expect(data.error).toMatch(/no data rows/i)
+    expect(mockCreateTable).not.toHaveBeenCalled()
+  })
+
+  it('returns 400 when the file precedes required fields', async () => {
+    const response = await POST(
+      makeRequest([
+        { name: 'file', filename: 'data.csv', value: csvWithRows(3) },
+        { name: 'workspaceId', value: 'workspace-1' },
+      ])
+    )
+
+    expect(response.status).toBe(400)
+    expect(mockCreateTable).not.toHaveBeenCalled()
+  })
+
+  it('returns 400 when no file part is present', async () => {
+    const response = await POST(makeRequest([{ name: 'workspaceId', value: 'workspace-1' }]))
+    expect(response.status).toBe(400)
+    expect(mockCreateTable).not.toHaveBeenCalled()
+  })
+
+  it('returns 400 with the reason when an insert exceeds the plan row limit', async () => {
+    mockBatchInsertRows.mockRejectedValueOnce(
+      new Error('This table has reached its row limit (1,000 rows) on your current plan.')
+    )
+    const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
+    const data = await response.json()
+
+    expect(response.status).toBe(400)
+    expect(data.error).toMatch(/row limit/)
+  })
+
+  it('rolls back the created table when a batch insert fails mid-stream', async () => {
+    mockBatchInsertRows
+      .mockResolvedValueOnce(Array.from({ length: 100 }, () => ({ id: 'row' })))
+      .mockRejectedValueOnce(new Error('insert boom'))
+
+    const response = await POST(makeRequest(uploadParts(csvWithRows(250))))
+
+    expect(response.status).toBe(500)
+    expect(mockDeleteTable).toHaveBeenCalledWith('tbl_1', expect.any(String))
+  })
+
+  it('returns 401 when unauthenticated', async () => {
+    hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ success: false })
+    const response = await POST(makeRequest(uploadParts(csvWithRows(3))))
+    expect(response.status).toBe(401)
+  })
+
+  it('returns 403 without write permission', async () => {
+    permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read')
+    const response = await POST(makeRequest(uploadParts(csvWithRows(3))))
+    expect(response.status).toBe(403)
+  })
+})

apps/sim/app/api/table/import-csv/route.ts

@@ -0,0 +1,244 @@
+import type { Readable } from 'node:stream'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { type NextRequest, NextResponse } from 'next/server'
+import { csvExtensionSchema, csvImportFormSchema } from '@/lib/api/contracts/tables'
+import { ianaTimezoneSchema } from '@/lib/api/contracts/user'
+import { getValidationErrorMessage } from '@/lib/api/server'
+import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import {
+  batchInsertRows,
+  CSV_MAX_BATCH_SIZE,
+  CSV_MAX_FILE_SIZE_BYTES,
+  CSV_SCHEMA_SAMPLE_SIZE,
+  coerceRowsForTable,
+  createCsvParser,
+  createTable,
+  deleteTable,
+  getWorkspaceTableLimits,
+  inferSchemaFromCsv,
+  sanitizeName,
+  TABLE_LIMITS,
+  type TableDefinition,
+  type TableSchema,
+} from '@/lib/table'
+import { getUserSettings } from '@/lib/users/queries'
+import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
+import {
+  csvProxyBodyCapResponse,
+  multipartErrorResponse,
+  normalizeColumn,
+  rowWriteErrorResponse,
+} from '@/app/api/table/utils'
+
+const logger = createLogger('TableImportCSV')
+
+export const runtime = 'nodejs'
+export const dynamic = 'force-dynamic'
+export const maxDuration = 300
+
+export const POST = withRouteHandler(async (request: NextRequest) => {
+  const requestId = generateRequestId()
+  let fileStream: Readable | undefined
+
+  try {
+    const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false })
+    if (!authResult.success || !authResult.userId) {
+      return NextResponse.json({ error: 'Authentication required' }, { status: 401 })
+    }
+    const userId = authResult.userId
+
+    const oversize = csvProxyBodyCapResponse(request)
+    if (oversize) return oversize
+
+    let parsed: Awaited<ReturnType<typeof readMultipart>>
+    try {
+      parsed = await readMultipart(request, {
+        maxFileBytes: CSV_MAX_FILE_SIZE_BYTES,
+        requiredFieldsBeforeFile: ['workspaceId'],
+        signal: request.signal,
+      })
+    } catch (err) {
+      if (isMultipartError(err)) return multipartErrorResponse(err)
+      throw err
+    }
+
+    const { fields, file } = parsed
+    if (!file) {
+      return NextResponse.json({ error: 'CSV file is required' }, { status: 400 })
+    }
+    fileStream = file.stream
+
+    const workspaceIdResult = csvImportFormSchema.shape.workspaceId.safeParse(fields.workspaceId)
+    if (!workspaceIdResult.success) {
+      return NextResponse.json(
+        { error: getValidationErrorMessage(workspaceIdResult.error) },
+        { status: 400 }
+      )
+    }
+    const workspaceId = workspaceIdResult.data
+
+    const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
+    if (permission !== 'write' && permission !== 'admin') {
+      return NextResponse.json({ error: 'Access denied' }, { status: 403 })
+    }
+
+    let timezone = (await getUserSettings(userId)).timezone ?? 'UTC'
+    if (fields.timezone) {
+      const timezoneResult = ianaTimezoneSchema.safeParse(fields.timezone)
+      if (!timezoneResult.success) {
+        return NextResponse.json(
+          { error: getValidationErrorMessage(timezoneResult.error) },
+          { status: 400 }
+        )
+      }
+      timezone = timezoneResult.data
+    }
+
+    const ext = file.filename.split('.').pop()?.toLowerCase()
+    const extensionResult = csvExtensionSchema.safeParse(ext)
+    if (!extensionResult.success) {
+      return NextResponse.json(
+        { error: getValidationErrorMessage(extensionResult.error) },
+        { status: 400 }
+      )
+    }
+    const delimiter = extensionResult.data === 'tsv' ? '\t' : ','
+
+    const parser = createCsvParser(delimiter)
+    // `.pipe` doesn't forward source errors; forward them so the iterator throws.
+    file.stream.on('error', (err) => parser.destroy(err))
+    file.stream.pipe(parser)
+
+    interface ImportState {
+      table: TableDefinition
+      schema: TableSchema
+      headerToColumn: Map<string, string>
+    }
+
+    const insertRows = async (
+      rows: Record<string, unknown>[],
+      state: ImportState,
+      currentRowCount: number
+    ) => {
+      if (rows.length === 0) return 0
+      const coerced = coerceRowsForTable(rows, state.schema, state.headerToColumn, { timezone })
+      const result = await batchInsertRows(
+        { tableId: state.table.id, rows: coerced, workspaceId, userId },
+        // The created table's rowCount is frozen at 0; pass the running total so the
+        // per-batch capacity check sees cumulative rows, not an always-empty table.
+        { ...state.table, rowCount: currentRowCount },
+        generateId().slice(0, 8)
+      )
+      return result.length
+    }
+
+    /** Infer the schema from the buffered sample and create the (empty) table. */
+    const buildTable = async (sampleRows: Record<string, unknown>[]): Promise<ImportState> => {
+      const inferred = inferSchemaFromCsv(Object.keys(sampleRows[0]), sampleRows)
+      const schema: TableSchema = { columns: inferred.columns.map(normalizeColumn) }
+      const planLimits = await getWorkspaceTableLimits(workspaceId)
+      const tableName = sanitizeName(file.filename.replace(/\.[^.]+$/, ''), 'imported_table').slice(
+        0,
+        TABLE_LIMITS.MAX_TABLE_NAME_LENGTH
+      )
+      const table = await createTable(
+        {
+          name: tableName,
+          description: `Imported from ${file.filename}`,
+          schema,
+          workspaceId,
+          userId,
+          maxTables: planLimits.maxTables,
+        },
+        requestId
+      )
+      // Coerce against the *created* schema so rows key by the ids `createTable`
+      // assigned (the local `schema` is the id-less inferred one).
+      return { table, schema: table.schema, headerToColumn: inferred.headerToColumn }
+    }
+
+    let state: ImportState | null = null
+    let inserted = 0
+    const sample: Record<string, unknown>[] = []
+    let batch: Record<string, unknown>[] = []
+
+    try {
+      for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
+        if (!state) {
+          sample.push(record)
+          if (sample.length >= CSV_SCHEMA_SAMPLE_SIZE) {
+            state = await buildTable(sample)
+            inserted += await insertRows(sample, state, inserted)
+          }
+          continue
+        }
+        batch.push(record)
+        if (batch.length >= CSV_MAX_BATCH_SIZE) {
+          inserted += await insertRows(batch, state, inserted)
+          batch = []
+        }
+      }
+
+      if (!state) {
+        if (sample.length === 0) {
+          return NextResponse.json({ error: 'CSV file has no data rows' }, { status: 400 })
+        }
+        state = await buildTable(sample)
+        inserted += await insertRows(sample, state, inserted)
+      } else {
+        inserted += await insertRows(batch, state, inserted)
+      }
+    } catch (streamError) {
+      if (state) await deleteTable(state.table.id, requestId).catch(() => {})
+      throw streamError
+    }
+
+    logger.info(`[${requestId}] CSV imported`, {
+      tableId: state.table.id,
+      fileName: file.filename,
+      columns: state.schema.columns.length,
+      rows: inserted,
+    })
+
+    return NextResponse.json({
+      success: true,
+      data: {
+        table: {
+          id: state.table.id,
+          name: state.table.name,
+          description: state.table.description,
+          schema: state.schema,
+          rowCount: inserted,
+        },
+      },
+    })
+  } catch (error) {
+    if (isMultipartError(error)) return multipartErrorResponse(error)
+
+    logger.error(`[${requestId}] CSV import failed:`, error)
+
+    // Row-write failures (e.g. the plan row-limit check) map to a 400 with the real reason.
+    const rowWriteError = rowWriteErrorResponse(error)
+    if (rowWriteError) return rowWriteError
+
+    const message = toError(error).message
+    const isClientError =
+      message.includes('maximum table limit') ||
+      message.includes('CSV file has no') ||
+      message.includes('Invalid table name') ||
+      message.includes('Invalid schema') ||
+      message.includes('already exists')
+
+    return NextResponse.json(
+      { error: isClientError ? message : 'Failed to import CSV' },
+      { status: isClientError ? 400 : 500 }
+    )
+  } finally {
+    fileStream?.destroy()
+  }
+})