chore: import upstream snapshot with attribution

apps/sim/app/api/table/[tableId]/import/route.test.ts

@@ -0,0 +1,536 @@
+/**
+ * @vitest-environment node
+ */
+import { hybridAuthMockFns } from '@sim/testing'
+import { NextRequest } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { TableDefinition } from '@/lib/table'
+
+const {
+  mockCheckAccess,
+  mockImportAppendRows,
+  mockImportReplaceRows,
+  mockDispatchAfterBatchInsert,
+  mockMarkTableImporting,
+  mockReleaseImportClaim,
+  mockGetMaxRowsPerTable,
+} = vi.hoisted(() => ({
+  mockCheckAccess: vi.fn(),
+  mockImportAppendRows: vi.fn(),
+  mockImportReplaceRows: vi.fn(),
+  mockDispatchAfterBatchInsert: vi.fn(),
+  mockMarkTableImporting: vi.fn(),
+  mockReleaseImportClaim: vi.fn(),
+  mockGetMaxRowsPerTable: vi.fn(),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn().mockReturnValue('deadbeefcafef00d'),
+  generateShortId: vi.fn().mockReturnValue('short-id'),
+}))
+
+vi.mock('@/app/api/table/utils', async () => {
+  const { NextResponse } = await import('next/server')
+  return {
+    checkAccess: mockCheckAccess,
+    accessError: (result: { status: number }) => {
+      const message = result.status === 404 ? 'Table not found' : 'Access denied'
+      return NextResponse.json({ error: message }, { status: result.status })
+    },
+    csvProxyBodyCapResponse: () => null,
+    multipartErrorResponse: (error: { code: string; message: string }) =>
+      NextResponse.json(
+        { error: error.message },
+        { status: error.code === 'FILE_TOO_LARGE' ? 413 : 400 }
+      ),
+  }
+})
+
+/**
+ * The route imports `importAppendRows` / `importReplaceRows` from
+ * `@/lib/table/import-data`. These functions own the import transaction (column
+ * adds + row writes); mocking that module replaces them without touching the
+ * other real helpers (`coerceRowsForTable`, `createCsvParser`, etc.) exported
+ * through the barrel.
+ */
+vi.mock('@/lib/table/import-data', () => ({
+  importAppendRows: mockImportAppendRows,
+  importReplaceRows: mockImportReplaceRows,
+}))
+
+vi.mock('@/lib/table/jobs/service', () => ({
+  markTableJobRunning: mockMarkTableImporting,
+  releaseJobClaim: mockReleaseImportClaim,
+}))
+
+vi.mock('@/lib/table/rows/service', () => ({
+  dispatchAfterBatchInsert: mockDispatchAfterBatchInsert,
+}))
+
+/** The append pre-check reads the workspace's current plan row limit, not the frozen `table.maxRows`. */
+vi.mock('@/lib/table/billing', () => ({
+  getMaxRowsPerTable: mockGetMaxRowsPerTable,
+  wouldExceedRowLimit: (limit: number, current: number, added: number) =>
+    limit >= 0 && current + added > limit,
+}))
+
+import { POST } from '@/app/api/table/[tableId]/import/route'
+
+function createCsvFile(contents: string, name = 'data.csv', type = 'text/csv'): File {
+  return new File([contents], name, { type })
+}
+
+function createFormData(
+  file: File,
+  options?: {
+    workspaceId?: string | null
+    mode?: string | null
+    mapping?: unknown
+    createColumns?: unknown
+  }
+): FormData {
+  // Text fields must precede the file part for the streaming parser.
+  const form = new FormData()
+  if (options?.workspaceId !== null) {
+    form.append('workspaceId', options?.workspaceId ?? 'workspace-1')
+  }
+  if (options?.mode !== null) {
+    form.append('mode', options?.mode ?? 'append')
+  }
+  if (options?.mapping !== undefined) {
+    form.append(
+      'mapping',
+      typeof options.mapping === 'string' ? options.mapping : JSON.stringify(options.mapping)
+    )
+  }
+  if (options?.createColumns !== undefined) {
+    form.append(
+      'createColumns',
+      typeof options.createColumns === 'string'
+        ? options.createColumns
+        : JSON.stringify(options.createColumns)
+    )
+  }
+  form.append('file', file)
+  return form
+}
+
+function buildTable(overrides: Partial<TableDefinition> = {}): TableDefinition {
+  return {
+    id: 'tbl_1',
+    name: 'People',
+    description: null,
+    schema: {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'age', type: 'number' },
+      ],
+    },
+    metadata: null,
+    rowCount: 0,
+    maxRows: 100,
+    workspaceId: 'workspace-1',
+    createdBy: 'user-1',
+    archivedAt: null,
+    createdAt: new Date('2024-01-01'),
+    updatedAt: new Date('2024-01-01'),
+    ...overrides,
+  }
+}
+
+/** Additions array the route passed to importAppendRows (2nd positional arg). */
+function appendAdditions(): { name: string; type: string }[] {
+  return mockImportAppendRows.mock.calls[0][1] as { name: string; type: string }[]
+}
+
+/** Rows array the route passed to importAppendRows (3rd positional arg). */
+function appendRows(): unknown[] {
+  return mockImportAppendRows.mock.calls[0][2] as unknown[]
+}
+
+async function callPost(form: FormData, { tableId }: { tableId: string } = { tableId: 'tbl_1' }) {
+  // Building the request from a FormData body gives a real multipart stream and
+  // boundary, exercising the streaming `readMultipart` parser end-to-end.
+  const req = new NextRequest(`http://localhost:3000/api/table/${tableId}/import`, {
+    method: 'POST',
+    body: form,
+  })
+  return POST(req, { params: Promise.resolve({ tableId }) })
+}
+
+describe('POST /api/table/[tableId]/import', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({
+      success: true,
+      userId: 'user-1',
+      authType: 'session',
+    })
+    mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() })
+    mockImportAppendRows.mockImplementation(
+      async (table: TableDefinition, _additions: unknown, rows: unknown[]) => ({
+        inserted: rows.map((_, i) => ({ id: `row_${i}` })),
+        table,
+      })
+    )
+    mockImportReplaceRows.mockResolvedValue({ deletedCount: 0, insertedCount: 0 })
+    mockMarkTableImporting.mockResolvedValue(true)
+    mockReleaseImportClaim.mockResolvedValue(undefined)
+    mockGetMaxRowsPerTable.mockResolvedValue(1_000_000)
+  })
+
+  it('returns 401 when the user is not authenticated', async () => {
+    hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({
+      success: false,
+      error: 'Authentication required',
+    })
+    const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
+    expect(response.status).toBe(401)
+  })
+
+  it('returns 409 when a background import already holds the table (claim lost)', async () => {
+    mockMarkTableImporting.mockResolvedValueOnce(false)
+    const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
+    expect(response.status).toBe(409)
+    expect(mockImportAppendRows).not.toHaveBeenCalled()
+    expect(mockImportReplaceRows).not.toHaveBeenCalled()
+    expect(mockReleaseImportClaim).not.toHaveBeenCalled()
+  })
+
+  it('releases the import claim after a successful write', async () => {
+    const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
+    expect(response.status).toBe(200)
+    expect(mockMarkTableImporting).toHaveBeenCalledWith('tbl_1', 'deadbeefcafef00d', 'import')
+    expect(mockReleaseImportClaim).toHaveBeenCalledWith('tbl_1', 'deadbeefcafef00d')
+  })
+
+  it('returns 400 when the mode is invalid', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'bogus' })
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/Invalid mode/)
+  })
+
+  it('returns 403 when the user lacks workspace write access', async () => {
+    mockCheckAccess.mockResolvedValueOnce({ ok: false, status: 403 })
+    const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
+    expect(response.status).toBe(403)
+  })
+
+  it('returns 400 when the target table is archived', async () => {
+    mockCheckAccess.mockResolvedValueOnce({
+      ok: true,
+      table: buildTable({ archivedAt: new Date('2024-01-02') }),
+    })
+    const response = await callPost(createFormData(createCsvFile('name,age\nAlice,30')))
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/archived/i)
+  })
+
+  it('returns 400 when the file part precedes the required fields', async () => {
+    // Build a raw multipart body with the file BEFORE workspaceId.
+    const boundary = '----orderboundary'
+    const body = Buffer.concat([
+      Buffer.from(
+        `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="data.csv"\r\nContent-Type: text/csv\r\n\r\nname,age\nAlice,30\r\n`
+      ),
+      Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="workspaceId"\r\n\r\n`),
+      Buffer.from('workspace-1\r\n'),
+      Buffer.from(`--${boundary}--\r\n`),
+    ])
+    const req = {
+      headers: new Headers({ 'content-type': `multipart/form-data; boundary=${boundary}` }),
+      body: new ReadableStream<Uint8Array>({
+        start(controller) {
+          controller.enqueue(new Uint8Array(body))
+          controller.close()
+        },
+      }),
+      signal: undefined,
+    } as unknown as NextRequest
+
+    const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
+    expect(response.status).toBe(400)
+    expect(mockImportAppendRows).not.toHaveBeenCalled()
+    expect(mockImportReplaceRows).not.toHaveBeenCalled()
+  })
+
+  it('returns 400 when the CSV is missing a required column', async () => {
+    const response = await callPost(createFormData(createCsvFile('age\n30')))
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/missing required columns/i)
+    expect(data.details?.missingRequired).toEqual(['name'])
+    expect(mockImportAppendRows).not.toHaveBeenCalled()
+  })
+
+  it('appends rows via importAppendRows', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
+    )
+    expect(response.status).toBe(200)
+    const data = await response.json()
+    expect(data.data.mode).toBe('append')
+    expect(data.data.insertedCount).toBe(2)
+    expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
+    expect(appendRows()).toEqual([
+      { name: 'Alice', age: 30 },
+      { name: 'Bob', age: 40 },
+    ])
+    expect(mockImportReplaceRows).not.toHaveBeenCalled()
+  })
+
+  it('accepts chunked multipart imports without a content-length header', async () => {
+    const form = createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
+    const req = new NextRequest('http://localhost:3000/api/table/tbl_1/import', {
+      method: 'POST',
+      body: form,
+    })
+
+    expect(req.headers.get('content-length')).toBeNull()
+
+    const response = await POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) })
+
+    expect(response.status).toBe(200)
+    expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects append when it would exceed the current plan row limit', async () => {
+    mockCheckAccess.mockResolvedValueOnce({ ok: true, table: buildTable({ rowCount: 99 }) })
+    mockGetMaxRowsPerTable.mockResolvedValueOnce(100)
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'append' })
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/exceed table row limit/)
+    expect(mockImportAppendRows).not.toHaveBeenCalled()
+  })
+
+  it('replaces rows via importReplaceRows', async () => {
+    mockImportReplaceRows.mockResolvedValueOnce({ deletedCount: 5, insertedCount: 2 })
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30\nBob,40'), { mode: 'replace' })
+    )
+    expect(response.status).toBe(200)
+    const data = await response.json()
+    expect(data.data.mode).toBe('replace')
+    expect(data.data.deletedCount).toBe(5)
+    expect(data.data.insertedCount).toBe(2)
+    expect(mockImportReplaceRows).toHaveBeenCalledTimes(1)
+    expect(mockImportAppendRows).not.toHaveBeenCalled()
+  })
+
+  it('uses an explicit mapping when provided', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('First Name,Years\nAlice,30\nBob,40', 'people.csv'), {
+        mode: 'append',
+        mapping: { 'First Name': 'name', Years: 'age' },
+      })
+    )
+    expect(response.status).toBe(200)
+    const data = await response.json()
+    expect(data.data.mappedColumns).toEqual(['First Name', 'Years'])
+    expect(appendRows()).toEqual([
+      { name: 'Alice', age: 30 },
+      { name: 'Bob', age: 40 },
+    ])
+  })
+
+  it('returns 400 when the mapping targets a non-existent column', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('a\nAlice'), {
+        mode: 'append',
+        mapping: { a: 'nonexistent' },
+      })
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/do not exist on the table/)
+  })
+
+  it('returns 400 when a mapping value is not a string or null', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30'), {
+        mode: 'append',
+        mapping: { name: 42 },
+      })
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/Mapping values must be/)
+  })
+
+  it('surfaces unique violations from importAppendRows as 400', async () => {
+    mockImportAppendRows.mockRejectedValueOnce(
+      new Error('Row 1: Column "name" must be unique. Value "Alice" already exists in row row_xxx')
+    )
+    const response = await callPost(
+      createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/must be unique/)
+    expect(data.data?.insertedCount).toBe(0)
+  })
+
+  it('accepts TSV files', async () => {
+    const response = await callPost(
+      createFormData(
+        createCsvFile('name\tage\nAlice\t30', 'data.tsv', 'text/tab-separated-values'),
+        { mode: 'append' }
+      )
+    )
+    expect(response.status).toBe(200)
+    expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
+  })
+
+  it('returns 400 for unsupported file extensions', async () => {
+    const response = await callPost(
+      createFormData(createCsvFile('name,age', 'data.json', 'application/json'))
+    )
+    expect(response.status).toBe(400)
+    const data = await response.json()
+    expect(data.error).toMatch(/CSV and TSV/)
+  })
+
+  describe('createColumns', () => {
+    it('auto-creates columns for unmapped CSV headers', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io\nBob,40,b@x.io'), {
+          mode: 'append',
+          createColumns: ['email'],
+        })
+      )
+      expect(response.status).toBe(200)
+      expect(mockImportAppendRows).toHaveBeenCalledTimes(1)
+      expect(appendAdditions()).toEqual([
+        expect.objectContaining({ name: 'email', type: 'string' }),
+      ])
+      // Existing columns have no id (legacy) → keyed by name; the new `email`
+      // column was assigned id `col_deadbeefcafef00d` (mocked generateId).
+      expect(appendRows()).toEqual([
+        { name: 'Alice', age: 30, col_deadbeefcafef00d: 'a@x.io' },
+        { name: 'Bob', age: 40, col_deadbeefcafef00d: 'b@x.io' },
+      ])
+    })
+
+    it('infers column type from CSV row values', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,score\nAlice,42\nBob,17'), {
+          mode: 'append',
+          createColumns: ['score'],
+        })
+      )
+      expect(response.status).toBe(200)
+      expect(appendAdditions()).toEqual([
+        expect.objectContaining({ name: 'score', type: 'number' }),
+      ])
+    })
+
+    it('dedupes when sanitized name collides with an existing column', async () => {
+      mockCheckAccess.mockResolvedValueOnce({
+        ok: true,
+        table: buildTable({
+          schema: {
+            columns: [
+              { name: 'name', type: 'string', required: true },
+              { name: 'age', type: 'number' },
+              { name: 'email', type: 'string' },
+            ],
+          },
+        }),
+      })
+      const response = await callPost(
+        createFormData(createCsvFile('name,age,Email\nAlice,30,a@x.io'), {
+          mode: 'append',
+          createColumns: ['Email'],
+        })
+      )
+      expect(response.status).toBe(200)
+      expect(appendAdditions()).toEqual([
+        expect.objectContaining({ name: 'Email_2', type: 'string' }),
+      ])
+    })
+
+    it('returns 400 when createColumns references a header not in the CSV', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,age\nAlice,30'), {
+          mode: 'append',
+          createColumns: ['nonexistent'],
+        })
+      )
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.error).toMatch(/unknown CSV headers/)
+      expect(mockImportAppendRows).not.toHaveBeenCalled()
+    })
+
+    it('returns 400 when createColumns is not an array of strings', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,age\nAlice,30'), {
+          mode: 'append',
+          createColumns: [1, 2],
+        })
+      )
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.error).toMatch(/createColumns must be a JSON array/)
+      expect(mockImportAppendRows).not.toHaveBeenCalled()
+    })
+
+    it('returns 400 when createColumns is invalid JSON', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,age\nAlice,30'), {
+          mode: 'append',
+          createColumns: '{not-json',
+        })
+      )
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.error).toMatch(/createColumns must be valid JSON/)
+    })
+
+    it('surfaces column-creation failures from importAppendRows as 400', async () => {
+      mockImportAppendRows.mockRejectedValueOnce(new Error('Column "email" already exists'))
+      const response = await callPost(
+        createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io'), {
+          mode: 'append',
+          createColumns: ['email'],
+        })
+      )
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.error).toMatch(/already exists/)
+    })
+
+    it('surfaces row insert failures without success when schema was mutated', async () => {
+      mockImportAppendRows.mockRejectedValueOnce(new Error('must be unique'))
+      const response = await callPost(
+        createFormData(createCsvFile('name,age,email\nAlice,30,a@x.io'), {
+          mode: 'append',
+          createColumns: ['email'],
+        })
+      )
+      // Route forwarded the column addition into the (now atomic) import op.
+      expect(appendAdditions()).toEqual([
+        expect.objectContaining({ name: 'email', type: 'string' }),
+      ])
+      expect(response.status).toBe(400)
+      const data = await response.json()
+      expect(data.success).toBeUndefined()
+      expect(data.error).toMatch(/must be unique/)
+    })
+
+    it('passes no additions when createColumns is omitted', async () => {
+      const response = await callPost(
+        createFormData(createCsvFile('name,age\nAlice,30'), { mode: 'append' })
+      )
+      expect(response.status).toBe(200)
+      expect(appendAdditions()).toEqual([])
+    })
+  })
+})