simstudioai--sim/apps/sim/lib/table/import.ts

/**
 * Shared CSV import helpers for user-defined tables.
 *
 * Used by:
 * - `POST /api/table/import-csv` (create new table from CSV — streams via {@link createCsvParser})
 * - `POST /api/table/[tableId]/import` (append/replace into existing table)
 * - Copilot `user-table` tool (`create_from_file`, `import_file` — buffers via {@link parseCsvBuffer})
 *
 * Keeping a single implementation avoids drift between HTTP and agent code paths.
 * Both the buffered ({@link parseCsvBuffer}) and streaming ({@link createCsvParser})
 * parsers share {@link csvParseOptions} so their behavior can't drift.
 */

import { type Options as CsvParseOptions, type Parser, parse as parseCsvStream } from 'csv-parse'
import { getColumnId } from '@/lib/table/column-keys'
import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates'
import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types'

/**
 * Single source of truth for the `csv-parse` options used by both the buffered
 * sync parser and the streaming parser. `columns: true` emits each record as an
 * object keyed by the (first-row) headers.
 */
export function csvParseOptions(delimiter = ','): CsvParseOptions {
  return {
    columns: true,
    skip_empty_lines: true,
    trim: true,
    relax_column_count: true,
    relax_quotes: true,
    skip_records_with_error: true,
    cast: false,
    bom: true,
    delimiter,
  }
}

/**
 * Returns a streaming `csv-parse` parser (a `Transform`/async-iterable). Pipe a
 * file stream into it and iterate records with `for await`; backpressure flows
 * back to the source while each record is processed. Use this for HTTP uploads
 * so the file is never fully buffered in memory.
 */
export function createCsvParser(delimiter = ','): Parser {
  return parseCsvStream(csvParseOptions(delimiter))
}

/** Narrower type than `COLUMN_TYPES` used internally for coercion. */
export type CsvColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json'

/** Number of CSV rows sampled when inferring column types for a new table. */
export const CSV_SCHEMA_SAMPLE_SIZE = 100

/**
 * Maximum rows inserted per import batch. Each batch is one `INSERT … VALUES` statement, and
 * Postgres caps bind parameters at 65,535 — at 9 params per row that's a hard ceiling of ~7,200
 * rows, so 5,000 keeps a margin while cutting per-batch overhead (validation, unique-constraint
 * check, ownership heartbeat) 5× vs the old 1,000.
 */
export const CSV_MAX_BATCH_SIZE = 5000

/** Maximum CSV/TSV file size accepted by import routes (25 MB). */
export const CSV_MAX_FILE_SIZE_BYTES = 25 * 1024 * 1024

/**
 * Error thrown when the user-supplied mapping or CSV does not line up with the
 * target table. Callers should translate this into a 400 response.
 */
export class CsvImportValidationError extends Error {
  readonly code = 'CSV_IMPORT_VALIDATION' as const
  readonly details: {
    missingRequired?: string[]
    duplicateTargets?: string[]
    unknownColumns?: string[]
    unknownHeaders?: string[]
  }

  constructor(
    message: string,
    details: {
      missingRequired?: string[]
      duplicateTargets?: string[]
      unknownColumns?: string[]
      unknownHeaders?: string[]
    } = {}
  ) {
    super(message)
    this.name = 'CsvImportValidationError'
    this.details = details
  }
}

/**
 * Parses a CSV/TSV payload using `csv-parse/sync`. Accepts a Node `Buffer`,
 * browser-friendly `Uint8Array`, or already-decoded string. A leading UTF-8 BOM
 * is stripped by csv-parse (`bom: true` in {@link csvParseOptions}).
 *
 * For HTTP uploads prefer {@link createCsvParser} so the file isn't buffered.
 */
export async function parseCsvBuffer(
  input: Buffer | Uint8Array | string,
  delimiter = ','
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
  const { parse } = await import('csv-parse/sync')

  let text: string
  if (typeof input === 'string') {
    text = input
  } else if (typeof Buffer !== 'undefined' && Buffer.isBuffer(input)) {
    text = input.toString('utf-8')
  } else {
    text = new TextDecoder('utf-8').decode(input as Uint8Array)
  }

  // double-cast-allowed: shared csvParseOptions() loses the `columns: true` literal that drives
  // csv-parse's record-vs-string[][] overload, but `columns: true` is always set so records are objects
  const parsed = parse(text, csvParseOptions(delimiter)) as unknown as Record<string, unknown>[]

  if (parsed.length === 0) {
    throw new Error('CSV file has no data rows')
  }

  const headers = Object.keys(parsed[0])
  if (headers.length === 0) {
    throw new Error('CSV file has no headers')
  }

  return { headers, rows: parsed }
}

/**
 * Infers a column type from a sample of non-empty values. Order matters: we
 * prefer narrower types (number > boolean > ISO date) and fall back to string.
 * JSON is never inferred automatically.
 */
export function inferColumnType(values: unknown[]): Exclude<CsvColumnType, 'json'> {
  const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
  if (nonEmpty.length === 0) return 'string'

  const allNumber = nonEmpty.every((v) => {
    const n = Number(v)
    return !Number.isNaN(n) && String(v).trim() !== ''
  })
  if (allNumber) return 'number'

  const allBoolean = nonEmpty.every((v) => {
    const s = String(v).toLowerCase()
    return s === 'true' || s === 'false'
  })
  if (allBoolean) return 'boolean'

  const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/
  const allDate = nonEmpty.every((v) => {
    const s = String(v)
    return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s))
  })
  if (allDate) return 'date'

  return 'string'
}

/**
 * Sanitizes a raw header into a valid column/table name. Strips disallowed
 * characters, collapses runs of underscores, and ensures the first character
 * is a letter or underscore (prefixing with `fallbackPrefix` otherwise).
 */
export function sanitizeName(raw: string, fallbackPrefix = 'col'): string {
  let name = raw
    .trim()
    .replace(/[^a-zA-Z0-9_]/g, '_')
    .replace(/_+/g, '_')
    .replace(/^_+|_+$/g, '')

  if (!name || /^\d/.test(name)) {
    name = `${fallbackPrefix}_${name}`
  }

  return name
}

/**
 * Returns column definitions inferred from CSV headers + sample rows. Duplicate
 * sanitized names are suffixed with `_2`, `_3`, etc. Also returns the header ->
 * column-name mapping used when coercing row values.
 */
export function inferSchemaFromCsv(
  headers: string[],
  rows: Record<string, unknown>[]
): { columns: ColumnDefinition[]; headerToColumn: Map<string, string> } {
  const sample = rows.slice(0, CSV_SCHEMA_SAMPLE_SIZE)
  const seen = new Set<string>()
  const headerToColumn = new Map<string, string>()

  const columns = headers.map((header) => {
    const base = sanitizeName(header)
    let colName = base
    let suffix = 2
    while (seen.has(colName.toLowerCase())) {
      colName = `${base}_${suffix}`
      suffix++
    }
    seen.add(colName.toLowerCase())
    headerToColumn.set(header, colName)

    return {
      name: colName,
      type: inferColumnType(sample.map((r) => r[header])),
    } satisfies ColumnDefinition
  })

  return { columns, headerToColumn }
}

/**
 * Coerces a single value to the requested column type. Returns `null` for
 * empty inputs or values that cannot be parsed (numbers/booleans). Dates fall
 * back to the original string when unparseable so that schema validation can
 * reject it with context rather than silently inserting `null`.
 */
export function coerceValue(
  value: unknown,
  colType: CsvColumnType,
  options?: NormalizeDateCellOptions
): string | number | boolean | null | Record<string, unknown> | unknown[] {
  if (value === null || value === undefined || value === '') return null
  switch (colType) {
    case 'number': {
      const n = Number(value)
      return Number.isNaN(n) ? null : n
    }
    case 'boolean': {
      const s = String(value).toLowerCase()
      if (s === 'true') return true
      if (s === 'false') return false
      return null
    }
    case 'date': {
      return normalizeDateCellValue(String(value), options) ?? String(value)
    }
    case 'json': {
      if (typeof value === 'object') return value as Record<string, unknown> | unknown[]
      try {
        return JSON.parse(String(value))
      } catch {
        return String(value)
      }
    }
    default:
      return String(value)
  }
}

/**
 * Mapping from raw CSV header to target column name, with `null` indicating
 * "do not import this column".
 */
export type CsvHeaderMapping = Record<string, string | null>

export interface CsvMappingValidationResult {
  /** Columns present in the CSV that landed on a real table column. */
  mappedHeaders: string[]
  /** Columns in the CSV that the user/client chose to skip. */
  skippedHeaders: string[]
  /** Target column names that ended up unmapped (resolved from the mapping). */
  unmappedColumns: string[]
  /** Effective header -> column map (after dropping unknown / null targets). */
  effectiveMap: Map<string, string>
}

/**
 * Validates a user-supplied mapping against the target table schema. Rejects
 * unknown target columns, duplicate targets, and required table columns that
 * are not covered by the CSV. Returns the normalized header -> column map.
 */
export function validateMapping(params: {
  csvHeaders: string[]
  mapping: CsvHeaderMapping
  tableSchema: TableSchema
}): CsvMappingValidationResult {
  const { csvHeaders, mapping, tableSchema } = params
  const columnByName = new Map(tableSchema.columns.map((c) => [c.name, c]))

  const unknownHeaders = Object.keys(mapping).filter((h) => !csvHeaders.includes(h))
  if (unknownHeaders.length > 0) {
    throw new CsvImportValidationError(
      `Mapping references unknown CSV headers: ${unknownHeaders.join(', ')}`,
      { unknownHeaders }
    )
  }

  const invalidTargets = Object.entries(mapping).filter(
    ([, target]) => target !== null && typeof target !== 'string'
  )
  if (invalidTargets.length > 0) {
    throw new CsvImportValidationError(
      `Mapping values must be a column name (string) or null, got: ${invalidTargets
        .map(([header]) => header)
        .join(', ')}`
    )
  }

  const targetsSeen = new Map<string, string[]>()
  const unknownColumns: string[] = []
  const effectiveMap = new Map<string, string>()
  const skippedHeaders: string[] = []

  for (const header of csvHeaders) {
    const target = header in mapping ? mapping[header] : undefined
    if (target === null || target === undefined) {
      skippedHeaders.push(header)
      continue
    }
    if (!columnByName.has(target)) {
      unknownColumns.push(target)
      continue
    }
    const existing = targetsSeen.get(target) ?? []
    existing.push(header)
    targetsSeen.set(target, existing)
    effectiveMap.set(header, target)
  }

  if (unknownColumns.length > 0) {
    throw new CsvImportValidationError(
      `Mapping references columns that do not exist on the table: ${unknownColumns.join(', ')}`,
      { unknownColumns }
    )
  }

  const duplicateTargets = [...targetsSeen.entries()]
    .filter(([, headers]) => headers.length > 1)
    .map(([col]) => col)
  if (duplicateTargets.length > 0) {
    throw new CsvImportValidationError(
      `Multiple CSV headers map to the same column(s): ${duplicateTargets.join(', ')}`,
      { duplicateTargets }
    )
  }

  const mappedTargets = new Set(effectiveMap.values())
  const unmappedColumns = tableSchema.columns
    .filter((c) => !mappedTargets.has(c.name))
    .map((c) => c.name)

  const missingRequired = tableSchema.columns
    .filter((c) => c.required && !mappedTargets.has(c.name))
    .map((c) => c.name)
  if (missingRequired.length > 0) {
    throw new CsvImportValidationError(
      `CSV is missing required columns: ${missingRequired.join(', ')}`,
      { missingRequired }
    )
  }

  return {
    mappedHeaders: [...effectiveMap.keys()],
    skippedHeaders,
    unmappedColumns,
    effectiveMap,
  }
}

/**
 * Builds an auto-mapping from CSV headers to table columns: prefers exact
 * sanitized-name matches and falls back to a case- and punctuation-insensitive
 * comparison. Unmapped headers are set to `null`.
 */
export function buildAutoMapping(csvHeaders: string[], tableSchema: TableSchema): CsvHeaderMapping {
  const mapping: CsvHeaderMapping = {}
  const columns = tableSchema.columns

  const exactByName = new Map(columns.map((c) => [c.name, c.name]))
  const loose = new Map<string, string>()
  for (const col of columns) {
    loose.set(col.name.toLowerCase().replace(/[^a-z0-9]/g, ''), col.name)
  }

  const usedTargets = new Set<string>()

  for (const header of csvHeaders) {
    const sanitized = sanitizeName(header)
    const exact = exactByName.get(sanitized)
    if (exact && !usedTargets.has(exact)) {
      mapping[header] = exact
      usedTargets.add(exact)
      continue
    }
    const key = header.toLowerCase().replace(/[^a-z0-9]/g, '')
    const fuzzy = loose.get(key)
    if (fuzzy && !usedTargets.has(fuzzy)) {
      mapping[header] = fuzzy
      usedTargets.add(fuzzy)
      continue
    }
    mapping[header] = null
  }

  return mapping
}

/**
 * Coerces parsed CSV rows into `RowData` objects keyed by the target column's
 * **stable id** (the row-data storage key), applying the column types declared in
 * `tableSchema`. Headers not present in `headerToColumn` are dropped. Missing
 * table columns remain unset (schema validation decides whether that's
 * acceptable). Pass the schema returned by `createTable` so ids are resolved.
 */
export function coerceRowsForTable(
  rows: Record<string, unknown>[],
  tableSchema: TableSchema,
  headerToColumn: Map<string, string>,
  options?: NormalizeDateCellOptions
): RowData[] {
  const colByName = new Map(tableSchema.columns.map((c) => [c.name, c]))

  return rows.map((row) => {
    const coerced: RowData = {}
    for (const [header, value] of Object.entries(row)) {
      const colName = headerToColumn.get(header)
      if (!colName) continue
      const col = colByName.get(colName)
      if (!col) continue
      const colType = (col.type as CsvColumnType) ?? 'string'
      coerced[getColumnId(col)] = coerceValue(value, colType, options) as RowData[string]
    }
    return coerced
  })
}

/**
 * Sanitizes raw JSON keys so they conform to the same column-name rules as CSV
 * headers, letting `inferSchemaFromCsv` and `coerceRowsForTable` be reused for
 * JSON imports. Collisions after sanitization are disambiguated with a trailing
 * underscore. Returns the headers and rows untouched when no key needs renaming.
 */
export function sanitizeJsonHeaders(
  headers: string[],
  rows: Record<string, unknown>[]
): { headers: string[]; rows: Record<string, unknown>[] } {
  const renamed = new Map<string, string>()
  const seen = new Set<string>()

  for (const raw of headers) {
    let safe = sanitizeName(raw)
    while (seen.has(safe)) safe = `${safe}_`
    seen.add(safe)
    renamed.set(raw, safe)
  }

  const noChange = headers.every((h) => renamed.get(h) === h)
  if (noChange) return { headers, rows }

  return {
    headers: headers.map((h) => renamed.get(h)!),
    rows: rows.map((row) => {
      const out: Record<string, unknown> = {}
      for (const [raw, safe] of renamed) {
        if (raw in row) out[safe] = row[raw]
      }
      return out
    }),
  }
}

/**
 * Parses a JSON payload that must be an array of plain objects into the same
 * `{ headers, rows }` shape produced by `parseCsvBuffer`. The header set is the
 * union of all object keys, sanitized via {@link sanitizeJsonHeaders}.
 */
export function parseJsonRows(buffer: Buffer | string): {
  headers: string[]
  rows: Record<string, unknown>[]
} {
  const text = typeof buffer === 'string' ? buffer : buffer.toString('utf-8')
  const parsed = JSON.parse(text)
  if (!Array.isArray(parsed)) {
    throw new Error('JSON file must contain an array of objects')
  }
  if (parsed.length === 0) {
    throw new Error('JSON file contains an empty array')
  }
  const headerSet = new Set<string>()
  for (const row of parsed) {
    if (typeof row !== 'object' || row === null || Array.isArray(row)) {
      throw new Error('Each element in the JSON array must be a plain object')
    }
    for (const key of Object.keys(row)) headerSet.add(key)
  }
  return sanitizeJsonHeaders([...headerSet], parsed)
}

/**
 * Parses a tabular upload (CSV, TSV, or JSON array-of-objects) into a uniform
 * `{ headers, rows }` shape, dispatching on file extension and falling back to
 * the MIME content type. Throws on unsupported formats so callers fail fast.
 */
export async function parseFileRows(
  buffer: Buffer,
  fileName: string,
  contentType?: string
): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
  const ext = fileName.split('.').pop()?.toLowerCase()
  if (ext === 'json' || contentType === 'application/json') {
    return parseJsonRows(buffer)
  }
  if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') {
    const delimiter = ext === 'tsv' ? '\t' : ','
    return parseCsvBuffer(buffer, delimiter)
  }
  throw new Error(`Unsupported file format: "${ext ?? fileName}". Supported: csv, tsv, json`)
}