chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,162 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
const { mockGenerateId } = vi.hoisted(() => ({
mockGenerateId: vi.fn(),
}))
vi.mock('@sim/utils/id', () => ({
generateId: mockGenerateId,
}))
import {
buildIdByName,
buildNameById,
filterNamesToIds,
generateColumnId,
getColumnId,
remapGroupColumnRefs,
rowDataIdToName,
rowDataNameToId,
sortNamesToIds,
withGeneratedColumnIds,
} from '@/lib/table/column-keys'
import type { TableSchema, WorkflowGroup } from '@/lib/table/types'
describe('getColumnId', () => {
it('returns the explicit id when present', () => {
expect(getColumnId({ id: 'col_abc', name: 'email' })).toBe('col_abc')
})
it('falls back to name for legacy id-less columns', () => {
expect(getColumnId({ name: 'email' })).toBe('email')
})
})
describe('generateColumnId', () => {
it('mints a col_-prefixed id with the uuid dashes stripped', () => {
mockGenerateId.mockReturnValue('11111111-2222-4333-8444-555566667777')
expect(generateColumnId()).toBe('col_11111111222243338444555566667777')
})
it('produces an id that satisfies NAME_PATTERN (valid JSONB key / filter field)', () => {
mockGenerateId.mockReturnValue('0a1b2c3d-4e5f-4607-8809-0a1b2c3d4e5f')
// Must start with a letter/underscore and contain only [a-z0-9_].
expect(generateColumnId()).toMatch(/^[a-z_][a-z0-9_]*$/i)
})
})
describe('name ↔ id maps', () => {
const schema: TableSchema = {
columns: [
{ id: 'col_1', name: 'email', type: 'string' },
{ name: 'age', type: 'number' }, // legacy: id == name
],
}
it('buildIdByName maps display name → storage id', () => {
expect(Object.fromEntries(buildIdByName(schema))).toEqual({ email: 'col_1', age: 'age' })
})
it('buildNameById maps storage id → display name', () => {
expect(Object.fromEntries(buildNameById(schema))).toEqual({ col_1: 'email', age: 'age' })
})
})
describe('row data translation', () => {
const schema: TableSchema = {
columns: [
{ id: 'col_1', name: 'email', type: 'string' },
{ name: 'age', type: 'number' },
],
}
const idByName = buildIdByName(schema)
const nameById = buildNameById(schema)
it('round-trips name → id → name', () => {
const wire = { email: 'a@b.c', age: 30 }
const stored = rowDataNameToId(wire, idByName)
expect(stored).toEqual({ col_1: 'a@b.c', age: 30 })
expect(rowDataIdToName(stored, nameById)).toEqual(wire)
})
it('drops keys with no matching column (orphans / unknowns)', () => {
expect(rowDataNameToId({ email: 'x', ghost: 1 }, idByName)).toEqual({ col_1: 'x' })
expect(rowDataIdToName({ col_1: 'x', col_gone: 9 }, nameById)).toEqual({ email: 'x' })
})
})
describe('filter / sort translation', () => {
const idByName = new Map([
['email', 'col_1'],
['age', 'col_2'],
])
it('translates field names, recurses $or/$and, passes through unknown fields', () => {
const filter = {
email: 'a@b.c',
$or: [{ age: { $gt: 18 } }, { createdAt: { $gt: '2024' } }],
}
expect(filterNamesToIds(filter, idByName)).toEqual({
col_1: 'a@b.c',
$or: [{ col_2: { $gt: 18 } }, { createdAt: { $gt: '2024' } }],
})
})
it('translates sort field names, passes through unknown', () => {
expect(sortNamesToIds({ email: 'asc', createdAt: 'desc' }, idByName)).toEqual({
col_1: 'asc',
createdAt: 'desc',
})
})
})
describe('withGeneratedColumnIds', () => {
it('stamps ids on id-less columns and remaps group refs name → id', () => {
mockGenerateId.mockReturnValueOnce('a').mockReturnValueOnce('b')
const schema: TableSchema = {
columns: [
{ name: 'email', type: 'string', workflowGroupId: 'g1' },
{ name: 'score', type: 'number', workflowGroupId: 'g1' },
],
workflowGroups: [
{
id: 'g1',
workflowId: 'wf',
outputs: [{ blockId: 'b', path: 'p', columnName: 'score' }],
dependencies: { columns: ['email'] },
inputMappings: [{ inputName: 'in', columnName: 'email' }],
},
],
}
const out = withGeneratedColumnIds(schema)
expect(out.columns[0].id).toBe('col_a')
expect(out.columns[1].id).toBe('col_b')
const g = out.workflowGroups![0]
expect(g.outputs[0].columnName).toBe('col_b') // score
expect(g.dependencies!.columns).toEqual(['col_a']) // email
expect(g.inputMappings![0].columnName).toBe('col_a')
})
it('is idempotent for columns that already have an id', () => {
const schema: TableSchema = {
columns: [{ id: 'col_keep', name: 'email', type: 'string' }],
}
expect(withGeneratedColumnIds(schema).columns[0].id).toBe('col_keep')
})
})
describe('remapGroupColumnRefs', () => {
it('rewrites refs that are names, leaves refs that are already ids', () => {
const idByName = new Map([['email', 'col_1']])
const group: WorkflowGroup = {
id: 'g',
workflowId: 'wf',
outputs: [{ blockId: 'b', path: 'p', columnName: 'email' }],
dependencies: { columns: ['col_existing'] },
}
const out = remapGroupColumnRefs(group, idByName)
expect(out.outputs[0].columnName).toBe('col_1')
expect(out.dependencies!.columns).toEqual(['col_existing'])
})
})
@@ -0,0 +1,115 @@
/**
* @vitest-environment node
*
* Unit-tests the result mapping and truncation logic of `findRowMatches`. The
* SQL itself runs against a mocked `db.execute`, so these assertions cover the
* JS-side shaping (ordinal coercion, column rename, LIMIT+1 truncation), not
* the query semantics — those need a real Postgres.
*/
import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
import { sql } from 'drizzle-orm'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { ColumnDefinition, TableDefinition } from '@/lib/table/types'
vi.mock('@sim/db', () => dbChainMock)
vi.mock('@/lib/table/sql', () => ({
buildFilterClause: vi.fn(() => sql`true`),
buildSortClause: vi.fn(() => sql`true`),
escapeLikePattern: vi.fn((s: string) => s),
}))
vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: vi.fn() }))
vi.mock('@/lib/table/workflow-columns', () => ({
assertValidSchema: vi.fn(),
scheduleRunsForRows: vi.fn(),
scheduleRunsForTable: vi.fn(),
stripGroupDeps: vi.fn(),
}))
vi.mock('@/lib/table/validation', () => ({
validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowValues: vi.fn(),
validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
getUniqueColumns: vi.fn(() => []),
checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
}))
import { findRowMatches } from '@/lib/table/rows/service'
import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
const COLUMNS: ColumnDefinition[] = [
{ name: 'name', type: 'string' },
{ name: 'email', type: 'string' },
]
const TABLE: TableDefinition = {
id: 'tbl-1',
name: 'People',
description: null,
schema: { columns: COLUMNS },
metadata: null,
rowCount: 0,
maxRows: 1000,
workspaceId: 'ws-1',
createdBy: 'user-1',
archivedAt: null,
createdAt: new Date('2024-01-01'),
updatedAt: new Date('2024-01-01'),
}
describe('findRowMatches', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})
it('returns empty without querying when the table has no columns', async () => {
const result = await findRowMatches({ ...TABLE, schema: { columns: [] } }, { q: 'x' }, 'req')
expect(result).toEqual({ matches: [], truncated: false })
expect(dbChainMockFns.execute).not.toHaveBeenCalled()
})
it('maps rows to matches, coercing the bigint ordinal and renaming the column', async () => {
dbChainMockFns.execute.mockResolvedValue([
{ ordinal: '2', id: 'r2', column_name: 'name' },
{ ordinal: 5, id: 'r5', column_name: 'email' },
])
const result = await findRowMatches(TABLE, { q: 'a' }, 'req')
expect(result.truncated).toBe(false)
expect(result.matches).toEqual([
{ ordinal: 2, rowId: 'r2', column: 'name' },
{ ordinal: 5, rowId: 'r5', column: 'email' },
])
})
it('flags truncation and caps the result when the DB returns LIMIT+1 rows', async () => {
const over = Array.from({ length: 1001 }, (_, i) => ({
ordinal: i,
id: `r${i}`,
column_name: 'name',
}))
dbChainMockFns.execute.mockResolvedValue(over)
const result = await findRowMatches(TABLE, { q: 'a' }, 'req')
expect(result.truncated).toBe(true)
expect(result.matches).toHaveLength(1000)
})
it('threads filter and sort through the SQL builders', async () => {
dbChainMockFns.execute.mockResolvedValue([])
await findRowMatches(
TABLE,
{ q: 'a', filter: { name: { $contains: 'a' } }, sort: { name: 'asc' } },
'req'
)
expect(buildFilterClause).toHaveBeenCalledWith(
{ name: { $contains: 'a' } },
expect.any(String),
COLUMNS
)
expect(buildSortClause).toHaveBeenCalledWith({ name: 'asc' }, expect.any(String), COLUMNS)
})
})
@@ -0,0 +1,107 @@
/**
* @vitest-environment node
*
* Lock-order regression guard: a column-creating CSV import must acquire the
* per-table row-order advisory lock (`user_table_rows_pos`) BEFORE writing
* `user_table_definitions`, matching the rows_pos → definitions order that plain
* inserts take via the row-count trigger. The opposite order deadlocks
* concurrent inserts on the same table.
*/
import { userTableDefinitions } from '@sim/db/schema'
import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { importAppendRows } from '@/lib/table/import-data'
import type { TableDefinition } from '@/lib/table/types'
vi.mock('@sim/db', () => dbChainMock)
vi.mock('@/lib/core/config/feature-flags', () => ({
isFeatureEnabled: vi.fn().mockResolvedValue(false),
}))
vi.mock('@/lib/table/validation', () => ({
validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowValues: vi.fn((row) => row),
validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
getUniqueColumns: vi.fn(() => []),
checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
}))
const TABLE: TableDefinition = {
id: 'tbl-1',
name: 'People',
description: null,
schema: { columns: [{ name: 'name', type: 'string' }] },
metadata: null,
rowCount: 0,
maxRows: 1000,
workspaceId: 'ws-1',
createdBy: 'user-1',
archivedAt: null,
createdAt: new Date('2024-01-01'),
updatedAt: new Date('2024-01-01'),
}
/**
* invocationCallOrder of the first `tx.execute(...)` whose SQL contains
* `substring` — checks the template strings, the interpolated values (the
* advisory key is passed as a value), and `sql.raw` output.
*/
function executeOrderContaining(substring: string): number {
const { calls, invocationCallOrder } = dbChainMockFns.execute.mock
for (let i = 0; i < calls.length; i++) {
const arg = calls[i][0] as { strings?: unknown; values?: unknown; rawSql?: unknown } | undefined
const haystacks: string[] = []
if (Array.isArray(arg?.strings)) {
haystacks.push(
...(arg.strings as unknown[]).filter((s): s is string => typeof s === 'string')
)
}
if (Array.isArray(arg?.values)) {
haystacks.push(...(arg.values as unknown[]).filter((v): v is string => typeof v === 'string'))
}
if (typeof arg?.rawSql === 'string') haystacks.push(arg.rawSql)
if (haystacks.some((s) => s.includes(substring))) return invocationCallOrder[i]
}
return -1
}
/** invocationCallOrder of the first `tx.update(table)` call. */
function updateOrderForTable(table: unknown): number {
const { calls, invocationCallOrder } = dbChainMockFns.update.mock
for (let i = 0; i < calls.length; i++) {
if (calls[i][0] === table) return invocationCallOrder[i]
}
return -1
}
describe('table import lock ordering', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})
it('acquires the rows_pos advisory before the user_table_definitions write when adding columns', async () => {
// The lock pre-acquire and the column-creating write both run at the top of
// the import, before the batch-insert loop. The loop's order-key aggregates
// aren't fully wired in this unit mock, so tolerate a downstream error after
// the locks under test have been recorded.
await importAppendRows(
TABLE,
[{ name: 'new_col', type: 'string' }],
[{ name: 'Alice', new_col: 'x' }],
{ workspaceId: 'ws-1', userId: 'user-1', requestId: 'req-1' }
).catch(() => {})
const rowsPosLockOrder = executeOrderContaining('user_table_rows_pos')
const definitionsWriteOrder = updateOrderForTable(userTableDefinitions)
expect(rowsPosLockOrder).toBeGreaterThan(0)
expect(definitionsWriteOrder).toBeGreaterThan(0)
expect(rowsPosLockOrder).toBeLessThan(definitionsWriteOrder)
})
})
@@ -0,0 +1,33 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { trimRowsToByteBudget } from '@/lib/table/rows/paging'
function row(id: string, bytes: number) {
// {"b":"aaa…"} serializes to bytes + 8 chars of envelope.
return { id, data: { b: 'a'.repeat(Math.max(0, bytes - 8)) } }
}
describe('trimRowsToByteBudget', () => {
it('returns all rows when they fit the budget', () => {
const rows = [row('r1', 100), row('r2', 100)]
expect(trimRowsToByteBudget(rows, 1000)).toBe(rows)
})
it('keeps the longest prefix within the budget', () => {
const rows = [row('r1', 400), row('r2', 400), row('r3', 400)]
const kept = trimRowsToByteBudget(rows, 900)
expect(kept.map((r) => r.id)).toEqual(['r1', 'r2'])
})
it('always keeps the first row even when it alone exceeds the budget', () => {
const rows = [row('r1', 5000), row('r2', 100)]
const kept = trimRowsToByteBudget(rows, 1000)
expect(kept.map((r) => r.id)).toEqual(['r1'])
})
it('returns empty input unchanged', () => {
expect(trimRowsToByteBudget([], 1000)).toEqual([])
})
})
@@ -0,0 +1,125 @@
/**
* @vitest-environment node
*
* Integration test asserting that `table.schema.columns` is forwarded to
* `buildFilterClause` from each service function that filters rows. This
* guards the contract that type-aware JSONB casts (numeric for numbers,
* timestamp for dates) are always available at the SQL builder layer — the
* latent bug that PR #4657 was originally fixing.
*/
import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
import { sql } from 'drizzle-orm'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
import type { ColumnDefinition, TableDefinition } from '@/lib/table/types'
vi.mock('@sim/db', () => dbChainMock)
vi.mock('@/lib/table/sql', () => ({
buildFilterClause: vi.fn(() => sql`true`),
buildSortClause: vi.fn(() => sql`true`),
}))
vi.mock('@/lib/table/trigger', () => ({
fireTableTrigger: vi.fn(),
}))
vi.mock('@/lib/table/workflow-columns', () => ({
assertValidSchema: vi.fn(),
scheduleRunsForRows: vi.fn(),
scheduleRunsForTable: vi.fn(),
stripGroupDeps: vi.fn(),
}))
vi.mock('@/lib/table/validation', () => ({
validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowValues: vi.fn(),
validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
getUniqueColumns: vi.fn(() => []),
checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
}))
import { deleteRowsByFilter, queryRows, updateRowsByFilter } from '@/lib/table/rows/service'
const COLUMNS: ColumnDefinition[] = [
{ name: 'name', type: 'string' },
{ name: 'birthDate', type: 'date' },
{ name: 'score', type: 'number' },
]
const TABLE: TableDefinition = {
id: 'tbl-1',
name: 'People',
description: null,
schema: { columns: COLUMNS },
metadata: null,
rowCount: 0,
maxRows: 1000,
workspaceId: 'ws-1',
createdBy: 'user-1',
archivedAt: null,
createdAt: new Date('2024-01-01'),
updatedAt: new Date('2024-01-01'),
}
describe('service filter threading', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})
it('queryRows forwards table.schema.columns to buildFilterClause', async () => {
await queryRows(
TABLE,
{ filter: { birthDate: { $gte: '2024-01-01' } }, includeTotal: false },
'req-1'
).catch(() => {})
expect(buildFilterClause).toHaveBeenCalledTimes(1)
expect(buildFilterClause).toHaveBeenCalledWith(
{ birthDate: { $gte: '2024-01-01' } },
expect.any(String),
COLUMNS
)
})
it('queryRows forwards columns to buildSortClause as well', async () => {
await queryRows(TABLE, { sort: { birthDate: 'asc' }, includeTotal: false }, 'req-1').catch(
() => {}
)
expect(buildSortClause).toHaveBeenCalledWith({ birthDate: 'asc' }, expect.any(String), COLUMNS)
})
it('updateRowsByFilter forwards table.schema.columns to buildFilterClause', async () => {
dbChainMockFns.where.mockResolvedValueOnce([])
await updateRowsByFilter(
TABLE,
{ filter: { birthDate: { $lt: '2024-06-01' } }, data: { name: 'x' } },
'req-1'
)
expect(buildFilterClause).toHaveBeenCalledTimes(1)
expect(buildFilterClause).toHaveBeenCalledWith(
{ birthDate: { $lt: '2024-06-01' } },
expect.any(String),
COLUMNS
)
})
it('deleteRowsByFilter forwards table.schema.columns to buildFilterClause', async () => {
dbChainMockFns.where.mockResolvedValueOnce([])
await deleteRowsByFilter(TABLE, { filter: { score: { $gt: 90 } } }, 'req-1')
expect(buildFilterClause).toHaveBeenCalledTimes(1)
expect(buildFilterClause).toHaveBeenCalledWith(
{ score: { $gt: 90 } },
expect.any(String),
COLUMNS
)
})
})
+462
View File
@@ -0,0 +1,462 @@
/**
* @vitest-environment node
*
* SQL Builder Unit Tests
*
* Tests the table SQL query builder. Assertions inspect the generated SQL
* string so cast selection (numeric vs timestamptz) is verified end-to-end.
*
* Rendering: `drizzle-orm` is globally mocked in `vitest.setup.ts`. The mock
* represents tagged-template fragments as `{ strings, values }`, raw fragments
* as `{ rawSql }`, and joined fragments as `{ fragments, separator }`. The
* local `renderSql` helper walks that shape recursively so we can assert real
* substrings like `::timestamptz` against the generated SQL.
*/
import { describe, expect, it } from 'vitest'
import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
import type { ColumnDefinition, Filter, Sort } from '@/lib/table/types'
type SqlNode =
| { strings: ArrayLike<string>; values: unknown[] }
| { rawSql: string }
| { fragments: unknown[]; separator: unknown }
| string
| number
| boolean
| null
| undefined
function isTemplateNode(n: unknown): n is { strings: ArrayLike<string>; values: unknown[] } {
return (
typeof n === 'object' &&
n !== null &&
'strings' in n &&
'values' in n &&
Array.isArray((n as { values: unknown[] }).values)
)
}
function isRawNode(n: unknown): n is { rawSql: string } {
return typeof n === 'object' && n !== null && 'rawSql' in n
}
function isJoinNode(n: unknown): n is { fragments: unknown[]; separator: unknown } {
return (
typeof n === 'object' &&
n !== null &&
'fragments' in n &&
Array.isArray((n as { fragments: unknown[] }).fragments)
)
}
/** Recursively render a mock SQL node into its generated SQL string. */
function renderSql(node: SqlNode | unknown): string {
if (node == null) return String(node)
if (isRawNode(node)) return node.rawSql
if (isJoinNode(node)) {
const sep = isRawNode(node.separator) ? node.separator.rawSql : ', '
return node.fragments.map(renderSql).join(sep)
}
if (isTemplateNode(node)) {
const parts: string[] = []
for (let i = 0; i < node.strings.length; i++) {
parts.push(node.strings[i])
if (i < node.values.length) {
parts.push(renderSql(node.values[i]))
}
}
return parts.join('')
}
if (typeof node === 'string') return `'${node}'`
return String(node)
}
function render(node: unknown): string {
return renderSql(node)
}
const TABLE = 'user_table_rows'
const NO_COLUMNS: ColumnDefinition[] = []
describe('SQL Builder', () => {
describe('buildFilterClause', () => {
it('returns undefined for empty filter', () => {
expect(buildFilterClause({}, TABLE, NO_COLUMNS)).toBeUndefined()
})
it('handles simple equality via JSONB containment', () => {
const out = render(buildFilterClause({ name: 'John' }, TABLE, NO_COLUMNS))
expect(out).toContain('user_table_rows.data @>')
expect(out).toContain('"name":"John"')
})
it('emits ::numeric cast for $gt on a number column', () => {
const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
const out = render(buildFilterClause({ age: { $gt: 18 } }, TABLE, cols))
expect(out).toContain(`(${TABLE}.data->>'age')::numeric > `)
expect(out).not.toContain('::timestamp')
})
it('falls back to ::numeric when column type is unknown', () => {
const out = render(buildFilterClause({ score: { $gte: 5 } }, TABLE, NO_COLUMNS))
expect(out).toContain(`(${TABLE}.data->>'score')::numeric >= `)
expect(out).not.toContain('::timestamp')
})
it('handles $eq operator', () => {
const out = render(buildFilterClause({ status: { $eq: 'active' } }, TABLE, NO_COLUMNS))
expect(out).toContain('"status":"active"')
})
it('handles $ne operator', () => {
const out = render(buildFilterClause({ status: { $ne: 'deleted' } }, TABLE, NO_COLUMNS))
expect(out).toContain('NOT (')
expect(out).toContain('"status":"deleted"')
})
it('handles $in with multiple values via OR of containments', () => {
const out = render(
buildFilterClause({ status: { $in: ['active', 'pending'] } }, TABLE, NO_COLUMNS)
)
expect(out).toContain(' OR ')
expect(out).toContain('"status":"active"')
expect(out).toContain('"status":"pending"')
})
it('handles $nin', () => {
const out = render(
buildFilterClause({ status: { $nin: ['deleted', 'archived'] } }, TABLE, NO_COLUMNS)
)
expect(out).toContain('NOT (')
expect(out).toContain(' AND ')
})
it('handles $contains as ILIKE', () => {
const out = render(buildFilterClause({ name: { $contains: 'john' } }, TABLE, NO_COLUMNS))
expect(out).toContain(`${TABLE}.data->>'name'`)
expect(out).toContain('ILIKE')
expect(out).toContain('%john%')
})
it('handles $ncontains as negated ILIKE that surfaces null cells', () => {
const out = render(buildFilterClause({ name: { $ncontains: 'john' } }, TABLE, NO_COLUMNS))
expect(out).toContain('IS NULL')
expect(out).toContain('NOT ILIKE')
expect(out).toContain('%john%')
})
it('handles $startsWith with a trailing wildcard only', () => {
const out = render(buildFilterClause({ name: { $startsWith: 'jo' } }, TABLE, NO_COLUMNS))
expect(out).toContain('ILIKE')
expect(out).toContain('jo%')
expect(out).not.toContain('%jo%')
})
it('handles $endsWith with a leading wildcard only', () => {
const out = render(buildFilterClause({ file: { $endsWith: '.pdf' } }, TABLE, NO_COLUMNS))
expect(out).toContain('ILIKE')
expect(out).toContain('%.pdf')
})
it('escapes ILIKE wildcards in pattern values', () => {
const out = render(buildFilterClause({ name: { $contains: '50%_off' } }, TABLE, NO_COLUMNS))
expect(out).toContain('50\\%\\_off')
})
it('rejects an empty pattern value rather than matching every row', () => {
for (const op of ['$contains', '$ncontains', '$startsWith', '$endsWith'] as const) {
expect(() =>
buildFilterClause({ name: { [op]: '' } } as Filter, TABLE, NO_COLUMNS)
).toThrow(/requires a non-empty value/)
}
})
it('handles $empty: true as null-or-empty-string check', () => {
const out = render(buildFilterClause({ phone: { $empty: true } }, TABLE, NO_COLUMNS))
expect(out).toContain(`${TABLE}.data->>'phone'`)
expect(out).toContain('IS NULL')
expect(out).toContain("= ''")
expect(out).toContain(' OR ')
})
it('handles $empty: false as present-and-non-empty check', () => {
const out = render(buildFilterClause({ phone: { $empty: false } }, TABLE, NO_COLUMNS))
expect(out).toContain('IS NOT NULL')
expect(out).toContain("<> ''")
expect(out).toContain(' AND ')
})
it('coerces string "true"/"false" $empty operands (lenient raw-API input)', () => {
const truthy = render(
buildFilterClause({ phone: { $empty: 'true' } } as Filter, TABLE, NO_COLUMNS)
)
expect(truthy).toContain('IS NULL')
const falsy = render(
buildFilterClause({ phone: { $empty: 'false' } } as Filter, TABLE, NO_COLUMNS)
)
expect(falsy).toContain('IS NOT NULL')
})
it('throws on a non-boolean $empty operand rather than silently inverting', () => {
expect(() =>
buildFilterClause({ phone: { $empty: 1 } } as unknown as Filter, TABLE, NO_COLUMNS)
).toThrow(/\$empty on column "phone" requires a boolean/)
})
it('joins multiple top-level conditions with AND', () => {
const out = render(
buildFilterClause({ status: 'active', age: { $gt: 18 } }, TABLE, NO_COLUMNS)
)
expect(out).toContain(' AND ')
})
it('handles $or logical operator', () => {
const out = render(
buildFilterClause({ $or: [{ status: 'active' }, { status: 'pending' }] }, TABLE, NO_COLUMNS)
)
expect(out).toContain(' OR ')
})
it('handles $and logical operator', () => {
const out = render(
buildFilterClause({ $and: [{ status: 'active' }, { age: { $gt: 18 } }] }, TABLE, NO_COLUMNS)
)
expect(out).toContain(' AND ')
})
it('handles nested $or and $and', () => {
const out = render(
buildFilterClause(
{ $or: [{ $and: [{ status: 'active' }, { verified: true }] }, { role: 'admin' }] },
TABLE,
NO_COLUMNS
)
)
expect(out).toContain(' OR ')
expect(out).toContain(' AND ')
})
it('skips undefined values', () => {
const result = buildFilterClause({ name: undefined, status: 'active' }, TABLE, NO_COLUMNS)
expect(result).toBeDefined()
})
it('handles boolean / null / numeric primitives', () => {
expect(render(buildFilterClause({ active: true }, TABLE, NO_COLUMNS))).toContain(
'"active":true'
)
expect(render(buildFilterClause({ deleted_at: null }, TABLE, NO_COLUMNS))).toContain(
'"deleted_at":null'
)
expect(render(buildFilterClause({ count: 42 }, TABLE, NO_COLUMNS))).toContain('"count":42')
})
it('throws on invalid field name', () => {
expect(() => buildFilterClause({ 'invalid-field': 'v' }, TABLE, NO_COLUMNS)).toThrow(
'Invalid field name'
)
})
it('throws on invalid operator', () => {
const f = { name: { $invalid: 'value' } } as unknown as Filter
expect(() => buildFilterClause(f, TABLE, NO_COLUMNS)).toThrow('Invalid operator')
})
})
describe('buildFilterClause > date column type', () => {
const dateCols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
it.each([
['$gt', '>'],
['$gte', '>='],
['$lt', '<'],
['$lte', '<='],
] as const)('emits ::timestamptz on both sides for %s on a date column', (operator, sqlOp) => {
const filter = { birthDate: { [operator]: '2024-01-01' } } as Filter
const out = render(buildFilterClause(filter, TABLE, dateCols))
expect(out).toContain(`(${TABLE}.data->>'birthDate')::timestamptz ${sqlOp} `)
expect(out).toContain('::timestamptz')
expect(out).not.toContain('::numeric')
// RHS cast — without it Postgres would compare as text (lexicographic).
expect(out.match(/::timestamptz/g)?.length).toBe(2)
})
it('combined range ($gte + $lte) emits two ::timestamptz pairs', () => {
const out = render(
buildFilterClause(
{ birthDate: { $gte: '2024-01-01', $lte: '2024-12-31' } },
TABLE,
dateCols
)
)
expect(out.match(/::timestamptz/g)?.length).toBe(4)
expect(out).not.toContain('::numeric')
expect(out).toContain(' AND ')
})
it('propagates date cast through nested $and', () => {
const out = render(
buildFilterClause(
{ $and: [{ birthDate: { $gte: '2024-01-01' } }, { birthDate: { $lt: '2025-01-01' } }] },
TABLE,
dateCols
)
)
expect(out).toContain('::timestamptz')
expect(out).not.toContain('::numeric')
})
it('propagates date cast through nested $or', () => {
const out = render(
buildFilterClause(
{ $or: [{ birthDate: { $lt: '2000-01-01' } }, { birthDate: { $gt: '2024-01-01' } }] },
TABLE,
dateCols
)
)
expect(out).toContain('::timestamptz')
expect(out).not.toContain('::numeric')
expect(out).toContain(' OR ')
})
it('a number column in the same query keeps ::numeric (no cross-contamination)', () => {
const cols: ColumnDefinition[] = [
{ name: 'birthDate', type: 'date' },
{ name: 'age', type: 'number' },
]
const out = render(
buildFilterClause({ birthDate: { $gte: '2024-01-01' }, age: { $gt: 18 } }, TABLE, cols)
)
expect(out).toContain('::timestamptz')
expect(out).toContain('::numeric')
})
})
describe('buildFilterClause > range operator value type validation', () => {
it('throws when $gt on a number column receives a string', () => {
const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
expect(() => buildFilterClause({ age: { $gt: 'eighteen' } } as Filter, TABLE, cols)).toThrow(
/column "age" \(number\) requires a number, got string/
)
})
it('throws when $gte on a date column receives a number', () => {
const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
expect(() =>
buildFilterClause({ birthDate: { $gte: 1704067200000 } } as Filter, TABLE, cols)
).toThrow(/column "birthDate" \(date\) requires a date string, got number/)
})
it('throws when $lt on an unknown column (numeric fallback) receives a string', () => {
expect(() =>
buildFilterClause({ score: { $lt: 'high' } } as Filter, TABLE, NO_COLUMNS)
).toThrow(/column "score" \(number\) requires a number, got string/)
})
it('accepts valid number on number column', () => {
const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
expect(() => buildFilterClause({ age: { $gt: 18 } }, TABLE, cols)).not.toThrow()
})
it('accepts valid ISO string on date column', () => {
const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
expect(() =>
buildFilterClause({ birthDate: { $gte: '2024-01-01' } }, TABLE, cols)
).not.toThrow()
})
})
describe('buildSortClause', () => {
it('returns undefined for empty sort', () => {
expect(buildSortClause({}, TABLE, NO_COLUMNS)).toBeUndefined()
})
it('sorts string columns as text (no cast)', () => {
const cols: ColumnDefinition[] = [{ name: 'name', type: 'string' }]
const out = render(buildSortClause({ name: 'asc' }, TABLE, cols))
expect(out).toBe(`${TABLE}.data->>'name' ASC`)
expect(out).not.toContain('::')
})
it('sorts number columns with ::numeric NULLS LAST', () => {
const cols: ColumnDefinition[] = [{ name: 'salary', type: 'number' }]
const out = render(buildSortClause({ salary: 'desc' }, TABLE, cols))
expect(out).toBe(`(${TABLE}.data->>'salary')::numeric DESC NULLS LAST`)
})
it('sorts date columns with ::timestamptz NULLS LAST', () => {
const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
const out = render(buildSortClause({ birthDate: 'asc' }, TABLE, cols))
expect(out).toBe(`(${TABLE}.data->>'birthDate')::timestamptz ASC NULLS LAST`)
})
it('sorts createdAt / updatedAt as direct column refs', () => {
expect(render(buildSortClause({ createdAt: 'desc' }, TABLE, NO_COLUMNS))).toBe(
`${TABLE}.createdAt DESC`
)
expect(render(buildSortClause({ updatedAt: 'asc' }, TABLE, NO_COLUMNS))).toBe(
`${TABLE}.updatedAt ASC`
)
})
it('combines multiple sort fields with commas', () => {
const cols: ColumnDefinition[] = [
{ name: 'name', type: 'string' },
{ name: 'salary', type: 'number' },
]
const out = render(buildSortClause({ name: 'asc', salary: 'desc' }, TABLE, cols))
expect(out).toBe(
`${TABLE}.data->>'name' ASC, (${TABLE}.data->>'salary')::numeric DESC NULLS LAST`
)
})
it('falls back to text sort for unknown column types', () => {
const sort: Sort = { unknownField: 'asc' }
const out = render(buildSortClause(sort, TABLE, NO_COLUMNS))
expect(out).toBe(`${TABLE}.data->>'unknownField' ASC`)
})
it('throws on invalid field name', () => {
const sort: Sort = { 'invalid-field': 'asc' }
expect(() => buildSortClause(sort, TABLE, NO_COLUMNS)).toThrow('Invalid field name')
})
it('throws on invalid direction', () => {
const sort = { name: 'invalid' as 'asc' | 'desc' }
expect(() => buildSortClause(sort, TABLE, NO_COLUMNS)).toThrow('Invalid sort direction')
})
})
describe('Field name validation', () => {
it('accepts valid identifiers', () => {
const valid = ['name', 'user_id', '_private', 'Count123', 'a']
for (const name of valid) {
expect(() => buildFilterClause({ [name]: 'v' }, TABLE, NO_COLUMNS)).not.toThrow()
}
})
it('rejects identifiers starting with a digit', () => {
expect(() => buildFilterClause({ '123name': 'v' }, TABLE, NO_COLUMNS)).toThrow(
'Invalid field name'
)
})
it('rejects identifiers with special characters', () => {
const invalid = ['field-name', 'field.name', 'field name', 'field@name']
for (const name of invalid) {
expect(() => buildFilterClause({ [name]: 'v' }, TABLE, NO_COLUMNS)).toThrow(
'Invalid field name'
)
}
})
it('rejects SQL injection attempts in field names', () => {
const attempts = ["'; DROP TABLE users; --", 'name OR 1=1', 'name; DELETE FROM']
for (const a of attempts) {
expect(() => buildFilterClause({ [a]: 'v' }, TABLE, NO_COLUMNS)).toThrow(
'Invalid field name'
)
}
})
})
})
@@ -0,0 +1,405 @@
/**
* @vitest-environment node
*/
import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { deleteColumn, renameColumn } from '@/lib/table/columns/service'
import {
batchInsertRows,
insertRow,
replaceTableRows,
updateRow,
upsertRow,
} from '@/lib/table/rows/service'
import type { TableDefinition } from '@/lib/table/types'
import { getUniqueColumns } from '@/lib/table/validation'
vi.mock('@sim/db', () => dbChainMock)
// Capacity is exercised in billing.test.ts; here it's a no-op so the timeout-scaling
// suites can use large synthetic row counts without tripping the plan limit.
vi.mock('@/lib/table/billing', () => ({
assertRowCapacity: vi.fn().mockResolvedValue(1_000_000),
notifyTableRowUsage: vi.fn(),
getMaxRowsPerTable: vi.fn().mockResolvedValue(1_000_000),
wouldExceedRowLimit: () => false,
TableRowLimitError: class TableRowLimitError extends Error {},
}))
vi.mock('@/lib/table/validation', () => ({
validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
coerceRowValues: vi.fn(),
validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
getUniqueColumns: vi.fn(() => []),
checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
}))
/**
* Inspects the queued `trx.execute(...)` calls for SQL containing `substring`.
* Works with both `sql\`...\`` (produces `{ strings, values }`) and `sql.raw(...)`
* (produces `{ rawSql }`) from the global drizzle mock.
*/
function findExecutedSqlContaining(substring: string): boolean {
return dbChainMockFns.execute.mock.calls.some(([arg]) => {
if (!arg || typeof arg !== 'object') return false
const a = arg as Record<string, unknown>
if (Array.isArray(a.strings)) {
return (a.strings as string[]).some((s) => typeof s === 'string' && s.includes(substring))
}
if (typeof a.rawSql === 'string') {
return (a.rawSql as string).includes(substring)
}
return false
})
}
function findExecutedRawSql(substring: string): string | undefined {
for (const [arg] of dbChainMockFns.execute.mock.calls) {
if (!arg || typeof arg !== 'object') continue
const raw = (arg as { rawSql?: unknown }).rawSql
if (typeof raw === 'string' && raw.includes(substring)) return raw
}
return undefined
}
const EXISTING_ROW = {
id: 'row-1',
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Alice', age: 30 },
position: 1,
createdAt: new Date('2024-01-01'),
updatedAt: new Date('2024-01-01'),
}
const TABLE: TableDefinition = {
id: 'tbl-1',
name: 'People',
description: null,
schema: {
columns: [
{ name: 'name', type: 'string' },
{ name: 'age', type: 'number' },
],
},
metadata: null,
rowCount: 0,
maxRows: 1000,
workspaceId: 'ws-1',
createdBy: 'user-1',
archivedAt: null,
createdAt: new Date('2024-01-01'),
updatedAt: new Date('2024-01-01'),
}
describe('updateRow — partial merge', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
dbChainMockFns.limit.mockResolvedValue([EXISTING_ROW])
})
it('preserves columns not included in the partial update', async () => {
const result = await updateRow(
{ tableId: 'tbl-1', rowId: 'row-1', data: { age: 31 }, workspaceId: 'ws-1' },
TABLE,
'req-1'
)
expect(result.data).toEqual({ name: 'Alice', age: 31 })
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({ data: { name: 'Alice', age: 31 } })
)
})
it('allows updating a single column without affecting others', async () => {
const result = await updateRow(
{ tableId: 'tbl-1', rowId: 'row-1', data: { name: 'Bob' }, workspaceId: 'ws-1' },
TABLE,
'req-1'
)
expect(result.data).toEqual({ name: 'Bob', age: 30 })
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({ data: { name: 'Bob', age: 30 } })
)
})
it('allows explicitly nulling a field while preserving others', async () => {
const result = await updateRow(
{ tableId: 'tbl-1', rowId: 'row-1', data: { age: null }, workspaceId: 'ws-1' },
TABLE,
'req-1'
)
expect(result.data).toEqual({ name: 'Alice', age: null })
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({ data: { name: 'Alice', age: null } })
)
})
it('handles a full-row update correctly (idempotent merge)', async () => {
const result = await updateRow(
{ tableId: 'tbl-1', rowId: 'row-1', data: { name: 'Bob', age: 25 }, workspaceId: 'ws-1' },
TABLE,
'req-1'
)
expect(result.data).toEqual({ name: 'Bob', age: 25 })
})
it('throws when the row does not exist', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([])
await expect(
updateRow(
{ tableId: 'tbl-1', rowId: 'row-missing', data: { age: 31 }, workspaceId: 'ws-1' },
TABLE,
'req-1'
)
).rejects.toThrow('Row not found')
})
})
describe('insertRow — position race safety (migration 0198 + advisory lock)', () => {
beforeEach(() => {
vi.resetAllMocks()
resetDbChainMock()
vi.mocked(getUniqueColumns).mockReturnValue([])
})
it('auto-position inserts acquire the per-table advisory lock before reading max(position)', async () => {
await expect(
insertRow({ tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1' }, TABLE, 'req-1')
).rejects.toBeDefined()
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
expect(findExecutedSqlContaining('hashtextextended')).toBe(true)
})
it('explicit-position inserts also acquire the advisory lock to serialize order-key minting', async () => {
await expect(
insertRow(
{ tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1', position: 5 },
TABLE,
'req-1'
)
).rejects.toBeDefined()
// A position-based insert resolves its order_key from the neighbor at that
// rank; the lock serializes concurrent minting at the same slot.
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
})
it('batchInsertRows acquires the advisory lock (always auto-positioned)', async () => {
await expect(
batchInsertRows(
{ tableId: 'tbl-1', rows: [{ name: 'a' }, { name: 'b' }], workspaceId: 'ws-1' },
TABLE,
'req-1'
)
).rejects.toBeDefined()
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
})
it('upsertRow skips the advisory lock on the update path (match found)', async () => {
vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
dbChainMockFns.limit.mockResolvedValueOnce([
{
id: 'row-1',
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Alice', age: 30 },
position: 0,
createdAt: new Date(),
updatedAt: new Date(),
},
])
dbChainMockFns.returning.mockResolvedValueOnce([
{
id: 'row-1',
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Alice', age: 31 },
position: 0,
createdAt: new Date(),
updatedAt: new Date(),
},
])
await upsertRow(
{
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Alice', age: 31 },
conflictTarget: 'name',
},
TABLE,
'req-1'
)
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(false)
})
it('upsertRow acquires the advisory lock on the insert path (no match)', async () => {
vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
// Initial existing-row check + post-lock re-check both find no match.
dbChainMockFns.limit.mockResolvedValueOnce([])
dbChainMockFns.limit.mockResolvedValueOnce([])
await expect(
upsertRow(
{
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Bob', age: 25 },
conflictTarget: 'name',
},
TABLE,
'req-1'
)
).rejects.toBeDefined()
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
})
it('upsertRow re-checks after acquiring the lock and switches to UPDATE when a racing tx inserted the row', async () => {
vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
// Initial existing-row check: no match (another tx has not committed yet).
dbChainMockFns.limit.mockResolvedValueOnce([])
// Post-lock re-check: a racing tx just inserted the row.
const racedRow = {
id: 'row-raced',
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Bob', age: 25 },
position: 0,
createdAt: new Date(),
updatedAt: new Date(),
}
dbChainMockFns.limit.mockResolvedValueOnce([racedRow])
// UPDATE returning the patched row.
dbChainMockFns.returning.mockResolvedValueOnce([
{ ...racedRow, data: { name: 'Bob', age: 26 } },
])
const result = await upsertRow(
{
tableId: 'tbl-1',
workspaceId: 'ws-1',
data: { name: 'Bob', age: 26 },
conflictTarget: 'name',
},
TABLE,
'req-1'
)
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
expect(result.operation).toBe('update')
expect(result.row.id).toBe('row-raced')
expect(dbChainMockFns.update).toHaveBeenCalled()
expect(dbChainMockFns.insert).not.toHaveBeenCalled()
})
})
describe('mutation paths — SET LOCAL timeouts', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})
it('insertRow sets the default 10s/3s/5s timeouts', async () => {
await expect(
insertRow({ tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1' }, TABLE, 'req-1')
).rejects.toBeDefined()
expect(findExecutedRawSql("SET LOCAL statement_timeout = '10000ms'")).toBeDefined()
expect(findExecutedRawSql("SET LOCAL lock_timeout = '3000ms'")).toBeDefined()
expect(
findExecutedRawSql("SET LOCAL idle_in_transaction_session_timeout = '5000ms'")
).toBeDefined()
})
it('batchInsertRows raises statement_timeout to 60s', async () => {
await expect(
batchInsertRows(
{ tableId: 'tbl-1', rows: [{ name: 'a' }], workspaceId: 'ws-1' },
TABLE,
'req-1'
)
).rejects.toBeDefined()
expect(findExecutedRawSql("SET LOCAL statement_timeout = '60000ms'")).toBeDefined()
})
it('replaceTableRows scales statement_timeout with (existing + new) row count', async () => {
const bigTable: TableDefinition = { ...TABLE, rowCount: 100_000, maxRows: 1_000_000 }
const payload = Array.from({ length: 50_000 }, (_, i) => ({ name: `row-${i}` }))
await replaceTableRows(
{ tableId: 'tbl-1', workspaceId: 'ws-1', rows: payload },
bigTable,
'req-1'
)
// (100_000 + 50_000) × 3ms/row = 450_000ms; above 120_000 floor, below 600_000 cap
expect(findExecutedRawSql("SET LOCAL statement_timeout = '450000ms'")).toBeDefined()
})
it('replaceTableRows caps scaled timeout at 10 minutes for very large tables', async () => {
const hugeTable: TableDefinition = { ...TABLE, rowCount: 10_000_000, maxRows: 20_000_000 }
await replaceTableRows({ tableId: 'tbl-1', workspaceId: 'ws-1', rows: [] }, hugeTable, 'req-1')
// 10M × 3ms = 30M ms, capped at 600_000ms (10 min)
expect(findExecutedRawSql("SET LOCAL statement_timeout = '600000ms'")).toBeDefined()
})
it('replaceTableRows uses the 120s floor on small tables', async () => {
const smallTable: TableDefinition = { ...TABLE, rowCount: 10 }
await replaceTableRows(
{ tableId: 'tbl-1', workspaceId: 'ws-1', rows: [{ name: 'a' }, { name: 'b' }] },
smallTable,
'req-1'
)
// 12 × 3ms = 36ms → floored at 120_000ms
expect(findExecutedRawSql("SET LOCAL statement_timeout = '120000ms'")).toBeDefined()
})
it('renameColumn is metadata-only — no per-row JSONB rewrite regardless of row count', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([{ ...TABLE, rowCount: 500_000 }])
await renameColumn({ tableId: 'tbl-1', oldName: 'name', newName: 'full_name' }, 'req-1')
// Row data is keyed by the column's stable id (unchanged by a rename), so no
// `user_table_rows` key rewrite is executed — and thus no scaled timeout.
expect(findExecutedSqlContaining('jsonb_build_object')).toBe(false)
})
it('deleteColumn uses the 60s floor on small tables', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([{ ...TABLE, rowCount: 100 }])
await deleteColumn({ tableId: 'tbl-1', columnName: 'age' }, 'req-1')
// 100 × 2ms = 200ms → floored at 60_000ms
expect(findExecutedRawSql("SET LOCAL statement_timeout = '60000ms'")).toBeDefined()
})
it('replaceTableRows acquires the per-table advisory lock to serialize concurrent replaces', async () => {
await replaceTableRows(
{ tableId: 'tbl-1', workspaceId: 'ws-1', rows: [{ name: 'a' }] },
{ ...TABLE, rowCount: 5 },
'req-1'
)
expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
expect(findExecutedSqlContaining('hashtextextended')).toBe(true)
})
})
@@ -0,0 +1,496 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { TABLE_LIMITS } from '@/lib/table/constants'
import {
type ColumnDefinition,
coerceRowToSchema,
coerceRowValues,
getUniqueColumns,
type TableSchema,
validateColumnDefinition,
validateRowAgainstSchema,
validateRowSize,
validateTableName,
validateTableSchema,
validateUniqueConstraints,
} from '@/lib/table/validation'
describe('Validation', () => {
describe('validateTableName', () => {
it('should accept valid table names', () => {
const validNames = ['users', 'user_data', '_private', 'Users123', 'a']
for (const name of validNames) {
const result = validateTableName(name)
expect(result.valid).toBe(true)
expect(result.errors).toHaveLength(0)
}
})
it('should reject empty name', () => {
const result = validateTableName('')
expect(result.valid).toBe(false)
expect(result.errors).toContain('Table name is required')
})
it('should reject null/undefined name', () => {
const result1 = validateTableName(null as unknown as string)
expect(result1.valid).toBe(false)
const result2 = validateTableName(undefined as unknown as string)
expect(result2.valid).toBe(false)
})
it('should reject names starting with number', () => {
const result = validateTableName('123table')
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must start with letter or underscore')
})
it('should reject names with special characters', () => {
const invalidNames = ['table-name', 'table.name', 'table name', 'table@name']
for (const name of invalidNames) {
const result = validateTableName(name)
expect(result.valid).toBe(false)
}
})
it('should reject names exceeding max length', () => {
const longName = 'a'.repeat(TABLE_LIMITS.MAX_TABLE_NAME_LENGTH + 1)
const result = validateTableName(longName)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('exceeds maximum length')
})
})
describe('validateColumnDefinition', () => {
it('should accept valid column definition', () => {
const column: ColumnDefinition = {
name: 'email',
type: 'string',
required: true,
unique: true,
}
const result = validateColumnDefinition(column)
expect(result.valid).toBe(true)
})
it('should accept all valid column types', () => {
const types = ['string', 'number', 'boolean', 'date', 'json'] as const
for (const type of types) {
const result = validateColumnDefinition({ name: 'test', type })
expect(result.valid).toBe(true)
}
})
it('should reject empty column name', () => {
const result = validateColumnDefinition({ name: '', type: 'string' })
expect(result.valid).toBe(false)
expect(result.errors).toContain('Column name is required')
})
it('should reject invalid column type', () => {
const result = validateColumnDefinition({
name: 'test',
type: 'invalid' as any,
})
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('invalid type')
})
it('should reject column name exceeding max length', () => {
const longName = 'a'.repeat(TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH + 1)
const result = validateColumnDefinition({ name: longName, type: 'string' })
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('exceeds maximum length')
})
})
describe('validateTableSchema', () => {
it('should accept valid schema', () => {
const schema: TableSchema = {
columns: [
{ name: 'id', type: 'string', required: true, unique: true },
{ name: 'name', type: 'string', required: true },
{ name: 'age', type: 'number' },
],
}
const result = validateTableSchema(schema)
expect(result.valid).toBe(true)
})
it('should reject empty columns array', () => {
const schema: TableSchema = { columns: [] }
const result = validateTableSchema(schema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Schema must have at least one column')
})
it('should reject duplicate column names', () => {
const schema: TableSchema = {
columns: [
{ name: 'id', type: 'string' },
{ name: 'ID', type: 'number' },
],
}
const result = validateTableSchema(schema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Duplicate column names found')
})
it('should reject null schema', () => {
const result = validateTableSchema(null as unknown as TableSchema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Schema is required')
})
it('should reject schema without columns array', () => {
const result = validateTableSchema({} as TableSchema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Schema must have columns array')
})
it('should reject schema exceeding max columns', () => {
const columns = Array.from({ length: TABLE_LIMITS.MAX_COLUMNS_PER_TABLE + 1 }, (_, i) => ({
name: `col_${i}`,
type: 'string' as const,
}))
const result = validateTableSchema({ columns })
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('exceeds maximum columns')
})
})
describe('validateRowSize', () => {
it('should accept row within size limit', () => {
const data = { name: 'test', value: 123 }
const result = validateRowSize(data)
expect(result.valid).toBe(true)
})
it('should reject row exceeding size limit', () => {
const largeString = 'a'.repeat(TABLE_LIMITS.MAX_ROW_SIZE_BYTES + 1)
const data = { content: largeString }
const result = validateRowSize(data)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('exceeds limit')
})
it('should measure UTF-8 bytes, not UTF-16 code units', () => {
// '工' is one UTF-16 code unit but three UTF-8 bytes — a char-count check
// would accept this row at ~1/3 of the real serialized size.
const chars = Math.ceil(TABLE_LIMITS.MAX_ROW_SIZE_BYTES / 3) + 1
const result = validateRowSize({ content: '工'.repeat(chars) })
expect(result.valid).toBe(false)
})
})
describe('validateRowAgainstSchema', () => {
const schema: TableSchema = {
columns: [
{ name: 'name', type: 'string', required: true },
{ name: 'age', type: 'number' },
{ name: 'active', type: 'boolean' },
{ name: 'created', type: 'date' },
{ name: 'metadata', type: 'json' },
],
}
it('should accept valid row data', () => {
const data = {
name: 'John',
age: 30,
active: true,
created: '2024-01-01',
metadata: { key: 'value' },
}
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
it('should reject missing required field', () => {
const data = { age: 30 }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Missing required field: name')
})
it('should reject wrong type for string field', () => {
const data = { name: 123 }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be string')
})
it('should reject wrong type for number field', () => {
const data = { name: 'John', age: 'thirty' }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be number')
})
it('should reject NaN for number field', () => {
const data = { name: 'John', age: Number.NaN }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be number')
})
it('should reject wrong type for boolean field', () => {
const data = { name: 'John', active: 'yes' }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be boolean')
})
it('should reject invalid date string', () => {
const data = { name: 'John', created: 'not-a-date' }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be valid date')
})
it('should accept valid ISO date string', () => {
const data = { name: 'John', created: '2024-01-15T10:30:00Z' }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
it('should accept Date object', () => {
const data = { name: 'John', created: new Date() }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
it('should allow null for optional fields', () => {
const data = { name: 'John', age: null }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
it('should allow undefined for optional fields', () => {
const data = { name: 'John' }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
it('should allow long strings — cell size is bounded by the row byte cap, not per-value', () => {
const longString = 'a'.repeat(100_000)
const data = { name: longString }
const result = validateRowAgainstSchema(data, schema)
expect(result.valid).toBe(true)
})
})
describe('coerceRowToSchema', () => {
const schema: TableSchema = {
columns: [
{ name: 'name', type: 'string', required: true },
{ name: 'age', type: 'number' },
{ name: 'founded', type: 'number', required: true },
{ name: 'active', type: 'boolean' },
{ name: 'created', type: 'date' },
{ name: 'metadata', type: 'json' },
],
}
it('coerces a numeric string to a number in place', () => {
const data = { name: 'Acme', founded: '1999' }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.founded).toBe(1999)
})
it('nulls an un-coercible value for an optional number column', () => {
const data = { name: 'Acme', founded: 2000, age: 'unknown' }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.age).toBeNull()
})
it('rejects an un-coercible value for a required number column', () => {
const data = { name: 'Acme', founded: 'unknown' }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('founded must be number')
expect(data.founded).toBe('unknown')
})
it('coerces a number to a string for a string column', () => {
const data = { name: 12345, founded: 2000 }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.name).toBe('12345')
})
it('coerces "true"/"false" strings to booleans', () => {
const data = { name: 'Acme', founded: 2000, active: 'false' }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.active).toBe(false)
})
it('coerces an epoch number to an ISO date string', () => {
const epoch = Date.parse('2024-01-15T00:00:00Z')
const data = { name: 'Acme', founded: 2000, created: epoch }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.created).toBe(new Date(epoch).toISOString())
})
it('coerces a Date instance to an ISO date string', () => {
const date = new Date('2024-01-15T00:00:00Z')
const data = { name: 'Acme', founded: 2000, created: date }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.created).toBe(date.toISOString())
})
it('nulls an out-of-range epoch number for an optional date column without throwing', () => {
const data = { name: 'Acme', founded: 2000, created: 1e20 }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.created).toBeNull()
})
it('nulls an invalid Date instance for an optional date column without throwing', () => {
const data = { name: 'Acme', founded: 2000, created: new Date('not-a-date') }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data.created).toBeNull()
})
it('leaves already-correct values untouched and passes through json', () => {
const data = { name: 'Acme', founded: 2000, metadata: { k: 'v' } }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(true)
expect(data).toEqual({ name: 'Acme', founded: 2000, metadata: { k: 'v' } })
})
it('still rejects a missing required field', () => {
const data = { name: 'Acme' }
const result = coerceRowToSchema(data, schema)
expect(result.valid).toBe(false)
expect(result.errors).toContain('Missing required field: founded')
})
})
describe('coerceRowValues', () => {
const schema: TableSchema = {
columns: [
{ name: 'name', type: 'string', required: true },
{ name: 'founded', type: 'number', required: true },
{ name: 'age', type: 'number' },
],
}
it('coerces a partial patch in place without flagging absent required fields', () => {
const patch = { age: '42' }
coerceRowValues(patch, schema)
expect(patch.age).toBe(42)
})
it('nulls an un-coercible optional value in a patch', () => {
const patch: { age: unknown } = { age: 'nope' }
coerceRowValues(patch as never, schema)
expect(patch.age).toBeNull()
})
it('leaves an un-coercible required value in place for downstream validation', () => {
const patch: { founded: unknown } = { founded: 'nope' }
coerceRowValues(patch as never, schema)
expect(patch.founded).toBe('nope')
})
})
describe('getUniqueColumns', () => {
it('should return only columns with unique=true', () => {
const schema: TableSchema = {
columns: [
{ name: 'id', type: 'string', unique: true },
{ name: 'email', type: 'string', unique: true },
{ name: 'name', type: 'string' },
{ name: 'count', type: 'number', unique: false },
],
}
const result = getUniqueColumns(schema)
expect(result).toHaveLength(2)
expect(result.map((c) => c.name)).toEqual(['id', 'email'])
})
it('should return empty array when no unique columns', () => {
const schema: TableSchema = {
columns: [
{ name: 'name', type: 'string' },
{ name: 'value', type: 'number' },
],
}
const result = getUniqueColumns(schema)
expect(result).toHaveLength(0)
})
})
describe('validateUniqueConstraints', () => {
const schema: TableSchema = {
columns: [
{ name: 'id', type: 'string', unique: true },
{ name: 'email', type: 'string', unique: true },
{ name: 'name', type: 'string' },
],
}
const existingRows = [
{ id: 'row1', data: { id: 'abc123', email: 'john@example.com', name: 'John' } },
{ id: 'row2', data: { id: 'def456', email: 'jane@example.com', name: 'Jane' } },
]
it('should accept data with unique values', () => {
const data = { id: 'xyz789', email: 'new@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(true)
})
it('should reject duplicate unique value', () => {
const data = { id: 'abc123', email: 'new@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(false)
expect(result.errors[0]).toContain('must be unique')
expect(result.errors[0]).toContain('abc123')
})
it('should be case-insensitive for string comparisons', () => {
const data = { id: 'ABC123', email: 'new@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(false)
})
it('should exclude specified row from checks (for updates)', () => {
const data = { id: 'abc123', email: 'john@example.com', name: 'John Updated' }
const result = validateUniqueConstraints(data, schema, existingRows, 'row1')
expect(result.valid).toBe(true)
})
it('should allow null values for unique columns', () => {
const data = { id: null, email: 'new@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(true)
})
it('should allow undefined values for unique columns', () => {
const data = { email: 'new@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(true)
})
it('should report multiple violations', () => {
const data = { id: 'abc123', email: 'john@example.com', name: 'New User' }
const result = validateUniqueConstraints(data, schema, existingRows)
expect(result.valid).toBe(false)
expect(result.errors).toHaveLength(2)
})
})
})