chore: import upstream snapshot with attribution

apps/sim/scripts/backfill-trace-spans.ts

@@ -0,0 +1,188 @@
+#!/usr/bin/env bun
+
+/**
+ * One-shot, idempotent, resumable backfill that externalizes inline heavy
+ * `execution_data` (traceSpans, finalOutput, workflowInput, ...) into the
+ * execution-context large-value store, matching the completion path (cost-stripped
+ * spans, trace pointer + markers, owner/dependency + execution_log reference
+ * registration). Skips running rows and rows already carrying the pointer.
+ *
+ * Requires object storage to be configured; self-hosted deployments without it
+ * keep `execution_data` inline (reads resolve inline transparently) and can skip
+ * this script entirely.
+ *
+ * NOTE: the companion `cost_total` / `models_used` backfill is done in SQL by
+ * migration 0220 (batched, idempotent), so it runs for everyone — including
+ * self-hosted — and is intentionally NOT part of this script.
+ *
+ * Usage:
+ *   DATABASE_URL=... bun apps/sim/scripts/backfill-trace-spans.ts [--max-batches=<n>]
+ */
+
+import { db } from '@sim/db'
+import { workflowExecutionLogs } from '@sim/db/schema'
+import { toError } from '@sim/utils/errors'
+import { and, asc, eq, gt, sql } from 'drizzle-orm'
+import {
+  collectLargeValueReferenceKeys,
+  replaceLargeValueReferenceKeysWithClient,
+} from '@/lib/execution/payloads/large-value-metadata'
+import {
+  externalizeExecutionData,
+  stripSpanCosts,
+  TRACE_STORE_REF_KEY,
+} from '@/lib/logs/execution/trace-store'
+
+const TRACE_BATCH_SIZE = 100
+
+/**
+ * Recursively counts trace spans (matching the completion path). Legacy rows
+ * predate the inline hasTraceSpans/traceSpanCount markers, so we derive them
+ * before externalizing — otherwise a post-expiry degraded read can't report
+ * "trace data expired (N spans)".
+ */
+function countTraceSpans(spans: unknown): number {
+  if (!Array.isArray(spans)) return 0
+  return spans.reduce(
+    (count: number, span) =>
+      count + 1 + countTraceSpans((span as { children?: unknown } | null)?.children),
+    0
+  )
+}
+
+interface Options {
+  maxBatches: number
+}
+
+function parseArgs(argv: string[]): Options {
+  const maxBatchesArg = argv.find((a) => a.startsWith('--max-batches='))
+  const maxBatches = maxBatchesArg
+    ? Number.parseInt(maxBatchesArg.slice('--max-batches='.length), 10)
+    : Number.POSITIVE_INFINITY
+
+  if (Number.isNaN(maxBatches) || maxBatches <= 0) {
+    throw new Error('--max-batches must be a positive integer')
+  }
+
+  return { maxBatches }
+}
+
+/** Externalize inline heavy execution_data into the large-value store. */
+async function backfillTraceStorage(
+  maxBatches: number
+): Promise<{ migrated: number; failed: number }> {
+  let migrated = 0
+  let failed = 0
+  // Keyset cursor by id: every row is visited at most once per run, so rows that
+  // can't be externalized (storage error, oversized) aren't re-selected into an
+  // infinite loop. A fresh re-run (cursor reset) retries any that failed.
+  let lastId = ''
+
+  for (let batch = 0; batch < maxBatches; batch++) {
+    const rows = await db
+      .select({
+        id: workflowExecutionLogs.id,
+        workspaceId: workflowExecutionLogs.workspaceId,
+        workflowId: workflowExecutionLogs.workflowId,
+        executionId: workflowExecutionLogs.executionId,
+        executionData: workflowExecutionLogs.executionData,
+      })
+      .from(workflowExecutionLogs)
+      .where(
+        and(
+          sql`${workflowExecutionLogs.endedAt} IS NOT NULL`,
+          // Skip deleted-workflow rows: externalization requires a workflowId.
+          sql`${workflowExecutionLogs.workflowId} IS NOT NULL`,
+          sql`${workflowExecutionLogs.executionData} ? 'traceSpans'`,
+          sql`NOT (${workflowExecutionLogs.executionData} ? ${TRACE_STORE_REF_KEY})`,
+          lastId ? gt(workflowExecutionLogs.id, lastId) : undefined
+        )
+      )
+      .orderBy(asc(workflowExecutionLogs.id))
+      .limit(TRACE_BATCH_SIZE)
+
+    if (rows.length === 0) break
+
+    for (const row of rows) {
+      try {
+        const executionData = (row.executionData ?? {}) as Record<string, unknown>
+        // Derive the inline markers legacy rows lack so externalizeExecutionData
+        // carries them onto the slim row (they survive object expiry).
+        const traceSpanCount = countTraceSpans(executionData.traceSpans)
+        executionData.hasTraceSpans = traceSpanCount > 0
+        executionData.traceSpanCount = traceSpanCount
+        stripSpanCosts(executionData.traceSpans)
+        // workspace_files.user_id (NOT NULL) needs the execution owner; legacy
+        // rows carry it under executionData.environment.userId. Rows without an
+        // owner can't be externalized — count them as failed and skip.
+        const environment = executionData.environment as { userId?: string } | undefined
+        const ownerUserId = environment?.userId
+        if (!ownerUserId) {
+          failed++
+          continue
+        }
+        const slim = await externalizeExecutionData(executionData, {
+          workspaceId: row.workspaceId,
+          workflowId: row.workflowId,
+          executionId: row.executionId,
+          userId: ownerUserId,
+        })
+
+        if (!(TRACE_STORE_REF_KEY in slim)) {
+          failed++
+          continue
+        }
+
+        await db.transaction(async (tx) => {
+          await tx
+            .update(workflowExecutionLogs)
+            .set({ executionData: slim })
+            .where(eq(workflowExecutionLogs.id, row.id))
+
+          await replaceLargeValueReferenceKeysWithClient(
+            tx,
+            {
+              workspaceId: row.workspaceId,
+              workflowId: row.workflowId,
+              executionId: row.executionId,
+              source: 'execution_log',
+            },
+            collectLargeValueReferenceKeys(slim)
+          )
+        })
+
+        migrated++
+      } catch (error) {
+        failed++
+        console.error(`  [trace] row ${row.id} failed: ${toError(error).message}`)
+      }
+    }
+
+    // Advance the cursor past this batch so failed rows aren't re-selected.
+    lastId = rows[rows.length - 1].id
+
+    console.log(`  [trace] batch ${batch + 1}: migrated ${migrated}, failed ${failed}`)
+  }
+
+  return { migrated, failed }
+}
+
+async function main(): Promise<void> {
+  const options = parseArgs(process.argv.slice(2))
+  const startedAt = Date.now()
+
+  console.log('Backfilling trace storage (externalizing execution_data)…')
+  const { migrated, failed } = await backfillTraceStorage(options.maxBatches)
+  console.log(`Trace storage done: ${migrated} migrated, ${failed} skipped/failed.`)
+
+  console.log(`Backfill complete in ${((Date.now() - startedAt) / 1000).toFixed(1)}s.`)
+}
+
+main()
+  .catch((err) => {
+    console.error('Backfill failed:', err)
+    process.exit(1)
+  })
+  .finally(() => {
+    process.exit(0)
+  })

apps/sim/scripts/build-pi-e2b-template.ts

@@ -0,0 +1,57 @@
+#!/usr/bin/env bun
+
+/**
+ * Builds the E2B sandbox template that powers the Pi Coding Agent cloud mode.
+ *
+ * Layers the `pi` CLI plus git onto E2B's `code-interpreter` base (which already
+ * ships node + python). The cloud backend runs `pi` and `git clone/commit/push`
+ * inside this sandbox, so both must resolve on PATH — the global npm bin and
+ * `/usr/bin` both are.
+ *
+ * Usage:
+ *   E2B_API_KEY=... bun run apps/sim/scripts/build-pi-e2b-template.ts [--name <name>] [--no-cache]
+ *
+ * After it builds, set the printed value in the Sim app's .env:
+ *   E2B_PI_TEMPLATE_ID=<name>
+ * `Sandbox.create` resolves by template name, so use the name (not the ID).
+ */
+
+import { defaultBuildLogger, Template } from '@e2b/code-interpreter'
+
+const DEFAULT_TEMPLATE_NAME = 'sim-pi'
+
+const piTemplate = Template()
+  .fromTemplate('code-interpreter-v1')
+  // git (+ ssh/certs) for clone/commit/push; ripgrep/fd give the agent fast
+  // file search from its bash tool; gh enables richer GitHub workflows.
+  .aptInstall(['git', 'gh', 'openssh-client', 'ca-certificates', 'ripgrep', 'fd-find'])
+  // The `pi` CLI the cloud backend invokes.
+  .npmInstall(['@earendil-works/pi-coding-agent'], { g: true })
+
+async function main() {
+  if (!process.env.E2B_API_KEY) {
+    console.error('E2B_API_KEY is required')
+    process.exit(1)
+  }
+
+  const args = process.argv.slice(2)
+  const nameIdx = args.indexOf('--name')
+  const templateName = nameIdx !== -1 ? args[nameIdx + 1] : DEFAULT_TEMPLATE_NAME
+  const skipCache = args.includes('--no-cache')
+
+  console.log(`Building Pi E2B template: ${templateName}`)
+  console.log(skipCache ? 'Cache: disabled\n' : 'Cache: enabled\n')
+
+  const result = await Template.build(piTemplate, templateName, {
+    onBuildLogs: defaultBuildLogger(),
+    ...(skipCache ? { skipCache: true } : {}),
+  })
+
+  console.log(`\nDone. Template ID: ${result.templateId}`)
+  console.log(`Set in .env: E2B_PI_TEMPLATE_ID=${templateName}`)
+}
+
+main().catch((error) => {
+  console.error('Build failed:', error)
+  process.exit(1)
+})

apps/sim/scripts/check-block-registry.ts

@@ -0,0 +1,320 @@
+#!/usr/bin/env bun
+
+/**
+ * CI check: enforces block-registry invariants that protect the runtime.
+ *
+ * Two checks run in sequence:
+ *
+ * 1. **Subblock ID stability** — diffs the current registry against a base ref
+ *    and fails if any subblock ID was removed without a corresponding entry in
+ *    `SUBBLOCK_ID_MIGRATIONS`. Removing IDs without a migration breaks
+ *    deployed workflows.
+ *
+ * 2. **Canonical-id contract** — for every (block, tool) pair where the tool
+ *    param is `required: true` and `visibility: 'user-only'`, the block must
+ *    expose a subBlock whose `id` or `canonicalParamId` equals the tool param
+ *    id. The serializer's pre-execution validator depends on this contract to
+ *    resolve values via direct lookup; mismatches false-flag fields as missing
+ *    at submit time.
+ *
+ * Usage:
+ *   bun run apps/sim/scripts/check-block-registry.ts [base-ref]
+ *
+ * `base-ref` defaults to `HEAD~1`. In a PR CI pipeline, pass the merge base:
+ *   bun run apps/sim/scripts/check-block-registry.ts origin/main
+ */
+
+import { execSync } from 'child_process'
+import { SUBBLOCK_ID_MIGRATIONS } from '@/lib/workflows/migrations/subblock-migrations'
+import { getAllBlocks, getBlockMeta } from '@/blocks/registry'
+import { tools as toolRegistry } from '@/tools/registry'
+
+const baseRef = process.argv[2] || 'HEAD~1'
+
+const gitRoot = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim()
+const gitOpts = { encoding: 'utf-8' as const, cwd: gitRoot }
+
+type IdMap = Record<string, Set<string>>
+
+/**
+ * Extracts subblock IDs from the `subBlocks: [ ... ]` section of a block
+ * definition. Only grabs the top-level `id:` of each subblock object —
+ * ignores nested IDs inside `options`, `columns`, etc.
+ */
+function extractSubBlockIds(source: string): string[] {
+  const startIdx = source.indexOf('subBlocks:')
+  if (startIdx === -1) return []
+
+  const bracketStart = source.indexOf('[', startIdx)
+  if (bracketStart === -1) return []
+
+  const ids: string[] = []
+  let braceDepth = 0
+  let bracketDepth = 0
+  let i = bracketStart + 1
+  bracketDepth = 1
+
+  while (i < source.length && bracketDepth > 0) {
+    const ch = source[i]
+
+    if (ch === '[') bracketDepth++
+    else if (ch === ']') {
+      bracketDepth--
+      if (bracketDepth === 0) break
+    } else if (ch === '{') {
+      braceDepth++
+      if (braceDepth === 1) {
+        const ahead = source.slice(i, i + 200)
+        const idMatch = ahead.match(/{\s*(?:\/\/[^\n]*\n\s*)*id:\s*['"]([^'"]+)['"]/)
+        if (idMatch) {
+          ids.push(idMatch[1])
+        }
+      }
+    } else if (ch === '}') {
+      braceDepth--
+    }
+
+    i++
+  }
+
+  return ids
+}
+
+function getCurrentIds(): IdMap {
+  const map: IdMap = {}
+  for (const block of getAllBlocks()) {
+    map[block.type] = new Set(block.subBlocks.map((sb) => sb.id))
+  }
+  return map
+}
+
+type PreviousIdsResult =
+  | { kind: 'skip'; reason: string }
+  | { kind: 'noop' }
+  | { kind: 'ok'; map: IdMap }
+
+function getPreviousIds(): PreviousIdsResult {
+  const registryPath = 'apps/sim/blocks/registry.ts'
+  const blocksDir = 'apps/sim/blocks/blocks'
+
+  let hasChanges = false
+  try {
+    const diff = execSync(
+      `git diff --name-only ${baseRef} HEAD -- ${registryPath} ${blocksDir}`,
+      gitOpts
+    ).trim()
+    hasChanges = diff.length > 0
+  } catch {
+    return { kind: 'skip', reason: 'Could not diff against base ref' }
+  }
+
+  if (!hasChanges) {
+    return { kind: 'noop' }
+  }
+
+  const map: IdMap = {}
+
+  try {
+    const blockFiles = execSync(`git ls-tree -r --name-only ${baseRef} -- ${blocksDir}`, gitOpts)
+      .trim()
+      .split('\n')
+      .filter((f) => f.endsWith('.ts') && !f.endsWith('.test.ts'))
+
+    for (const filePath of blockFiles) {
+      let content: string
+      try {
+        content = execSync(`git show ${baseRef}:${filePath}`, gitOpts)
+      } catch {
+        continue
+      }
+
+      const typeMatch = content.match(/BlockConfig\s*=\s*\{[\s\S]*?type:\s*['"]([^'"]+)['"]/)
+      if (!typeMatch) continue
+      const blockType = typeMatch[1]
+
+      const ids = extractSubBlockIds(content)
+      if (ids.length === 0) continue
+
+      map[blockType] = new Set(ids)
+    }
+  } catch (err) {
+    return { kind: 'skip', reason: `Could not read previous block files from ${baseRef}: ${err}` }
+  }
+
+  return { kind: 'ok', map }
+}
+
+type CheckResult =
+  | { kind: 'pass'; message: string }
+  | { kind: 'skip'; message: string }
+  | { kind: 'fail'; errors: string[] }
+
+function checkSubblockIdStability(): CheckResult {
+  const previous = getPreviousIds()
+
+  if (previous.kind === 'skip') {
+    return { kind: 'skip', message: `${previous.reason} — skipping subblock ID stability check` }
+  }
+  if (previous.kind === 'noop') {
+    return {
+      kind: 'skip',
+      message: 'No block definition changes detected — skipping subblock ID stability check',
+    }
+  }
+
+  const current = getCurrentIds()
+  const errors: string[] = []
+
+  for (const [blockType, prevIds] of Object.entries(previous.map)) {
+    const currIds = current[blockType]
+    if (!currIds) continue
+
+    const migrations = SUBBLOCK_ID_MIGRATIONS[blockType] ?? {}
+
+    for (const oldId of prevIds) {
+      if (currIds.has(oldId)) continue
+      if (oldId in migrations) continue
+
+      errors.push(
+        `Block "${blockType}": subblock ID "${oldId}" was removed.\n` +
+          `  → Add a migration in SUBBLOCK_ID_MIGRATIONS (lib/workflows/migrations/subblock-migrations.ts)\n` +
+          `    mapping "${oldId}" to its replacement ID.`
+      )
+    }
+  }
+
+  if (errors.length === 0) {
+    return { kind: 'pass', message: 'Subblock ID stability check passed' }
+  }
+  return { kind: 'fail', errors }
+}
+
+function checkCanonicalIdContract(): CheckResult {
+  const errors: string[] = []
+
+  for (const block of getAllBlocks()) {
+    const access: string[] = block.tools?.access ?? []
+    if (access.length === 0) continue
+
+    // A subBlock with `canonicalParamId` has its raw `id` deleted from `params` during
+    // canonical-group resolution in `extractParams` (serializer/index.ts), so the raw id is
+    // NOT a valid lookup key at execution time — only the canonical is. Tool params must
+    // align with the canonical, not the raw id.
+    const subBlockKeys = new Set<string>()
+    for (const sb of block.subBlocks ?? []) {
+      const canonical = (sb as { canonicalParamId?: string }).canonicalParamId
+      if (canonical) {
+        subBlockKeys.add(canonical)
+      } else if (sb.id) {
+        subBlockKeys.add(sb.id)
+      }
+    }
+
+    for (const toolId of access) {
+      const tool = toolRegistry[toolId]
+      if (!tool) continue
+
+      for (const [paramId, paramConfig] of Object.entries(tool.params ?? {})) {
+        if (!paramConfig || typeof paramConfig !== 'object') continue
+        const required = (paramConfig as { required?: boolean }).required === true
+        const userOnly = (paramConfig as { visibility?: string }).visibility === 'user-only'
+        if (!required || !userOnly) continue
+
+        if (!subBlockKeys.has(paramId)) {
+          errors.push(
+            `Block "${block.type}" → tool "${toolId}": required user-only param "${paramId}" has no subBlock with id or canonicalParamId === "${paramId}".\n` +
+              '  → Rename a subBlock id or canonicalParamId to match the tool param id,\n' +
+              "    and update the block's inputs + tools.config.params mapper to read from that key."
+          )
+        }
+      }
+    }
+  }
+
+  if (errors.length === 0) {
+    return { kind: 'pass', message: 'Canonical-id contract check passed' }
+  }
+  return { kind: 'fail', errors }
+}
+
+/**
+ * Every catalog-visible integration block must expose a `BlockMeta` entry in
+ * `BLOCK_META_REGISTRY`. The integration detail pages
+ * (`getTemplatesForBlock`/`getSuggestedSkillsForBlock`) and the landing catalog
+ * read metas by block type; a `tools`-category block that ships in the toolbar
+ * without a meta renders an empty detail page (no templates, no skills).
+ *
+ * "Catalog-visible integration" mirrors the `isIntegrationBlock` predicate in
+ * `scripts/generate-docs.ts`: `category === 'tools' && !hideFromToolbar`. Core
+ * primitives (`agent`/`api`/`function`/…), hidden/legacy base versions, and
+ * first-party `blocks`-category blocks intentionally carry no meta and are not
+ * checked. `getBlockMeta` resolves through the version suffix, so a versioned
+ * block (e.g. `github_v2`) passes via its base meta.
+ */
+function checkIntegrationMetaCoverage(): CheckResult {
+  const errors: string[] = []
+
+  for (const block of getAllBlocks()) {
+    // Unreleased preview blocks ship no BlockMeta until GA (they are absent
+    // from every catalog surface), so meta coverage must not force one. The
+    // registry projection already hides them here (no visibility context in a
+    // script), but the explicit check keeps this true regardless.
+    const isCatalogIntegration =
+      block.category === 'tools' && !block.hideFromToolbar && !block.preview
+    if (!isCatalogIntegration) continue
+
+    if (!getBlockMeta(block.type)) {
+      errors.push(
+        `Block "${block.type}" is a catalog integration (category: 'tools', not hidden) but has no BlockMeta.\n` +
+          `  → Export a \`${block.type}\`-keyed \`{Service}BlockMeta\` (tags + templates + skills) from its block file\n` +
+          '    and register it in BLOCK_META_REGISTRY (apps/sim/blocks/registry-maps.ts), alphabetically.'
+      )
+    }
+  }
+
+  if (errors.length === 0) {
+    return { kind: 'pass', message: 'Integration meta coverage check passed' }
+  }
+  return { kind: 'fail', errors }
+}
+
+function reportResult(label: string, failureHeader: string, result: CheckResult): boolean {
+  if (result.kind === 'pass') {
+    console.log(`✓ ${result.message}`)
+    return true
+  }
+  if (result.kind === 'skip') {
+    console.log(`⚠ ${result.message}`)
+    return true
+  }
+  console.error(`\n✗ ${label} FAILED\n`)
+  if (failureHeader) console.error(`${failureHeader}\n`)
+  for (const err of result.errors) {
+    console.error(`  ${err}\n`)
+  }
+  return false
+}
+
+const stabilityResult = checkSubblockIdStability()
+const canonicalResult = checkCanonicalIdContract()
+const metaCoverageResult = checkIntegrationMetaCoverage()
+
+const stabilityOk = reportResult(
+  'Subblock ID stability check',
+  'Removing subblock IDs breaks deployed workflows.\nEither revert the rename or add a migration entry.',
+  stabilityResult
+)
+
+const canonicalOk = reportResult(
+  'Canonical-id contract check',
+  "Misaligned ids cause the serializer's pre-execution validator to false-flag fields as missing at submit time.",
+  canonicalResult
+)
+
+const metaCoverageOk = reportResult(
+  'Integration meta coverage check',
+  'Catalog integrations without a BlockMeta render empty detail pages (no templates, no skills).',
+  metaCoverageResult
+)
+
+process.exit(stabilityOk && canonicalOk && metaCoverageOk ? 0 : 1)

apps/sim/scripts/copilot-messages-reconcile.ts

@@ -0,0 +1,81 @@
+#!/usr/bin/env bun
+
+/**
+ * One-shot reconciliation: copy any messages present in
+ * `copilot_chats.messages` JSONB but missing from `copilot_messages`.
+ *
+ * Idempotent via `ON CONFLICT (chat_id, message_id) DO NOTHING`. Safe to
+ * re-run. Intended to be run manually before cutting reads over to the new
+ * table (R+1 of the dual-write rollout), or after a known dual-write outage.
+ *
+ * Usage:
+ *   DATABASE_URL=... bun apps/sim/scripts/copilot-messages-reconcile.ts [--since=<interval>]
+ *
+ * Examples:
+ *   bun apps/sim/scripts/copilot-messages-reconcile.ts
+ *   bun apps/sim/scripts/copilot-messages-reconcile.ts --since='7 days'
+ *   bun apps/sim/scripts/copilot-messages-reconcile.ts --since='1 hour'
+ *
+ * Omit --since to reconcile the entire table.
+ */
+
+import { sql } from 'drizzle-orm'
+import { db } from '../../../packages/db/db.js'
+
+function parseSinceArg(argv: string[]): string | null {
+  const arg = argv.find((a) => a.startsWith('--since='))
+  if (!arg) return null
+  const value = arg.slice('--since='.length).trim()
+  if (!value) {
+    throw new Error('--since requires a value, e.g. --since="7 days"')
+  }
+  if (!/^[\w\s]+$/.test(value)) {
+    throw new Error(`--since value must be a simple interval like "7 days"; got: ${value}`)
+  }
+  return value
+}
+
+async function main(): Promise<void> {
+  const since = parseSinceArg(process.argv.slice(2))
+  const windowClause = since
+    ? sql`AND c.updated_at > now() - ${sql.raw(`interval '${since}'`)}`
+    : sql``
+
+  console.log(
+    since
+      ? `Reconciling copilot_messages for chats updated in the last ${since}…`
+      : 'Reconciling copilot_messages across the full copilot_chats table…'
+  )
+  const startedAt = Date.now()
+
+  const result = await db.execute(sql`
+    INSERT INTO copilot_messages (chat_id, message_id, role, content, model, created_at, updated_at)
+    SELECT
+      c.id,
+      msg.value->>'id',
+      msg.value->>'role',
+      msg.value,
+      c.model,
+      (msg.value->>'timestamp')::timestamptz,
+      (msg.value->>'timestamp')::timestamptz
+    FROM copilot_chats c
+    CROSS JOIN LATERAL jsonb_array_elements(c.messages) AS msg(value)
+    WHERE jsonb_typeof(c.messages) = 'array'
+      AND jsonb_array_length(c.messages) > 0
+      ${windowClause}
+    ON CONFLICT (chat_id, message_id) DO NOTHING
+  `)
+
+  const elapsedMs = Date.now() - startedAt
+  const rowCount = (result as { rowCount?: number }).rowCount ?? 0
+  console.log(`Inserted ${rowCount} new rows in ${(elapsedMs / 1000).toFixed(1)}s.`)
+}
+
+main()
+  .catch((err) => {
+    console.error('Reconciliation failed:', err)
+    process.exit(1)
+  })
+  .finally(() => {
+    process.exit(0)
+  })

apps/sim/scripts/export-workflow.ts

@@ -0,0 +1,120 @@
+#!/usr/bin/env bun
+
+/**
+ * Export workflow JSON from database
+ *
+ * Usage:
+ *   bun apps/sim/scripts/export-workflow.ts <workflow-id>
+ *
+ * This script exports a workflow in the same format as the export API route.
+ * It fetches the workflow state from normalized tables, combines it with metadata
+ * and variables, sanitizes it, and outputs the JSON.
+ *
+ * Make sure DATABASE_URL or POSTGRES_URL is set in your environment.
+ */
+
+// Suppress console logs from imported modules - only JSON should go to stdout
+const originalConsole = {
+  log: console.log,
+  warn: console.warn,
+  error: console.error,
+}
+console.log = () => {}
+console.warn = () => {}
+console.error = () => {}
+
+import { writeFileSync } from 'fs'
+import { eq } from 'drizzle-orm'
+import { db } from '../../../packages/db/db.js'
+import { workflow } from '../../../packages/db/schema.js'
+import { loadWorkflowFromNormalizedTables } from '../lib/workflows/persistence/utils.js'
+import { sanitizeForExport } from '../lib/workflows/sanitization/json-sanitizer.js'
+
+// ---------- CLI argument parsing ----------
+const args = process.argv.slice(2)
+const workflowId = args[0]
+const outputFile = args[1] // Optional output filename
+
+if (!workflowId) {
+  process.stderr.write(
+    'Usage: bun apps/sim/scripts/export-workflow.ts <workflow-id> [output-file]\n'
+  )
+  process.stderr.write('\n')
+  process.stderr.write('Examples:\n')
+  process.stderr.write('  bun apps/sim/scripts/export-workflow.ts abc123\n')
+  process.stderr.write('  bun apps/sim/scripts/export-workflow.ts abc123 workflow.json\n')
+  process.stderr.write('\n')
+  process.stderr.write('Make sure DATABASE_URL or POSTGRES_URL is set in your environment.\n')
+  process.exit(1)
+}
+
+// ---------- Main export function ----------
+async function exportWorkflow(workflowId: string, outputFile?: string): Promise<void> {
+  try {
+    // Fetch workflow metadata
+    const [workflowData] = await db
+      .select()
+      .from(workflow)
+      .where(eq(workflow.id, workflowId))
+      .limit(1)
+
+    if (!workflowData) {
+      process.stderr.write(`Error: Workflow ${workflowId} not found\n`)
+      process.exit(1)
+    }
+
+    // Load workflow from normalized tables
+    const normalizedData = await loadWorkflowFromNormalizedTables(workflowId)
+
+    if (!normalizedData) {
+      process.stderr.write(`Error: Workflow ${workflowId} has no normalized data\n`)
+      process.exit(1)
+    }
+
+    // Get variables in Record format (as stored in database)
+    type VariableType = 'string' | 'number' | 'boolean' | 'object' | 'array' | 'plain'
+    const workflowVariables = workflowData.variables as
+      | Record<string, { id: string; name: string; type: VariableType; value: unknown }>
+      | undefined
+
+    // Prepare export state - match the exact format from the UI
+    const workflowState = {
+      blocks: normalizedData.blocks,
+      edges: normalizedData.edges,
+      loops: normalizedData.loops,
+      parallels: normalizedData.parallels,
+      metadata: {
+        name: workflowData.name,
+        description: workflowData.description ?? undefined,
+        exportedAt: new Date().toISOString(),
+      },
+      variables: workflowVariables,
+    }
+
+    // Sanitize and export - this returns { version, exportedAt, state }
+    const exportState = sanitizeForExport(workflowState)
+    const jsonString = JSON.stringify(exportState, null, 2)
+
+    // Write to file or stdout
+    if (outputFile) {
+      writeFileSync(outputFile, jsonString, 'utf-8')
+      process.stderr.write(`Workflow exported to ${outputFile}\n`)
+    } else {
+      // Output the JSON to stdout only
+      process.stdout.write(`${jsonString}\n`)
+    }
+  } catch (error) {
+    process.stderr.write(`Error exporting workflow: ${error}\n`)
+    process.exit(1)
+  }
+}
+
+// ---------- Execute ----------
+exportWorkflow(workflowId, outputFile)
+  .then(() => {
+    process.exit(0)
+  })
+  .catch((error) => {
+    process.stderr.write(`Unexpected error: ${error}\n`)
+    process.exit(1)
+  })

apps/sim/scripts/load/README.md

@@ -0,0 +1,107 @@
+# Workflow Load Tests
+
+These local-only Artillery scenarios exercise `POST /api/workflows/[id]/execute` in async mode.
+
+## Requirements
+
+- The app should be running locally, for example with `bun run dev:full`
+- Each scenario needs valid workflow IDs and API keys
+- All scenarios default to `http://localhost:3000`
+
+The default rates are tuned for these local limits:
+
+- `ADMISSION_GATE_MAX_INFLIGHT=500`
+
+The admission gate caps total in-flight requests per pod.
+
+## Baseline Concurrency
+
+Use this to ramp traffic into one workflow and observe normal queueing behavior.
+
+Default profile:
+
+- Starts at `2` requests per second
+- Ramps to `8` requests per second
+- Holds there for `20` seconds
+- Good for validating queueing against a Free workspace concurrency of `5`
+
+```bash
+WORKFLOW_ID=<workflow-id> \
+SIM_API_KEY=<api-key> \
+bun run load:workflow:baseline
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `WARMUP_DURATION`
+- `WARMUP_RATE`
+- `PEAK_RATE`
+- `HOLD_DURATION`
+
+For higher-plan workspaces, a good local starting point is:
+
+- Pro: `PEAK_RATE=20` to `40`
+- Team or Enterprise: `PEAK_RATE=50` to `100`
+
+## Queueing Waves
+
+Use this to send repeated bursts to one workflow in the same workspace.
+
+Default profile:
+
+- Wave 1: `6` requests per second for `10` seconds
+- Wave 2: `8` requests per second for `15` seconds
+- Wave 3: `10` requests per second for `20` seconds
+- Quiet gaps: `5` seconds
+
+```bash
+WORKFLOW_ID=<workflow-id> \
+SIM_API_KEY=<api-key> \
+bun run load:workflow:waves
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `WAVE_ONE_DURATION`
+- `WAVE_ONE_RATE`
+- `QUIET_DURATION`
+- `WAVE_TWO_DURATION`
+- `WAVE_TWO_RATE`
+- `WAVE_THREE_DURATION`
+- `WAVE_THREE_RATE`
+
+## Two-Workspace Isolation
+
+Use this to send mixed traffic to two workflows from different workspaces and compare whether one workspace's queue pressure appears to affect the other.
+
+Default profile:
+
+- Total rate: `9` requests per second for `30` seconds
+- Weight split: `8:1`
+- In practice this sends heavy pressure to workspace A while still sending a light stream to workspace B
+
+```bash
+WORKFLOW_ID_A=<workspace-a-workflow-id> \
+SIM_API_KEY_A=<workspace-a-api-key> \
+WORKFLOW_ID_B=<workspace-b-workflow-id> \
+SIM_API_KEY_B=<workspace-b-api-key> \
+bun run load:workflow:isolation
+```
+
+Optional variables:
+
+- `BASE_URL`
+- `ISOLATION_DURATION`
+- `TOTAL_RATE`
+- `WORKSPACE_A_WEIGHT`
+- `WORKSPACE_B_WEIGHT`
+
+## Notes
+
+- `load:workflow` is an alias for `load:workflow:baseline`
+- All scenarios send `x-execution-mode: async`
+- Artillery output will show request counts and response codes, which is usually enough for quick local verification
+- At these defaults, you should see `429` responses when approaching `ADMISSION_GATE_MAX_INFLIGHT=500`
+- If you still see lots of `429` or `ETIMEDOUT` responses locally, lower the rates again before increasing durations

apps/sim/scripts/load/workflow-concurrency.yml

@@ -0,0 +1,24 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.WARMUP_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WARMUP_RATE }}"
+      rampTo: "{{ $processEnvironment.PEAK_RATE }}"
+      name: baseline-ramp
+    - duration: "{{ $processEnvironment.HOLD_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.PEAK_RATE }}"
+      name: baseline-hold
+  defaults:
+    headers:
+      content-type: application/json
+      x-api-key: "{{ $processEnvironment.SIM_API_KEY }}"
+      x-execution-mode: async
+scenarios:
+  - name: baseline-workflow-concurrency
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID }}/execute"
+          json:
+            input:
+              source: artillery-baseline
+              runId: "{{ $uuid }}"

apps/sim/scripts/load/workflow-isolation.yml

@@ -0,0 +1,35 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.ISOLATION_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.TOTAL_RATE }}"
+      name: mixed-workspace-load
+  defaults:
+    headers:
+      content-type: application/json
+      x-execution-mode: async
+scenarios:
+  - name: workspace-a-traffic
+    weight: "{{ $processEnvironment.WORKSPACE_A_WEIGHT }}"
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID_A }}/execute"
+          headers:
+            x-api-key: "{{ $processEnvironment.SIM_API_KEY_A }}"
+          json:
+            input:
+              source: artillery-isolation
+              workspace: a
+              runId: "{{ $uuid }}"
+  - name: workspace-b-traffic
+    weight: "{{ $processEnvironment.WORKSPACE_B_WEIGHT }}"
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID_B }}/execute"
+          headers:
+            x-api-key: "{{ $processEnvironment.SIM_API_KEY_B }}"
+          json:
+            input:
+              source: artillery-isolation
+              workspace: b
+              runId: "{{ $uuid }}"

apps/sim/scripts/load/workflow-waves.yml

@@ -0,0 +1,33 @@
+config:
+  target: "{{ $processEnvironment.BASE_URL }}"
+  phases:
+    - duration: "{{ $processEnvironment.WAVE_ONE_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_ONE_RATE }}"
+      name: wave-one
+    - duration: "{{ $processEnvironment.QUIET_DURATION }}"
+      arrivalRate: 1
+      name: quiet-gap
+    - duration: "{{ $processEnvironment.WAVE_TWO_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_TWO_RATE }}"
+      name: wave-two
+    - duration: "{{ $processEnvironment.QUIET_DURATION }}"
+      arrivalRate: 1
+      name: quiet-gap-two
+    - duration: "{{ $processEnvironment.WAVE_THREE_DURATION }}"
+      arrivalRate: "{{ $processEnvironment.WAVE_THREE_RATE }}"
+      name: wave-three
+  defaults:
+    headers:
+      content-type: application/json
+      x-api-key: "{{ $processEnvironment.SIM_API_KEY }}"
+      x-execution-mode: async
+scenarios:
+  - name: workflow-queue-waves
+    flow:
+      - post:
+          url: "/api/workflows/{{ $processEnvironment.WORKFLOW_ID }}/execute"
+          json:
+            input:
+              source: artillery-waves
+              runId: "{{ $uuid }}"
+              waveProfile: single-workspace

apps/sim/scripts/process-docs.ts

@@ -0,0 +1,262 @@
+#!/usr/bin/env bun
+
+import path from 'path'
+import { db } from '@sim/db'
+import { docsEmbeddings } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { sql } from 'drizzle-orm'
+import { type DocChunk, DocsChunker } from '@/lib/chunkers'
+import { isDev } from '@/lib/core/config/env-flags'
+
+const logger = createLogger('ProcessDocs')
+
+interface ProcessingOptions {
+  /** Clear existing docs embeddings before processing */
+  clearExisting?: boolean
+  /** Path to docs directory */
+  docsPath?: string
+  /** Base URL for generating links */
+  baseUrl?: string
+  /** Chunk size in tokens */
+  chunkSize?: number
+  /** Minimum chunk size in characters */
+  minCharactersPerChunk?: number
+  /** Overlap between chunks in tokens */
+  chunkOverlap?: number
+  /** Dry run - only display results, don't save to DB */
+  dryRun?: boolean
+  /** Verbose output */
+  verbose?: boolean
+}
+
+/**
+ * Process documentation files and optionally save embeddings to database
+ */
+async function processDocs(options: ProcessingOptions = {}) {
+  const config = {
+    docsPath: options.docsPath || path.join(process.cwd(), '../../apps/docs/content/docs/en'),
+    baseUrl: options.baseUrl || (isDev ? 'http://localhost:4000' : 'https://docs.sim.ai'),
+    chunkSize: options.chunkSize || 1024,
+    minCharactersPerChunk: options.minCharactersPerChunk || 100,
+    chunkOverlap: options.chunkOverlap || 200,
+    clearExisting: options.clearExisting ?? false,
+    dryRun: options.dryRun ?? false,
+    verbose: options.verbose ?? false,
+  }
+
+  let processedChunks = 0
+  let failedChunks = 0
+
+  try {
+    logger.info('🚀 Starting docs processing with config:', {
+      docsPath: config.docsPath,
+      baseUrl: config.baseUrl,
+      chunkSize: config.chunkSize,
+      clearExisting: config.clearExisting,
+      dryRun: config.dryRun,
+    })
+
+    // Initialize the chunker
+    const chunker = new DocsChunker({
+      chunkSize: config.chunkSize,
+      minCharactersPerChunk: config.minCharactersPerChunk,
+      chunkOverlap: config.chunkOverlap,
+      baseUrl: config.baseUrl,
+    })
+
+    // Process all .mdx files
+    logger.info(`📚 Processing docs from: ${config.docsPath}`)
+    const chunks = await chunker.chunkAllDocs(config.docsPath)
+
+    if (chunks.length === 0) {
+      logger.warn('⚠️ No chunks generated from docs')
+      return { success: false, processedChunks: 0, failedChunks: 0 }
+    }
+
+    logger.info(`📊 Generated ${chunks.length} chunks with embeddings`)
+
+    // Group chunks by document for summary
+    const chunksByDoc = chunks.reduce<Record<string, DocChunk[]>>((acc, chunk) => {
+      if (!acc[chunk.sourceDocument]) {
+        acc[chunk.sourceDocument] = []
+      }
+      acc[chunk.sourceDocument].push(chunk)
+      return acc
+    }, {})
+
+    // Display summary
+    logger.info(`\n=== DOCUMENT SUMMARY ===`)
+    for (const [doc, docChunks] of Object.entries(chunksByDoc)) {
+      logger.info(`${doc}: ${docChunks.length} chunks`)
+    }
+
+    // Display sample chunks in verbose or dry-run mode
+    if (config.verbose || config.dryRun) {
+      logger.info(`\n=== SAMPLE CHUNKS ===`)
+      chunks.slice(0, 3).forEach((chunk, index) => {
+        logger.info(`\nChunk ${index + 1}:`)
+        logger.info(`  Source: ${chunk.sourceDocument}`)
+        logger.info(`  Header: ${chunk.headerText} (Level ${chunk.headerLevel})`)
+        logger.info(`  Link: ${chunk.headerLink}`)
+        logger.info(`  Tokens: ${chunk.tokenCount}`)
+        logger.info(`  Embedding: ${chunk.embedding.length} dimensions (${chunk.embeddingModel})`)
+        if (config.verbose) {
+          logger.info(`  Text Preview: ${chunk.text.substring(0, 200)}...`)
+        }
+      })
+    }
+
+    // If dry run, stop here
+    if (config.dryRun) {
+      logger.info('\n✅ Dry run complete - no data saved to database')
+      return { success: true, processedChunks: chunks.length, failedChunks: 0 }
+    }
+
+    // Clear existing embeddings if requested
+    if (config.clearExisting) {
+      logger.info('🗑️ Clearing existing docs embeddings...')
+      try {
+        await db.delete(docsEmbeddings)
+        logger.info(`✅ Successfully deleted existing embeddings`)
+      } catch (error) {
+        logger.error('❌ Failed to delete existing embeddings:', error)
+        throw new Error('Failed to clear existing embeddings')
+      }
+    }
+
+    // Save chunks to database in batches
+    const batchSize = 10
+    logger.info(`💾 Saving chunks to database (batch size: ${batchSize})...`)
+
+    for (let i = 0; i < chunks.length; i += batchSize) {
+      const batch = chunks.slice(i, i + batchSize)
+
+      try {
+        const batchData = batch.map((chunk) => ({
+          chunkText: chunk.text,
+          sourceDocument: chunk.sourceDocument,
+          sourceLink: chunk.headerLink,
+          headerText: chunk.headerText,
+          headerLevel: chunk.headerLevel,
+          tokenCount: chunk.tokenCount,
+          embedding: chunk.embedding,
+          embeddingModel: chunk.embeddingModel,
+          metadata: chunk.metadata,
+        }))
+
+        await db.insert(docsEmbeddings).values(batchData)
+        processedChunks += batch.length
+
+        if (i % (batchSize * 5) === 0 || i + batchSize >= chunks.length) {
+          logger.info(
+            `  💾 Saved ${Math.min(i + batchSize, chunks.length)}/${chunks.length} chunks`
+          )
+        }
+      } catch (error) {
+        logger.error(`❌ Failed to save batch ${Math.floor(i / batchSize) + 1}:`, error)
+        failedChunks += batch.length
+      }
+    }
+
+    // Verify final count
+    const savedCount = await db
+      .select({ count: sql<number>`count(*)` })
+      .from(docsEmbeddings)
+      .then((res) => res[0]?.count || 0)
+
+    logger.info(
+      `\n✅ Processing complete!\n` +
+        `   📊 Total chunks: ${chunks.length}\n` +
+        `   ✅ Processed: ${processedChunks}\n` +
+        `   ❌ Failed: ${failedChunks}\n` +
+        `   💾 Total in DB: ${savedCount}`
+    )
+
+    return { success: failedChunks === 0, processedChunks, failedChunks }
+  } catch (error) {
+    logger.error('❌ Fatal error during processing:', error)
+    return { success: false, processedChunks, failedChunks }
+  }
+}
+
+/**
+ * Main entry point with CLI argument parsing
+ */
+async function main() {
+  const args = process.argv.slice(2)
+
+  const options: ProcessingOptions = {
+    clearExisting: args.includes('--clear'),
+    dryRun: args.includes('--dry-run'),
+    verbose: args.includes('--verbose'),
+  }
+
+  // Parse custom path if provided
+  const pathIndex = args.indexOf('--path')
+  if (pathIndex !== -1 && args[pathIndex + 1]) {
+    options.docsPath = args[pathIndex + 1]
+  }
+
+  // Parse custom base URL if provided
+  const urlIndex = args.indexOf('--url')
+  if (urlIndex !== -1 && args[urlIndex + 1]) {
+    options.baseUrl = args[urlIndex + 1]
+  }
+
+  // Parse chunk size if provided
+  const chunkSizeIndex = args.indexOf('--chunk-size')
+  if (chunkSizeIndex !== -1 && args[chunkSizeIndex + 1]) {
+    options.chunkSize = Number.parseInt(args[chunkSizeIndex + 1], 10)
+  }
+
+  // Show help if requested
+  if (args.includes('--help') || args.includes('-h')) {
+    console.log(`
+📚 Process Documentation Script
+
+Usage: bun run process-docs.ts [options]
+
+By default, processes English (en) documentation only.
+Note: Use --clear flag when changing language scope to remove old embeddings.
+
+Options:
+  --clear          Clear existing embeddings before processing
+  --dry-run        Process and display results without saving to DB
+  --verbose        Show detailed output including text previews
+  --path <path>    Custom path to docs directory (default: docs/en)
+  --url <url>      Custom base URL for links
+  --chunk-size <n> Custom chunk size in tokens (default: 1024)
+  --help, -h       Show this help message
+
+Examples:
+  # Dry run to test chunking (English docs)
+  bun run process-docs.ts --dry-run
+
+  # Process and save to database (English docs)
+  bun run process-docs.ts
+
+  # Clear existing and reprocess (English docs)
+  bun run process-docs.ts --clear
+
+  # Process a different language
+  bun run process-docs.ts --path ../../apps/docs/content/docs/es
+
+  # Custom path with verbose output
+  bun run process-docs.ts --path ./my-docs --verbose
+    `)
+    process.exit(0)
+  }
+
+  const result = await processDocs(options)
+  process.exit(result.success ? 0 : 1)
+}
+
+// Run if executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((error) => {
+    logger.error('Fatal error:', error)
+    process.exit(1)
+  })
+}
+
+export { processDocs }

apps/sim/scripts/repair-table-order-key-collation.ts

@@ -0,0 +1,169 @@
+#!/usr/bin/env bun
+
+/**
+ * One-off repair for `user_table_rows.order_key` rows mis-ordered by the
+ * collation bug fixed in migration 0228.
+ *
+ * Fractional `order_key`s are base-62 strings the fractional-indexing library
+ * compares BYTEWISE (ASCII: `0-9 < A-Z < a-z`). Before migration 0228 the column
+ * compared under the database's `en_US.UTF-8` locale, where lowercase interleaves
+ * with/precedes uppercase ("a0" < "Zz", the opposite of bytewise). Keys minted in
+ * that window were anchored to the wrong neighbors, so a table's keys can be
+ * out of order — or duplicated — under bytewise comparison. That makes inserts
+ * throw `generateKeyBetween`'s `a >= b` assertion and rows display out of order.
+ *
+ * This script finds every table whose `order_key`s are mis-assigned: walking rows
+ * in their authoritative `position, id` order, a row's key is `>=` the next row's
+ * under bytewise (`COLLATE "C"`) comparison. That covers swapped keys (a low
+ * position holding a bytewise-larger key than a higher one) and duplicates. Each
+ * flagged table is re-keyed from `position` order — the legacy authoritative order
+ * the original backfill also used — minting a fresh, evenly-spaced, distinct run
+ * with `nKeysBetween`.
+ *
+ * Distinct from the `0001_backfill_table_order_keys` script migration
+ * (`packages/db/script-migrations/`), which keys tables with NULL keys;
+ * this one repairs tables that are fully keyed but bytewise-disordered. Run it
+ * AFTER migration 0228 so the re-key writes and sorts under `COLLATE "C"`.
+ *
+ * Per-table-atomic: each table is re-keyed inside one transaction holding the
+ * same per-table advisory lock the app uses for inserts, so a concurrent insert
+ * can't interleave. Idempotent: a table whose keys are already distinct and
+ * ordered is never selected, so a re-run after a partial failure is safe.
+ *
+ * Usage:
+ *   DATABASE_URL=... bun run apps/sim/scripts/repair-table-order-key-collation.ts
+ *   DATABASE_URL=... bun run apps/sim/scripts/repair-table-order-key-collation.ts --dry-run
+ */
+
+import { userTableRows } from '@sim/db/schema'
+import { getErrorMessage } from '@sim/utils/errors'
+import { asc, eq, sql } from 'drizzle-orm'
+import { drizzle } from 'drizzle-orm/postgres-js'
+import postgres from 'postgres'
+import { nKeysBetween } from '@/lib/table/order-key'
+
+/** Keeps each VALUES list well under Postgres's 65535-bound-param and JS call-stack ceilings. */
+const WRITE_CHUNK_SIZE = 5000
+
+export async function runRepair(): Promise<void> {
+  const dryRun = process.argv.includes('--dry-run')
+  const connectionString = process.env.DATABASE_URL ?? process.env.POSTGRES_URL
+  if (!connectionString) {
+    console.error('Missing DATABASE_URL or POSTGRES_URL')
+    process.exit(1)
+  }
+
+  const client = postgres(connectionString, {
+    prepare: false,
+    idle_timeout: 20,
+    connect_timeout: 30,
+    max: 5,
+    onnotice: () => {},
+  })
+  const db = drizzle(client)
+
+  const stats = { tables: 0, tablesKeyed: 0, rowsKeyed: 0, failed: 0 }
+
+  try {
+    // Tables with a bytewise (`COLLATE "C"`) inversion or duplicate among their
+    // non-null keys. Walking rows in their authoritative `position, id` order (the
+    // order the re-key writes), a healthy table has strictly INCREASING keys; flag
+    // any table where a row's key is `>=` the next row's. Ordering by `position`
+    // (not by `order_key`) is what makes this detect actual mis-assignment — e.g.
+    // pos 0 holding "a0" while pos 1 holds "Zz" (bytewise "Zz" < "a0") — and not
+    // just adjacent duplicates. The explicit `COLLATE "C"` keeps the comparison
+    // bytewise whether or not migration 0228 has been applied yet.
+    const pending = await db.execute<{ table_id: string }>(sql`
+      SELECT DISTINCT table_id FROM (
+        SELECT
+          table_id,
+          order_key,
+          LEAD(order_key) OVER (
+            PARTITION BY table_id ORDER BY position, id
+          ) AS next_key
+        FROM user_table_rows
+        WHERE order_key IS NOT NULL
+      ) t
+      WHERE next_key IS NOT NULL AND order_key COLLATE "C" >= next_key COLLATE "C"
+    `)
+
+    console.log(
+      `Repair starting — ${pending.length} table(s) with mis-ordered keys${dryRun ? ' [DRY RUN]' : ''}`
+    )
+
+    for (const { table_id: tableId } of pending) {
+      stats.tables += 1
+      try {
+        if (dryRun) {
+          // Sizing only — count outside any transaction/lock so we never serialize
+          // live inserts on the table (taking the advisory lock just to count would
+          // make the dry run the opposite of safe).
+          const [row] = await db
+            .select({ rowCount: sql<number>`count(*)`.mapWith(Number) })
+            .from(userTableRows)
+            .where(eq(userTableRows.tableId, tableId))
+          stats.tablesKeyed += 1
+          stats.rowsKeyed += row.rowCount
+          console.log(`  ${tableId}: would re-key ${row.rowCount} rows`)
+          continue
+        }
+
+        const keyed = await db.transaction(async (trx) => {
+          // Serialize with concurrent inserts on this table (same lock the app uses).
+          await trx.execute(
+            sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_rows_pos:${tableId}`}, 0))`
+          )
+          const rows = await trx
+            .select({ id: userTableRows.id })
+            .from(userTableRows)
+            .where(eq(userTableRows.tableId, tableId))
+            .orderBy(asc(userTableRows.position), asc(userTableRows.id))
+
+          if (rows.length === 0) return 0
+          const keys = nKeysBetween(null, null, rows.length)
+
+          // Chunked UPDATE … FROM (VALUES …) mapping id → key (see WRITE_CHUNK_SIZE).
+          for (let start = 0; start < rows.length; start += WRITE_CHUNK_SIZE) {
+            const chunk = rows.slice(start, start + WRITE_CHUNK_SIZE)
+            const values = sql.join(
+              chunk.map((r, i) => sql`(${r.id}, ${keys[start + i]})`),
+              sql`, `
+            )
+            await trx.execute(sql`
+              UPDATE user_table_rows AS t
+              SET order_key = v.order_key
+              FROM (VALUES ${values}) AS v(id, order_key)
+              WHERE t.id = v.id AND t.table_id = ${tableId}
+            `)
+          }
+          return rows.length
+        })
+        stats.tablesKeyed += 1
+        stats.rowsKeyed += keyed
+        console.log(`  ${tableId}: re-keyed ${keyed} rows`)
+      } catch (error) {
+        stats.failed += 1
+        console.error(`  ${tableId}: FAILED — ${getErrorMessage(error)}`)
+      }
+    }
+
+    const verb = dryRun ? 'to re-key' : 're-keyed'
+    console.log(`Repair complete.${dryRun ? ' [DRY RUN — no rows written]' : ''}`)
+    console.log(`  tables scanned: ${stats.tables}`)
+    console.log(`  tables ${verb}: ${stats.tablesKeyed}`)
+    console.log(`  rows ${verb}: ${stats.rowsKeyed}`)
+    console.log(`  failed: ${stats.failed}`)
+    if (stats.failed > 0) process.exitCode = 1
+  } finally {
+    await client.end({ timeout: 5 }).catch(() => {})
+  }
+}
+
+if ((import.meta as { main?: boolean }).main) {
+  try {
+    await runRepair()
+  } catch (error) {
+    console.error('Repair aborted:', getErrorMessage(error))
+    process.exitCode = 1
+  }
+}