chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,78 @@
/**
* @vitest-environment node
*
* Tests for the pure helpers used by `backfill-api-key-hash.ts`. The script
* itself does I/O against Postgres and is not tested here; idempotency is a
* property of the helpers — running them twice on the same input produces the
* same output, so re-running the script after an interruption recomputes the
* same hash for every row.
*/
import { createCipheriv } from 'crypto'
import { describe, expect, it } from 'vitest'
import { deriveKeyHashForStoredKey, hashApiKey, isEncryptedKey } from './backfill-api-key-hash'
const FIXED_ENCRYPTION_KEY = '0'.repeat(64)
function encryptForTest(plainKey: string, keyHex: string): string {
const key = Buffer.from(keyHex, 'hex')
const iv = Buffer.from('11'.repeat(16), 'hex')
const cipher = createCipheriv('aes-256-gcm', key, iv, { authTagLength: 16 })
let encrypted = cipher.update(plainKey, 'utf8', 'hex')
encrypted += cipher.final('hex')
const authTag = cipher.getAuthTag()
return `${iv.toString('hex')}:${encrypted}:${authTag.toString('hex')}`
}
describe('hashApiKey', () => {
it('is deterministic', () => {
expect(hashApiKey('sim_abc')).toBe(hashApiKey('sim_abc'))
})
it('returns a 64-char hex digest', () => {
expect(hashApiKey('sim_abc')).toMatch(/^[0-9a-f]{64}$/)
})
})
describe('isEncryptedKey', () => {
it('detects "iv:encrypted:authTag" format', () => {
expect(isEncryptedKey('aa:bb:cc')).toBe(true)
})
it('rejects plain text', () => {
expect(isEncryptedKey('sim_plain')).toBe(false)
})
it('rejects strings with the wrong number of colons', () => {
expect(isEncryptedKey('a:b')).toBe(false)
expect(isEncryptedKey('a:b:c:d')).toBe(false)
})
})
describe('deriveKeyHashForStoredKey — backfill idempotency', () => {
it('hashes a legacy plain-text row', () => {
const hash = deriveKeyHashForStoredKey('sim_legacy-plain', null)
expect(hash).toBe(hashApiKey('sim_legacy-plain'))
})
it('decrypts + hashes an encrypted row, matching the plaintext hash', () => {
const plainKey = 'sk-sim-some-example-key'
const encrypted = encryptForTest(plainKey, FIXED_ENCRYPTION_KEY)
const hash = deriveKeyHashForStoredKey(encrypted, FIXED_ENCRYPTION_KEY)
expect(hash).toBe(hashApiKey(plainKey))
})
it('produces the same hash when re-run on the same row', () => {
const plainKey = 'sk-sim-idempotent'
const encrypted = encryptForTest(plainKey, FIXED_ENCRYPTION_KEY)
const first = deriveKeyHashForStoredKey(encrypted, FIXED_ENCRYPTION_KEY)
const second = deriveKeyHashForStoredKey(encrypted, FIXED_ENCRYPTION_KEY)
expect(first).toBe(second)
})
it('throws when the row looks encrypted but no encryption key is supplied', () => {
const encrypted = encryptForTest('sk-sim-x', FIXED_ENCRYPTION_KEY)
expect(() => deriveKeyHashForStoredKey(encrypted, null)).toThrow(/API_ENCRYPTION_KEY/)
})
})
@@ -0,0 +1,230 @@
#!/usr/bin/env bun
/**
* Backfills the `key_hash` column on the `api_key` table.
*
* The authentication hot path is being rewritten to look up API keys by the
* SHA-256 hash of the plain-text key — a single indexed equality lookup rather
* than a full-table scan + AES-GCM decrypt loop. This script populates
* `key_hash` for every existing row so the new fast path can match historic
* keys.
*
* For each row where `key_hash IS NULL`:
* - If `key` is in encrypted format (iv:encrypted:authTag), decrypt it using
* `API_ENCRYPTION_KEY` to recover the plain-text key.
* - Otherwise treat `key` as legacy plain text.
* - Compute `sha256(plainKey)` and update the row.
*
* The script is idempotent: it only touches rows where `key_hash IS NULL`, and
* re-running after a partial failure continues where it left off.
*
* Usage:
* POSTGRES_URL=... API_ENCRYPTION_KEY=... \
* bun run packages/db/scripts/backfill-api-key-hash.ts
* # or
* POSTGRES_URL=... API_ENCRYPTION_KEY=... \
* bun run packages/db/scripts/backfill-api-key-hash.ts --dry-run
*/
import { createCipheriv, createDecipheriv, createHash } from 'crypto'
import { getErrorMessage } from '@sim/utils/errors'
import { and, eq, isNull, sql } from 'drizzle-orm'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
import { apiKey } from '../schema'
const BATCH_SIZE = 500
export function isEncryptedKey(storedKey: string): boolean {
return storedKey.includes(':') && storedKey.split(':').length === 3
}
export function hashApiKey(plainKey: string): string {
return createHash('sha256').update(plainKey, 'utf8').digest('hex')
}
function decryptApiKey(encryptedValue: string, apiEncryptionKey: string): string {
const parts = encryptedValue.split(':')
if (parts.length !== 3) {
return encryptedValue
}
const key = Buffer.from(apiEncryptionKey, 'hex')
const [ivHex, encrypted, authTagHex] = parts
if (!ivHex || !encrypted || !authTagHex) {
throw new Error('Invalid encrypted api_key format. Expected "iv:encrypted:authTag"')
}
const iv = Buffer.from(ivHex, 'hex')
const authTag = Buffer.from(authTagHex, 'hex')
const decipher = createDecipheriv('aes-256-gcm', key, iv, { authTagLength: 16 })
decipher.setAuthTag(authTag)
let decrypted = decipher.update(encrypted, 'hex', 'utf8')
decrypted += decipher.final('utf8')
return decrypted
}
/**
* Computes the hash to write to `key_hash` for a row during backfill. Pure:
* no I/O, no globals — safe to import from tests. Throws when the stored
* value looks encrypted but the caller has no encryption key.
*/
export function deriveKeyHashForStoredKey(
storedKey: string,
apiEncryptionKey: string | null
): string {
if (isEncryptedKey(storedKey)) {
if (!apiEncryptionKey) {
throw new Error('API_ENCRYPTION_KEY is required to decrypt an encrypted stored key')
}
return hashApiKey(decryptApiKey(storedKey, apiEncryptionKey))
}
return hashApiKey(storedKey)
}
interface BackfillStats {
scanned: number
updated: number
skippedEncryptedNoKey: number
failed: number
}
export async function runBackfill(): Promise<void> {
const dryRun = process.argv.includes('--dry-run')
const connectionString = process.env.POSTGRES_URL ?? process.env.DATABASE_URL
if (!connectionString) {
console.error('Missing POSTGRES_URL or DATABASE_URL environment variable')
process.exit(1)
}
const apiEncryptionKey = process.env.API_ENCRYPTION_KEY ?? null
if (!apiEncryptionKey) {
console.warn(
'API_ENCRYPTION_KEY is not set. Rows whose stored key is encrypted will fail to decrypt. ' +
'Only rows whose stored key is already plain text will be backfilled in this run.'
)
} else if (apiEncryptionKey.length !== 64) {
console.error('API_ENCRYPTION_KEY must be a 64-character hex string (32 bytes)')
process.exit(1)
}
assertCryptoRoundTrip(apiEncryptionKey)
const postgresClient = postgres(connectionString, {
prepare: false,
idle_timeout: 20,
connect_timeout: 30,
max: 5,
onnotice: () => {},
})
const db = drizzle(postgresClient)
const stats: BackfillStats = {
scanned: 0,
updated: 0,
skippedEncryptedNoKey: 0,
failed: 0,
}
try {
const [{ count: pendingBefore }] = await db
.select({ count: sql<number>`count(*)::int` })
.from(apiKey)
.where(isNull(apiKey.keyHash))
console.log(
`Backfill starting — ${pendingBefore} row(s) with NULL key_hash${dryRun ? ' [DRY RUN]' : ''}`
)
for (;;) {
const rows = await db
.select({ id: apiKey.id, key: apiKey.key })
.from(apiKey)
.where(isNull(apiKey.keyHash))
.limit(BATCH_SIZE)
if (rows.length === 0) break
await Promise.all(
rows.map(async (row) => {
stats.scanned += 1
try {
if (isEncryptedKey(row.key) && !apiEncryptionKey) {
stats.skippedEncryptedNoKey += 1
return
}
const keyHash = deriveKeyHashForStoredKey(row.key, apiEncryptionKey)
if (dryRun) {
stats.updated += 1
return
}
await db
.update(apiKey)
.set({ keyHash })
.where(and(eq(apiKey.id, row.id), isNull(apiKey.keyHash)))
stats.updated += 1
} catch (error) {
stats.failed += 1
console.error(`Failed to backfill api_key id=${row.id}: ${getErrorMessage(error)}`)
}
})
)
console.log(
` progress: scanned=${stats.scanned} updated=${stats.updated} skipped=${stats.skippedEncryptedNoKey} failed=${stats.failed}`
)
if (dryRun) break
}
const [{ count: pendingAfter }] = await db
.select({ count: sql<number>`count(*)::int` })
.from(apiKey)
.where(isNull(apiKey.keyHash))
console.log('Backfill complete.')
console.log(` scanned: ${stats.scanned}`)
console.log(` updated: ${stats.updated}`)
console.log(` skipped (no api key): ${stats.skippedEncryptedNoKey}`)
console.log(` failed: ${stats.failed}`)
console.log(` remaining null: ${pendingAfter}`)
if (stats.failed > 0 || pendingAfter > 0) {
process.exitCode = 1
}
} finally {
await postgresClient.end({ timeout: 5 }).catch(() => {})
}
}
/** Fails fast if the AES-GCM round-trip disagrees with itself in this env. */
function assertCryptoRoundTrip(apiEncryptionKey: string | null): void {
if (!apiEncryptionKey) return
const key = Buffer.from(apiEncryptionKey, 'hex')
const sample = 'sk-sim-roundtrip-test-value'
const iv = Buffer.from('00'.repeat(16), 'hex')
const cipher = createCipheriv('aes-256-gcm', key, iv, { authTagLength: 16 })
let encrypted = cipher.update(sample, 'utf8', 'hex')
encrypted += cipher.final('hex')
const authTag = cipher.getAuthTag()
const assembled = `${iv.toString('hex')}:${encrypted}:${authTag.toString('hex')}`
const roundTripped = decryptApiKey(assembled, apiEncryptionKey)
if (roundTripped !== sample) {
throw new Error('Crypto self-test failed — refusing to run backfill')
}
}
if ((import.meta as { main?: boolean }).main) {
try {
await runBackfill()
} catch (error) {
console.error('Backfill aborted:', getErrorMessage(error))
process.exitCode = 1
}
}
@@ -0,0 +1,171 @@
#!/usr/bin/env bun
/**
* Deregister SSO Provider Script
*
* This script removes an SSO provider from the database for a specific user.
*
* Usage: bun run packages/db/scripts/deregister-sso-provider.ts
*
* Required Environment Variables:
* DATABASE_URL=your-database-url
* SSO_USER_EMAIL=user@domain.com (user whose SSO provider to remove)
* SSO_PROVIDER_ID=provider-id (optional, if not provided will remove all providers for user)
*/
import { getErrorMessage } from '@sim/utils/errors'
import { and, eq } from 'drizzle-orm'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
import { ssoProvider, user } from '../schema'
const logger = {
info: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.log(
`[${timestamp}] [INFO] [DeregisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
error: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.error(
`[${timestamp}] [ERROR] [DeregisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
warn: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.warn(
`[${timestamp}] [WARN] [DeregisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
}
const CONNECTION_STRING = process.env.POSTGRES_URL ?? process.env.DATABASE_URL
if (!CONNECTION_STRING) {
console.error('❌ POSTGRES_URL or DATABASE_URL environment variable is required')
process.exit(1)
}
const postgresClient = postgres(CONNECTION_STRING, {
prepare: false,
idle_timeout: 20,
connect_timeout: 30,
max: 10,
onnotice: () => {},
})
const db = drizzle(postgresClient)
async function getUser(email: string): Promise<{ id: string; email: string } | null> {
try {
const users = await db.select().from(user).where(eq(user.email, email))
if (users.length === 0) {
logger.error(`No user found with email: ${email}`)
return null
}
return { id: users[0].id, email: users[0].email }
} catch (error) {
logger.error('Failed to query user:', error)
return null
}
}
async function deregisterSSOProvider(): Promise<boolean> {
try {
const userEmail = process.env.SSO_USER_EMAIL
if (!userEmail) {
logger.error('❌ SSO_USER_EMAIL environment variable is required')
logger.error('')
logger.error('Example usage:')
logger.error(
' SSO_USER_EMAIL=admin@company.com bun run packages/db/scripts/deregister-sso-provider.ts'
)
logger.error('')
logger.error('Optional: SSO_PROVIDER_ID=provider-id (to remove specific provider)')
return false
}
const targetUser = await getUser(userEmail)
if (!targetUser) {
return false
}
logger.info(`Found user: ${targetUser.email} (ID: ${targetUser.id})`)
const providers = await db
.select()
.from(ssoProvider)
.where(eq(ssoProvider.userId, targetUser.id))
if (providers.length === 0) {
logger.warn(`No SSO providers found for user: ${targetUser.email}`)
return false
}
logger.info(`Found ${providers.length} SSO provider(s) for user ${targetUser.email}`)
for (const provider of providers) {
logger.info(` - Provider ID: ${provider.providerId}, Domain: ${provider.domain}`)
}
const specificProviderId = process.env.SSO_PROVIDER_ID
if (specificProviderId) {
const providerToDelete = providers.find((p) => p.providerId === specificProviderId)
if (!providerToDelete) {
logger.error(`Provider '${specificProviderId}' not found for user ${targetUser.email}`)
return false
}
await db
.delete(ssoProvider)
.where(
and(eq(ssoProvider.userId, targetUser.id), eq(ssoProvider.providerId, specificProviderId))
)
logger.info(
`✅ Successfully deleted SSO provider '${specificProviderId}' for user ${targetUser.email}`
)
} else {
await db.delete(ssoProvider).where(eq(ssoProvider.userId, targetUser.id))
logger.info(
`✅ Successfully deleted all ${providers.length} SSO provider(s) for user ${targetUser.email}`
)
}
return true
} catch (error) {
logger.error('❌ Failed to deregister SSO provider:', {
error: getErrorMessage(error, 'Unknown error'),
stack: error instanceof Error ? error.stack : undefined,
})
return false
} finally {
try {
await postgresClient.end({ timeout: 5 })
} catch {}
}
}
async function main() {
console.log('🗑️ Deregister SSO Provider Script')
console.log('====================================')
console.log('This script removes SSO provider records from the database.\n')
const success = await deregisterSSOProvider()
if (success) {
console.log('\n🎉 SSO provider deregistration completed successfully!')
process.exit(0)
} else {
console.log('\n💥 SSO deregistration failed. Check the logs above for details.')
process.exit(1)
}
}
main().catch((error) => {
logger.error('Script execution failed:', { error })
process.exit(1)
})
@@ -0,0 +1,756 @@
#!/usr/bin/env bun
// Self-contained script for migrating block-level API keys into workspace BYOK keys.
// Iterates per workspace. Original block-level values are left untouched for safety.
// Handles both literal keys ("sk-xxx...") and env var references ("{{VAR_NAME}}").
//
// Usage:
// # Step 1 — Dry run: audit for conflicts + preview inserts (no DB writes)
// # Outputs migrate-byok-workspace-ids.txt for the live run.
// bun run packages/db/scripts/migrate-block-api-keys-to-byok.ts --dry-run \
// --map jina=jina --map perplexity=perplexity --map google_books=google_cloud
//
// # Step 2 — Live run: insert BYOK keys (--from-file is required)
// bun run packages/db/scripts/migrate-block-api-keys-to-byok.ts \
// --map jina=jina --map perplexity=perplexity --map google_books=google_cloud \
// --from-file migrate-byok-workspace-ids.txt
//
// # Optionally scope dry run to specific users (repeatable)
// bun run packages/db/scripts/migrate-block-api-keys-to-byok.ts --dry-run \
// --map jina=jina --user user_abc123 --user user_def456
import { createCipheriv, createDecipheriv, randomBytes } from 'crypto'
import { appendFileSync, readFileSync, writeFileSync } from 'fs'
import { resolve } from 'path'
import { sleep } from '@sim/utils/helpers'
import { generateId } from '@sim/utils/id'
import { eq, sql } from 'drizzle-orm'
import { index, json, jsonb, pgTable, text, timestamp } from 'drizzle-orm/pg-core'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
// ---------- CLI ----------
const DRY_RUN = process.argv.includes('--dry-run')
function parseMapArgs(): Record<string, string> {
const mapping: Record<string, string> = {}
const args = process.argv.slice(2)
for (let i = 0; i < args.length; i++) {
if (args[i] === '--map' && args[i + 1]) {
const [blockType, providerId] = args[i + 1].split('=')
if (blockType && providerId) {
mapping[blockType] = providerId
} else {
console.error(
`Invalid --map value: "${args[i + 1]}". Expected format: blockType=providerId`
)
process.exit(1)
}
i++
}
}
return mapping
}
const BLOCK_TYPE_TO_PROVIDER = parseMapArgs()
if (Object.keys(BLOCK_TYPE_TO_PROVIDER).length === 0) {
console.error('No --map arguments provided. Specify at least one: --map blockType=providerId')
console.error(
'Example: --map jina=jina --map perplexity=perplexity --map google_books=google_cloud'
)
process.exit(1)
}
function parseUserArgs(): string[] {
const users: string[] = []
const args = process.argv.slice(2)
for (let i = 0; i < args.length; i++) {
if (args[i] === '--user' && args[i + 1]) {
users.push(args[i + 1])
i++
}
}
return users
}
const USER_FILTER = parseUserArgs()
function parseFromFileArg(): string | null {
const args = process.argv.slice(2)
for (let i = 0; i < args.length; i++) {
if (args[i] === '--from-file' && args[i + 1]) {
return args[i + 1]
}
}
return null
}
const FROM_FILE = parseFromFileArg()
if (!DRY_RUN && !FROM_FILE) {
console.error('Live runs require --from-file. Run with --dry-run first to generate the file.')
process.exit(1)
}
if (DRY_RUN && FROM_FILE) {
console.error(
'--from-file cannot be used with --dry-run. Dry runs always discover workspaces from the database.'
)
process.exit(1)
}
// ---------- Env ----------
function getEnv(name: string): string | undefined {
if (typeof process !== 'undefined' && process.env && name in process.env) {
return process.env[name]
}
return undefined
}
const CONNECTION_STRING = getEnv('POSTGRES_URL') ?? getEnv('DATABASE_URL')
if (!CONNECTION_STRING) {
console.error('Missing POSTGRES_URL or DATABASE_URL environment variable')
process.exit(1)
}
const ENCRYPTION_KEY = getEnv('ENCRYPTION_KEY')
if (!ENCRYPTION_KEY || ENCRYPTION_KEY.length !== 64) {
console.error('ENCRYPTION_KEY must be set to a 64-character hex string (32 bytes)')
process.exit(1)
}
// ---------- Encryption (mirrors apps/sim/lib/core/security/encryption.ts) ----------
function getEncryptionKeyBuffer(): Buffer {
return Buffer.from(ENCRYPTION_KEY!, 'hex')
}
async function encryptSecret(secret: string): Promise<string> {
const iv = randomBytes(16)
const key = getEncryptionKeyBuffer()
const cipher = createCipheriv('aes-256-gcm', key, iv, { authTagLength: 16 })
let encrypted = cipher.update(secret, 'utf8', 'hex')
encrypted += cipher.final('hex')
const authTag = cipher.getAuthTag()
return `${iv.toString('hex')}:${encrypted}:${authTag.toString('hex')}`
}
async function decryptSecret(encryptedValue: string): Promise<string> {
const parts = encryptedValue.split(':')
const ivHex = parts[0]
const authTagHex = parts[parts.length - 1]
const encrypted = parts.slice(1, -1).join(':')
if (!ivHex || !encrypted || !authTagHex) {
throw new Error('Invalid encrypted value format. Expected "iv:encrypted:authTag"')
}
const key = getEncryptionKeyBuffer()
const iv = Buffer.from(ivHex, 'hex')
const authTag = Buffer.from(authTagHex, 'hex')
const decipher = createDecipheriv('aes-256-gcm', key, iv, { authTagLength: 16 })
decipher.setAuthTag(authTag)
let decrypted = decipher.update(encrypted, 'hex', 'utf8')
decrypted += decipher.final('utf8')
return decrypted
}
// ---------- Schema ----------
const workspaceTable = pgTable('workspace', {
id: text('id').primaryKey(),
ownerId: text('owner_id').notNull(),
})
const workflow = pgTable('workflow', {
id: text('id').primaryKey(),
userId: text('user_id').notNull(),
workspaceId: text('workspace_id'),
name: text('name').notNull(),
})
const workflowBlocks = pgTable(
'workflow_blocks',
{
id: text('id').primaryKey(),
workflowId: text('workflow_id').notNull(),
type: text('type').notNull(),
name: text('name').notNull(),
subBlocks: jsonb('sub_blocks').notNull().default('{}'),
createdAt: timestamp('created_at').notNull().defaultNow(),
updatedAt: timestamp('updated_at').notNull().defaultNow(),
},
(table) => ({
workflowIdIdx: index('workflow_blocks_workflow_id_idx').on(table.workflowId),
})
)
const workspaceBYOKKeys = pgTable(
'workspace_byok_keys',
{
id: text('id').primaryKey(),
workspaceId: text('workspace_id').notNull(),
providerId: text('provider_id').notNull(),
encryptedApiKey: text('encrypted_api_key').notNull(),
createdBy: text('created_by'),
createdAt: timestamp('created_at').notNull().defaultNow(),
updatedAt: timestamp('updated_at').notNull().defaultNow(),
},
(table) => ({
workspaceProviderIdx: index('workspace_byok_workspace_provider_idx').on(
table.workspaceId,
table.providerId
),
workspaceIdx: index('workspace_byok_workspace_idx').on(table.workspaceId),
})
)
const environment = pgTable('environment', {
id: text('id').primaryKey(),
userId: text('user_id').notNull(),
variables: json('variables').notNull(),
})
const workspaceEnvironment = pgTable('workspace_environment', {
id: text('id').primaryKey(),
workspaceId: text('workspace_id').notNull(),
variables: json('variables').notNull().default('{}'),
})
const WORKSPACE_CONCURRENCY = 100
const WORKSPACE_BATCH_SIZE = 1000
const SLEEP_MS = 30_000
// ---------- DB ----------
const postgresClient = postgres(CONNECTION_STRING, {
prepare: false,
idle_timeout: 20,
connect_timeout: 30,
max: 10,
onnotice: () => {},
})
const db = drizzle(postgresClient)
// ---------- Helpers ----------
const TOOL_INPUT_SUBBLOCK_IDS: Record<string, string> = {
agent: 'tools',
human_in_the_loop: 'notification',
}
const ENV_VAR_PATTERN = /^\{\{([^}]+)\}\}$/
function isEnvVarReference(value: string): boolean {
return ENV_VAR_PATTERN.test(value)
}
function extractEnvVarName(value: string): string | null {
const match = ENV_VAR_PATTERN.exec(value)
return match ? match[1].trim() : null
}
function maskKey(key: string): string {
if (key.length <= 8) return '•'.repeat(8)
return key.slice(0, 4) + '•'.repeat(Math.min(key.length - 8, 12)) + key.slice(-4)
}
function parseToolInputValue(value: unknown): any[] {
if (Array.isArray(value)) return value
if (typeof value === 'string') {
try {
const parsed = JSON.parse(value)
if (Array.isArray(parsed)) return parsed
} catch {}
}
return []
}
type RawKeyRef = {
rawValue: string
blockName: string
workflowId: string
workflowName: string
userId: string
}
type EnvLookup = {
wsEnvVars: Record<string, string>
personalEnvCache: Map<string, Record<string, string>>
}
type KeySource = 'plaintext' | 'workspace' | 'personal'
const KEY_SOURCE_PRIORITY: Record<KeySource, number> = {
plaintext: 0,
workspace: 1,
personal: 2,
}
interface ResolveKeyContext {
workspaceId: string
workspaceOwnerId: string | null
}
type MigrationStats = {
workspacesProcessed: number
workspacesSkipped: number
conflicts: number
inserted: number
skippedExisting: number
errors: number
envVarFailures: number
}
type WorkspaceResult = {
stats: MigrationStats
shouldWriteWorkspaceId: boolean
}
function createEmptyStats(): MigrationStats {
return {
workspacesProcessed: 0,
workspacesSkipped: 0,
conflicts: 0,
inserted: 0,
skippedExisting: 0,
errors: 0,
envVarFailures: 0,
}
}
function mergeStats(target: MigrationStats, source: MigrationStats) {
target.workspacesProcessed += source.workspacesProcessed
target.workspacesSkipped += source.workspacesSkipped
target.conflicts += source.conflicts
target.inserted += source.inserted
target.skippedExisting += source.skippedExisting
target.errors += source.errors
target.envVarFailures += source.envVarFailures
}
async function resolveKey(
ref: RawKeyRef,
env: EnvLookup,
ctx: ResolveKeyContext
): Promise<{ key: string | null; source: KeySource; envVarFailed: boolean }> {
if (!isEnvVarReference(ref.rawValue)) {
return { key: ref.rawValue, source: 'plaintext', envVarFailed: false }
}
const varName = extractEnvVarName(ref.rawValue)
if (!varName) return { key: null, source: 'personal', envVarFailed: true }
const personalVars = env.personalEnvCache.get(ref.userId)
const wsValue = env.wsEnvVars[varName]
const personalValue = personalVars?.[varName]
const encryptedValue = wsValue ?? personalValue
const source: KeySource = wsValue ? 'workspace' : 'personal'
const logPrefix =
`workspace=${ctx.workspaceId} owner=${ctx.workspaceOwnerId ?? 'unknown'}` +
` workflow=${ref.workflowId} user=${ref.userId}`
if (!encryptedValue) {
console.warn(
` [WARN] Env var "${varName}" not found — ${logPrefix} "${ref.blockName}" in "${ref.workflowName}"`
)
return { key: null, source, envVarFailed: true }
}
try {
const decrypted = await decryptSecret(encryptedValue)
return { key: decrypted, source, envVarFailed: false }
} catch (error) {
console.warn(
` [WARN] Failed to decrypt env var "${varName}" — ${logPrefix} "${ref.blockName}" in "${ref.workflowName}": ${error}`
)
return { key: null, source, envVarFailed: true }
}
}
async function processWorkspace(
workspaceId: string,
allBlockTypes: string[],
userFilter: ReturnType<typeof sql>,
total: number,
index: number
): Promise<WorkspaceResult> {
const stats = createEmptyStats()
try {
const [blocks, wsRows] = await Promise.all([
db
.select({
blockId: workflowBlocks.id,
blockName: workflowBlocks.name,
blockType: workflowBlocks.type,
subBlocks: workflowBlocks.subBlocks,
workflowId: workflow.id,
workflowName: workflow.name,
userId: workflow.userId,
})
.from(workflowBlocks)
.innerJoin(workflow, eq(workflowBlocks.workflowId, workflow.id))
.where(
sql`${workflow.workspaceId} = ${workspaceId} AND ${workflowBlocks.type} IN (${sql.join(
allBlockTypes.map((t) => sql`${t}`),
sql`, `
)})${userFilter}`
),
db
.select({ ownerId: workspaceTable.ownerId })
.from(workspaceTable)
.where(eq(workspaceTable.id, workspaceId))
.limit(1),
])
const workspaceOwnerId = wsRows[0]?.ownerId ?? null
console.log(
`[${index}/${total}] [Workspace ${workspaceId}] ${blocks.length} blocks, owner=${workspaceOwnerId ?? 'unknown'}`
)
const providerKeys = new Map<string, RawKeyRef[]>()
for (const block of blocks) {
const subBlocks = block.subBlocks as Record<string, { value?: any }>
const providerId = BLOCK_TYPE_TO_PROVIDER[block.blockType]
if (providerId) {
const val = subBlocks?.apiKey?.value
if (typeof val === 'string' && val.trim()) {
const refs = providerKeys.get(providerId) ?? []
refs.push({
rawValue: val,
blockName: block.blockName,
workflowId: block.workflowId,
workflowName: block.workflowName,
userId: block.userId,
})
providerKeys.set(providerId, refs)
}
}
const toolInputId = TOOL_INPUT_SUBBLOCK_IDS[block.blockType]
if (toolInputId) {
const tools = parseToolInputValue(subBlocks?.[toolInputId]?.value)
for (const tool of tools) {
const toolType = tool?.type as string | undefined
const toolApiKey = tool?.params?.apiKey as string | undefined
if (!toolType || !toolApiKey || !toolApiKey.trim()) continue
const toolProviderId = BLOCK_TYPE_TO_PROVIDER[toolType]
if (!toolProviderId) continue
const refs = providerKeys.get(toolProviderId) ?? []
refs.push({
rawValue: toolApiKey,
blockName: `${block.blockName} > tool "${tool.title || toolType}"`,
workflowId: block.workflowId,
workflowName: block.workflowName,
userId: block.userId,
})
providerKeys.set(toolProviderId, refs)
}
}
}
if (providerKeys.size === 0) {
console.log(` [${index}/${total}] No API keys found, skipping\n`)
stats.workspacesSkipped++
return { stats, shouldWriteWorkspaceId: false }
}
const needsEnvVars = [...providerKeys.values()]
.flat()
.some((ref) => isEnvVarReference(ref.rawValue))
let wsEnvVars: Record<string, string> = {}
const personalEnvCache = new Map<string, Record<string, string>>()
if (needsEnvVars) {
const wsEnvRows = await db
.select()
.from(workspaceEnvironment)
.where(sql`${workspaceEnvironment.workspaceId} = ${workspaceId}`)
.limit(1)
if (wsEnvRows[0]) {
wsEnvVars = (wsEnvRows[0].variables as Record<string, string>) || {}
}
const userIds = [...new Set([...providerKeys.values()].flat().map((r) => r.userId))]
if (userIds.length > 0) {
const personalRows = await db
.select()
.from(environment)
.where(
sql`${environment.userId} IN (${sql.join(
userIds.map((id) => sql`${id}`),
sql`, `
)})`
)
for (const row of personalRows) {
personalEnvCache.set(row.userId, (row.variables as Record<string, string>) || {})
}
}
}
const envLookup: EnvLookup = { wsEnvVars, personalEnvCache }
stats.workspacesProcessed++
const existingBYOKProviders = new Set<string>()
const existingRows = await db
.select({ providerId: workspaceBYOKKeys.providerId })
.from(workspaceBYOKKeys)
.where(eq(workspaceBYOKKeys.workspaceId, workspaceId))
for (const row of existingRows) {
existingBYOKProviders.add(row.providerId)
}
let hasNewInserts = false
for (const [providerId, refs] of providerKeys) {
const resolved: { ref: RawKeyRef; key: string; source: KeySource }[] = []
const resolveCtx: ResolveKeyContext = { workspaceId, workspaceOwnerId }
for (const ref of refs) {
const { key, source, envVarFailed } = await resolveKey(ref, envLookup, resolveCtx)
if (envVarFailed) stats.envVarFailures++
if (!key?.trim()) continue
if (source === 'personal' && ref.userId !== workspaceOwnerId) {
console.log(
` [SKIP-PERSONAL] Ignoring non-owner personal key from user=${ref.userId} workflow=${ref.workflowId} "${ref.blockName}" in "${ref.workflowName}"`
)
continue
}
resolved.push({ ref, key: key.trim(), source })
}
if (resolved.length === 0) continue
resolved.sort((a, b) => KEY_SOURCE_PRIORITY[a.source] - KEY_SOURCE_PRIORITY[b.source])
const distinctKeys = new Set(resolved.map((r) => r.key))
if (distinctKeys.size > 1) {
stats.conflicts++
console.log(` [CONFLICT] provider "${providerId}": ${distinctKeys.size} distinct keys`)
for (const { ref, key, source } of resolved) {
const isOwner = ref.userId === workspaceOwnerId ? ' (owner)' : ''
const display = isEnvVarReference(ref.rawValue)
? `${ref.rawValue} -> ${maskKey(key)}`
: maskKey(ref.rawValue)
console.log(
` [${source}] user=${ref.userId}${isOwner} workflow=${ref.workflowId} "${ref.blockName}" in "${ref.workflowName}": ${display}`
)
}
const chosenIsOwner = resolved[0].ref.userId === workspaceOwnerId ? ', owner' : ''
console.log(
` Using highest-priority key (${resolved[0].source}${chosenIsOwner}, user=${resolved[0].ref.userId})`
)
}
const chosen = resolved[0]
if (DRY_RUN) {
if (existingBYOKProviders.has(providerId)) {
console.log(` [DRY RUN] BYOK already exists for provider "${providerId}", skipping`)
stats.skippedExisting++
continue
}
hasNewInserts = true
console.log(
` [DRY RUN] Would insert BYOK for provider "${providerId}": ${maskKey(chosen.key)}`
)
continue
}
if (existingBYOKProviders.has(providerId)) {
console.log(` [SKIP] BYOK already exists for provider "${providerId}"`)
stats.skippedExisting++
continue
}
try {
const encrypted = await encryptSecret(chosen.key)
await db.insert(workspaceBYOKKeys).values({
id: generateId(),
workspaceId,
providerId,
encryptedApiKey: encrypted,
createdBy: chosen.ref.userId,
})
existingBYOKProviders.add(providerId)
console.log(` [INSERT] BYOK for provider "${providerId}": ${maskKey(chosen.key)}`)
stats.inserted++
} catch (error) {
console.error(` [ERROR] Failed to insert BYOK for provider "${providerId}":`, error)
stats.errors++
}
}
console.log(` [${index}/${total}] Done with workspace ${workspaceId}\n`)
return { stats, shouldWriteWorkspaceId: DRY_RUN && hasNewInserts }
} catch (error) {
console.error(` [ERROR] Failed workspace ${workspaceId}:`, error)
stats.errors++
return { stats, shouldWriteWorkspaceId: false }
}
}
// ---------- Main ----------
async function run() {
console.log(`Mode: ${DRY_RUN ? 'DRY RUN (audit + preview)' : 'LIVE'}`)
console.log(
`Mappings: ${Object.entries(BLOCK_TYPE_TO_PROVIDER)
.map(([b, p]) => `${b}=${p}`)
.join(', ')}`
)
console.log(`Users: ${USER_FILTER.length > 0 ? USER_FILTER.join(', ') : 'all'}`)
if (FROM_FILE) console.log(`From file: ${FROM_FILE}`)
console.log('---\n')
const stats = createEmptyStats()
let workspaceIdsWritten = 0
try {
// 1. Get distinct workspace IDs that have matching blocks
const mappedBlockTypes = Object.keys(BLOCK_TYPE_TO_PROVIDER)
const agentTypes = Object.keys(TOOL_INPUT_SUBBLOCK_IDS)
const allBlockTypes = [...new Set([...mappedBlockTypes, ...agentTypes])]
const userFilter =
USER_FILTER.length > 0
? sql` AND ${workflow.userId} IN (${sql.join(
USER_FILTER.map((id) => sql`${id}`),
sql`, `
)})`
: sql``
let workspaceIds: string[]
if (DRY_RUN) {
const workspaceIdRows = await db
.selectDistinct({ workspaceId: workflow.workspaceId })
.from(workflowBlocks)
.innerJoin(workflow, eq(workflowBlocks.workflowId, workflow.id))
.where(
sql`${workflow.workspaceId} IS NOT NULL AND ${workflowBlocks.type} IN (${sql.join(
allBlockTypes.map((t) => sql`${t}`),
sql`, `
)})${userFilter}`
)
workspaceIds = workspaceIdRows
.map((r) => r.workspaceId)
.filter((id): id is string => id !== null)
console.log(`Found ${workspaceIds.length} workspaces with candidate blocks\n`)
const outPath = resolve('migrate-byok-workspace-ids.txt')
writeFileSync(outPath, '')
console.log(`[DRY RUN] Will write workspace IDs with keys to ${outPath}\n`)
} else {
const raw = readFileSync(resolve(FROM_FILE!), 'utf-8')
workspaceIds = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean)
console.log(`Loaded ${workspaceIds.length} workspace IDs from ${FROM_FILE}\n`)
}
// 2. Process workspaces in parallel groups of 100, pausing for 60s after each 1000
for (let batchStart = 0; batchStart < workspaceIds.length; batchStart += WORKSPACE_BATCH_SIZE) {
const batch = workspaceIds.slice(batchStart, batchStart + WORKSPACE_BATCH_SIZE)
console.log(
`[BATCH] Processing workspaces ${batchStart + 1}-${batchStart + batch.length} of ${workspaceIds.length}`
)
for (
let concurrencyStart = 0;
concurrencyStart < batch.length;
concurrencyStart += WORKSPACE_CONCURRENCY
) {
const workspaceChunk = batch.slice(
concurrencyStart,
concurrencyStart + WORKSPACE_CONCURRENCY
)
const results = await Promise.all(
workspaceChunk.map((workspaceId, chunkIndex) =>
processWorkspace(
workspaceId,
allBlockTypes,
userFilter,
workspaceIds.length,
batchStart + concurrencyStart + chunkIndex + 1
)
)
)
if (DRY_RUN) {
const workspaceIdsWithKeys = results
.map((result, resultIndex) =>
result.shouldWriteWorkspaceId ? workspaceChunk[resultIndex] : null
)
.filter((id): id is string => id !== null)
if (workspaceIdsWithKeys.length > 0) {
appendFileSync(
resolve('migrate-byok-workspace-ids.txt'),
`${workspaceIdsWithKeys.join('\n')}\n`
)
workspaceIdsWritten += workspaceIdsWithKeys.length
}
}
for (const result of results) {
mergeStats(stats, result.stats)
}
}
if (batchStart + batch.length < workspaceIds.length) {
console.log(
` [THROTTLE] ${batchStart + batch.length}/${workspaceIds.length} workspaces — sleeping ${SLEEP_MS / 1000}s`
)
await sleep(SLEEP_MS)
}
}
// 3. Summary
console.log('---')
console.log('Summary:')
console.log(` Workspaces processed: ${stats.workspacesProcessed}`)
console.log(` Workspaces skipped (no keys): ${stats.workspacesSkipped}`)
console.log(` BYOK keys inserted: ${stats.inserted}`)
console.log(` BYOK keys skipped (already existed): ${stats.skippedExisting}`)
console.log(` Conflicts (multiple distinct keys): ${stats.conflicts}`)
console.log(` Insert errors: ${stats.errors}`)
console.log(` Env var resolution failures: ${stats.envVarFailures}`)
if (DRY_RUN) {
console.log(
`\n[DRY RUN] Wrote ${workspaceIdsWritten} workspace IDs (with new keys to insert) to migrate-byok-workspace-ids.txt`
)
console.log('[DRY RUN] No changes were made to the database.')
console.log('Run without --dry-run to apply changes.')
} else {
console.log('\nMigration completed successfully!')
}
} catch (error) {
console.error('Fatal error:', error)
process.exit(1)
} finally {
try {
await postgresClient.end({ timeout: 5 })
} catch {}
}
}
run()
.then(() => {
console.log('\nDone!')
process.exit(0)
})
.catch((error) => {
console.error('Unexpected error:', error)
process.exit(1)
})
@@ -0,0 +1,371 @@
#!/usr/bin/env bun
// This script is intentionally self-contained for execution in the migrations image.
// Do not import from the main app code; duplicate minimal schema and DB setup here.
// Workspace-internal packages (`@sim/*`) are permitted since they ship in the migrations image.
import { generateId } from '@sim/utils/id'
import { sql } from 'drizzle-orm'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
// ---------- Minimal env helpers ----------
function getEnv(name: string): string | undefined {
if (typeof process !== 'undefined' && process.env && name in process.env) {
return process.env[name]
}
return undefined
}
const CONNECTION_STRING = getEnv('POSTGRES_URL') ?? getEnv('DATABASE_URL')
if (!CONNECTION_STRING) {
console.error('Missing POSTGRES_URL or DATABASE_URL environment variable')
process.exit(1)
}
// ---------- Minimal schema (only what we need) ----------
import { boolean, index, integer, json, jsonb, pgTable, text, timestamp } from 'drizzle-orm/pg-core'
// Tables referenced by the script
const workflow = pgTable(
'workflow',
{
id: text('id').primaryKey(),
userId: text('user_id').notNull(),
name: text('name').notNull(),
isDeployed: boolean('is_deployed').notNull().default(false),
deployedState: json('deployed_state'),
deployedAt: timestamp('deployed_at'),
},
(table) => ({
userIdIdx: index('workflow_user_id_idx').on(table.userId),
})
)
const workflowBlocks = pgTable(
'workflow_blocks',
{
id: text('id').primaryKey(),
workflowId: text('workflow_id').notNull(),
type: text('type').notNull(),
name: text('name').notNull(),
positionX: text('position_x').notNull(),
positionY: text('position_y').notNull(),
enabled: boolean('enabled').notNull().default(true),
horizontalHandles: boolean('horizontal_handles').notNull().default(true),
isWide: boolean('is_wide').notNull().default(false),
advancedMode: boolean('advanced_mode').notNull().default(false),
triggerMode: boolean('trigger_mode').notNull().default(false),
height: text('height').notNull().default('0'),
subBlocks: jsonb('sub_blocks').notNull().default('{}'),
outputs: jsonb('outputs').notNull().default('{}'),
data: jsonb('data').default('{}'),
parentId: text('parent_id'),
extent: text('extent'),
createdAt: timestamp('created_at').notNull().defaultNow(),
updatedAt: timestamp('updated_at').notNull().defaultNow(),
},
(table) => ({
workflowIdIdx: index('workflow_blocks_workflow_id_idx').on(table.workflowId),
})
)
const workflowEdges = pgTable(
'workflow_edges',
{
id: text('id').primaryKey(),
workflowId: text('workflow_id').notNull(),
sourceBlockId: text('source_block_id').notNull(),
targetBlockId: text('target_block_id').notNull(),
sourceHandle: text('source_handle'),
targetHandle: text('target_handle'),
createdAt: timestamp('created_at').notNull().defaultNow(),
},
(table) => ({
workflowIdIdx: index('workflow_edges_workflow_id_idx').on(table.workflowId),
})
)
const workflowSubflows = pgTable(
'workflow_subflows',
{
id: text('id').primaryKey(),
workflowId: text('workflow_id').notNull(),
type: text('type').notNull(),
config: jsonb('config').notNull().default('{}'),
createdAt: timestamp('created_at').notNull().defaultNow(),
updatedAt: timestamp('updated_at').notNull().defaultNow(),
},
(table) => ({
workflowIdIdx: index('workflow_subflows_workflow_id_idx').on(table.workflowId),
})
)
const workflowDeploymentVersion = pgTable(
'workflow_deployment_version',
{
id: text('id').primaryKey(),
workflowId: text('workflow_id').notNull(),
version: integer('version').notNull(),
state: json('state').notNull(),
isActive: boolean('is_active').notNull().default(false),
createdAt: timestamp('created_at').notNull().defaultNow(),
createdBy: text('created_by'),
},
(table) => ({
workflowIdIdx: index('workflow_deployment_version_workflow_id_idx').on(table.workflowId),
})
)
// ---------- DB client ----------
const postgresClient = postgres(CONNECTION_STRING, {
prepare: false,
idle_timeout: 20,
connect_timeout: 30,
max: 10,
onnotice: () => {},
})
const db = drizzle(postgresClient)
// ---------- Minimal types ----------
type WorkflowState = {
blocks: Record<string, any>
edges: Array<{
id: string
source: string
target: string
sourceHandle?: string | null
targetHandle?: string | null
}>
loops: Record<string, any>
parallels: Record<string, any>
}
// ---------- Normalized loader (inline of loadWorkflowFromNormalizedTables) ----------
async function loadWorkflowFromNormalizedTables(workflowId: string) {
const [blocks, edges, subflows] = await Promise.all([
db.select().from(workflowBlocks).where(sql`${workflowBlocks.workflowId} = ${workflowId}`),
db.select().from(workflowEdges).where(sql`${workflowEdges.workflowId} = ${workflowId}`),
db.select().from(workflowSubflows).where(sql`${workflowSubflows.workflowId} = ${workflowId}`),
])
if (blocks.length === 0) return null
const blocksMap: Record<string, any> = {}
for (const block of blocks as any[]) {
const parentId = (block.parentId as string | null) || null
const extent = (block.extent as string | null) || null
blocksMap[block.id] = {
id: block.id,
type: block.type,
name: block.name,
position: {
x: Number(block.positionX),
y: Number(block.positionY),
},
enabled: block.enabled,
horizontalHandles: block.horizontalHandles,
isWide: block.isWide,
advancedMode: block.advancedMode,
triggerMode: block.triggerMode,
height: Number(block.height),
subBlocks: block.subBlocks || {},
outputs: block.outputs || {},
data: {
...(block.data || {}),
...(parentId && { parentId }),
...(extent && { extent }),
},
parentId,
extent,
}
}
const edgesArray = (edges as any[]).map((edge) => ({
id: edge.id,
source: edge.sourceBlockId,
target: edge.targetBlockId,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
}))
const loops: Record<string, any> = {}
const parallels: Record<string, any> = {}
for (const sub of subflows as any[]) {
const config = sub.config || {}
if (sub.type === 'loop') {
loops[sub.id] = { id: sub.id, ...config }
} else if (sub.type === 'parallel') {
parallels[sub.id] = { id: sub.id, ...config }
}
}
return {
blocks: blocksMap,
edges: edgesArray,
loops,
parallels,
isFromNormalizedTables: true,
}
}
// ---------- Migration ----------
const DRY_RUN = process.argv.includes('--dry-run')
const BATCH_SIZE = 50
async function migrateWorkflows() {
console.log('Starting deployment version migration...')
console.log(`Mode: ${DRY_RUN ? 'DRY RUN' : 'LIVE'}`)
console.log(`Batch size: ${BATCH_SIZE}`)
console.log('---')
try {
const workflows = await db
.select({
id: workflow.id,
name: workflow.name,
isDeployed: workflow.isDeployed,
deployedState: workflow.deployedState,
deployedAt: workflow.deployedAt,
userId: workflow.userId,
})
.from(workflow)
console.log(`Found ${workflows.length} workflows to process`)
const existingVersions = await db
.select({ workflowId: workflowDeploymentVersion.workflowId })
.from(workflowDeploymentVersion)
const existingWorkflowIds = new Set(existingVersions.map((v) => v.workflowId as string))
console.log(`${existingWorkflowIds.size} workflows already have deployment versions`)
let successCount = 0
let skipCount = 0
let errorCount = 0
for (let i = 0; i < workflows.length; i += BATCH_SIZE) {
const batch = workflows.slice(i, i + BATCH_SIZE)
console.log(
`\nProcessing batch ${Math.floor(i / BATCH_SIZE) + 1} (workflows ${i + 1}-${Math.min(i + BATCH_SIZE, workflows.length)})`
)
const deploymentVersions: Array<{
id: string
workflowId: string
version: number
state: WorkflowState
createdAt: Date
createdBy: string
isActive: boolean
}> = []
for (const wf of batch as any[]) {
if (existingWorkflowIds.has(wf.id)) {
console.log(` [SKIP] ${wf.id} (${wf.name}) - already has deployment version`)
skipCount++
continue
}
let state: WorkflowState | null = null
if (wf.deployedState) {
state = wf.deployedState as WorkflowState
console.log(` [DEPLOYED] ${wf.id} (${wf.name}) - using existing deployedState`)
} else {
const normalized = await loadWorkflowFromNormalizedTables(wf.id)
if (normalized) {
state = {
blocks: normalized.blocks,
edges: normalized.edges,
loops: normalized.loops,
parallels: normalized.parallels,
}
console.log(
` [NORMALIZED] ${wf.id} (${wf.name}) - loaded from normalized tables (was deployed: ${wf.isDeployed})`
)
} else {
console.log(` [SKIP] ${wf.id} (${wf.name}) - no state available`)
skipCount++
continue
}
}
if (state) {
deploymentVersions.push({
id: generateId(),
workflowId: wf.id,
version: 1,
state,
createdAt: wf.deployedAt || new Date(),
createdBy: wf.userId || 'migration',
isActive: true,
})
successCount++
}
}
if (deploymentVersions.length > 0) {
if (DRY_RUN) {
console.log(` [DRY RUN] Would insert ${deploymentVersions.length} deployment versions`)
console.log(` [DRY RUN] Would mark ${deploymentVersions.length} workflows as deployed`)
} else {
try {
await db.insert(workflowDeploymentVersion).values(deploymentVersions)
console.log(` [SUCCESS] Inserted ${deploymentVersions.length} deployment versions`)
const workflowIds = deploymentVersions.map((v) => v.workflowId)
await db
.update(workflow)
.set({
isDeployed: true,
deployedAt: new Date(),
})
.where(
sql`${workflow.id} IN (${sql.join(
workflowIds.map((id) => sql`${id}`),
sql`, `
)})`
)
console.log(` [SUCCESS] Marked ${workflowIds.length} workflows as deployed`)
} catch (error) {
console.error(` [ERROR] Failed to insert batch:`, error)
errorCount += deploymentVersions.length
successCount -= deploymentVersions.length
}
}
}
}
console.log('\n---')
console.log('Migration Summary:')
console.log(` Success: ${successCount} workflows`)
console.log(` Skipped: ${skipCount} workflows`)
console.log(` Errors: ${errorCount} workflows`)
if (DRY_RUN) {
console.log('\n[DRY RUN] No changes were made to the database.')
console.log('Run without --dry-run flag to apply changes.')
} else {
console.log('\nMigration completed successfully!')
}
} catch (error) {
console.error('Fatal error during migration:', error)
process.exit(1)
} finally {
try {
await postgresClient.end({ timeout: 5 })
} catch {}
}
}
migrateWorkflows()
.then(() => {
console.log('\nDone!')
process.exit(0)
})
.catch((error) => {
console.error('Unexpected error:', error)
process.exit(1)
})
+325
View File
@@ -0,0 +1,325 @@
import { getPostgresErrorCode } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { backoffWithJitter } from '@sim/utils/retry'
import { drizzle } from 'drizzle-orm/postgres-js'
import { migrate } from 'drizzle-orm/postgres-js/migrator'
import postgres from 'postgres'
import { runScriptMigrations } from '../script-migrations/index'
/**
* Concurrent-index convention: plain `CREATE INDEX` write-blocks large/hot
* tables, and CONCURRENTLY cannot run inside drizzle's migration transaction.
* For indexes on big tables, edit the generated SQL to:
*
* COMMIT;--> statement-breakpoint
* SET lock_timeout = 0;--> statement-breakpoint
* CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_name" ON "table" (...);--> statement-breakpoint
* SET lock_timeout = '5s';
*
* The embedded COMMIT ends the batch transaction, so everything after it (in
* this and later pending files) runs in autocommit and must be idempotent
* (`IF NOT EXISTS` etc.) — a failed run replays unjournaled files from the top.
* A failed CONCURRENTLY build leaves an INVALID index that `IF NOT EXISTS`
* skips; `warnOnInvalidIndexes` below surfaces those.
*/
/**
* Prefer a direct (non-pooled) DSN: session advisory locks and session `SET`s
* are unsupported through PgBouncer transaction pooling. Falls back to
* DATABASE_URL for setups that connect directly anyway.
*/
const url = process.env.MIGRATION_DATABASE_URL || process.env.DATABASE_URL
if (!url) {
console.error('ERROR: Missing DATABASE_URL environment variable.')
console.error('Ensure packages/db/.env is configured.')
process.exit(1)
}
/**
* The pid guard is only sound on a direct connection — through transaction
* pooling, consecutive statements legitimately land on different backends.
*/
const hasDirectMigrationUrl = Boolean(process.env.MIGRATION_DATABASE_URL)
/**
* `max_lifetime: null` pins the session for the whole run: the postgres-js
* default recycles the connection after 3060 min, silently dropping the
* session advisory lock and `SET`s.
*/
const client = postgres(url, {
max: 1,
connect_timeout: 10,
max_lifetime: null,
connection: { application_name: 'sim-migrate' },
})
/**
* Cross-process migration lock. drizzle's `migrate()` has no built-in lock, so
* concurrent runners (one per app replica at deploy time) must be serialized.
* Acquisition is a bounded try-lock loop: a plain `pg_advisory_lock` wait let
* one wedged runner silently hang every other runner and the whole deploy.
*/
const MIGRATION_LOCK_KEY = 4_961_002_270n
const LOCK_ACQUIRE_DEADLINE_MS = 30 * 60_000
const LOCK_RETRY_INTERVAL_MS = 5_000
/**
* Max time a migration statement may queue for a table lock (SQLSTATE 55P03 on
* expiry). Without it, DDL waiting on an AccessExclusiveLock queues every other
* query on the table behind it — a table-wide stall for the whole wait.
*/
const DDL_LOCK_TIMEOUT = '5s'
const MAX_MIGRATE_ATTEMPTS = 8
const MIGRATE_RETRY_BACKOFF = { baseMs: 2_000, maxMs: 30_000 } as const
const CONNECT_MAX_ATTEMPTS = 10
const CONNECT_RETRY_BACKOFF = { baseMs: 1_000, maxMs: 15_000 } as const
/**
* Error codes that mean the database was momentarily unreachable rather than
* the migration being wrong: chiefly `53300` (too_many_connections — every
* non-superuser slot was taken, surfaced as "remaining connection slots are
* reserved for roles with the SUPERUSER attribute"), the `08xxx`
* connection_exception class, and the postgres-js driver's own transport
* codes. These are retried while opening the session; anything else is fatal.
*/
const TRANSIENT_CONNECT_CODES = new Set([
'53300',
'53400',
'CONNECT_TIMEOUT',
'CONNECTION_CLOSED',
'CONNECTION_DESTROYED',
'CONNECTION_ENDED',
'ECONNREFUSED',
'ECONNRESET',
'ETIMEDOUT',
'EHOSTUNREACH',
'ENOTFOUND',
])
function isTransientConnectError(error: unknown): boolean {
const code = getPostgresErrorCode(error)
if (!code) return false
return TRANSIENT_CONNECT_CODES.has(code) || code.startsWith('08')
}
/** Backend pid of the lock-holding session; a change means the lock was lost. */
let lockSessionPid = 0
try {
await connectWithRetry()
await acquireMigrationLock()
try {
await runMigrationsWithRetry()
console.log('Migrations applied successfully.')
await assertLockSessionHeld()
await runScriptMigrations(client)
await warnOnInvalidIndexes()
} finally {
await releaseMigrationLock()
}
} catch (error) {
console.error('ERROR: Migration failed.')
printMigrationError(error)
process.exit(1)
} finally {
await client.end()
}
/**
* Open the migration session before taking the advisory lock, retrying
* transient connection failures with bounded backoff. The deploy database can
* briefly exhaust every non-superuser connection slot at peak (`53300`); the
* migration is a single short-lived session, so waiting out a spike that frees
* within seconds is far safer than failing the whole deploy. Non-transient
* errors (auth, unknown host config, etc.) still fail fast.
*/
async function connectWithRetry(): Promise<void> {
for (let attempt = 1; ; attempt++) {
try {
await client`SELECT 1`
return
} catch (error) {
if (!isTransientConnectError(error) || attempt >= CONNECT_MAX_ATTEMPTS) throw error
const delayMs = backoffWithJitter(attempt, null, CONNECT_RETRY_BACKOFF)
console.warn(
`WARN: database unavailable (${getPostgresErrorCode(error)}); ` +
`attempt ${attempt}/${CONNECT_MAX_ATTEMPTS}, retrying in ${Math.round(delayMs)}ms.`
)
await sleep(delayMs)
}
}
}
/**
* Acquire the cross-process migration lock, failing loudly after the deadline
* instead of blocking forever behind a wedged runner.
*/
async function acquireMigrationLock(): Promise<void> {
const deadline = Date.now() + LOCK_ACQUIRE_DEADLINE_MS
for (;;) {
const [{ locked, pid }] =
await client`SELECT pg_try_advisory_lock(${MIGRATION_LOCK_KEY}) AS locked, pg_backend_pid() AS pid`
if (locked) {
lockSessionPid = pid
return
}
if (Date.now() >= deadline) {
throw new Error(
`Timed out after ${LOCK_ACQUIRE_DEADLINE_MS}ms waiting for the migration advisory lock; ` +
'another runner is likely stuck mid-migration. Investigate before retrying.'
)
}
await sleep(LOCK_RETRY_INTERVAL_MS)
}
}
/**
* Run pending migrations, retrying on lock timeout (55P03, found anywhere in
* the wrapped `cause` chain). Each attempt re-verifies the lock session (pid)
* and re-asserts the session timeouts — a migration file may have changed them,
* and `SET` cannot be parameterized, hence `client.unsafe` with constants.
* Replays are safe: drizzle rolls the batch back on failure, and post-COMMIT
* CONCURRENTLY statements are idempotent by convention.
*/
/**
* Verify the session still holds the migration advisory lock: a changed
* backend pid means the connection was recycled and the lock silently dropped.
* Only sound on a direct connection — see `hasDirectMigrationUrl`.
*/
async function assertLockSessionHeld(): Promise<void> {
if (!hasDirectMigrationUrl) return
const [{ pid }] = await client`SELECT pg_backend_pid() AS pid`
if (pid !== lockSessionPid) {
throw new Error(
`Database session changed mid-run (backend pid ${lockSessionPid} -> ${pid}); ` +
'the migration advisory lock was lost. Aborting so a fresh runner can retry safely.'
)
}
}
async function runMigrationsWithRetry(): Promise<void> {
for (let attempt = 1; ; attempt++) {
await assertLockSessionHeld()
await client.unsafe('SET statement_timeout = 0')
await client.unsafe(`SET lock_timeout = '${DDL_LOCK_TIMEOUT}'`)
try {
await migrate(drizzle(client), { migrationsFolder: './migrations' })
return
} catch (error) {
const isLockTimeout = getPostgresErrorCode(error) === '55P03'
if (!isLockTimeout || attempt >= MAX_MIGRATE_ATTEMPTS) throw error
const delayMs = backoffWithJitter(attempt, null, MIGRATE_RETRY_BACKOFF)
console.warn(
`WARN: migration DDL hit lock_timeout (attempt ${attempt}/${MAX_MIGRATE_ATTEMPTS}); ` +
`retrying in ${Math.round(delayMs)}ms.`
)
await sleep(delayMs)
}
}
}
/**
* A failed CONCURRENTLY build leaves an INVALID index that `IF NOT EXISTS`
* silently skips forever — surface it (warn only; the migration committed).
*/
async function warnOnInvalidIndexes(): Promise<void> {
try {
const rows = await client`
SELECT n.nspname AS schema, c.relname AS index
FROM pg_index i
JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE NOT i.indisvalid
`
for (const row of rows) {
console.warn(
`WARN: invalid index ${row.schema}.${row.index} — a CONCURRENTLY build failed partway. ` +
'Drop and rebuild it; IF NOT EXISTS will keep skipping it.'
)
}
} catch (checkError) {
console.warn('WARN: could not check for invalid indexes.', checkError)
}
}
/**
* Unlock errors are swallowed: the session lock auto-releases on disconnect,
* and a thrown unlock would falsely report a committed migration as failed.
*/
async function releaseMigrationLock(): Promise<void> {
try {
await client`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})`
} catch (unlockError) {
console.error(
'WARN: pg_advisory_unlock failed; the session lock will auto-release on disconnect.',
unlockError
)
}
}
/**
* Print every diagnostic field a Postgres driver puts on a thrown error. The default
* `error.message` loses the constraint name, affected table/column, PG code, and hint —
* which are usually what you need to diagnose a failed migration.
*/
function printMigrationError(error: unknown): void {
if (!(error instanceof Error)) {
console.error(error)
return
}
console.error(`message: ${error.message}`)
const pgFields = [
'code',
'severity',
'severity_local',
'detail',
'hint',
'schema',
'schema_name',
'table',
'table_name',
'column',
'column_name',
'constraint',
'constraint_name',
'data_type',
'where',
'internal_query',
'internal_position',
'position',
'routine',
'file',
'line',
] as const
const err = error as Record<string, unknown>
for (const field of pgFields) {
const value = err[field]
if (value !== undefined && value !== null && value !== '') {
console.error(`${field}: ${String(value)}`)
}
}
if (err.query && typeof err.query === 'string') {
console.error('\nfailing query:')
console.error(err.query)
}
if (err.parameters !== undefined) {
console.error('\nparameters:')
console.error(err.parameters)
}
if (error.cause) {
console.error('\ncause:')
printMigrationError(error.cause)
}
if (error.stack) {
console.error('\nstack:')
console.error(error.stack)
}
}
@@ -0,0 +1,705 @@
#!/usr/bin/env bun
/**
* Direct Database SSO Registration Script (Better Auth Best Practice)
*
* This script bypasses the authentication requirement by directly inserting
* SSO provider records into the database, following the exact same logic
* as Better Auth's registerSSOProvider endpoint.
*
* Usage: bun run packages/db/scripts/register-sso-provider.ts
*
* Required Environment Variables:
* SSO_ENABLED=true
* SSO_PROVIDER_TYPE=oidc|saml
* SSO_PROVIDER_ID=your-provider-id
* SSO_ISSUER=https://your-idp-url
* SSO_DOMAIN=your-email-domain.com
* SSO_USER_EMAIL=admin@yourdomain.com (must be existing user)
*
* OIDC Providers:
* SSO_OIDC_CLIENT_ID=your_client_id
* SSO_OIDC_CLIENT_SECRET=your_client_secret
* SSO_OIDC_SCOPES=openid,profile,email (optional)
* SSO_OIDC_TOKEN_ENDPOINT_AUTH=client_secret_post|client_secret_basic (optional, defaults to client_secret_post)
* SSO_OIDC_SKIP_USERINFO_ENDPOINT=true (optional; reads claims from the verified ID token
* instead of calling the discovered UserInfo endpoint, matching better-auth's ID-token
* path in its OIDC callback. Use this for IdPs whose UserInfo endpoint omits claims that
* are present on the ID token, e.g. Microsoft Entra ID's Graph userinfo endpoint dropping
* `email` for some tenants)
*
* SAML Providers:
* SSO_SAML_ENTRY_POINT=https://your-idp/sso
* SSO_SAML_CERT=your-certificate-pem-string
* SSO_SAML_CALLBACK_URL=https://yourdomain.com/api/auth/sso/saml2/callback/provider-id
* SSO_SAML_SP_METADATA=<custom-sp-metadata-xml> (optional, auto-generated if not provided)
* SSO_SAML_IDP_METADATA=<idp-metadata-xml> (optional)
* SSO_SAML_AUDIENCE=https://yourdomain.com (optional, defaults to SSO_ISSUER)
* SSO_SAML_WANT_ASSERTIONS_SIGNED=true (optional, defaults to false)
*/
import { getErrorMessage } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { eq } from 'drizzle-orm'
import { drizzle } from 'drizzle-orm/postgres-js'
import postgres from 'postgres'
import { ssoProvider, user } from '../schema'
interface SSOMapping {
id: string
email: string
name: string
image?: string
}
interface OIDCConfig {
clientId: string
clientSecret: string
scopes?: string[]
pkce?: boolean
authorizationEndpoint?: string
tokenEndpoint?: string
userInfoEndpoint?: string
skipUserInfoEndpoint?: boolean
jwksEndpoint?: string
discoveryEndpoint?: string
tokenEndpointAuthentication?: 'client_secret_post' | 'client_secret_basic'
}
interface SAMLConfig {
issuer?: string
entryPoint: string
cert: string
callbackUrl?: string
audience?: string
wantAssertionsSigned?: boolean
signatureAlgorithm?: string
digestAlgorithm?: string
identifierFormat?: string
idpMetadata?: {
metadata?: string
entityID?: string
cert?: string
privateKey?: string
privateKeyPass?: string
isAssertionEncrypted?: boolean
encPrivateKey?: string
encPrivateKeyPass?: string
singleSignOnService?: Array<{
Binding: string
Location: string
}>
}
spMetadata?: {
metadata?: string
entityID?: string
binding?: string
privateKey?: string
privateKeyPass?: string
isAssertionEncrypted?: boolean
encPrivateKey?: string
encPrivateKeyPass?: string
}
privateKey?: string
decryptionPvk?: string
additionalParams?: Record<string, unknown>
}
interface SSOProviderConfig {
providerId: string
issuer: string
domain: string
providerType: 'oidc' | 'saml'
mapping?: SSOMapping
oidcConfig?: OIDCConfig
samlConfig?: SAMLConfig
}
const logger = {
info: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.log(
`[${timestamp}] [INFO] [RegisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
error: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.error(
`[${timestamp}] [ERROR] [RegisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
warn: (message: string, meta?: any) => {
const timestamp = new Date().toISOString()
console.warn(
`[${timestamp}] [WARN] [RegisterSSODB] ${message}`,
meta ? JSON.stringify(meta, null, 2) : ''
)
},
}
const CONNECTION_STRING = process.env.POSTGRES_URL ?? process.env.DATABASE_URL
if (!CONNECTION_STRING) {
console.error('❌ POSTGRES_URL or DATABASE_URL environment variable is required')
process.exit(1)
}
const postgresClient = postgres(CONNECTION_STRING, {
prepare: false,
idle_timeout: 20,
connect_timeout: 30,
max: 10,
onnotice: () => {},
})
const db = drizzle(postgresClient)
interface SSOProviderData {
id: string
issuer: string
domain: string
oidcConfig?: string
samlConfig?: string
userId: string
providerId: string
organizationId?: string
}
function buildSSOConfigFromEnv(): SSOProviderConfig | null {
const enabled = process.env.SSO_ENABLED === 'true'
if (!enabled) return null
const providerId = process.env.SSO_PROVIDER_ID
const issuer = process.env.SSO_ISSUER
const domain = process.env.SSO_DOMAIN
const providerType = process.env.SSO_PROVIDER_TYPE as 'oidc' | 'saml'
if (!providerId || !issuer || !domain || !providerType) {
return null
}
const config: SSOProviderConfig = {
providerId,
issuer,
domain,
providerType,
}
config.mapping = {
id:
process.env.SSO_MAPPING_ID ||
(providerType === 'oidc'
? 'sub'
: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier'),
email:
process.env.SSO_MAPPING_EMAIL ||
(providerType === 'oidc'
? 'email'
: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'),
name:
process.env.SSO_MAPPING_NAME ||
(providerType === 'oidc'
? 'name'
: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'),
image: process.env.SSO_MAPPING_IMAGE || (providerType === 'oidc' ? 'picture' : undefined),
}
if (providerType === 'oidc') {
const clientId = process.env.SSO_OIDC_CLIENT_ID
const clientSecret = process.env.SSO_OIDC_CLIENT_SECRET
if (!clientId || !clientSecret) {
return null
}
config.oidcConfig = {
clientId,
clientSecret,
scopes: process.env.SSO_OIDC_SCOPES?.split(',').map((s) => s.trim()) || [
'openid',
'profile',
'email',
],
pkce: process.env.SSO_OIDC_PKCE !== 'false',
authorizationEndpoint: process.env.SSO_OIDC_AUTHORIZATION_ENDPOINT,
tokenEndpoint: process.env.SSO_OIDC_TOKEN_ENDPOINT,
tokenEndpointAuthentication:
process.env.SSO_OIDC_TOKEN_ENDPOINT_AUTH === 'client_secret_post' ||
process.env.SSO_OIDC_TOKEN_ENDPOINT_AUTH === 'client_secret_basic'
? process.env.SSO_OIDC_TOKEN_ENDPOINT_AUTH
: undefined,
userInfoEndpoint: process.env.SSO_OIDC_USERINFO_ENDPOINT,
skipUserInfoEndpoint: process.env.SSO_OIDC_SKIP_USERINFO_ENDPOINT === 'true',
jwksEndpoint: process.env.SSO_OIDC_JWKS_ENDPOINT,
discoveryEndpoint:
process.env.SSO_OIDC_DISCOVERY_ENDPOINT ||
`${issuer.replace(/\/$/, '')}/.well-known/openid-configuration`,
}
} else if (providerType === 'saml') {
const entryPoint = process.env.SSO_SAML_ENTRY_POINT
const cert = process.env.SSO_SAML_CERT
if (!entryPoint || !cert) {
return null
}
const appBaseUrl = (
process.env.NEXT_PUBLIC_APP_URL ||
process.env.BETTER_AUTH_URL ||
''
).replace(/\/$/, '')
const escapeXml = (str: string) =>
str.replace(/[<>&"']/g, (c) => {
switch (c) {
case '<':
return '&lt;'
case '>':
return '&gt;'
case '&':
return '&amp;'
case '"':
return '&quot;'
case "'":
return '&apos;'
default:
return c
}
})
const callbackUrl =
process.env.SSO_SAML_CALLBACK_URL || `${appBaseUrl}/api/auth/sso/saml2/callback/${providerId}`
let spMetadata = process.env.SSO_SAML_SP_METADATA
if (!spMetadata) {
spMetadata = `<?xml version="1.0" encoding="UTF-8"?>
<md:EntityDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" entityID="${escapeXml(appBaseUrl)}">
<md:SPSSODescriptor AuthnRequestsSigned="false" WantAssertionsSigned="false" protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
<md:AssertionConsumerService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" Location="${escapeXml(callbackUrl)}" index="1"/>
</md:SPSSODescriptor>
</md:EntityDescriptor>`
}
const idpMetadataXml = process.env.SSO_SAML_IDP_METADATA
let computedIdpMetadata: string
if (idpMetadataXml) {
computedIdpMetadata = idpMetadataXml
} else {
const certBase64 = cert
.replace(/-----BEGIN CERTIFICATE-----/g, '')
.replace(/-----END CERTIFICATE-----/g, '')
.replace(/\s/g, '')
const escapedEntryPoint = escapeXml(entryPoint)
computedIdpMetadata = `<?xml version="1.0"?>
<EntityDescriptor xmlns="urn:oasis:names:tc:SAML:2.0:metadata" entityID="${escapeXml(issuer)}">
<IDPSSODescriptor WantAuthnRequestsSigned="false" protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
<KeyDescriptor use="signing">
<ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
<ds:X509Data>
<ds:X509Certificate>${certBase64}</ds:X509Certificate>
</ds:X509Data>
</ds:KeyInfo>
</KeyDescriptor>
<SingleSignOnService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" Location="${escapedEntryPoint}"/>
<SingleSignOnService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" Location="${escapedEntryPoint}"/>
</IDPSSODescriptor>
</EntityDescriptor>`
}
config.samlConfig = {
issuer,
entryPoint,
cert,
callbackUrl,
audience: process.env.SSO_SAML_AUDIENCE || issuer,
wantAssertionsSigned: process.env.SSO_SAML_WANT_ASSERTIONS_SIGNED === 'true',
signatureAlgorithm: process.env.SSO_SAML_SIGNATURE_ALGORITHM,
digestAlgorithm: process.env.SSO_SAML_DIGEST_ALGORITHM,
identifierFormat: process.env.SSO_SAML_IDENTIFIER_FORMAT,
spMetadata: {
metadata: spMetadata,
entityID: appBaseUrl,
},
idpMetadata: {
metadata: computedIdpMetadata,
},
}
}
return config
}
function getExampleEnvVars(
providerType: 'oidc' | 'saml',
provider?: string
): Record<string, string> {
const baseVars = {
SSO_ENABLED: 'true',
SSO_PROVIDER_TYPE: providerType,
SSO_PROVIDER_ID: provider || (providerType === 'oidc' ? 'okta' : 'adfs'),
SSO_DOMAIN: 'yourcompany.com',
SSO_USER_EMAIL: 'admin@yourcompany.com',
}
if (providerType === 'oidc') {
const examples: Record<string, Record<string, string>> = {
okta: {
...baseVars,
SSO_PROVIDER_ID: 'okta',
SSO_ISSUER: 'https://dev-123456.okta.com/oauth2/default',
SSO_OIDC_CLIENT_ID: '0oavhncxymgOpe06E697',
SSO_OIDC_CLIENT_SECRET: 'your-client-secret',
SSO_OIDC_SCOPES: 'openid,profile,email',
},
'azure-ad': {
...baseVars,
SSO_PROVIDER_ID: 'azure-ad',
SSO_ISSUER: 'https://login.microsoftonline.com/{tenant-id}/v2.0',
SSO_OIDC_CLIENT_ID: 'your-application-id',
SSO_OIDC_CLIENT_SECRET: 'your-client-secret',
SSO_MAPPING_ID: 'oid',
SSO_OIDC_SKIP_USERINFO_ENDPOINT: 'true',
},
generic: {
...baseVars,
SSO_PROVIDER_ID: 'custom-oidc',
SSO_ISSUER: 'https://idp.example.com',
SSO_OIDC_CLIENT_ID: 'your-client-id',
SSO_OIDC_CLIENT_SECRET: 'your-client-secret',
SSO_OIDC_AUTHORIZATION_ENDPOINT: 'https://idp.example.com/auth',
SSO_OIDC_TOKEN_ENDPOINT: 'https://idp.example.com/token',
SSO_OIDC_USERINFO_ENDPOINT: 'https://idp.example.com/userinfo',
},
}
return examples[provider || 'okta'] || examples.generic
}
return {
...baseVars,
SSO_PROVIDER_ID: 'adfs',
SSO_ISSUER: 'https://adfs.company.com',
SSO_SAML_ENTRY_POINT: 'https://adfs.company.com/adfs/ls/',
SSO_SAML_CERT:
'-----BEGIN CERTIFICATE-----\nMIIDBjCCAe4CAQAwDQYJKoZIhvcNAQEFBQAwEjEQMA4GA1UEAwwHYWRmcy...\n-----END CERTIFICATE-----',
SSO_SAML_AUDIENCE: 'https://yourapp.com',
SSO_SAML_WANT_ASSERTIONS_SIGNED: 'true',
SSO_MAPPING_ID: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier',
SSO_MAPPING_EMAIL: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress',
SSO_MAPPING_NAME: 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name',
}
}
async function getAdminUser(): Promise<{ id: string; email: string } | null> {
const adminEmail = process.env.SSO_USER_EMAIL
if (!adminEmail) {
logger.error('SSO_USER_EMAIL is required to identify the admin user')
return null
}
try {
const users = await db.select().from(user).where(eq(user.email, adminEmail))
if (users.length === 0) {
logger.error(`No user found with email: ${adminEmail}`)
logger.error('Please ensure this user exists in your database first')
return null
}
return { id: users[0].id, email: users[0].email }
} catch (error) {
logger.error('Failed to query user:', error)
return null
}
}
async function registerSSOProvider(): Promise<boolean> {
try {
const ssoConfig = buildSSOConfigFromEnv()
if (!ssoConfig) {
logger.error('❌ No valid SSO configuration found in environment variables')
logger.error('')
logger.error('📝 Required environment variables:')
logger.error('For OIDC providers (like Okta, Azure AD):')
const oidcExample = getExampleEnvVars('oidc', 'okta')
for (const [key, value] of Object.entries(oidcExample)) {
logger.error(` ${key}=${value}`)
}
logger.error(' SSO_USER_EMAIL=admin@yourdomain.com')
logger.error('')
logger.error('For SAML providers (like ADFS):')
const samlExample = getExampleEnvVars('saml')
for (const [key, value] of Object.entries(samlExample)) {
logger.error(` ${key}=${value}`)
}
logger.error(' SSO_USER_EMAIL=admin@yourdomain.com')
return false
}
const adminUser = await getAdminUser()
if (!adminUser) {
return false
}
logger.info('Registering SSO provider directly in database...', {
providerId: ssoConfig.providerId,
providerType: ssoConfig.providerType,
domain: ssoConfig.domain,
adminUser: adminUser.email,
})
try {
new URL(ssoConfig.issuer)
} catch {
logger.error('Invalid issuer. Must be a valid URL:', ssoConfig.issuer)
return false
}
if (
ssoConfig.providerType === 'saml' &&
!process.env.NEXT_PUBLIC_APP_URL &&
!process.env.BETTER_AUTH_URL
) {
logger.error(
'NEXT_PUBLIC_APP_URL or BETTER_AUTH_URL is required for SAML — it is used as the SP entity ID in SP metadata. Set one of these env vars.'
)
return false
}
if (ssoConfig.providerType === 'oidc' && ssoConfig.oidcConfig) {
const needsDiscovery =
!ssoConfig.oidcConfig.authorizationEndpoint ||
!ssoConfig.oidcConfig.tokenEndpoint ||
!ssoConfig.oidcConfig.jwksEndpoint
if (needsDiscovery) {
const discoveryUrl =
ssoConfig.oidcConfig.discoveryEndpoint ||
`${ssoConfig.issuer.replace(/\/$/, '')}/.well-known/openid-configuration`
logger.info('Fetching OIDC discovery document for missing endpoints...', {
discoveryUrl,
hasAuthEndpoint: !!ssoConfig.oidcConfig.authorizationEndpoint,
hasTokenEndpoint: !!ssoConfig.oidcConfig.tokenEndpoint,
hasJwksEndpoint: !!ssoConfig.oidcConfig.jwksEndpoint,
})
try {
const response = await fetch(discoveryUrl, {
headers: { Accept: 'application/json' },
})
if (!response.ok) {
logger.error('Failed to fetch OIDC discovery document', {
status: response.status,
statusText: response.statusText,
})
logger.error(
'Provide all endpoints explicitly via SSO_OIDC_AUTHORIZATION_ENDPOINT, SSO_OIDC_TOKEN_ENDPOINT, SSO_OIDC_JWKS_ENDPOINT'
)
return false
}
const discovery = await response.json()
ssoConfig.oidcConfig.authorizationEndpoint =
ssoConfig.oidcConfig.authorizationEndpoint || discovery.authorization_endpoint
ssoConfig.oidcConfig.tokenEndpoint =
ssoConfig.oidcConfig.tokenEndpoint || discovery.token_endpoint
ssoConfig.oidcConfig.userInfoEndpoint =
ssoConfig.oidcConfig.userInfoEndpoint || discovery.userinfo_endpoint
ssoConfig.oidcConfig.jwksEndpoint =
ssoConfig.oidcConfig.jwksEndpoint || discovery.jwks_uri
logger.info('Merged OIDC endpoints (user-provided + discovery)', {
authorizationEndpoint: ssoConfig.oidcConfig.authorizationEndpoint,
tokenEndpoint: ssoConfig.oidcConfig.tokenEndpoint,
userInfoEndpoint: ssoConfig.oidcConfig.userInfoEndpoint,
jwksEndpoint: ssoConfig.oidcConfig.jwksEndpoint,
})
} catch (error) {
logger.error('Error fetching OIDC discovery document', {
error: getErrorMessage(error, 'Unknown error'),
discoveryUrl,
})
logger.error(
'Please provide explicit endpoints via SSO_OIDC_AUTHORIZATION_ENDPOINT, SSO_OIDC_TOKEN_ENDPOINT, SSO_OIDC_JWKS_ENDPOINT'
)
return false
}
} else {
logger.info('Using explicitly provided OIDC endpoints (all present)', {
authorizationEndpoint: ssoConfig.oidcConfig.authorizationEndpoint,
tokenEndpoint: ssoConfig.oidcConfig.tokenEndpoint,
userInfoEndpoint: ssoConfig.oidcConfig.userInfoEndpoint,
jwksEndpoint: ssoConfig.oidcConfig.jwksEndpoint,
})
}
if (ssoConfig.oidcConfig.skipUserInfoEndpoint) {
ssoConfig.oidcConfig.userInfoEndpoint = undefined
logger.info('Skipping UserInfo endpoint: claims will be read from the verified ID token')
}
if (
!ssoConfig.oidcConfig.authorizationEndpoint ||
!ssoConfig.oidcConfig.tokenEndpoint ||
!ssoConfig.oidcConfig.jwksEndpoint
) {
const missing: string[] = []
if (!ssoConfig.oidcConfig.authorizationEndpoint)
missing.push('SSO_OIDC_AUTHORIZATION_ENDPOINT')
if (!ssoConfig.oidcConfig.tokenEndpoint) missing.push('SSO_OIDC_TOKEN_ENDPOINT')
if (!ssoConfig.oidcConfig.jwksEndpoint) missing.push('SSO_OIDC_JWKS_ENDPOINT')
logger.error('Missing required OIDC endpoints after discovery merge', {
missing,
authorizationEndpoint: ssoConfig.oidcConfig.authorizationEndpoint,
tokenEndpoint: ssoConfig.oidcConfig.tokenEndpoint,
jwksEndpoint: ssoConfig.oidcConfig.jwksEndpoint,
})
logger.error(`Please provide: ${missing.join(', ')}`)
return false
}
}
const existingProviders = await db
.select()
.from(ssoProvider)
.where(eq(ssoProvider.providerId, ssoConfig.providerId))
if (existingProviders.length > 0) {
logger.warn(`SSO provider with ID '${ssoConfig.providerId}' already exists`)
logger.info('Updating existing provider...')
}
const providerData: SSOProviderData = {
id: generateId(),
issuer: ssoConfig.issuer,
domain: ssoConfig.domain,
userId: adminUser.id,
providerId: ssoConfig.providerId,
organizationId: process.env.SSO_ORGANIZATION_ID || undefined,
}
if (ssoConfig.providerType === 'oidc' && ssoConfig.oidcConfig) {
const oidcConfig = {
issuer: ssoConfig.issuer,
clientId: ssoConfig.oidcConfig.clientId,
clientSecret: ssoConfig.oidcConfig.clientSecret,
authorizationEndpoint: ssoConfig.oidcConfig.authorizationEndpoint,
tokenEndpoint: ssoConfig.oidcConfig.tokenEndpoint,
// Default to client_secret_post: better-auth sends client_secret_basic
// credentials without URL-encoding per RFC 6749 §2.3.1, so '+' in secrets
// is decoded as space by OIDC providers, causing invalid_client errors.
tokenEndpointAuthentication:
ssoConfig.oidcConfig.tokenEndpointAuthentication ?? 'client_secret_post',
jwksEndpoint: ssoConfig.oidcConfig.jwksEndpoint,
pkce: ssoConfig.oidcConfig.pkce,
discoveryEndpoint:
ssoConfig.oidcConfig.discoveryEndpoint ||
`${ssoConfig.issuer}/.well-known/openid-configuration`,
mapping: ssoConfig.mapping,
scopes: ssoConfig.oidcConfig.scopes,
userInfoEndpoint: ssoConfig.oidcConfig.userInfoEndpoint,
overrideUserInfo: false,
}
providerData.oidcConfig = JSON.stringify(oidcConfig)
}
if (ssoConfig.providerType === 'saml' && ssoConfig.samlConfig) {
const samlConfig = {
issuer: ssoConfig.issuer,
entryPoint: ssoConfig.samlConfig.entryPoint,
cert: ssoConfig.samlConfig.cert,
callbackUrl: ssoConfig.samlConfig.callbackUrl,
audience: ssoConfig.samlConfig.audience,
idpMetadata: ssoConfig.samlConfig.idpMetadata,
spMetadata: ssoConfig.samlConfig.spMetadata,
wantAssertionsSigned: ssoConfig.samlConfig.wantAssertionsSigned,
signatureAlgorithm: ssoConfig.samlConfig.signatureAlgorithm,
digestAlgorithm: ssoConfig.samlConfig.digestAlgorithm,
identifierFormat: ssoConfig.samlConfig.identifierFormat,
privateKey: ssoConfig.samlConfig.privateKey,
decryptionPvk: ssoConfig.samlConfig.decryptionPvk,
additionalParams: ssoConfig.samlConfig.additionalParams,
mapping: ssoConfig.mapping,
}
providerData.samlConfig = JSON.stringify(samlConfig)
}
if (existingProviders.length > 0) {
await db
.update(ssoProvider)
.set({
issuer: providerData.issuer,
domain: providerData.domain,
oidcConfig: providerData.oidcConfig,
samlConfig: providerData.samlConfig,
userId: providerData.userId,
organizationId: providerData.organizationId,
})
.where(eq(ssoProvider.providerId, ssoConfig.providerId))
} else {
await db.insert(ssoProvider).values(providerData)
}
logger.info('✅ SSO provider registered successfully in database!', {
providerId: ssoConfig.providerId,
providerType: ssoConfig.providerType,
domain: ssoConfig.domain,
id: providerData.id,
})
logger.info('🔗 Users can now sign in using SSO')
const baseUrl =
process.env.NEXT_PUBLIC_APP_URL || process.env.BETTER_AUTH_URL || 'https://your-domain.com'
const callbackPath =
ssoConfig.providerType === 'saml'
? `api/auth/sso/saml2/callback/${ssoConfig.providerId}`
: `api/auth/sso/callback/${ssoConfig.providerId}`
logger.info(
`📋 Callback URL (configure this in your identity provider): ${baseUrl}/${callbackPath}`
)
return true
} catch (error) {
logger.error('❌ Failed to register SSO provider:', {
error: getErrorMessage(error, 'Unknown error'),
errorType: typeof error,
errorDetails: JSON.stringify(error),
stack: error instanceof Error ? error.stack : undefined,
})
return false
} finally {
try {
await postgresClient.end({ timeout: 5 })
} catch {}
}
}
async function main() {
console.log('🔐 Direct Database SSO Registration Script (Better Auth Best Practice)')
console.log('====================================================================')
console.log('This script directly inserts SSO provider records into the database.')
console.log("It follows Better Auth's exact registerSSOProvider logic.\n")
const success = await registerSSOProvider()
if (success) {
console.log('🎉 SSO setup completed successfully!')
console.log()
console.log('Next steps:')
console.log('1. Configure the callback URL in your identity provider')
console.log('2. Restart your application if needed')
console.log('3. Users can now sign in with SSO!')
process.exit(0)
} else {
console.log('💥 SSO setup failed. Check the logs above for details.')
process.exit(1)
}
}
main().catch((error) => {
logger.error('Script execution failed:', { error })
process.exit(1)
})
@@ -0,0 +1,221 @@
/// <reference types="node" />
/**
* Seed script to populate the stress_test_users table.
*
* Usage:
* cd packages/db && bun run scripts/seed-stress-test-users.ts
*/
import { generateId } from '@sim/utils/id'
import { randomFloat, randomInt, randomItem } from '@sim/utils/random'
import { eq, type InferInsertModel } from 'drizzle-orm'
import { db, userTableDefinitions, userTableRows } from '../index'
const WORKSPACE_ID = '098d71e1-6a36-47e3-874d-818faee0bfe8'
const TABLE_NAME = 'stress_test_users'
const NUM_ROWS = 100000
interface UserRow {
name: string
email: string
age: number
department: string
salary: number
active: boolean
hire_date: string
country: string
}
const departments = [
'Engineering',
'Sales',
'Marketing',
'HR',
'Finance',
'Operations',
'Legal',
'Product',
]
const countries = [
'USA',
'UK',
'Germany',
'France',
'Canada',
'Australia',
'Japan',
'India',
'Brazil',
'Singapore',
]
const firstNames = [
'James',
'Mary',
'John',
'Patricia',
'Robert',
'Jennifer',
'Michael',
'Linda',
'William',
'Elizabeth',
'David',
'Barbara',
'Richard',
'Susan',
'Joseph',
'Jessica',
'Thomas',
'Sarah',
'Charles',
'Karen',
]
const lastNames = [
'Smith',
'Johnson',
'Williams',
'Brown',
'Jones',
'Garcia',
'Miller',
'Davis',
'Rodriguez',
'Martinez',
'Hernandez',
'Lopez',
'Gonzalez',
'Wilson',
'Anderson',
'Thomas',
'Taylor',
'Moore',
'Jackson',
'Martin',
]
function randomDate(start: Date, end: Date): string {
const date = new Date(start.getTime() + randomFloat() * (end.getTime() - start.getTime()))
return date.toISOString().split('T')[0]
}
function generateUserRow(index: number): UserRow {
const firstName = randomItem(firstNames)
const lastName = randomItem(lastNames)
const domain = randomItem(['gmail.com', 'yahoo.com', 'outlook.com', 'company.com', 'work.org'])
return {
name: `${firstName} ${lastName}`,
email: `${firstName.toLowerCase()}.${lastName.toLowerCase()}${index}@${domain}`,
age: randomInt(22, 65),
department: randomItem(departments),
salary: randomInt(40000, 200000),
active: randomFloat() > 0.1, // 90% active
hire_date: randomDate(new Date('2015-01-01'), new Date('2024-12-31')),
country: randomItem(countries),
}
}
async function main() {
console.log(`Seeding ${TABLE_NAME} table for workspace ${WORKSPACE_ID}...`)
// Get user ID for created_by
const userResult = (await db.execute(`SELECT id FROM "user" LIMIT 1`)) as { id: string }[]
const userId = Array.isArray(userResult) && userResult[0] ? userResult[0].id : 'system'
console.log(`Using user ID: ${userId}`)
// Check if table already exists
const existingTable = await db
.select()
.from(userTableDefinitions)
.where(eq(userTableDefinitions.workspaceId, WORKSPACE_ID))
.limit(1)
let tableId: string
if (existingTable.length > 0 && existingTable[0].name === TABLE_NAME) {
tableId = existingTable[0].id
console.log(`Table ${TABLE_NAME} already exists (${tableId}), clearing existing rows...`)
// Delete existing rows
await db.delete(userTableRows).where(eq(userTableRows.tableId, tableId))
// Reset row count (trigger will update it as we insert)
await db
.update(userTableDefinitions)
.set({ rowCount: 0, updatedAt: new Date() })
.where(eq(userTableDefinitions.id, tableId))
} else {
// Create table
tableId = `tbl_${generateId().replace(/-/g, '')}`
const now = new Date()
const tableSchema = {
columns: [
{ name: 'name', type: 'string', required: true },
{ name: 'email', type: 'string', required: true, unique: true },
{ name: 'age', type: 'number', required: true },
{ name: 'department', type: 'string', required: true },
{ name: 'salary', type: 'number', required: true },
{ name: 'active', type: 'boolean', required: true },
{ name: 'hire_date', type: 'string', required: true },
{ name: 'country', type: 'string', required: true },
],
}
await db.insert(userTableDefinitions).values({
id: tableId,
workspaceId: WORKSPACE_ID,
name: TABLE_NAME,
description: 'Stress test table with sample user data',
schema: tableSchema,
maxRows: 10000,
createdBy: userId,
createdAt: now,
updatedAt: now,
})
console.log(`Created table ${TABLE_NAME} (${tableId})`)
}
// Generate and insert rows in batches
const batchSize = 1000
const now = new Date()
console.log(`Inserting ${NUM_ROWS} rows in batches of ${batchSize}...`)
for (let i = 0; i < NUM_ROWS; i += batchSize) {
const batch: InferInsertModel<typeof userTableRows>[] = []
const endIdx = Math.min(i + batchSize, NUM_ROWS)
for (let j = i; j < endIdx; j++) {
batch.push({
id: `row_${generateId().replace(/-/g, '')}`,
tableId,
workspaceId: WORKSPACE_ID,
data: generateUserRow(j),
createdBy: userId,
createdAt: now,
updatedAt: now,
})
}
await db.insert(userTableRows).values(batch)
console.log(` Inserted rows ${i + 1} to ${endIdx}`)
}
// Verify final row count
const finalTable = await db
.select({ rowCount: userTableDefinitions.rowCount })
.from(userTableDefinitions)
.where(eq(userTableDefinitions.id, tableId))
.limit(1)
console.log(`\nDone! Table ${TABLE_NAME} now has ${finalTable[0]?.rowCount ?? 0} rows.`)
process.exit(0)
}
main().catch((err) => {
console.error('Error seeding data:', err)
process.exit(1)
})