chore: import upstream snapshot with attribution

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/route.ts

@@ -0,0 +1,190 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { db } from '@sim/db'
+import { dataDrains } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getPostgresErrorCode } from '@sim/utils/errors'
+import { and, eq, ne } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import {
+  deleteDataDrainContract,
+  getDataDrainContract,
+  updateDataDrainContract,
+} from '@/lib/api/contracts/data-drains'
+import { parseRequest, validationErrorResponse } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
+import { getDestination } from '@/lib/data-drains/destinations/registry'
+import { encryptCredentials } from '@/lib/data-drains/encryption'
+import { serializeDrain } from '@/lib/data-drains/serializers'
+
+const logger = createLogger('DataDrainAPI')
+
+type RouteContext = { params: Promise<{ id: string; drainId: string }> }
+
+export const GET = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: false })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(getDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+  return NextResponse.json({ drain: serializeDrain(drain) })
+})
+
+export const PUT = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: true })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(updateDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const body = parsed.data.body
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+
+  if (body.name !== undefined && body.name !== drain.name) {
+    const [conflict] = await db
+      .select({ id: dataDrains.id })
+      .from(dataDrains)
+      .where(
+        and(
+          eq(dataDrains.organizationId, organizationId),
+          eq(dataDrains.name, body.name),
+          ne(dataDrains.id, drainId)
+        )
+      )
+      .limit(1)
+    if (conflict) {
+      return NextResponse.json(
+        { error: 'A data drain with this name already exists in this organization' },
+        { status: 409 }
+      )
+    }
+  }
+
+  if (body.source !== undefined && body.source !== drain.source) {
+    return NextResponse.json({ error: 'source cannot be changed after creation' }, { status: 400 })
+  }
+
+  const updates: Partial<typeof dataDrains.$inferInsert> = { updatedAt: new Date() }
+  if (body.name !== undefined) updates.name = body.name
+  if (body.scheduleCadence !== undefined) updates.scheduleCadence = body.scheduleCadence
+  if (body.enabled !== undefined) updates.enabled = body.enabled
+
+  if (body.destinationType !== undefined && body.destinationType !== drain.destinationType) {
+    return NextResponse.json(
+      { error: 'destinationType cannot be changed after creation' },
+      { status: 400 }
+    )
+  }
+  if (body.destinationConfig !== undefined || body.destinationCredentials !== undefined) {
+    const destination = getDestination(drain.destinationType)
+    if (body.destinationConfig !== undefined) {
+      const configResult = destination.configSchema.safeParse(body.destinationConfig)
+      if (!configResult.success) return validationErrorResponse(configResult.error)
+      updates.destinationConfig = configResult.data as Record<string, unknown>
+    }
+    if (body.destinationCredentials !== undefined) {
+      const credentialsResult = destination.credentialsSchema.safeParse(body.destinationCredentials)
+      if (!credentialsResult.success) return validationErrorResponse(credentialsResult.error)
+      updates.destinationCredentials = await encryptCredentials(credentialsResult.data)
+    }
+  }
+
+  let updated: typeof dataDrains.$inferSelect | undefined
+  try {
+    ;[updated] = await db
+      .update(dataDrains)
+      .set(updates)
+      .where(eq(dataDrains.id, drainId))
+      .returning()
+  } catch (error) {
+    if (getPostgresErrorCode(error) === '23505') {
+      return NextResponse.json(
+        { error: 'A data drain with this name already exists in this organization' },
+        { status: 409 }
+      )
+    }
+    throw error
+  }
+
+  if (!updated) {
+    // Concurrent DELETE landed between loadDrain() and this UPDATE.
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+
+  logger.info('Data drain updated', { drainId, organizationId })
+
+  recordAudit({
+    workspaceId: null,
+    actorId: access.session.user.id,
+    action: AuditAction.DATA_DRAIN_UPDATED,
+    resourceType: AuditResourceType.DATA_DRAIN,
+    resourceId: drainId,
+    actorName: access.session.user.name ?? undefined,
+    actorEmail: access.session.user.email ?? undefined,
+    resourceName: updated.name,
+    description: `Updated data drain '${updated.name}'`,
+    metadata: {
+      organizationId,
+      changes: {
+        name: body.name,
+        source: body.source,
+        scheduleCadence: body.scheduleCadence,
+        enabled: body.enabled,
+        destinationConfigChanged: body.destinationConfig !== undefined,
+        destinationCredentialsChanged: body.destinationCredentials !== undefined,
+      },
+    },
+    request,
+  })
+
+  return NextResponse.json({ drain: serializeDrain(updated) })
+})
+
+export const DELETE = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: true })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(deleteDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+
+  await db.delete(dataDrains).where(eq(dataDrains.id, drainId))
+
+  logger.info('Data drain deleted', { drainId, organizationId })
+
+  recordAudit({
+    workspaceId: null,
+    actorId: access.session.user.id,
+    action: AuditAction.DATA_DRAIN_DELETED,
+    resourceType: AuditResourceType.DATA_DRAIN,
+    resourceId: drainId,
+    actorName: access.session.user.name ?? undefined,
+    actorEmail: access.session.user.email ?? undefined,
+    resourceName: drain.name,
+    description: `Deleted data drain '${drain.name}'`,
+    metadata: {
+      organizationId,
+      source: drain.source,
+      destinationType: drain.destinationType,
+    },
+    request,
+  })
+
+  return NextResponse.json({ success: true as const })
+})

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/run/route.ts

@@ -0,0 +1,76 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { db } from '@sim/db'
+import { dataDrainRuns } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { and, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { runDataDrainContract } from '@/lib/api/contracts/data-drains'
+import { parseRequest } from '@/lib/api/server'
+import { getJobQueue } from '@/lib/core/async-jobs'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
+
+const logger = createLogger('DataDrainRunAPI')
+
+type RouteContext = { params: Promise<{ id: string; drainId: string }> }
+
+export const POST = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: true })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(runDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+  if (!drain.enabled) {
+    return NextResponse.json(
+      { error: 'Cannot run a disabled drain. Enable it first.' },
+      { status: 400 }
+    )
+  }
+
+  // Reject obvious double-fires up-front. The job-queue concurrencyKey is the
+  // real serialization barrier (it covers the gap between enqueue and the
+  // runner inserting the `running` row), but this gives the user immediate
+  // feedback when a run is already in flight.
+  const [inFlight] = await db
+    .select({ id: dataDrainRuns.id })
+    .from(dataDrainRuns)
+    .where(and(eq(dataDrainRuns.drainId, drainId), eq(dataDrainRuns.status, 'running')))
+    .limit(1)
+  if (inFlight) {
+    return NextResponse.json(
+      { error: 'A run is already in progress for this drain' },
+      { status: 409 }
+    )
+  }
+
+  const queue = await getJobQueue()
+  const jobId = await queue.enqueue(
+    'run-data-drain',
+    { drainId, trigger: 'manual' },
+    { concurrencyKey: `data-drain:${drainId}` }
+  )
+
+  logger.info('Manually enqueued data drain run', { drainId, organizationId, jobId })
+
+  recordAudit({
+    workspaceId: null,
+    actorId: access.session.user.id,
+    action: AuditAction.DATA_DRAIN_RAN,
+    resourceType: AuditResourceType.DATA_DRAIN,
+    resourceId: drainId,
+    actorName: access.session.user.name ?? undefined,
+    actorEmail: access.session.user.email ?? undefined,
+    resourceName: drain.name,
+    description: `Triggered manual run for data drain '${drain.name}'`,
+    metadata: { organizationId, jobId, trigger: 'manual' },
+    request,
+  })
+
+  return NextResponse.json({ jobId })
+})

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/runs/route.ts

@@ -0,0 +1,37 @@
+import { db } from '@sim/db'
+import { dataDrainRuns } from '@sim/db/schema'
+import { desc, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { listDataDrainRunsContract } from '@/lib/api/contracts/data-drains'
+import { parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
+import { serializeDrainRun } from '@/lib/data-drains/serializers'
+
+const DEFAULT_LIMIT = 25
+
+type RouteContext = { params: Promise<{ id: string; drainId: string }> }
+
+export const GET = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: false })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(listDataDrainRunsContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+
+  const limit = parsed.data.query?.limit ?? DEFAULT_LIMIT
+  const runs = await db
+    .select()
+    .from(dataDrainRuns)
+    .where(eq(dataDrainRuns.drainId, drainId))
+    .orderBy(desc(dataDrainRuns.startedAt))
+    .limit(limit)
+
+  return NextResponse.json({ runs: runs.map(serializeDrainRun) })
+})

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts

@@ -0,0 +1,91 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { type NextRequest, NextResponse } from 'next/server'
+import { testDataDrainContract } from '@/lib/api/contracts/data-drains'
+import { parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
+import { getDestination } from '@/lib/data-drains/destinations/registry'
+import { decryptCredentials } from '@/lib/data-drains/encryption'
+
+const logger = createLogger('DataDrainTestAPI')
+
+const TEST_TIMEOUT_MS = 10_000
+
+type RouteContext = { params: Promise<{ id: string; drainId: string }> }
+
+export const POST = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId, drainId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: true })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(testDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const drain = await loadDrain(organizationId, drainId)
+  if (!drain) {
+    return NextResponse.json({ error: 'Data drain not found' }, { status: 404 })
+  }
+
+  const destination = getDestination(drain.destinationType)
+  if (!destination.test) {
+    return NextResponse.json(
+      { error: `Destination '${drain.destinationType}' does not support connection testing` },
+      { status: 400 }
+    )
+  }
+
+  const config = destination.configSchema.parse(drain.destinationConfig)
+  const credentials = destination.credentialsSchema.parse(
+    await decryptCredentials(drain.destinationCredentials)
+  )
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS)
+  try {
+    await destination.test({ config, credentials, signal: controller.signal })
+    recordAudit({
+      workspaceId: null,
+      actorId: access.session.user.id,
+      action: AuditAction.DATA_DRAIN_TESTED,
+      resourceType: AuditResourceType.DATA_DRAIN,
+      resourceId: drainId,
+      actorName: access.session.user.name ?? undefined,
+      actorEmail: access.session.user.email ?? undefined,
+      resourceName: drain.name,
+      description: `Tested connection for data drain '${drain.name}' (success)`,
+      metadata: { organizationId, destinationType: drain.destinationType, outcome: 'success' },
+      request,
+    })
+    return NextResponse.json({ ok: true as const })
+  } catch (error) {
+    const message = toError(error).message
+    logger.warn('Data drain test connection failed', {
+      drainId,
+      destinationType: drain.destinationType,
+      error: message,
+    })
+    recordAudit({
+      workspaceId: null,
+      actorId: access.session.user.id,
+      action: AuditAction.DATA_DRAIN_TESTED,
+      resourceType: AuditResourceType.DATA_DRAIN,
+      resourceId: drainId,
+      actorName: access.session.user.name ?? undefined,
+      actorEmail: access.session.user.email ?? undefined,
+      resourceName: drain.name,
+      description: `Tested connection for data drain '${drain.name}' (failed)`,
+      metadata: {
+        organizationId,
+        destinationType: drain.destinationType,
+        outcome: 'failed',
+        error: message,
+      },
+      request,
+    })
+    return NextResponse.json({ error: message }, { status: 400 })
+  } finally {
+    clearTimeout(timeout)
+  }
+})

apps/sim/app/api/organizations/[id]/data-drains/route.ts

@@ -0,0 +1,136 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { db } from '@sim/db'
+import { dataDrains } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getPostgresErrorCode } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, asc, eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { createDataDrainContract, listDataDrainsContract } from '@/lib/api/contracts/data-drains'
+import { parseRequest, validationErrorResponse } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { authorizeDrainAccess } from '@/lib/data-drains/access'
+import { getDestination } from '@/lib/data-drains/destinations/registry'
+import { encryptCredentials } from '@/lib/data-drains/encryption'
+import { serializeDrain } from '@/lib/data-drains/serializers'
+
+const logger = createLogger('DataDrainsAPI')
+
+type RouteContext = { params: Promise<{ id: string }> }
+
+export const GET = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: false })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(listDataDrainsContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const rows = await db
+    .select()
+    .from(dataDrains)
+    .where(eq(dataDrains.organizationId, organizationId))
+    .orderBy(asc(dataDrains.createdAt))
+
+  return NextResponse.json({ drains: rows.map(serializeDrain) })
+})
+
+export const POST = withRouteHandler(async (request: NextRequest, context: RouteContext) => {
+  const { id: organizationId } = await context.params
+  const access = await authorizeDrainAccess(organizationId, { requireMutating: true })
+  if (!access.ok) return access.response
+
+  const parsed = await parseRequest(createDataDrainContract, request, context)
+  if (!parsed.success) return parsed.response
+
+  const body = parsed.data.body
+
+  if (!body.destinationCredentials) {
+    return NextResponse.json(
+      { error: 'destinationCredentials is required when creating a drain' },
+      { status: 400 }
+    )
+  }
+  const destination = getDestination(body.destinationType)
+  const configResult = destination.configSchema.safeParse(body.destinationConfig)
+  if (!configResult.success) return validationErrorResponse(configResult.error)
+  const credentialsResult = destination.credentialsSchema.safeParse(body.destinationCredentials)
+  if (!credentialsResult.success) return validationErrorResponse(credentialsResult.error)
+  const encryptedCredentials = await encryptCredentials(credentialsResult.data)
+
+  const [existing] = await db
+    .select({ id: dataDrains.id })
+    .from(dataDrains)
+    .where(and(eq(dataDrains.organizationId, organizationId), eq(dataDrains.name, body.name)))
+    .limit(1)
+  if (existing) {
+    return NextResponse.json(
+      { error: 'A data drain with this name already exists in this organization' },
+      { status: 409 }
+    )
+  }
+
+  const id = generateId()
+  const now = new Date()
+  let inserted: typeof dataDrains.$inferSelect | undefined
+  try {
+    ;[inserted] = await db
+      .insert(dataDrains)
+      .values({
+        id,
+        organizationId,
+        name: body.name,
+        source: body.source,
+        destinationType: body.destinationType,
+        destinationConfig: configResult.data as Record<string, unknown>,
+        destinationCredentials: encryptedCredentials,
+        scheduleCadence: body.scheduleCadence,
+        enabled: body.enabled ?? true,
+        cursor: null,
+        createdBy: access.session.user.id,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .returning()
+  } catch (error) {
+    if (getPostgresErrorCode(error) === '23505') {
+      return NextResponse.json(
+        { error: 'A data drain with this name already exists in this organization' },
+        { status: 409 }
+      )
+    }
+    throw error
+  }
+
+  if (!inserted) {
+    throw new Error('Insert returned no row')
+  }
+
+  logger.info('Data drain created', {
+    drainId: id,
+    organizationId,
+    source: body.source,
+    destinationType: body.destinationType,
+  })
+
+  recordAudit({
+    workspaceId: null,
+    actorId: access.session.user.id,
+    action: AuditAction.DATA_DRAIN_CREATED,
+    resourceType: AuditResourceType.DATA_DRAIN,
+    resourceId: id,
+    actorName: access.session.user.name ?? undefined,
+    actorEmail: access.session.user.email ?? undefined,
+    resourceName: body.name,
+    description: `Created data drain '${body.name}'`,
+    metadata: {
+      organizationId,
+      source: body.source,
+      destinationType: body.destinationType,
+      scheduleCadence: body.scheduleCadence,
+    },
+    request,
+  })
+
+  return NextResponse.json({ drain: serializeDrain(inserted) }, { status: 201 })
+})