chore: import upstream snapshot with attribution

apps/sim/lib/data-drains/sources/audit-logs.ts

@@ -0,0 +1,79 @@
+import { dbReplica } from '@sim/db'
+import { auditLog } from '@sim/db/schema'
+import { and, inArray, isNull, or, sql } from 'drizzle-orm'
+import {
+  decodeTimeCursor,
+  encodeTimeCursor,
+  timeCursorOrderBy,
+  timeCursorPredicate,
+  timeCursorStabilityBound,
+} from '@/lib/data-drains/sources/cursor'
+import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
+import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
+
+type AuditLogRow = typeof auditLog.$inferSelect
+
+/**
+ * Drains audit events scoped to the organization: rows from any of the org's
+ * workspaces, plus org-level rows (`workspace_id IS NULL`) where
+ * `metadata->>'organizationId'` matches. Audit-log writers consistently set
+ * `metadata.organizationId` for org-scoped actions even though the table has
+ * no dedicated FK column.
+ */
+async function* pages(input: SourcePageInput): AsyncIterable<AuditLogRow[]> {
+  const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId)
+
+  const orgScopedClause = and(
+    isNull(auditLog.workspaceId),
+    sql`${auditLog.metadata}->>'organizationId' = ${input.organizationId}`
+  )
+  const scopeClause =
+    workspaceIds.length === 0
+      ? orgScopedClause
+      : or(inArray(auditLog.workspaceId, workspaceIds), orgScopedClause)
+
+  let cursor = decodeTimeCursor(input.cursor)
+  while (!input.signal.aborted) {
+    const cursorClause = timeCursorPredicate(auditLog.createdAt, auditLog.id, cursor)
+
+    const rows = await dbReplica
+      .select()
+      .from(auditLog)
+      .where(and(scopeClause, timeCursorStabilityBound(auditLog.createdAt), cursorClause))
+      .orderBy(...timeCursorOrderBy(auditLog.createdAt, auditLog.id))
+      .limit(input.chunkSize)
+
+    if (rows.length === 0) return
+    yield rows
+    const last = rows[rows.length - 1]
+    cursor = { ts: last.createdAt.toISOString(), id: last.id }
+    if (rows.length < input.chunkSize) return
+  }
+}
+
+export const auditLogsSource: DrainSource<AuditLogRow> = {
+  type: 'audit_logs',
+  displayName: 'Audit logs',
+  pages,
+  serialize(row) {
+    return {
+      id: row.id,
+      workspaceId: row.workspaceId,
+      actorId: row.actorId,
+      actorName: row.actorName,
+      actorEmail: row.actorEmail,
+      action: row.action,
+      resourceType: row.resourceType,
+      resourceId: row.resourceId,
+      resourceName: row.resourceName,
+      description: row.description,
+      metadata: row.metadata,
+      ipAddress: row.ipAddress,
+      userAgent: row.userAgent,
+      createdAt: row.createdAt.toISOString(),
+    }
+  },
+  cursorAfter(row): Cursor {
+    return encodeTimeCursor({ ts: row.createdAt.toISOString(), id: row.id })
+  },
+}

apps/sim/lib/data-drains/sources/copilot-chats.ts

@@ -0,0 +1,131 @@
+import { dbReplica } from '@sim/db'
+import { copilotChats, copilotMessages } from '@sim/db/schema'
+import { and, asc, inArray, isNull, sql } from 'drizzle-orm'
+import {
+  decodeTimeCursor,
+  encodeTimeCursor,
+  timeCursorOrderBy,
+  timeCursorPredicate,
+  timeCursorStabilityBound,
+} from '@/lib/data-drains/sources/cursor'
+import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
+import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
+
+/**
+ * The transcript no longer lives on `copilot_chats.messages` — it is assembled
+ * per page from the normalized `copilot_messages` table, so `messages` is the
+ * ordered list of message `content` objects rather than the DB column.
+ */
+type CopilotChatRow = Omit<typeof copilotChats.$inferSelect, 'messages'> & {
+  messages: unknown[]
+}
+
+/** Chat metadata columns, excluding the legacy `messages` JSONB. */
+const chatColumns = {
+  id: copilotChats.id,
+  userId: copilotChats.userId,
+  workflowId: copilotChats.workflowId,
+  workspaceId: copilotChats.workspaceId,
+  type: copilotChats.type,
+  title: copilotChats.title,
+  model: copilotChats.model,
+  conversationId: copilotChats.conversationId,
+  previewYaml: copilotChats.previewYaml,
+  planArtifact: copilotChats.planArtifact,
+  config: copilotChats.config,
+  resources: copilotChats.resources,
+  lastSeenAt: copilotChats.lastSeenAt,
+  pinned: copilotChats.pinned,
+  createdAt: copilotChats.createdAt,
+  updatedAt: copilotChats.updatedAt,
+} as const
+
+/**
+ * Cursor is `createdAt` (immutable) but rows themselves are mutable —
+ * `messages`, `title`, `lastSeenAt`, etc. are updated in-place over the chat's
+ * lifetime. This means a chat exported once will not be re-exported when its
+ * messages change. Consumers who need the latest state should periodically
+ * full-refresh from a separate snapshot job; drains are append-mostly by
+ * design and `data-drains` is not a CDC pipeline.
+ */
+async function* pages(input: SourcePageInput): AsyncIterable<CopilotChatRow[]> {
+  const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId)
+  if (workspaceIds.length === 0) return
+
+  let cursor = decodeTimeCursor(input.cursor)
+  while (!input.signal.aborted) {
+    const cursorClause = timeCursorPredicate(copilotChats.createdAt, copilotChats.id, cursor)
+
+    const metaRows = await dbReplica
+      .select(chatColumns)
+      .from(copilotChats)
+      .where(
+        and(
+          inArray(copilotChats.workspaceId, workspaceIds),
+          timeCursorStabilityBound(copilotChats.createdAt),
+          cursorClause
+        )
+      )
+      .orderBy(...timeCursorOrderBy(copilotChats.createdAt, copilotChats.id))
+      .limit(input.chunkSize)
+
+    if (metaRows.length === 0) return
+
+    const chatIds = metaRows.map((r) => r.id)
+    const messageRows = await dbReplica
+      .select({ chatId: copilotMessages.chatId, content: copilotMessages.content })
+      .from(copilotMessages)
+      .where(and(inArray(copilotMessages.chatId, chatIds), isNull(copilotMessages.deletedAt)))
+      .orderBy(
+        asc(copilotMessages.chatId),
+        sql`${copilotMessages.seq} asc nulls last`,
+        asc(copilotMessages.createdAt),
+        asc(copilotMessages.id)
+      )
+    const messagesByChat = new Map<string, unknown[]>()
+    for (const m of messageRows) {
+      const existing = messagesByChat.get(m.chatId)
+      if (existing) existing.push(m.content)
+      else messagesByChat.set(m.chatId, [m.content])
+    }
+
+    const rows: CopilotChatRow[] = metaRows.map((r) => ({
+      ...r,
+      messages: messagesByChat.get(r.id) ?? [],
+    }))
+
+    yield rows
+    const last = metaRows[metaRows.length - 1]
+    cursor = { ts: last.createdAt.toISOString(), id: last.id }
+    if (metaRows.length < input.chunkSize) return
+  }
+}
+
+export const copilotChatsSource: DrainSource<CopilotChatRow> = {
+  type: 'copilot_chats',
+  displayName: 'Chats',
+  pages,
+  serialize(row) {
+    return {
+      id: row.id,
+      userId: row.userId,
+      workflowId: row.workflowId,
+      workspaceId: row.workspaceId,
+      type: row.type,
+      title: row.title,
+      messages: row.messages,
+      model: row.model,
+      conversationId: row.conversationId,
+      previewYaml: row.previewYaml,
+      planArtifact: row.planArtifact,
+      config: row.config,
+      resources: row.resources,
+      lastSeenAt: row.lastSeenAt ? row.lastSeenAt.toISOString() : null,
+      createdAt: row.createdAt.toISOString(),
+      updatedAt: row.updatedAt.toISOString(),
+    }
+  },
+  cursorAfter(row): Cursor {
+    return encodeTimeCursor({ ts: row.createdAt.toISOString(), id: row.id })
+  },
+}

apps/sim/lib/data-drains/sources/copilot-runs.ts

@@ -0,0 +1,79 @@
+import { dbReplica } from '@sim/db'
+import { copilotRuns } from '@sim/db/schema'
+import { and, inArray, isNotNull } from 'drizzle-orm'
+import {
+  decodeTimeCursor,
+  encodeTimeCursor,
+  timeCursorOrderBy,
+  timeCursorPredicate,
+  timeCursorStabilityBound,
+} from '@/lib/data-drains/sources/cursor'
+import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
+import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
+
+type CopilotRunRow = typeof copilotRuns.$inferSelect
+
+/**
+ * Cursors on terminal `completedAt` so in-flight runs (mutable `status`,
+ * `error`, `completedAt`) are not exported until they reach a terminal state.
+ */
+async function* pages(input: SourcePageInput): AsyncIterable<CopilotRunRow[]> {
+  const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId)
+  if (workspaceIds.length === 0) return
+
+  let cursor = decodeTimeCursor(input.cursor)
+  while (!input.signal.aborted) {
+    const cursorClause = timeCursorPredicate(copilotRuns.completedAt, copilotRuns.id, cursor)
+
+    const rows = await dbReplica
+      .select()
+      .from(copilotRuns)
+      .where(
+        and(
+          inArray(copilotRuns.workspaceId, workspaceIds),
+          isNotNull(copilotRuns.completedAt),
+          timeCursorStabilityBound(copilotRuns.completedAt),
+          cursorClause
+        )
+      )
+      .orderBy(...timeCursorOrderBy(copilotRuns.completedAt, copilotRuns.id))
+      .limit(input.chunkSize)
+
+    if (rows.length === 0) return
+    yield rows
+    const last = rows[rows.length - 1]
+    cursor = { ts: last.completedAt!.toISOString(), id: last.id }
+    if (rows.length < input.chunkSize) return
+  }
+}
+
+export const copilotRunsSource: DrainSource<CopilotRunRow> = {
+  type: 'copilot_runs',
+  displayName: 'Chat runs',
+  pages,
+  serialize(row) {
+    return {
+      id: row.id,
+      executionId: row.executionId,
+      parentRunId: row.parentRunId,
+      chatId: row.chatId,
+      userId: row.userId,
+      workflowId: row.workflowId,
+      workspaceId: row.workspaceId,
+      streamId: row.streamId,
+      agent: row.agent,
+      model: row.model,
+      provider: row.provider,
+      status: row.status,
+      requestContext: row.requestContext,
+      startedAt: row.startedAt.toISOString(),
+      completedAt: row.completedAt ? row.completedAt.toISOString() : null,
+      createdAt: row.createdAt.toISOString(),
+      updatedAt: row.updatedAt.toISOString(),
+      error: row.error,
+    }
+  },
+  cursorAfter(row): Cursor {
+    return encodeTimeCursor({ ts: row.completedAt!.toISOString(), id: row.id })
+  },
+}

apps/sim/lib/data-drains/sources/cursor.test.ts

@@ -0,0 +1,26 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { decodeTimeCursor, encodeTimeCursor } from '@/lib/data-drains/sources/cursor'
+
+describe('time cursor encoding', () => {
+  it('round-trips a valid cursor', () => {
+    const value = { ts: '2026-01-01T00:00:00.000Z', id: 'row-1' }
+    expect(decodeTimeCursor(encodeTimeCursor(value))).toEqual(value)
+  })
+
+  it('returns null for null input', () => {
+    expect(decodeTimeCursor(null)).toBeNull()
+  })
+
+  it('returns null for malformed JSON', () => {
+    expect(decodeTimeCursor('not-json')).toBeNull()
+  })
+
+  it('returns null when shape is wrong', () => {
+    expect(decodeTimeCursor(JSON.stringify({ ts: 1, id: 'x' }))).toBeNull()
+    expect(decodeTimeCursor(JSON.stringify({ ts: '2026', id: 5 }))).toBeNull()
+    expect(decodeTimeCursor(JSON.stringify({}))).toBeNull()
+  })
+})

apps/sim/lib/data-drains/sources/cursor.ts

@@ -0,0 +1,67 @@
+import { type SQL, sql } from 'drizzle-orm'
+import type { PgColumn } from 'drizzle-orm/pg-core'
+import type { Cursor } from '@/lib/data-drains/types'
+
+/**
+ * Composite cursor for time-ordered tables. Pairs a timestamp with the row's id
+ * so chunks split across rows that share a timestamp pick up cleanly without
+ * skipping or duplicating.
+ */
+export interface TimeCursor {
+  ts: string
+  id: string
+}
+
+export function encodeTimeCursor(value: TimeCursor): Cursor {
+  return JSON.stringify(value)
+}
+
+export function decodeTimeCursor(cursor: Cursor): TimeCursor | null {
+  if (!cursor) return null
+  try {
+    const parsed = JSON.parse(cursor) as TimeCursor
+    if (typeof parsed?.ts !== 'string' || typeof parsed?.id !== 'string') return null
+    return parsed
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Builds a strict-greater-than predicate over a `(timestampCol, idCol)` pair.
+ *
+ * Postgres `timestamp` columns store microsecond precision but JS `Date`
+ * round-trips at millisecond precision, so the cursor only ever captures
+ * millisecond-truncated timestamps. We compare in millisecond buckets via
+ * `date_trunc('milliseconds', col)` so the predicate's notion of order matches
+ * `timeCursorOrderBy` exactly. If ORDER BY used raw microseconds while the
+ * predicate used millisecond buckets, a row sorted later by µs but with a
+ * lexicographically earlier id than the cursor row would be skipped forever.
+ */
+export function timeCursorPredicate(
+  timestampCol: PgColumn,
+  idCol: PgColumn,
+  cursor: TimeCursor | null
+): SQL | undefined {
+  if (!cursor) return undefined
+  return sql`(date_trunc('milliseconds', ${timestampCol}), ${idCol}) > (${new Date(cursor.ts)}, ${cursor.id})`
+}
+
+/**
+ * ORDER BY fragments paired with `timeCursorPredicate`. Both must agree on
+ * millisecond bucketing so cursor advancement never skips rows.
+ */
+export function timeCursorOrderBy(timestampCol: PgColumn, idCol: PgColumn): [SQL, SQL] {
+  return [sql`date_trunc('milliseconds', ${timestampCol}) asc`, sql`${idCol} asc`]
+}
+
+/**
+ * Excludes rows newer than a short stability window. Timestamp cursors assume
+ * rows become visible in timestamp order, but out-of-order commits and replica
+ * lag can surface an earlier-stamped row after the cursor has advanced past it
+ * — permanently skipping it. Leaving the freshest rows for the next run bounds
+ * both.
+ */
+export function timeCursorStabilityBound(timestampCol: PgColumn): SQL {
+  return sql`${timestampCol} <= now() - interval '5 minutes'`
+}

apps/sim/lib/data-drains/sources/helpers.ts

@@ -0,0 +1,15 @@
+import { db } from '@sim/db'
+import { workspace } from '@sim/db/schema'
+import { eq } from 'drizzle-orm'
+
+/**
+ * Returns the IDs of all workspaces belonging to the organization. Used by
+ * sources whose underlying tables are workspace-scoped rather than org-scoped.
+ */
+export async function getOrganizationWorkspaceIds(organizationId: string): Promise<string[]> {
+  const rows = await db
+    .select({ id: workspace.id })
+    .from(workspace)
+    .where(eq(workspace.organizationId, organizationId))
+  return rows.map((row) => row.id)
+}

apps/sim/lib/data-drains/sources/job-logs.ts

@@ -0,0 +1,74 @@
+import { dbReplica } from '@sim/db'
+import { jobExecutionLogs } from '@sim/db/schema'
+import { and, inArray, isNotNull } from 'drizzle-orm'
+import {
+  decodeTimeCursor,
+  encodeTimeCursor,
+  timeCursorOrderBy,
+  timeCursorPredicate,
+  timeCursorStabilityBound,
+} from '@/lib/data-drains/sources/cursor'
+import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
+import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
+
+type JobLogRow = typeof jobExecutionLogs.$inferSelect
+
+/**
+ * Cursors on terminal `endedAt` so in-flight rows (mutable `status`, `endedAt`,
+ * `totalDurationMs`, `executionData`) are not exported until finalized.
+ */
+async function* pages(input: SourcePageInput): AsyncIterable<JobLogRow[]> {
+  const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId)
+  if (workspaceIds.length === 0) return
+
+  let cursor = decodeTimeCursor(input.cursor)
+  while (!input.signal.aborted) {
+    const cursorClause = timeCursorPredicate(jobExecutionLogs.endedAt, jobExecutionLogs.id, cursor)
+
+    const rows = await dbReplica
+      .select()
+      .from(jobExecutionLogs)
+      .where(
+        and(
+          inArray(jobExecutionLogs.workspaceId, workspaceIds),
+          isNotNull(jobExecutionLogs.endedAt),
+          timeCursorStabilityBound(jobExecutionLogs.endedAt),
+          cursorClause
+        )
+      )
+      .orderBy(...timeCursorOrderBy(jobExecutionLogs.endedAt, jobExecutionLogs.id))
+      .limit(input.chunkSize)
+
+    if (rows.length === 0) return
+    yield rows
+    const last = rows[rows.length - 1]
+    cursor = { ts: last.endedAt!.toISOString(), id: last.id }
+    if (rows.length < input.chunkSize) return
+  }
+}
+
+export const jobLogsSource: DrainSource<JobLogRow> = {
+  type: 'job_logs',
+  displayName: 'Job execution logs',
+  pages,
+  serialize(row) {
+    return {
+      id: row.id,
+      executionId: row.executionId,
+      scheduleId: row.scheduleId,
+      workspaceId: row.workspaceId,
+      level: row.level,
+      status: row.status,
+      trigger: row.trigger,
+      startedAt: row.startedAt.toISOString(),
+      endedAt: row.endedAt ? row.endedAt.toISOString() : null,
+      totalDurationMs: row.totalDurationMs,
+      executionData: row.executionData,
+      cost: row.cost,
+      createdAt: row.createdAt.toISOString(),
+    }
+  },
+  cursorAfter(row): Cursor {
+    return encodeTimeCursor({ ts: row.endedAt!.toISOString(), id: row.id })
+  },
+}

apps/sim/lib/data-drains/sources/registry.ts

@@ -0,0 +1,18 @@
+import { auditLogsSource } from '@/lib/data-drains/sources/audit-logs'
+import { copilotChatsSource } from '@/lib/data-drains/sources/copilot-chats'
+import { copilotRunsSource } from '@/lib/data-drains/sources/copilot-runs'
+import { jobLogsSource } from '@/lib/data-drains/sources/job-logs'
+import { workflowLogsSource } from '@/lib/data-drains/sources/workflow-logs'
+import type { DrainSource, SourceType } from '@/lib/data-drains/types'
+
+export const SOURCE_REGISTRY = {
+  workflow_logs: workflowLogsSource,
+  job_logs: jobLogsSource,
+  audit_logs: auditLogsSource,
+  copilot_chats: copilotChatsSource,
+  copilot_runs: copilotRunsSource,
+} as const satisfies Record<SourceType, DrainSource>
+
+export function getSource(type: SourceType): DrainSource {
+  return SOURCE_REGISTRY[type]
+}

apps/sim/lib/data-drains/sources/workflow-logs.ts

@@ -0,0 +1,103 @@
+import { dbReplica } from '@sim/db'
+import { workflowExecutionLogs } from '@sim/db/schema'
+import { and, inArray, isNotNull } from 'drizzle-orm'
+import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import {
+  decodeTimeCursor,
+  encodeTimeCursor,
+  timeCursorOrderBy,
+  timeCursorPredicate,
+  timeCursorStabilityBound,
+} from '@/lib/data-drains/sources/cursor'
+import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
+import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
+import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
+
+type WorkflowLogRow = typeof workflowExecutionLogs.$inferSelect
+
+/**
+ * Cursors on `endedAt` (terminal timestamp) rather than `startedAt`. A running
+ * row's mutable fields (`endedAt`, `status`, `totalDurationMs`, `executionData`)
+ * would otherwise be exported mid-flight and never re-emitted with their final
+ * values. Filtering on `endedAt IS NOT NULL` guarantees rows are immutable
+ * once visible to the drain.
+ */
+async function* pages(input: SourcePageInput): AsyncIterable<WorkflowLogRow[]> {
+  const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId)
+  if (workspaceIds.length === 0) return
+
+  let cursor = decodeTimeCursor(input.cursor)
+  while (!input.signal.aborted) {
+    const cursorClause = timeCursorPredicate(
+      workflowExecutionLogs.endedAt,
+      workflowExecutionLogs.id,
+      cursor
+    )
+
+    const rows = await dbReplica
+      .select()
+      .from(workflowExecutionLogs)
+      .where(
+        and(
+          inArray(workflowExecutionLogs.workspaceId, workspaceIds),
+          isNotNull(workflowExecutionLogs.endedAt),
+          timeCursorStabilityBound(workflowExecutionLogs.endedAt),
+          cursorClause
+        )
+      )
+      .orderBy(...timeCursorOrderBy(workflowExecutionLogs.endedAt, workflowExecutionLogs.id))
+      .limit(input.chunkSize)
+
+    if (rows.length === 0) return
+
+    // Heavy execution data may live in object storage; resolve pointers (bounded
+    // concurrency) so the drain exports full execution data, not the slim row.
+    // Use the order-preserving returned array (the util's documented contract)
+    // and write back, rather than mutating rows inside the mapper.
+    const materialized = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) =>
+      materializeExecutionData(row.executionData as Record<string, unknown> | null, {
+        workspaceId: row.workspaceId,
+        workflowId: row.workflowId,
+        executionId: row.executionId,
+      })
+    )
+    for (let i = 0; i < rows.length; i++) {
+      rows[i].executionData = materialized[i] as WorkflowLogRow['executionData']
+    }
+
+    yield rows
+    const last = rows[rows.length - 1]
+    cursor = { ts: last.endedAt!.toISOString(), id: last.id }
+    if (rows.length < input.chunkSize) return
+  }
+}
+
+export const workflowLogsSource: DrainSource<WorkflowLogRow> = {
+  type: 'workflow_logs',
+  displayName: 'Workflow execution logs',
+  pages,
+  serialize(row) {
+    return {
+      id: row.id,
+      executionId: row.executionId,
+      workflowId: row.workflowId,
+      workspaceId: row.workspaceId,
+      stateSnapshotId: row.stateSnapshotId,
+      deploymentVersionId: row.deploymentVersionId,
+      level: row.level,
+      status: row.status,
+      trigger: row.trigger,
+      startedAt: row.startedAt.toISOString(),
+      endedAt: row.endedAt ? row.endedAt.toISOString() : null,
+      totalDurationMs: row.totalDurationMs,
+      executionData: row.executionData,
+      // cost_total projection of the usage_log ledger (not the deprecated jsonb).
+      cost: row.costTotal != null ? { total: Number(row.costTotal) } : null,
+      files: row.files,
+      createdAt: row.createdAt.toISOString(),
+    }
+  },
+  cursorAfter(row): Cursor {
+    return encodeTimeCursor({ ts: row.endedAt!.toISOString(), id: row.id })
+  },
+}