chore: import upstream snapshot with attribution

packages/workflow-persistence/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@sim/workflow-persistence",
+  "version": "0.1.0",
+  "private": true,
+  "sideEffects": false,
+  "type": "module",
+  "license": "Apache-2.0",
+  "engines": {
+    "bun": ">=1.2.13",
+    "node": ">=20.0.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    },
+    "./load": {
+      "types": "./src/load.ts",
+      "default": "./src/load.ts"
+    },
+    "./save": {
+      "types": "./src/save.ts",
+      "default": "./src/save.ts"
+    },
+    "./subblocks": {
+      "types": "./src/subblocks.ts",
+      "default": "./src/subblocks.ts"
+    },
+    "./subflow-helpers": {
+      "types": "./src/subflow-helpers.ts",
+      "default": "./src/subflow-helpers.ts"
+    },
+    "./types": {
+      "types": "./src/types.ts",
+      "default": "./src/types.ts"
+    }
+  },
+  "scripts": {
+    "type-check": "tsc --noEmit",
+    "lint": "biome check --write --unsafe .",
+    "lint:check": "biome check .",
+    "format": "biome format --write .",
+    "format:check": "biome format ."
+  },
+  "dependencies": {
+    "@sim/db": "workspace:*",
+    "@sim/logger": "workspace:*",
+    "@sim/utils": "workspace:*",
+    "@sim/workflow-types": "workspace:*",
+    "drizzle-orm": "^0.45.2",
+    "reactflow": "^11.11.4"
+  },
+  "devDependencies": {
+    "@sim/tsconfig": "workspace:*",
+    "@types/node": "24.2.1",
+    "typescript": "^7.0.2"
+  }
+}

packages/workflow-persistence/src/index.ts

@@ -0,0 +1,19 @@
+export {
+  loadWorkflowFromNormalizedTablesRaw,
+  persistMigratedBlocks,
+  type RawNormalizedWorkflow,
+} from './load'
+export { saveWorkflowToNormalizedTables } from './save'
+export {
+  DEFAULT_SUBBLOCK_TYPE,
+  mergeSubBlockValues,
+  mergeSubblockStateWithValues,
+} from './subblocks'
+export {
+  convertLoopBlockToLoop,
+  convertParallelBlockToParallel,
+  findChildNodes,
+  generateLoopBlocks,
+  generateParallelBlocks,
+} from './subflow-helpers'
+export type { DbOrTx, NormalizedWorkflowData } from './types'

packages/workflow-persistence/src/load.ts

@@ -0,0 +1,217 @@
+import { db, workflow, workflowBlocks, workflowEdges, workflowSubflows } from '@sim/db'
+import { createLogger } from '@sim/logger'
+import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow'
+import { SUBFLOW_TYPES } from '@sim/workflow-types/workflow'
+import { and, eq, isNull } from 'drizzle-orm'
+import type { Edge } from 'reactflow'
+import { clampParallelBatchSize } from './subflow-helpers'
+import type { DbOrTx, NormalizedWorkflowData } from './types'
+
+const logger = createLogger('WorkflowPersistenceLoad')
+
+export interface RawNormalizedWorkflow extends NormalizedWorkflowData {
+  workspaceId: string
+  blockUpdatedAtById: Record<string, Date | null>
+}
+
+/**
+ * Load workflow state from normalized tables without running block migrations.
+ * Block migrations (credential rewrites, subblock ID migrations, canonical-mode
+ * backfill, tool sanitization) depend on the block/tool registry that lives in
+ * the Next app and should not be pulled into leaf services. Callers that want
+ * migrated state should wrap this with their own migration pipeline.
+ *
+ * Invariant: downstream migrations must not mutate `block.data.collection`,
+ * `block.data.whileCondition`, or `block.data.doWhileCondition`. Those fields
+ * are patched here from the subflow config on the pre-migration block, and
+ * callers re-sync only `loop.enabled`/`parallel.enabled` from the migrated
+ * block. If a future migration rewrites these data fields, the loop/parallel
+ * config on the returned object will silently diverge from the migrated block.
+ */
+export async function loadWorkflowFromNormalizedTablesRaw(
+  workflowId: string,
+  externalTx?: DbOrTx
+): Promise<RawNormalizedWorkflow | null> {
+  try {
+    const tx = externalTx ?? db
+    const [blocks, edges, subflows, [workflowRow]] = await Promise.all([
+      tx.select().from(workflowBlocks).where(eq(workflowBlocks.workflowId, workflowId)),
+      tx.select().from(workflowEdges).where(eq(workflowEdges.workflowId, workflowId)),
+      tx.select().from(workflowSubflows).where(eq(workflowSubflows.workflowId, workflowId)),
+      tx
+        .select({ workspaceId: workflow.workspaceId })
+        .from(workflow)
+        .where(eq(workflow.id, workflowId))
+        .limit(1),
+    ])
+
+    if (blocks.length === 0) {
+      return null
+    }
+
+    if (!workflowRow?.workspaceId) {
+      throw new Error(`Workflow ${workflowId} has no workspace`)
+    }
+
+    const blocksMap: Record<string, BlockState> = {}
+    const blockUpdatedAtById: Record<string, Date | null> = {}
+    blocks.forEach((block) => {
+      const blockData = (block.data ?? {}) as BlockState['data']
+
+      const assembled: BlockState = {
+        id: block.id,
+        type: block.type,
+        name: block.name,
+        position: {
+          x: Number(block.positionX),
+          y: Number(block.positionY),
+        },
+        enabled: block.enabled,
+        horizontalHandles: block.horizontalHandles,
+        advancedMode: block.advancedMode,
+        triggerMode: block.triggerMode,
+        height: Number(block.height),
+        subBlocks: (block.subBlocks as BlockState['subBlocks']) || {},
+        outputs: (block.outputs as BlockState['outputs']) || {},
+        data: blockData,
+        locked: block.locked,
+      }
+
+      blocksMap[block.id] = assembled
+      blockUpdatedAtById[block.id] = block.updatedAt ?? null
+    })
+
+    const edgesArray: Edge[] = edges.map((edge) => ({
+      id: edge.id,
+      source: edge.sourceBlockId,
+      target: edge.targetBlockId,
+      sourceHandle: edge.sourceHandle ?? undefined,
+      targetHandle: edge.targetHandle ?? undefined,
+      type: 'default',
+      data: {},
+    }))
+
+    const loops: Record<string, Loop> = {}
+    const parallels: Record<string, Parallel> = {}
+
+    subflows.forEach((subflow) => {
+      const config = (subflow.config ?? {}) as Partial<Loop & Parallel>
+
+      if (subflow.type === SUBFLOW_TYPES.LOOP) {
+        const loopType =
+          (config as Loop).loopType === 'for' ||
+          (config as Loop).loopType === 'forEach' ||
+          (config as Loop).loopType === 'while' ||
+          (config as Loop).loopType === 'doWhile'
+            ? (config as Loop).loopType
+            : 'for'
+
+        const loop: Loop = {
+          id: subflow.id,
+          nodes: Array.isArray((config as Loop).nodes) ? (config as Loop).nodes : [],
+          iterations:
+            typeof (config as Loop).iterations === 'number' ? (config as Loop).iterations : 1,
+          loopType,
+          forEachItems: (config as Loop).forEachItems ?? '',
+          whileCondition: (config as Loop).whileCondition ?? '',
+          doWhileCondition: (config as Loop).doWhileCondition ?? '',
+          enabled: blocksMap[subflow.id]?.enabled ?? true,
+        }
+        loops[subflow.id] = loop
+
+        if (blocksMap[subflow.id]) {
+          const block = blocksMap[subflow.id]
+          blocksMap[subflow.id] = {
+            ...block,
+            data: {
+              ...block.data,
+              collection: loop.forEachItems ?? block.data?.collection ?? '',
+              whileCondition: loop.whileCondition ?? block.data?.whileCondition ?? '',
+              doWhileCondition: loop.doWhileCondition ?? block.data?.doWhileCondition ?? '',
+            },
+          }
+        }
+      } else if (subflow.type === SUBFLOW_TYPES.PARALLEL) {
+        const parallel: Parallel = {
+          id: subflow.id,
+          nodes: Array.isArray((config as Parallel).nodes) ? (config as Parallel).nodes : [],
+          count: typeof (config as Parallel).count === 'number' ? (config as Parallel).count : 5,
+          distribution: (config as Parallel).distribution ?? '',
+          parallelType:
+            (config as Parallel).parallelType === 'count' ||
+            (config as Parallel).parallelType === 'collection'
+              ? (config as Parallel).parallelType
+              : 'count',
+          batchSize: clampParallelBatchSize((config as Parallel).batchSize),
+          enabled: blocksMap[subflow.id]?.enabled ?? true,
+        }
+        parallels[subflow.id] = parallel
+
+        if (blocksMap[subflow.id]) {
+          const block = blocksMap[subflow.id]
+          blocksMap[subflow.id] = {
+            ...block,
+            data: {
+              ...block.data,
+              count: parallel.count,
+              collection: parallel.distribution ?? block.data?.collection ?? '',
+              parallelType: parallel.parallelType,
+              batchSize: parallel.batchSize,
+            },
+          }
+        }
+      } else {
+        logger.warn(`Unknown subflow type: ${subflow.type} for subflow ${subflow.id}`)
+      }
+    })
+
+    return {
+      blocks: blocksMap,
+      edges: edgesArray,
+      loops,
+      parallels,
+      isFromNormalizedTables: true,
+      workspaceId: workflowRow.workspaceId,
+      blockUpdatedAtById,
+    }
+  } catch (error) {
+    logger.error(`Error loading workflow ${workflowId} from normalized tables:`, error)
+    return null
+  }
+}
+
+export async function persistMigratedBlocks(
+  workflowId: string,
+  originalBlocks: Record<string, BlockState>,
+  migratedBlocks: Record<string, BlockState>,
+  blockUpdatedAtById: Record<string, Date | null> = {}
+): Promise<void> {
+  try {
+    for (const [blockId, block] of Object.entries(migratedBlocks)) {
+      if (block !== originalBlocks[blockId]) {
+        const hasExpectedUpdatedAt = Object.hasOwn(blockUpdatedAtById, blockId)
+        const expectedUpdatedAt = blockUpdatedAtById[blockId]
+        const whereClause = hasExpectedUpdatedAt
+          ? and(
+              eq(workflowBlocks.id, blockId),
+              eq(workflowBlocks.workflowId, workflowId),
+              expectedUpdatedAt === null
+                ? isNull(workflowBlocks.updatedAt)
+                : eq(workflowBlocks.updatedAt, expectedUpdatedAt)
+            )
+          : and(eq(workflowBlocks.id, blockId), eq(workflowBlocks.workflowId, workflowId))
+
+        await db
+          .update(workflowBlocks)
+          .set({
+            subBlocks: block.subBlocks,
+            data: block.data,
+            updatedAt: new Date(),
+          })
+          .where(whereClause)
+      }
+    }
+  } catch (err) {
+    logger.warn('Failed to persist block migrations', { workflowId, error: err })
+  }
+}

packages/workflow-persistence/src/save.ts

@@ -0,0 +1,108 @@
+import { db, workflowBlocks, workflowEdges, workflowSubflows } from '@sim/db'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import type { BlockState, WorkflowState } from '@sim/workflow-types/workflow'
+import { SUBFLOW_TYPES } from '@sim/workflow-types/workflow'
+import type { InferInsertModel } from 'drizzle-orm'
+import { eq } from 'drizzle-orm'
+import { generateLoopBlocks, generateParallelBlocks } from './subflow-helpers'
+import type { DbOrTx } from './types'
+
+const logger = createLogger('WorkflowPersistenceSave')
+
+type SubflowInsert = InferInsertModel<typeof workflowSubflows>
+
+export async function saveWorkflowToNormalizedTables(
+  workflowId: string,
+  state: WorkflowState,
+  externalTx?: DbOrTx
+): Promise<{ success: boolean; error?: string }> {
+  const blockRecords = state.blocks as Record<string, BlockState>
+  const canonicalLoops = generateLoopBlocks(blockRecords)
+  const canonicalParallels = generateParallelBlocks(blockRecords)
+
+  const execute = async (tx: DbOrTx) => {
+    await Promise.all([
+      tx.delete(workflowBlocks).where(eq(workflowBlocks.workflowId, workflowId)),
+      tx.delete(workflowEdges).where(eq(workflowEdges.workflowId, workflowId)),
+      tx.delete(workflowSubflows).where(eq(workflowSubflows.workflowId, workflowId)),
+    ])
+
+    if (Object.keys(state.blocks).length > 0) {
+      const blockInserts = Object.values(state.blocks).map((block) => ({
+        id: block.id,
+        workflowId,
+        type: block.type,
+        name: block.name || '',
+        positionX: String(block.position?.x || 0),
+        positionY: String(block.position?.y || 0),
+        enabled: block.enabled ?? true,
+        horizontalHandles: block.horizontalHandles ?? true,
+        advancedMode: block.advancedMode ?? false,
+        triggerMode: block.triggerMode ?? false,
+        height: String(block.height || 0),
+        subBlocks: block.subBlocks || {},
+        outputs: block.outputs || {},
+        data: block.data || {},
+        parentId: block.data?.parentId || null,
+        extent: block.data?.extent || null,
+        locked: block.locked ?? false,
+      }))
+
+      await tx.insert(workflowBlocks).values(blockInserts)
+    }
+
+    if (state.edges.length > 0) {
+      const edgeInserts = state.edges.map((edge) => ({
+        id: edge.id,
+        workflowId,
+        sourceBlockId: edge.source,
+        targetBlockId: edge.target,
+        sourceHandle: edge.sourceHandle || null,
+        targetHandle: edge.targetHandle || null,
+      }))
+
+      await tx.insert(workflowEdges).values(edgeInserts)
+    }
+
+    const subflowInserts: SubflowInsert[] = []
+
+    Object.values(canonicalLoops).forEach((loop) => {
+      subflowInserts.push({
+        id: loop.id,
+        workflowId,
+        type: SUBFLOW_TYPES.LOOP,
+        config: loop,
+      })
+    })
+
+    Object.values(canonicalParallels).forEach((parallel) => {
+      subflowInserts.push({
+        id: parallel.id,
+        workflowId,
+        type: SUBFLOW_TYPES.PARALLEL,
+        config: parallel,
+      })
+    })
+
+    if (subflowInserts.length > 0) {
+      await tx.insert(workflowSubflows).values(subflowInserts)
+    }
+  }
+
+  if (externalTx) {
+    await execute(externalTx)
+    return { success: true }
+  }
+
+  try {
+    await db.transaction(execute)
+    return { success: true }
+  } catch (error) {
+    logger.error(`Error saving workflow ${workflowId} to normalized tables:`, error)
+    return {
+      success: false,
+      error: toError(error).message,
+    }
+  }
+}

packages/workflow-persistence/src/subblocks.ts

@@ -0,0 +1,81 @@
+import { filterUndefined } from '@sim/utils/object'
+import type { BlockState, SubBlockState } from '@sim/workflow-types/workflow'
+
+export const DEFAULT_SUBBLOCK_TYPE = 'short-input'
+
+/**
+ * Merges subblock values into the provided subblock structures.
+ * Falls back to a default subblock shape when a value has no structure.
+ * @param subBlocks - Existing subblock definitions from the workflow
+ * @param values - Stored subblock values keyed by subblock id
+ * @returns Merged subblock structures with updated values
+ */
+export function mergeSubBlockValues(
+  subBlocks: Record<string, unknown> | undefined,
+  values: Record<string, unknown> | undefined
+): Record<string, unknown> {
+  const merged = { ...(subBlocks || {}) } as Record<string, any>
+
+  if (!values) return merged
+
+  Object.entries(values).forEach(([subBlockId, value]) => {
+    if (merged[subBlockId] && typeof merged[subBlockId] === 'object') {
+      merged[subBlockId] = {
+        ...(merged[subBlockId] as Record<string, unknown>),
+        value,
+      }
+      return
+    }
+
+    merged[subBlockId] = {
+      id: subBlockId,
+      type: DEFAULT_SUBBLOCK_TYPE,
+      value,
+    }
+  })
+
+  return merged
+}
+
+/**
+ * Merges workflow block states with explicit subblock values while maintaining block structure.
+ * Values that are null or undefined do not override existing subblock values.
+ * @param blocks - Block configurations from workflow state
+ * @param subBlockValues - Subblock values keyed by blockId -> subBlockId -> value
+ * @param blockId - Optional specific block ID to merge (merges all if not provided)
+ * @returns Merged block states with updated subblocks
+ */
+export function mergeSubblockStateWithValues(
+  blocks: Record<string, BlockState>,
+  subBlockValues: Record<string, Record<string, unknown>> = {},
+  blockId?: string
+): Record<string, BlockState> {
+  const blocksToProcess = blockId ? { [blockId]: blocks[blockId] } : blocks
+
+  return Object.entries(blocksToProcess).reduce(
+    (acc, [id, block]) => {
+      if (!block) {
+        return acc
+      }
+
+      const blockSubBlocks = block.subBlocks || {}
+      const blockValues = subBlockValues[id] || {}
+      const filteredValues = Object.fromEntries(
+        Object.entries(filterUndefined(blockValues)).filter(([, value]) => value !== null)
+      )
+
+      const mergedSubBlocks = mergeSubBlockValues(blockSubBlocks, filteredValues) as Record<
+        string,
+        SubBlockState
+      >
+
+      acc[id] = {
+        ...block,
+        subBlocks: mergedSubBlocks,
+      }
+
+      return acc
+    },
+    {} as Record<string, BlockState>
+  )
+}

packages/workflow-persistence/src/subflow-helpers.ts

@@ -0,0 +1,106 @@
+import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow'
+
+const DEFAULT_LOOP_ITERATIONS = 5
+const DEFAULT_PARALLEL_BATCH_SIZE = 20
+const MAX_PARALLEL_BATCH_SIZE = 20
+
+export function clampParallelBatchSize(batchSize: unknown): number {
+  const parsed = typeof batchSize === 'number' ? batchSize : Number.parseInt(String(batchSize), 10)
+  if (Number.isNaN(parsed)) {
+    return DEFAULT_PARALLEL_BATCH_SIZE
+  }
+  return Math.max(1, Math.min(MAX_PARALLEL_BATCH_SIZE, parsed))
+}
+
+export function findChildNodes(containerId: string, blocks: Record<string, BlockState>): string[] {
+  return Object.values(blocks)
+    .filter((block) => block.data?.parentId === containerId)
+    .map((block) => block.id)
+}
+
+export function convertLoopBlockToLoop(
+  loopBlockId: string,
+  blocks: Record<string, BlockState>
+): Loop | undefined {
+  const loopBlock = blocks[loopBlockId]
+  if (!loopBlock || loopBlock.type !== 'loop') return undefined
+
+  const loopType = loopBlock.data?.loopType || 'for'
+
+  const loop: Loop = {
+    id: loopBlockId,
+    nodes: findChildNodes(loopBlockId, blocks),
+    iterations: loopBlock.data?.count || DEFAULT_LOOP_ITERATIONS,
+    loopType,
+    enabled: loopBlock.enabled,
+  }
+
+  loop.forEachItems = loopBlock.data?.collection || ''
+  loop.whileCondition = loopBlock.data?.whileCondition || ''
+  loop.doWhileCondition = loopBlock.data?.doWhileCondition || ''
+
+  return loop
+}
+
+export function convertParallelBlockToParallel(
+  parallelBlockId: string,
+  blocks: Record<string, BlockState>
+): Parallel | undefined {
+  const parallelBlock = blocks[parallelBlockId]
+  if (!parallelBlock || parallelBlock.type !== 'parallel') return undefined
+
+  const parallelType = parallelBlock.data?.parallelType || 'count'
+
+  const validParallelTypes = ['collection', 'count'] as const
+  const validatedParallelType = validParallelTypes.includes(parallelType as any)
+    ? parallelType
+    : 'collection'
+
+  const distribution =
+    validatedParallelType === 'collection' ? parallelBlock.data?.collection || '' : undefined
+
+  const count = parallelBlock.data?.count || 5
+  const batchSize = clampParallelBatchSize(parallelBlock.data?.batchSize)
+
+  return {
+    id: parallelBlockId,
+    nodes: findChildNodes(parallelBlockId, blocks),
+    distribution,
+    count,
+    parallelType: validatedParallelType,
+    batchSize,
+    enabled: parallelBlock.enabled,
+  }
+}
+
+export function generateLoopBlocks(blocks: Record<string, BlockState>): Record<string, Loop> {
+  const loops: Record<string, Loop> = {}
+
+  Object.entries(blocks)
+    .filter(([_, block]) => block.type === 'loop')
+    .forEach(([id, block]) => {
+      const loop = convertLoopBlockToLoop(id, blocks)
+      if (loop) {
+        loops[id] = loop
+      }
+    })
+
+  return loops
+}
+
+export function generateParallelBlocks(
+  blocks: Record<string, BlockState>
+): Record<string, Parallel> {
+  const parallels: Record<string, Parallel> = {}
+
+  Object.entries(blocks)
+    .filter(([_, block]) => block.type === 'parallel')
+    .forEach(([id, block]) => {
+      const parallel = convertParallelBlockToParallel(id, blocks)
+      if (parallel) {
+        parallels[id] = parallel
+      }
+    })
+
+  return parallels
+}

packages/workflow-persistence/src/types.ts

@@ -0,0 +1,23 @@
+import type { db } from '@sim/db'
+import type * as schema from '@sim/db/schema'
+import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow'
+import type { ExtractTablesWithRelations } from 'drizzle-orm'
+import type { PgTransaction } from 'drizzle-orm/pg-core'
+import type { PostgresJsQueryResultHKT } from 'drizzle-orm/postgres-js'
+import type { Edge } from 'reactflow'
+
+export type DbOrTx =
+  | typeof db
+  | PgTransaction<
+      PostgresJsQueryResultHKT,
+      typeof schema,
+      ExtractTablesWithRelations<typeof schema>
+    >
+
+export interface NormalizedWorkflowData {
+  blocks: Record<string, BlockState>
+  edges: Edge[]
+  loops: Record<string, Loop>
+  parallels: Record<string, Parallel>
+  isFromNormalizedTables: boolean
+}

packages/workflow-persistence/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "@sim/tsconfig/library.json",
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}