chore: import upstream snapshot with attribution

apps/sim/stores/operation-queue/store.test.ts

@@ -0,0 +1,488 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { registerEmitFunctions, useOperationQueueStore } from '@/stores/operation-queue/store'
+
+describe('operation queue room gating', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    useOperationQueueStore.setState({
+      operations: [],
+      workflowOperationVersions: {},
+      isProcessing: false,
+      hasOperationError: false,
+    })
+    registerEmitFunctions(vi.fn(), vi.fn(), vi.fn(), null)
+  })
+
+  afterEach(() => {
+    useOperationQueueStore.setState({
+      operations: [],
+      workflowOperationVersions: {},
+      isProcessing: false,
+      hasOperationError: false,
+    })
+    registerEmitFunctions(vi.fn(), vi.fn(), vi.fn(), null)
+  })
+
+  it('does not process workflow operations while no workflow is registered', () => {
+    const workflowEmit = vi.fn()
+    registerEmitFunctions(workflowEmit, vi.fn(), vi.fn(), null)
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    expect(workflowEmit).not.toHaveBeenCalled()
+  })
+
+  it('waits until the matching workflow is registered before emitting', () => {
+    const workflowEmit = vi.fn()
+    registerEmitFunctions(workflowEmit, vi.fn(), vi.fn(), null)
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    registerEmitFunctions(workflowEmit, vi.fn(), vi.fn(), 'workflow-b')
+    expect(workflowEmit).not.toHaveBeenCalled()
+
+    registerEmitFunctions(workflowEmit, vi.fn(), vi.fn(), 'workflow-a')
+    expect(workflowEmit).toHaveBeenCalledWith(
+      'workflow-a',
+      'replace-state',
+      'workflow',
+      { state: {} },
+      'op-1'
+    )
+
+    useOperationQueueStore.getState().confirmOperation('op-1')
+  })
+
+  it('reverts the operation to pending without retrying when the emit is skipped', () => {
+    const skippingEmit = vi.fn(() => false)
+    registerEmitFunctions(skippingEmit, vi.fn(), vi.fn(), 'workflow-a')
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    expect(skippingEmit).toHaveBeenCalledTimes(1)
+
+    const state = useOperationQueueStore.getState()
+    expect(state.isProcessing).toBe(false)
+    expect(state.hasOperationError).toBe(false)
+    expect(state.operations).toEqual([
+      expect.objectContaining({ id: 'op-1', status: 'pending', retryCount: 0 }),
+    ])
+  })
+
+  it('emits a previously skipped operation once the room becomes joinable', () => {
+    const skippingEmit = vi.fn(() => false)
+    registerEmitFunctions(skippingEmit, vi.fn(), vi.fn(), 'workflow-a')
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    expect(skippingEmit).toHaveBeenCalledTimes(1)
+
+    const sendingEmit = vi.fn(() => true)
+    registerEmitFunctions(sendingEmit, vi.fn(), vi.fn(), 'workflow-a')
+
+    expect(sendingEmit).toHaveBeenCalledWith(
+      'workflow-a',
+      'replace-state',
+      'workflow',
+      { state: {} },
+      'op-1'
+    )
+    expect(useOperationQueueStore.getState().operations).toEqual([
+      expect.objectContaining({ id: 'op-1', status: 'processing' }),
+    ])
+
+    useOperationQueueStore.getState().confirmOperation('op-1')
+  })
+
+  it('triggers offline mode for a non-retryable failure and recovers via clearError', () => {
+    registerEmitFunctions(
+      vi.fn(() => true),
+      vi.fn(),
+      vi.fn(),
+      'workflow-a'
+    )
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    useOperationQueueStore.getState().failOperation('op-1', false)
+
+    expect(useOperationQueueStore.getState().hasOperationError).toBe(true)
+    expect(useOperationQueueStore.getState().operations).toEqual([])
+
+    useOperationQueueStore.getState().clearError()
+
+    expect(useOperationQueueStore.getState().hasOperationError).toBe(false)
+  })
+
+  it('triggers offline mode once retries exhaust for retryable failures', () => {
+    registerEmitFunctions(
+      vi.fn(() => true),
+      vi.fn(),
+      vi.fn(),
+      'workflow-a'
+    )
+
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    useOperationQueueStore.getState().failOperation('op-1', true)
+    useOperationQueueStore.getState().failOperation('op-1', true)
+    useOperationQueueStore.getState().failOperation('op-1', true)
+    expect(useOperationQueueStore.getState().hasOperationError).toBe(false)
+
+    useOperationQueueStore.getState().failOperation('op-1', true)
+
+    expect(useOperationQueueStore.getState().hasOperationError).toBe(true)
+    expect(useOperationQueueStore.getState().operations).toEqual([])
+  })
+
+  it('reports pending operations per workflow', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().hasPendingOperations('workflow-a')).toBe(true)
+    expect(useOperationQueueStore.getState().hasPendingOperations('workflow-b')).toBe(false)
+  })
+
+  it('tracks local operation activity per workflow', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().workflowOperationVersions['workflow-a']).toBe(1)
+    expect(
+      useOperationQueueStore.getState().workflowOperationVersions['workflow-b']
+    ).toBeUndefined()
+  })
+
+  it('coalesces pending subblock updates to the latest value for the same field', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'subblock-update',
+        target: 'subblock',
+        payload: {
+          blockId: 'block-1',
+          subblockId: 'prompt',
+          value: 'old value',
+        },
+      },
+    })
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-2',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'subblock-update',
+        target: 'subblock',
+        payload: {
+          blockId: 'block-1',
+          subblockId: 'prompt',
+          value: 'new value',
+        },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().operations).toEqual([
+      expect.objectContaining({
+        id: 'op-2',
+        operation: expect.objectContaining({
+          payload: expect.objectContaining({ value: 'new value' }),
+        }),
+      }),
+    ])
+  })
+
+  it('does not coalesce matching subblock updates across workflows', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'subblock-update',
+        target: 'subblock',
+        payload: {
+          blockId: 'block-1',
+          subblockId: 'prompt',
+          value: 'workflow a value',
+        },
+      },
+    })
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-2',
+      workflowId: 'workflow-b',
+      userId: 'user-1',
+      operation: {
+        operation: 'subblock-update',
+        target: 'subblock',
+        payload: {
+          blockId: 'block-1',
+          subblockId: 'prompt',
+          value: 'workflow b value',
+        },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().operations).toEqual([
+      expect.objectContaining({
+        id: 'op-1',
+        workflowId: 'workflow-a',
+      }),
+      expect.objectContaining({
+        id: 'op-2',
+        workflowId: 'workflow-b',
+      }),
+    ])
+  })
+
+  it('coalesces variable field updates without dropping unrelated fields', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'variable-update',
+        target: 'variable',
+        payload: {
+          variableId: 'variable-1',
+          field: 'value',
+          value: 'old value',
+        },
+      },
+    })
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-2',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'variable-update',
+        target: 'variable',
+        payload: {
+          variableId: 'variable-1',
+          field: 'name',
+          value: 'Variable Name',
+        },
+      },
+    })
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-3',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'variable-update',
+        target: 'variable',
+        payload: {
+          variableId: 'variable-1',
+          field: 'value',
+          value: 'new value',
+        },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().operations).toEqual([
+      expect.objectContaining({
+        id: 'op-2',
+        operation: expect.objectContaining({
+          payload: expect.objectContaining({ field: 'name', value: 'Variable Name' }),
+        }),
+      }),
+      expect.objectContaining({
+        id: 'op-3',
+        operation: expect.objectContaining({
+          payload: expect.objectContaining({ field: 'value', value: 'new value' }),
+        }),
+      }),
+    ])
+  })
+
+  it('does not coalesce matching variable updates across workflows', () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'variable-update',
+        target: 'variable',
+        payload: {
+          variableId: 'variable-1',
+          field: 'value',
+          value: 'workflow a value',
+        },
+      },
+    })
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-2',
+      workflowId: 'workflow-b',
+      userId: 'user-1',
+      operation: {
+        operation: 'variable-update',
+        target: 'variable',
+        payload: {
+          variableId: 'variable-1',
+          field: 'value',
+          value: 'workflow b value',
+        },
+      },
+    })
+
+    expect(useOperationQueueStore.getState().operations).toEqual([
+      expect.objectContaining({
+        id: 'op-1',
+        workflowId: 'workflow-a',
+      }),
+      expect.objectContaining({
+        id: 'op-2',
+        workflowId: 'workflow-b',
+      }),
+    ])
+  })
+
+  it('waits for matching workflow operations to drain', async () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    const drained = useOperationQueueStore.getState().waitForWorkflowOperations('workflow-a')
+    useOperationQueueStore.getState().confirmOperation('op-1')
+
+    await expect(drained).resolves.toBe(true)
+  })
+
+  it('does not wait on operations from other workflows', async () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    await expect(
+      useOperationQueueStore.getState().waitForWorkflowOperations('workflow-b')
+    ).resolves.toBe(true)
+  })
+
+  it('stops waiting when an operation error is reported', async () => {
+    useOperationQueueStore.getState().addToQueue({
+      id: 'op-1',
+      workflowId: 'workflow-a',
+      userId: 'user-1',
+      operation: {
+        operation: 'replace-state',
+        target: 'workflow',
+        payload: { state: {} },
+      },
+    })
+
+    const drained = useOperationQueueStore.getState().waitForWorkflowOperations('workflow-a')
+    useOperationQueueStore.setState({ hasOperationError: true })
+
+    await expect(drained).resolves.toBe(false)
+  })
+
+  it('stops waiting when matching workflow operations do not drain before timeout', async () => {
+    vi.useFakeTimers()
+    try {
+      useOperationQueueStore.getState().addToQueue({
+        id: 'op-1',
+        workflowId: 'workflow-a',
+        userId: 'user-1',
+        operation: {
+          operation: 'replace-state',
+          target: 'workflow',
+          payload: { state: {} },
+        },
+      })
+
+      const drained = useOperationQueueStore.getState().waitForWorkflowOperations('workflow-a', 100)
+      await vi.advanceTimersByTimeAsync(100)
+
+      await expect(drained).resolves.toBe(false)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

apps/sim/stores/operation-queue/store.ts

@@ -0,0 +1,679 @@
+import { createLogger } from '@sim/logger'
+import { create } from 'zustand'
+import type {
+  OperationQueueState,
+  QueuedOperation,
+  SubblockUpdateEmit,
+  VariableUpdateEmit,
+  WorkflowOperationEmit,
+} from './types'
+
+function isBlockStillPresent(blockId: string | undefined): boolean {
+  if (!blockId) return true
+  try {
+    const { useWorkflowStore } = require('@/stores/workflows/workflow/store')
+    return Boolean(useWorkflowStore.getState().blocks[blockId])
+  } catch {
+    return true
+  }
+}
+
+function isVariableStillPresent(variableId: string | undefined): boolean {
+  if (!variableId) return true
+  try {
+    const { useVariablesStore } = require('@/stores/variables/store')
+    return Boolean(useVariablesStore.getState().variables[variableId])
+  } catch {
+    return true
+  }
+}
+
+const logger = createLogger('OperationQueue')
+
+/** Timeout for subblock/variable operations before considering them failed */
+const SUBBLOCK_VARIABLE_TIMEOUT_MS = 15000
+/** Timeout for structural operations before considering them failed */
+const STRUCTURAL_TIMEOUT_MS = 5000
+/** Maximum retry attempts for subblock/variable operations */
+const SUBBLOCK_VARIABLE_MAX_RETRIES = 5
+/** Maximum retry attempts for structural operations */
+const STRUCTURAL_MAX_RETRIES = 3
+/** Maximum retry delay cap for subblock/variable operations */
+const SUBBLOCK_VARIABLE_MAX_RETRY_DELAY_MS = 3000
+/** Base retry delay multiplier (1s, 2s, 3s for linear) */
+const RETRY_DELAY_BASE_MS = 1000
+
+const retryTimeouts = new Map<string, NodeJS.Timeout>()
+const operationTimeouts = new Map<string, NodeJS.Timeout>()
+const DEFAULT_WORKFLOW_DRAIN_TIMEOUT_MS = 20000
+
+let emitWorkflowOperation: WorkflowOperationEmit | null = null
+let emitSubblockUpdate: SubblockUpdateEmit | null = null
+let emitVariableUpdate: VariableUpdateEmit | null = null
+
+export function registerEmitFunctions(
+  workflowEmit: WorkflowOperationEmit,
+  subblockEmit: SubblockUpdateEmit,
+  variableEmit: VariableUpdateEmit,
+  workflowId: string | null
+) {
+  emitWorkflowOperation = workflowEmit
+  emitSubblockUpdate = subblockEmit
+  emitVariableUpdate = variableEmit
+  currentRegisteredWorkflowId = workflowId
+  if (workflowId) {
+    useOperationQueueStore.getState().processNextOperation()
+  }
+}
+
+let currentRegisteredWorkflowId: string | null = null
+
+/** Targets whose payload id refers to a canvas block (subflow ids are loop/parallel blocks). */
+const BLOCK_SCOPED_TARGETS = ['block', 'subblock', 'subflow']
+
+/**
+ * Drops a failed operation whose target entity no longer exists locally (e.g. it
+ * was removed by a remote collaborator while the operation was in flight), so a
+ * stale per-entity failure does not trip offline mode. Applies only to block-,
+ * subblock-, subflow-, and variable-scoped operations; structural operations
+ * (workflow/blocks/edges) have no single target entity and always fall through
+ * to offline mode. Returns true when the operation was dropped.
+ */
+function dropOperationForMissingTarget(operation: QueuedOperation): boolean {
+  const { target, payload } = operation.operation
+
+  const isVariableOperation = target === 'variable'
+  if (!isVariableOperation && !BLOCK_SCOPED_TARGETS.includes(target)) {
+    return false
+  }
+
+  const targetId = isVariableOperation
+    ? payload?.variableId || payload?.id
+    : payload?.blockId || payload?.id
+  const targetStillPresent = isVariableOperation
+    ? isVariableStillPresent(targetId)
+    : isBlockStillPresent(targetId)
+
+  if (!targetId || targetStillPresent) {
+    return false
+  }
+
+  logger.debug(
+    isVariableOperation
+      ? 'Dropping failed operation for deleted variable'
+      : 'Dropping failed operation for deleted block',
+    {
+      operationId: operation.id,
+      targetId,
+    }
+  )
+  useOperationQueueStore.setState((s) => ({
+    operations: s.operations.filter((op) => op.id !== operation.id),
+    isProcessing: false,
+  }))
+  useOperationQueueStore.getState().processNextOperation()
+  return true
+}
+
+export const useOperationQueueStore = create<OperationQueueState>((set, get) => ({
+  operations: [],
+  workflowOperationVersions: {},
+  isProcessing: false,
+  hasOperationError: false,
+
+  addToQueue: (operation) => {
+    set((state) => ({
+      workflowOperationVersions: {
+        ...state.workflowOperationVersions,
+        [operation.workflowId]: (state.workflowOperationVersions[operation.workflowId] ?? 0) + 1,
+      },
+    }))
+
+    let shouldDropPendingOperation = (_op: QueuedOperation) => false
+
+    if (
+      operation.operation.operation === 'subblock-update' &&
+      operation.operation.target === 'subblock'
+    ) {
+      const { blockId, subblockId } = operation.operation.payload
+      shouldDropPendingOperation = (op) =>
+        op.status === 'pending' &&
+        op.workflowId === operation.workflowId &&
+        op.operation.operation === 'subblock-update' &&
+        op.operation.target === 'subblock' &&
+        op.operation.payload?.blockId === blockId &&
+        op.operation.payload?.subblockId === subblockId
+    }
+
+    if (
+      operation.operation.operation === 'variable-update' &&
+      operation.operation.target === 'variable'
+    ) {
+      const { variableId, field } = operation.operation.payload
+      shouldDropPendingOperation = (op) =>
+        op.status === 'pending' &&
+        op.workflowId === operation.workflowId &&
+        op.operation.operation === 'variable-update' &&
+        op.operation.target === 'variable' &&
+        op.operation.payload?.variableId === variableId &&
+        op.operation.payload?.field === field
+    }
+
+    const state = get()
+
+    const existingOp = state.operations.find((op) => op.id === operation.id)
+    if (existingOp) {
+      logger.debug('Skipping duplicate operation ID', {
+        operationId: operation.id,
+        existingStatus: existingOp.status,
+      })
+      return
+    }
+
+    const duplicateContent = state.operations.find(
+      (op) =>
+        !shouldDropPendingOperation(op) &&
+        op.operation.operation === operation.operation.operation &&
+        op.operation.target === operation.operation.target &&
+        op.workflowId === operation.workflowId &&
+        ((operation.operation.target === 'block' &&
+          op.operation.payload?.id === operation.operation.payload?.id) ||
+          (operation.operation.target !== 'block' &&
+            JSON.stringify(op.operation.payload) === JSON.stringify(operation.operation.payload)))
+    )
+
+    const isReplaceStateWorkflowOp =
+      operation.operation.target === 'workflow' && operation.operation.operation === 'replace-state'
+
+    if (duplicateContent && !isReplaceStateWorkflowOp) {
+      logger.debug('Skipping duplicate operation content', {
+        operationId: operation.id,
+        existingOperationId: duplicateContent.id,
+        operation: operation.operation.operation,
+        target: operation.operation.target,
+        existingStatus: duplicateContent.status,
+        payload:
+          operation.operation.target === 'block'
+            ? { id: operation.operation.payload?.id }
+            : operation.operation.payload,
+      })
+      return
+    }
+
+    const queuedOp: QueuedOperation = {
+      ...operation,
+      timestamp: Date.now(),
+      retryCount: 0,
+      status: 'pending',
+    }
+
+    logger.debug('Adding operation to queue', {
+      operationId: queuedOp.id,
+      operation: queuedOp.operation,
+    })
+
+    set((state) => ({
+      operations: [...state.operations.filter((op) => !shouldDropPendingOperation(op)), queuedOp],
+    }))
+
+    get().processNextOperation()
+  },
+
+  confirmOperation: (operationId) => {
+    const state = get()
+    const operation = state.operations.find((op) => op.id === operationId)
+    const newOperations = state.operations.filter((op) => op.id !== operationId)
+
+    const retryTimeout = retryTimeouts.get(operationId)
+    if (retryTimeout) {
+      clearTimeout(retryTimeout)
+      retryTimeouts.delete(operationId)
+    }
+
+    const operationTimeout = operationTimeouts.get(operationId)
+    if (operationTimeout) {
+      clearTimeout(operationTimeout)
+      operationTimeouts.delete(operationId)
+    }
+
+    logger.debug('Removing operation from queue', {
+      operationId,
+      remainingOps: newOperations.length,
+    })
+
+    set({ operations: newOperations, isProcessing: false })
+
+    get().processNextOperation()
+  },
+
+  failOperation: (operationId: string, retryable = true) => {
+    const state = get()
+    const operation = state.operations.find((op) => op.id === operationId)
+    if (!operation) {
+      logger.warn('Attempted to fail operation that does not exist in queue', { operationId })
+      return
+    }
+
+    const operationTimeout = operationTimeouts.get(operationId)
+    if (operationTimeout) {
+      clearTimeout(operationTimeout)
+      operationTimeouts.delete(operationId)
+    }
+
+    if (!retryable) {
+      if (dropOperationForMissingTarget(operation)) {
+        return
+      }
+
+      logger.error(
+        'Operation failed with non-retryable error - state out of sync, triggering offline mode',
+        {
+          operationId,
+          operation: operation.operation.operation,
+          target: operation.operation.target,
+        }
+      )
+
+      get().triggerOfflineMode()
+      return
+    }
+
+    const isSubblockOrVariable =
+      (operation.operation.operation === 'subblock-update' &&
+        operation.operation.target === 'subblock') ||
+      (operation.operation.operation === 'variable-update' &&
+        operation.operation.target === 'variable')
+
+    const maxRetries = isSubblockOrVariable ? SUBBLOCK_VARIABLE_MAX_RETRIES : STRUCTURAL_MAX_RETRIES
+
+    if (operation.retryCount < maxRetries) {
+      const newRetryCount = operation.retryCount + 1
+      // Faster retries for subblock/variable, exponential for structural
+      const delay = isSubblockOrVariable
+        ? Math.min(RETRY_DELAY_BASE_MS * newRetryCount, SUBBLOCK_VARIABLE_MAX_RETRY_DELAY_MS)
+        : 2 ** newRetryCount * RETRY_DELAY_BASE_MS
+
+      logger.warn(
+        `Operation failed, retrying in ${delay}ms (attempt ${newRetryCount}/${maxRetries})`,
+        {
+          operationId,
+          retryCount: newRetryCount,
+          operation: operation.operation.operation,
+        }
+      )
+
+      set((state) => ({
+        operations: state.operations.map((op) =>
+          op.id === operationId
+            ? { ...op, retryCount: newRetryCount, status: 'pending' as const }
+            : op
+        ),
+        isProcessing: false,
+      }))
+
+      const timeout = setTimeout(() => {
+        retryTimeouts.delete(operationId)
+        get().processNextOperation()
+      }, delay)
+
+      retryTimeouts.set(operationId, timeout)
+    } else {
+      if (dropOperationForMissingTarget(operation)) {
+        return
+      }
+
+      logger.error('Operation failed after max retries, triggering offline mode', {
+        operationId,
+        operation: operation.operation.operation,
+        retryCount: operation.retryCount,
+      })
+      get().triggerOfflineMode()
+    }
+  },
+
+  handleOperationTimeout: (operationId: string) => {
+    const state = get()
+    const operation = state.operations.find((op) => op.id === operationId)
+    if (!operation) {
+      logger.debug('Ignoring timeout for operation not in queue', { operationId })
+      return
+    }
+
+    logger.warn('Operation timeout detected - treating as failure to trigger retries', {
+      operationId,
+    })
+
+    get().failOperation(operationId)
+  },
+
+  processNextOperation: () => {
+    const state = get()
+
+    if (state.isProcessing) {
+      return
+    }
+
+    if (!currentRegisteredWorkflowId) {
+      return
+    }
+
+    const nextOperation = state.operations.find(
+      (op) => op.status === 'pending' && op.workflowId === currentRegisteredWorkflowId
+    )
+    if (!nextOperation) {
+      return
+    }
+
+    set((state) => ({
+      operations: state.operations.map((op) =>
+        op.id === nextOperation.id ? { ...op, status: 'processing' as const } : op
+      ),
+      isProcessing: true,
+    }))
+
+    logger.debug('Processing operation sequentially', {
+      operationId: nextOperation.id,
+      operation: nextOperation.operation,
+      retryCount: nextOperation.retryCount,
+    })
+
+    const { operation: op, target, payload } = nextOperation.operation
+    let emitted = false
+    if (op === 'subblock-update' && target === 'subblock') {
+      if (emitSubblockUpdate) {
+        emitted = emitSubblockUpdate(
+          payload.blockId,
+          payload.subblockId,
+          payload.value,
+          nextOperation.id,
+          nextOperation.workflowId
+        )
+      }
+    } else if (op === 'variable-update' && target === 'variable') {
+      if (emitVariableUpdate) {
+        emitted = emitVariableUpdate(
+          payload.variableId,
+          payload.field,
+          payload.value,
+          nextOperation.id,
+          nextOperation.workflowId
+        )
+      }
+    } else {
+      if (emitWorkflowOperation) {
+        emitted = emitWorkflowOperation(
+          nextOperation.workflowId,
+          op,
+          target,
+          payload,
+          nextOperation.id
+        )
+      }
+    }
+
+    if (!emitted) {
+      logger.debug('Emit skipped for operation - leaving it pending until the room is joinable', {
+        operationId: nextOperation.id,
+        operation: nextOperation.operation.operation,
+        workflowId: nextOperation.workflowId,
+      })
+      set((state) => ({
+        operations: state.operations.map((o) =>
+          o.id === nextOperation.id ? { ...o, status: 'pending' as const } : o
+        ),
+        isProcessing: false,
+      }))
+      return
+    }
+
+    const isSubblockOrVariable =
+      (nextOperation.operation.operation === 'subblock-update' &&
+        nextOperation.operation.target === 'subblock') ||
+      (nextOperation.operation.operation === 'variable-update' &&
+        nextOperation.operation.target === 'variable')
+    const timeoutDuration = isSubblockOrVariable
+      ? SUBBLOCK_VARIABLE_TIMEOUT_MS
+      : STRUCTURAL_TIMEOUT_MS
+
+    const timeoutId = setTimeout(() => {
+      logger.warn(`Operation timeout - no server response after ${timeoutDuration}ms`, {
+        operationId: nextOperation.id,
+        operation: nextOperation.operation.operation,
+      })
+      operationTimeouts.delete(nextOperation.id)
+      get().handleOperationTimeout(nextOperation.id)
+    }, timeoutDuration)
+
+    operationTimeouts.set(nextOperation.id, timeoutId)
+  },
+
+  hasPendingOperations: (workflowId: string) => {
+    return get().operations.some((op) => op.workflowId === workflowId)
+  },
+
+  waitForWorkflowOperations: (
+    workflowId: string,
+    timeoutMs = DEFAULT_WORKFLOW_DRAIN_TIMEOUT_MS
+  ) => {
+    if (!get().hasPendingOperations(workflowId)) {
+      return Promise.resolve(true)
+    }
+
+    return new Promise((resolve) => {
+      let unsubscribe = () => {}
+      const timeout = setTimeout(() => {
+        unsubscribe()
+        resolve(false)
+      }, timeoutMs)
+
+      unsubscribe = useOperationQueueStore.subscribe((state) => {
+        if (state.hasOperationError) {
+          clearTimeout(timeout)
+          unsubscribe()
+          resolve(false)
+          return
+        }
+
+        if (!state.operations.some((op) => op.workflowId === workflowId)) {
+          clearTimeout(timeout)
+          unsubscribe()
+          resolve(true)
+        }
+      })
+    })
+  },
+
+  cancelOperationsForBlock: (blockId: string) => {
+    logger.debug('Canceling all operations for block', { blockId })
+
+    const state = get()
+    const operationsToCancel = state.operations.filter((op) => {
+      const { target, payload, operation } = op.operation
+
+      if (target === 'block' && payload?.id === blockId) return true
+
+      if (target === 'subblock' && payload?.blockId === blockId) return true
+
+      if (target === 'blocks') {
+        if (operation === 'batch-add-blocks' && Array.isArray(payload?.blocks)) {
+          return payload.blocks.some((b: { id: string }) => b.id === blockId)
+        }
+        if (operation === 'batch-remove-blocks' && Array.isArray(payload?.ids)) {
+          return payload.ids.includes(blockId)
+        }
+        if (operation === 'batch-update-positions' && Array.isArray(payload?.updates)) {
+          return payload.updates.some((u: { id: string }) => u.id === blockId)
+        }
+      }
+
+      return false
+    })
+
+    operationsToCancel.forEach((op) => {
+      const operationTimeout = operationTimeouts.get(op.id)
+      if (operationTimeout) {
+        clearTimeout(operationTimeout)
+        operationTimeouts.delete(op.id)
+      }
+
+      const retryTimeout = retryTimeouts.get(op.id)
+      if (retryTimeout) {
+        clearTimeout(retryTimeout)
+        retryTimeouts.delete(op.id)
+      }
+    })
+
+    const newOperations = state.operations.filter((op) => {
+      const { target, payload, operation } = op.operation
+
+      if (target === 'block' && payload?.id === blockId) return false
+
+      if (target === 'subblock' && payload?.blockId === blockId) return false
+
+      if (target === 'blocks') {
+        if (operation === 'batch-add-blocks' && Array.isArray(payload?.blocks)) {
+          if (payload.blocks.some((b: { id: string }) => b.id === blockId)) return false
+        }
+        if (operation === 'batch-remove-blocks' && Array.isArray(payload?.ids)) {
+          if (payload.ids.includes(blockId)) return false
+        }
+        if (operation === 'batch-update-positions' && Array.isArray(payload?.updates)) {
+          if (payload.updates.some((u: { id: string }) => u.id === blockId)) return false
+        }
+      }
+
+      return true
+    })
+
+    set({
+      operations: newOperations,
+      isProcessing: false,
+    })
+
+    logger.debug('Cancelled operations for block', {
+      blockId,
+      cancelledOperations: operationsToCancel.length,
+    })
+
+    get().processNextOperation()
+  },
+
+  cancelOperationsForVariable: (variableId: string) => {
+    logger.debug('Canceling all operations for variable', { variableId })
+
+    const state = get()
+    const operationsToCancel = state.operations.filter(
+      (op) =>
+        (op.operation.target === 'variable' && op.operation.payload?.variableId === variableId) ||
+        (op.operation.target === 'variable' &&
+          op.operation.payload?.sourceVariableId === variableId)
+    )
+
+    operationsToCancel.forEach((op) => {
+      const operationTimeout = operationTimeouts.get(op.id)
+      if (operationTimeout) {
+        clearTimeout(operationTimeout)
+        operationTimeouts.delete(op.id)
+      }
+
+      const retryTimeout = retryTimeouts.get(op.id)
+      if (retryTimeout) {
+        clearTimeout(retryTimeout)
+        retryTimeouts.delete(op.id)
+      }
+    })
+
+    const newOperations = state.operations.filter(
+      (op) =>
+        !(
+          (op.operation.target === 'variable' && op.operation.payload?.variableId === variableId) ||
+          (op.operation.target === 'variable' &&
+            op.operation.payload?.sourceVariableId === variableId)
+        )
+    )
+
+    set({
+      operations: newOperations,
+      isProcessing: false,
+    })
+
+    logger.debug('Cancelled operations for variable', {
+      variableId,
+      cancelledOperations: operationsToCancel.length,
+    })
+
+    get().processNextOperation()
+  },
+
+  cancelOperationsForWorkflow: (workflowId: string) => {
+    const state = get()
+    retryTimeouts.forEach((timeout, opId) => {
+      const op = state.operations.find((o) => o.id === opId)
+      if (op && op.workflowId === workflowId) {
+        clearTimeout(timeout)
+        retryTimeouts.delete(opId)
+      }
+    })
+    operationTimeouts.forEach((timeout, opId) => {
+      const op = state.operations.find((o) => o.id === opId)
+      if (op && op.workflowId === workflowId) {
+        clearTimeout(timeout)
+        operationTimeouts.delete(opId)
+      }
+    })
+    set((s) => ({
+      operations: s.operations.filter((op) => op.workflowId !== workflowId),
+      isProcessing: false,
+    }))
+  },
+
+  triggerOfflineMode: () => {
+    logger.error('Operation failed after retries - triggering offline mode')
+
+    retryTimeouts.forEach((timeout) => clearTimeout(timeout))
+    retryTimeouts.clear()
+    operationTimeouts.forEach((timeout) => clearTimeout(timeout))
+    operationTimeouts.clear()
+
+    set({
+      operations: [],
+      isProcessing: false,
+      hasOperationError: true,
+    })
+  },
+
+  clearError: () => {
+    set({ hasOperationError: false })
+  },
+}))
+
+/**
+ * Hook to access operation queue state and actions.
+ * Uses getState() for actions to avoid unnecessary re-renders.
+ * Only subscribes to the specific state values needed.
+ */
+export function useOperationQueue() {
+  const hasOperationError = useOperationQueueStore((state) => state.hasOperationError)
+
+  const actions = useOperationQueueStore.getState()
+
+  return {
+    get queue() {
+      return useOperationQueueStore.getState().operations
+    },
+    get isProcessing() {
+      return useOperationQueueStore.getState().isProcessing
+    },
+    hasOperationError,
+    addToQueue: actions.addToQueue,
+    confirmOperation: actions.confirmOperation,
+    failOperation: actions.failOperation,
+    processNextOperation: actions.processNextOperation,
+    hasPendingOperations: actions.hasPendingOperations,
+    waitForWorkflowOperations: actions.waitForWorkflowOperations,
+    cancelOperationsForBlock: actions.cancelOperationsForBlock,
+    cancelOperationsForVariable: actions.cancelOperationsForVariable,
+    triggerOfflineMode: actions.triggerOfflineMode,
+    clearError: actions.clearError,
+  }
+}

apps/sim/stores/operation-queue/types.ts

@@ -0,0 +1,64 @@
+/**
+ * Emit functions return whether the payload was actually sent over the socket.
+ * A `false` return means the emit was skipped (room not joined/visible) and the
+ * operation should stay pending instead of waiting on a confirmation timeout.
+ */
+export type WorkflowOperationEmit = (
+  workflowId: string,
+  operation: string,
+  target: string,
+  payload: any,
+  operationId?: string
+) => boolean
+
+export type SubblockUpdateEmit = (
+  blockId: string,
+  subblockId: string,
+  value: any,
+  operationId: string | undefined,
+  workflowId: string
+) => boolean
+
+export type VariableUpdateEmit = (
+  variableId: string,
+  field: string,
+  value: any,
+  operationId: string | undefined,
+  workflowId: string
+) => boolean
+
+export interface QueuedOperation {
+  id: string
+  operation: {
+    operation: string
+    target: string
+    payload: any
+  }
+  workflowId: string
+  timestamp: number
+  retryCount: number
+  status: 'pending' | 'processing' | 'confirmed' | 'failed'
+  userId: string
+}
+
+export interface OperationQueueState {
+  operations: QueuedOperation[]
+  workflowOperationVersions: Record<string, number>
+  isProcessing: boolean
+  hasOperationError: boolean
+
+  addToQueue: (operation: Omit<QueuedOperation, 'timestamp' | 'retryCount' | 'status'>) => void
+  confirmOperation: (operationId: string) => void
+  failOperation: (operationId: string, retryable?: boolean) => void
+  handleOperationTimeout: (operationId: string) => void
+  processNextOperation: () => void
+  hasPendingOperations: (workflowId: string) => boolean
+  waitForWorkflowOperations: (workflowId: string, timeoutMs?: number) => Promise<boolean>
+  cancelOperationsForBlock: (blockId: string) => void
+  cancelOperationsForVariable: (variableId: string) => void
+
+  cancelOperationsForWorkflow: (workflowId: string) => void
+
+  triggerOfflineMode: () => void
+  clearError: () => void
+}