chore: import upstream snapshot with attribution

apps/sim/app/api/cron/cleanup-soft-deletes/route.ts

@@ -0,0 +1,25 @@
+import { createLogger } from '@sim/logger'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+
+export const dynamic = 'force-dynamic'
+
+const logger = createLogger('SoftDeleteCleanupAPI')
+
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  try {
+    const authError = verifyCronAuth(request, 'soft-delete cleanup')
+    if (authError) return authError
+
+    const result = await dispatchCleanupJobs('cleanup-soft-deletes')
+
+    logger.info('Soft-delete cleanup jobs dispatched', result)
+
+    return NextResponse.json({ triggered: true, ...result })
+  } catch (error) {
+    logger.error('Failed to dispatch soft-delete cleanup jobs:', { error })
+    return NextResponse.json({ error: 'Failed to dispatch soft-delete cleanup' }, { status: 500 })
+  }
+})

apps/sim/app/api/cron/cleanup-stale-executions/route.ts

@@ -0,0 +1,247 @@
+import { asyncJobs, db } from '@sim/db'
+import { tableJobs, workflowExecutionLogs } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { and, eq, inArray, lt, sql } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { JOB_RETENTION_HOURS, JOB_STATUS } from '@/lib/core/async-jobs'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { deleteFile } from '@/lib/uploads/core/storage-service'
+
+const logger = createLogger('CleanupStaleExecutions')
+
+const STALE_THRESHOLD_MS = getMaxExecutionTimeout() + 5 * 60 * 1000
+const STALE_THRESHOLD_MINUTES = Math.ceil(STALE_THRESHOLD_MS / 60000)
+const MAX_INT32 = 2_147_483_647
+/** Terminal table-jobs older than this are pruned; only the latest job per table is ever read. */
+const TABLE_JOB_RETENTION_HOURS = 24
+
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  try {
+    const authError = verifyCronAuth(request, 'Stale execution cleanup')
+    if (authError) {
+      return authError
+    }
+
+    logger.info('Starting stale execution cleanup job')
+
+    const staleThreshold = new Date(Date.now() - STALE_THRESHOLD_MINUTES * 60 * 1000)
+
+    const staleExecutions = await db
+      .select({
+        id: workflowExecutionLogs.id,
+        executionId: workflowExecutionLogs.executionId,
+        workflowId: workflowExecutionLogs.workflowId,
+        startedAt: workflowExecutionLogs.startedAt,
+      })
+      .from(workflowExecutionLogs)
+      .where(
+        and(
+          eq(workflowExecutionLogs.status, 'running'),
+          lt(workflowExecutionLogs.startedAt, staleThreshold)
+        )
+      )
+      .limit(100)
+
+    logger.info(`Found ${staleExecutions.length} stale executions to clean up`)
+
+    let cleaned = 0
+    let failed = 0
+
+    for (const execution of staleExecutions) {
+      try {
+        const staleDurationMs = Date.now() - new Date(execution.startedAt).getTime()
+        const staleDurationMinutes = Math.round(staleDurationMs / 60000)
+        const totalDurationMs = Math.min(staleDurationMs, MAX_INT32)
+
+        await db
+          .update(workflowExecutionLogs)
+          .set({
+            status: 'failed',
+            endedAt: new Date(),
+            totalDurationMs,
+            executionData: sql`jsonb_set(
+              COALESCE(execution_data, '{}'::jsonb),
+              ARRAY['error'],
+              to_jsonb(${`Execution terminated: worker timeout or crash after ${staleDurationMinutes} minutes`}::text)
+            )`,
+          })
+          .where(eq(workflowExecutionLogs.id, execution.id))
+
+        logger.info(`Cleaned up stale execution ${execution.executionId}`, {
+          workflowId: execution.workflowId,
+          staleDurationMinutes,
+        })
+
+        cleaned++
+      } catch (error) {
+        logger.error(`Failed to clean up execution ${execution.executionId}:`, {
+          error: toError(error).message,
+        })
+        failed++
+      }
+    }
+
+    logger.info(`Stale execution cleanup completed. Cleaned: ${cleaned}, Failed: ${failed}`)
+
+    // Clean up stale async jobs (stuck in processing)
+    let asyncJobsMarkedFailed = 0
+
+    try {
+      const staleAsyncJobs = await db
+        .update(asyncJobs)
+        .set({
+          status: JOB_STATUS.FAILED,
+          completedAt: new Date(),
+          error: `Job terminated: stuck in processing for more than ${STALE_THRESHOLD_MINUTES} minutes`,
+          updatedAt: new Date(),
+        })
+        .where(
+          and(eq(asyncJobs.status, JOB_STATUS.PROCESSING), lt(asyncJobs.startedAt, staleThreshold))
+        )
+        .returning({ id: asyncJobs.id })
+
+      asyncJobsMarkedFailed = staleAsyncJobs.length
+      if (asyncJobsMarkedFailed > 0) {
+        logger.info(`Marked ${asyncJobsMarkedFailed} stale async jobs as failed`)
+      }
+    } catch (error) {
+      logger.error('Failed to clean up stale async jobs:', {
+        error: toError(error).message,
+      })
+    }
+
+    // Mark stale table jobs (import, export, or delete) as failed. Jobs run detached on the web container
+    // and are lost if the pod is killed mid-run. `updated_at` is bumped by progress updates, so a
+    // `running` job with no recent update has stalled (not merely slow). Committed work is left in
+    // place (no rollback); the user retries. Also prune long-settled terminal jobs so the table
+    // doesn't grow unbounded (the latest job per table is what list/detail reads surface).
+    let staleTableJobsMarkedFailed = 0
+    try {
+      const now = new Date()
+      const staleJobs = await db
+        .update(tableJobs)
+        .set({
+          status: 'failed',
+          error: `Job terminated: no progress for more than ${STALE_THRESHOLD_MINUTES} minutes (worker timeout or crash)`,
+          completedAt: now,
+          updatedAt: now,
+        })
+        .where(and(eq(tableJobs.status, 'running'), lt(tableJobs.updatedAt, staleThreshold)))
+        .returning({ id: tableJobs.id })
+
+      staleTableJobsMarkedFailed = staleJobs.length
+      if (staleTableJobsMarkedFailed > 0) {
+        logger.info(`Marked ${staleTableJobsMarkedFailed} stale table jobs as failed`)
+      }
+
+      const terminalRetention = new Date(Date.now() - TABLE_JOB_RETENTION_HOURS * 60 * 60 * 1000)
+      const pruned = await db
+        .delete(tableJobs)
+        .where(
+          and(
+            inArray(tableJobs.status, ['ready', 'failed', 'canceled']),
+            lt(tableJobs.updatedAt, terminalRetention)
+          )
+        )
+        .returning({ type: tableJobs.type, payload: tableJobs.payload })
+
+      // Pruned export jobs carry the generated file's storage key — delete the file with the job
+      // so the exports prefix doesn't accumulate. Best-effort: a miss just orphans one object.
+      for (const job of pruned) {
+        if (job.type !== 'export') continue
+        const resultKey = (job.payload as { resultKey?: string } | null)?.resultKey
+        if (!resultKey) continue
+        await deleteFile({ key: resultKey, context: 'workspace' }).catch((err) => {
+          logger.warn('Failed to delete pruned export file', {
+            resultKey,
+            error: toError(err).message,
+          })
+        })
+      }
+    } catch (error) {
+      logger.error('Failed to clean up stale table jobs:', {
+        error: toError(error).message,
+      })
+    }
+
+    // Clean up stale pending jobs (never started, e.g., due to server crash before startJob())
+    let stalePendingJobsMarkedFailed = 0
+
+    try {
+      const stalePendingJobs = await db
+        .update(asyncJobs)
+        .set({
+          status: JOB_STATUS.FAILED,
+          completedAt: new Date(),
+          error: `Job terminated: stuck in pending state for more than ${STALE_THRESHOLD_MINUTES} minutes (never started)`,
+          updatedAt: new Date(),
+        })
+        .where(
+          and(eq(asyncJobs.status, JOB_STATUS.PENDING), lt(asyncJobs.createdAt, staleThreshold))
+        )
+        .returning({ id: asyncJobs.id })
+
+      stalePendingJobsMarkedFailed = stalePendingJobs.length
+      if (stalePendingJobsMarkedFailed > 0) {
+        logger.info(`Marked ${stalePendingJobsMarkedFailed} stale pending jobs as failed`)
+      }
+    } catch (error) {
+      logger.error('Failed to clean up stale pending jobs:', {
+        error: toError(error).message,
+      })
+    }
+
+    // Delete completed/failed jobs older than retention period
+    const retentionThreshold = new Date(Date.now() - JOB_RETENTION_HOURS * 60 * 60 * 1000)
+    let asyncJobsDeleted = 0
+
+    try {
+      const deletedJobs = await db
+        .delete(asyncJobs)
+        .where(
+          and(
+            inArray(asyncJobs.status, [JOB_STATUS.COMPLETED, JOB_STATUS.FAILED]),
+            lt(asyncJobs.completedAt, retentionThreshold)
+          )
+        )
+        .returning({ id: asyncJobs.id })
+
+      asyncJobsDeleted = deletedJobs.length
+      if (asyncJobsDeleted > 0) {
+        logger.info(
+          `Deleted ${asyncJobsDeleted} old async jobs (retention: ${JOB_RETENTION_HOURS}h)`
+        )
+      }
+    } catch (error) {
+      logger.error('Failed to delete old async jobs:', {
+        error: toError(error).message,
+      })
+    }
+
+    return NextResponse.json({
+      success: true,
+      executions: {
+        found: staleExecutions.length,
+        cleaned,
+        failed,
+        thresholdMinutes: STALE_THRESHOLD_MINUTES,
+      },
+      asyncJobs: {
+        staleProcessingMarkedFailed: asyncJobsMarkedFailed,
+        stalePendingMarkedFailed: stalePendingJobsMarkedFailed,
+        oldDeleted: asyncJobsDeleted,
+        staleThresholdMinutes: STALE_THRESHOLD_MINUTES,
+        retentionHours: JOB_RETENTION_HOURS,
+      },
+      tableJobs: {
+        staleMarkedFailed: staleTableJobsMarkedFailed,
+      },
+    })
+  } catch (error) {
+    logger.error('Error in stale execution cleanup job:', error)
+    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+  }
+})

apps/sim/app/api/cron/cleanup-tasks/route.ts

@@ -0,0 +1,25 @@
+import { createLogger } from '@sim/logger'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+
+export const dynamic = 'force-dynamic'
+
+const logger = createLogger('TaskCleanupAPI')
+
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  try {
+    const authError = verifyCronAuth(request, 'task cleanup')
+    if (authError) return authError
+
+    const result = await dispatchCleanupJobs('cleanup-tasks')
+
+    logger.info('Task cleanup jobs dispatched', result)
+
+    return NextResponse.json({ triggered: true, ...result })
+  } catch (error) {
+    logger.error('Failed to dispatch task cleanup jobs:', { error })
+    return NextResponse.json({ error: 'Failed to dispatch task cleanup' }, { status: 500 })
+  }
+})

apps/sim/app/api/cron/reconcile-billing-seats/route.test.ts

@@ -0,0 +1,92 @@
+/**
+ * Tests for the billing seat reconciliation cron route.
+ *
+ * @vitest-environment node
+ */
+import { createMockRequest } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockVerifyCronAuth, mockReconcileTeamSeatDrift, mockFindDeadLetteredEvents } = vi.hoisted(
+  () => ({
+    mockVerifyCronAuth: vi.fn().mockReturnValue(null),
+    mockReconcileTeamSeatDrift: vi.fn(),
+    mockFindDeadLetteredEvents: vi.fn(),
+  })
+)
+
+vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mockVerifyCronAuth }))
+vi.mock('@/lib/billing/organizations/seat-drift', () => ({
+  reconcileTeamSeatDrift: mockReconcileTeamSeatDrift,
+}))
+vi.mock('@/lib/core/outbox/service', () => ({ findDeadLetteredEvents: mockFindDeadLetteredEvents }))
+vi.mock('@/lib/billing/webhooks/outbox-handlers', () => ({
+  OUTBOX_EVENT_TYPES: {
+    STRIPE_SYNC_SUBSCRIPTION_SEATS: 'stripe.sync-subscription-seats',
+    STRIPE_SYNC_CANCEL_AT_PERIOD_END: 'stripe.sync-cancel-at-period-end',
+  },
+}))
+
+import { GET } from './route'
+
+function createRequest() {
+  return createMockRequest(
+    'GET',
+    undefined,
+    {},
+    'http://localhost:3000/api/cron/reconcile-billing-seats'
+  )
+}
+
+describe('reconcile-billing-seats cron route', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockVerifyCronAuth.mockReturnValue(null)
+    mockReconcileTeamSeatDrift.mockResolvedValue({ drifted: 0, reconciled: 0 })
+    mockFindDeadLetteredEvents.mockResolvedValue([])
+  })
+
+  it('returns the auth error when cron auth fails', async () => {
+    mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never)
+
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(401)
+    expect(mockReconcileTeamSeatDrift).not.toHaveBeenCalled()
+  })
+
+  it('runs the drift sweep and reports dead-lettered billing syncs', async () => {
+    mockReconcileTeamSeatDrift.mockResolvedValue({ drifted: 2, reconciled: 1 })
+    mockFindDeadLetteredEvents.mockResolvedValue([
+      {
+        id: 'evt-1',
+        eventType: 'stripe.sync-subscription-seats',
+        payload: { subscriptionId: 'sub-1' },
+        lastError: 'card declined',
+      },
+    ])
+
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(200)
+    const data = await response.json()
+    expect(data).toMatchObject({
+      success: true,
+      drift: { drifted: 2, reconciled: 1 },
+      deadLetteredBillingSyncs: 1,
+    })
+    expect(mockFindDeadLetteredEvents).toHaveBeenCalledWith([
+      'stripe.sync-subscription-seats',
+      'stripe.sync-cancel-at-period-end',
+    ])
+  })
+
+  it('returns 500 when the sweep throws', async () => {
+    mockReconcileTeamSeatDrift.mockRejectedValueOnce(new Error('boom'))
+
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(500)
+    const data = await response.json()
+    expect(data.success).toBe(false)
+  })
+})

apps/sim/app/api/cron/reconcile-billing-seats/route.ts

@@ -0,0 +1,79 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { reconcileTeamSeatDrift } from '@/lib/billing/organizations/seat-drift'
+import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers'
+import { findDeadLetteredEvents } from '@/lib/core/outbox/service'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+
+const logger = createLogger('BillingSeatReconcileCron')
+
+export const dynamic = 'force-dynamic'
+
+const BILLING_SYNC_EVENT_TYPES = [
+  OUTBOX_EVENT_TYPES.STRIPE_SYNC_SUBSCRIPTION_SEATS,
+  OUTBOX_EVENT_TYPES.STRIPE_SYNC_CANCEL_AT_PERIOD_END,
+]
+
+/**
+ * Periodic billing-seat reconciliation. Self-heals Team organizations whose
+ * stored seat count drifted from their member count, and reports any
+ * dead-lettered Stripe seat/cancel sync events so a member who has access but
+ * whose seat charge never synced is surfaced for manual remediation rather than
+ * silently under-billed.
+ *
+ * Scheduled in helm/sim/values.yaml under cronjobs.jobs.reconcileBillingSeats.
+ */
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const requestId = generateRequestId()
+
+  const authError = verifyCronAuth(request, 'Billing seat reconciliation')
+  if (authError) {
+    return authError
+  }
+
+  try {
+    const drift = await reconcileTeamSeatDrift()
+
+    const deadLettered = await findDeadLetteredEvents(BILLING_SYNC_EVENT_TYPES)
+    if (deadLettered.length > 0) {
+      logger.error(
+        'Dead-lettered billing sync events require manual remediation — a billing state change (seat charge or cancellation) never reached Stripe',
+        {
+          requestId,
+          count: deadLettered.length,
+          events: deadLettered.map((event) => ({
+            id: event.id,
+            eventType: event.eventType,
+            subscriptionId: (event.payload as { subscriptionId?: string } | null)?.subscriptionId,
+            lastError: event.lastError,
+          })),
+        }
+      )
+    }
+
+    logger.info('Billing seat reconciliation completed', {
+      requestId,
+      ...drift,
+      deadLetteredBillingSyncs: deadLettered.length,
+    })
+
+    return NextResponse.json({
+      success: true,
+      requestId,
+      drift,
+      deadLetteredBillingSyncs: deadLettered.length,
+    })
+  } catch (error) {
+    logger.error('Billing seat reconciliation failed', {
+      requestId,
+      error: toError(error).message,
+    })
+    return NextResponse.json(
+      { success: false, requestId, error: toError(error).message },
+      { status: 500 }
+    )
+  }
+})

apps/sim/app/api/cron/reconcile-inbox-entitlement/route.ts

@@ -0,0 +1,66 @@
+import { db, workspace } from '@sim/db'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { eq } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { hasWorkspaceInboxGraceAccess } from '@/lib/billing/core/subscription'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { disableInbox } from '@/lib/mothership/inbox/lifecycle'
+
+const logger = createLogger('InboxEntitlementReconcileCron')
+
+export const dynamic = 'force-dynamic'
+
+/**
+ * Periodic inbox (Sim Mailer) entitlement reconciliation. Releases the AgentMail
+ * inbox + webhook for any workspace whose provisioned inbox has outlived its
+ * plan — i.e. `inboxEnabled` is still true but the billing entity no longer
+ * holds an entitled (active or `past_due`) Max/Enterprise subscription.
+ *
+ * Teardown keys off {@link hasWorkspaceInboxGraceAccess}, which tolerates
+ * `past_due`, so a transient payment failure never destroys a paying customer's
+ * inbox — only a genuinely terminal plan (canceled/downgraded) is reclaimed.
+ * `disableInbox` swallows AgentMail delete failures, so a rerun or a race with a
+ * manual disable is a no-op. On self-hosted / billing-disabled deployments the
+ * grace check returns true for every workspace, making this sweep inert.
+ *
+ * Scheduled in helm/sim/values.yaml under cronjobs.jobs.reconcileInboxEntitlement.
+ */
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const authError = verifyCronAuth(request, 'Inbox entitlement reconciliation')
+  if (authError) {
+    return authError
+  }
+
+  const enabledWorkspaces = await db
+    .select({ id: workspace.id })
+    .from(workspace)
+    .where(eq(workspace.inboxEnabled, true))
+
+  let disabled = 0
+  for (const ws of enabledWorkspaces) {
+    try {
+      if (await hasWorkspaceInboxGraceAccess(ws.id)) {
+        continue
+      }
+      await disableInbox(ws.id)
+      disabled++
+      logger.info('Reclaimed inbox for workspace with terminated Sim Mailer entitlement', {
+        workspaceId: ws.id,
+      })
+    } catch (error) {
+      logger.error('Failed to reconcile inbox entitlement for workspace', {
+        workspaceId: ws.id,
+        error: getErrorMessage(error, 'Unknown error'),
+      })
+    }
+  }
+
+  logger.info('Inbox entitlement reconciliation complete', {
+    checked: enabledWorkspaces.length,
+    disabled,
+  })
+
+  return NextResponse.json({ checked: enabledWorkspaces.length, disabled })
+})

apps/sim/app/api/cron/renew-subscriptions/route.test.ts

@@ -0,0 +1,91 @@
+/**
+ * Tests for the Teams subscription renewal cron route.
+ *
+ * @vitest-environment node
+ */
+import {
+  authOAuthUtilsMock,
+  createMockRequest,
+  dbChainMock,
+  dbChainMockFns,
+  redisConfigMock,
+  redisConfigMockFns,
+  resetDbChainMock,
+} from '@sim/testing'
+import { sleep } from '@sim/utils/helpers'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockVerifyCronAuth } = vi.hoisted(() => ({
+  mockVerifyCronAuth: vi.fn().mockReturnValue(null),
+}))
+
+vi.mock('@/lib/auth/internal', () => ({
+  verifyCronAuth: mockVerifyCronAuth,
+}))
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+vi.mock('@sim/db', () => dbChainMock)
+vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
+
+import { GET } from './route'
+
+function createRequest() {
+  return createMockRequest(
+    'GET',
+    undefined,
+    {},
+    'http://localhost:3000/api/cron/renew-subscriptions'
+  )
+}
+
+const flushMicrotasks = () => sleep(0)
+
+describe('Teams subscription renewal route (fire-and-forget)', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    redisConfigMockFns.mockAcquireLock.mockResolvedValue(true)
+    redisConfigMockFns.mockReleaseLock.mockResolvedValue(true)
+    mockVerifyCronAuth.mockReturnValue(null)
+  })
+
+  it('returns the auth error when cron auth fails', async () => {
+    mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never)
+
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(401)
+    expect(redisConfigMockFns.mockAcquireLock).not.toHaveBeenCalled()
+  })
+
+  it('acknowledges with 202 and renews in the background after acquiring the lock', async () => {
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(202)
+    const data = await response.json()
+    expect(data).toMatchObject({ status: 'started' })
+    expect(redisConfigMockFns.mockAcquireLock).toHaveBeenCalledWith(
+      'teams-subscription-renewal-lock',
+      expect.any(String),
+      expect.any(Number)
+    )
+
+    await flushMicrotasks()
+    expect(dbChainMockFns.select).toHaveBeenCalled()
+    expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
+      'teams-subscription-renewal-lock',
+      expect.any(String)
+    )
+  })
+
+  it('skips with 202 when the lock is already held', async () => {
+    redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
+
+    const response = await GET(createRequest())
+
+    expect(response.status).toBe(202)
+    const data = await response.json()
+    expect(data).toMatchObject({ status: 'skip' })
+    expect(dbChainMockFns.select).not.toHaveBeenCalled()
+  })
+})

apps/sim/app/api/cron/renew-subscriptions/route.ts

@@ -0,0 +1,274 @@
+import { db } from '@sim/db'
+import { webhook as webhookTable } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { generateShortId } from '@sim/utils/id'
+import { and, eq, or } from 'drizzle-orm'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { acquireLock, releaseLock } from '@/lib/core/config/redis'
+import { runDetached } from '@/lib/core/utils/background'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils'
+import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
+
+const logger = createLogger('TeamsSubscriptionRenewal')
+
+const LOCK_KEY = 'teams-subscription-renewal-lock'
+/** Lock TTL in seconds — generous enough to cover the Graph API renewal loop. */
+const LOCK_TTL_SECONDS = 300
+
+/** Microsoft Graph subscriptions are hard-capped at ~3 days. */
+const MAX_LIFETIME_MINUTES = 4230
+
+/**
+ * Recreate a Teams chat subscription from scratch after the existing one has
+ * actually expired on Microsoft's side (PATCH returns 404/410). Without this,
+ * a subscription that expires while every renewal attempt in its 48h window
+ * failed (revoked consent, prolonged Graph outage, etc.) would stay dead
+ * forever — the webhook remains `isActive` but never receives events again.
+ */
+async function recreateSubscription(
+  webhook: Record<string, unknown>,
+  config: Record<string, any>,
+  accessToken: string
+): Promise<{ id: string; expirationDateTime: string } | null> {
+  const chatId = config.chatId as string | undefined
+  if (!chatId) {
+    logger.error(`Missing chatId for webhook ${webhook.id}, cannot recreate subscription`)
+    return null
+  }
+
+  const notificationUrl = getNotificationUrl(webhook)
+  const expirationDateTime = new Date(Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000).toISOString()
+
+  const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${accessToken}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      changeType: 'created,updated',
+      notificationUrl,
+      lifecycleNotificationUrl: notificationUrl,
+      resource: `/chats/${chatId}/messages`,
+      includeResourceData: false,
+      expirationDateTime,
+      clientState: webhook.id,
+    }),
+  })
+
+  if (!res.ok) {
+    const error = await res.json()
+    logger.error(`Failed to recreate Teams subscription for webhook ${webhook.id}`, {
+      status: res.status,
+      error: error.error,
+    })
+    return null
+  }
+
+  const payload = await res.json()
+  return { id: payload.id as string, expirationDateTime: payload.expirationDateTime as string }
+}
+
+/**
+ * Renews Microsoft Teams chat subscriptions that are close to expiring.
+ *
+ * Teams subscriptions expire after ~3 days and must be renewed. Runs detached
+ * from the HTTP response so the cron caller does not wait for the Graph API loop.
+ */
+async function renewExpiringSubscriptions(): Promise<{
+  checked: number
+  renewed: number
+  failed: number
+  total: number
+}> {
+  logger.info('Starting Teams subscription renewal job')
+
+  let totalRenewed = 0
+  let totalFailed = 0
+  let totalChecked = 0
+
+  const webhooksWithWorkflows = await db
+    .select({
+      webhook: webhookTable,
+    })
+    .from(webhookTable)
+    .where(
+      and(
+        eq(webhookTable.isActive, true),
+        or(
+          eq(webhookTable.provider, 'microsoft-teams'),
+          eq(webhookTable.provider, 'microsoftteams')
+        )
+      )
+    )
+
+  logger.info(
+    `Found ${webhooksWithWorkflows.length} active Teams webhooks, checking for expiring subscriptions`
+  )
+
+  /** Renew any subscription expiring within the next 48 hours. */
+  const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000)
+
+  for (const { webhook } of webhooksWithWorkflows) {
+    const config = (webhook.providerConfig as Record<string, any>) || {}
+
+    if (config.triggerId !== 'microsoftteams_chat_subscription') continue
+
+    const expirationStr = config.subscriptionExpiration as string | undefined
+    if (!expirationStr) continue
+
+    const expiresAt = new Date(expirationStr)
+    if (expiresAt > renewalThreshold) continue
+
+    totalChecked++
+    const requestId = `renewal-${webhook.id}`
+
+    try {
+      logger.info(
+        `Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})`
+      )
+
+      const credentialId = config.credentialId as string | undefined
+      const externalSubscriptionId = config.externalSubscriptionId as string | undefined
+
+      if (!credentialId || !externalSubscriptionId) {
+        logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`)
+        totalFailed++
+        continue
+      }
+
+      const credentialOwner = await getCredentialOwner(credentialId, requestId)
+      if (!credentialOwner) {
+        logger.error(`Credential owner not found for credential ${credentialId}`)
+        totalFailed++
+        continue
+      }
+
+      const accessToken = await refreshAccessTokenIfNeeded(
+        credentialOwner.accountId,
+        credentialOwner.userId,
+        requestId
+      )
+
+      if (!accessToken) {
+        logger.error(`Failed to get access token for webhook ${webhook.id}`)
+        totalFailed++
+        continue
+      }
+
+      const newExpirationDateTime = new Date(
+        Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000
+      ).toISOString()
+
+      const res = await fetch(
+        `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`,
+        {
+          method: 'PATCH',
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ expirationDateTime: newExpirationDateTime }),
+        }
+      )
+
+      let newSubscriptionId: string | undefined
+      let newExpiration: string | undefined
+
+      if (!res.ok) {
+        const error = await res.json()
+        logger.error(
+          `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`,
+          { status: res.status, error: error.error }
+        )
+
+        if (res.status === 404 || res.status === 410) {
+          const recreated = await recreateSubscription(webhook, config, accessToken)
+          if (!recreated) {
+            totalFailed++
+            continue
+          }
+          newSubscriptionId = recreated.id
+          newExpiration = recreated.expirationDateTime
+          logger.info(
+            `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})`
+          )
+        } else {
+          totalFailed++
+          continue
+        }
+      } else {
+        const payload = await res.json()
+        newExpiration = payload.expirationDateTime as string
+      }
+
+      const updatedConfig = {
+        ...config,
+        ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}),
+        subscriptionExpiration: newExpiration,
+      }
+
+      await db
+        .update(webhookTable)
+        .set({ providerConfig: updatedConfig, updatedAt: new Date() })
+        .where(eq(webhookTable.id, webhook.id))
+
+      logger.info(
+        `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}`
+      )
+      totalRenewed++
+    } catch (error) {
+      logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error)
+      totalFailed++
+    }
+  }
+
+  logger.info(
+    `Teams subscription renewal job completed. Checked: ${totalChecked}, Renewed: ${totalRenewed}, Failed: ${totalFailed}`
+  )
+
+  return {
+    checked: totalChecked,
+    renewed: totalRenewed,
+    failed: totalFailed,
+    total: webhooksWithWorkflows.length,
+  }
+}
+
+/**
+ * Cron endpoint to renew Microsoft Teams chat subscriptions before they expire.
+ * Configured in helm/sim/values.yaml under cronjobs.jobs.renewSubscriptions.
+ *
+ * Acknowledges the cron call immediately and renews subscriptions in the
+ * background; a Redis lock prevents overlapping runs.
+ */
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const authError = verifyCronAuth(request, 'Teams subscription renewal')
+  if (authError) {
+    return authError
+  }
+
+  const lockValue = generateShortId()
+  const locked = await acquireLock(LOCK_KEY, lockValue, LOCK_TTL_SECONDS)
+  if (!locked) {
+    return NextResponse.json(
+      { success: true, message: 'Renewal already in progress – skipped', status: 'skip' },
+      { status: 202 }
+    )
+  }
+
+  runDetached('teams-subscription-renewal', async () => {
+    try {
+      await renewExpiringSubscriptions()
+    } finally {
+      await releaseLock(LOCK_KEY, lockValue).catch(() => {})
+    }
+  })
+
+  return NextResponse.json(
+    { success: true, message: 'Teams subscription renewal started', status: 'started' },
+    { status: 202 }
+  )
+})

apps/sim/app/api/cron/run-data-drains/route.ts

@@ -0,0 +1,30 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { type NextRequest, NextResponse } from 'next/server'
+import { verifyCronAuth } from '@/lib/auth/internal'
+import { isBillingEnabled, isDataDrainsEnabled } from '@/lib/core/config/env-flags'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { dispatchDueDrains } from '@/lib/data-drains/dispatcher'
+
+const logger = createLogger('CronRunDataDrains')
+
+export const GET = withRouteHandler(async (request: NextRequest) => {
+  const authError = verifyCronAuth(request, 'Data drain dispatcher')
+  if (authError) return authError
+
+  // Self-hosted opt-in: skip dispatch entirely when the deployment hasn't
+  // enabled drains. Sim Cloud (billing enabled) gates per-org by enterprise
+  // plan inside the dispatcher's join.
+  if (!isBillingEnabled && !isDataDrainsEnabled) {
+    return NextResponse.json({ success: true, dispatched: 0, skipped: 'disabled' })
+  }
+
+  try {
+    const result = await dispatchDueDrains()
+    logger.info('Data drain dispatcher run complete', result)
+    return NextResponse.json({ success: true, ...result })
+  } catch (error) {
+    logger.error('Data drain dispatcher run failed', { error: toError(error).message })
+    return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
+  }
+})