d25d482dc2
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
188 lines
6.4 KiB
TypeScript
188 lines
6.4 KiB
TypeScript
import { db } from '@sim/db'
|
||
import { pausedExecutions } from '@sim/db/schema'
|
||
import { createLogger } from '@sim/logger'
|
||
import { toError } from '@sim/utils/errors'
|
||
import { generateShortId } from '@sim/utils/id'
|
||
import { and, asc, inArray, isNotNull, lte } from 'drizzle-orm'
|
||
import { type NextRequest, NextResponse } from 'next/server'
|
||
import { verifyCronAuth } from '@/lib/auth/internal'
|
||
import { acquireLock, releaseLock } from '@/lib/core/config/redis'
|
||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||
import {
|
||
computeEarliestResumeAt,
|
||
PauseResumeManager,
|
||
} from '@/lib/workflows/executor/human-in-the-loop-manager'
|
||
import type { PausePoint } from '@/executor/types'
|
||
|
||
const logger = createLogger('TimePauseResumePoll')
|
||
|
||
export const dynamic = 'force-dynamic'
|
||
export const maxDuration = 120
|
||
|
||
const LOCK_KEY = 'time-pause-resume-poll-lock'
|
||
const LOCK_TTL_SECONDS = 180
|
||
const POLL_BATCH_LIMIT = 200
|
||
|
||
interface DispatchFailure {
|
||
executionId: string
|
||
contextId: string
|
||
error: string
|
||
}
|
||
|
||
interface RowResult {
|
||
dispatched: number
|
||
failures: DispatchFailure[]
|
||
}
|
||
|
||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||
const requestId = generateShortId()
|
||
|
||
const authError = verifyCronAuth(request, 'Time-pause resume poll')
|
||
if (authError) return authError
|
||
|
||
const lockAcquired = await acquireLock(LOCK_KEY, requestId, LOCK_TTL_SECONDS)
|
||
if (!lockAcquired) {
|
||
return NextResponse.json(
|
||
{ success: true, message: 'Polling already in progress – skipped', requestId },
|
||
{ status: 202 }
|
||
)
|
||
}
|
||
|
||
try {
|
||
const now = new Date()
|
||
|
||
const dueRows = await db
|
||
.select({
|
||
id: pausedExecutions.id,
|
||
executionId: pausedExecutions.executionId,
|
||
workflowId: pausedExecutions.workflowId,
|
||
pausePoints: pausedExecutions.pausePoints,
|
||
metadata: pausedExecutions.metadata,
|
||
})
|
||
.from(pausedExecutions)
|
||
.where(
|
||
and(
|
||
// 'partially_resumed' rows occur when a chained-pause workflow advanced past
|
||
// an earlier wait — e.g. wait1 → agent → wait2 — and now wait2's time pause
|
||
// is the one waiting for the cron. Include it alongside fresh 'paused' rows.
|
||
inArray(pausedExecutions.status, ['paused', 'partially_resumed']),
|
||
isNotNull(pausedExecutions.nextResumeAt),
|
||
lte(pausedExecutions.nextResumeAt, now)
|
||
)
|
||
)
|
||
.orderBy(asc(pausedExecutions.nextResumeAt))
|
||
.limit(POLL_BATCH_LIMIT)
|
||
|
||
const results = await Promise.all(dueRows.map((row) => dispatchRow(row, now)))
|
||
const dispatched = results.reduce((sum, r) => sum + r.dispatched, 0)
|
||
const failures = results.flatMap((r) => r.failures)
|
||
|
||
logger.info('Time-pause resume poll completed', {
|
||
requestId,
|
||
claimedRows: dueRows.length,
|
||
dispatched,
|
||
failureCount: failures.length,
|
||
})
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
requestId,
|
||
claimedRows: dueRows.length,
|
||
dispatched,
|
||
failures,
|
||
})
|
||
} catch (error) {
|
||
const message = toError(error).message
|
||
logger.error('Time-pause resume poll failed', { requestId, error: message })
|
||
return NextResponse.json({ success: false, requestId, error: message }, { status: 500 })
|
||
} finally {
|
||
await releaseLock(LOCK_KEY, requestId).catch(() => {})
|
||
}
|
||
})
|
||
|
||
interface DueRow {
|
||
id: string
|
||
executionId: string
|
||
workflowId: string
|
||
pausePoints: unknown
|
||
metadata: unknown
|
||
}
|
||
|
||
async function dispatchRow(row: DueRow, now: Date): Promise<RowResult> {
|
||
const points = (row.pausePoints ?? {}) as Record<string, PausePoint>
|
||
const metadata = (row.metadata ?? {}) as Record<string, unknown>
|
||
const userId = typeof metadata.executorUserId === 'string' ? metadata.executorUserId : ''
|
||
|
||
const eligiblePoints = Object.values(points).filter(
|
||
(point) =>
|
||
point.pauseKind === 'time' && (!point.resumeStatus || point.resumeStatus === 'paused')
|
||
)
|
||
const duePoints = eligiblePoints.filter((point) => {
|
||
if (!point.resumeAt) return false
|
||
const at = new Date(point.resumeAt)
|
||
return !Number.isNaN(at.getTime()) && at <= now
|
||
})
|
||
|
||
const failures: DispatchFailure[] = []
|
||
let dispatched = 0
|
||
|
||
for (const point of duePoints) {
|
||
if (!point.contextId) continue
|
||
try {
|
||
const enqueueResult = await PauseResumeManager.enqueueOrStartResume({
|
||
executionId: row.executionId,
|
||
workflowId: row.workflowId,
|
||
contextId: point.contextId,
|
||
resumeInput: {},
|
||
userId,
|
||
allowedPauseKinds: ['time'],
|
||
})
|
||
|
||
if (enqueueResult.status === 'starting') {
|
||
// Route through `executeResumeJob` (not `PauseResumeManager.startResumeExecution`
|
||
// directly) so cell-context restoration + cascade-loop continuation
|
||
// fires. This is the same primitive the trigger.dev `resumeExecutionTask`
|
||
// wraps — calling it directly handles both trigger.dev-disabled local
|
||
// dev and trigger.dev-enabled prod identically.
|
||
const { executeResumeJob } = await import('@/background/resume-execution')
|
||
void executeResumeJob({
|
||
resumeEntryId: enqueueResult.resumeEntryId,
|
||
resumeExecutionId: enqueueResult.resumeExecutionId,
|
||
pausedExecutionId: enqueueResult.pausedExecution.id,
|
||
contextId: enqueueResult.contextId,
|
||
resumeInput: enqueueResult.resumeInput,
|
||
userId: enqueueResult.userId,
|
||
workflowId: row.workflowId,
|
||
parentExecutionId: row.executionId,
|
||
}).catch((error) => {
|
||
logger.error('Background time-pause resume failed', {
|
||
executionId: row.executionId,
|
||
contextId: point.contextId,
|
||
error: toError(error).message,
|
||
})
|
||
})
|
||
}
|
||
dispatched++
|
||
} catch (error) {
|
||
const message = toError(error).message
|
||
logger.warn('Failed to dispatch time-pause resume', {
|
||
executionId: row.executionId,
|
||
contextId: point.contextId,
|
||
error: message,
|
||
})
|
||
failures.push({ executionId: row.executionId, contextId: point.contextId, error: message })
|
||
}
|
||
}
|
||
|
||
// We never auto-retry a failed dispatch: workflow blocks aren't idempotent, and
|
||
// an operator must investigate stranded rows by hand. The status='paused' guard
|
||
// also prevents clobbering when a concurrent manual resume has already advanced
|
||
// the row's state since we read it.
|
||
await PauseResumeManager.setNextResumeAt({
|
||
pausedExecutionId: row.id,
|
||
nextResumeAt: computeEarliestResumeAt(eligiblePoints, { after: now }),
|
||
})
|
||
|
||
return { dispatched, failures }
|
||
}
|