import { trace } from '@opentelemetry/api' import { db } from '@sim/db' import { account, webhook } from '@sim/db/schema' import { createLogger, runWithRequestContext } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { task } from '@trigger.dev/sdk' import { eq } from 'drizzle-orm' import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types' import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits' import { IdempotencyService, webhookIdempotency } from '@/lib/core/idempotency' import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { type WebhookAttachment, WebhookAttachmentProcessor, } from '@/lib/webhooks/attachment-processor' import { resolveWebhookRecordProviderConfig } from '@/lib/webhooks/env-resolver' import { getProviderHandler } from '@/lib/webhooks/providers' import { executeWorkflowCore, wasExecutionFinalizedByCore, } from '@/lib/workflows/executor/execution-core' import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence' import { loadDeployedWorkflowState } from '@/lib/workflows/persistence/utils' import { resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' import { WEBHOOK_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits' import { getBlock } from '@/blocks' import { ExecutionSnapshot } from '@/executor/execution/snapshot' import type { ExecutionMetadata } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' import { hasExecutionResult } from '@/executor/utils/errors' import { safeAssign } from '@/tools/safe-assign' import { getTrigger, isTriggerValid } from '@/triggers' const logger = createLogger('TriggerWebhookExecution') type WebhookAttachmentInput = Omit & { data: unknown } function isSerializedBuffer(value: unknown): value is { type: 'Buffer'; data: number[] } { return isRecordLike(value) && value.type === 'Buffer' && Array.isArray(value.data) } function hasSupportedAttachmentData(value: unknown): boolean { return ( Buffer.isBuffer(value) || typeof value === 'string' || value instanceof ArrayBuffer || ArrayBuffer.isView(value) || Array.isArray(value) || isSerializedBuffer(value) ) } function toAttachmentBuffer(data: unknown, name: string): Buffer { if (Buffer.isBuffer(data)) { return data } if (isSerializedBuffer(data)) { return Buffer.from(data.data) } if (data instanceof ArrayBuffer) { return Buffer.from(data) } if (ArrayBuffer.isView(data)) { return Buffer.from(data.buffer, data.byteOffset, data.byteLength) } if (Array.isArray(data)) { return Buffer.from(data) } if (typeof data === 'string') { const trimmed = data.trim() if (trimmed.startsWith('data:')) { const [, base64Data] = trimmed.split(',') return Buffer.from(base64Data ?? '', 'base64') } return Buffer.from(trimmed, 'base64') } throw new Error(`Attachment '${name}' has unsupported data format`) } function isWebhookAttachmentInput(value: unknown): value is WebhookAttachmentInput { if (!isRecordLike(value)) { return false } return ( typeof value.name === 'string' && typeof value.size === 'number' && hasSupportedAttachmentData(value.data) && (value.contentType === undefined || typeof value.contentType === 'string') && (value.mimeType === undefined || typeof value.mimeType === 'string') ) } function normalizeWebhookAttachment(value: unknown): WebhookAttachment | null { if (!isWebhookAttachmentInput(value)) { return null } return { name: value.name, data: toAttachmentBuffer(value.data, value.name), contentType: value.contentType, mimeType: value.mimeType, size: value.size, } } function normalizeWebhookAttachments(value: unknown): WebhookAttachment[] { if (!Array.isArray(value)) { return [] } return value.flatMap((attachment) => { const normalized = normalizeWebhookAttachment(attachment) return normalized ? [normalized] : [] }) } export function buildWebhookCorrelation( payload: WebhookExecutionPayload ): AsyncExecutionCorrelation { const executionId = payload.executionId || generateId() const requestId = payload.requestId || payload.correlation?.requestId || executionId.slice(0, 8) return { executionId, requestId, source: 'webhook', workflowId: payload.workflowId, webhookId: payload.webhookId, path: payload.path, provider: payload.provider, triggerType: payload.correlation?.triggerType || 'webhook', } } /** * Process trigger outputs based on their schema definitions. * Finds outputs marked as 'file' or 'file[]' and uploads them to execution storage. */ async function processTriggerFileOutputs( input: unknown, triggerOutputs: Record, context: { workspaceId: string workflowId: string executionId: string requestId: string userId?: string }, path = '' ): Promise { if (!input || typeof input !== 'object') { return input } const processed = (Array.isArray(input) ? [] : {}) as Record for (const [key, value] of Object.entries(input)) { const currentPath = path ? `${path}.${key}` : key const outputDef = triggerOutputs[key] as Record | undefined if (outputDef?.type === 'file[]' && Array.isArray(value)) { try { processed[key] = await WebhookAttachmentProcessor.processAttachments( normalizeWebhookAttachments(value), context ) } catch (error) { processed[key] = [] } } else if (outputDef?.type === 'file' && value) { const attachment = normalizeWebhookAttachment(value) if (!attachment) { processed[key] = value continue } try { const [processedFile] = await WebhookAttachmentProcessor.processAttachments( [attachment], context ) processed[key] = processedFile } catch (error) { logger.error(`[${context.requestId}] Error processing ${currentPath}:`, error) processed[key] = value } } else if ( outputDef && typeof outputDef === 'object' && (outputDef.type === 'object' || outputDef.type === 'json') && outputDef.properties ) { processed[key] = await processTriggerFileOutputs( value, outputDef.properties as Record, context, currentPath ) } else if (outputDef && typeof outputDef === 'object' && !outputDef.type) { processed[key] = await processTriggerFileOutputs( value, outputDef as Record, context, currentPath ) } else { processed[key] = value } } return processed } export type WebhookExecutionPayload = { webhookId: string workflowId: string userId: string executionId?: string requestId?: string correlation?: AsyncExecutionCorrelation provider: string body: unknown headers: Record path: string blockId?: string workspaceId?: string credentialId?: string /** Epoch ms when the webhook HTTP request was first received (for dispatch-latency metrics). */ webhookReceivedAt?: number /** Epoch ms of the originating provider interaction (e.g. Slack x-slack-request-timestamp). */ triggerTimestampMs?: number /** * Billing actor resolved by the webhook route, set ONLY for in-process inline * execution that runs microseconds after resolution. The background pass reuses * it to skip the redundant billed-account lookup. Deliberately absent on queued * (Trigger.dev) and persisted payloads — a deferred run could outlive a * billed-account change, so it re-resolves the current actor instead. */ resolvedActorUserId?: string } export async function executeWebhookJob(payload: WebhookExecutionPayload) { const correlation = buildWebhookCorrelation(payload) const executionId = correlation.executionId const requestId = correlation.requestId return runWithRequestContext({ requestId }, async () => { logger.info(`[${requestId}] Starting webhook execution`, { webhookId: payload.webhookId, workflowId: payload.workflowId, provider: payload.provider, userId: payload.userId, executionId, }) const idempotencyKey = IdempotencyService.createWebhookIdempotencyKey( payload.webhookId, payload.headers, payload.body, payload.provider ) const runOperation = async () => { return await executeWebhookJobInternal(payload, correlation) } return await webhookIdempotency.executeWithIdempotency( payload.provider, idempotencyKey, runOperation ) }) } export async function resolveWebhookExecutionProviderConfig< T extends { id: string; providerConfig?: unknown }, >( webhookRecord: T, provider: string, userId: string, workspaceId?: string ): Promise }> { try { return await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId) } catch (error) { const errorMessage = toError(error).message throw new Error( `Failed to resolve webhook provider config for ${provider} webhook ${webhookRecord.id}: ${errorMessage}` ) } } async function resolveCredentialAccountUserId(credentialId: string): Promise { const resolved = await resolveOAuthAccountId(credentialId) if (!resolved) { return undefined } const [credentialRecord] = await db .select({ userId: account.userId }) .from(account) .where(eq(account.id, resolved.accountId)) .limit(1) return credentialRecord?.userId } /** * Handle execution result status (timeout, pause, resume). * Shared between all provider paths to eliminate duplication. */ async function handleExecutionResult( executionResult: ExecutionResult, ctx: { loggingSession: LoggingSession timeoutController: ReturnType requestId: string executionId: string workflowId: string } ) { if ( executionResult.status === 'cancelled' && ctx.timeoutController.isTimedOut() && ctx.timeoutController.timeoutMs ) { const timeoutErrorMessage = getTimeoutErrorMessage(null, ctx.timeoutController.timeoutMs) logger.info(`[${ctx.requestId}] Webhook execution timed out`, { timeoutMs: ctx.timeoutController.timeoutMs, }) await ctx.loggingSession.markAsFailed(timeoutErrorMessage) } else { await handlePostExecutionPauseState({ result: executionResult, workflowId: ctx.workflowId, executionId: ctx.executionId, loggingSession: ctx.loggingSession, }) } await ctx.loggingSession.waitForPostExecution() } async function executeWebhookJobInternal( payload: WebhookExecutionPayload, correlation: AsyncExecutionCorrelation ) { const { executionId, requestId } = correlation const loggingSession = new LoggingSession( payload.workflowId, executionId, payload.provider, requestId ) const preprocessResult = await preprocessExecution({ workflowId: payload.workflowId, userId: payload.userId, triggerType: 'webhook', executionId, requestId, triggerData: { correlation }, checkRateLimit: false, checkDeployment: false, skipUsageLimits: true, workspaceId: payload.workspaceId, loggingSession, /** * Reuse the route-resolved actor only for inline execution (set on the * in-process payload). When absent — queued/Trigger.dev runs — preprocessing * re-resolves the current billed account. Either way the ban and * archived-workflow gates run fresh against the resolved actor. */ resolvedActorUserId: payload.resolvedActorUserId, }) if (!preprocessResult.success) { throw new Error(preprocessResult.error?.message || 'Preprocessing failed in background job') } const { workflowRecord, executionTimeout } = preprocessResult if (!workflowRecord) { throw new Error(`Workflow ${payload.workflowId} not found during preprocessing`) } const workspaceId = workflowRecord.workspaceId if (!workspaceId) { throw new Error(`Workflow ${payload.workflowId} has no associated workspace`) } const workflowVariables = (workflowRecord.variables as Record) || {} const asyncTimeout = executionTimeout?.async ?? 120_000 const timeoutController = createTimeoutAbortController(asyncTimeout) let deploymentVersionId: string | undefined try { const [workflowData, webhookRows, resolvedCredentialUserId] = await Promise.all([ loadDeployedWorkflowState(payload.workflowId, workspaceId), db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1), payload.credentialId ? resolveCredentialAccountUserId(payload.credentialId) : Promise.resolve(undefined), ]) const credentialAccountUserId = resolvedCredentialUserId if (payload.credentialId && !credentialAccountUserId) { logger.warn( `[${requestId}] Failed to resolve credential account for credential ${payload.credentialId}` ) } if (!workflowData) { throw new Error( 'Workflow state not found. The workflow may not be deployed or the deployment data may be corrupted.' ) } const { blocks, edges, loops, parallels } = workflowData deploymentVersionId = 'deploymentVersionId' in workflowData ? (workflowData.deploymentVersionId as string) : undefined const handler = getProviderHandler(payload.provider) let input: Record | null = null let skipMessage: string | undefined const webhookRecord = webhookRows[0] if (!webhookRecord) { throw new Error(`Webhook record not found: ${payload.webhookId}`) } const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig( webhookRecord, payload.provider, workflowRecord.userId, workspaceId ) if (handler.formatInput) { const result = await handler.formatInput({ webhook: resolvedWebhookRecord, workflow: { id: payload.workflowId, userId: payload.userId }, body: payload.body, headers: payload.headers, requestId, }) input = result.input as Record | null skipMessage = result.skip?.message } else { input = payload.body as Record | null } if (!input && handler.handleEmptyInput) { const skipResult = handler.handleEmptyInput(requestId) if (skipResult) { skipMessage = skipResult.message } } if (skipMessage) { await loggingSession.safeStart({ userId: payload.userId, workspaceId, variables: {}, triggerData: { isTest: false, correlation, }, deploymentVersionId, }) await loggingSession.safeComplete({ endedAt: new Date().toISOString(), totalDurationMs: 0, finalOutput: { message: skipMessage }, traceSpans: [], }) return { success: true, workflowId: payload.workflowId, executionId, output: { message: skipMessage }, executedAt: new Date().toISOString(), } } if (input && payload.blockId && blocks[payload.blockId]) { try { const triggerBlock = blocks[payload.blockId] const rawSelectedTriggerId = triggerBlock?.subBlocks?.selectedTriggerId?.value const rawTriggerId = triggerBlock?.subBlocks?.triggerId?.value let resolvedTriggerId = [rawSelectedTriggerId, rawTriggerId].find( (candidate): candidate is string => typeof candidate === 'string' && isTriggerValid(candidate) ) if (!resolvedTriggerId) { const blockConfig = getBlock(triggerBlock.type) if (blockConfig?.category === 'triggers' && isTriggerValid(triggerBlock.type)) { resolvedTriggerId = triggerBlock.type } else if (triggerBlock.triggerMode && blockConfig?.triggers?.enabled) { const available = blockConfig.triggers?.available?.[0] if (available && isTriggerValid(available)) { resolvedTriggerId = available } } } if (resolvedTriggerId) { const triggerConfig = getTrigger(resolvedTriggerId) if (triggerConfig.outputs) { const processedInput = await processTriggerFileOutputs(input, triggerConfig.outputs, { workspaceId, workflowId: payload.workflowId, executionId, requestId, userId: payload.userId, }) safeAssign(input, processedInput as Record) } } } catch (error) { logger.error(`[${requestId}] Error processing trigger file outputs:`, error) } } if (input && handler.processInputFiles && payload.blockId && blocks[payload.blockId]) { try { await handler.processInputFiles({ input, blocks, blockId: payload.blockId, workspaceId, workflowId: payload.workflowId, executionId, requestId, userId: payload.userId, }) } catch (error) { logger.error(`[${requestId}] Error processing provider-specific files:`, error) } } logger.info(`[${requestId}] Executing workflow for ${payload.provider} webhook`) const metadata: ExecutionMetadata = { requestId, executionId, workflowId: payload.workflowId, workspaceId, userId: payload.userId, sessionUserId: undefined, workflowUserId: workflowRecord.userId, triggerType: payload.provider || 'webhook', triggerBlockId: payload.blockId, useDraftState: false, startTime: new Date().toISOString(), isClientSession: false, credentialAccountUserId, correlation, workflowStateOverride: { blocks, edges, loops: loops || {}, parallels: parallels || {}, deploymentVersionId, }, } const triggerInput = input || {} /** * Surface the pre-execution latency that per-block timings cannot see: the * gap between webhook receipt and the first block running, and — for * trigger_id-bound providers like Slack — the true age of the interaction * against its 3s expiry window. Logged structured so it is queryable/alarmable. */ if (payload.webhookReceivedAt !== undefined || payload.triggerTimestampMs !== undefined) { const now = Date.now() logger.info(`[${requestId}] Webhook dispatch latency`, { workflowId: payload.workflowId, provider: payload.provider, dispatchLatencyMs: payload.webhookReceivedAt !== undefined ? now - payload.webhookReceivedAt : undefined, triggerAgeMs: payload.triggerTimestampMs !== undefined ? now - payload.triggerTimestampMs : undefined, }) } const snapshot = new ExecutionSnapshot( metadata, workflowRecord, triggerInput, workflowVariables, [] ) const executionResult = await executeWorkflowCore({ snapshot, callbacks: {}, loggingSession, includeFileBase64: false, base64MaxBytes: undefined, abortSignal: timeoutController.signal, }) await handleExecutionResult(executionResult, { loggingSession, timeoutController, requestId, executionId, workflowId: payload.workflowId, }) logger.info(`[${requestId}] Webhook execution completed`, { success: executionResult.success, workflowId: payload.workflowId, provider: payload.provider, }) return { success: executionResult.success, workflowId: payload.workflowId, executionId, output: executionResult.output, executedAt: new Date().toISOString(), provider: payload.provider, } } catch (error: unknown) { const errorMessage = toError(error).message const errorStack = error instanceof Error ? error.stack : undefined logger.error(`[${requestId}] Webhook execution failed`, { error: errorMessage, stack: errorStack, workflowId: payload.workflowId, provider: payload.provider, }) // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the // signal is reliable and the failure is fully persisted before we decide fault vs error. await loggingSession.waitForPostExecution() // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.) // is finalized by core and already recorded in the execution logs. That is a user/workflow error, // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. if (wasExecutionFinalizedByCore(error, executionId)) { // Record the exception on the run span so it stays visible in traces without // marking the span as ERROR — that status is what faults the trigger.dev run. trace.getActiveSpan()?.recordException(toError(error)) return { success: false, workflowId: payload.workflowId, executionId, output: hasExecutionResult(error) ? error.executionResult.output : {}, executedAt: new Date().toISOString(), provider: payload.provider, } } try { await loggingSession.safeStart({ userId: payload.userId, workspaceId, variables: {}, triggerData: { isTest: false, correlation, }, deploymentVersionId, }) const executionResult = hasExecutionResult(error) ? error.executionResult : { success: false, output: {}, logs: [], } const { traceSpans } = buildTraceSpans(executionResult) await loggingSession.safeCompleteWithError({ endedAt: new Date().toISOString(), totalDurationMs: 0, error: { message: errorMessage || 'Webhook execution failed', stackTrace: errorStack, }, traceSpans, }) } catch (loggingError) { logger.error(`[${requestId}] Failed to complete logging session`, loggingError) } throw error } finally { timeoutController.cleanup() } } export const webhookExecution = task({ id: 'webhook-execution', machine: 'medium-1x', retry: { maxAttempts: 1, }, queue: { concurrencyLimit: WEBHOOK_EXECUTION_CONCURRENCY_LIMIT, }, run: async (payload: WebhookExecutionPayload) => executeWebhookJob(payload), })