chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,111 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
describe('readCanonicalTriggerValue', () => {
it('returns the canonical value when present', () => {
expect(readCanonicalTriggerValue('canon', 'basic', 'advanced')).toBe('canon')
})
it('falls back basic-first when the canonical key is absent (transitional)', () => {
expect(readCanonicalTriggerValue(undefined, 'basic', 'advanced')).toBe('basic')
})
it('falls back to the advanced value when canonical and basic are absent', () => {
expect(readCanonicalTriggerValue(undefined, undefined, 'advanced')).toBe('advanced')
})
it('treats empty strings as unset', () => {
expect(readCanonicalTriggerValue('', '', 'advanced')).toBe('advanced')
expect(readCanonicalTriggerValue('', '')).toBeUndefined()
})
it('ignores non-string members', () => {
expect(readCanonicalTriggerValue(null, 42 as unknown, 'advanced')).toBe('advanced')
expect(readCanonicalTriggerValue(undefined, undefined)).toBeUndefined()
})
/**
* No-repoint invariant: for every config shape the previous app version could
* have stored, the new canonical-first read must resolve to the SAME resource
* the old basic-first read returned — whether the row has been backfilled
* (canonical key present) or not (transitional fallback). The legacy
* basic-first read is reproduced inline as the source of truth.
*/
describe('no-repoint across legacy and backfilled shapes', () => {
interface GoogleDriveConfig {
folderId?: string
manualFolderId?: string
}
// The exact pre-change Google Drive poller read (basic key === canonical key).
const legacyDriveRead = (c: GoogleDriveConfig) => c.folderId || c.manualFolderId
// Mirror of the migration 0253 backfill: fill the canonical key basic-first
// ONLY when absent. For Drive the canonical key IS the basic key (folderId).
const backfillDrive = (c: GoogleDriveConfig): GoogleDriveConfig =>
c.folderId ? c : { ...c, folderId: c.folderId || c.manualFolderId }
const driveShapes: GoogleDriveConfig[] = [
{ folderId: 'basic-only' },
{ manualFolderId: 'advanced-only' },
// Drift: stale basic + active advanced both stored. Current poller reads basic.
{ folderId: 'STALE', manualFolderId: 'ACTIVE' },
{},
]
it.each(driveShapes)('Drive legacy config %o reads same resource as before', (config) => {
const before = legacyDriveRead(config)
const after = readCanonicalTriggerValue(config.folderId, config.manualFolderId)
expect(after).toBe(before || undefined)
})
it.each(driveShapes)('Drive backfilled config %o reads same resource as before', (config) => {
const before = legacyDriveRead(config)
const backfilled = backfillDrive(config)
const after = readCanonicalTriggerValue(backfilled.folderId, backfilled.manualFolderId)
expect(after).toBe(before || undefined)
})
interface TableConfig {
tableId?: string
tableSelector?: string
manualTableId?: string
}
// The exact pre-change table reader (already canonical-first, but tableId was
// never written, so it fell through to the raw keys).
const legacyTableRead = (c: TableConfig) => c.tableId ?? c.tableSelector ?? c.manualTableId
// Mirror of the migration 0253 backfill for table (canonical key is distinct).
const backfillTable = (c: TableConfig): TableConfig =>
c.tableId ? c : { ...c, tableId: c.tableSelector ?? c.manualTableId }
const tableShapes: TableConfig[] = [
{ tableSelector: 'basic-only' },
{ manualTableId: 'advanced-only' },
{ tableSelector: 'STALE', manualTableId: 'ACTIVE' },
{ tableId: 'collapsed', tableSelector: 'STALE', manualTableId: 'ACTIVE' },
{},
]
it.each(tableShapes)('table legacy config %o reads same resource as before', (config) => {
const before = legacyTableRead(config)
const after = readCanonicalTriggerValue(
config.tableId,
config.tableSelector,
config.manualTableId
)
expect(after).toBe(before ?? undefined)
})
it.each(tableShapes)('table backfilled config %o reads same resource as before', (config) => {
const before = legacyTableRead(config)
const backfilled = backfillTable(config)
const after = readCanonicalTriggerValue(
backfilled.tableId,
backfilled.tableSelector,
backfilled.manualTableId
)
expect(after).toBe(before ?? undefined)
})
})
})
@@ -0,0 +1,31 @@
/**
* Resolve the live resource id for a canonical trigger field from a deployed
* `webhook.providerConfig`.
*
* The canonical key — written at the deploy boundary by `buildProviderConfig`
* and populated on pre-existing rows by migration `0253` — is authoritative and
* is read first. The `transitionalFallback` values are read basic-first ONLY
* when the canonical key is absent, i.e. for configs deployed before the
* canonical key existed and not yet backfilled (including webhooks created by
* the previous app version during a deploy cutover).
*
* The fallback is TRANSITIONAL: once the backfill is confirmed and every
* deployed config carries the canonical key, it can be deleted in a follow-up
* contract phase and callers can read the canonical key alone. It exists solely
* for migration safety, not as permanent precedence logic.
*
* Empty strings are treated as unset (matching the previous basic-first `||`
* reads); returns `undefined` when no member is set.
*
* @param canonicalValue - value stored under the group's `canonicalParamId`
* @param transitionalFallback - the raw subblock values in basic-first order
*/
export function readCanonicalTriggerValue(
canonicalValue: unknown,
...transitionalFallback: unknown[]
): string | undefined {
for (const value of [canonicalValue, ...transitionalFallback]) {
if (typeof value === 'string' && value.length > 0) return value
}
return undefined
}
+595
View File
@@ -0,0 +1,595 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
import type { GmailAttachment } from '@/tools/gmail/types'
import { downloadAttachments, extractAttachmentInfo } from '@/tools/gmail/utils'
interface GmailWebhookConfig {
labelIds: string[]
labelFilterBehavior: 'INCLUDE' | 'EXCLUDE'
markAsRead: boolean
searchQuery?: string
maxEmailsPerPoll?: number
lastCheckedTimestamp?: string
historyId?: string
includeAttachments?: boolean
includeRawEmail?: boolean
}
interface GmailEmail {
id: string
threadId: string
historyId?: string
labelIds?: string[]
payload?: Record<string, unknown>
snippet?: string
internalDate?: string
}
interface SimplifiedEmail {
id: string
threadId: string
subject: string
from: string
to: string
cc: string
date: string | null
bodyText: string
bodyHtml: string
labels: string[]
hasAttachments: boolean
attachments: GmailAttachment[]
}
interface GmailWebhookPayload {
email: SimplifiedEmail
timestamp: string
rawEmail?: GmailEmail
}
export const gmailPollingHandler: PollingProviderHandler = {
provider: 'gmail',
label: 'Gmail',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-email', requestId)
const config = getProviderConfig<GmailWebhookConfig>(webhookData.providerConfig)
const now = new Date()
const { emails, latestHistoryId } = await fetchNewEmails(
accessToken,
config,
requestId,
logger
)
if (!emails || !emails.length) {
await updateWebhookProviderConfig(
webhookId,
{
lastCheckedTimestamp: now.toISOString(),
...(latestHistoryId || config.historyId
? { historyId: latestHistoryId || config.historyId }
: {}),
},
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
return 'success'
}
logger.info(`[${requestId}] Found ${emails.length} new emails for webhook ${webhookId}`)
const { processedCount, failedCount } = await processEmails(
emails,
webhookData,
workflowData,
config,
accessToken,
requestId,
logger
)
await updateWebhookProviderConfig(
webhookId,
{
lastCheckedTimestamp: now.toISOString(),
...(latestHistoryId || config.historyId
? { historyId: latestHistoryId || config.historyId }
: {}),
},
logger
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
logger.error(`[${requestId}] Error processing Gmail webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function fetchNewEmails(
accessToken: string,
config: GmailWebhookConfig,
requestId: string,
logger: Logger
) {
try {
const useHistoryApi = !!config.historyId
let emails: GmailEmail[] = []
let latestHistoryId = config.historyId
if (useHistoryApi) {
const messageIds = new Set<string>()
let pageToken: string | undefined
do {
let historyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/history?startHistoryId=${config.historyId}&historyTypes=messageAdded`
if (pageToken) {
historyUrl += `&pageToken=${pageToken}`
}
const historyResponse = await fetch(historyUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!historyResponse.ok) {
const status = historyResponse.status
const errorData = await historyResponse.json().catch(() => ({}))
logger.error(`[${requestId}] Gmail history API error:`, {
status,
statusText: historyResponse.statusText,
error: errorData,
})
if (status === 403 || status === 429) {
throw new Error(
`Gmail API error ${status} — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
)
}
logger.info(`[${requestId}] Falling back to search API after history API error ${status}`)
const searchResult = await searchEmails(accessToken, config, requestId, logger)
if (searchResult.emails.length === 0) {
const freshHistoryId = await getGmailProfileHistoryId(accessToken, requestId, logger)
if (freshHistoryId) {
logger.info(
`[${requestId}] Fetched fresh historyId ${freshHistoryId} after invalid historyId (was: ${config.historyId})`
)
return { emails: [], latestHistoryId: freshHistoryId }
}
}
return searchResult
}
const historyData = await historyResponse.json()
if (historyData.historyId) {
latestHistoryId = historyData.historyId
}
if (historyData.history) {
for (const history of historyData.history) {
if (history.messagesAdded) {
for (const messageAdded of history.messagesAdded) {
messageIds.add(messageAdded.message.id)
}
}
}
}
pageToken = historyData.nextPageToken
} while (pageToken)
if (!messageIds.size) {
return { emails: [], latestHistoryId }
}
const sortedIds = [...messageIds].sort().reverse()
const idsToFetch = sortedIds.slice(0, config.maxEmailsPerPoll || 25)
logger.info(`[${requestId}] Processing ${idsToFetch.length} emails from history API`)
const emailResults = await Promise.allSettled(
idsToFetch.map((messageId) => getEmailDetails(accessToken, messageId))
)
const rejected = emailResults.filter((r) => r.status === 'rejected')
if (rejected.length > 0) {
logger.warn(`[${requestId}] Failed to fetch ${rejected.length} email details`)
}
emails = emailResults
.filter(
(result): result is PromiseFulfilledResult<GmailEmail> => result.status === 'fulfilled'
)
.map((result) => result.value)
emails = filterEmailsByLabels(emails, config)
} else {
return searchEmails(accessToken, config, requestId, logger)
}
return { emails, latestHistoryId }
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error fetching new emails:`, errorMessage)
throw error
}
}
function buildGmailSearchQuery(config: {
labelIds?: string[]
labelFilterBehavior?: 'INCLUDE' | 'EXCLUDE'
searchQuery?: string
}): string {
let labelQuery = ''
if (config.labelIds && config.labelIds.length > 0) {
const labelParts = config.labelIds.map((label) => `label:${label}`).join(' OR ')
labelQuery =
config.labelFilterBehavior === 'INCLUDE'
? config.labelIds.length > 1
? `(${labelParts})`
: labelParts
: config.labelIds.length > 1
? `-(${labelParts})`
: `-${labelParts}`
}
let searchQueryPart = ''
if (config.searchQuery?.trim()) {
searchQueryPart = config.searchQuery.trim()
if (searchQueryPart.includes(' OR ') || searchQueryPart.includes(' AND ')) {
searchQueryPart = `(${searchQueryPart})`
}
}
let baseQuery = ''
if (labelQuery && searchQueryPart) {
baseQuery = `${labelQuery} ${searchQueryPart}`
} else if (searchQueryPart) {
baseQuery = searchQueryPart
} else if (labelQuery) {
baseQuery = labelQuery
} else {
baseQuery = 'in:inbox'
}
return baseQuery
}
async function searchEmails(
accessToken: string,
config: GmailWebhookConfig,
requestId: string,
logger: Logger
) {
try {
const baseQuery = buildGmailSearchQuery(config)
let timeConstraint = ''
if (config.lastCheckedTimestamp) {
const lastCheckedTime = new Date(config.lastCheckedTimestamp)
const now = new Date()
const minutesSinceLastCheck = (now.getTime() - lastCheckedTime.getTime()) / (60 * 1000)
if (minutesSinceLastCheck < 60) {
const bufferSeconds = Math.max(1 * 60 * 2, 180)
const cutoffTime = new Date(lastCheckedTime.getTime() - bufferSeconds * 1000)
const timestamp = Math.floor(cutoffTime.getTime() / 1000)
timeConstraint = ` after:${timestamp}`
} else if (minutesSinceLastCheck < 24 * 60) {
const hours = Math.ceil(minutesSinceLastCheck / 60) + 1
timeConstraint = ` newer_than:${hours}h`
} else {
const days = Math.min(Math.ceil(minutesSinceLastCheck / (24 * 60)), 7) + 1
timeConstraint = ` newer_than:${days}d`
}
} else {
timeConstraint = ' newer_than:1d'
}
const query = `${baseQuery}${timeConstraint}`
const searchUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${encodeURIComponent(query)}&maxResults=${config.maxEmailsPerPoll || 25}`
const searchResponse = await fetch(searchUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!searchResponse.ok) {
const errorData = await searchResponse.json()
logger.error(`[${requestId}] Gmail search API error:`, {
status: searchResponse.status,
statusText: searchResponse.statusText,
query,
error: errorData,
})
throw new Error(
`Gmail API error: ${searchResponse.status} ${searchResponse.statusText} - ${JSON.stringify(errorData)}`
)
}
const searchData = await searchResponse.json()
if (!searchData.messages || !searchData.messages.length) {
logger.info(`[${requestId}] No emails found matching query: ${query}`)
return { emails: [], latestHistoryId: config.historyId }
}
const idsToFetch = searchData.messages.slice(0, config.maxEmailsPerPoll || 25)
let latestHistoryId = config.historyId
logger.info(
`[${requestId}] Processing ${idsToFetch.length} emails from search API (total matches: ${searchData.messages.length})`
)
const emailResults = await Promise.allSettled(
idsToFetch.map((message: { id: string }) => getEmailDetails(accessToken, message.id))
)
const rejected = emailResults.filter((r) => r.status === 'rejected')
if (rejected.length > 0) {
logger.warn(`[${requestId}] Failed to fetch ${rejected.length} email details`)
}
const emails = emailResults
.filter(
(result): result is PromiseFulfilledResult<GmailEmail> => result.status === 'fulfilled'
)
.map((result) => result.value)
if (emails.length > 0 && emails[0].historyId) {
latestHistoryId = emails[0].historyId
}
return { emails, latestHistoryId }
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error searching emails:`, errorMessage)
throw error
}
}
async function getGmailProfileHistoryId(
accessToken: string,
requestId: string,
logger: Logger
): Promise<string | null> {
try {
const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
logger.warn(
`[${requestId}] Failed to fetch Gmail profile for fresh historyId: ${response.status}`
)
return null
}
const profile = await response.json()
return (profile.historyId as string | undefined) ?? null
} catch (error) {
logger.warn(`[${requestId}] Error fetching Gmail profile:`, error)
return null
}
}
async function getEmailDetails(accessToken: string, messageId: string): Promise<GmailEmail> {
const messageUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}?format=full`
const messageResponse = await fetch(messageUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!messageResponse.ok) {
const errorData = await messageResponse.json().catch(() => ({}))
throw new Error(
`Failed to fetch email details for message ${messageId}: ${messageResponse.status} ${messageResponse.statusText} - ${JSON.stringify(errorData)}`
)
}
return await messageResponse.json()
}
function filterEmailsByLabels(emails: GmailEmail[], config: GmailWebhookConfig): GmailEmail[] {
if (!config.labelIds.length) {
return emails
}
return emails.filter((email) => {
const emailLabels = email.labelIds || []
const hasMatchingLabel = config.labelIds.some((configLabel) =>
emailLabels.includes(configLabel)
)
return config.labelFilterBehavior === 'INCLUDE' ? hasMatchingLabel : !hasMatchingLabel
})
}
async function processEmails(
emails: GmailEmail[],
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
config: GmailWebhookConfig,
accessToken: string,
requestId: string,
logger: Logger
) {
let processedCount = 0
let failedCount = 0
for (const email of emails) {
try {
await pollingIdempotency.executeWithIdempotency(
'gmail',
`${webhookData.id}:${email.id}`,
async () => {
const headers: Record<string, string> = {}
const payload = email.payload as Record<string, unknown> | undefined
if (payload?.headers && Array.isArray(payload.headers)) {
for (const header of payload.headers as { name: string; value: string }[]) {
headers[header.name.toLowerCase()] = header.value
}
}
let textContent = ''
let htmlContent = ''
const extractContent = (part: Record<string, unknown>) => {
if (!part) return
if (part.mimeType === 'text/plain') {
const body = part.body as { data?: string } | undefined
if (body?.data) {
textContent = Buffer.from(body.data, 'base64').toString('utf-8')
}
} else if (part.mimeType === 'text/html') {
const body = part.body as { data?: string } | undefined
if (body?.data) {
htmlContent = Buffer.from(body.data, 'base64').toString('utf-8')
}
}
if (part.parts && Array.isArray(part.parts)) {
for (const subPart of part.parts) {
extractContent(subPart as Record<string, unknown>)
}
}
}
if (payload) {
extractContent(payload)
}
let date: string | null = null
if (headers.date) {
try {
date = new Date(headers.date).toISOString()
} catch (_e) {}
} else if (email.internalDate) {
date = new Date(Number.parseInt(email.internalDate)).toISOString()
}
let attachments: GmailAttachment[] = []
const hasAttachments = payload ? extractAttachmentInfo(payload).length > 0 : false
if (config.includeAttachments && hasAttachments && payload) {
try {
const attachmentInfo = extractAttachmentInfo(payload)
attachments = await downloadAttachments(email.id, attachmentInfo, accessToken)
} catch (error) {
logger.error(
`[${requestId}] Error downloading attachments for email ${email.id}:`,
error
)
}
}
const simplifiedEmail: SimplifiedEmail = {
id: email.id,
threadId: email.threadId,
subject: headers.subject || '[No Subject]',
from: headers.from || '',
to: headers.to || '',
cc: headers.cc || '',
date,
bodyText: textContent,
bodyHtml: htmlContent,
labels: email.labelIds || [],
hasAttachments,
attachments,
}
const webhookPayload: GmailWebhookPayload = {
email: simplifiedEmail,
timestamp: new Date().toISOString(),
...(config.includeRawEmail ? { rawEmail: email } : {}),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
webhookPayload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for email ${email.id}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
if (config.markAsRead) {
await markEmailAsRead(accessToken, email.id, logger)
}
return { emailId: email.id, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed email ${email.id} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing email ${email.id}:`, errorMessage)
failedCount++
}
}
return { processedCount, failedCount }
}
async function markEmailAsRead(accessToken: string, messageId: string, logger: Logger) {
const modifyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}/modify`
try {
const response = await fetch(modifyUrl, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ removeLabelIds: ['UNREAD'] }),
})
if (!response.ok) {
await response.body?.cancel().catch(() => {})
throw new Error(
`Failed to mark email ${messageId} as read: ${response.status} ${response.statusText}`
)
}
} catch (error) {
logger.error(`Error marking email ${messageId} as read:`, error)
throw error
}
}
@@ -0,0 +1,358 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3'
const MAX_EVENTS_PER_POLL = 50
const MAX_PAGES = 10
type CalendarEventTypeFilter = '' | 'created' | 'updated' | 'cancelled'
interface GoogleCalendarWebhookConfig {
calendarId?: string
manualCalendarId?: string
eventTypeFilter?: CalendarEventTypeFilter
searchTerm?: string
lastCheckedTimestamp?: string
maxEventsPerPoll?: number
}
interface CalendarEventAttendee {
email: string
displayName?: string
responseStatus?: string
self?: boolean
organizer?: boolean
}
interface CalendarEventPerson {
email: string
displayName?: string
self?: boolean
}
interface CalendarEventTime {
dateTime?: string
date?: string
timeZone?: string
}
interface CalendarEvent {
id: string
status: string
htmlLink?: string
created?: string
updated?: string
summary?: string
description?: string
location?: string
start?: CalendarEventTime
end?: CalendarEventTime
attendees?: CalendarEventAttendee[]
creator?: CalendarEventPerson
organizer?: CalendarEventPerson
recurringEventId?: string
}
interface SimplifiedCalendarEvent {
id: string
status: string
eventType: 'created' | 'updated' | 'cancelled'
summary: string | null
eventDescription: string | null
location: string | null
htmlLink: string | null
start: CalendarEventTime | null
end: CalendarEventTime | null
created: string | null
updated: string | null
attendees: CalendarEventAttendee[] | null
creator: CalendarEventPerson | null
organizer: CalendarEventPerson | null
}
interface GoogleCalendarWebhookPayload {
event: SimplifiedCalendarEvent
calendarId: string
timestamp: string
}
export const googleCalendarPollingHandler: PollingProviderHandler = {
provider: 'google-calendar',
label: 'Google Calendar',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-calendar', requestId)
const config = getProviderConfig<GoogleCalendarWebhookConfig>(webhookData.providerConfig)
// Canonical key `calendarId` first; `manualCalendarId` is a transitional basic-first
// fallback. Defaults to the user's primary calendar when none is set.
const calendarId =
readCanonicalTriggerValue(config.calendarId, config.manualCalendarId) || 'primary'
// First poll: seed timestamp, emit nothing
if (!config.lastCheckedTimestamp) {
await updateWebhookProviderConfig(
webhookId,
{ lastCheckedTimestamp: new Date(Date.now() - 30_000).toISOString() },
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] First poll for webhook ${webhookId}, seeded timestamp`)
return 'success'
}
// Fetch changed events since last poll
const events = await fetchChangedEvents(accessToken, calendarId, config, requestId, logger)
if (!events.length) {
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No changed events for webhook ${webhookId}`)
return 'success'
}
logger.info(`[${requestId}] Found ${events.length} changed events for webhook ${webhookId}`)
const { processedCount, failedCount, latestUpdated } = await processEvents(
events,
calendarId,
config.eventTypeFilter,
webhookData,
workflowData,
requestId,
logger
)
// Advance cursor to latestUpdated - 5s for clock-skew overlap, but never regress
// below the previous cursor — this prevents an infinite re-fetch loop when all
// returned events are filtered client-side and latestUpdated is within 5s of the cursor.
const newTimestamp =
failedCount > 0
? config.lastCheckedTimestamp
: latestUpdated
? new Date(
Math.max(
new Date(latestUpdated).getTime() - 5000,
new Date(config.lastCheckedTimestamp).getTime()
)
).toISOString()
: config.lastCheckedTimestamp
await updateWebhookProviderConfig(webhookId, { lastCheckedTimestamp: newTimestamp }, logger)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} events failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} events for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
logger.error(`[${requestId}] Error processing Google Calendar webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function fetchChangedEvents(
accessToken: string,
calendarId: string,
config: GoogleCalendarWebhookConfig,
requestId: string,
logger: Logger
): Promise<CalendarEvent[]> {
const allEvents: CalendarEvent[] = []
const maxEvents = config.maxEventsPerPoll || MAX_EVENTS_PER_POLL
let pageToken: string | undefined
let pages = 0
do {
pages++
const params = new URLSearchParams({
updatedMin: config.lastCheckedTimestamp!,
singleEvents: 'true',
showDeleted: 'true',
maxResults: String(Math.min(maxEvents, 250)),
})
if (pageToken) {
params.set('pageToken', pageToken)
}
if (config.searchTerm) {
params.set('q', config.searchTerm)
}
const encodedCalendarId = encodeURIComponent(calendarId)
const url = `${CALENDAR_API_BASE}/calendars/${encodedCalendarId}/events?${params.toString()}`
const response = await fetch(url, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const status = response.status
const errorData = await response.json().catch(() => ({}))
if (status === 403 || status === 429) {
throw new Error(
`Calendar API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
)
}
throw new Error(`Failed to fetch calendar events: ${status} - ${JSON.stringify(errorData)}`)
}
const data = await response.json()
const events = (data.items || []) as CalendarEvent[]
allEvents.push(...events)
pageToken = data.nextPageToken as string | undefined
// Stop if we have enough events or hit the page limit
if (allEvents.length >= maxEvents || pages >= MAX_PAGES) {
break
}
} while (pageToken)
return allEvents.slice(0, maxEvents)
}
function determineEventType(event: CalendarEvent): 'created' | 'updated' | 'cancelled' {
if (event.status === 'cancelled') {
return 'cancelled'
}
// If created and updated are within 5 seconds, treat as newly created
if (event.created && event.updated) {
const createdTime = new Date(event.created).getTime()
const updatedTime = new Date(event.updated).getTime()
if (Math.abs(updatedTime - createdTime) < 5000) {
return 'created'
}
}
return 'updated'
}
function simplifyEvent(
event: CalendarEvent,
eventType?: 'created' | 'updated' | 'cancelled'
): SimplifiedCalendarEvent {
return {
id: event.id,
status: event.status,
eventType: eventType ?? determineEventType(event),
summary: event.summary ?? null,
eventDescription: event.description ?? null,
location: event.location ?? null,
htmlLink: event.htmlLink ?? null,
start: event.start ?? null,
end: event.end ?? null,
created: event.created ?? null,
updated: event.updated ?? null,
attendees: event.attendees ?? null,
creator: event.creator ?? null,
organizer: event.organizer ?? null,
}
}
async function processEvents(
events: CalendarEvent[],
calendarId: string,
eventTypeFilter: CalendarEventTypeFilter | undefined,
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
requestId: string,
logger: Logger
): Promise<{ processedCount: number; failedCount: number; latestUpdated: string | null }> {
let processedCount = 0
let failedCount = 0
let latestUpdated: string | null = null
for (const event of events) {
// Track the latest `updated` timestamp for clock-skew-free state tracking
if (event.updated) {
if (!latestUpdated || event.updated > latestUpdated) {
latestUpdated = event.updated
}
}
// Client-side event type filter — skip before idempotency so filtered events aren't cached
const computedEventType = determineEventType(event)
if (eventTypeFilter && computedEventType !== eventTypeFilter) {
continue
}
try {
// Idempotency key includes `updated` so re-edits of the same event re-trigger
const idempotencyKey = `${webhookData.id}:${event.id}:${event.updated || event.created || ''}`
await pollingIdempotency.executeWithIdempotency(
'google-calendar',
idempotencyKey,
async () => {
const simplified = simplifyEvent(event, computedEventType)
const payload: GoogleCalendarWebhookPayload = {
event: simplified,
calendarId,
timestamp: new Date().toISOString(),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for event ${event.id}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
return { eventId: event.id, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed event ${event.id} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing event ${event.id}:`, errorMessage)
failedCount++
}
}
return { processedCount, failedCount, latestUpdated }
}
@@ -0,0 +1,435 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
const MAX_FILES_PER_POLL = 50
const MAX_KNOWN_FILE_IDS = 1000
const MAX_PAGES = 10
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3'
type DriveEventTypeFilter = '' | 'created' | 'modified' | 'deleted' | 'created_or_modified'
interface GoogleDriveWebhookConfig {
folderId?: string
manualFolderId?: string
mimeTypeFilter?: string
includeSharedDrives?: boolean
eventTypeFilter?: DriveEventTypeFilter
maxFilesPerPoll?: number
pageToken?: string
knownFileIds?: string[]
}
interface DriveChangeEntry {
kind: string
type: string
time: string
removed: boolean
fileId: string
file?: DriveFileMetadata
}
interface DriveFileMetadata {
id: string
name: string
mimeType: string
modifiedTime: string
createdTime?: string
size?: string
webViewLink?: string
parents?: string[]
lastModifyingUser?: { displayName?: string; emailAddress?: string }
shared?: boolean
starred?: boolean
trashed?: boolean
}
interface GoogleDriveWebhookPayload {
file: DriveFileMetadata | { id: string }
eventType: 'created' | 'modified' | 'deleted'
timestamp: string
}
const FILE_FIELDS = [
'id',
'name',
'mimeType',
'modifiedTime',
'createdTime',
'size',
'webViewLink',
'parents',
'lastModifyingUser',
'shared',
'starred',
'trashed',
].join(',')
export const googleDrivePollingHandler: PollingProviderHandler = {
provider: 'google-drive',
label: 'Google Drive',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-drive', requestId)
const config = getProviderConfig<GoogleDriveWebhookConfig>(webhookData.providerConfig)
// First poll (or re-seed after 410): seed page token, preserve any existing known file IDs.
if (!config.pageToken) {
const startPageToken = await getStartPageToken(accessToken, config, requestId, logger)
await updateWebhookProviderConfig(
webhookId,
{ pageToken: startPageToken, knownFileIds: config.knownFileIds ?? [] },
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] First poll for webhook ${webhookId}, seeded pageToken: ${startPageToken}`
)
return 'success'
}
const { changes, newStartPageToken } = await fetchChanges(
accessToken,
config,
requestId,
logger
)
if (!changes.length) {
await updateWebhookProviderConfig(webhookId, { pageToken: newStartPageToken }, logger)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No changes found for webhook ${webhookId}`)
return 'success'
}
const filteredChanges = filterChanges(changes, config)
if (!filteredChanges.length) {
await updateWebhookProviderConfig(webhookId, { pageToken: newStartPageToken }, logger)
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] ${changes.length} changes found but none match filters for webhook ${webhookId}`
)
return 'success'
}
logger.info(
`[${requestId}] Found ${filteredChanges.length} matching changes for webhook ${webhookId}`
)
const { processedCount, failedCount, newKnownFileIds } = await processChanges(
filteredChanges,
config,
webhookData,
workflowData,
requestId,
logger
)
const existingKnownIds = config.knownFileIds || []
const mergedKnownIds = [...new Set([...newKnownFileIds, ...existingKnownIds])].slice(
0,
MAX_KNOWN_FILE_IDS
)
const anyFailed = failedCount > 0
await updateWebhookProviderConfig(
webhookId,
{
pageToken: anyFailed ? config.pageToken : newStartPageToken,
knownFileIds: anyFailed ? existingKnownIds : mergedKnownIds,
},
logger
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} changes failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} changes for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
if (error instanceof Error && error.name === 'DrivePageTokenInvalidError') {
await updateWebhookProviderConfig(webhookId, { pageToken: undefined }, logger)
await markWebhookSuccess(webhookId, logger)
logger.warn(
`[${requestId}] Drive page token invalid for webhook ${webhookId}, re-seeding on next poll`
)
return 'success'
}
if (error instanceof Error && error.name === 'DriveRateLimitError') {
await markWebhookSuccess(webhookId, logger)
logger.warn(
`[${requestId}] Drive API rate limited for webhook ${webhookId}, skipping to retry next poll cycle`
)
return 'success'
}
logger.error(`[${requestId}] Error processing Google Drive webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
const DRIVE_RATE_LIMIT_REASONS = new Set(['rateLimitExceeded', 'userRateLimitExceeded'])
/** Returns true only for quota/rate-limit 403s, not permission errors. */
function isDriveRateLimitError(status: number, errorData: Record<string, unknown>): boolean {
if (status !== 403) return false
const reason = (errorData as { error?: { errors?: { reason?: string }[] } })?.error?.errors?.[0]
?.reason
return reason !== undefined && DRIVE_RATE_LIMIT_REASONS.has(reason)
}
async function getStartPageToken(
accessToken: string,
config: GoogleDriveWebhookConfig,
requestId: string,
logger: Logger
): Promise<string> {
const params = new URLSearchParams()
if (config.includeSharedDrives) {
params.set('supportsAllDrives', 'true')
}
const url = `${DRIVE_API_BASE}/changes/startPageToken?${params.toString()}`
const response = await fetch(url, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const status = response.status
const errorData = await response.json().catch(() => ({}))
if (status === 429 || isDriveRateLimitError(status, errorData)) {
const err = new Error(`Drive API rate limit (${status}): ${JSON.stringify(errorData)}`)
err.name = 'DriveRateLimitError'
throw err
}
throw new Error(
`Failed to get Drive start page token: ${status} - ${JSON.stringify(errorData)}`
)
}
const data = await response.json()
return data.startPageToken as string
}
async function fetchChanges(
accessToken: string,
config: GoogleDriveWebhookConfig,
requestId: string,
logger: Logger
): Promise<{ changes: DriveChangeEntry[]; newStartPageToken: string }> {
const allChanges: DriveChangeEntry[] = []
let currentPageToken = config.pageToken!
let newStartPageToken: string | undefined
let lastNextPageToken: string | undefined
const maxFiles = config.maxFilesPerPoll || MAX_FILES_PER_POLL
let pages = 0
while (true) {
pages++
const params = new URLSearchParams({
pageToken: currentPageToken,
pageSize: String(Math.min(maxFiles, 100)),
fields: `nextPageToken,newStartPageToken,changes(kind,type,time,removed,fileId,file(${FILE_FIELDS}))`,
restrictToMyDrive: config.includeSharedDrives ? 'false' : 'true',
})
if (config.includeSharedDrives) {
params.set('supportsAllDrives', 'true')
params.set('includeItemsFromAllDrives', 'true')
}
const url = `${DRIVE_API_BASE}/changes?${params.toString()}`
const response = await fetch(url, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const status = response.status
const errorData = await response.json().catch(() => ({}))
if (status === 410) {
const err = new Error('Drive page token is no longer valid')
err.name = 'DrivePageTokenInvalidError'
throw err
}
if (status === 429 || isDriveRateLimitError(status, errorData)) {
const err = new Error(`Drive API rate limit (${status}): ${JSON.stringify(errorData)}`)
err.name = 'DriveRateLimitError'
throw err
}
throw new Error(`Failed to fetch Drive changes: ${status} - ${JSON.stringify(errorData)}`)
}
const data = await response.json()
const changes = (data.changes || []) as DriveChangeEntry[]
allChanges.push(...changes)
if (data.newStartPageToken) {
newStartPageToken = data.newStartPageToken as string
}
const hasMore = !!data.nextPageToken
const overLimit = allChanges.length >= maxFiles
if (!hasMore || overLimit || pages >= MAX_PAGES) {
if (hasMore) {
lastNextPageToken = data.nextPageToken as string
}
break
}
lastNextPageToken = data.nextPageToken as string
currentPageToken = data.nextPageToken as string
}
// When allChanges exceeds maxFiles (multi-page overshoot), resume mid-list via lastNextPageToken.
// Otherwise resume from newStartPageToken (end of change list) or lastNextPageToken (MAX_PAGES hit).
const slicingOccurs = allChanges.length > maxFiles
const resumeToken = slicingOccurs
? (lastNextPageToken ?? newStartPageToken!)
: (newStartPageToken ?? lastNextPageToken!)
return { changes: allChanges.slice(0, maxFiles), newStartPageToken: resumeToken }
}
function filterChanges(
changes: DriveChangeEntry[],
config: GoogleDriveWebhookConfig
): DriveChangeEntry[] {
return changes.filter((change) => {
if (change.removed) return true
const file = change.file
if (!file) return false
if (file.trashed) return false
// Canonical key `folderId` first; `manualFolderId` is a transitional basic-first fallback.
const folderId = readCanonicalTriggerValue(config.folderId, config.manualFolderId)
if (folderId) {
if (!file.parents || !file.parents.includes(folderId)) {
return false
}
}
if (config.mimeTypeFilter) {
if (config.mimeTypeFilter.endsWith('/')) {
if (!file.mimeType.startsWith(config.mimeTypeFilter)) {
return false
}
} else if (file.mimeType !== config.mimeTypeFilter) {
return false
}
}
return true
})
}
async function processChanges(
changes: DriveChangeEntry[],
config: GoogleDriveWebhookConfig,
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
requestId: string,
logger: Logger
): Promise<{ processedCount: number; failedCount: number; newKnownFileIds: string[] }> {
let processedCount = 0
let failedCount = 0
const newKnownFileIds: string[] = []
const knownFileIdsSet = new Set(config.knownFileIds || [])
for (const change of changes) {
let eventType: 'created' | 'modified' | 'deleted'
if (change.removed) {
eventType = 'deleted'
} else if (!knownFileIdsSet.has(change.fileId)) {
eventType = 'created'
} else {
eventType = 'modified'
}
// Track file as known regardless of filter so future changes are correctly classified
if (!change.removed) {
newKnownFileIds.push(change.fileId)
}
// Apply event type filter before idempotency so filtered events aren't cached
const filter = config.eventTypeFilter
if (filter) {
const skip = filter === 'created_or_modified' ? eventType === 'deleted' : eventType !== filter
if (skip) continue
}
try {
const idempotencyKey = `${webhookData.id}:${change.fileId}:${change.time || change.fileId}`
await pollingIdempotency.executeWithIdempotency('google-drive', idempotencyKey, async () => {
const payload: GoogleDriveWebhookPayload = {
file: change.file || { id: change.fileId },
eventType,
timestamp: new Date().toISOString(),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for file ${change.fileId}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
return { fileId: change.fileId, processed: true }
})
logger.info(
`[${requestId}] Successfully processed change for file ${change.fileId} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(
`[${requestId}] Error processing change for file ${change.fileId}:`,
errorMessage
)
failedCount++
}
}
return { processedCount, failedCount, newKnownFileIds }
}
@@ -0,0 +1,468 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
const MAX_ROWS_PER_POLL = 100
/** Maximum number of leading rows to scan when auto-detecting the header row. */
const HEADER_SCAN_ROWS = 10
type ValueRenderOption = 'FORMATTED_VALUE' | 'UNFORMATTED_VALUE' | 'FORMULA'
type DateTimeRenderOption = 'SERIAL_NUMBER' | 'FORMATTED_STRING'
interface GoogleSheetsWebhookConfig {
spreadsheetId?: string
manualSpreadsheetId?: string
sheetName?: string
manualSheetName?: string
valueRenderOption?: ValueRenderOption
dateTimeRenderOption?: DateTimeRenderOption
/** 1-indexed row number of the last row seeded or processed. */
lastIndexChecked?: number
lastModifiedTime?: string
lastCheckedTimestamp?: string
maxRowsPerPoll?: number
}
interface GoogleSheetsWebhookPayload {
row: Record<string, string> | null
rawRow: string[]
headers: string[]
rowNumber: number
spreadsheetId: string
sheetName: string
timestamp: string
}
export const googleSheetsPollingHandler: PollingProviderHandler = {
provider: 'google-sheets',
label: 'Google Sheets',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-sheets', requestId)
const config = getProviderConfig<GoogleSheetsWebhookConfig>(webhookData.providerConfig)
// Canonical keys (`spreadsheetId`/`sheetName`) first; the `manual*` keys are a transitional
// basic-first fallback for configs deployed before the canonical key existed.
const spreadsheetId = readCanonicalTriggerValue(
config.spreadsheetId,
config.manualSpreadsheetId
)
const sheetName = readCanonicalTriggerValue(config.sheetName, config.manualSheetName)
const now = new Date()
if (!spreadsheetId || !sheetName) {
logger.error(`[${requestId}] Missing spreadsheetId or sheetName for webhook ${webhookId}`)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
const { unchanged: skipPoll, currentModifiedTime } = await isDriveFileUnchanged(
accessToken,
spreadsheetId,
config.lastModifiedTime,
requestId,
logger
)
if (skipPoll) {
await updateWebhookProviderConfig(
webhookId,
{ lastCheckedTimestamp: now.toISOString() },
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] Sheet not modified since last poll for webhook ${webhookId}`)
return 'success'
}
const valueRender = config.valueRenderOption || 'FORMATTED_VALUE'
const dateTimeRender = config.dateTimeRenderOption || 'SERIAL_NUMBER'
const {
rowCount: currentRowCount,
headers,
headerRowIndex,
} = await fetchSheetState(
accessToken,
spreadsheetId,
sheetName,
valueRender,
dateTimeRender,
requestId,
logger
)
// First poll: seed state, emit nothing
if (config.lastIndexChecked === undefined) {
await updateWebhookProviderConfig(
webhookId,
{
lastIndexChecked: currentRowCount,
lastModifiedTime: currentModifiedTime ?? config.lastModifiedTime,
lastCheckedTimestamp: now.toISOString(),
},
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] First poll for webhook ${webhookId}, seeded row index: ${currentRowCount}`
)
return 'success'
}
if (currentRowCount <= config.lastIndexChecked) {
if (currentRowCount < config.lastIndexChecked) {
logger.warn(
`[${requestId}] Row count decreased from ${config.lastIndexChecked} to ${currentRowCount} for webhook ${webhookId}`
)
}
await updateWebhookProviderConfig(
webhookId,
{
lastIndexChecked: currentRowCount,
lastModifiedTime: currentModifiedTime ?? config.lastModifiedTime,
lastCheckedTimestamp: now.toISOString(),
},
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new rows for webhook ${webhookId}`)
return 'success'
}
const newRowCount = currentRowCount - config.lastIndexChecked
const maxRows = config.maxRowsPerPoll || MAX_ROWS_PER_POLL
const rowsToFetch = Math.min(newRowCount, maxRows)
const startRow = config.lastIndexChecked + 1
const endRow = config.lastIndexChecked + rowsToFetch
// Skip past the header row (and any blank rows above it) so it is never
// emitted as a data event.
const adjustedStartRow =
headerRowIndex > 0 ? Math.max(startRow, headerRowIndex + 1) : startRow
logger.info(
`[${requestId}] Found ${newRowCount} new rows for webhook ${webhookId}, processing rows ${adjustedStartRow}-${endRow}`
)
// Entire batch is header/blank rows — advance pointer and skip fetch.
if (adjustedStartRow > endRow) {
const hasRemainingRows = rowsToFetch < newRowCount
await updateWebhookProviderConfig(
webhookId,
{
lastIndexChecked: config.lastIndexChecked + rowsToFetch,
lastModifiedTime: hasRemainingRows
? config.lastModifiedTime
: (currentModifiedTime ?? config.lastModifiedTime),
lastCheckedTimestamp: now.toISOString(),
},
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Batch ${startRow}-${endRow} contained only header/blank rows for webhook ${webhookId}, advancing pointer`
)
return 'success'
}
const newRows = await fetchRowRange(
accessToken,
spreadsheetId,
sheetName,
adjustedStartRow,
endRow,
valueRender,
dateTimeRender,
requestId,
logger
)
const { processedCount, failedCount } = await processRows(
newRows,
headers,
adjustedStartRow,
spreadsheetId,
sheetName,
webhookData,
workflowData,
requestId,
logger
)
const rowsAdvanced = failedCount > 0 ? 0 : rowsToFetch
const newLastIndexChecked = config.lastIndexChecked + rowsAdvanced
const hasRemainingOrFailed = rowsAdvanced < newRowCount
await updateWebhookProviderConfig(
webhookId,
{
lastIndexChecked: newLastIndexChecked,
lastModifiedTime: hasRemainingOrFailed
? config.lastModifiedTime
: (currentModifiedTime ?? config.lastModifiedTime),
lastCheckedTimestamp: now.toISOString(),
},
logger
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} rows failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} rows for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
logger.error(`[${requestId}] Error processing Google Sheets webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function isDriveFileUnchanged(
accessToken: string,
spreadsheetId: string,
lastModifiedTime: string | undefined,
requestId: string,
logger: Logger
): Promise<{ unchanged: boolean; currentModifiedTime?: string }> {
try {
const currentModifiedTime = await getDriveFileModifiedTime(accessToken, spreadsheetId, logger)
if (!lastModifiedTime || !currentModifiedTime) {
return { unchanged: false, currentModifiedTime }
}
return { unchanged: currentModifiedTime === lastModifiedTime, currentModifiedTime }
} catch (error) {
logger.warn(`[${requestId}] Drive modifiedTime check failed, proceeding with Sheets API`)
return { unchanged: false }
}
}
async function getDriveFileModifiedTime(
accessToken: string,
fileId: string,
logger: Logger
): Promise<string | undefined> {
try {
const response = await fetch(
`https://www.googleapis.com/drive/v3/files/${fileId}?fields=modifiedTime`,
{ headers: { Authorization: `Bearer ${accessToken}` } }
)
if (!response.ok) return undefined
const data = await response.json()
return data.modifiedTime as string | undefined
} catch {
return undefined
}
}
/**
* Fetches the sheet (A:Z) and returns the row count, auto-detected headers,
* and the 1-indexed header row number in a single API call.
*
* The Sheets API omits trailing empty rows, so `rows.length` equals the last
* non-empty row in columns AZ. Header detection scans the first
* {@link HEADER_SCAN_ROWS} rows for the first non-empty row. Returns
* `headerRowIndex = 0` when no header is found within the scan window.
*/
async function fetchSheetState(
accessToken: string,
spreadsheetId: string,
sheetName: string,
valueRenderOption: ValueRenderOption,
dateTimeRenderOption: DateTimeRenderOption,
requestId: string,
logger: Logger
): Promise<{ rowCount: number; headers: string[]; headerRowIndex: number }> {
const encodedSheet = encodeURIComponent(sheetName)
const params = new URLSearchParams({
majorDimension: 'ROWS',
fields: 'values',
valueRenderOption,
dateTimeRenderOption,
})
const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!A:Z?${params.toString()}`
const response = await fetch(url, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const status = response.status
const errorData = await response.json().catch(() => ({}))
if (status === 403 || status === 429) {
throw new Error(
`Sheets API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
)
}
throw new Error(
`Failed to fetch sheet state: ${status} ${response.statusText} - ${JSON.stringify(errorData)}`
)
}
const data = await response.json()
const rows = (data.values as string[][] | undefined) ?? []
const rowCount = rows.length
let headers: string[] = []
let headerRowIndex = 0
for (let i = 0; i < Math.min(rows.length, HEADER_SCAN_ROWS); i++) {
const row = rows[i]
if (row?.some((cell) => cell !== '')) {
headers = row
headerRowIndex = i + 1
break
}
}
return { rowCount, headers, headerRowIndex }
}
async function fetchRowRange(
accessToken: string,
spreadsheetId: string,
sheetName: string,
startRow: number,
endRow: number,
valueRenderOption: ValueRenderOption,
dateTimeRenderOption: DateTimeRenderOption,
requestId: string,
logger: Logger
): Promise<string[][]> {
const encodedSheet = encodeURIComponent(sheetName)
const params = new URLSearchParams({
fields: 'values',
valueRenderOption,
dateTimeRenderOption,
})
const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!${startRow}:${endRow}?${params.toString()}`
const response = await fetch(url, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const status = response.status
const errorData = await response.json().catch(() => ({}))
if (status === 403 || status === 429) {
throw new Error(
`Sheets API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
)
}
throw new Error(
`Failed to fetch rows ${startRow}-${endRow}: ${status} ${response.statusText} - ${JSON.stringify(errorData)}`
)
}
const data = await response.json()
return (data.values as string[][]) ?? []
}
async function processRows(
rows: string[][],
headers: string[],
startRowIndex: number,
spreadsheetId: string,
sheetName: string,
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
requestId: string,
logger: Logger
): Promise<{ processedCount: number; failedCount: number }> {
let processedCount = 0
let failedCount = 0
for (let i = 0; i < rows.length; i++) {
const row = rows[i]
const rowNumber = startRowIndex + i
// Skip empty rows — don't fire a workflow run with no data.
if (!row || row.length === 0) {
logger.info(`[${requestId}] Skipping empty row ${rowNumber} for webhook ${webhookData.id}`)
continue
}
try {
await pollingIdempotency.executeWithIdempotency(
'google-sheets',
`${webhookData.id}:${spreadsheetId}:${sheetName}:row${rowNumber}`,
async () => {
let mappedRow: Record<string, string> | null = null
if (headers.length > 0) {
mappedRow = {}
for (let j = 0; j < headers.length; j++) {
mappedRow[headers[j] || `Column ${j + 1}`] = row[j] ?? ''
}
for (let j = headers.length; j < row.length; j++) {
mappedRow[`Column ${j + 1}`] = row[j] ?? ''
}
}
const payload: GoogleSheetsWebhookPayload = {
row: mappedRow,
rawRow: row,
headers,
rowNumber,
spreadsheetId,
sheetName,
timestamp: new Date().toISOString(),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for row ${rowNumber}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
return { rowNumber, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed row ${rowNumber} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing row ${rowNumber}:`, errorMessage)
failedCount++
}
}
return { processedCount, failedCount }
}
@@ -0,0 +1,95 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { buildUserFilters } from '@/lib/webhooks/polling/hubspot'
describe('buildUserFilters', () => {
it('translates pipeline/stage/owner shortcuts into EQ filters', () => {
const filters = buildUserFilters({
objectType: 'deal',
pipelineId: 'pipeline-1',
stageId: 'stage-1',
ownerId: 'owner-1',
})
expect(filters).toEqual([
{ propertyName: 'pipeline', operator: 'EQ', value: 'pipeline-1' },
{ propertyName: 'dealstage', operator: 'EQ', value: 'stage-1' },
{ propertyName: 'hubspot_owner_id', operator: 'EQ', value: 'owner-1' },
])
})
it('uses ticket-specific pipeline/stage property names', () => {
const filters = buildUserFilters({
objectType: 'ticket',
pipelineId: 'pipeline-1',
stageId: 'stage-1',
})
expect(filters).toEqual([
{ propertyName: 'hs_pipeline', operator: 'EQ', value: 'pipeline-1' },
{ propertyName: 'hs_pipeline_stage', operator: 'EQ', value: 'stage-1' },
])
})
it('parses advanced JSON filters and preserves values arrays', () => {
const filters = buildUserFilters({
filters: JSON.stringify([
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
{ propertyName: 'dealstage', operator: 'IN', values: ['a', 'b'] },
]),
})
expect(filters).toEqual([
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
{ propertyName: 'dealstage', operator: 'IN', values: ['a', 'b'] },
])
})
it('drops filter entries with an unrecognized operator', () => {
const filters = buildUserFilters({
filters: JSON.stringify([
{ propertyName: 'amount', operator: 'STARTS_WITH', value: '1' },
{ propertyName: 'amount', operator: 'GT', value: '1' },
]),
})
expect(filters).toEqual([{ propertyName: 'amount', operator: 'GT', value: '1' }])
})
it('ignores malformed JSON filters without throwing', () => {
expect(() => buildUserFilters({ filters: 'not json' })).not.toThrow()
expect(buildUserFilters({ filters: 'not json' })).toEqual([])
})
it('allows exactly the HubSpot per-group limit of combined filters', () => {
const filters = buildUserFilters({
objectType: 'deal',
pipelineId: 'pipeline-1',
stageId: 'stage-1',
ownerId: 'owner-1',
filters: JSON.stringify([{ propertyName: 'amount', operator: 'GT', value: '1000' }]),
})
// 3 shortcuts + 1 advanced = 4, exactly MAX_USER_FILTERS.
expect(filters).toHaveLength(4)
})
it('throws rather than silently dropping filters when the combined count exceeds the limit', () => {
// Filters within a filterGroup are AND-combined, so silently dropping one would widen
// the match set instead of narrowing it — throwing surfaces the misconfiguration loudly.
expect(() =>
buildUserFilters({
objectType: 'deal',
pipelineId: 'pipeline-1',
stageId: 'stage-1',
ownerId: 'owner-1',
filters: JSON.stringify([
{ propertyName: 'amount', operator: 'GT', value: '1000' },
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
]),
})
).toThrow(/exceeding the 4-filter limit/)
})
})
+990
View File
@@ -0,0 +1,990 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
type HubSpotBuiltInObjectType = 'contact' | 'company' | 'deal' | 'ticket'
type HubSpotEventType = 'created' | 'updated' | 'property_changed'
interface FilterClause {
propertyName: string
operator: string
value?: string
values?: string[]
}
interface HubSpotWebhookConfig {
credentialId?: string
/**
* Built-in slug, 'custom' (defers to customObjectTypeId), 'list_membership',
* or a raw HubSpot custom object type id ('2-12345').
*/
objectType?: string
customObjectTypeId?: string
listId?: string
eventType?: HubSpotEventType
targetPropertyName?: string
properties?: string[] | string
pipelineId?: string
stageId?: string
ownerId?: string
/** User-supplied AND-combined filters — string list or JSON-array string. */
filters?: string | FilterClause[]
maxRecordsPerPoll?: number
lastSeenTimestampMs?: string
lastSeenObjectId?: string
/**
* List-membership cursor — the `after` value to pass to the next
* `/lists/{id}/memberships/join-order` request. The HubSpot endpoint walks
* ASC by default (oldest first); we use cursor-based resume because
* `before`-mode (DESC) has ambiguous bootstrap semantics.
*/
lastSeenMembershipCursor?: string
/**
* True once we've walked to the end of the list once. While this is false we
* are still in the seed pass — we paginate forward without emitting so the
* workflow doesn't see a flood of historical members on activation.
*/
membershipSeedComplete?: boolean
/**
* Snapshot of the watched property's last-seen value per record (property_changed event).
* Persisted as an entries array (not a `Record`) because HubSpot record ids are numeric
* strings, and JS engines enumerate integer-indexed object keys in numeric order
* regardless of insertion — which would break LRU-style trimming. Array order is stable.
*/
propertySnapshot?: {
property: string
/** `[recordId, value]` pairs in LRU order — oldest first. */
entries: Array<[string, string | null]>
}
lastCheckedTimestamp?: string
}
/**
* In-memory snapshot state used during a single poll. Uses a Map (not a plain object) so
* insertion-order iteration is honored even for numeric-string keys.
*/
interface PropertySnapshotState {
property: string
values: Map<string, string | null>
}
interface HubSpotSearchResult {
id: string
properties: Record<string, string | null>
createdAt: string
updatedAt: string
archived: boolean
}
interface HubSpotSearchResponse {
total: number
results: HubSpotSearchResult[]
paging?: { next?: { after?: string } }
}
const HUBSPOT_PAGE_LIMIT = 100
const DEFAULT_MAX_RECORDS = 50
const MAX_MAX_RECORDS = 1000
/** HubSpot Search API: 10k result hard cap, 5 req/s rate limit. */
const MAX_PAGES_PER_POLL = 10
/** Cap on property-change snapshot size to bound providerConfig payload. */
const MAX_SNAPSHOT_SIZE = 1000
/**
* HubSpot Search API caps each filterGroup at 6 filters (developers.hubspot.com/docs/api/crm/search).
* `buildBody` reserves 2 slots in Group B (filterProperty EQ + hs_object_id GT), so
* user-supplied filters (pipeline/stage/owner shortcuts plus advanced JSON filters) must
* leave room for those — cap at 4 so Group B never exceeds 6.
*/
const MAX_USER_FILTERS = 4
const BUILT_IN_PATH: Record<HubSpotBuiltInObjectType, string> = {
contact: 'contacts',
company: 'companies',
deal: 'deals',
ticket: 'tickets',
}
const VALID_OPERATORS = new Set([
'EQ',
'NEQ',
'CONTAINS_TOKEN',
'NOT_CONTAINS_TOKEN',
'GT',
'GTE',
'LT',
'LTE',
'BETWEEN',
'IN',
'NOT_IN',
'HAS_PROPERTY',
'NOT_HAS_PROPERTY',
])
function resolveSearchPath(objectType: string): string {
if (objectType in BUILT_IN_PATH) {
return BUILT_IN_PATH[objectType as HubSpotBuiltInObjectType]
}
return objectType
}
/** Contacts use `lastmodifieddate`; the `hs_lastmodifieddate` property is null on contacts. */
function resolveModifiedDateProperty(objectType: string): string {
return objectType === 'contact' ? 'lastmodifieddate' : 'hs_lastmodifieddate'
}
const DEFAULT_PROPERTIES: Record<HubSpotBuiltInObjectType, string[]> = {
contact: [
'firstname',
'lastname',
'email',
'phone',
'company',
'lifecyclestage',
'hs_lead_status',
'hubspot_owner_id',
'createdate',
'lastmodifieddate',
],
company: [
'name',
'domain',
'industry',
'lifecyclestage',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
deal: [
'dealname',
'amount',
'dealstage',
'pipeline',
'closedate',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
ticket: [
'subject',
'content',
'hs_pipeline',
'hs_pipeline_stage',
'hs_ticket_priority',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
}
export const hubspotPollingHandler: PollingProviderHandler = {
provider: 'hubspot',
label: 'HubSpot',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const accessToken = await resolveOAuthCredential(webhookData, 'hubspot', requestId)
const config = getProviderConfig<HubSpotWebhookConfig>(webhookData.providerConfig)
if (config.objectType === 'list_membership') {
return await pollListMembership(ctx, config, accessToken)
}
return await pollSearchBased(ctx, config, accessToken)
} catch (error) {
logger.error(`[${requestId}] Error processing HubSpot webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function pollSearchBased(
ctx: PollWebhookContext,
config: HubSpotWebhookConfig,
accessToken: string
): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const objectType = resolveObjectType(config)
const eventType = config.eventType
if (!objectType) {
throw new Error(`HubSpot webhook ${webhookId} is missing objectType`)
}
if (eventType !== 'created' && eventType !== 'updated' && eventType !== 'property_changed') {
throw new Error(`HubSpot webhook ${webhookId} is missing or has invalid eventType`)
}
if (eventType === 'property_changed' && !config.targetPropertyName?.trim()) {
throw new Error(
`HubSpot webhook ${webhookId} uses property_changed event but has no targetPropertyName`
)
}
// property_changed walks the modified-date stream and diffs the watched property locally.
const filterProperty =
eventType === 'created' ? 'createdate' : resolveModifiedDateProperty(objectType)
const nowMs = Date.now()
if (!config.lastSeenTimestampMs) {
await updateWebhookProviderConfig(
webhookId,
{
lastSeenTimestampMs: String(nowMs),
lastCheckedTimestamp: new Date(nowMs).toISOString(),
...(eventType === 'property_changed'
? {
propertySnapshot: {
property: config.targetPropertyName?.trim() ?? '',
entries: [],
},
}
: {}),
},
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Seeded HubSpot webhook ${webhookId} watermark to ${nowMs} (${objectType}/${eventType}/${filterProperty})`
)
return 'success'
}
const watermarkMs = Number(config.lastSeenTimestampMs)
if (!Number.isFinite(watermarkMs)) {
throw new Error(
`HubSpot webhook ${webhookId} has corrupt watermark ${config.lastSeenTimestampMs}`
)
}
const userFilters = buildUserFilters(config, logger, requestId)
const properties = resolveRequestedProperties(config, objectType, filterProperty, userFilters)
const maxRecords = Math.min(
Math.max(config.maxRecordsPerPoll ?? DEFAULT_MAX_RECORDS, 1),
MAX_MAX_RECORDS
)
const lastSeenObjectId = config.lastSeenObjectId
const records = await fetchHubSpotChanges({
accessToken,
objectType,
filterProperty,
watermarkMs,
lastSeenObjectId,
properties,
userFilters,
maxRecords,
requestId,
logger,
})
if (records.length === 0) {
await updateWebhookProviderConfig(
webhookId,
{ lastCheckedTimestamp: new Date(nowMs).toISOString() },
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new HubSpot ${objectType} ${eventType} for webhook ${webhookId}`)
return 'success'
}
logger.info(
`[${requestId}] Found ${records.length} HubSpot ${objectType} ${eventType} candidates for webhook ${webhookId}`
)
const targetProperty = config.targetPropertyName?.trim() || undefined
const snapshotForRun =
eventType === 'property_changed' && targetProperty
? resolvePropertySnapshot(config, targetProperty)
: null
const { processedCount, failedCount, skippedCount, highestSeenMs, maxIdAtHighestTimestamp } =
await processRecords(
records,
webhookData,
workflowData,
objectType,
eventType,
filterProperty,
targetProperty,
snapshotForRun,
requestId,
logger
)
const newTimestampMs = highestSeenMs > watermarkMs ? highestSeenMs : watermarkMs
const newObjectId = maxIdAtHighestTimestamp || lastSeenObjectId || ''
const update: Record<string, unknown> = {
lastSeenTimestampMs: String(newTimestampMs),
lastSeenObjectId: newObjectId,
lastCheckedTimestamp: new Date(nowMs).toISOString(),
}
if (snapshotForRun) {
update.propertySnapshot = serializeSnapshot(snapshotForRun)
}
await updateWebhookProviderConfig(webhookId, update, logger)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} HubSpot records failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Processed ${processedCount} HubSpot records${skippedCount ? `, skipped ${skippedCount} (no property change)` : ''}${failedCount ? `, ${failedCount} failed` : ''} for webhook ${webhookId}`
)
return 'success'
}
async function pollListMembership(
ctx: PollWebhookContext,
config: HubSpotWebhookConfig,
accessToken: string
): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const listId = config.listId?.trim()
if (!listId) {
throw new Error(`HubSpot list_membership trigger ${webhookId} is missing listId`)
}
const nowMs = Date.now()
const seedComplete = config.membershipSeedComplete === true
const maxRecords = Math.min(
Math.max(config.maxRecordsPerPoll ?? DEFAULT_MAX_RECORDS, 1),
MAX_MAX_RECORDS
)
// The HubSpot endpoint walks ASC by default. We resume from a stored `after` cursor —
// empty cursor means "from the beginning of the list". During the seed pass we paginate
// forward without emitting; once we reach the end (no `paging.next.after`) we mark the
// seed complete and re-fetch from the cursor-to-last-page on each normal poll. New
// members appended to the list show up on subsequent fetches; the idempotency layer
// dedups the records we've already seen on the boundary page.
const result = await fetchListMembershipPages({
listId,
accessToken,
initialAfter: config.lastSeenMembershipCursor?.trim() || undefined,
pageLimit: seedComplete ? maxRecords : MAX_PAGES_PER_POLL * HUBSPOT_PAGE_LIMIT,
requestId,
logger,
})
if (!seedComplete) {
// Seed phase: don't emit. Just save the cursor and (when reached) flip the flag.
const update: Record<string, unknown> = {
lastCheckedTimestamp: new Date(nowMs).toISOString(),
lastSeenMembershipCursor: result.resumeCursor ?? '',
}
if (result.reachedEnd) {
update.membershipSeedComplete = true
logger.info(
`[${requestId}] HubSpot list_membership ${webhookId} seed complete (list=${listId})`
)
} else {
logger.info(
`[${requestId}] HubSpot list_membership ${webhookId} seed in progress (list=${listId}, scanned ${result.scanned})`
)
}
await updateWebhookProviderConfig(webhookId, update, logger)
await markWebhookSuccess(webhookId, logger)
return 'success'
}
if (result.records.length === 0) {
await updateWebhookProviderConfig(
webhookId,
{
lastCheckedTimestamp: new Date(nowMs).toISOString(),
lastSeenMembershipCursor: result.resumeCursor ?? '',
},
logger
)
await markWebhookSuccess(webhookId, logger)
return 'success'
}
logger.info(
`[${requestId}] Found ${result.records.length} HubSpot list memberships for webhook ${webhookId}`
)
let processedCount = 0
let failedCount = 0
for (const member of result.records) {
try {
await pollingIdempotency.executeWithIdempotency(
'hubspot',
`${webhookId}:list_membership:${listId}:${member.recordId}:${member.membershipTimestamp}`,
async () => {
const payload = {
objectType: 'list_membership',
eventType: 'joined',
objectId: member.recordId,
occurredAt: member.membershipTimestamp,
listId,
timestamp: new Date().toISOString(),
}
const wfResult = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!wfResult.success) {
throw new Error(
`Webhook processing failed (${wfResult.statusCode}): ${wfResult.error ?? 'unknown'}`
)
}
return { recordId: member.recordId, processed: true }
}
)
processedCount++
} catch (error) {
failedCount++
logger.error(
`[${requestId}] Error processing HubSpot list membership ${member.recordId}:`,
getErrorMessage(error, 'Unknown error')
)
}
}
// HubSpot's `paging.next.after` is page-granular — there's no per-record cursor we can
// freeze on. Advance the cursor only when the entire batch succeeded; otherwise replay
// the page next poll and let idempotency dedup the records that already landed.
const advanceCursor = failedCount === 0
const nextCursor = advanceCursor
? (result.resumeCursor ?? '')
: (config.lastSeenMembershipCursor?.trim() ?? '')
await updateWebhookProviderConfig(
webhookId,
{
lastSeenMembershipCursor: nextCursor,
lastCheckedTimestamp: new Date(nowMs).toISOString(),
},
logger
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
return 'success'
}
function resolveObjectType(config: HubSpotWebhookConfig): string {
const raw = config.objectType?.trim()
if (raw === 'custom') {
return config.customObjectTypeId?.trim() ?? ''
}
return raw ?? ''
}
function resolveRequestedProperties(
config: HubSpotWebhookConfig,
objectType: string,
filterProperty: string,
userFilters: FilterClause[]
): string[] {
const requested = new Set<string>()
const userProperties = Array.isArray(config.properties)
? config.properties
: typeof config.properties === 'string'
? config.properties.split(/[\n,]/)
: []
for (const name of userProperties) {
const trimmed = name.trim()
if (trimmed) requested.add(trimmed)
}
if (requested.size === 0 && objectType in BUILT_IN_PATH) {
for (const name of DEFAULT_PROPERTIES[objectType as HubSpotBuiltInObjectType]) {
requested.add(name)
}
}
requested.add('createdate')
requested.add(filterProperty)
if (config.targetPropertyName?.trim()) {
requested.add(config.targetPropertyName.trim())
}
for (const f of userFilters) {
if (f.propertyName) requested.add(f.propertyName)
}
return [...requested]
}
export function buildUserFilters(
config: HubSpotWebhookConfig,
logger?: Logger,
requestId?: string
): FilterClause[] {
const filters: FilterClause[] = []
if (config.pipelineId?.trim()) {
const property = config.objectType === 'ticket' ? 'hs_pipeline' : 'pipeline'
filters.push({ propertyName: property, operator: 'EQ', value: config.pipelineId.trim() })
}
if (config.stageId?.trim()) {
const property = config.objectType === 'ticket' ? 'hs_pipeline_stage' : 'dealstage'
filters.push({ propertyName: property, operator: 'EQ', value: config.stageId.trim() })
}
if (config.ownerId?.trim()) {
filters.push({ propertyName: 'hubspot_owner_id', operator: 'EQ', value: config.ownerId.trim() })
}
const raw = config.filters
if (raw) {
try {
const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw
if (Array.isArray(parsed)) {
for (const entry of parsed) {
if (!entry || typeof entry !== 'object') continue
const propertyName = String((entry as FilterClause).propertyName ?? '').trim()
const operator = String((entry as FilterClause).operator ?? '').trim()
if (!propertyName || !VALID_OPERATORS.has(operator)) continue
const clause: FilterClause = { propertyName, operator }
if (Array.isArray((entry as FilterClause).values)) {
clause.values = (entry as FilterClause).values
} else if ((entry as FilterClause).value !== undefined) {
clause.value = String((entry as FilterClause).value)
}
filters.push(clause)
}
}
} catch (error) {
logger?.warn(
`[${requestId ?? ''}] Could not parse user filters as JSON, ignoring:`,
getErrorMessage(error, 'parse error')
)
}
}
// Filters within a filterGroup are AND-combined, so dropping any of them would silently
// widen the match set (matching records the user's config meant to exclude) rather than
// just failing the request — a worse outcome than a loud poll failure. Throw instead so
// the webhook is marked failed and the user can trim their filter configuration.
if (filters.length > MAX_USER_FILTERS) {
throw new Error(
`[${requestId ?? ''}] HubSpot webhook has ${filters.length} combined filters (pipeline/stage/owner shortcuts + advanced filters), exceeding the ${MAX_USER_FILTERS}-filter limit HubSpot's Search API allows alongside the reserved cursor filters. Reduce the number of filters.`
)
}
return filters
}
function resolvePropertySnapshot(
config: HubSpotWebhookConfig,
property: string
): PropertySnapshotState {
const existing = config.propertySnapshot
if (existing && existing.property === property && Array.isArray(existing.entries)) {
return { property, values: new Map(existing.entries) }
}
// Property changed since last config — start fresh so we don't compare against stale values.
return { property, values: new Map() }
}
function serializeSnapshot(state: PropertySnapshotState): {
property: string
entries: Array<[string, string | null]>
} {
let entries = [...state.values.entries()]
// Drop oldest by insertion order; Map preserves it regardless of key type.
if (entries.length > MAX_SNAPSHOT_SIZE) {
entries = entries.slice(entries.length - MAX_SNAPSHOT_SIZE)
}
return { property: state.property, entries }
}
interface FetchArgs {
accessToken: string
objectType: string
filterProperty: string
watermarkMs: number
lastSeenObjectId?: string
properties: string[]
userFilters: FilterClause[]
maxRecords: number
requestId: string
logger: Logger
}
async function fetchHubSpotChanges(args: FetchArgs): Promise<HubSpotSearchResult[]> {
const {
accessToken,
objectType,
filterProperty,
watermarkMs,
lastSeenObjectId,
properties,
userFilters,
maxRecords,
requestId,
logger,
} = args
const url = `https://api.hubapi.com/crm/v3/objects/${encodeURIComponent(resolveSearchPath(objectType))}/search`
const accumulated: HubSpotSearchResult[] = []
let after: string | undefined
let pages = 0
// Two OR-combined filter groups give a strict monotonic cursor over (timestamp, id):
// A: filterProperty > watermark (next timestamps)
// B: filterProperty == watermark AND id > lastSeenObjectId (more ids at boundary)
// User filters AND into both groups so they apply regardless of which side matches.
// Group B is dropped on the first poll after seeding so we don't emit boundary
// records the seed point already skipped past.
const buildBody = (cursor?: string) => {
const groupA: FilterClause[] = [
{ propertyName: filterProperty, operator: 'GT', value: String(watermarkMs) },
...userFilters,
]
const groups = [{ filters: groupA }]
if (lastSeenObjectId) {
const groupB: FilterClause[] = [
{ propertyName: filterProperty, operator: 'EQ', value: String(watermarkMs) },
{ propertyName: 'hs_object_id', operator: 'GT', value: String(lastSeenObjectId) },
...userFilters,
]
groups.push({ filters: groupB })
}
return {
filterGroups: groups,
sorts: [{ propertyName: filterProperty, direction: 'ASCENDING' }],
properties,
limit: HUBSPOT_PAGE_LIMIT,
...(cursor ? { after: cursor } : {}),
}
}
do {
const response = await fetch(url, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(buildBody(after)),
})
if (!response.ok) {
const errorBody = await response.text().catch(() => '')
logger.error(`[${requestId}] HubSpot search API error:`, {
status: response.status,
objectType,
error: errorBody,
})
throw new Error(`HubSpot search API ${response.status}: ${errorBody.slice(0, 500)}`)
}
const data = (await response.json()) as HubSpotSearchResponse
if (data.results?.length) accumulated.push(...data.results)
after = data.paging?.next?.after
pages++
if (accumulated.length >= maxRecords) break
if (pages >= MAX_PAGES_PER_POLL) {
logger.warn(
`[${requestId}] HubSpot poll hit MAX_PAGES_PER_POLL=${MAX_PAGES_PER_POLL} — remaining records will roll over to next poll`
)
break
}
} while (after)
// HubSpot's intra-timestamp ordering is undocumented. Sorting locally by (timestamp, id)
// before slicing keeps the lowest-(timestamp, id) records, so any unemitted record is
// guaranteed to match Group A or Group B on the next poll. Without this the cursor could
// overshoot and silently skip records.
accumulated.sort((a, b) => {
const aTs = extractPropertyTimestampMs(a, filterProperty)
const bTs = extractPropertyTimestampMs(b, filterProperty)
if (aTs !== bTs) {
if (!Number.isFinite(aTs)) return 1
if (!Number.isFinite(bTs)) return -1
return aTs - bTs
}
return compareObjectIds(a.id, b.id)
})
return accumulated.slice(0, maxRecords)
}
function extractPropertyTimestampMs(record: HubSpotSearchResult, propertyName: string): number {
const raw = record.properties?.[propertyName]
if (raw) {
const ms = Date.parse(raw)
if (Number.isFinite(ms)) return ms
}
const fallback = propertyName === 'createdate' ? record.createdAt : record.updatedAt
return fallback ? Date.parse(fallback) : Number.NaN
}
async function processRecords(
records: HubSpotSearchResult[],
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
objectType: string,
eventType: HubSpotEventType,
filterProperty: string,
targetProperty: string | undefined,
snapshot: PropertySnapshotState | null,
requestId: string,
logger: Logger
): Promise<{
processedCount: number
failedCount: number
skippedCount: number
highestSeenMs: number
maxIdAtHighestTimestamp: string
}> {
let processedCount = 0
let failedCount = 0
let skippedCount = 0
let highestSeenMs = 0
let maxIdAtHighestTimestamp = ''
// Stop advancing the cursor at the first failure so that the failed record and all later
// records (sorted ASC) get re-fetched on the next poll. Without this gate, a transient
// failure on a record at a high timestamp would advance the cursor past it permanently.
let cursorFrozen = false
for (const record of records) {
const occurredAtMs = extractPropertyTimestampMs(record, filterProperty)
let previousValue: string | null | undefined
let propertyValue: string | null | undefined
let handledBySkip = false
if (eventType === 'property_changed' && targetProperty && snapshot) {
propertyValue = record.properties?.[targetProperty] ?? null
const had = snapshot.values.has(record.id)
previousValue = had ? snapshot.values.get(record.id) : undefined
if (had && (previousValue ?? null) === (propertyValue ?? null)) {
// Touch the snapshot so this record's entry moves to the end of the LRU order.
// Map.delete + Map.set re-inserts at the tail, regardless of key type.
snapshot.values.delete(record.id)
snapshot.values.set(record.id, propertyValue ?? null)
skippedCount++
handledBySkip = true
}
// Note: we do NOT pre-update the snapshot before processing. If emission fails the
// record must re-fetch on the next poll AND still appear as a change vs. the prior
// snapshot — otherwise we'd silently skip it on retry.
}
let handledSuccessfully = handledBySkip
if (!handledBySkip) {
try {
await pollingIdempotency.executeWithIdempotency(
'hubspot',
`${webhookData.id}:${objectType}:${eventType}:${record.id}:${Number.isFinite(occurredAtMs) ? occurredAtMs : record.updatedAt}`,
async () => {
const payload: Record<string, unknown> = {
objectType,
eventType,
objectId: record.id,
occurredAt: Number.isFinite(occurredAtMs)
? new Date(occurredAtMs).toISOString()
: record.updatedAt,
properties: record.properties,
createdAt: record.createdAt,
updatedAt: record.updatedAt,
archived: record.archived,
timestamp: new Date().toISOString(),
}
if (eventType === 'property_changed' && targetProperty) {
payload.propertyName = targetProperty
payload.propertyValue = propertyValue ?? null
payload.previousValue = previousValue ?? null
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
throw new Error(
`Webhook processing failed (${result.statusCode}): ${result.error ?? 'unknown'}`
)
}
return { recordId: record.id, processed: true }
}
)
processedCount++
handledSuccessfully = true
if (eventType === 'property_changed' && targetProperty && snapshot) {
// Same delete+set dance so an updated value moves to the tail of the LRU order.
snapshot.values.delete(record.id)
snapshot.values.set(record.id, propertyValue ?? null)
}
} catch (error) {
failedCount++
cursorFrozen = true
logger.error(
`[${requestId}] Error processing HubSpot ${objectType} ${record.id}:`,
getErrorMessage(error, 'Unknown error')
)
}
}
// Advance the cursor only for records handled (emitted or intentionally skipped) WITHOUT
// any prior failure in this batch. Records are pre-sorted (timestamp ASC, id ASC), so
// the watermark we persist is the highest contiguously-successful (timestamp, id) pair.
// Anything after the first failure stays unfrozen so it gets re-fetched next poll.
if (handledSuccessfully && !cursorFrozen && Number.isFinite(occurredAtMs)) {
if (occurredAtMs > highestSeenMs) {
highestSeenMs = occurredAtMs
maxIdAtHighestTimestamp = record.id
} else if (occurredAtMs === highestSeenMs) {
if (compareObjectIds(record.id, maxIdAtHighestTimestamp) > 0) {
maxIdAtHighestTimestamp = record.id
}
}
}
}
return {
processedCount,
failedCount,
skippedCount,
highestSeenMs,
maxIdAtHighestTimestamp,
}
}
interface ListMembershipRecord {
recordId: string
membershipTimestamp: string
}
interface FetchListMembershipPagesArgs {
listId: string
accessToken: string
initialAfter: string | undefined
/** Soft cap on emitted records (normal mode) or pages × HUBSPOT_PAGE_LIMIT (seed mode). */
pageLimit: number
requestId: string
logger: Logger
}
interface FetchListMembershipPagesResult {
records: ListMembershipRecord[]
/** Cursor to persist for the next poll; empty if we are at the very start. */
resumeCursor: string | undefined
/** True when the API stopped returning `paging.next.after` — we've consumed everything. */
reachedEnd: boolean
/** Total records scanned across pages (useful for seed-progress logging). */
scanned: number
}
/**
* Walks `/lists/{listId}/memberships/join-order` ASC from an optional cursor until either
* (a) the per-poll page cap is reached or (b) the API stops returning a next cursor.
*
* The endpoint returns members in ascending join-order by default. We never use `before`
* (DESC mode) because its bootstrap semantics — what value to pass for "newest first" —
* are undocumented in HubSpot's SDK type and behave inconsistently in practice. ASC + a
* stored cursor is provably correct and lets new members appear on subsequent polls as
* they're appended past our cursor's position.
*/
async function fetchListMembershipPages(
args: FetchListMembershipPagesArgs
): Promise<FetchListMembershipPagesResult> {
const { listId, accessToken, initialAfter, pageLimit, requestId, logger } = args
const records: ListMembershipRecord[] = []
let after: string | undefined = initialAfter
let pages = 0
let reachedEnd = false
let scanned = 0
while (pages < MAX_PAGES_PER_POLL) {
const params = new URLSearchParams({ limit: String(HUBSPOT_PAGE_LIMIT) })
if (after) params.set('after', after)
const url = `https://api.hubapi.com/crm/v3/lists/${encodeURIComponent(listId)}/memberships/join-order?${params.toString()}`
const response = await fetch(url, { headers: { Authorization: `Bearer ${accessToken}` } })
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error(
`[${requestId}] HubSpot list memberships fetch failed ${response.status}: ${errorText}`
)
throw new Error(
`HubSpot list memberships fetch ${response.status}: ${errorText.slice(0, 500)}`
)
}
const data = (await response.json()) as {
results?: Array<{ recordId: string; membershipTimestamp: string }>
paging?: { next?: { after?: string } }
}
const batch = data.results ?? []
scanned += batch.length
const nextAfter = data.paging?.next?.after
for (const m of batch) {
records.push({
recordId: m.recordId,
membershipTimestamp: m.membershipTimestamp,
})
}
pages++
if (!nextAfter) {
reachedEnd = true
break
}
after = nextAfter
if (records.length >= pageLimit) break
}
return {
records,
// If we walked to the end, hold onto the cursor we LAST used so the next poll re-fetches
// the tail page and picks up any members appended since. If we stopped early, the next
// cursor walks us forward through the rest of the list.
resumeCursor: after,
reachedEnd,
scanned,
}
}
/** Numeric compare for HubSpot ids (decimal strings, treated numerically by GT/LT). */
function compareObjectIds(a: string, b: string): number {
if (!a) return b ? -1 : 0
if (!b) return 1
const an = Number(a)
const bn = Number(b)
if (Number.isFinite(an) && Number.isFinite(bn)) {
if (an === bn) return 0
return an > bn ? 1 : -1
}
if (a === b) return 0
return a > b ? 1 : -1
}
+608
View File
@@ -0,0 +1,608 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import type { FetchMessageObject, MailboxLockObject } from 'imapflow'
import { ImapFlow } from 'imapflow'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { validateDatabaseHost } from '@/lib/core/security/input-validation.server'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
interface ImapWebhookConfig {
host: string
port: number
secure: boolean
username: string
password: string
mailbox: string | string[]
searchCriteria: string | Record<string, unknown>
markAsRead: boolean
includeAttachments: boolean
lastProcessedUid?: number
lastProcessedUidByMailbox?: Record<string, number>
uidValidityByMailbox?: Record<string, string>
lastCheckedTimestamp?: string
maxEmailsPerPoll?: number
}
interface ImapAttachment {
name: string
data: Buffer
mimeType: string
size: number
}
interface SimplifiedImapEmail {
uid: string
messageId: string
subject: string
from: string
to: string
cc: string
date: string | null
bodyText: string
bodyHtml: string
mailbox: string
hasAttachments: boolean
attachments: ImapAttachment[]
}
interface ImapWebhookPayload {
messageId: string
subject: string
from: string
to: string
cc: string
date: string | null
bodyText: string
bodyHtml: string
mailbox: string
hasAttachments: boolean
attachments: ImapAttachment[]
email: SimplifiedImapEmail
timestamp: string
}
export const imapPollingHandler: PollingProviderHandler = {
provider: 'imap',
label: 'IMAP',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const config = getProviderConfig<ImapWebhookConfig>(webhookData.providerConfig)
if (!config.host || !config.username || !config.password) {
logger.error(`[${requestId}] Missing IMAP credentials for webhook ${webhookId}`)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
const hostValidation = await validateDatabaseHost(config.host, 'host')
if (!hostValidation.isValid) {
logger.error(
`[${requestId}] IMAP host validation failed for webhook ${webhookId}: ${hostValidation.error}`
)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
const client = new ImapFlow({
host: hostValidation.resolvedIP!,
servername: config.host,
port: config.port || 993,
secure: config.secure ?? true,
auth: {
user: config.username,
pass: config.password,
},
tls: { rejectUnauthorized: true },
logger: false,
})
let emails: Awaited<ReturnType<typeof fetchNewEmails>>['emails'] = []
let latestUidByMailbox: Record<string, number> = {}
let uidValidityByMailbox: Record<string, string> = {}
try {
await client.connect()
const result = await fetchNewEmails(client, config, requestId, logger)
emails = result.emails
latestUidByMailbox = result.latestUidByMailbox
uidValidityByMailbox = result.uidValidityByMailbox
const pollTimestamp = new Date().toISOString()
if (!emails.length) {
await updateImapState(
webhookId,
latestUidByMailbox,
pollTimestamp,
config,
logger,
uidValidityByMailbox
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
await client.logout()
return 'success'
}
logger.info(`[${requestId}] Found ${emails.length} new emails for webhook ${webhookId}`)
const { processedCount, failedCount } = await processEmails(
emails,
webhookData,
workflowData,
config,
client,
requestId,
logger
)
await updateImapState(
webhookId,
latestUidByMailbox,
pollTimestamp,
config,
logger,
uidValidityByMailbox
)
await client.logout()
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (innerError) {
try {
await client.logout()
} catch {}
throw innerError
}
} catch (error) {
logger.error(`[${requestId}] Error processing IMAP webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function updateImapState(
webhookId: string,
uidByMailbox: Record<string, number>,
timestamp: string,
config: ImapWebhookConfig,
logger: Logger,
uidValidityByMailbox: Record<string, string>
) {
const existingUidByMailbox = config.lastProcessedUidByMailbox || {}
const prevUidValidity = config.uidValidityByMailbox || {}
const resetMailboxes = new Set(
Object.entries(uidValidityByMailbox)
.filter(
([mailbox, validity]) =>
prevUidValidity[mailbox] !== undefined && prevUidValidity[mailbox] !== validity
)
.map(([mailbox]) => mailbox)
)
const mergedUidByMailbox: Record<string, number> = {}
for (const [mailbox, uid] of Object.entries(existingUidByMailbox)) {
if (!resetMailboxes.has(mailbox)) {
mergedUidByMailbox[mailbox] = uid
}
}
for (const [mailbox, uid] of Object.entries(uidByMailbox)) {
if (resetMailboxes.has(mailbox)) {
mergedUidByMailbox[mailbox] = uid
} else {
mergedUidByMailbox[mailbox] = Math.max(uid, mergedUidByMailbox[mailbox] || 0)
}
}
await updateWebhookProviderConfig(
webhookId,
{
lastProcessedUidByMailbox: mergedUidByMailbox,
lastCheckedTimestamp: timestamp,
uidValidityByMailbox,
},
logger
)
}
async function fetchNewEmails(
client: ImapFlow,
config: ImapWebhookConfig,
requestId: string,
logger: Logger
) {
const emails: Array<{
uid: number
mailboxPath: string
envelope: FetchMessageObject['envelope']
bodyStructure: FetchMessageObject['bodyStructure']
source?: Buffer
}> = []
const mailboxes = getMailboxesToCheck(config)
const latestUidByMailbox: Record<string, number> = { ...(config.lastProcessedUidByMailbox || {}) }
const uidValidityByMailbox: Record<string, string> = { ...(config.uidValidityByMailbox || {}) }
const maxEmails = config.maxEmailsPerPoll || 25
let totalEmailsCollected = 0
for (const mailboxPath of mailboxes) {
if (totalEmailsCollected >= maxEmails) break
try {
const mailbox = await client.mailboxOpen(mailboxPath)
const currentUidValidity = mailbox.uidValidity.toString()
const storedUidValidity = uidValidityByMailbox[mailboxPath]
if (storedUidValidity && storedUidValidity !== currentUidValidity) {
logger.warn(
`[${requestId}] UIDVALIDITY changed for ${mailboxPath} (${storedUidValidity} -> ${currentUidValidity}), discarding stored UID`
)
delete latestUidByMailbox[mailboxPath]
}
uidValidityByMailbox[mailboxPath] = currentUidValidity
let searchCriteria: Record<string, unknown> = { unseen: true }
if (config.searchCriteria) {
if (typeof config.searchCriteria === 'object') {
searchCriteria = config.searchCriteria
} else if (typeof config.searchCriteria === 'string') {
try {
searchCriteria = JSON.parse(config.searchCriteria)
} catch {
logger.warn(`[${requestId}] Invalid search criteria JSON, using default`)
}
}
}
const lastUidForMailbox = latestUidByMailbox[mailboxPath]
if (lastUidForMailbox) {
searchCriteria = { ...searchCriteria, uid: `${lastUidForMailbox + 1}:*` }
}
if (config.lastCheckedTimestamp) {
const lastChecked = new Date(config.lastCheckedTimestamp)
const bufferTime = new Date(lastChecked.getTime() - 60000)
searchCriteria = { ...searchCriteria, since: bufferTime }
} else {
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
searchCriteria = { ...searchCriteria, since: oneDayAgo }
}
let messageUids: number[] = []
try {
const searchResult = await client.search(searchCriteria, { uid: true })
messageUids = searchResult === false ? [] : searchResult
} catch {
continue
}
if (messageUids.length === 0) continue
messageUids.sort((a, b) => a - b)
const remainingSlots = maxEmails - totalEmailsCollected
const uidsToProcess = messageUids.slice(0, remainingSlots)
for await (const msg of client.fetch(
uidsToProcess,
{ uid: true, envelope: true, bodyStructure: true, source: true },
{ uid: true }
)) {
emails.push({
uid: msg.uid,
mailboxPath,
envelope: msg.envelope,
bodyStructure: msg.bodyStructure,
source: msg.source,
})
if (msg.uid > (latestUidByMailbox[mailboxPath] || 0)) {
latestUidByMailbox[mailboxPath] = msg.uid
}
totalEmailsCollected++
}
} catch (mailboxError) {
logger.warn(`[${requestId}] Error processing mailbox ${mailboxPath}:`, mailboxError)
}
}
return { emails, latestUidByMailbox, uidValidityByMailbox }
}
function getMailboxesToCheck(config: ImapWebhookConfig): string[] {
if (!config.mailbox || (Array.isArray(config.mailbox) && config.mailbox.length === 0)) {
return ['INBOX']
}
if (Array.isArray(config.mailbox)) {
return config.mailbox
}
return [config.mailbox]
}
function parseEmailAddress(
addr: { name?: string; address?: string } | { name?: string; address?: string }[] | undefined
): string {
if (!addr) return ''
if (Array.isArray(addr)) {
return addr
.map((a) => (a.name ? `${a.name} <${a.address}>` : a.address || ''))
.filter(Boolean)
.join(', ')
}
return addr.name ? `${addr.name} <${addr.address}>` : addr.address || ''
}
function extractTextFromSource(source: Buffer): { text: string; html: string } {
const content = source.toString('utf-8')
let text = ''
let html = ''
const parts = content.split(/--[^\r\n]+/)
for (const part of parts) {
const lowerPart = part.toLowerCase()
if (lowerPart.includes('content-type: text/plain')) {
const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i)
if (match) {
text = match[1].trim()
if (lowerPart.includes('quoted-printable')) {
text = text
.replace(/=\r?\n/g, '')
.replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
}
if (lowerPart.includes('base64')) {
try {
text = Buffer.from(text.replace(/\s/g, ''), 'base64').toString('utf-8')
} catch {}
}
}
} else if (lowerPart.includes('content-type: text/html')) {
const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i)
if (match) {
html = match[1].trim()
if (lowerPart.includes('quoted-printable')) {
html = html
.replace(/=\r?\n/g, '')
.replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
}
if (lowerPart.includes('base64')) {
try {
html = Buffer.from(html.replace(/\s/g, ''), 'base64').toString('utf-8')
} catch {}
}
}
}
}
if (!text && !html) {
const bodyMatch = content.match(/\r?\n\r?\n([\s\S]+)$/)
if (bodyMatch) {
text = bodyMatch[1].trim()
}
}
return { text, html }
}
function extractAttachmentsFromSource(
source: Buffer,
bodyStructure: FetchMessageObject['bodyStructure']
): ImapAttachment[] {
const attachments: ImapAttachment[] = []
if (!bodyStructure) return attachments
const content = source.toString('utf-8')
const parts = content.split(/--[^\r\n]+/)
for (const part of parts) {
const lowerPart = part.toLowerCase()
const dispositionMatch = part.match(
/content-disposition:\s*attachment[^;]*;\s*filename="?([^"\r\n]+)"?/i
)
const filenameMatch = part.match(/name="?([^"\r\n]+)"?/i)
const contentTypeMatch = part.match(/content-type:\s*([^;\r\n]+)/i)
if (
dispositionMatch ||
(filenameMatch && !lowerPart.includes('text/plain') && !lowerPart.includes('text/html'))
) {
const filename = dispositionMatch?.[1] || filenameMatch?.[1] || 'attachment'
const mimeType = contentTypeMatch?.[1]?.trim() || 'application/octet-stream'
const dataMatch = part.match(/\r?\n\r?\n([\s\S]*?)$/i)
if (dataMatch) {
const data = dataMatch[1].trim()
if (lowerPart.includes('base64')) {
try {
const buffer = Buffer.from(data.replace(/\s/g, ''), 'base64')
attachments.push({
name: filename,
data: buffer,
mimeType,
size: buffer.length,
})
} catch {}
}
}
}
}
return attachments
}
function hasAttachmentsInBodyStructure(structure: FetchMessageObject['bodyStructure']): boolean {
if (!structure) return false
if (structure.disposition === 'attachment') return true
if (structure.disposition === 'inline' && structure.dispositionParameters?.filename) return true
if (structure.childNodes && Array.isArray(structure.childNodes)) {
return structure.childNodes.some((child) => hasAttachmentsInBodyStructure(child))
}
return false
}
async function processEmails(
emails: Array<{
uid: number
mailboxPath: string
envelope: FetchMessageObject['envelope']
bodyStructure: FetchMessageObject['bodyStructure']
source?: Buffer
}>,
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
config: ImapWebhookConfig,
client: ImapFlow,
requestId: string,
logger: Logger
) {
let processedCount = 0
let failedCount = 0
let currentOpenMailbox: string | null = null
const lockState: { lock: MailboxLockObject | null } = { lock: null }
try {
for (const email of emails) {
try {
await pollingIdempotency.executeWithIdempotency(
'imap',
`${webhookData.id}:${email.mailboxPath}:${email.uid}`,
async () => {
const envelope = email.envelope
const { text: bodyText, html: bodyHtml } = email.source
? extractTextFromSource(email.source)
: { text: '', html: '' }
let attachments: ImapAttachment[] = []
const hasAttachments = hasAttachmentsInBodyStructure(email.bodyStructure)
if (config.includeAttachments && hasAttachments && email.source) {
attachments = extractAttachmentsFromSource(email.source, email.bodyStructure)
}
const simplifiedEmail: SimplifiedImapEmail = {
uid: String(email.uid),
messageId: envelope?.messageId || '',
subject: envelope?.subject || '[No Subject]',
from: parseEmailAddress(envelope?.from),
to: parseEmailAddress(envelope?.to),
cc: parseEmailAddress(envelope?.cc),
date: envelope?.date ? new Date(envelope.date).toISOString() : null,
bodyText,
bodyHtml,
mailbox: email.mailboxPath,
hasAttachments,
attachments,
}
const payload: ImapWebhookPayload = {
messageId: simplifiedEmail.messageId,
subject: simplifiedEmail.subject,
from: simplifiedEmail.from,
to: simplifiedEmail.to,
cc: simplifiedEmail.cc,
date: simplifiedEmail.date,
bodyText: simplifiedEmail.bodyText,
bodyHtml: simplifiedEmail.bodyHtml,
mailbox: simplifiedEmail.mailbox,
hasAttachments: simplifiedEmail.hasAttachments,
attachments: simplifiedEmail.attachments,
email: simplifiedEmail,
timestamp: new Date().toISOString(),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for email ${email.uid}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
if (config.markAsRead) {
try {
if (currentOpenMailbox !== email.mailboxPath) {
if (lockState.lock) {
lockState.lock.release()
lockState.lock = null
}
lockState.lock = await client.getMailboxLock(email.mailboxPath)
currentOpenMailbox = email.mailboxPath
}
await client.messageFlagsAdd(email.uid, ['\\Seen'], { uid: true })
} catch (flagError) {
logger.warn(
`[${requestId}] Failed to mark message ${email.uid} as read:`,
flagError
)
}
}
return { emailUid: email.uid, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed email ${email.uid} from ${email.mailboxPath} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing email ${email.uid}:`, errorMessage)
failedCount++
}
}
} finally {
if (lockState.lock) {
try {
lockState.lock.release()
} catch {}
}
}
return { processedCount, failedCount }
}
+2
View File
@@ -0,0 +1,2 @@
export { pollProvider } from '@/lib/webhooks/polling/orchestrator'
export { VALID_POLLING_PROVIDERS } from '@/lib/webhooks/polling/registry'
@@ -0,0 +1,46 @@
import { createLogger } from '@sim/logger'
import { generateShortId } from '@sim/utils/id'
import { getPollingHandler } from '@/lib/webhooks/polling/registry'
import type { PollSummary } from '@/lib/webhooks/polling/types'
import { fetchActiveWebhooks, runWithConcurrency } from '@/lib/webhooks/polling/utils'
/** Poll all active webhooks for a given provider. */
export async function pollProvider(providerName: string): Promise<PollSummary> {
const handler = getPollingHandler(providerName)
if (!handler) {
throw new Error(`Unknown polling provider: ${providerName}`)
}
const logger = createLogger(`${handler.label}PollingService`)
logger.info(`Starting ${handler.label} webhook polling`)
const activeWebhooks = await fetchActiveWebhooks(handler.provider)
if (!activeWebhooks.length) {
logger.info(`No active ${handler.label} webhooks found`)
return { total: 0, successful: 0, failed: 0 }
}
logger.info(`Found ${activeWebhooks.length} active ${handler.label} webhooks`)
const { successCount, failureCount } = await runWithConcurrency(
activeWebhooks,
async (entry) => {
const requestId = generateShortId()
return handler.pollWebhook({
webhookData: entry.webhook,
workflowData: entry.workflow,
requestId,
logger,
})
},
logger
)
const summary: PollSummary = {
total: activeWebhooks.length,
successful: successCount,
failed: failureCount,
}
logger.info(`${handler.label} polling completed`, summary)
return summary
}
+570
View File
@@ -0,0 +1,570 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { htmlToText } from 'html-to-text'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
resolveOAuthCredential,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
interface OutlookWebhookConfig {
credentialId: string
folderIds?: string[]
folderFilterBehavior?: 'INCLUDE' | 'EXCLUDE'
markAsRead?: boolean
maxEmailsPerPoll?: number
lastCheckedTimestamp?: string
includeAttachments?: boolean
includeRawEmail?: boolean
}
interface OutlookEmail {
id: string
conversationId: string
subject: string
bodyPreview: string
body: {
contentType: string
content: string
}
from: {
emailAddress: {
name: string
address: string
}
}
toRecipients: Array<{
emailAddress: {
name: string
address: string
}
}>
ccRecipients?: Array<{
emailAddress: {
name: string
address: string
}
}>
receivedDateTime: string
sentDateTime: string
hasAttachments: boolean
isRead: boolean
parentFolderId: string
}
interface OutlookAttachment {
name: string
data: Buffer
contentType: string
size: number
}
interface SimplifiedOutlookEmail {
id: string
conversationId: string
subject: string
from: string
to: string
cc: string
date: string
bodyText: string
bodyHtml: string
hasAttachments: boolean
attachments: OutlookAttachment[]
isRead: boolean
folderId: string
messageId: string
threadId: string
}
interface OutlookWebhookPayload {
email: SimplifiedOutlookEmail
timestamp: string
rawEmail?: OutlookEmail
}
function convertHtmlToPlainText(html: string): string {
if (!html) return ''
return htmlToText(html, {
wordwrap: false,
selectors: [
{ selector: 'a', options: { hideLinkHrefIfSameAsText: true, noAnchorUrl: true } },
{ selector: 'img', format: 'skip' },
{ selector: 'script', format: 'skip' },
{ selector: 'style', format: 'skip' },
],
preserveNewlines: true,
})
}
export const outlookPollingHandler: PollingProviderHandler = {
provider: 'outlook',
label: 'Outlook',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
logger.info(`[${requestId}] Processing Outlook webhook: ${webhookId}`)
const accessToken = await resolveOAuthCredential(webhookData, 'outlook', requestId)
const config = getProviderConfig<OutlookWebhookConfig>(webhookData.providerConfig)
const now = new Date()
const { emails } = await fetchNewOutlookEmails(accessToken, config, requestId, logger)
if (!emails || !emails.length) {
await updateWebhookProviderConfig(
webhookId,
{ lastCheckedTimestamp: now.toISOString() },
logger
)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
return 'success'
}
logger.info(`[${requestId}] Found ${emails.length} emails for webhook ${webhookId}`)
const { processedCount, failedCount } = await processOutlookEmails(
emails,
webhookData,
workflowData,
config,
accessToken,
requestId,
logger
)
await updateWebhookProviderConfig(
webhookId,
{ lastCheckedTimestamp: now.toISOString() },
logger
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
logger.error(`[${requestId}] Error processing Outlook webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
/** Hard cap on total emails fetched per poll to prevent unbounded pagination loops. */
const OUTLOOK_HARD_MAX_EMAILS = 200
/** Number of items to request per Graph API page. Decoupled from the total cap so pagination actually runs. */
const OUTLOOK_PAGE_SIZE = 50
async function fetchNewOutlookEmails(
accessToken: string,
config: OutlookWebhookConfig,
requestId: string,
logger: Logger
) {
try {
const apiUrl = 'https://graph.microsoft.com/v1.0/me/messages'
const params = new URLSearchParams()
params.append(
'$select',
'id,conversationId,subject,bodyPreview,body,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,isRead,parentFolderId'
)
params.append('$orderby', 'receivedDateTime desc')
const maxEmails = Math.min(config.maxEmailsPerPoll || 25, OUTLOOK_HARD_MAX_EMAILS)
params.append('$top', OUTLOOK_PAGE_SIZE.toString())
if (config.lastCheckedTimestamp) {
const lastChecked = new Date(config.lastCheckedTimestamp)
const bufferTime = new Date(lastChecked.getTime() - 60000)
params.append('$filter', `receivedDateTime gt ${bufferTime.toISOString()}`)
}
const allEmails: OutlookEmail[] = []
let nextUrl: string | undefined = `${apiUrl}?${params.toString()}`
logger.info(`[${requestId}] Fetching emails from: ${nextUrl}`)
while (nextUrl && allEmails.length < maxEmails) {
const response = await fetchWithRetry(nextUrl, {
headers: {
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorData = await response
.json()
.catch(() => ({ error: { message: 'Unknown error' } }))
logger.error(`[${requestId}] Microsoft Graph API error:`, {
status: response.status,
statusText: response.statusText,
error: errorData,
})
throw new Error(
`Microsoft Graph API error: ${response.status} ${response.statusText} - ${JSON.stringify(errorData)}`
)
}
const data = await response.json()
const pageEmails: OutlookEmail[] = data.value || []
const remaining = maxEmails - allEmails.length
allEmails.push(...pageEmails.slice(0, remaining))
nextUrl =
allEmails.length < maxEmails ? (data['@odata.nextLink'] as string | undefined) : undefined
if (pageEmails.length === 0) break
}
logger.info(`[${requestId}] Fetched ${allEmails.length} emails total`)
const emails = allEmails
let resolvedFolderIds: Map<string, string> | undefined
let skipFolderFilter = false
if (config.folderIds && config.folderIds.length > 0) {
const wellKnownFolders = config.folderIds.filter(isWellKnownFolderName)
if (wellKnownFolders.length > 0) {
resolvedFolderIds = await resolveWellKnownFolderIds(
accessToken,
config.folderIds,
requestId,
logger
)
if (resolvedFolderIds.size < wellKnownFolders.length) {
logger.warn(
`[${requestId}] Could not resolve all well-known folders (${resolvedFolderIds.size}/${wellKnownFolders.length}) — skipping folder filter to avoid incorrect results`
)
skipFolderFilter = true
}
}
}
const filteredEmails = skipFolderFilter
? emails
: filterEmailsByFolder(emails, config, resolvedFolderIds)
logger.info(
`[${requestId}] Fetched ${emails.length} emails, ${filteredEmails.length} after filtering`
)
return { emails: filteredEmails }
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error fetching new Outlook emails:`, errorMessage)
throw error
}
}
const OUTLOOK_WELL_KNOWN_FOLDERS = new Set([
'inbox',
'drafts',
'sentitems',
'deleteditems',
'junkemail',
'archive',
'outbox',
])
function isWellKnownFolderName(folderId: string): boolean {
return OUTLOOK_WELL_KNOWN_FOLDERS.has(folderId.toLowerCase())
}
async function resolveWellKnownFolderId(
accessToken: string,
folderName: string,
requestId: string,
logger: Logger
): Promise<string | null> {
try {
const response = await fetchWithRetry(
`https://graph.microsoft.com/v1.0/me/mailFolders/${folderName}`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
}
)
if (!response.ok) {
logger.warn(
`[${requestId}] Failed to resolve well-known folder '${folderName}': ${response.status}`
)
return null
}
const folder = await response.json()
return folder.id || null
} catch (error) {
logger.error(`[${requestId}] Error resolving well-known folder '${folderName}':`, error)
return null
}
}
async function resolveWellKnownFolderIds(
accessToken: string,
folderIds: string[],
requestId: string,
logger: Logger
): Promise<Map<string, string>> {
const resolvedIds = new Map<string, string>()
const wellKnownFolders = folderIds.filter(isWellKnownFolderName)
if (wellKnownFolders.length === 0) return resolvedIds
const resolutions = await Promise.all(
wellKnownFolders.map(async (folderName) => {
const actualId = await resolveWellKnownFolderId(accessToken, folderName, requestId, logger)
return { folderName, actualId }
})
)
for (const { folderName, actualId } of resolutions) {
if (actualId) {
resolvedIds.set(folderName.toLowerCase(), actualId)
}
}
logger.info(
`[${requestId}] Resolved ${resolvedIds.size}/${wellKnownFolders.length} well-known folders`
)
return resolvedIds
}
function filterEmailsByFolder(
emails: OutlookEmail[],
config: OutlookWebhookConfig,
resolvedFolderIds?: Map<string, string>
): OutlookEmail[] {
if (!config.folderIds || !config.folderIds.length) return emails
const actualFolderIds = config.folderIds.map((configFolder) => {
if (resolvedFolderIds && isWellKnownFolderName(configFolder)) {
const resolvedId = resolvedFolderIds.get(configFolder.toLowerCase())
if (resolvedId) return resolvedId
}
return configFolder
})
return emails.filter((email) => {
const emailFolderId = email.parentFolderId
const hasMatchingFolder = actualFolderIds.some(
(folderId) => emailFolderId.toLowerCase() === folderId.toLowerCase()
)
return config.folderFilterBehavior === 'INCLUDE' ? hasMatchingFolder : !hasMatchingFolder
})
}
async function processOutlookEmails(
emails: OutlookEmail[],
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
config: OutlookWebhookConfig,
accessToken: string,
requestId: string,
logger: Logger
) {
let processedCount = 0
let failedCount = 0
for (const email of emails) {
try {
await pollingIdempotency.executeWithIdempotency(
'outlook',
`${webhookData.id}:${email.id}`,
async () => {
let attachments: OutlookAttachment[] = []
if (config.includeAttachments && email.hasAttachments) {
try {
attachments = await downloadOutlookAttachments(
accessToken,
email.id,
requestId,
logger
)
} catch (error) {
logger.error(
`[${requestId}] Error downloading attachments for email ${email.id}:`,
error
)
}
}
const simplifiedEmail: SimplifiedOutlookEmail = {
id: email.id,
conversationId: email.conversationId,
subject: email.subject || '',
from: email.from?.emailAddress?.address || '',
to: email.toRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
cc: email.ccRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
date: email.receivedDateTime,
bodyText: (() => {
const content = email.body?.content || ''
const type = (email.body?.contentType || '').toLowerCase()
if (!content) return email.bodyPreview || ''
if (type === 'text' || type === 'text/plain') return content
return convertHtmlToPlainText(content)
})(),
bodyHtml: email.body?.content || '',
hasAttachments: email.hasAttachments,
attachments,
isRead: email.isRead,
folderId: email.parentFolderId,
messageId: email.id,
threadId: email.conversationId,
}
const payload: OutlookWebhookPayload = {
email: simplifiedEmail,
timestamp: new Date().toISOString(),
}
if (config.includeRawEmail) {
payload.rawEmail = email
}
logger.info(
`[${requestId}] Processing email: ${email.subject} from ${email.from?.emailAddress?.address}`
)
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for email ${email.id}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
if (config.markAsRead) {
await markOutlookEmailAsRead(accessToken, email.id, logger)
}
return { emailId: email.id, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed email ${email.id} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
logger.error(`[${requestId}] Error processing email ${email.id}:`, error)
failedCount++
}
}
return { processedCount, failedCount }
}
async function downloadOutlookAttachments(
accessToken: string,
messageId: string,
requestId: string,
logger: Logger
): Promise<OutlookAttachment[]> {
const attachments: OutlookAttachment[] = []
try {
const response = await fetchWithRetry(
`https://graph.microsoft.com/v1.0/me/messages/${messageId}/attachments`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
}
)
if (!response.ok) {
logger.error(`[${requestId}] Failed to fetch attachments for message ${messageId}`)
return attachments
}
const data = await response.json()
const attachmentsList = data.value || []
for (const attachment of attachmentsList) {
try {
if (attachment['@odata.type'] === '#microsoft.graph.fileAttachment') {
const contentBytes = attachment.contentBytes
if (contentBytes) {
const buffer = Buffer.from(contentBytes, 'base64')
attachments.push({
name: attachment.name,
data: buffer,
contentType: attachment.contentType,
size: attachment.size,
})
}
}
} catch (error) {
logger.error(
`[${requestId}] Error processing attachment ${attachment.id} for message ${messageId}:`,
error
)
}
}
logger.info(
`[${requestId}] Downloaded ${attachments.length} attachments for message ${messageId}`
)
} catch (error) {
logger.error(`[${requestId}] Error downloading attachments for message ${messageId}:`, error)
}
return attachments
}
async function markOutlookEmailAsRead(accessToken: string, messageId: string, logger: Logger) {
try {
const response = await fetchWithRetry(
`https://graph.microsoft.com/v1.0/me/messages/${messageId}`,
{
method: 'PATCH',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ isRead: true }),
}
)
if (!response.ok) {
logger.error(
`Failed to mark email ${messageId} as read:`,
response.status,
await response.text()
)
}
} catch (error) {
logger.error(`Error marking email ${messageId} as read:`, error)
}
}
+27
View File
@@ -0,0 +1,27 @@
import { gmailPollingHandler } from '@/lib/webhooks/polling/gmail'
import { googleCalendarPollingHandler } from '@/lib/webhooks/polling/google-calendar'
import { googleDrivePollingHandler } from '@/lib/webhooks/polling/google-drive'
import { googleSheetsPollingHandler } from '@/lib/webhooks/polling/google-sheets'
import { hubspotPollingHandler } from '@/lib/webhooks/polling/hubspot'
import { imapPollingHandler } from '@/lib/webhooks/polling/imap'
import { outlookPollingHandler } from '@/lib/webhooks/polling/outlook'
import { rssPollingHandler } from '@/lib/webhooks/polling/rss'
import type { PollingProviderHandler } from '@/lib/webhooks/polling/types'
const POLLING_HANDLERS: Record<string, PollingProviderHandler> = {
gmail: gmailPollingHandler,
'google-calendar': googleCalendarPollingHandler,
'google-drive': googleDrivePollingHandler,
'google-sheets': googleSheetsPollingHandler,
hubspot: hubspotPollingHandler,
imap: imapPollingHandler,
outlook: outlookPollingHandler,
rss: rssPollingHandler,
}
export const VALID_POLLING_PROVIDERS = new Set(Object.keys(POLLING_HANDLERS))
/** Look up the polling handler for a provider. */
export function getPollingHandler(provider: string): PollingProviderHandler | undefined {
return POLLING_HANDLERS[provider]
}
+374
View File
@@ -0,0 +1,374 @@
import type { Logger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import Parser from 'rss-parser'
import { pollingIdempotency } from '@/lib/core/idempotency/service'
import {
secureFetchWithPinnedIP,
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import {
getProviderConfig,
type PollingProviderHandler,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
const MAX_GUIDS_TO_TRACK = 500
interface RssWebhookConfig {
feedUrl: string
lastCheckedTimestamp?: string
lastSeenGuids?: string[]
etag?: string
lastModified?: string
}
interface RssItem {
title?: string
link?: string
pubDate?: string
guid?: string
description?: string
content?: string
contentSnippet?: string
author?: string
creator?: string
categories?: string[]
enclosure?: {
url: string
type?: string
length?: string | number
}
isoDate?: string
[key: string]: unknown
}
interface RssFeed {
title?: string
link?: string
description?: string
items: RssItem[]
}
interface RssWebhookPayload {
title?: string
link?: string
pubDate?: string
item: RssItem
feed: {
title?: string
link?: string
description?: string
}
timestamp: string
}
const parser = new Parser({
timeout: 30000,
headers: {
'User-Agent': 'Sim/1.0 RSS Poller',
},
})
export const rssPollingHandler: PollingProviderHandler = {
provider: 'rss',
label: 'RSS',
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
try {
const config = getProviderConfig<RssWebhookConfig>(webhookData.providerConfig)
if (!config?.feedUrl) {
logger.error(`[${requestId}] Missing feedUrl for webhook ${webhookId}`)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
const now = new Date()
const {
feed,
items: newItems,
etag,
lastModified,
} = await fetchNewRssItems(config, requestId, logger)
if (!newItems.length) {
await updateRssState(webhookId, now.toISOString(), [], config, logger, etag, lastModified)
await markWebhookSuccess(webhookId, logger)
logger.info(`[${requestId}] No new items found for webhook ${webhookId}`)
return 'success'
}
logger.info(`[${requestId}] Found ${newItems.length} new items for webhook ${webhookId}`)
const { processedCount, failedCount } = await processRssItems(
newItems,
feed,
webhookData,
workflowData,
requestId,
logger
)
const newGuids = newItems
.map(
(item) =>
item.guid ||
item.link ||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
)
.filter((guid) => guid.length > 0)
await updateRssState(
webhookId,
now.toISOString(),
newGuids,
config,
logger,
etag,
lastModified
)
if (failedCount > 0 && processedCount === 0) {
await markWebhookFailed(webhookId, logger)
logger.warn(
`[${requestId}] All ${failedCount} items failed to process for webhook ${webhookId}`
)
return 'failure'
}
await markWebhookSuccess(webhookId, logger)
logger.info(
`[${requestId}] Successfully processed ${processedCount} items for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
)
return 'success'
} catch (error) {
logger.error(`[${requestId}] Error processing RSS webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
}
async function updateRssState(
webhookId: string,
timestamp: string,
newGuids: string[],
config: RssWebhookConfig,
logger: Logger,
etag?: string,
lastModified?: string
) {
const existingGuids = config.lastSeenGuids || []
const allGuids = [...newGuids, ...existingGuids].slice(0, MAX_GUIDS_TO_TRACK)
await updateWebhookProviderConfig(
webhookId,
{
lastCheckedTimestamp: timestamp,
lastSeenGuids: allGuids,
...(etag !== undefined ? { etag } : {}),
...(lastModified !== undefined ? { lastModified } : {}),
},
logger
)
}
async function fetchNewRssItems(
config: RssWebhookConfig,
requestId: string,
logger: Logger
): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> {
try {
const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl')
if (!urlValidation.isValid) {
logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`)
throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`)
}
const headers: Record<string, string> = {
'User-Agent': 'Sim/1.0 RSS Poller',
Accept: 'application/rss+xml, application/xml, text/xml, */*',
}
if (config.etag) {
headers['If-None-Match'] = config.etag
}
if (config.lastModified) {
headers['If-Modified-Since'] = config.lastModified
}
const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, {
headers,
timeout: 30000,
})
if (response.status === 304) {
logger.info(`[${requestId}] RSS feed not modified (304) for ${config.feedUrl}`)
return {
feed: { items: [] } as RssFeed,
items: [],
etag: response.headers.get('etag') ?? config.etag,
lastModified: response.headers.get('last-modified') ?? config.lastModified,
}
}
if (!response.ok) {
await response.text().catch(() => {})
throw new Error(`Failed to fetch RSS feed: ${response.status} ${response.statusText}`)
}
const newEtag = response.headers.get('etag') ?? undefined
const newLastModified = response.headers.get('last-modified') ?? undefined
const xmlContent = await response.text()
const feed = await parser.parseString(xmlContent)
if (!feed.items || !feed.items.length) {
return { feed: feed as RssFeed, items: [], etag: newEtag, lastModified: newLastModified }
}
const lastCheckedTime = config.lastCheckedTimestamp
? new Date(config.lastCheckedTimestamp)
: null
const lastSeenGuids = new Set(config.lastSeenGuids || [])
const newItems = feed.items.filter((item) => {
const itemGuid =
item.guid ||
item.link ||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
if (itemGuid && lastSeenGuids.has(itemGuid)) {
return false
}
if (lastCheckedTime && item.isoDate) {
const itemDate = new Date(item.isoDate)
if (itemDate <= lastCheckedTime) {
return false
}
}
return true
})
newItems.sort((a, b) => {
const dateA = a.isoDate ? new Date(a.isoDate).getTime() : 0
const dateB = b.isoDate ? new Date(b.isoDate).getTime() : 0
return dateB - dateA
})
const limitedItems = newItems.slice(0, 25)
logger.info(
`[${requestId}] Found ${newItems.length} new items (processing ${limitedItems.length})`
)
return {
feed: feed as RssFeed,
items: limitedItems as RssItem[],
etag: newEtag,
lastModified: newLastModified,
}
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error fetching RSS feed:`, errorMessage)
throw error
}
}
async function processRssItems(
items: RssItem[],
feed: RssFeed,
webhookData: PollWebhookContext['webhookData'],
workflowData: PollWebhookContext['workflowData'],
requestId: string,
logger: Logger
): Promise<{ processedCount: number; failedCount: number }> {
let processedCount = 0
let failedCount = 0
for (const item of items) {
try {
const itemGuid =
item.guid ||
item.link ||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
if (!itemGuid) {
logger.warn(
`[${requestId}] Skipping RSS item with no identifiable GUID for webhook ${webhookData.id}`
)
continue
}
await pollingIdempotency.executeWithIdempotency(
'rss',
`${webhookData.id}:${itemGuid}`,
async () => {
const payload: RssWebhookPayload = {
title: item.title,
link: item.link,
pubDate: item.pubDate,
item: {
title: item.title,
link: item.link,
pubDate: item.pubDate,
guid: item.guid,
description: item.description,
content: item.content,
contentSnippet: item.contentSnippet,
author: item.author || item.creator,
categories: item.categories,
enclosure: item.enclosure,
isoDate: item.isoDate,
},
feed: {
title: feed.title,
link: feed.link,
description: feed.description,
},
timestamp: new Date().toISOString(),
}
const result = await processPolledWebhookEvent(
webhookData,
workflowData,
payload,
requestId
)
if (!result.success) {
logger.error(
`[${requestId}] Failed to process webhook for item ${itemGuid}:`,
result.statusCode,
result.error
)
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
}
return { itemGuid, processed: true }
}
)
logger.info(
`[${requestId}] Successfully processed item ${item.title || itemGuid} for webhook ${webhookData.id}`
)
processedCount++
} catch (error) {
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing item:`, errorMessage)
failedCount++
}
}
return { processedCount, failedCount }
}
+55
View File
@@ -0,0 +1,55 @@
import type { webhook, workflow } from '@sim/db/schema'
import type { Logger } from '@sim/logger'
import { isRecordLike } from '@sim/utils/object'
/** Summary returned after polling all webhooks for a provider. */
export interface PollSummary {
total: number
successful: number
failed: number
}
/** Context passed to a provider handler when processing one webhook. */
export interface PollWebhookContext {
webhookData: WebhookRecord
workflowData: WorkflowRecord
requestId: string
logger: Logger
}
export type WebhookRecord = typeof webhook.$inferSelect
export type WorkflowRecord = typeof workflow.$inferSelect
export function getProviderConfigRecord(
providerConfig: WebhookRecord['providerConfig']
): Record<string, unknown> {
return isRecordLike(providerConfig) ? providerConfig : {}
}
export function getProviderConfig<T extends object>(
providerConfig: WebhookRecord['providerConfig']
): T {
return getProviderConfigRecord(providerConfig) as T
}
/**
* Strategy interface for provider-specific polling behavior.
* Mirrors `WebhookProviderHandler` from `providers/types.ts`.
*
* Each provider implements `pollWebhook()` — the full inner loop for one webhook:
* validate config, resolve credentials, fetch new items, process each via
* `processPolledWebhookEvent()` (wrapped in `pollingIdempotency`), update state.
*/
export interface PollingProviderHandler {
/** Provider name used in DB queries (e.g. 'gmail', 'rss'). */
readonly provider: string
/** Display label for log messages (e.g. 'Gmail', 'RSS'). */
readonly label: string
/**
* Process a single webhook entry.
* Return 'success' (even if 0 new items) or 'failure'.
*/
pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'>
}
+185
View File
@@ -0,0 +1,185 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockUpdate, mockSet, mockWhere, mockSelect, mockSelectRows, sqlCalls } = vi.hoisted(() => {
const mockSelectRows = vi.fn()
return {
mockUpdate: vi.fn(),
mockSet: vi.fn(),
mockWhere: vi.fn(),
mockSelectRows,
mockSelect: vi.fn(() => ({
from: vi.fn(() => ({
where: vi.fn(() => ({
limit: mockSelectRows,
})),
})),
})),
sqlCalls: [] as Array<{ strings: readonly string[]; values: unknown[] }>,
}
})
vi.mock('@sim/db', () => ({ db: { update: mockUpdate, select: mockSelect } }))
vi.mock('@sim/db/schema', () => ({
webhook: {
id: 'webhook.id',
providerConfig: 'webhook.providerConfig',
updatedAt: 'webhook.updatedAt',
},
account: {},
workflow: {},
workflowDeploymentVersion: {},
}))
vi.mock('drizzle-orm', () => ({
sql: (strings: readonly string[], ...values: unknown[]) => {
const node = { strings, values }
sqlCalls.push(node)
return node
},
and: vi.fn(),
eq: vi.fn((field: unknown, value: unknown) => ({ field, value })),
isNull: vi.fn(),
ne: vi.fn(),
or: vi.fn(),
}))
vi.mock('@/app/api/auth/oauth/utils', () => ({
getOAuthToken: vi.fn(),
refreshAccessTokenIfNeeded: vi.fn(),
resolveOAuthAccountId: vi.fn(),
}))
vi.mock('@/triggers/constants', () => ({ MAX_CONSECUTIVE_FAILURES: 5 }))
import type { WebhookRecord } from '@/lib/webhooks/polling/types'
import { resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils'
import {
getOAuthToken,
refreshAccessTokenIfNeeded,
resolveOAuthAccountId,
} from '@/app/api/auth/oauth/utils'
const logger = { error: vi.fn() } as never
function allInterpolatedValues(): unknown[] {
return sqlCalls.flatMap((c) => c.values)
}
function allSqlText(): string {
return sqlCalls.map((c) => c.strings.join('')).join(' ')
}
describe('updateWebhookProviderConfig (atomic jsonb merge)', () => {
beforeEach(() => {
vi.clearAllMocks()
sqlCalls.length = 0
mockWhere.mockResolvedValue(undefined)
mockSet.mockReturnValue({ where: mockWhere })
mockUpdate.mockReturnValue({ set: mockSet })
})
it('merges defined keys (null preserved) and removes undefined keys', async () => {
await updateWebhookProviderConfig(
'wh-1',
{ historyId: 'h1', cleared: undefined, nulled: null },
logger
)
expect(mockUpdate).toHaveBeenCalledTimes(1)
expect(allInterpolatedValues()).toContain(JSON.stringify({ historyId: 'h1', nulled: null }))
expect(allInterpolatedValues()).toContainEqual(['cleared'])
})
it('uses merge only (no key-removal expression) when nothing is undefined', async () => {
await updateWebhookProviderConfig('wh-1', { historyId: 'h1' }, logger)
expect(allInterpolatedValues()).toContain(JSON.stringify({ historyId: 'h1' }))
expect(allInterpolatedValues().some((v) => Array.isArray(v))).toBe(false)
})
it('casts the json column to jsonb for the merge and back to json for storage', async () => {
await updateWebhookProviderConfig('wh-1', { historyId: 'h1', cleared: undefined }, logger)
const sqlText = allSqlText()
// Column (interpolated as a value) is cast to jsonb: `COALESCE(<col>::jsonb, ...)`
expect(sqlText).toContain('COALESCE(::jsonb')
// Merge runs in jsonb space, result cast back to the json column: `(<expr>)::json`
expect(sqlText).toContain(')::json')
})
})
describe('resolveOAuthCredential (single-credential polling)', () => {
const makeWebhook = (providerConfig: Record<string, unknown>): WebhookRecord =>
({ id: 'wh-1', providerConfig }) as unknown as WebhookRecord
beforeEach(() => {
vi.clearAllMocks()
mockSelectRows.mockResolvedValue([])
})
it('resolves via credentialId: account lookup then token refresh', async () => {
vi.mocked(resolveOAuthAccountId).mockResolvedValue({
accountId: 'acc-1',
} as Awaited<ReturnType<typeof resolveOAuthAccountId>>)
mockSelectRows.mockResolvedValue([{ userId: 'owner-1' }])
vi.mocked(refreshAccessTokenIfNeeded).mockResolvedValue('tok-abc')
const token = await resolveOAuthCredential(
makeWebhook({ credentialId: 'cred-1' }),
'google-email',
'req-1'
)
expect(token).toBe('tok-abc')
expect(resolveOAuthAccountId).toHaveBeenCalledWith('cred-1')
expect(refreshAccessTokenIfNeeded).toHaveBeenCalledWith('acc-1', 'owner-1', 'req-1')
expect(getOAuthToken).not.toHaveBeenCalled()
})
it('throws when the credential cannot be resolved to an OAuth account', async () => {
vi.mocked(resolveOAuthAccountId).mockResolvedValue(null)
await expect(
resolveOAuthCredential(makeWebhook({ credentialId: 'cred-gone' }), 'google-email', 'req-1')
).rejects.toThrow('Failed to resolve OAuth account for credential cred-gone')
})
it('throws when the resolved account row does not exist', async () => {
vi.mocked(resolveOAuthAccountId).mockResolvedValue({
accountId: 'acc-missing',
} as Awaited<ReturnType<typeof resolveOAuthAccountId>>)
mockSelectRows.mockResolvedValue([])
await expect(
resolveOAuthCredential(makeWebhook({ credentialId: 'cred-1' }), 'google-email', 'req-1')
).rejects.toThrow('Credential cred-1 not found for webhook wh-1')
})
it('falls back to the legacy userId path via getOAuthToken', async () => {
vi.mocked(getOAuthToken).mockResolvedValue('tok-legacy')
const token = await resolveOAuthCredential(
makeWebhook({ userId: 'user-1' }),
'outlook',
'req-1'
)
expect(token).toBe('tok-legacy')
expect(getOAuthToken).toHaveBeenCalledWith('user-1', 'outlook')
expect(resolveOAuthAccountId).not.toHaveBeenCalled()
})
it('throws when neither credentialId nor userId is present', async () => {
await expect(resolveOAuthCredential(makeWebhook({}), 'gmail', 'req-1')).rejects.toThrow(
'Missing credential info for webhook wh-1'
)
})
it('throws when the legacy userId path yields no token', async () => {
vi.mocked(getOAuthToken).mockResolvedValue(null)
await expect(
resolveOAuthCredential(makeWebhook({ userId: 'user-1' }), 'outlook', 'req-1')
).rejects.toThrow('Failed to get outlook access token for webhook wh-1')
})
})
+221
View File
@@ -0,0 +1,221 @@
import { db } from '@sim/db'
import { account, webhook, workflow, workflowDeploymentVersion } from '@sim/db/schema'
import type { Logger } from '@sim/logger'
import { and, eq, isNull, ne, or, sql } from 'drizzle-orm'
import type { WebhookRecord, WorkflowRecord } from '@/lib/webhooks/polling/types'
import {
getOAuthToken,
refreshAccessTokenIfNeeded,
resolveOAuthAccountId,
} from '@/app/api/auth/oauth/utils'
import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants'
/** Concurrency limit for parallel webhook processing. Standardized across all providers. */
export const CONCURRENCY = 10
/** Increment the webhook's failure count. Auto-disables after MAX_CONSECUTIVE_FAILURES. */
export async function markWebhookFailed(webhookId: string, logger: Logger): Promise<void> {
try {
const result = await db
.update(webhook)
.set({
failedCount: sql`COALESCE(${webhook.failedCount}, 0) + 1`,
lastFailedAt: new Date(),
updatedAt: new Date(),
})
.where(eq(webhook.id, webhookId))
.returning({ failedCount: webhook.failedCount })
const newFailedCount = result[0]?.failedCount || 0
if (newFailedCount >= MAX_CONSECUTIVE_FAILURES) {
await db
.update(webhook)
.set({
isActive: false,
updatedAt: new Date(),
})
.where(eq(webhook.id, webhookId))
logger.warn(
`Webhook ${webhookId} auto-disabled after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`
)
}
} catch (err) {
logger.error(`Failed to mark webhook ${webhookId} as failed:`, err)
}
}
/** Reset the webhook's failure count on successful poll. */
export async function markWebhookSuccess(webhookId: string, logger: Logger): Promise<void> {
try {
await db
.update(webhook)
.set({
failedCount: 0,
updatedAt: new Date(),
})
.where(
and(eq(webhook.id, webhookId), or(isNull(webhook.failedCount), ne(webhook.failedCount, 0)))
)
} catch (err) {
logger.error(`Failed to mark webhook ${webhookId} as successful:`, err)
}
}
/** Fetch all active webhooks for a provider, joined with their workflow. */
export async function fetchActiveWebhooks(
provider: string
): Promise<{ webhook: WebhookRecord; workflow: WorkflowRecord }[]> {
const rows = await db
.select({ webhook, workflow })
.from(webhook)
.innerJoin(workflow, eq(webhook.workflowId, workflow.id))
.leftJoin(
workflowDeploymentVersion,
and(
eq(workflowDeploymentVersion.workflowId, workflow.id),
eq(workflowDeploymentVersion.isActive, true)
)
)
.where(
and(
eq(webhook.provider, provider),
eq(webhook.isActive, true),
isNull(webhook.archivedAt),
eq(workflow.isDeployed, true),
isNull(workflow.archivedAt),
or(
eq(webhook.deploymentVersionId, workflowDeploymentVersion.id),
and(isNull(workflowDeploymentVersion.id), isNull(webhook.deploymentVersionId))
)
)
)
return rows
}
/**
* Run an async function over entries with bounded concurrency.
* Returns aggregate success/failure counts.
*/
export async function runWithConcurrency(
entries: { webhook: WebhookRecord; workflow: WorkflowRecord }[],
processFn: (entry: {
webhook: WebhookRecord
workflow: WorkflowRecord
}) => Promise<'success' | 'failure'>,
logger: Logger
): Promise<{ successCount: number; failureCount: number }> {
const running: Promise<void>[] = []
let successCount = 0
let failureCount = 0
for (const entry of entries) {
const promise: Promise<void> = processFn(entry)
.then((result) => {
if (result === 'success') {
successCount++
} else {
failureCount++
}
})
.catch((err) => {
logger.error('Unexpected error in webhook processing:', err)
failureCount++
})
.finally(() => {
const idx = running.indexOf(promise)
if (idx !== -1) running.splice(idx, 1)
})
running.push(promise)
if (running.length >= CONCURRENCY) {
await Promise.race(running)
}
}
await Promise.allSettled(running)
return { successCount, failureCount }
}
/**
* Atomically merge provider-specific config fields into `webhook.provider_config`.
* Each provider passes its specific state updates (historyId, lastSeenGuids, etc.).
*
* The column is `json` (not `jsonb`), which has no merge operators, so the existing
* value is cast to `jsonb` for the `||`/`-` merge and the result cast back to `json`
* for storage. Casting is required — a bare `jsonb` expression cannot be assigned to
* the `json` column.
*/
export async function updateWebhookProviderConfig(
webhookId: string,
configUpdates: Record<string, unknown>,
logger: Logger
): Promise<void> {
try {
const defined: Record<string, unknown> = {}
const removedKeys: string[] = []
for (const [key, value] of Object.entries(configUpdates)) {
if (value === undefined) removedKeys.push(key)
else defined[key] = value
}
const merged = sql`COALESCE(${webhook.providerConfig}::jsonb, '{}'::jsonb) || ${JSON.stringify(defined)}::jsonb`
const nextConfig = removedKeys.length > 0 ? sql`(${merged}) - ${removedKeys}::text[]` : merged
await db
.update(webhook)
.set({
providerConfig: sql`(${nextConfig})::json`,
updatedAt: new Date(),
})
.where(eq(webhook.id, webhookId))
} catch (err) {
logger.error(`Failed to update webhook ${webhookId} config:`, err)
}
}
/**
* Resolve OAuth credentials for a webhook. Shared by Gmail and Outlook.
* Returns the access token or throws on failure.
*/
export async function resolveOAuthCredential(
webhookData: WebhookRecord,
oauthProvider: string,
requestId: string
): Promise<string> {
const metadata = webhookData.providerConfig as Record<string, unknown> | null
const credentialId = metadata?.credentialId as string | undefined
const userId = metadata?.userId as string | undefined
if (!credentialId && !userId) {
throw new Error(`Missing credential info for webhook ${webhookData.id}`)
}
let accessToken: string | null = null
if (credentialId) {
const resolved = await resolveOAuthAccountId(credentialId)
if (!resolved) {
throw new Error(
`Failed to resolve OAuth account for credential ${credentialId}, webhook ${webhookData.id}`
)
}
const rows = await db.select().from(account).where(eq(account.id, resolved.accountId)).limit(1)
if (!rows.length) {
throw new Error(`Credential ${credentialId} not found for webhook ${webhookData.id}`)
}
const ownerUserId = rows[0].userId
accessToken = await refreshAccessTokenIfNeeded(resolved.accountId, ownerUserId, requestId)
} else if (userId) {
accessToken = await getOAuthToken(userId, oauthProvider)
}
if (!accessToken) {
throw new Error(`Failed to get ${oauthProvider} access token for webhook ${webhookData.id}`)
}
return accessToken
}