chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
|
||||
|
||||
describe('readCanonicalTriggerValue', () => {
|
||||
it('returns the canonical value when present', () => {
|
||||
expect(readCanonicalTriggerValue('canon', 'basic', 'advanced')).toBe('canon')
|
||||
})
|
||||
|
||||
it('falls back basic-first when the canonical key is absent (transitional)', () => {
|
||||
expect(readCanonicalTriggerValue(undefined, 'basic', 'advanced')).toBe('basic')
|
||||
})
|
||||
|
||||
it('falls back to the advanced value when canonical and basic are absent', () => {
|
||||
expect(readCanonicalTriggerValue(undefined, undefined, 'advanced')).toBe('advanced')
|
||||
})
|
||||
|
||||
it('treats empty strings as unset', () => {
|
||||
expect(readCanonicalTriggerValue('', '', 'advanced')).toBe('advanced')
|
||||
expect(readCanonicalTriggerValue('', '')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('ignores non-string members', () => {
|
||||
expect(readCanonicalTriggerValue(null, 42 as unknown, 'advanced')).toBe('advanced')
|
||||
expect(readCanonicalTriggerValue(undefined, undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
/**
|
||||
* No-repoint invariant: for every config shape the previous app version could
|
||||
* have stored, the new canonical-first read must resolve to the SAME resource
|
||||
* the old basic-first read returned — whether the row has been backfilled
|
||||
* (canonical key present) or not (transitional fallback). The legacy
|
||||
* basic-first read is reproduced inline as the source of truth.
|
||||
*/
|
||||
describe('no-repoint across legacy and backfilled shapes', () => {
|
||||
interface GoogleDriveConfig {
|
||||
folderId?: string
|
||||
manualFolderId?: string
|
||||
}
|
||||
// The exact pre-change Google Drive poller read (basic key === canonical key).
|
||||
const legacyDriveRead = (c: GoogleDriveConfig) => c.folderId || c.manualFolderId
|
||||
// Mirror of the migration 0253 backfill: fill the canonical key basic-first
|
||||
// ONLY when absent. For Drive the canonical key IS the basic key (folderId).
|
||||
const backfillDrive = (c: GoogleDriveConfig): GoogleDriveConfig =>
|
||||
c.folderId ? c : { ...c, folderId: c.folderId || c.manualFolderId }
|
||||
|
||||
const driveShapes: GoogleDriveConfig[] = [
|
||||
{ folderId: 'basic-only' },
|
||||
{ manualFolderId: 'advanced-only' },
|
||||
// Drift: stale basic + active advanced both stored. Current poller reads basic.
|
||||
{ folderId: 'STALE', manualFolderId: 'ACTIVE' },
|
||||
{},
|
||||
]
|
||||
|
||||
it.each(driveShapes)('Drive legacy config %o reads same resource as before', (config) => {
|
||||
const before = legacyDriveRead(config)
|
||||
const after = readCanonicalTriggerValue(config.folderId, config.manualFolderId)
|
||||
expect(after).toBe(before || undefined)
|
||||
})
|
||||
|
||||
it.each(driveShapes)('Drive backfilled config %o reads same resource as before', (config) => {
|
||||
const before = legacyDriveRead(config)
|
||||
const backfilled = backfillDrive(config)
|
||||
const after = readCanonicalTriggerValue(backfilled.folderId, backfilled.manualFolderId)
|
||||
expect(after).toBe(before || undefined)
|
||||
})
|
||||
|
||||
interface TableConfig {
|
||||
tableId?: string
|
||||
tableSelector?: string
|
||||
manualTableId?: string
|
||||
}
|
||||
// The exact pre-change table reader (already canonical-first, but tableId was
|
||||
// never written, so it fell through to the raw keys).
|
||||
const legacyTableRead = (c: TableConfig) => c.tableId ?? c.tableSelector ?? c.manualTableId
|
||||
// Mirror of the migration 0253 backfill for table (canonical key is distinct).
|
||||
const backfillTable = (c: TableConfig): TableConfig =>
|
||||
c.tableId ? c : { ...c, tableId: c.tableSelector ?? c.manualTableId }
|
||||
|
||||
const tableShapes: TableConfig[] = [
|
||||
{ tableSelector: 'basic-only' },
|
||||
{ manualTableId: 'advanced-only' },
|
||||
{ tableSelector: 'STALE', manualTableId: 'ACTIVE' },
|
||||
{ tableId: 'collapsed', tableSelector: 'STALE', manualTableId: 'ACTIVE' },
|
||||
{},
|
||||
]
|
||||
|
||||
it.each(tableShapes)('table legacy config %o reads same resource as before', (config) => {
|
||||
const before = legacyTableRead(config)
|
||||
const after = readCanonicalTriggerValue(
|
||||
config.tableId,
|
||||
config.tableSelector,
|
||||
config.manualTableId
|
||||
)
|
||||
expect(after).toBe(before ?? undefined)
|
||||
})
|
||||
|
||||
it.each(tableShapes)('table backfilled config %o reads same resource as before', (config) => {
|
||||
const before = legacyTableRead(config)
|
||||
const backfilled = backfillTable(config)
|
||||
const after = readCanonicalTriggerValue(
|
||||
backfilled.tableId,
|
||||
backfilled.tableSelector,
|
||||
backfilled.manualTableId
|
||||
)
|
||||
expect(after).toBe(before ?? undefined)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Resolve the live resource id for a canonical trigger field from a deployed
|
||||
* `webhook.providerConfig`.
|
||||
*
|
||||
* The canonical key — written at the deploy boundary by `buildProviderConfig`
|
||||
* and populated on pre-existing rows by migration `0253` — is authoritative and
|
||||
* is read first. The `transitionalFallback` values are read basic-first ONLY
|
||||
* when the canonical key is absent, i.e. for configs deployed before the
|
||||
* canonical key existed and not yet backfilled (including webhooks created by
|
||||
* the previous app version during a deploy cutover).
|
||||
*
|
||||
* The fallback is TRANSITIONAL: once the backfill is confirmed and every
|
||||
* deployed config carries the canonical key, it can be deleted in a follow-up
|
||||
* contract phase and callers can read the canonical key alone. It exists solely
|
||||
* for migration safety, not as permanent precedence logic.
|
||||
*
|
||||
* Empty strings are treated as unset (matching the previous basic-first `||`
|
||||
* reads); returns `undefined` when no member is set.
|
||||
*
|
||||
* @param canonicalValue - value stored under the group's `canonicalParamId`
|
||||
* @param transitionalFallback - the raw subblock values in basic-first order
|
||||
*/
|
||||
export function readCanonicalTriggerValue(
|
||||
canonicalValue: unknown,
|
||||
...transitionalFallback: unknown[]
|
||||
): string | undefined {
|
||||
for (const value of [canonicalValue, ...transitionalFallback]) {
|
||||
if (typeof value === 'string' && value.length > 0) return value
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -0,0 +1,595 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
import type { GmailAttachment } from '@/tools/gmail/types'
|
||||
import { downloadAttachments, extractAttachmentInfo } from '@/tools/gmail/utils'
|
||||
|
||||
interface GmailWebhookConfig {
|
||||
labelIds: string[]
|
||||
labelFilterBehavior: 'INCLUDE' | 'EXCLUDE'
|
||||
markAsRead: boolean
|
||||
searchQuery?: string
|
||||
maxEmailsPerPoll?: number
|
||||
lastCheckedTimestamp?: string
|
||||
historyId?: string
|
||||
includeAttachments?: boolean
|
||||
includeRawEmail?: boolean
|
||||
}
|
||||
|
||||
interface GmailEmail {
|
||||
id: string
|
||||
threadId: string
|
||||
historyId?: string
|
||||
labelIds?: string[]
|
||||
payload?: Record<string, unknown>
|
||||
snippet?: string
|
||||
internalDate?: string
|
||||
}
|
||||
|
||||
interface SimplifiedEmail {
|
||||
id: string
|
||||
threadId: string
|
||||
subject: string
|
||||
from: string
|
||||
to: string
|
||||
cc: string
|
||||
date: string | null
|
||||
bodyText: string
|
||||
bodyHtml: string
|
||||
labels: string[]
|
||||
hasAttachments: boolean
|
||||
attachments: GmailAttachment[]
|
||||
}
|
||||
|
||||
interface GmailWebhookPayload {
|
||||
email: SimplifiedEmail
|
||||
timestamp: string
|
||||
rawEmail?: GmailEmail
|
||||
}
|
||||
|
||||
export const gmailPollingHandler: PollingProviderHandler = {
|
||||
provider: 'gmail',
|
||||
label: 'Gmail',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'google-email', requestId)
|
||||
|
||||
const config = getProviderConfig<GmailWebhookConfig>(webhookData.providerConfig)
|
||||
const now = new Date()
|
||||
|
||||
const { emails, latestHistoryId } = await fetchNewEmails(
|
||||
accessToken,
|
||||
config,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
if (!emails || !emails.length) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
...(latestHistoryId || config.historyId
|
||||
? { historyId: latestHistoryId || config.historyId }
|
||||
: {}),
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Found ${emails.length} new emails for webhook ${webhookId}`)
|
||||
|
||||
const { processedCount, failedCount } = await processEmails(
|
||||
emails,
|
||||
webhookData,
|
||||
workflowData,
|
||||
config,
|
||||
accessToken,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
...(latestHistoryId || config.historyId
|
||||
? { historyId: latestHistoryId || config.historyId }
|
||||
: {}),
|
||||
},
|
||||
logger
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing Gmail webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function fetchNewEmails(
|
||||
accessToken: string,
|
||||
config: GmailWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
try {
|
||||
const useHistoryApi = !!config.historyId
|
||||
let emails: GmailEmail[] = []
|
||||
let latestHistoryId = config.historyId
|
||||
|
||||
if (useHistoryApi) {
|
||||
const messageIds = new Set<string>()
|
||||
let pageToken: string | undefined
|
||||
|
||||
do {
|
||||
let historyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/history?startHistoryId=${config.historyId}&historyTypes=messageAdded`
|
||||
if (pageToken) {
|
||||
historyUrl += `&pageToken=${pageToken}`
|
||||
}
|
||||
|
||||
const historyResponse = await fetch(historyUrl, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!historyResponse.ok) {
|
||||
const status = historyResponse.status
|
||||
const errorData = await historyResponse.json().catch(() => ({}))
|
||||
logger.error(`[${requestId}] Gmail history API error:`, {
|
||||
status,
|
||||
statusText: historyResponse.statusText,
|
||||
error: errorData,
|
||||
})
|
||||
|
||||
if (status === 403 || status === 429) {
|
||||
throw new Error(
|
||||
`Gmail API error ${status} — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Falling back to search API after history API error ${status}`)
|
||||
const searchResult = await searchEmails(accessToken, config, requestId, logger)
|
||||
if (searchResult.emails.length === 0) {
|
||||
const freshHistoryId = await getGmailProfileHistoryId(accessToken, requestId, logger)
|
||||
if (freshHistoryId) {
|
||||
logger.info(
|
||||
`[${requestId}] Fetched fresh historyId ${freshHistoryId} after invalid historyId (was: ${config.historyId})`
|
||||
)
|
||||
return { emails: [], latestHistoryId: freshHistoryId }
|
||||
}
|
||||
}
|
||||
return searchResult
|
||||
}
|
||||
|
||||
const historyData = await historyResponse.json()
|
||||
|
||||
if (historyData.historyId) {
|
||||
latestHistoryId = historyData.historyId
|
||||
}
|
||||
|
||||
if (historyData.history) {
|
||||
for (const history of historyData.history) {
|
||||
if (history.messagesAdded) {
|
||||
for (const messageAdded of history.messagesAdded) {
|
||||
messageIds.add(messageAdded.message.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pageToken = historyData.nextPageToken
|
||||
} while (pageToken)
|
||||
|
||||
if (!messageIds.size) {
|
||||
return { emails: [], latestHistoryId }
|
||||
}
|
||||
|
||||
const sortedIds = [...messageIds].sort().reverse()
|
||||
const idsToFetch = sortedIds.slice(0, config.maxEmailsPerPoll || 25)
|
||||
logger.info(`[${requestId}] Processing ${idsToFetch.length} emails from history API`)
|
||||
|
||||
const emailResults = await Promise.allSettled(
|
||||
idsToFetch.map((messageId) => getEmailDetails(accessToken, messageId))
|
||||
)
|
||||
const rejected = emailResults.filter((r) => r.status === 'rejected')
|
||||
if (rejected.length > 0) {
|
||||
logger.warn(`[${requestId}] Failed to fetch ${rejected.length} email details`)
|
||||
}
|
||||
emails = emailResults
|
||||
.filter(
|
||||
(result): result is PromiseFulfilledResult<GmailEmail> => result.status === 'fulfilled'
|
||||
)
|
||||
.map((result) => result.value)
|
||||
|
||||
emails = filterEmailsByLabels(emails, config)
|
||||
} else {
|
||||
return searchEmails(accessToken, config, requestId, logger)
|
||||
}
|
||||
|
||||
return { emails, latestHistoryId }
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error fetching new emails:`, errorMessage)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function buildGmailSearchQuery(config: {
|
||||
labelIds?: string[]
|
||||
labelFilterBehavior?: 'INCLUDE' | 'EXCLUDE'
|
||||
searchQuery?: string
|
||||
}): string {
|
||||
let labelQuery = ''
|
||||
if (config.labelIds && config.labelIds.length > 0) {
|
||||
const labelParts = config.labelIds.map((label) => `label:${label}`).join(' OR ')
|
||||
labelQuery =
|
||||
config.labelFilterBehavior === 'INCLUDE'
|
||||
? config.labelIds.length > 1
|
||||
? `(${labelParts})`
|
||||
: labelParts
|
||||
: config.labelIds.length > 1
|
||||
? `-(${labelParts})`
|
||||
: `-${labelParts}`
|
||||
}
|
||||
|
||||
let searchQueryPart = ''
|
||||
if (config.searchQuery?.trim()) {
|
||||
searchQueryPart = config.searchQuery.trim()
|
||||
if (searchQueryPart.includes(' OR ') || searchQueryPart.includes(' AND ')) {
|
||||
searchQueryPart = `(${searchQueryPart})`
|
||||
}
|
||||
}
|
||||
|
||||
let baseQuery = ''
|
||||
if (labelQuery && searchQueryPart) {
|
||||
baseQuery = `${labelQuery} ${searchQueryPart}`
|
||||
} else if (searchQueryPart) {
|
||||
baseQuery = searchQueryPart
|
||||
} else if (labelQuery) {
|
||||
baseQuery = labelQuery
|
||||
} else {
|
||||
baseQuery = 'in:inbox'
|
||||
}
|
||||
|
||||
return baseQuery
|
||||
}
|
||||
|
||||
async function searchEmails(
|
||||
accessToken: string,
|
||||
config: GmailWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
try {
|
||||
const baseQuery = buildGmailSearchQuery(config)
|
||||
let timeConstraint = ''
|
||||
|
||||
if (config.lastCheckedTimestamp) {
|
||||
const lastCheckedTime = new Date(config.lastCheckedTimestamp)
|
||||
const now = new Date()
|
||||
const minutesSinceLastCheck = (now.getTime() - lastCheckedTime.getTime()) / (60 * 1000)
|
||||
|
||||
if (minutesSinceLastCheck < 60) {
|
||||
const bufferSeconds = Math.max(1 * 60 * 2, 180)
|
||||
const cutoffTime = new Date(lastCheckedTime.getTime() - bufferSeconds * 1000)
|
||||
const timestamp = Math.floor(cutoffTime.getTime() / 1000)
|
||||
timeConstraint = ` after:${timestamp}`
|
||||
} else if (minutesSinceLastCheck < 24 * 60) {
|
||||
const hours = Math.ceil(minutesSinceLastCheck / 60) + 1
|
||||
timeConstraint = ` newer_than:${hours}h`
|
||||
} else {
|
||||
const days = Math.min(Math.ceil(minutesSinceLastCheck / (24 * 60)), 7) + 1
|
||||
timeConstraint = ` newer_than:${days}d`
|
||||
}
|
||||
} else {
|
||||
timeConstraint = ' newer_than:1d'
|
||||
}
|
||||
|
||||
const query = `${baseQuery}${timeConstraint}`
|
||||
const searchUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${encodeURIComponent(query)}&maxResults=${config.maxEmailsPerPoll || 25}`
|
||||
|
||||
const searchResponse = await fetch(searchUrl, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!searchResponse.ok) {
|
||||
const errorData = await searchResponse.json()
|
||||
logger.error(`[${requestId}] Gmail search API error:`, {
|
||||
status: searchResponse.status,
|
||||
statusText: searchResponse.statusText,
|
||||
query,
|
||||
error: errorData,
|
||||
})
|
||||
throw new Error(
|
||||
`Gmail API error: ${searchResponse.status} ${searchResponse.statusText} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
const searchData = await searchResponse.json()
|
||||
|
||||
if (!searchData.messages || !searchData.messages.length) {
|
||||
logger.info(`[${requestId}] No emails found matching query: ${query}`)
|
||||
return { emails: [], latestHistoryId: config.historyId }
|
||||
}
|
||||
|
||||
const idsToFetch = searchData.messages.slice(0, config.maxEmailsPerPoll || 25)
|
||||
let latestHistoryId = config.historyId
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Processing ${idsToFetch.length} emails from search API (total matches: ${searchData.messages.length})`
|
||||
)
|
||||
|
||||
const emailResults = await Promise.allSettled(
|
||||
idsToFetch.map((message: { id: string }) => getEmailDetails(accessToken, message.id))
|
||||
)
|
||||
const rejected = emailResults.filter((r) => r.status === 'rejected')
|
||||
if (rejected.length > 0) {
|
||||
logger.warn(`[${requestId}] Failed to fetch ${rejected.length} email details`)
|
||||
}
|
||||
const emails = emailResults
|
||||
.filter(
|
||||
(result): result is PromiseFulfilledResult<GmailEmail> => result.status === 'fulfilled'
|
||||
)
|
||||
.map((result) => result.value)
|
||||
|
||||
if (emails.length > 0 && emails[0].historyId) {
|
||||
latestHistoryId = emails[0].historyId
|
||||
}
|
||||
|
||||
return { emails, latestHistoryId }
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error searching emails:`, errorMessage)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function getGmailProfileHistoryId(
|
||||
accessToken: string,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
if (!response.ok) {
|
||||
logger.warn(
|
||||
`[${requestId}] Failed to fetch Gmail profile for fresh historyId: ${response.status}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
const profile = await response.json()
|
||||
return (profile.historyId as string | undefined) ?? null
|
||||
} catch (error) {
|
||||
logger.warn(`[${requestId}] Error fetching Gmail profile:`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function getEmailDetails(accessToken: string, messageId: string): Promise<GmailEmail> {
|
||||
const messageUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}?format=full`
|
||||
|
||||
const messageResponse = await fetch(messageUrl, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!messageResponse.ok) {
|
||||
const errorData = await messageResponse.json().catch(() => ({}))
|
||||
throw new Error(
|
||||
`Failed to fetch email details for message ${messageId}: ${messageResponse.status} ${messageResponse.statusText} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
return await messageResponse.json()
|
||||
}
|
||||
|
||||
function filterEmailsByLabels(emails: GmailEmail[], config: GmailWebhookConfig): GmailEmail[] {
|
||||
if (!config.labelIds.length) {
|
||||
return emails
|
||||
}
|
||||
|
||||
return emails.filter((email) => {
|
||||
const emailLabels = email.labelIds || []
|
||||
const hasMatchingLabel = config.labelIds.some((configLabel) =>
|
||||
emailLabels.includes(configLabel)
|
||||
)
|
||||
return config.labelFilterBehavior === 'INCLUDE' ? hasMatchingLabel : !hasMatchingLabel
|
||||
})
|
||||
}
|
||||
|
||||
async function processEmails(
|
||||
emails: GmailEmail[],
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
config: GmailWebhookConfig,
|
||||
accessToken: string,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
for (const email of emails) {
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'gmail',
|
||||
`${webhookData.id}:${email.id}`,
|
||||
async () => {
|
||||
const headers: Record<string, string> = {}
|
||||
const payload = email.payload as Record<string, unknown> | undefined
|
||||
if (payload?.headers && Array.isArray(payload.headers)) {
|
||||
for (const header of payload.headers as { name: string; value: string }[]) {
|
||||
headers[header.name.toLowerCase()] = header.value
|
||||
}
|
||||
}
|
||||
|
||||
let textContent = ''
|
||||
let htmlContent = ''
|
||||
|
||||
const extractContent = (part: Record<string, unknown>) => {
|
||||
if (!part) return
|
||||
|
||||
if (part.mimeType === 'text/plain') {
|
||||
const body = part.body as { data?: string } | undefined
|
||||
if (body?.data) {
|
||||
textContent = Buffer.from(body.data, 'base64').toString('utf-8')
|
||||
}
|
||||
} else if (part.mimeType === 'text/html') {
|
||||
const body = part.body as { data?: string } | undefined
|
||||
if (body?.data) {
|
||||
htmlContent = Buffer.from(body.data, 'base64').toString('utf-8')
|
||||
}
|
||||
}
|
||||
|
||||
if (part.parts && Array.isArray(part.parts)) {
|
||||
for (const subPart of part.parts) {
|
||||
extractContent(subPart as Record<string, unknown>)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (payload) {
|
||||
extractContent(payload)
|
||||
}
|
||||
|
||||
let date: string | null = null
|
||||
if (headers.date) {
|
||||
try {
|
||||
date = new Date(headers.date).toISOString()
|
||||
} catch (_e) {}
|
||||
} else if (email.internalDate) {
|
||||
date = new Date(Number.parseInt(email.internalDate)).toISOString()
|
||||
}
|
||||
|
||||
let attachments: GmailAttachment[] = []
|
||||
const hasAttachments = payload ? extractAttachmentInfo(payload).length > 0 : false
|
||||
|
||||
if (config.includeAttachments && hasAttachments && payload) {
|
||||
try {
|
||||
const attachmentInfo = extractAttachmentInfo(payload)
|
||||
attachments = await downloadAttachments(email.id, attachmentInfo, accessToken)
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[${requestId}] Error downloading attachments for email ${email.id}:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const simplifiedEmail: SimplifiedEmail = {
|
||||
id: email.id,
|
||||
threadId: email.threadId,
|
||||
subject: headers.subject || '[No Subject]',
|
||||
from: headers.from || '',
|
||||
to: headers.to || '',
|
||||
cc: headers.cc || '',
|
||||
date,
|
||||
bodyText: textContent,
|
||||
bodyHtml: htmlContent,
|
||||
labels: email.labelIds || [],
|
||||
hasAttachments,
|
||||
attachments,
|
||||
}
|
||||
|
||||
const webhookPayload: GmailWebhookPayload = {
|
||||
email: simplifiedEmail,
|
||||
timestamp: new Date().toISOString(),
|
||||
...(config.includeRawEmail ? { rawEmail: email } : {}),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
webhookPayload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for email ${email.id}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
if (config.markAsRead) {
|
||||
await markEmailAsRead(accessToken, email.id, logger)
|
||||
}
|
||||
|
||||
return { emailId: email.id, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed email ${email.id} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error processing email ${email.id}:`, errorMessage)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount }
|
||||
}
|
||||
|
||||
async function markEmailAsRead(accessToken: string, messageId: string, logger: Logger) {
|
||||
const modifyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}/modify`
|
||||
|
||||
try {
|
||||
const response = await fetch(modifyUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ removeLabelIds: ['UNREAD'] }),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {})
|
||||
throw new Error(
|
||||
`Failed to mark email ${messageId} as read: ${response.status} ${response.statusText}`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error marking email ${messageId} as read:`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3'
|
||||
const MAX_EVENTS_PER_POLL = 50
|
||||
const MAX_PAGES = 10
|
||||
|
||||
type CalendarEventTypeFilter = '' | 'created' | 'updated' | 'cancelled'
|
||||
|
||||
interface GoogleCalendarWebhookConfig {
|
||||
calendarId?: string
|
||||
manualCalendarId?: string
|
||||
eventTypeFilter?: CalendarEventTypeFilter
|
||||
searchTerm?: string
|
||||
lastCheckedTimestamp?: string
|
||||
maxEventsPerPoll?: number
|
||||
}
|
||||
|
||||
interface CalendarEventAttendee {
|
||||
email: string
|
||||
displayName?: string
|
||||
responseStatus?: string
|
||||
self?: boolean
|
||||
organizer?: boolean
|
||||
}
|
||||
|
||||
interface CalendarEventPerson {
|
||||
email: string
|
||||
displayName?: string
|
||||
self?: boolean
|
||||
}
|
||||
|
||||
interface CalendarEventTime {
|
||||
dateTime?: string
|
||||
date?: string
|
||||
timeZone?: string
|
||||
}
|
||||
|
||||
interface CalendarEvent {
|
||||
id: string
|
||||
status: string
|
||||
htmlLink?: string
|
||||
created?: string
|
||||
updated?: string
|
||||
summary?: string
|
||||
description?: string
|
||||
location?: string
|
||||
start?: CalendarEventTime
|
||||
end?: CalendarEventTime
|
||||
attendees?: CalendarEventAttendee[]
|
||||
creator?: CalendarEventPerson
|
||||
organizer?: CalendarEventPerson
|
||||
recurringEventId?: string
|
||||
}
|
||||
|
||||
interface SimplifiedCalendarEvent {
|
||||
id: string
|
||||
status: string
|
||||
eventType: 'created' | 'updated' | 'cancelled'
|
||||
summary: string | null
|
||||
eventDescription: string | null
|
||||
location: string | null
|
||||
htmlLink: string | null
|
||||
start: CalendarEventTime | null
|
||||
end: CalendarEventTime | null
|
||||
created: string | null
|
||||
updated: string | null
|
||||
attendees: CalendarEventAttendee[] | null
|
||||
creator: CalendarEventPerson | null
|
||||
organizer: CalendarEventPerson | null
|
||||
}
|
||||
|
||||
interface GoogleCalendarWebhookPayload {
|
||||
event: SimplifiedCalendarEvent
|
||||
calendarId: string
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
export const googleCalendarPollingHandler: PollingProviderHandler = {
|
||||
provider: 'google-calendar',
|
||||
label: 'Google Calendar',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'google-calendar', requestId)
|
||||
|
||||
const config = getProviderConfig<GoogleCalendarWebhookConfig>(webhookData.providerConfig)
|
||||
// Canonical key `calendarId` first; `manualCalendarId` is a transitional basic-first
|
||||
// fallback. Defaults to the user's primary calendar when none is set.
|
||||
const calendarId =
|
||||
readCanonicalTriggerValue(config.calendarId, config.manualCalendarId) || 'primary'
|
||||
|
||||
// First poll: seed timestamp, emit nothing
|
||||
if (!config.lastCheckedTimestamp) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ lastCheckedTimestamp: new Date(Date.now() - 30_000).toISOString() },
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] First poll for webhook ${webhookId}, seeded timestamp`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
// Fetch changed events since last poll
|
||||
const events = await fetchChangedEvents(accessToken, calendarId, config, requestId, logger)
|
||||
|
||||
if (!events.length) {
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No changed events for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Found ${events.length} changed events for webhook ${webhookId}`)
|
||||
|
||||
const { processedCount, failedCount, latestUpdated } = await processEvents(
|
||||
events,
|
||||
calendarId,
|
||||
config.eventTypeFilter,
|
||||
webhookData,
|
||||
workflowData,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
// Advance cursor to latestUpdated - 5s for clock-skew overlap, but never regress
|
||||
// below the previous cursor — this prevents an infinite re-fetch loop when all
|
||||
// returned events are filtered client-side and latestUpdated is within 5s of the cursor.
|
||||
const newTimestamp =
|
||||
failedCount > 0
|
||||
? config.lastCheckedTimestamp
|
||||
: latestUpdated
|
||||
? new Date(
|
||||
Math.max(
|
||||
new Date(latestUpdated).getTime() - 5000,
|
||||
new Date(config.lastCheckedTimestamp).getTime()
|
||||
)
|
||||
).toISOString()
|
||||
: config.lastCheckedTimestamp
|
||||
await updateWebhookProviderConfig(webhookId, { lastCheckedTimestamp: newTimestamp }, logger)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} events failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} events for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing Google Calendar webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function fetchChangedEvents(
|
||||
accessToken: string,
|
||||
calendarId: string,
|
||||
config: GoogleCalendarWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<CalendarEvent[]> {
|
||||
const allEvents: CalendarEvent[] = []
|
||||
const maxEvents = config.maxEventsPerPoll || MAX_EVENTS_PER_POLL
|
||||
let pageToken: string | undefined
|
||||
let pages = 0
|
||||
|
||||
do {
|
||||
pages++
|
||||
const params = new URLSearchParams({
|
||||
updatedMin: config.lastCheckedTimestamp!,
|
||||
singleEvents: 'true',
|
||||
showDeleted: 'true',
|
||||
maxResults: String(Math.min(maxEvents, 250)),
|
||||
})
|
||||
|
||||
if (pageToken) {
|
||||
params.set('pageToken', pageToken)
|
||||
}
|
||||
|
||||
if (config.searchTerm) {
|
||||
params.set('q', config.searchTerm)
|
||||
}
|
||||
|
||||
const encodedCalendarId = encodeURIComponent(calendarId)
|
||||
const url = `${CALENDAR_API_BASE}/calendars/${encodedCalendarId}/events?${params.toString()}`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const status = response.status
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
|
||||
if (status === 403 || status === 429) {
|
||||
throw new Error(
|
||||
`Calendar API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Failed to fetch calendar events: ${status} - ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const events = (data.items || []) as CalendarEvent[]
|
||||
allEvents.push(...events)
|
||||
|
||||
pageToken = data.nextPageToken as string | undefined
|
||||
|
||||
// Stop if we have enough events or hit the page limit
|
||||
if (allEvents.length >= maxEvents || pages >= MAX_PAGES) {
|
||||
break
|
||||
}
|
||||
} while (pageToken)
|
||||
|
||||
return allEvents.slice(0, maxEvents)
|
||||
}
|
||||
|
||||
function determineEventType(event: CalendarEvent): 'created' | 'updated' | 'cancelled' {
|
||||
if (event.status === 'cancelled') {
|
||||
return 'cancelled'
|
||||
}
|
||||
|
||||
// If created and updated are within 5 seconds, treat as newly created
|
||||
if (event.created && event.updated) {
|
||||
const createdTime = new Date(event.created).getTime()
|
||||
const updatedTime = new Date(event.updated).getTime()
|
||||
if (Math.abs(updatedTime - createdTime) < 5000) {
|
||||
return 'created'
|
||||
}
|
||||
}
|
||||
|
||||
return 'updated'
|
||||
}
|
||||
|
||||
function simplifyEvent(
|
||||
event: CalendarEvent,
|
||||
eventType?: 'created' | 'updated' | 'cancelled'
|
||||
): SimplifiedCalendarEvent {
|
||||
return {
|
||||
id: event.id,
|
||||
status: event.status,
|
||||
eventType: eventType ?? determineEventType(event),
|
||||
summary: event.summary ?? null,
|
||||
eventDescription: event.description ?? null,
|
||||
location: event.location ?? null,
|
||||
htmlLink: event.htmlLink ?? null,
|
||||
start: event.start ?? null,
|
||||
end: event.end ?? null,
|
||||
created: event.created ?? null,
|
||||
updated: event.updated ?? null,
|
||||
attendees: event.attendees ?? null,
|
||||
creator: event.creator ?? null,
|
||||
organizer: event.organizer ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
async function processEvents(
|
||||
events: CalendarEvent[],
|
||||
calendarId: string,
|
||||
eventTypeFilter: CalendarEventTypeFilter | undefined,
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ processedCount: number; failedCount: number; latestUpdated: string | null }> {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
let latestUpdated: string | null = null
|
||||
|
||||
for (const event of events) {
|
||||
// Track the latest `updated` timestamp for clock-skew-free state tracking
|
||||
if (event.updated) {
|
||||
if (!latestUpdated || event.updated > latestUpdated) {
|
||||
latestUpdated = event.updated
|
||||
}
|
||||
}
|
||||
|
||||
// Client-side event type filter — skip before idempotency so filtered events aren't cached
|
||||
const computedEventType = determineEventType(event)
|
||||
if (eventTypeFilter && computedEventType !== eventTypeFilter) {
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
// Idempotency key includes `updated` so re-edits of the same event re-trigger
|
||||
const idempotencyKey = `${webhookData.id}:${event.id}:${event.updated || event.created || ''}`
|
||||
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'google-calendar',
|
||||
idempotencyKey,
|
||||
async () => {
|
||||
const simplified = simplifyEvent(event, computedEventType)
|
||||
|
||||
const payload: GoogleCalendarWebhookPayload = {
|
||||
event: simplified,
|
||||
calendarId,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for event ${event.id}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
return { eventId: event.id, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed event ${event.id} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error processing event ${event.id}:`, errorMessage)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount, latestUpdated }
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
const MAX_FILES_PER_POLL = 50
|
||||
const MAX_KNOWN_FILE_IDS = 1000
|
||||
const MAX_PAGES = 10
|
||||
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3'
|
||||
|
||||
type DriveEventTypeFilter = '' | 'created' | 'modified' | 'deleted' | 'created_or_modified'
|
||||
|
||||
interface GoogleDriveWebhookConfig {
|
||||
folderId?: string
|
||||
manualFolderId?: string
|
||||
mimeTypeFilter?: string
|
||||
includeSharedDrives?: boolean
|
||||
eventTypeFilter?: DriveEventTypeFilter
|
||||
maxFilesPerPoll?: number
|
||||
pageToken?: string
|
||||
knownFileIds?: string[]
|
||||
}
|
||||
|
||||
interface DriveChangeEntry {
|
||||
kind: string
|
||||
type: string
|
||||
time: string
|
||||
removed: boolean
|
||||
fileId: string
|
||||
file?: DriveFileMetadata
|
||||
}
|
||||
|
||||
interface DriveFileMetadata {
|
||||
id: string
|
||||
name: string
|
||||
mimeType: string
|
||||
modifiedTime: string
|
||||
createdTime?: string
|
||||
size?: string
|
||||
webViewLink?: string
|
||||
parents?: string[]
|
||||
lastModifyingUser?: { displayName?: string; emailAddress?: string }
|
||||
shared?: boolean
|
||||
starred?: boolean
|
||||
trashed?: boolean
|
||||
}
|
||||
|
||||
interface GoogleDriveWebhookPayload {
|
||||
file: DriveFileMetadata | { id: string }
|
||||
eventType: 'created' | 'modified' | 'deleted'
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
const FILE_FIELDS = [
|
||||
'id',
|
||||
'name',
|
||||
'mimeType',
|
||||
'modifiedTime',
|
||||
'createdTime',
|
||||
'size',
|
||||
'webViewLink',
|
||||
'parents',
|
||||
'lastModifyingUser',
|
||||
'shared',
|
||||
'starred',
|
||||
'trashed',
|
||||
].join(',')
|
||||
|
||||
export const googleDrivePollingHandler: PollingProviderHandler = {
|
||||
provider: 'google-drive',
|
||||
label: 'Google Drive',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'google-drive', requestId)
|
||||
|
||||
const config = getProviderConfig<GoogleDriveWebhookConfig>(webhookData.providerConfig)
|
||||
|
||||
// First poll (or re-seed after 410): seed page token, preserve any existing known file IDs.
|
||||
if (!config.pageToken) {
|
||||
const startPageToken = await getStartPageToken(accessToken, config, requestId, logger)
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ pageToken: startPageToken, knownFileIds: config.knownFileIds ?? [] },
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] First poll for webhook ${webhookId}, seeded pageToken: ${startPageToken}`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const { changes, newStartPageToken } = await fetchChanges(
|
||||
accessToken,
|
||||
config,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
if (!changes.length) {
|
||||
await updateWebhookProviderConfig(webhookId, { pageToken: newStartPageToken }, logger)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No changes found for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const filteredChanges = filterChanges(changes, config)
|
||||
|
||||
if (!filteredChanges.length) {
|
||||
await updateWebhookProviderConfig(webhookId, { pageToken: newStartPageToken }, logger)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] ${changes.length} changes found but none match filters for webhook ${webhookId}`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Found ${filteredChanges.length} matching changes for webhook ${webhookId}`
|
||||
)
|
||||
|
||||
const { processedCount, failedCount, newKnownFileIds } = await processChanges(
|
||||
filteredChanges,
|
||||
config,
|
||||
webhookData,
|
||||
workflowData,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
const existingKnownIds = config.knownFileIds || []
|
||||
const mergedKnownIds = [...new Set([...newKnownFileIds, ...existingKnownIds])].slice(
|
||||
0,
|
||||
MAX_KNOWN_FILE_IDS
|
||||
)
|
||||
|
||||
const anyFailed = failedCount > 0
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
pageToken: anyFailed ? config.pageToken : newStartPageToken,
|
||||
knownFileIds: anyFailed ? existingKnownIds : mergedKnownIds,
|
||||
},
|
||||
logger
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} changes failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} changes for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === 'DrivePageTokenInvalidError') {
|
||||
await updateWebhookProviderConfig(webhookId, { pageToken: undefined }, logger)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] Drive page token invalid for webhook ${webhookId}, re-seeding on next poll`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
if (error instanceof Error && error.name === 'DriveRateLimitError') {
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] Drive API rate limited for webhook ${webhookId}, skipping to retry next poll cycle`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
logger.error(`[${requestId}] Error processing Google Drive webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
const DRIVE_RATE_LIMIT_REASONS = new Set(['rateLimitExceeded', 'userRateLimitExceeded'])
|
||||
|
||||
/** Returns true only for quota/rate-limit 403s, not permission errors. */
|
||||
function isDriveRateLimitError(status: number, errorData: Record<string, unknown>): boolean {
|
||||
if (status !== 403) return false
|
||||
const reason = (errorData as { error?: { errors?: { reason?: string }[] } })?.error?.errors?.[0]
|
||||
?.reason
|
||||
return reason !== undefined && DRIVE_RATE_LIMIT_REASONS.has(reason)
|
||||
}
|
||||
|
||||
async function getStartPageToken(
|
||||
accessToken: string,
|
||||
config: GoogleDriveWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<string> {
|
||||
const params = new URLSearchParams()
|
||||
if (config.includeSharedDrives) {
|
||||
params.set('supportsAllDrives', 'true')
|
||||
}
|
||||
|
||||
const url = `${DRIVE_API_BASE}/changes/startPageToken?${params.toString()}`
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const status = response.status
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
if (status === 429 || isDriveRateLimitError(status, errorData)) {
|
||||
const err = new Error(`Drive API rate limit (${status}): ${JSON.stringify(errorData)}`)
|
||||
err.name = 'DriveRateLimitError'
|
||||
throw err
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to get Drive start page token: ${status} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
return data.startPageToken as string
|
||||
}
|
||||
|
||||
async function fetchChanges(
|
||||
accessToken: string,
|
||||
config: GoogleDriveWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ changes: DriveChangeEntry[]; newStartPageToken: string }> {
|
||||
const allChanges: DriveChangeEntry[] = []
|
||||
let currentPageToken = config.pageToken!
|
||||
let newStartPageToken: string | undefined
|
||||
let lastNextPageToken: string | undefined
|
||||
const maxFiles = config.maxFilesPerPoll || MAX_FILES_PER_POLL
|
||||
let pages = 0
|
||||
|
||||
while (true) {
|
||||
pages++
|
||||
const params = new URLSearchParams({
|
||||
pageToken: currentPageToken,
|
||||
pageSize: String(Math.min(maxFiles, 100)),
|
||||
fields: `nextPageToken,newStartPageToken,changes(kind,type,time,removed,fileId,file(${FILE_FIELDS}))`,
|
||||
restrictToMyDrive: config.includeSharedDrives ? 'false' : 'true',
|
||||
})
|
||||
|
||||
if (config.includeSharedDrives) {
|
||||
params.set('supportsAllDrives', 'true')
|
||||
params.set('includeItemsFromAllDrives', 'true')
|
||||
}
|
||||
|
||||
const url = `${DRIVE_API_BASE}/changes?${params.toString()}`
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const status = response.status
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
if (status === 410) {
|
||||
const err = new Error('Drive page token is no longer valid')
|
||||
err.name = 'DrivePageTokenInvalidError'
|
||||
throw err
|
||||
}
|
||||
if (status === 429 || isDriveRateLimitError(status, errorData)) {
|
||||
const err = new Error(`Drive API rate limit (${status}): ${JSON.stringify(errorData)}`)
|
||||
err.name = 'DriveRateLimitError'
|
||||
throw err
|
||||
}
|
||||
throw new Error(`Failed to fetch Drive changes: ${status} - ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const changes = (data.changes || []) as DriveChangeEntry[]
|
||||
allChanges.push(...changes)
|
||||
|
||||
if (data.newStartPageToken) {
|
||||
newStartPageToken = data.newStartPageToken as string
|
||||
}
|
||||
|
||||
const hasMore = !!data.nextPageToken
|
||||
const overLimit = allChanges.length >= maxFiles
|
||||
|
||||
if (!hasMore || overLimit || pages >= MAX_PAGES) {
|
||||
if (hasMore) {
|
||||
lastNextPageToken = data.nextPageToken as string
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
lastNextPageToken = data.nextPageToken as string
|
||||
currentPageToken = data.nextPageToken as string
|
||||
}
|
||||
|
||||
// When allChanges exceeds maxFiles (multi-page overshoot), resume mid-list via lastNextPageToken.
|
||||
// Otherwise resume from newStartPageToken (end of change list) or lastNextPageToken (MAX_PAGES hit).
|
||||
const slicingOccurs = allChanges.length > maxFiles
|
||||
const resumeToken = slicingOccurs
|
||||
? (lastNextPageToken ?? newStartPageToken!)
|
||||
: (newStartPageToken ?? lastNextPageToken!)
|
||||
|
||||
return { changes: allChanges.slice(0, maxFiles), newStartPageToken: resumeToken }
|
||||
}
|
||||
|
||||
function filterChanges(
|
||||
changes: DriveChangeEntry[],
|
||||
config: GoogleDriveWebhookConfig
|
||||
): DriveChangeEntry[] {
|
||||
return changes.filter((change) => {
|
||||
if (change.removed) return true
|
||||
|
||||
const file = change.file
|
||||
if (!file) return false
|
||||
|
||||
if (file.trashed) return false
|
||||
|
||||
// Canonical key `folderId` first; `manualFolderId` is a transitional basic-first fallback.
|
||||
const folderId = readCanonicalTriggerValue(config.folderId, config.manualFolderId)
|
||||
if (folderId) {
|
||||
if (!file.parents || !file.parents.includes(folderId)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if (config.mimeTypeFilter) {
|
||||
if (config.mimeTypeFilter.endsWith('/')) {
|
||||
if (!file.mimeType.startsWith(config.mimeTypeFilter)) {
|
||||
return false
|
||||
}
|
||||
} else if (file.mimeType !== config.mimeTypeFilter) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
async function processChanges(
|
||||
changes: DriveChangeEntry[],
|
||||
config: GoogleDriveWebhookConfig,
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ processedCount: number; failedCount: number; newKnownFileIds: string[] }> {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
const newKnownFileIds: string[] = []
|
||||
const knownFileIdsSet = new Set(config.knownFileIds || [])
|
||||
|
||||
for (const change of changes) {
|
||||
let eventType: 'created' | 'modified' | 'deleted'
|
||||
if (change.removed) {
|
||||
eventType = 'deleted'
|
||||
} else if (!knownFileIdsSet.has(change.fileId)) {
|
||||
eventType = 'created'
|
||||
} else {
|
||||
eventType = 'modified'
|
||||
}
|
||||
|
||||
// Track file as known regardless of filter so future changes are correctly classified
|
||||
if (!change.removed) {
|
||||
newKnownFileIds.push(change.fileId)
|
||||
}
|
||||
|
||||
// Apply event type filter before idempotency so filtered events aren't cached
|
||||
const filter = config.eventTypeFilter
|
||||
if (filter) {
|
||||
const skip = filter === 'created_or_modified' ? eventType === 'deleted' : eventType !== filter
|
||||
if (skip) continue
|
||||
}
|
||||
|
||||
try {
|
||||
const idempotencyKey = `${webhookData.id}:${change.fileId}:${change.time || change.fileId}`
|
||||
|
||||
await pollingIdempotency.executeWithIdempotency('google-drive', idempotencyKey, async () => {
|
||||
const payload: GoogleDriveWebhookPayload = {
|
||||
file: change.file || { id: change.fileId },
|
||||
eventType,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for file ${change.fileId}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
return { fileId: change.fileId, processed: true }
|
||||
})
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed change for file ${change.fileId} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(
|
||||
`[${requestId}] Error processing change for file ${change.fileId}:`,
|
||||
errorMessage
|
||||
)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount, newKnownFileIds }
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
const MAX_ROWS_PER_POLL = 100
|
||||
|
||||
/** Maximum number of leading rows to scan when auto-detecting the header row. */
|
||||
const HEADER_SCAN_ROWS = 10
|
||||
|
||||
type ValueRenderOption = 'FORMATTED_VALUE' | 'UNFORMATTED_VALUE' | 'FORMULA'
|
||||
type DateTimeRenderOption = 'SERIAL_NUMBER' | 'FORMATTED_STRING'
|
||||
|
||||
interface GoogleSheetsWebhookConfig {
|
||||
spreadsheetId?: string
|
||||
manualSpreadsheetId?: string
|
||||
sheetName?: string
|
||||
manualSheetName?: string
|
||||
valueRenderOption?: ValueRenderOption
|
||||
dateTimeRenderOption?: DateTimeRenderOption
|
||||
/** 1-indexed row number of the last row seeded or processed. */
|
||||
lastIndexChecked?: number
|
||||
lastModifiedTime?: string
|
||||
lastCheckedTimestamp?: string
|
||||
maxRowsPerPoll?: number
|
||||
}
|
||||
|
||||
interface GoogleSheetsWebhookPayload {
|
||||
row: Record<string, string> | null
|
||||
rawRow: string[]
|
||||
headers: string[]
|
||||
rowNumber: number
|
||||
spreadsheetId: string
|
||||
sheetName: string
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
export const googleSheetsPollingHandler: PollingProviderHandler = {
|
||||
provider: 'google-sheets',
|
||||
label: 'Google Sheets',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'google-sheets', requestId)
|
||||
|
||||
const config = getProviderConfig<GoogleSheetsWebhookConfig>(webhookData.providerConfig)
|
||||
// Canonical keys (`spreadsheetId`/`sheetName`) first; the `manual*` keys are a transitional
|
||||
// basic-first fallback for configs deployed before the canonical key existed.
|
||||
const spreadsheetId = readCanonicalTriggerValue(
|
||||
config.spreadsheetId,
|
||||
config.manualSpreadsheetId
|
||||
)
|
||||
const sheetName = readCanonicalTriggerValue(config.sheetName, config.manualSheetName)
|
||||
const now = new Date()
|
||||
|
||||
if (!spreadsheetId || !sheetName) {
|
||||
logger.error(`[${requestId}] Missing spreadsheetId or sheetName for webhook ${webhookId}`)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
const { unchanged: skipPoll, currentModifiedTime } = await isDriveFileUnchanged(
|
||||
accessToken,
|
||||
spreadsheetId,
|
||||
config.lastModifiedTime,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
if (skipPoll) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ lastCheckedTimestamp: now.toISOString() },
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] Sheet not modified since last poll for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const valueRender = config.valueRenderOption || 'FORMATTED_VALUE'
|
||||
const dateTimeRender = config.dateTimeRenderOption || 'SERIAL_NUMBER'
|
||||
|
||||
const {
|
||||
rowCount: currentRowCount,
|
||||
headers,
|
||||
headerRowIndex,
|
||||
} = await fetchSheetState(
|
||||
accessToken,
|
||||
spreadsheetId,
|
||||
sheetName,
|
||||
valueRender,
|
||||
dateTimeRender,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
// First poll: seed state, emit nothing
|
||||
if (config.lastIndexChecked === undefined) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastIndexChecked: currentRowCount,
|
||||
lastModifiedTime: currentModifiedTime ?? config.lastModifiedTime,
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] First poll for webhook ${webhookId}, seeded row index: ${currentRowCount}`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
if (currentRowCount <= config.lastIndexChecked) {
|
||||
if (currentRowCount < config.lastIndexChecked) {
|
||||
logger.warn(
|
||||
`[${requestId}] Row count decreased from ${config.lastIndexChecked} to ${currentRowCount} for webhook ${webhookId}`
|
||||
)
|
||||
}
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastIndexChecked: currentRowCount,
|
||||
lastModifiedTime: currentModifiedTime ?? config.lastModifiedTime,
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new rows for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const newRowCount = currentRowCount - config.lastIndexChecked
|
||||
const maxRows = config.maxRowsPerPoll || MAX_ROWS_PER_POLL
|
||||
const rowsToFetch = Math.min(newRowCount, maxRows)
|
||||
const startRow = config.lastIndexChecked + 1
|
||||
const endRow = config.lastIndexChecked + rowsToFetch
|
||||
|
||||
// Skip past the header row (and any blank rows above it) so it is never
|
||||
// emitted as a data event.
|
||||
const adjustedStartRow =
|
||||
headerRowIndex > 0 ? Math.max(startRow, headerRowIndex + 1) : startRow
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Found ${newRowCount} new rows for webhook ${webhookId}, processing rows ${adjustedStartRow}-${endRow}`
|
||||
)
|
||||
|
||||
// Entire batch is header/blank rows — advance pointer and skip fetch.
|
||||
if (adjustedStartRow > endRow) {
|
||||
const hasRemainingRows = rowsToFetch < newRowCount
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastIndexChecked: config.lastIndexChecked + rowsToFetch,
|
||||
lastModifiedTime: hasRemainingRows
|
||||
? config.lastModifiedTime
|
||||
: (currentModifiedTime ?? config.lastModifiedTime),
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Batch ${startRow}-${endRow} contained only header/blank rows for webhook ${webhookId}, advancing pointer`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const newRows = await fetchRowRange(
|
||||
accessToken,
|
||||
spreadsheetId,
|
||||
sheetName,
|
||||
adjustedStartRow,
|
||||
endRow,
|
||||
valueRender,
|
||||
dateTimeRender,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
const { processedCount, failedCount } = await processRows(
|
||||
newRows,
|
||||
headers,
|
||||
adjustedStartRow,
|
||||
spreadsheetId,
|
||||
sheetName,
|
||||
webhookData,
|
||||
workflowData,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
const rowsAdvanced = failedCount > 0 ? 0 : rowsToFetch
|
||||
const newLastIndexChecked = config.lastIndexChecked + rowsAdvanced
|
||||
const hasRemainingOrFailed = rowsAdvanced < newRowCount
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastIndexChecked: newLastIndexChecked,
|
||||
lastModifiedTime: hasRemainingOrFailed
|
||||
? config.lastModifiedTime
|
||||
: (currentModifiedTime ?? config.lastModifiedTime),
|
||||
lastCheckedTimestamp: now.toISOString(),
|
||||
},
|
||||
logger
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} rows failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} rows for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing Google Sheets webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function isDriveFileUnchanged(
|
||||
accessToken: string,
|
||||
spreadsheetId: string,
|
||||
lastModifiedTime: string | undefined,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ unchanged: boolean; currentModifiedTime?: string }> {
|
||||
try {
|
||||
const currentModifiedTime = await getDriveFileModifiedTime(accessToken, spreadsheetId, logger)
|
||||
if (!lastModifiedTime || !currentModifiedTime) {
|
||||
return { unchanged: false, currentModifiedTime }
|
||||
}
|
||||
return { unchanged: currentModifiedTime === lastModifiedTime, currentModifiedTime }
|
||||
} catch (error) {
|
||||
logger.warn(`[${requestId}] Drive modifiedTime check failed, proceeding with Sheets API`)
|
||||
return { unchanged: false }
|
||||
}
|
||||
}
|
||||
|
||||
async function getDriveFileModifiedTime(
|
||||
accessToken: string,
|
||||
fileId: string,
|
||||
logger: Logger
|
||||
): Promise<string | undefined> {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`https://www.googleapis.com/drive/v3/files/${fileId}?fields=modifiedTime`,
|
||||
{ headers: { Authorization: `Bearer ${accessToken}` } }
|
||||
)
|
||||
if (!response.ok) return undefined
|
||||
const data = await response.json()
|
||||
return data.modifiedTime as string | undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the sheet (A:Z) and returns the row count, auto-detected headers,
|
||||
* and the 1-indexed header row number in a single API call.
|
||||
*
|
||||
* The Sheets API omits trailing empty rows, so `rows.length` equals the last
|
||||
* non-empty row in columns A–Z. Header detection scans the first
|
||||
* {@link HEADER_SCAN_ROWS} rows for the first non-empty row. Returns
|
||||
* `headerRowIndex = 0` when no header is found within the scan window.
|
||||
*/
|
||||
async function fetchSheetState(
|
||||
accessToken: string,
|
||||
spreadsheetId: string,
|
||||
sheetName: string,
|
||||
valueRenderOption: ValueRenderOption,
|
||||
dateTimeRenderOption: DateTimeRenderOption,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ rowCount: number; headers: string[]; headerRowIndex: number }> {
|
||||
const encodedSheet = encodeURIComponent(sheetName)
|
||||
const params = new URLSearchParams({
|
||||
majorDimension: 'ROWS',
|
||||
fields: 'values',
|
||||
valueRenderOption,
|
||||
dateTimeRenderOption,
|
||||
})
|
||||
const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!A:Z?${params.toString()}`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const status = response.status
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
if (status === 403 || status === 429) {
|
||||
throw new Error(
|
||||
`Sheets API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to fetch sheet state: ${status} ${response.statusText} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const rows = (data.values as string[][] | undefined) ?? []
|
||||
const rowCount = rows.length
|
||||
|
||||
let headers: string[] = []
|
||||
let headerRowIndex = 0
|
||||
for (let i = 0; i < Math.min(rows.length, HEADER_SCAN_ROWS); i++) {
|
||||
const row = rows[i]
|
||||
if (row?.some((cell) => cell !== '')) {
|
||||
headers = row
|
||||
headerRowIndex = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return { rowCount, headers, headerRowIndex }
|
||||
}
|
||||
|
||||
async function fetchRowRange(
|
||||
accessToken: string,
|
||||
spreadsheetId: string,
|
||||
sheetName: string,
|
||||
startRow: number,
|
||||
endRow: number,
|
||||
valueRenderOption: ValueRenderOption,
|
||||
dateTimeRenderOption: DateTimeRenderOption,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<string[][]> {
|
||||
const encodedSheet = encodeURIComponent(sheetName)
|
||||
const params = new URLSearchParams({
|
||||
fields: 'values',
|
||||
valueRenderOption,
|
||||
dateTimeRenderOption,
|
||||
})
|
||||
const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!${startRow}:${endRow}?${params.toString()}`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const status = response.status
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
if (status === 403 || status === 429) {
|
||||
throw new Error(
|
||||
`Sheets API rate limit (${status}) — skipping to retry next poll cycle: ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
throw new Error(
|
||||
`Failed to fetch rows ${startRow}-${endRow}: ${status} ${response.statusText} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
return (data.values as string[][]) ?? []
|
||||
}
|
||||
|
||||
async function processRows(
|
||||
rows: string[][],
|
||||
headers: string[],
|
||||
startRowIndex: number,
|
||||
spreadsheetId: string,
|
||||
sheetName: string,
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ processedCount: number; failedCount: number }> {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const row = rows[i]
|
||||
const rowNumber = startRowIndex + i
|
||||
|
||||
// Skip empty rows — don't fire a workflow run with no data.
|
||||
if (!row || row.length === 0) {
|
||||
logger.info(`[${requestId}] Skipping empty row ${rowNumber} for webhook ${webhookData.id}`)
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'google-sheets',
|
||||
`${webhookData.id}:${spreadsheetId}:${sheetName}:row${rowNumber}`,
|
||||
async () => {
|
||||
let mappedRow: Record<string, string> | null = null
|
||||
if (headers.length > 0) {
|
||||
mappedRow = {}
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
mappedRow[headers[j] || `Column ${j + 1}`] = row[j] ?? ''
|
||||
}
|
||||
for (let j = headers.length; j < row.length; j++) {
|
||||
mappedRow[`Column ${j + 1}`] = row[j] ?? ''
|
||||
}
|
||||
}
|
||||
|
||||
const payload: GoogleSheetsWebhookPayload = {
|
||||
row: mappedRow,
|
||||
rawRow: row,
|
||||
headers,
|
||||
rowNumber,
|
||||
spreadsheetId,
|
||||
sheetName,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for row ${rowNumber}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
return { rowNumber, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed row ${rowNumber} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error processing row ${rowNumber}:`, errorMessage)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount }
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildUserFilters } from '@/lib/webhooks/polling/hubspot'
|
||||
|
||||
describe('buildUserFilters', () => {
|
||||
it('translates pipeline/stage/owner shortcuts into EQ filters', () => {
|
||||
const filters = buildUserFilters({
|
||||
objectType: 'deal',
|
||||
pipelineId: 'pipeline-1',
|
||||
stageId: 'stage-1',
|
||||
ownerId: 'owner-1',
|
||||
})
|
||||
|
||||
expect(filters).toEqual([
|
||||
{ propertyName: 'pipeline', operator: 'EQ', value: 'pipeline-1' },
|
||||
{ propertyName: 'dealstage', operator: 'EQ', value: 'stage-1' },
|
||||
{ propertyName: 'hubspot_owner_id', operator: 'EQ', value: 'owner-1' },
|
||||
])
|
||||
})
|
||||
|
||||
it('uses ticket-specific pipeline/stage property names', () => {
|
||||
const filters = buildUserFilters({
|
||||
objectType: 'ticket',
|
||||
pipelineId: 'pipeline-1',
|
||||
stageId: 'stage-1',
|
||||
})
|
||||
|
||||
expect(filters).toEqual([
|
||||
{ propertyName: 'hs_pipeline', operator: 'EQ', value: 'pipeline-1' },
|
||||
{ propertyName: 'hs_pipeline_stage', operator: 'EQ', value: 'stage-1' },
|
||||
])
|
||||
})
|
||||
|
||||
it('parses advanced JSON filters and preserves values arrays', () => {
|
||||
const filters = buildUserFilters({
|
||||
filters: JSON.stringify([
|
||||
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
|
||||
{ propertyName: 'dealstage', operator: 'IN', values: ['a', 'b'] },
|
||||
]),
|
||||
})
|
||||
|
||||
expect(filters).toEqual([
|
||||
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
|
||||
{ propertyName: 'dealstage', operator: 'IN', values: ['a', 'b'] },
|
||||
])
|
||||
})
|
||||
|
||||
it('drops filter entries with an unrecognized operator', () => {
|
||||
const filters = buildUserFilters({
|
||||
filters: JSON.stringify([
|
||||
{ propertyName: 'amount', operator: 'STARTS_WITH', value: '1' },
|
||||
{ propertyName: 'amount', operator: 'GT', value: '1' },
|
||||
]),
|
||||
})
|
||||
|
||||
expect(filters).toEqual([{ propertyName: 'amount', operator: 'GT', value: '1' }])
|
||||
})
|
||||
|
||||
it('ignores malformed JSON filters without throwing', () => {
|
||||
expect(() => buildUserFilters({ filters: 'not json' })).not.toThrow()
|
||||
expect(buildUserFilters({ filters: 'not json' })).toEqual([])
|
||||
})
|
||||
|
||||
it('allows exactly the HubSpot per-group limit of combined filters', () => {
|
||||
const filters = buildUserFilters({
|
||||
objectType: 'deal',
|
||||
pipelineId: 'pipeline-1',
|
||||
stageId: 'stage-1',
|
||||
ownerId: 'owner-1',
|
||||
filters: JSON.stringify([{ propertyName: 'amount', operator: 'GT', value: '1000' }]),
|
||||
})
|
||||
|
||||
// 3 shortcuts + 1 advanced = 4, exactly MAX_USER_FILTERS.
|
||||
expect(filters).toHaveLength(4)
|
||||
})
|
||||
|
||||
it('throws rather than silently dropping filters when the combined count exceeds the limit', () => {
|
||||
// Filters within a filterGroup are AND-combined, so silently dropping one would widen
|
||||
// the match set instead of narrowing it — throwing surfaces the misconfiguration loudly.
|
||||
expect(() =>
|
||||
buildUserFilters({
|
||||
objectType: 'deal',
|
||||
pipelineId: 'pipeline-1',
|
||||
stageId: 'stage-1',
|
||||
ownerId: 'owner-1',
|
||||
filters: JSON.stringify([
|
||||
{ propertyName: 'amount', operator: 'GT', value: '1000' },
|
||||
{ propertyName: 'lifecyclestage', operator: 'EQ', value: 'customer' },
|
||||
]),
|
||||
})
|
||||
).toThrow(/exceeding the 4-filter limit/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,990 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
type HubSpotBuiltInObjectType = 'contact' | 'company' | 'deal' | 'ticket'
|
||||
type HubSpotEventType = 'created' | 'updated' | 'property_changed'
|
||||
|
||||
interface FilterClause {
|
||||
propertyName: string
|
||||
operator: string
|
||||
value?: string
|
||||
values?: string[]
|
||||
}
|
||||
|
||||
interface HubSpotWebhookConfig {
|
||||
credentialId?: string
|
||||
/**
|
||||
* Built-in slug, 'custom' (defers to customObjectTypeId), 'list_membership',
|
||||
* or a raw HubSpot custom object type id ('2-12345').
|
||||
*/
|
||||
objectType?: string
|
||||
customObjectTypeId?: string
|
||||
listId?: string
|
||||
eventType?: HubSpotEventType
|
||||
targetPropertyName?: string
|
||||
properties?: string[] | string
|
||||
pipelineId?: string
|
||||
stageId?: string
|
||||
ownerId?: string
|
||||
/** User-supplied AND-combined filters — string list or JSON-array string. */
|
||||
filters?: string | FilterClause[]
|
||||
maxRecordsPerPoll?: number
|
||||
lastSeenTimestampMs?: string
|
||||
lastSeenObjectId?: string
|
||||
/**
|
||||
* List-membership cursor — the `after` value to pass to the next
|
||||
* `/lists/{id}/memberships/join-order` request. The HubSpot endpoint walks
|
||||
* ASC by default (oldest first); we use cursor-based resume because
|
||||
* `before`-mode (DESC) has ambiguous bootstrap semantics.
|
||||
*/
|
||||
lastSeenMembershipCursor?: string
|
||||
/**
|
||||
* True once we've walked to the end of the list once. While this is false we
|
||||
* are still in the seed pass — we paginate forward without emitting so the
|
||||
* workflow doesn't see a flood of historical members on activation.
|
||||
*/
|
||||
membershipSeedComplete?: boolean
|
||||
/**
|
||||
* Snapshot of the watched property's last-seen value per record (property_changed event).
|
||||
* Persisted as an entries array (not a `Record`) because HubSpot record ids are numeric
|
||||
* strings, and JS engines enumerate integer-indexed object keys in numeric order
|
||||
* regardless of insertion — which would break LRU-style trimming. Array order is stable.
|
||||
*/
|
||||
propertySnapshot?: {
|
||||
property: string
|
||||
/** `[recordId, value]` pairs in LRU order — oldest first. */
|
||||
entries: Array<[string, string | null]>
|
||||
}
|
||||
lastCheckedTimestamp?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory snapshot state used during a single poll. Uses a Map (not a plain object) so
|
||||
* insertion-order iteration is honored even for numeric-string keys.
|
||||
*/
|
||||
interface PropertySnapshotState {
|
||||
property: string
|
||||
values: Map<string, string | null>
|
||||
}
|
||||
|
||||
interface HubSpotSearchResult {
|
||||
id: string
|
||||
properties: Record<string, string | null>
|
||||
createdAt: string
|
||||
updatedAt: string
|
||||
archived: boolean
|
||||
}
|
||||
|
||||
interface HubSpotSearchResponse {
|
||||
total: number
|
||||
results: HubSpotSearchResult[]
|
||||
paging?: { next?: { after?: string } }
|
||||
}
|
||||
|
||||
const HUBSPOT_PAGE_LIMIT = 100
|
||||
const DEFAULT_MAX_RECORDS = 50
|
||||
const MAX_MAX_RECORDS = 1000
|
||||
/** HubSpot Search API: 10k result hard cap, 5 req/s rate limit. */
|
||||
const MAX_PAGES_PER_POLL = 10
|
||||
/** Cap on property-change snapshot size to bound providerConfig payload. */
|
||||
const MAX_SNAPSHOT_SIZE = 1000
|
||||
/**
|
||||
* HubSpot Search API caps each filterGroup at 6 filters (developers.hubspot.com/docs/api/crm/search).
|
||||
* `buildBody` reserves 2 slots in Group B (filterProperty EQ + hs_object_id GT), so
|
||||
* user-supplied filters (pipeline/stage/owner shortcuts plus advanced JSON filters) must
|
||||
* leave room for those — cap at 4 so Group B never exceeds 6.
|
||||
*/
|
||||
const MAX_USER_FILTERS = 4
|
||||
|
||||
const BUILT_IN_PATH: Record<HubSpotBuiltInObjectType, string> = {
|
||||
contact: 'contacts',
|
||||
company: 'companies',
|
||||
deal: 'deals',
|
||||
ticket: 'tickets',
|
||||
}
|
||||
|
||||
const VALID_OPERATORS = new Set([
|
||||
'EQ',
|
||||
'NEQ',
|
||||
'CONTAINS_TOKEN',
|
||||
'NOT_CONTAINS_TOKEN',
|
||||
'GT',
|
||||
'GTE',
|
||||
'LT',
|
||||
'LTE',
|
||||
'BETWEEN',
|
||||
'IN',
|
||||
'NOT_IN',
|
||||
'HAS_PROPERTY',
|
||||
'NOT_HAS_PROPERTY',
|
||||
])
|
||||
|
||||
function resolveSearchPath(objectType: string): string {
|
||||
if (objectType in BUILT_IN_PATH) {
|
||||
return BUILT_IN_PATH[objectType as HubSpotBuiltInObjectType]
|
||||
}
|
||||
return objectType
|
||||
}
|
||||
|
||||
/** Contacts use `lastmodifieddate`; the `hs_lastmodifieddate` property is null on contacts. */
|
||||
function resolveModifiedDateProperty(objectType: string): string {
|
||||
return objectType === 'contact' ? 'lastmodifieddate' : 'hs_lastmodifieddate'
|
||||
}
|
||||
|
||||
const DEFAULT_PROPERTIES: Record<HubSpotBuiltInObjectType, string[]> = {
|
||||
contact: [
|
||||
'firstname',
|
||||
'lastname',
|
||||
'email',
|
||||
'phone',
|
||||
'company',
|
||||
'lifecyclestage',
|
||||
'hs_lead_status',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'lastmodifieddate',
|
||||
],
|
||||
company: [
|
||||
'name',
|
||||
'domain',
|
||||
'industry',
|
||||
'lifecyclestage',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
deal: [
|
||||
'dealname',
|
||||
'amount',
|
||||
'dealstage',
|
||||
'pipeline',
|
||||
'closedate',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
ticket: [
|
||||
'subject',
|
||||
'content',
|
||||
'hs_pipeline',
|
||||
'hs_pipeline_stage',
|
||||
'hs_ticket_priority',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
}
|
||||
|
||||
export const hubspotPollingHandler: PollingProviderHandler = {
|
||||
provider: 'hubspot',
|
||||
label: 'HubSpot',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'hubspot', requestId)
|
||||
const config = getProviderConfig<HubSpotWebhookConfig>(webhookData.providerConfig)
|
||||
|
||||
if (config.objectType === 'list_membership') {
|
||||
return await pollListMembership(ctx, config, accessToken)
|
||||
}
|
||||
return await pollSearchBased(ctx, config, accessToken)
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing HubSpot webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function pollSearchBased(
|
||||
ctx: PollWebhookContext,
|
||||
config: HubSpotWebhookConfig,
|
||||
accessToken: string
|
||||
): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
const objectType = resolveObjectType(config)
|
||||
const eventType = config.eventType
|
||||
if (!objectType) {
|
||||
throw new Error(`HubSpot webhook ${webhookId} is missing objectType`)
|
||||
}
|
||||
if (eventType !== 'created' && eventType !== 'updated' && eventType !== 'property_changed') {
|
||||
throw new Error(`HubSpot webhook ${webhookId} is missing or has invalid eventType`)
|
||||
}
|
||||
if (eventType === 'property_changed' && !config.targetPropertyName?.trim()) {
|
||||
throw new Error(
|
||||
`HubSpot webhook ${webhookId} uses property_changed event but has no targetPropertyName`
|
||||
)
|
||||
}
|
||||
|
||||
// property_changed walks the modified-date stream and diffs the watched property locally.
|
||||
const filterProperty =
|
||||
eventType === 'created' ? 'createdate' : resolveModifiedDateProperty(objectType)
|
||||
const nowMs = Date.now()
|
||||
|
||||
if (!config.lastSeenTimestampMs) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastSeenTimestampMs: String(nowMs),
|
||||
lastCheckedTimestamp: new Date(nowMs).toISOString(),
|
||||
...(eventType === 'property_changed'
|
||||
? {
|
||||
propertySnapshot: {
|
||||
property: config.targetPropertyName?.trim() ?? '',
|
||||
entries: [],
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Seeded HubSpot webhook ${webhookId} watermark to ${nowMs} (${objectType}/${eventType}/${filterProperty})`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
const watermarkMs = Number(config.lastSeenTimestampMs)
|
||||
if (!Number.isFinite(watermarkMs)) {
|
||||
throw new Error(
|
||||
`HubSpot webhook ${webhookId} has corrupt watermark ${config.lastSeenTimestampMs}`
|
||||
)
|
||||
}
|
||||
|
||||
const userFilters = buildUserFilters(config, logger, requestId)
|
||||
const properties = resolveRequestedProperties(config, objectType, filterProperty, userFilters)
|
||||
const maxRecords = Math.min(
|
||||
Math.max(config.maxRecordsPerPoll ?? DEFAULT_MAX_RECORDS, 1),
|
||||
MAX_MAX_RECORDS
|
||||
)
|
||||
const lastSeenObjectId = config.lastSeenObjectId
|
||||
|
||||
const records = await fetchHubSpotChanges({
|
||||
accessToken,
|
||||
objectType,
|
||||
filterProperty,
|
||||
watermarkMs,
|
||||
lastSeenObjectId,
|
||||
properties,
|
||||
userFilters,
|
||||
maxRecords,
|
||||
requestId,
|
||||
logger,
|
||||
})
|
||||
|
||||
if (records.length === 0) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ lastCheckedTimestamp: new Date(nowMs).toISOString() },
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new HubSpot ${objectType} ${eventType} for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Found ${records.length} HubSpot ${objectType} ${eventType} candidates for webhook ${webhookId}`
|
||||
)
|
||||
|
||||
const targetProperty = config.targetPropertyName?.trim() || undefined
|
||||
const snapshotForRun =
|
||||
eventType === 'property_changed' && targetProperty
|
||||
? resolvePropertySnapshot(config, targetProperty)
|
||||
: null
|
||||
|
||||
const { processedCount, failedCount, skippedCount, highestSeenMs, maxIdAtHighestTimestamp } =
|
||||
await processRecords(
|
||||
records,
|
||||
webhookData,
|
||||
workflowData,
|
||||
objectType,
|
||||
eventType,
|
||||
filterProperty,
|
||||
targetProperty,
|
||||
snapshotForRun,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
const newTimestampMs = highestSeenMs > watermarkMs ? highestSeenMs : watermarkMs
|
||||
const newObjectId = maxIdAtHighestTimestamp || lastSeenObjectId || ''
|
||||
|
||||
const update: Record<string, unknown> = {
|
||||
lastSeenTimestampMs: String(newTimestampMs),
|
||||
lastSeenObjectId: newObjectId,
|
||||
lastCheckedTimestamp: new Date(nowMs).toISOString(),
|
||||
}
|
||||
if (snapshotForRun) {
|
||||
update.propertySnapshot = serializeSnapshot(snapshotForRun)
|
||||
}
|
||||
await updateWebhookProviderConfig(webhookId, update, logger)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} HubSpot records failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Processed ${processedCount} HubSpot records${skippedCount ? `, skipped ${skippedCount} (no property change)` : ''}${failedCount ? `, ${failedCount} failed` : ''} for webhook ${webhookId}`
|
||||
)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
async function pollListMembership(
|
||||
ctx: PollWebhookContext,
|
||||
config: HubSpotWebhookConfig,
|
||||
accessToken: string
|
||||
): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
const listId = config.listId?.trim()
|
||||
if (!listId) {
|
||||
throw new Error(`HubSpot list_membership trigger ${webhookId} is missing listId`)
|
||||
}
|
||||
const nowMs = Date.now()
|
||||
const seedComplete = config.membershipSeedComplete === true
|
||||
const maxRecords = Math.min(
|
||||
Math.max(config.maxRecordsPerPoll ?? DEFAULT_MAX_RECORDS, 1),
|
||||
MAX_MAX_RECORDS
|
||||
)
|
||||
|
||||
// The HubSpot endpoint walks ASC by default. We resume from a stored `after` cursor —
|
||||
// empty cursor means "from the beginning of the list". During the seed pass we paginate
|
||||
// forward without emitting; once we reach the end (no `paging.next.after`) we mark the
|
||||
// seed complete and re-fetch from the cursor-to-last-page on each normal poll. New
|
||||
// members appended to the list show up on subsequent fetches; the idempotency layer
|
||||
// dedups the records we've already seen on the boundary page.
|
||||
const result = await fetchListMembershipPages({
|
||||
listId,
|
||||
accessToken,
|
||||
initialAfter: config.lastSeenMembershipCursor?.trim() || undefined,
|
||||
pageLimit: seedComplete ? maxRecords : MAX_PAGES_PER_POLL * HUBSPOT_PAGE_LIMIT,
|
||||
requestId,
|
||||
logger,
|
||||
})
|
||||
|
||||
if (!seedComplete) {
|
||||
// Seed phase: don't emit. Just save the cursor and (when reached) flip the flag.
|
||||
const update: Record<string, unknown> = {
|
||||
lastCheckedTimestamp: new Date(nowMs).toISOString(),
|
||||
lastSeenMembershipCursor: result.resumeCursor ?? '',
|
||||
}
|
||||
if (result.reachedEnd) {
|
||||
update.membershipSeedComplete = true
|
||||
logger.info(
|
||||
`[${requestId}] HubSpot list_membership ${webhookId} seed complete (list=${listId})`
|
||||
)
|
||||
} else {
|
||||
logger.info(
|
||||
`[${requestId}] HubSpot list_membership ${webhookId} seed in progress (list=${listId}, scanned ${result.scanned})`
|
||||
)
|
||||
}
|
||||
await updateWebhookProviderConfig(webhookId, update, logger)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
if (result.records.length === 0) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastCheckedTimestamp: new Date(nowMs).toISOString(),
|
||||
lastSeenMembershipCursor: result.resumeCursor ?? '',
|
||||
},
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Found ${result.records.length} HubSpot list memberships for webhook ${webhookId}`
|
||||
)
|
||||
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
for (const member of result.records) {
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'hubspot',
|
||||
`${webhookId}:list_membership:${listId}:${member.recordId}:${member.membershipTimestamp}`,
|
||||
async () => {
|
||||
const payload = {
|
||||
objectType: 'list_membership',
|
||||
eventType: 'joined',
|
||||
objectId: member.recordId,
|
||||
occurredAt: member.membershipTimestamp,
|
||||
listId,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
const wfResult = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
if (!wfResult.success) {
|
||||
throw new Error(
|
||||
`Webhook processing failed (${wfResult.statusCode}): ${wfResult.error ?? 'unknown'}`
|
||||
)
|
||||
}
|
||||
return { recordId: member.recordId, processed: true }
|
||||
}
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
failedCount++
|
||||
logger.error(
|
||||
`[${requestId}] Error processing HubSpot list membership ${member.recordId}:`,
|
||||
getErrorMessage(error, 'Unknown error')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// HubSpot's `paging.next.after` is page-granular — there's no per-record cursor we can
|
||||
// freeze on. Advance the cursor only when the entire batch succeeded; otherwise replay
|
||||
// the page next poll and let idempotency dedup the records that already landed.
|
||||
const advanceCursor = failedCount === 0
|
||||
const nextCursor = advanceCursor
|
||||
? (result.resumeCursor ?? '')
|
||||
: (config.lastSeenMembershipCursor?.trim() ?? '')
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastSeenMembershipCursor: nextCursor,
|
||||
lastCheckedTimestamp: new Date(nowMs).toISOString(),
|
||||
},
|
||||
logger
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
function resolveObjectType(config: HubSpotWebhookConfig): string {
|
||||
const raw = config.objectType?.trim()
|
||||
if (raw === 'custom') {
|
||||
return config.customObjectTypeId?.trim() ?? ''
|
||||
}
|
||||
return raw ?? ''
|
||||
}
|
||||
|
||||
function resolveRequestedProperties(
|
||||
config: HubSpotWebhookConfig,
|
||||
objectType: string,
|
||||
filterProperty: string,
|
||||
userFilters: FilterClause[]
|
||||
): string[] {
|
||||
const requested = new Set<string>()
|
||||
|
||||
const userProperties = Array.isArray(config.properties)
|
||||
? config.properties
|
||||
: typeof config.properties === 'string'
|
||||
? config.properties.split(/[\n,]/)
|
||||
: []
|
||||
|
||||
for (const name of userProperties) {
|
||||
const trimmed = name.trim()
|
||||
if (trimmed) requested.add(trimmed)
|
||||
}
|
||||
|
||||
if (requested.size === 0 && objectType in BUILT_IN_PATH) {
|
||||
for (const name of DEFAULT_PROPERTIES[objectType as HubSpotBuiltInObjectType]) {
|
||||
requested.add(name)
|
||||
}
|
||||
}
|
||||
|
||||
requested.add('createdate')
|
||||
requested.add(filterProperty)
|
||||
if (config.targetPropertyName?.trim()) {
|
||||
requested.add(config.targetPropertyName.trim())
|
||||
}
|
||||
for (const f of userFilters) {
|
||||
if (f.propertyName) requested.add(f.propertyName)
|
||||
}
|
||||
|
||||
return [...requested]
|
||||
}
|
||||
|
||||
export function buildUserFilters(
|
||||
config: HubSpotWebhookConfig,
|
||||
logger?: Logger,
|
||||
requestId?: string
|
||||
): FilterClause[] {
|
||||
const filters: FilterClause[] = []
|
||||
|
||||
if (config.pipelineId?.trim()) {
|
||||
const property = config.objectType === 'ticket' ? 'hs_pipeline' : 'pipeline'
|
||||
filters.push({ propertyName: property, operator: 'EQ', value: config.pipelineId.trim() })
|
||||
}
|
||||
if (config.stageId?.trim()) {
|
||||
const property = config.objectType === 'ticket' ? 'hs_pipeline_stage' : 'dealstage'
|
||||
filters.push({ propertyName: property, operator: 'EQ', value: config.stageId.trim() })
|
||||
}
|
||||
if (config.ownerId?.trim()) {
|
||||
filters.push({ propertyName: 'hubspot_owner_id', operator: 'EQ', value: config.ownerId.trim() })
|
||||
}
|
||||
|
||||
const raw = config.filters
|
||||
if (raw) {
|
||||
try {
|
||||
const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw
|
||||
if (Array.isArray(parsed)) {
|
||||
for (const entry of parsed) {
|
||||
if (!entry || typeof entry !== 'object') continue
|
||||
const propertyName = String((entry as FilterClause).propertyName ?? '').trim()
|
||||
const operator = String((entry as FilterClause).operator ?? '').trim()
|
||||
if (!propertyName || !VALID_OPERATORS.has(operator)) continue
|
||||
const clause: FilterClause = { propertyName, operator }
|
||||
if (Array.isArray((entry as FilterClause).values)) {
|
||||
clause.values = (entry as FilterClause).values
|
||||
} else if ((entry as FilterClause).value !== undefined) {
|
||||
clause.value = String((entry as FilterClause).value)
|
||||
}
|
||||
filters.push(clause)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger?.warn(
|
||||
`[${requestId ?? ''}] Could not parse user filters as JSON, ignoring:`,
|
||||
getErrorMessage(error, 'parse error')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Filters within a filterGroup are AND-combined, so dropping any of them would silently
|
||||
// widen the match set (matching records the user's config meant to exclude) rather than
|
||||
// just failing the request — a worse outcome than a loud poll failure. Throw instead so
|
||||
// the webhook is marked failed and the user can trim their filter configuration.
|
||||
if (filters.length > MAX_USER_FILTERS) {
|
||||
throw new Error(
|
||||
`[${requestId ?? ''}] HubSpot webhook has ${filters.length} combined filters (pipeline/stage/owner shortcuts + advanced filters), exceeding the ${MAX_USER_FILTERS}-filter limit HubSpot's Search API allows alongside the reserved cursor filters. Reduce the number of filters.`
|
||||
)
|
||||
}
|
||||
|
||||
return filters
|
||||
}
|
||||
|
||||
function resolvePropertySnapshot(
|
||||
config: HubSpotWebhookConfig,
|
||||
property: string
|
||||
): PropertySnapshotState {
|
||||
const existing = config.propertySnapshot
|
||||
if (existing && existing.property === property && Array.isArray(existing.entries)) {
|
||||
return { property, values: new Map(existing.entries) }
|
||||
}
|
||||
// Property changed since last config — start fresh so we don't compare against stale values.
|
||||
return { property, values: new Map() }
|
||||
}
|
||||
|
||||
function serializeSnapshot(state: PropertySnapshotState): {
|
||||
property: string
|
||||
entries: Array<[string, string | null]>
|
||||
} {
|
||||
let entries = [...state.values.entries()]
|
||||
// Drop oldest by insertion order; Map preserves it regardless of key type.
|
||||
if (entries.length > MAX_SNAPSHOT_SIZE) {
|
||||
entries = entries.slice(entries.length - MAX_SNAPSHOT_SIZE)
|
||||
}
|
||||
return { property: state.property, entries }
|
||||
}
|
||||
|
||||
interface FetchArgs {
|
||||
accessToken: string
|
||||
objectType: string
|
||||
filterProperty: string
|
||||
watermarkMs: number
|
||||
lastSeenObjectId?: string
|
||||
properties: string[]
|
||||
userFilters: FilterClause[]
|
||||
maxRecords: number
|
||||
requestId: string
|
||||
logger: Logger
|
||||
}
|
||||
|
||||
async function fetchHubSpotChanges(args: FetchArgs): Promise<HubSpotSearchResult[]> {
|
||||
const {
|
||||
accessToken,
|
||||
objectType,
|
||||
filterProperty,
|
||||
watermarkMs,
|
||||
lastSeenObjectId,
|
||||
properties,
|
||||
userFilters,
|
||||
maxRecords,
|
||||
requestId,
|
||||
logger,
|
||||
} = args
|
||||
|
||||
const url = `https://api.hubapi.com/crm/v3/objects/${encodeURIComponent(resolveSearchPath(objectType))}/search`
|
||||
const accumulated: HubSpotSearchResult[] = []
|
||||
let after: string | undefined
|
||||
let pages = 0
|
||||
|
||||
// Two OR-combined filter groups give a strict monotonic cursor over (timestamp, id):
|
||||
// A: filterProperty > watermark (next timestamps)
|
||||
// B: filterProperty == watermark AND id > lastSeenObjectId (more ids at boundary)
|
||||
// User filters AND into both groups so they apply regardless of which side matches.
|
||||
// Group B is dropped on the first poll after seeding so we don't emit boundary
|
||||
// records the seed point already skipped past.
|
||||
const buildBody = (cursor?: string) => {
|
||||
const groupA: FilterClause[] = [
|
||||
{ propertyName: filterProperty, operator: 'GT', value: String(watermarkMs) },
|
||||
...userFilters,
|
||||
]
|
||||
const groups = [{ filters: groupA }]
|
||||
if (lastSeenObjectId) {
|
||||
const groupB: FilterClause[] = [
|
||||
{ propertyName: filterProperty, operator: 'EQ', value: String(watermarkMs) },
|
||||
{ propertyName: 'hs_object_id', operator: 'GT', value: String(lastSeenObjectId) },
|
||||
...userFilters,
|
||||
]
|
||||
groups.push({ filters: groupB })
|
||||
}
|
||||
return {
|
||||
filterGroups: groups,
|
||||
sorts: [{ propertyName: filterProperty, direction: 'ASCENDING' }],
|
||||
properties,
|
||||
limit: HUBSPOT_PAGE_LIMIT,
|
||||
...(cursor ? { after: cursor } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(buildBody(after)),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
logger.error(`[${requestId}] HubSpot search API error:`, {
|
||||
status: response.status,
|
||||
objectType,
|
||||
error: errorBody,
|
||||
})
|
||||
throw new Error(`HubSpot search API ${response.status}: ${errorBody.slice(0, 500)}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as HubSpotSearchResponse
|
||||
if (data.results?.length) accumulated.push(...data.results)
|
||||
|
||||
after = data.paging?.next?.after
|
||||
pages++
|
||||
|
||||
if (accumulated.length >= maxRecords) break
|
||||
if (pages >= MAX_PAGES_PER_POLL) {
|
||||
logger.warn(
|
||||
`[${requestId}] HubSpot poll hit MAX_PAGES_PER_POLL=${MAX_PAGES_PER_POLL} — remaining records will roll over to next poll`
|
||||
)
|
||||
break
|
||||
}
|
||||
} while (after)
|
||||
|
||||
// HubSpot's intra-timestamp ordering is undocumented. Sorting locally by (timestamp, id)
|
||||
// before slicing keeps the lowest-(timestamp, id) records, so any unemitted record is
|
||||
// guaranteed to match Group A or Group B on the next poll. Without this the cursor could
|
||||
// overshoot and silently skip records.
|
||||
accumulated.sort((a, b) => {
|
||||
const aTs = extractPropertyTimestampMs(a, filterProperty)
|
||||
const bTs = extractPropertyTimestampMs(b, filterProperty)
|
||||
if (aTs !== bTs) {
|
||||
if (!Number.isFinite(aTs)) return 1
|
||||
if (!Number.isFinite(bTs)) return -1
|
||||
return aTs - bTs
|
||||
}
|
||||
return compareObjectIds(a.id, b.id)
|
||||
})
|
||||
|
||||
return accumulated.slice(0, maxRecords)
|
||||
}
|
||||
|
||||
function extractPropertyTimestampMs(record: HubSpotSearchResult, propertyName: string): number {
|
||||
const raw = record.properties?.[propertyName]
|
||||
if (raw) {
|
||||
const ms = Date.parse(raw)
|
||||
if (Number.isFinite(ms)) return ms
|
||||
}
|
||||
const fallback = propertyName === 'createdate' ? record.createdAt : record.updatedAt
|
||||
return fallback ? Date.parse(fallback) : Number.NaN
|
||||
}
|
||||
|
||||
async function processRecords(
|
||||
records: HubSpotSearchResult[],
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
objectType: string,
|
||||
eventType: HubSpotEventType,
|
||||
filterProperty: string,
|
||||
targetProperty: string | undefined,
|
||||
snapshot: PropertySnapshotState | null,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{
|
||||
processedCount: number
|
||||
failedCount: number
|
||||
skippedCount: number
|
||||
highestSeenMs: number
|
||||
maxIdAtHighestTimestamp: string
|
||||
}> {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
let skippedCount = 0
|
||||
let highestSeenMs = 0
|
||||
let maxIdAtHighestTimestamp = ''
|
||||
// Stop advancing the cursor at the first failure so that the failed record and all later
|
||||
// records (sorted ASC) get re-fetched on the next poll. Without this gate, a transient
|
||||
// failure on a record at a high timestamp would advance the cursor past it permanently.
|
||||
let cursorFrozen = false
|
||||
|
||||
for (const record of records) {
|
||||
const occurredAtMs = extractPropertyTimestampMs(record, filterProperty)
|
||||
|
||||
let previousValue: string | null | undefined
|
||||
let propertyValue: string | null | undefined
|
||||
let handledBySkip = false
|
||||
if (eventType === 'property_changed' && targetProperty && snapshot) {
|
||||
propertyValue = record.properties?.[targetProperty] ?? null
|
||||
const had = snapshot.values.has(record.id)
|
||||
previousValue = had ? snapshot.values.get(record.id) : undefined
|
||||
if (had && (previousValue ?? null) === (propertyValue ?? null)) {
|
||||
// Touch the snapshot so this record's entry moves to the end of the LRU order.
|
||||
// Map.delete + Map.set re-inserts at the tail, regardless of key type.
|
||||
snapshot.values.delete(record.id)
|
||||
snapshot.values.set(record.id, propertyValue ?? null)
|
||||
skippedCount++
|
||||
handledBySkip = true
|
||||
}
|
||||
// Note: we do NOT pre-update the snapshot before processing. If emission fails the
|
||||
// record must re-fetch on the next poll AND still appear as a change vs. the prior
|
||||
// snapshot — otherwise we'd silently skip it on retry.
|
||||
}
|
||||
|
||||
let handledSuccessfully = handledBySkip
|
||||
if (!handledBySkip) {
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'hubspot',
|
||||
`${webhookData.id}:${objectType}:${eventType}:${record.id}:${Number.isFinite(occurredAtMs) ? occurredAtMs : record.updatedAt}`,
|
||||
async () => {
|
||||
const payload: Record<string, unknown> = {
|
||||
objectType,
|
||||
eventType,
|
||||
objectId: record.id,
|
||||
occurredAt: Number.isFinite(occurredAtMs)
|
||||
? new Date(occurredAtMs).toISOString()
|
||||
: record.updatedAt,
|
||||
properties: record.properties,
|
||||
createdAt: record.createdAt,
|
||||
updatedAt: record.updatedAt,
|
||||
archived: record.archived,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
if (eventType === 'property_changed' && targetProperty) {
|
||||
payload.propertyName = targetProperty
|
||||
payload.propertyValue = propertyValue ?? null
|
||||
payload.previousValue = previousValue ?? null
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`Webhook processing failed (${result.statusCode}): ${result.error ?? 'unknown'}`
|
||||
)
|
||||
}
|
||||
return { recordId: record.id, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
processedCount++
|
||||
handledSuccessfully = true
|
||||
if (eventType === 'property_changed' && targetProperty && snapshot) {
|
||||
// Same delete+set dance so an updated value moves to the tail of the LRU order.
|
||||
snapshot.values.delete(record.id)
|
||||
snapshot.values.set(record.id, propertyValue ?? null)
|
||||
}
|
||||
} catch (error) {
|
||||
failedCount++
|
||||
cursorFrozen = true
|
||||
logger.error(
|
||||
`[${requestId}] Error processing HubSpot ${objectType} ${record.id}:`,
|
||||
getErrorMessage(error, 'Unknown error')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Advance the cursor only for records handled (emitted or intentionally skipped) WITHOUT
|
||||
// any prior failure in this batch. Records are pre-sorted (timestamp ASC, id ASC), so
|
||||
// the watermark we persist is the highest contiguously-successful (timestamp, id) pair.
|
||||
// Anything after the first failure stays unfrozen so it gets re-fetched next poll.
|
||||
if (handledSuccessfully && !cursorFrozen && Number.isFinite(occurredAtMs)) {
|
||||
if (occurredAtMs > highestSeenMs) {
|
||||
highestSeenMs = occurredAtMs
|
||||
maxIdAtHighestTimestamp = record.id
|
||||
} else if (occurredAtMs === highestSeenMs) {
|
||||
if (compareObjectIds(record.id, maxIdAtHighestTimestamp) > 0) {
|
||||
maxIdAtHighestTimestamp = record.id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
processedCount,
|
||||
failedCount,
|
||||
skippedCount,
|
||||
highestSeenMs,
|
||||
maxIdAtHighestTimestamp,
|
||||
}
|
||||
}
|
||||
|
||||
interface ListMembershipRecord {
|
||||
recordId: string
|
||||
membershipTimestamp: string
|
||||
}
|
||||
|
||||
interface FetchListMembershipPagesArgs {
|
||||
listId: string
|
||||
accessToken: string
|
||||
initialAfter: string | undefined
|
||||
/** Soft cap on emitted records (normal mode) or pages × HUBSPOT_PAGE_LIMIT (seed mode). */
|
||||
pageLimit: number
|
||||
requestId: string
|
||||
logger: Logger
|
||||
}
|
||||
|
||||
interface FetchListMembershipPagesResult {
|
||||
records: ListMembershipRecord[]
|
||||
/** Cursor to persist for the next poll; empty if we are at the very start. */
|
||||
resumeCursor: string | undefined
|
||||
/** True when the API stopped returning `paging.next.after` — we've consumed everything. */
|
||||
reachedEnd: boolean
|
||||
/** Total records scanned across pages (useful for seed-progress logging). */
|
||||
scanned: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks `/lists/{listId}/memberships/join-order` ASC from an optional cursor until either
|
||||
* (a) the per-poll page cap is reached or (b) the API stops returning a next cursor.
|
||||
*
|
||||
* The endpoint returns members in ascending join-order by default. We never use `before`
|
||||
* (DESC mode) because its bootstrap semantics — what value to pass for "newest first" —
|
||||
* are undocumented in HubSpot's SDK type and behave inconsistently in practice. ASC + a
|
||||
* stored cursor is provably correct and lets new members appear on subsequent polls as
|
||||
* they're appended past our cursor's position.
|
||||
*/
|
||||
async function fetchListMembershipPages(
|
||||
args: FetchListMembershipPagesArgs
|
||||
): Promise<FetchListMembershipPagesResult> {
|
||||
const { listId, accessToken, initialAfter, pageLimit, requestId, logger } = args
|
||||
|
||||
const records: ListMembershipRecord[] = []
|
||||
let after: string | undefined = initialAfter
|
||||
let pages = 0
|
||||
let reachedEnd = false
|
||||
let scanned = 0
|
||||
|
||||
while (pages < MAX_PAGES_PER_POLL) {
|
||||
const params = new URLSearchParams({ limit: String(HUBSPOT_PAGE_LIMIT) })
|
||||
if (after) params.set('after', after)
|
||||
const url = `https://api.hubapi.com/crm/v3/lists/${encodeURIComponent(listId)}/memberships/join-order?${params.toString()}`
|
||||
const response = await fetch(url, { headers: { Authorization: `Bearer ${accessToken}` } })
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error(
|
||||
`[${requestId}] HubSpot list memberships fetch failed ${response.status}: ${errorText}`
|
||||
)
|
||||
throw new Error(
|
||||
`HubSpot list memberships fetch ${response.status}: ${errorText.slice(0, 500)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
results?: Array<{ recordId: string; membershipTimestamp: string }>
|
||||
paging?: { next?: { after?: string } }
|
||||
}
|
||||
const batch = data.results ?? []
|
||||
scanned += batch.length
|
||||
const nextAfter = data.paging?.next?.after
|
||||
|
||||
for (const m of batch) {
|
||||
records.push({
|
||||
recordId: m.recordId,
|
||||
membershipTimestamp: m.membershipTimestamp,
|
||||
})
|
||||
}
|
||||
|
||||
pages++
|
||||
if (!nextAfter) {
|
||||
reachedEnd = true
|
||||
break
|
||||
}
|
||||
after = nextAfter
|
||||
if (records.length >= pageLimit) break
|
||||
}
|
||||
|
||||
return {
|
||||
records,
|
||||
// If we walked to the end, hold onto the cursor we LAST used so the next poll re-fetches
|
||||
// the tail page and picks up any members appended since. If we stopped early, the next
|
||||
// cursor walks us forward through the rest of the list.
|
||||
resumeCursor: after,
|
||||
reachedEnd,
|
||||
scanned,
|
||||
}
|
||||
}
|
||||
|
||||
/** Numeric compare for HubSpot ids (decimal strings, treated numerically by GT/LT). */
|
||||
function compareObjectIds(a: string, b: string): number {
|
||||
if (!a) return b ? -1 : 0
|
||||
if (!b) return 1
|
||||
const an = Number(a)
|
||||
const bn = Number(b)
|
||||
if (Number.isFinite(an) && Number.isFinite(bn)) {
|
||||
if (an === bn) return 0
|
||||
return an > bn ? 1 : -1
|
||||
}
|
||||
if (a === b) return 0
|
||||
return a > b ? 1 : -1
|
||||
}
|
||||
@@ -0,0 +1,608 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import type { FetchMessageObject, MailboxLockObject } from 'imapflow'
|
||||
import { ImapFlow } from 'imapflow'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import { validateDatabaseHost } from '@/lib/core/security/input-validation.server'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
interface ImapWebhookConfig {
|
||||
host: string
|
||||
port: number
|
||||
secure: boolean
|
||||
username: string
|
||||
password: string
|
||||
mailbox: string | string[]
|
||||
searchCriteria: string | Record<string, unknown>
|
||||
markAsRead: boolean
|
||||
includeAttachments: boolean
|
||||
lastProcessedUid?: number
|
||||
lastProcessedUidByMailbox?: Record<string, number>
|
||||
uidValidityByMailbox?: Record<string, string>
|
||||
lastCheckedTimestamp?: string
|
||||
maxEmailsPerPoll?: number
|
||||
}
|
||||
|
||||
interface ImapAttachment {
|
||||
name: string
|
||||
data: Buffer
|
||||
mimeType: string
|
||||
size: number
|
||||
}
|
||||
|
||||
interface SimplifiedImapEmail {
|
||||
uid: string
|
||||
messageId: string
|
||||
subject: string
|
||||
from: string
|
||||
to: string
|
||||
cc: string
|
||||
date: string | null
|
||||
bodyText: string
|
||||
bodyHtml: string
|
||||
mailbox: string
|
||||
hasAttachments: boolean
|
||||
attachments: ImapAttachment[]
|
||||
}
|
||||
|
||||
interface ImapWebhookPayload {
|
||||
messageId: string
|
||||
subject: string
|
||||
from: string
|
||||
to: string
|
||||
cc: string
|
||||
date: string | null
|
||||
bodyText: string
|
||||
bodyHtml: string
|
||||
mailbox: string
|
||||
hasAttachments: boolean
|
||||
attachments: ImapAttachment[]
|
||||
email: SimplifiedImapEmail
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
export const imapPollingHandler: PollingProviderHandler = {
|
||||
provider: 'imap',
|
||||
label: 'IMAP',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const config = getProviderConfig<ImapWebhookConfig>(webhookData.providerConfig)
|
||||
|
||||
if (!config.host || !config.username || !config.password) {
|
||||
logger.error(`[${requestId}] Missing IMAP credentials for webhook ${webhookId}`)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
const hostValidation = await validateDatabaseHost(config.host, 'host')
|
||||
if (!hostValidation.isValid) {
|
||||
logger.error(
|
||||
`[${requestId}] IMAP host validation failed for webhook ${webhookId}: ${hostValidation.error}`
|
||||
)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
const client = new ImapFlow({
|
||||
host: hostValidation.resolvedIP!,
|
||||
servername: config.host,
|
||||
port: config.port || 993,
|
||||
secure: config.secure ?? true,
|
||||
auth: {
|
||||
user: config.username,
|
||||
pass: config.password,
|
||||
},
|
||||
tls: { rejectUnauthorized: true },
|
||||
logger: false,
|
||||
})
|
||||
|
||||
let emails: Awaited<ReturnType<typeof fetchNewEmails>>['emails'] = []
|
||||
let latestUidByMailbox: Record<string, number> = {}
|
||||
let uidValidityByMailbox: Record<string, string> = {}
|
||||
|
||||
try {
|
||||
await client.connect()
|
||||
|
||||
const result = await fetchNewEmails(client, config, requestId, logger)
|
||||
emails = result.emails
|
||||
latestUidByMailbox = result.latestUidByMailbox
|
||||
uidValidityByMailbox = result.uidValidityByMailbox
|
||||
|
||||
const pollTimestamp = new Date().toISOString()
|
||||
|
||||
if (!emails.length) {
|
||||
await updateImapState(
|
||||
webhookId,
|
||||
latestUidByMailbox,
|
||||
pollTimestamp,
|
||||
config,
|
||||
logger,
|
||||
uidValidityByMailbox
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
|
||||
await client.logout()
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Found ${emails.length} new emails for webhook ${webhookId}`)
|
||||
|
||||
const { processedCount, failedCount } = await processEmails(
|
||||
emails,
|
||||
webhookData,
|
||||
workflowData,
|
||||
config,
|
||||
client,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
await updateImapState(
|
||||
webhookId,
|
||||
latestUidByMailbox,
|
||||
pollTimestamp,
|
||||
config,
|
||||
logger,
|
||||
uidValidityByMailbox
|
||||
)
|
||||
|
||||
await client.logout()
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (innerError) {
|
||||
try {
|
||||
await client.logout()
|
||||
} catch {}
|
||||
throw innerError
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing IMAP webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function updateImapState(
|
||||
webhookId: string,
|
||||
uidByMailbox: Record<string, number>,
|
||||
timestamp: string,
|
||||
config: ImapWebhookConfig,
|
||||
logger: Logger,
|
||||
uidValidityByMailbox: Record<string, string>
|
||||
) {
|
||||
const existingUidByMailbox = config.lastProcessedUidByMailbox || {}
|
||||
const prevUidValidity = config.uidValidityByMailbox || {}
|
||||
|
||||
const resetMailboxes = new Set(
|
||||
Object.entries(uidValidityByMailbox)
|
||||
.filter(
|
||||
([mailbox, validity]) =>
|
||||
prevUidValidity[mailbox] !== undefined && prevUidValidity[mailbox] !== validity
|
||||
)
|
||||
.map(([mailbox]) => mailbox)
|
||||
)
|
||||
|
||||
const mergedUidByMailbox: Record<string, number> = {}
|
||||
|
||||
for (const [mailbox, uid] of Object.entries(existingUidByMailbox)) {
|
||||
if (!resetMailboxes.has(mailbox)) {
|
||||
mergedUidByMailbox[mailbox] = uid
|
||||
}
|
||||
}
|
||||
|
||||
for (const [mailbox, uid] of Object.entries(uidByMailbox)) {
|
||||
if (resetMailboxes.has(mailbox)) {
|
||||
mergedUidByMailbox[mailbox] = uid
|
||||
} else {
|
||||
mergedUidByMailbox[mailbox] = Math.max(uid, mergedUidByMailbox[mailbox] || 0)
|
||||
}
|
||||
}
|
||||
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastProcessedUidByMailbox: mergedUidByMailbox,
|
||||
lastCheckedTimestamp: timestamp,
|
||||
uidValidityByMailbox,
|
||||
},
|
||||
logger
|
||||
)
|
||||
}
|
||||
|
||||
async function fetchNewEmails(
|
||||
client: ImapFlow,
|
||||
config: ImapWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
const emails: Array<{
|
||||
uid: number
|
||||
mailboxPath: string
|
||||
envelope: FetchMessageObject['envelope']
|
||||
bodyStructure: FetchMessageObject['bodyStructure']
|
||||
source?: Buffer
|
||||
}> = []
|
||||
|
||||
const mailboxes = getMailboxesToCheck(config)
|
||||
const latestUidByMailbox: Record<string, number> = { ...(config.lastProcessedUidByMailbox || {}) }
|
||||
const uidValidityByMailbox: Record<string, string> = { ...(config.uidValidityByMailbox || {}) }
|
||||
|
||||
const maxEmails = config.maxEmailsPerPoll || 25
|
||||
let totalEmailsCollected = 0
|
||||
|
||||
for (const mailboxPath of mailboxes) {
|
||||
if (totalEmailsCollected >= maxEmails) break
|
||||
|
||||
try {
|
||||
const mailbox = await client.mailboxOpen(mailboxPath)
|
||||
|
||||
const currentUidValidity = mailbox.uidValidity.toString()
|
||||
const storedUidValidity = uidValidityByMailbox[mailboxPath]
|
||||
|
||||
if (storedUidValidity && storedUidValidity !== currentUidValidity) {
|
||||
logger.warn(
|
||||
`[${requestId}] UIDVALIDITY changed for ${mailboxPath} (${storedUidValidity} -> ${currentUidValidity}), discarding stored UID`
|
||||
)
|
||||
delete latestUidByMailbox[mailboxPath]
|
||||
}
|
||||
uidValidityByMailbox[mailboxPath] = currentUidValidity
|
||||
|
||||
let searchCriteria: Record<string, unknown> = { unseen: true }
|
||||
if (config.searchCriteria) {
|
||||
if (typeof config.searchCriteria === 'object') {
|
||||
searchCriteria = config.searchCriteria
|
||||
} else if (typeof config.searchCriteria === 'string') {
|
||||
try {
|
||||
searchCriteria = JSON.parse(config.searchCriteria)
|
||||
} catch {
|
||||
logger.warn(`[${requestId}] Invalid search criteria JSON, using default`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const lastUidForMailbox = latestUidByMailbox[mailboxPath]
|
||||
|
||||
if (lastUidForMailbox) {
|
||||
searchCriteria = { ...searchCriteria, uid: `${lastUidForMailbox + 1}:*` }
|
||||
}
|
||||
|
||||
if (config.lastCheckedTimestamp) {
|
||||
const lastChecked = new Date(config.lastCheckedTimestamp)
|
||||
const bufferTime = new Date(lastChecked.getTime() - 60000)
|
||||
searchCriteria = { ...searchCriteria, since: bufferTime }
|
||||
} else {
|
||||
const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000)
|
||||
searchCriteria = { ...searchCriteria, since: oneDayAgo }
|
||||
}
|
||||
|
||||
let messageUids: number[] = []
|
||||
try {
|
||||
const searchResult = await client.search(searchCriteria, { uid: true })
|
||||
messageUids = searchResult === false ? [] : searchResult
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
|
||||
if (messageUids.length === 0) continue
|
||||
|
||||
messageUids.sort((a, b) => a - b)
|
||||
const remainingSlots = maxEmails - totalEmailsCollected
|
||||
const uidsToProcess = messageUids.slice(0, remainingSlots)
|
||||
|
||||
for await (const msg of client.fetch(
|
||||
uidsToProcess,
|
||||
{ uid: true, envelope: true, bodyStructure: true, source: true },
|
||||
{ uid: true }
|
||||
)) {
|
||||
emails.push({
|
||||
uid: msg.uid,
|
||||
mailboxPath,
|
||||
envelope: msg.envelope,
|
||||
bodyStructure: msg.bodyStructure,
|
||||
source: msg.source,
|
||||
})
|
||||
if (msg.uid > (latestUidByMailbox[mailboxPath] || 0)) {
|
||||
latestUidByMailbox[mailboxPath] = msg.uid
|
||||
}
|
||||
totalEmailsCollected++
|
||||
}
|
||||
} catch (mailboxError) {
|
||||
logger.warn(`[${requestId}] Error processing mailbox ${mailboxPath}:`, mailboxError)
|
||||
}
|
||||
}
|
||||
|
||||
return { emails, latestUidByMailbox, uidValidityByMailbox }
|
||||
}
|
||||
|
||||
function getMailboxesToCheck(config: ImapWebhookConfig): string[] {
|
||||
if (!config.mailbox || (Array.isArray(config.mailbox) && config.mailbox.length === 0)) {
|
||||
return ['INBOX']
|
||||
}
|
||||
if (Array.isArray(config.mailbox)) {
|
||||
return config.mailbox
|
||||
}
|
||||
return [config.mailbox]
|
||||
}
|
||||
|
||||
function parseEmailAddress(
|
||||
addr: { name?: string; address?: string } | { name?: string; address?: string }[] | undefined
|
||||
): string {
|
||||
if (!addr) return ''
|
||||
if (Array.isArray(addr)) {
|
||||
return addr
|
||||
.map((a) => (a.name ? `${a.name} <${a.address}>` : a.address || ''))
|
||||
.filter(Boolean)
|
||||
.join(', ')
|
||||
}
|
||||
return addr.name ? `${addr.name} <${addr.address}>` : addr.address || ''
|
||||
}
|
||||
|
||||
function extractTextFromSource(source: Buffer): { text: string; html: string } {
|
||||
const content = source.toString('utf-8')
|
||||
let text = ''
|
||||
let html = ''
|
||||
|
||||
const parts = content.split(/--[^\r\n]+/)
|
||||
|
||||
for (const part of parts) {
|
||||
const lowerPart = part.toLowerCase()
|
||||
|
||||
if (lowerPart.includes('content-type: text/plain')) {
|
||||
const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i)
|
||||
if (match) {
|
||||
text = match[1].trim()
|
||||
if (lowerPart.includes('quoted-printable')) {
|
||||
text = text
|
||||
.replace(/=\r?\n/g, '')
|
||||
.replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
|
||||
}
|
||||
if (lowerPart.includes('base64')) {
|
||||
try {
|
||||
text = Buffer.from(text.replace(/\s/g, ''), 'base64').toString('utf-8')
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
} else if (lowerPart.includes('content-type: text/html')) {
|
||||
const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i)
|
||||
if (match) {
|
||||
html = match[1].trim()
|
||||
if (lowerPart.includes('quoted-printable')) {
|
||||
html = html
|
||||
.replace(/=\r?\n/g, '')
|
||||
.replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16)))
|
||||
}
|
||||
if (lowerPart.includes('base64')) {
|
||||
try {
|
||||
html = Buffer.from(html.replace(/\s/g, ''), 'base64').toString('utf-8')
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!text && !html) {
|
||||
const bodyMatch = content.match(/\r?\n\r?\n([\s\S]+)$/)
|
||||
if (bodyMatch) {
|
||||
text = bodyMatch[1].trim()
|
||||
}
|
||||
}
|
||||
|
||||
return { text, html }
|
||||
}
|
||||
|
||||
function extractAttachmentsFromSource(
|
||||
source: Buffer,
|
||||
bodyStructure: FetchMessageObject['bodyStructure']
|
||||
): ImapAttachment[] {
|
||||
const attachments: ImapAttachment[] = []
|
||||
if (!bodyStructure) return attachments
|
||||
|
||||
const content = source.toString('utf-8')
|
||||
const parts = content.split(/--[^\r\n]+/)
|
||||
|
||||
for (const part of parts) {
|
||||
const lowerPart = part.toLowerCase()
|
||||
|
||||
const dispositionMatch = part.match(
|
||||
/content-disposition:\s*attachment[^;]*;\s*filename="?([^"\r\n]+)"?/i
|
||||
)
|
||||
const filenameMatch = part.match(/name="?([^"\r\n]+)"?/i)
|
||||
const contentTypeMatch = part.match(/content-type:\s*([^;\r\n]+)/i)
|
||||
|
||||
if (
|
||||
dispositionMatch ||
|
||||
(filenameMatch && !lowerPart.includes('text/plain') && !lowerPart.includes('text/html'))
|
||||
) {
|
||||
const filename = dispositionMatch?.[1] || filenameMatch?.[1] || 'attachment'
|
||||
const mimeType = contentTypeMatch?.[1]?.trim() || 'application/octet-stream'
|
||||
|
||||
const dataMatch = part.match(/\r?\n\r?\n([\s\S]*?)$/i)
|
||||
if (dataMatch) {
|
||||
const data = dataMatch[1].trim()
|
||||
|
||||
if (lowerPart.includes('base64')) {
|
||||
try {
|
||||
const buffer = Buffer.from(data.replace(/\s/g, ''), 'base64')
|
||||
attachments.push({
|
||||
name: filename,
|
||||
data: buffer,
|
||||
mimeType,
|
||||
size: buffer.length,
|
||||
})
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return attachments
|
||||
}
|
||||
|
||||
function hasAttachmentsInBodyStructure(structure: FetchMessageObject['bodyStructure']): boolean {
|
||||
if (!structure) return false
|
||||
if (structure.disposition === 'attachment') return true
|
||||
if (structure.disposition === 'inline' && structure.dispositionParameters?.filename) return true
|
||||
if (structure.childNodes && Array.isArray(structure.childNodes)) {
|
||||
return structure.childNodes.some((child) => hasAttachmentsInBodyStructure(child))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
async function processEmails(
|
||||
emails: Array<{
|
||||
uid: number
|
||||
mailboxPath: string
|
||||
envelope: FetchMessageObject['envelope']
|
||||
bodyStructure: FetchMessageObject['bodyStructure']
|
||||
source?: Buffer
|
||||
}>,
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
config: ImapWebhookConfig,
|
||||
client: ImapFlow,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
let currentOpenMailbox: string | null = null
|
||||
const lockState: { lock: MailboxLockObject | null } = { lock: null }
|
||||
|
||||
try {
|
||||
for (const email of emails) {
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'imap',
|
||||
`${webhookData.id}:${email.mailboxPath}:${email.uid}`,
|
||||
async () => {
|
||||
const envelope = email.envelope
|
||||
|
||||
const { text: bodyText, html: bodyHtml } = email.source
|
||||
? extractTextFromSource(email.source)
|
||||
: { text: '', html: '' }
|
||||
|
||||
let attachments: ImapAttachment[] = []
|
||||
const hasAttachments = hasAttachmentsInBodyStructure(email.bodyStructure)
|
||||
|
||||
if (config.includeAttachments && hasAttachments && email.source) {
|
||||
attachments = extractAttachmentsFromSource(email.source, email.bodyStructure)
|
||||
}
|
||||
|
||||
const simplifiedEmail: SimplifiedImapEmail = {
|
||||
uid: String(email.uid),
|
||||
messageId: envelope?.messageId || '',
|
||||
subject: envelope?.subject || '[No Subject]',
|
||||
from: parseEmailAddress(envelope?.from),
|
||||
to: parseEmailAddress(envelope?.to),
|
||||
cc: parseEmailAddress(envelope?.cc),
|
||||
date: envelope?.date ? new Date(envelope.date).toISOString() : null,
|
||||
bodyText,
|
||||
bodyHtml,
|
||||
mailbox: email.mailboxPath,
|
||||
hasAttachments,
|
||||
attachments,
|
||||
}
|
||||
|
||||
const payload: ImapWebhookPayload = {
|
||||
messageId: simplifiedEmail.messageId,
|
||||
subject: simplifiedEmail.subject,
|
||||
from: simplifiedEmail.from,
|
||||
to: simplifiedEmail.to,
|
||||
cc: simplifiedEmail.cc,
|
||||
date: simplifiedEmail.date,
|
||||
bodyText: simplifiedEmail.bodyText,
|
||||
bodyHtml: simplifiedEmail.bodyHtml,
|
||||
mailbox: simplifiedEmail.mailbox,
|
||||
hasAttachments: simplifiedEmail.hasAttachments,
|
||||
attachments: simplifiedEmail.attachments,
|
||||
email: simplifiedEmail,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for email ${email.uid}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
if (config.markAsRead) {
|
||||
try {
|
||||
if (currentOpenMailbox !== email.mailboxPath) {
|
||||
if (lockState.lock) {
|
||||
lockState.lock.release()
|
||||
lockState.lock = null
|
||||
}
|
||||
lockState.lock = await client.getMailboxLock(email.mailboxPath)
|
||||
currentOpenMailbox = email.mailboxPath
|
||||
}
|
||||
await client.messageFlagsAdd(email.uid, ['\\Seen'], { uid: true })
|
||||
} catch (flagError) {
|
||||
logger.warn(
|
||||
`[${requestId}] Failed to mark message ${email.uid} as read:`,
|
||||
flagError
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return { emailUid: email.uid, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed email ${email.uid} from ${email.mailboxPath} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error processing email ${email.uid}:`, errorMessage)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (lockState.lock) {
|
||||
try {
|
||||
lockState.lock.release()
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount }
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { pollProvider } from '@/lib/webhooks/polling/orchestrator'
|
||||
export { VALID_POLLING_PROVIDERS } from '@/lib/webhooks/polling/registry'
|
||||
@@ -0,0 +1,46 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateShortId } from '@sim/utils/id'
|
||||
import { getPollingHandler } from '@/lib/webhooks/polling/registry'
|
||||
import type { PollSummary } from '@/lib/webhooks/polling/types'
|
||||
import { fetchActiveWebhooks, runWithConcurrency } from '@/lib/webhooks/polling/utils'
|
||||
|
||||
/** Poll all active webhooks for a given provider. */
|
||||
export async function pollProvider(providerName: string): Promise<PollSummary> {
|
||||
const handler = getPollingHandler(providerName)
|
||||
if (!handler) {
|
||||
throw new Error(`Unknown polling provider: ${providerName}`)
|
||||
}
|
||||
|
||||
const logger = createLogger(`${handler.label}PollingService`)
|
||||
logger.info(`Starting ${handler.label} webhook polling`)
|
||||
|
||||
const activeWebhooks = await fetchActiveWebhooks(handler.provider)
|
||||
if (!activeWebhooks.length) {
|
||||
logger.info(`No active ${handler.label} webhooks found`)
|
||||
return { total: 0, successful: 0, failed: 0 }
|
||||
}
|
||||
|
||||
logger.info(`Found ${activeWebhooks.length} active ${handler.label} webhooks`)
|
||||
|
||||
const { successCount, failureCount } = await runWithConcurrency(
|
||||
activeWebhooks,
|
||||
async (entry) => {
|
||||
const requestId = generateShortId()
|
||||
return handler.pollWebhook({
|
||||
webhookData: entry.webhook,
|
||||
workflowData: entry.workflow,
|
||||
requestId,
|
||||
logger,
|
||||
})
|
||||
},
|
||||
logger
|
||||
)
|
||||
|
||||
const summary: PollSummary = {
|
||||
total: activeWebhooks.length,
|
||||
successful: successCount,
|
||||
failed: failureCount,
|
||||
}
|
||||
logger.info(`${handler.label} polling completed`, summary)
|
||||
return summary
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { htmlToText } from 'html-to-text'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
resolveOAuthCredential,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
interface OutlookWebhookConfig {
|
||||
credentialId: string
|
||||
folderIds?: string[]
|
||||
folderFilterBehavior?: 'INCLUDE' | 'EXCLUDE'
|
||||
markAsRead?: boolean
|
||||
maxEmailsPerPoll?: number
|
||||
lastCheckedTimestamp?: string
|
||||
includeAttachments?: boolean
|
||||
includeRawEmail?: boolean
|
||||
}
|
||||
|
||||
interface OutlookEmail {
|
||||
id: string
|
||||
conversationId: string
|
||||
subject: string
|
||||
bodyPreview: string
|
||||
body: {
|
||||
contentType: string
|
||||
content: string
|
||||
}
|
||||
from: {
|
||||
emailAddress: {
|
||||
name: string
|
||||
address: string
|
||||
}
|
||||
}
|
||||
toRecipients: Array<{
|
||||
emailAddress: {
|
||||
name: string
|
||||
address: string
|
||||
}
|
||||
}>
|
||||
ccRecipients?: Array<{
|
||||
emailAddress: {
|
||||
name: string
|
||||
address: string
|
||||
}
|
||||
}>
|
||||
receivedDateTime: string
|
||||
sentDateTime: string
|
||||
hasAttachments: boolean
|
||||
isRead: boolean
|
||||
parentFolderId: string
|
||||
}
|
||||
|
||||
interface OutlookAttachment {
|
||||
name: string
|
||||
data: Buffer
|
||||
contentType: string
|
||||
size: number
|
||||
}
|
||||
|
||||
interface SimplifiedOutlookEmail {
|
||||
id: string
|
||||
conversationId: string
|
||||
subject: string
|
||||
from: string
|
||||
to: string
|
||||
cc: string
|
||||
date: string
|
||||
bodyText: string
|
||||
bodyHtml: string
|
||||
hasAttachments: boolean
|
||||
attachments: OutlookAttachment[]
|
||||
isRead: boolean
|
||||
folderId: string
|
||||
messageId: string
|
||||
threadId: string
|
||||
}
|
||||
|
||||
interface OutlookWebhookPayload {
|
||||
email: SimplifiedOutlookEmail
|
||||
timestamp: string
|
||||
rawEmail?: OutlookEmail
|
||||
}
|
||||
|
||||
function convertHtmlToPlainText(html: string): string {
|
||||
if (!html) return ''
|
||||
return htmlToText(html, {
|
||||
wordwrap: false,
|
||||
selectors: [
|
||||
{ selector: 'a', options: { hideLinkHrefIfSameAsText: true, noAnchorUrl: true } },
|
||||
{ selector: 'img', format: 'skip' },
|
||||
{ selector: 'script', format: 'skip' },
|
||||
{ selector: 'style', format: 'skip' },
|
||||
],
|
||||
preserveNewlines: true,
|
||||
})
|
||||
}
|
||||
|
||||
export const outlookPollingHandler: PollingProviderHandler = {
|
||||
provider: 'outlook',
|
||||
label: 'Outlook',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
logger.info(`[${requestId}] Processing Outlook webhook: ${webhookId}`)
|
||||
|
||||
const accessToken = await resolveOAuthCredential(webhookData, 'outlook', requestId)
|
||||
const config = getProviderConfig<OutlookWebhookConfig>(webhookData.providerConfig)
|
||||
const now = new Date()
|
||||
|
||||
const { emails } = await fetchNewOutlookEmails(accessToken, config, requestId, logger)
|
||||
|
||||
if (!emails || !emails.length) {
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ lastCheckedTimestamp: now.toISOString() },
|
||||
logger
|
||||
)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Found ${emails.length} emails for webhook ${webhookId}`)
|
||||
|
||||
const { processedCount, failedCount } = await processOutlookEmails(
|
||||
emails,
|
||||
webhookData,
|
||||
workflowData,
|
||||
config,
|
||||
accessToken,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{ lastCheckedTimestamp: now.toISOString() },
|
||||
logger
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing Outlook webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
/** Hard cap on total emails fetched per poll to prevent unbounded pagination loops. */
|
||||
const OUTLOOK_HARD_MAX_EMAILS = 200
|
||||
|
||||
/** Number of items to request per Graph API page. Decoupled from the total cap so pagination actually runs. */
|
||||
const OUTLOOK_PAGE_SIZE = 50
|
||||
|
||||
async function fetchNewOutlookEmails(
|
||||
accessToken: string,
|
||||
config: OutlookWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
try {
|
||||
const apiUrl = 'https://graph.microsoft.com/v1.0/me/messages'
|
||||
const params = new URLSearchParams()
|
||||
|
||||
params.append(
|
||||
'$select',
|
||||
'id,conversationId,subject,bodyPreview,body,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,isRead,parentFolderId'
|
||||
)
|
||||
params.append('$orderby', 'receivedDateTime desc')
|
||||
const maxEmails = Math.min(config.maxEmailsPerPoll || 25, OUTLOOK_HARD_MAX_EMAILS)
|
||||
params.append('$top', OUTLOOK_PAGE_SIZE.toString())
|
||||
|
||||
if (config.lastCheckedTimestamp) {
|
||||
const lastChecked = new Date(config.lastCheckedTimestamp)
|
||||
const bufferTime = new Date(lastChecked.getTime() - 60000)
|
||||
params.append('$filter', `receivedDateTime gt ${bufferTime.toISOString()}`)
|
||||
}
|
||||
const allEmails: OutlookEmail[] = []
|
||||
let nextUrl: string | undefined = `${apiUrl}?${params.toString()}`
|
||||
logger.info(`[${requestId}] Fetching emails from: ${nextUrl}`)
|
||||
|
||||
while (nextUrl && allEmails.length < maxEmails) {
|
||||
const response = await fetchWithRetry(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Microsoft Graph API error:`, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
error: errorData,
|
||||
})
|
||||
throw new Error(
|
||||
`Microsoft Graph API error: ${response.status} ${response.statusText} - ${JSON.stringify(errorData)}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const pageEmails: OutlookEmail[] = data.value || []
|
||||
const remaining = maxEmails - allEmails.length
|
||||
allEmails.push(...pageEmails.slice(0, remaining))
|
||||
|
||||
nextUrl =
|
||||
allEmails.length < maxEmails ? (data['@odata.nextLink'] as string | undefined) : undefined
|
||||
|
||||
if (pageEmails.length === 0) break
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Fetched ${allEmails.length} emails total`)
|
||||
|
||||
const emails = allEmails
|
||||
|
||||
let resolvedFolderIds: Map<string, string> | undefined
|
||||
let skipFolderFilter = false
|
||||
if (config.folderIds && config.folderIds.length > 0) {
|
||||
const wellKnownFolders = config.folderIds.filter(isWellKnownFolderName)
|
||||
if (wellKnownFolders.length > 0) {
|
||||
resolvedFolderIds = await resolveWellKnownFolderIds(
|
||||
accessToken,
|
||||
config.folderIds,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
if (resolvedFolderIds.size < wellKnownFolders.length) {
|
||||
logger.warn(
|
||||
`[${requestId}] Could not resolve all well-known folders (${resolvedFolderIds.size}/${wellKnownFolders.length}) — skipping folder filter to avoid incorrect results`
|
||||
)
|
||||
skipFolderFilter = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const filteredEmails = skipFolderFilter
|
||||
? emails
|
||||
: filterEmailsByFolder(emails, config, resolvedFolderIds)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Fetched ${emails.length} emails, ${filteredEmails.length} after filtering`
|
||||
)
|
||||
|
||||
return { emails: filteredEmails }
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error fetching new Outlook emails:`, errorMessage)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const OUTLOOK_WELL_KNOWN_FOLDERS = new Set([
|
||||
'inbox',
|
||||
'drafts',
|
||||
'sentitems',
|
||||
'deleteditems',
|
||||
'junkemail',
|
||||
'archive',
|
||||
'outbox',
|
||||
])
|
||||
|
||||
function isWellKnownFolderName(folderId: string): boolean {
|
||||
return OUTLOOK_WELL_KNOWN_FOLDERS.has(folderId.toLowerCase())
|
||||
}
|
||||
|
||||
async function resolveWellKnownFolderId(
|
||||
accessToken: string,
|
||||
folderName: string,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`https://graph.microsoft.com/v1.0/me/mailFolders/${folderName}`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn(
|
||||
`[${requestId}] Failed to resolve well-known folder '${folderName}': ${response.status}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
const folder = await response.json()
|
||||
return folder.id || null
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error resolving well-known folder '${folderName}':`, error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveWellKnownFolderIds(
|
||||
accessToken: string,
|
||||
folderIds: string[],
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<Map<string, string>> {
|
||||
const resolvedIds = new Map<string, string>()
|
||||
const wellKnownFolders = folderIds.filter(isWellKnownFolderName)
|
||||
if (wellKnownFolders.length === 0) return resolvedIds
|
||||
|
||||
const resolutions = await Promise.all(
|
||||
wellKnownFolders.map(async (folderName) => {
|
||||
const actualId = await resolveWellKnownFolderId(accessToken, folderName, requestId, logger)
|
||||
return { folderName, actualId }
|
||||
})
|
||||
)
|
||||
|
||||
for (const { folderName, actualId } of resolutions) {
|
||||
if (actualId) {
|
||||
resolvedIds.set(folderName.toLowerCase(), actualId)
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Resolved ${resolvedIds.size}/${wellKnownFolders.length} well-known folders`
|
||||
)
|
||||
return resolvedIds
|
||||
}
|
||||
|
||||
function filterEmailsByFolder(
|
||||
emails: OutlookEmail[],
|
||||
config: OutlookWebhookConfig,
|
||||
resolvedFolderIds?: Map<string, string>
|
||||
): OutlookEmail[] {
|
||||
if (!config.folderIds || !config.folderIds.length) return emails
|
||||
|
||||
const actualFolderIds = config.folderIds.map((configFolder) => {
|
||||
if (resolvedFolderIds && isWellKnownFolderName(configFolder)) {
|
||||
const resolvedId = resolvedFolderIds.get(configFolder.toLowerCase())
|
||||
if (resolvedId) return resolvedId
|
||||
}
|
||||
return configFolder
|
||||
})
|
||||
|
||||
return emails.filter((email) => {
|
||||
const emailFolderId = email.parentFolderId
|
||||
const hasMatchingFolder = actualFolderIds.some(
|
||||
(folderId) => emailFolderId.toLowerCase() === folderId.toLowerCase()
|
||||
)
|
||||
return config.folderFilterBehavior === 'INCLUDE' ? hasMatchingFolder : !hasMatchingFolder
|
||||
})
|
||||
}
|
||||
|
||||
async function processOutlookEmails(
|
||||
emails: OutlookEmail[],
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
config: OutlookWebhookConfig,
|
||||
accessToken: string,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
) {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
for (const email of emails) {
|
||||
try {
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'outlook',
|
||||
`${webhookData.id}:${email.id}`,
|
||||
async () => {
|
||||
let attachments: OutlookAttachment[] = []
|
||||
if (config.includeAttachments && email.hasAttachments) {
|
||||
try {
|
||||
attachments = await downloadOutlookAttachments(
|
||||
accessToken,
|
||||
email.id,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[${requestId}] Error downloading attachments for email ${email.id}:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const simplifiedEmail: SimplifiedOutlookEmail = {
|
||||
id: email.id,
|
||||
conversationId: email.conversationId,
|
||||
subject: email.subject || '',
|
||||
from: email.from?.emailAddress?.address || '',
|
||||
to: email.toRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
|
||||
cc: email.ccRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
|
||||
date: email.receivedDateTime,
|
||||
bodyText: (() => {
|
||||
const content = email.body?.content || ''
|
||||
const type = (email.body?.contentType || '').toLowerCase()
|
||||
if (!content) return email.bodyPreview || ''
|
||||
if (type === 'text' || type === 'text/plain') return content
|
||||
return convertHtmlToPlainText(content)
|
||||
})(),
|
||||
bodyHtml: email.body?.content || '',
|
||||
hasAttachments: email.hasAttachments,
|
||||
attachments,
|
||||
isRead: email.isRead,
|
||||
folderId: email.parentFolderId,
|
||||
messageId: email.id,
|
||||
threadId: email.conversationId,
|
||||
}
|
||||
|
||||
const payload: OutlookWebhookPayload = {
|
||||
email: simplifiedEmail,
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
if (config.includeRawEmail) {
|
||||
payload.rawEmail = email
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Processing email: ${email.subject} from ${email.from?.emailAddress?.address}`
|
||||
)
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for email ${email.id}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
if (config.markAsRead) {
|
||||
await markOutlookEmailAsRead(accessToken, email.id, logger)
|
||||
}
|
||||
|
||||
return { emailId: email.id, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed email ${email.id} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing email ${email.id}:`, error)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount }
|
||||
}
|
||||
|
||||
async function downloadOutlookAttachments(
|
||||
accessToken: string,
|
||||
messageId: string,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<OutlookAttachment[]> {
|
||||
const attachments: OutlookAttachment[] = []
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`https://graph.microsoft.com/v1.0/me/messages/${messageId}/attachments`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`[${requestId}] Failed to fetch attachments for message ${messageId}`)
|
||||
return attachments
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const attachmentsList = data.value || []
|
||||
|
||||
for (const attachment of attachmentsList) {
|
||||
try {
|
||||
if (attachment['@odata.type'] === '#microsoft.graph.fileAttachment') {
|
||||
const contentBytes = attachment.contentBytes
|
||||
if (contentBytes) {
|
||||
const buffer = Buffer.from(contentBytes, 'base64')
|
||||
attachments.push({
|
||||
name: attachment.name,
|
||||
data: buffer,
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`[${requestId}] Error processing attachment ${attachment.id} for message ${messageId}:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Downloaded ${attachments.length} attachments for message ${messageId}`
|
||||
)
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error downloading attachments for message ${messageId}:`, error)
|
||||
}
|
||||
|
||||
return attachments
|
||||
}
|
||||
|
||||
async function markOutlookEmailAsRead(accessToken: string, messageId: string, logger: Logger) {
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`https://graph.microsoft.com/v1.0/me/messages/${messageId}`,
|
||||
{
|
||||
method: 'PATCH',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ isRead: true }),
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(
|
||||
`Failed to mark email ${messageId} as read:`,
|
||||
response.status,
|
||||
await response.text()
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error marking email ${messageId} as read:`, error)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { gmailPollingHandler } from '@/lib/webhooks/polling/gmail'
|
||||
import { googleCalendarPollingHandler } from '@/lib/webhooks/polling/google-calendar'
|
||||
import { googleDrivePollingHandler } from '@/lib/webhooks/polling/google-drive'
|
||||
import { googleSheetsPollingHandler } from '@/lib/webhooks/polling/google-sheets'
|
||||
import { hubspotPollingHandler } from '@/lib/webhooks/polling/hubspot'
|
||||
import { imapPollingHandler } from '@/lib/webhooks/polling/imap'
|
||||
import { outlookPollingHandler } from '@/lib/webhooks/polling/outlook'
|
||||
import { rssPollingHandler } from '@/lib/webhooks/polling/rss'
|
||||
import type { PollingProviderHandler } from '@/lib/webhooks/polling/types'
|
||||
|
||||
const POLLING_HANDLERS: Record<string, PollingProviderHandler> = {
|
||||
gmail: gmailPollingHandler,
|
||||
'google-calendar': googleCalendarPollingHandler,
|
||||
'google-drive': googleDrivePollingHandler,
|
||||
'google-sheets': googleSheetsPollingHandler,
|
||||
hubspot: hubspotPollingHandler,
|
||||
imap: imapPollingHandler,
|
||||
outlook: outlookPollingHandler,
|
||||
rss: rssPollingHandler,
|
||||
}
|
||||
|
||||
export const VALID_POLLING_PROVIDERS = new Set(Object.keys(POLLING_HANDLERS))
|
||||
|
||||
/** Look up the polling handler for a provider. */
|
||||
export function getPollingHandler(provider: string): PollingProviderHandler | undefined {
|
||||
return POLLING_HANDLERS[provider]
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import Parser from 'rss-parser'
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import {
|
||||
secureFetchWithPinnedIP,
|
||||
validateUrlWithDNS,
|
||||
} from '@/lib/core/security/input-validation.server'
|
||||
import {
|
||||
getProviderConfig,
|
||||
type PollingProviderHandler,
|
||||
type PollWebhookContext,
|
||||
} from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
markWebhookFailed,
|
||||
markWebhookSuccess,
|
||||
updateWebhookProviderConfig,
|
||||
} from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
const MAX_GUIDS_TO_TRACK = 500
|
||||
|
||||
interface RssWebhookConfig {
|
||||
feedUrl: string
|
||||
lastCheckedTimestamp?: string
|
||||
lastSeenGuids?: string[]
|
||||
etag?: string
|
||||
lastModified?: string
|
||||
}
|
||||
|
||||
interface RssItem {
|
||||
title?: string
|
||||
link?: string
|
||||
pubDate?: string
|
||||
guid?: string
|
||||
description?: string
|
||||
content?: string
|
||||
contentSnippet?: string
|
||||
author?: string
|
||||
creator?: string
|
||||
categories?: string[]
|
||||
enclosure?: {
|
||||
url: string
|
||||
type?: string
|
||||
length?: string | number
|
||||
}
|
||||
isoDate?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
interface RssFeed {
|
||||
title?: string
|
||||
link?: string
|
||||
description?: string
|
||||
items: RssItem[]
|
||||
}
|
||||
|
||||
interface RssWebhookPayload {
|
||||
title?: string
|
||||
link?: string
|
||||
pubDate?: string
|
||||
item: RssItem
|
||||
feed: {
|
||||
title?: string
|
||||
link?: string
|
||||
description?: string
|
||||
}
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
const parser = new Parser({
|
||||
timeout: 30000,
|
||||
headers: {
|
||||
'User-Agent': 'Sim/1.0 RSS Poller',
|
||||
},
|
||||
})
|
||||
|
||||
export const rssPollingHandler: PollingProviderHandler = {
|
||||
provider: 'rss',
|
||||
label: 'RSS',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
const config = getProviderConfig<RssWebhookConfig>(webhookData.providerConfig)
|
||||
|
||||
if (!config?.feedUrl) {
|
||||
logger.error(`[${requestId}] Missing feedUrl for webhook ${webhookId}`)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
const now = new Date()
|
||||
const {
|
||||
feed,
|
||||
items: newItems,
|
||||
etag,
|
||||
lastModified,
|
||||
} = await fetchNewRssItems(config, requestId, logger)
|
||||
|
||||
if (!newItems.length) {
|
||||
await updateRssState(webhookId, now.toISOString(), [], config, logger, etag, lastModified)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(`[${requestId}] No new items found for webhook ${webhookId}`)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Found ${newItems.length} new items for webhook ${webhookId}`)
|
||||
|
||||
const { processedCount, failedCount } = await processRssItems(
|
||||
newItems,
|
||||
feed,
|
||||
webhookData,
|
||||
workflowData,
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
|
||||
const newGuids = newItems
|
||||
.map(
|
||||
(item) =>
|
||||
item.guid ||
|
||||
item.link ||
|
||||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
|
||||
)
|
||||
.filter((guid) => guid.length > 0)
|
||||
|
||||
await updateRssState(
|
||||
webhookId,
|
||||
now.toISOString(),
|
||||
newGuids,
|
||||
config,
|
||||
logger,
|
||||
etag,
|
||||
lastModified
|
||||
)
|
||||
|
||||
if (failedCount > 0 && processedCount === 0) {
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
logger.warn(
|
||||
`[${requestId}] All ${failedCount} items failed to process for webhook ${webhookId}`
|
||||
)
|
||||
return 'failure'
|
||||
}
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed ${processedCount} items for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
|
||||
)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing RSS webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
async function updateRssState(
|
||||
webhookId: string,
|
||||
timestamp: string,
|
||||
newGuids: string[],
|
||||
config: RssWebhookConfig,
|
||||
logger: Logger,
|
||||
etag?: string,
|
||||
lastModified?: string
|
||||
) {
|
||||
const existingGuids = config.lastSeenGuids || []
|
||||
const allGuids = [...newGuids, ...existingGuids].slice(0, MAX_GUIDS_TO_TRACK)
|
||||
|
||||
await updateWebhookProviderConfig(
|
||||
webhookId,
|
||||
{
|
||||
lastCheckedTimestamp: timestamp,
|
||||
lastSeenGuids: allGuids,
|
||||
...(etag !== undefined ? { etag } : {}),
|
||||
...(lastModified !== undefined ? { lastModified } : {}),
|
||||
},
|
||||
logger
|
||||
)
|
||||
}
|
||||
|
||||
async function fetchNewRssItems(
|
||||
config: RssWebhookConfig,
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> {
|
||||
try {
|
||||
const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl')
|
||||
if (!urlValidation.isValid) {
|
||||
logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`)
|
||||
throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`)
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'User-Agent': 'Sim/1.0 RSS Poller',
|
||||
Accept: 'application/rss+xml, application/xml, text/xml, */*',
|
||||
}
|
||||
if (config.etag) {
|
||||
headers['If-None-Match'] = config.etag
|
||||
}
|
||||
if (config.lastModified) {
|
||||
headers['If-Modified-Since'] = config.lastModified
|
||||
}
|
||||
|
||||
const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, {
|
||||
headers,
|
||||
timeout: 30000,
|
||||
})
|
||||
|
||||
if (response.status === 304) {
|
||||
logger.info(`[${requestId}] RSS feed not modified (304) for ${config.feedUrl}`)
|
||||
return {
|
||||
feed: { items: [] } as RssFeed,
|
||||
items: [],
|
||||
etag: response.headers.get('etag') ?? config.etag,
|
||||
lastModified: response.headers.get('last-modified') ?? config.lastModified,
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
await response.text().catch(() => {})
|
||||
throw new Error(`Failed to fetch RSS feed: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
const newEtag = response.headers.get('etag') ?? undefined
|
||||
const newLastModified = response.headers.get('last-modified') ?? undefined
|
||||
|
||||
const xmlContent = await response.text()
|
||||
const feed = await parser.parseString(xmlContent)
|
||||
|
||||
if (!feed.items || !feed.items.length) {
|
||||
return { feed: feed as RssFeed, items: [], etag: newEtag, lastModified: newLastModified }
|
||||
}
|
||||
|
||||
const lastCheckedTime = config.lastCheckedTimestamp
|
||||
? new Date(config.lastCheckedTimestamp)
|
||||
: null
|
||||
const lastSeenGuids = new Set(config.lastSeenGuids || [])
|
||||
|
||||
const newItems = feed.items.filter((item) => {
|
||||
const itemGuid =
|
||||
item.guid ||
|
||||
item.link ||
|
||||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
|
||||
|
||||
if (itemGuid && lastSeenGuids.has(itemGuid)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (lastCheckedTime && item.isoDate) {
|
||||
const itemDate = new Date(item.isoDate)
|
||||
if (itemDate <= lastCheckedTime) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
newItems.sort((a, b) => {
|
||||
const dateA = a.isoDate ? new Date(a.isoDate).getTime() : 0
|
||||
const dateB = b.isoDate ? new Date(b.isoDate).getTime() : 0
|
||||
return dateB - dateA
|
||||
})
|
||||
|
||||
const limitedItems = newItems.slice(0, 25)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Found ${newItems.length} new items (processing ${limitedItems.length})`
|
||||
)
|
||||
|
||||
return {
|
||||
feed: feed as RssFeed,
|
||||
items: limitedItems as RssItem[],
|
||||
etag: newEtag,
|
||||
lastModified: newLastModified,
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error fetching RSS feed:`, errorMessage)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function processRssItems(
|
||||
items: RssItem[],
|
||||
feed: RssFeed,
|
||||
webhookData: PollWebhookContext['webhookData'],
|
||||
workflowData: PollWebhookContext['workflowData'],
|
||||
requestId: string,
|
||||
logger: Logger
|
||||
): Promise<{ processedCount: number; failedCount: number }> {
|
||||
let processedCount = 0
|
||||
let failedCount = 0
|
||||
|
||||
for (const item of items) {
|
||||
try {
|
||||
const itemGuid =
|
||||
item.guid ||
|
||||
item.link ||
|
||||
(item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
|
||||
|
||||
if (!itemGuid) {
|
||||
logger.warn(
|
||||
`[${requestId}] Skipping RSS item with no identifiable GUID for webhook ${webhookData.id}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
await pollingIdempotency.executeWithIdempotency(
|
||||
'rss',
|
||||
`${webhookData.id}:${itemGuid}`,
|
||||
async () => {
|
||||
const payload: RssWebhookPayload = {
|
||||
title: item.title,
|
||||
link: item.link,
|
||||
pubDate: item.pubDate,
|
||||
item: {
|
||||
title: item.title,
|
||||
link: item.link,
|
||||
pubDate: item.pubDate,
|
||||
guid: item.guid,
|
||||
description: item.description,
|
||||
content: item.content,
|
||||
contentSnippet: item.contentSnippet,
|
||||
author: item.author || item.creator,
|
||||
categories: item.categories,
|
||||
enclosure: item.enclosure,
|
||||
isoDate: item.isoDate,
|
||||
},
|
||||
feed: {
|
||||
title: feed.title,
|
||||
link: feed.link,
|
||||
description: feed.description,
|
||||
},
|
||||
timestamp: new Date().toISOString(),
|
||||
}
|
||||
|
||||
const result = await processPolledWebhookEvent(
|
||||
webhookData,
|
||||
workflowData,
|
||||
payload,
|
||||
requestId
|
||||
)
|
||||
|
||||
if (!result.success) {
|
||||
logger.error(
|
||||
`[${requestId}] Failed to process webhook for item ${itemGuid}:`,
|
||||
result.statusCode,
|
||||
result.error
|
||||
)
|
||||
throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
|
||||
}
|
||||
|
||||
return { itemGuid, processed: true }
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Successfully processed item ${item.title || itemGuid} for webhook ${webhookData.id}`
|
||||
)
|
||||
processedCount++
|
||||
} catch (error) {
|
||||
const errorMessage = getErrorMessage(error, 'Unknown error')
|
||||
logger.error(`[${requestId}] Error processing item:`, errorMessage)
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return { processedCount, failedCount }
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { webhook, workflow } from '@sim/db/schema'
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { isRecordLike } from '@sim/utils/object'
|
||||
|
||||
/** Summary returned after polling all webhooks for a provider. */
|
||||
export interface PollSummary {
|
||||
total: number
|
||||
successful: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
/** Context passed to a provider handler when processing one webhook. */
|
||||
export interface PollWebhookContext {
|
||||
webhookData: WebhookRecord
|
||||
workflowData: WorkflowRecord
|
||||
requestId: string
|
||||
logger: Logger
|
||||
}
|
||||
|
||||
export type WebhookRecord = typeof webhook.$inferSelect
|
||||
export type WorkflowRecord = typeof workflow.$inferSelect
|
||||
|
||||
export function getProviderConfigRecord(
|
||||
providerConfig: WebhookRecord['providerConfig']
|
||||
): Record<string, unknown> {
|
||||
return isRecordLike(providerConfig) ? providerConfig : {}
|
||||
}
|
||||
|
||||
export function getProviderConfig<T extends object>(
|
||||
providerConfig: WebhookRecord['providerConfig']
|
||||
): T {
|
||||
return getProviderConfigRecord(providerConfig) as T
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy interface for provider-specific polling behavior.
|
||||
* Mirrors `WebhookProviderHandler` from `providers/types.ts`.
|
||||
*
|
||||
* Each provider implements `pollWebhook()` — the full inner loop for one webhook:
|
||||
* validate config, resolve credentials, fetch new items, process each via
|
||||
* `processPolledWebhookEvent()` (wrapped in `pollingIdempotency`), update state.
|
||||
*/
|
||||
export interface PollingProviderHandler {
|
||||
/** Provider name used in DB queries (e.g. 'gmail', 'rss'). */
|
||||
readonly provider: string
|
||||
|
||||
/** Display label for log messages (e.g. 'Gmail', 'RSS'). */
|
||||
readonly label: string
|
||||
|
||||
/**
|
||||
* Process a single webhook entry.
|
||||
* Return 'success' (even if 0 new items) or 'failure'.
|
||||
*/
|
||||
pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'>
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockUpdate, mockSet, mockWhere, mockSelect, mockSelectRows, sqlCalls } = vi.hoisted(() => {
|
||||
const mockSelectRows = vi.fn()
|
||||
return {
|
||||
mockUpdate: vi.fn(),
|
||||
mockSet: vi.fn(),
|
||||
mockWhere: vi.fn(),
|
||||
mockSelectRows,
|
||||
mockSelect: vi.fn(() => ({
|
||||
from: vi.fn(() => ({
|
||||
where: vi.fn(() => ({
|
||||
limit: mockSelectRows,
|
||||
})),
|
||||
})),
|
||||
})),
|
||||
sqlCalls: [] as Array<{ strings: readonly string[]; values: unknown[] }>,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@sim/db', () => ({ db: { update: mockUpdate, select: mockSelect } }))
|
||||
vi.mock('@sim/db/schema', () => ({
|
||||
webhook: {
|
||||
id: 'webhook.id',
|
||||
providerConfig: 'webhook.providerConfig',
|
||||
updatedAt: 'webhook.updatedAt',
|
||||
},
|
||||
account: {},
|
||||
workflow: {},
|
||||
workflowDeploymentVersion: {},
|
||||
}))
|
||||
vi.mock('drizzle-orm', () => ({
|
||||
sql: (strings: readonly string[], ...values: unknown[]) => {
|
||||
const node = { strings, values }
|
||||
sqlCalls.push(node)
|
||||
return node
|
||||
},
|
||||
and: vi.fn(),
|
||||
eq: vi.fn((field: unknown, value: unknown) => ({ field, value })),
|
||||
isNull: vi.fn(),
|
||||
ne: vi.fn(),
|
||||
or: vi.fn(),
|
||||
}))
|
||||
vi.mock('@/app/api/auth/oauth/utils', () => ({
|
||||
getOAuthToken: vi.fn(),
|
||||
refreshAccessTokenIfNeeded: vi.fn(),
|
||||
resolveOAuthAccountId: vi.fn(),
|
||||
}))
|
||||
vi.mock('@/triggers/constants', () => ({ MAX_CONSECUTIVE_FAILURES: 5 }))
|
||||
|
||||
import type { WebhookRecord } from '@/lib/webhooks/polling/types'
|
||||
import { resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils'
|
||||
import {
|
||||
getOAuthToken,
|
||||
refreshAccessTokenIfNeeded,
|
||||
resolveOAuthAccountId,
|
||||
} from '@/app/api/auth/oauth/utils'
|
||||
|
||||
const logger = { error: vi.fn() } as never
|
||||
|
||||
function allInterpolatedValues(): unknown[] {
|
||||
return sqlCalls.flatMap((c) => c.values)
|
||||
}
|
||||
|
||||
function allSqlText(): string {
|
||||
return sqlCalls.map((c) => c.strings.join('')).join(' ')
|
||||
}
|
||||
|
||||
describe('updateWebhookProviderConfig (atomic jsonb merge)', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
sqlCalls.length = 0
|
||||
mockWhere.mockResolvedValue(undefined)
|
||||
mockSet.mockReturnValue({ where: mockWhere })
|
||||
mockUpdate.mockReturnValue({ set: mockSet })
|
||||
})
|
||||
|
||||
it('merges defined keys (null preserved) and removes undefined keys', async () => {
|
||||
await updateWebhookProviderConfig(
|
||||
'wh-1',
|
||||
{ historyId: 'h1', cleared: undefined, nulled: null },
|
||||
logger
|
||||
)
|
||||
|
||||
expect(mockUpdate).toHaveBeenCalledTimes(1)
|
||||
expect(allInterpolatedValues()).toContain(JSON.stringify({ historyId: 'h1', nulled: null }))
|
||||
expect(allInterpolatedValues()).toContainEqual(['cleared'])
|
||||
})
|
||||
|
||||
it('uses merge only (no key-removal expression) when nothing is undefined', async () => {
|
||||
await updateWebhookProviderConfig('wh-1', { historyId: 'h1' }, logger)
|
||||
|
||||
expect(allInterpolatedValues()).toContain(JSON.stringify({ historyId: 'h1' }))
|
||||
expect(allInterpolatedValues().some((v) => Array.isArray(v))).toBe(false)
|
||||
})
|
||||
|
||||
it('casts the json column to jsonb for the merge and back to json for storage', async () => {
|
||||
await updateWebhookProviderConfig('wh-1', { historyId: 'h1', cleared: undefined }, logger)
|
||||
|
||||
const sqlText = allSqlText()
|
||||
// Column (interpolated as a value) is cast to jsonb: `COALESCE(<col>::jsonb, ...)`
|
||||
expect(sqlText).toContain('COALESCE(::jsonb')
|
||||
// Merge runs in jsonb space, result cast back to the json column: `(<expr>)::json`
|
||||
expect(sqlText).toContain(')::json')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveOAuthCredential (single-credential polling)', () => {
|
||||
const makeWebhook = (providerConfig: Record<string, unknown>): WebhookRecord =>
|
||||
({ id: 'wh-1', providerConfig }) as unknown as WebhookRecord
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockSelectRows.mockResolvedValue([])
|
||||
})
|
||||
|
||||
it('resolves via credentialId: account lookup then token refresh', async () => {
|
||||
vi.mocked(resolveOAuthAccountId).mockResolvedValue({
|
||||
accountId: 'acc-1',
|
||||
} as Awaited<ReturnType<typeof resolveOAuthAccountId>>)
|
||||
mockSelectRows.mockResolvedValue([{ userId: 'owner-1' }])
|
||||
vi.mocked(refreshAccessTokenIfNeeded).mockResolvedValue('tok-abc')
|
||||
|
||||
const token = await resolveOAuthCredential(
|
||||
makeWebhook({ credentialId: 'cred-1' }),
|
||||
'google-email',
|
||||
'req-1'
|
||||
)
|
||||
|
||||
expect(token).toBe('tok-abc')
|
||||
expect(resolveOAuthAccountId).toHaveBeenCalledWith('cred-1')
|
||||
expect(refreshAccessTokenIfNeeded).toHaveBeenCalledWith('acc-1', 'owner-1', 'req-1')
|
||||
expect(getOAuthToken).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('throws when the credential cannot be resolved to an OAuth account', async () => {
|
||||
vi.mocked(resolveOAuthAccountId).mockResolvedValue(null)
|
||||
|
||||
await expect(
|
||||
resolveOAuthCredential(makeWebhook({ credentialId: 'cred-gone' }), 'google-email', 'req-1')
|
||||
).rejects.toThrow('Failed to resolve OAuth account for credential cred-gone')
|
||||
})
|
||||
|
||||
it('throws when the resolved account row does not exist', async () => {
|
||||
vi.mocked(resolveOAuthAccountId).mockResolvedValue({
|
||||
accountId: 'acc-missing',
|
||||
} as Awaited<ReturnType<typeof resolveOAuthAccountId>>)
|
||||
mockSelectRows.mockResolvedValue([])
|
||||
|
||||
await expect(
|
||||
resolveOAuthCredential(makeWebhook({ credentialId: 'cred-1' }), 'google-email', 'req-1')
|
||||
).rejects.toThrow('Credential cred-1 not found for webhook wh-1')
|
||||
})
|
||||
|
||||
it('falls back to the legacy userId path via getOAuthToken', async () => {
|
||||
vi.mocked(getOAuthToken).mockResolvedValue('tok-legacy')
|
||||
|
||||
const token = await resolveOAuthCredential(
|
||||
makeWebhook({ userId: 'user-1' }),
|
||||
'outlook',
|
||||
'req-1'
|
||||
)
|
||||
|
||||
expect(token).toBe('tok-legacy')
|
||||
expect(getOAuthToken).toHaveBeenCalledWith('user-1', 'outlook')
|
||||
expect(resolveOAuthAccountId).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('throws when neither credentialId nor userId is present', async () => {
|
||||
await expect(resolveOAuthCredential(makeWebhook({}), 'gmail', 'req-1')).rejects.toThrow(
|
||||
'Missing credential info for webhook wh-1'
|
||||
)
|
||||
})
|
||||
|
||||
it('throws when the legacy userId path yields no token', async () => {
|
||||
vi.mocked(getOAuthToken).mockResolvedValue(null)
|
||||
|
||||
await expect(
|
||||
resolveOAuthCredential(makeWebhook({ userId: 'user-1' }), 'outlook', 'req-1')
|
||||
).rejects.toThrow('Failed to get outlook access token for webhook wh-1')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,221 @@
|
||||
import { db } from '@sim/db'
|
||||
import { account, webhook, workflow, workflowDeploymentVersion } from '@sim/db/schema'
|
||||
import type { Logger } from '@sim/logger'
|
||||
import { and, eq, isNull, ne, or, sql } from 'drizzle-orm'
|
||||
import type { WebhookRecord, WorkflowRecord } from '@/lib/webhooks/polling/types'
|
||||
import {
|
||||
getOAuthToken,
|
||||
refreshAccessTokenIfNeeded,
|
||||
resolveOAuthAccountId,
|
||||
} from '@/app/api/auth/oauth/utils'
|
||||
import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants'
|
||||
|
||||
/** Concurrency limit for parallel webhook processing. Standardized across all providers. */
|
||||
export const CONCURRENCY = 10
|
||||
|
||||
/** Increment the webhook's failure count. Auto-disables after MAX_CONSECUTIVE_FAILURES. */
|
||||
export async function markWebhookFailed(webhookId: string, logger: Logger): Promise<void> {
|
||||
try {
|
||||
const result = await db
|
||||
.update(webhook)
|
||||
.set({
|
||||
failedCount: sql`COALESCE(${webhook.failedCount}, 0) + 1`,
|
||||
lastFailedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(webhook.id, webhookId))
|
||||
.returning({ failedCount: webhook.failedCount })
|
||||
|
||||
const newFailedCount = result[0]?.failedCount || 0
|
||||
if (newFailedCount >= MAX_CONSECUTIVE_FAILURES) {
|
||||
await db
|
||||
.update(webhook)
|
||||
.set({
|
||||
isActive: false,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(webhook.id, webhookId))
|
||||
|
||||
logger.warn(
|
||||
`Webhook ${webhookId} auto-disabled after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`Failed to mark webhook ${webhookId} as failed:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
/** Reset the webhook's failure count on successful poll. */
|
||||
export async function markWebhookSuccess(webhookId: string, logger: Logger): Promise<void> {
|
||||
try {
|
||||
await db
|
||||
.update(webhook)
|
||||
.set({
|
||||
failedCount: 0,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
and(eq(webhook.id, webhookId), or(isNull(webhook.failedCount), ne(webhook.failedCount, 0)))
|
||||
)
|
||||
} catch (err) {
|
||||
logger.error(`Failed to mark webhook ${webhookId} as successful:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch all active webhooks for a provider, joined with their workflow. */
|
||||
export async function fetchActiveWebhooks(
|
||||
provider: string
|
||||
): Promise<{ webhook: WebhookRecord; workflow: WorkflowRecord }[]> {
|
||||
const rows = await db
|
||||
.select({ webhook, workflow })
|
||||
.from(webhook)
|
||||
.innerJoin(workflow, eq(webhook.workflowId, workflow.id))
|
||||
.leftJoin(
|
||||
workflowDeploymentVersion,
|
||||
and(
|
||||
eq(workflowDeploymentVersion.workflowId, workflow.id),
|
||||
eq(workflowDeploymentVersion.isActive, true)
|
||||
)
|
||||
)
|
||||
.where(
|
||||
and(
|
||||
eq(webhook.provider, provider),
|
||||
eq(webhook.isActive, true),
|
||||
isNull(webhook.archivedAt),
|
||||
eq(workflow.isDeployed, true),
|
||||
isNull(workflow.archivedAt),
|
||||
or(
|
||||
eq(webhook.deploymentVersionId, workflowDeploymentVersion.id),
|
||||
and(isNull(workflowDeploymentVersion.id), isNull(webhook.deploymentVersionId))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an async function over entries with bounded concurrency.
|
||||
* Returns aggregate success/failure counts.
|
||||
*/
|
||||
export async function runWithConcurrency(
|
||||
entries: { webhook: WebhookRecord; workflow: WorkflowRecord }[],
|
||||
processFn: (entry: {
|
||||
webhook: WebhookRecord
|
||||
workflow: WorkflowRecord
|
||||
}) => Promise<'success' | 'failure'>,
|
||||
logger: Logger
|
||||
): Promise<{ successCount: number; failureCount: number }> {
|
||||
const running: Promise<void>[] = []
|
||||
let successCount = 0
|
||||
let failureCount = 0
|
||||
|
||||
for (const entry of entries) {
|
||||
const promise: Promise<void> = processFn(entry)
|
||||
.then((result) => {
|
||||
if (result === 'success') {
|
||||
successCount++
|
||||
} else {
|
||||
failureCount++
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error('Unexpected error in webhook processing:', err)
|
||||
failureCount++
|
||||
})
|
||||
.finally(() => {
|
||||
const idx = running.indexOf(promise)
|
||||
if (idx !== -1) running.splice(idx, 1)
|
||||
})
|
||||
|
||||
running.push(promise)
|
||||
|
||||
if (running.length >= CONCURRENCY) {
|
||||
await Promise.race(running)
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.allSettled(running)
|
||||
|
||||
return { successCount, failureCount }
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically merge provider-specific config fields into `webhook.provider_config`.
|
||||
* Each provider passes its specific state updates (historyId, lastSeenGuids, etc.).
|
||||
*
|
||||
* The column is `json` (not `jsonb`), which has no merge operators, so the existing
|
||||
* value is cast to `jsonb` for the `||`/`-` merge and the result cast back to `json`
|
||||
* for storage. Casting is required — a bare `jsonb` expression cannot be assigned to
|
||||
* the `json` column.
|
||||
*/
|
||||
export async function updateWebhookProviderConfig(
|
||||
webhookId: string,
|
||||
configUpdates: Record<string, unknown>,
|
||||
logger: Logger
|
||||
): Promise<void> {
|
||||
try {
|
||||
const defined: Record<string, unknown> = {}
|
||||
const removedKeys: string[] = []
|
||||
for (const [key, value] of Object.entries(configUpdates)) {
|
||||
if (value === undefined) removedKeys.push(key)
|
||||
else defined[key] = value
|
||||
}
|
||||
|
||||
const merged = sql`COALESCE(${webhook.providerConfig}::jsonb, '{}'::jsonb) || ${JSON.stringify(defined)}::jsonb`
|
||||
const nextConfig = removedKeys.length > 0 ? sql`(${merged}) - ${removedKeys}::text[]` : merged
|
||||
|
||||
await db
|
||||
.update(webhook)
|
||||
.set({
|
||||
providerConfig: sql`(${nextConfig})::json`,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(webhook.id, webhookId))
|
||||
} catch (err) {
|
||||
logger.error(`Failed to update webhook ${webhookId} config:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve OAuth credentials for a webhook. Shared by Gmail and Outlook.
|
||||
* Returns the access token or throws on failure.
|
||||
*/
|
||||
export async function resolveOAuthCredential(
|
||||
webhookData: WebhookRecord,
|
||||
oauthProvider: string,
|
||||
requestId: string
|
||||
): Promise<string> {
|
||||
const metadata = webhookData.providerConfig as Record<string, unknown> | null
|
||||
const credentialId = metadata?.credentialId as string | undefined
|
||||
const userId = metadata?.userId as string | undefined
|
||||
|
||||
if (!credentialId && !userId) {
|
||||
throw new Error(`Missing credential info for webhook ${webhookData.id}`)
|
||||
}
|
||||
|
||||
let accessToken: string | null = null
|
||||
|
||||
if (credentialId) {
|
||||
const resolved = await resolveOAuthAccountId(credentialId)
|
||||
if (!resolved) {
|
||||
throw new Error(
|
||||
`Failed to resolve OAuth account for credential ${credentialId}, webhook ${webhookData.id}`
|
||||
)
|
||||
}
|
||||
const rows = await db.select().from(account).where(eq(account.id, resolved.accountId)).limit(1)
|
||||
if (!rows.length) {
|
||||
throw new Error(`Credential ${credentialId} not found for webhook ${webhookData.id}`)
|
||||
}
|
||||
const ownerUserId = rows[0].userId
|
||||
accessToken = await refreshAccessTokenIfNeeded(resolved.accountId, ownerUserId, requestId)
|
||||
} else if (userId) {
|
||||
accessToken = await getOAuthToken(userId, oauthProvider)
|
||||
}
|
||||
|
||||
if (!accessToken) {
|
||||
throw new Error(`Failed to get ${oauthProvider} access token for webhook ${webhookData.id}`)
|
||||
}
|
||||
|
||||
return accessToken
|
||||
}
|
||||
Reference in New Issue
Block a user