chore: import upstream snapshot with attribution

apps/sim/lib/webhooks/polling/outlook.ts

@@ -0,0 +1,570 @@
+import type { Logger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { htmlToText } from 'html-to-text'
+import { pollingIdempotency } from '@/lib/core/idempotency/service'
+import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
+import {
+  getProviderConfig,
+  type PollingProviderHandler,
+  type PollWebhookContext,
+} from '@/lib/webhooks/polling/types'
+import {
+  markWebhookFailed,
+  markWebhookSuccess,
+  resolveOAuthCredential,
+  updateWebhookProviderConfig,
+} from '@/lib/webhooks/polling/utils'
+import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
+
+interface OutlookWebhookConfig {
+  credentialId: string
+  folderIds?: string[]
+  folderFilterBehavior?: 'INCLUDE' | 'EXCLUDE'
+  markAsRead?: boolean
+  maxEmailsPerPoll?: number
+  lastCheckedTimestamp?: string
+  includeAttachments?: boolean
+  includeRawEmail?: boolean
+}
+
+interface OutlookEmail {
+  id: string
+  conversationId: string
+  subject: string
+  bodyPreview: string
+  body: {
+    contentType: string
+    content: string
+  }
+  from: {
+    emailAddress: {
+      name: string
+      address: string
+    }
+  }
+  toRecipients: Array<{
+    emailAddress: {
+      name: string
+      address: string
+    }
+  }>
+  ccRecipients?: Array<{
+    emailAddress: {
+      name: string
+      address: string
+    }
+  }>
+  receivedDateTime: string
+  sentDateTime: string
+  hasAttachments: boolean
+  isRead: boolean
+  parentFolderId: string
+}
+
+interface OutlookAttachment {
+  name: string
+  data: Buffer
+  contentType: string
+  size: number
+}
+
+interface SimplifiedOutlookEmail {
+  id: string
+  conversationId: string
+  subject: string
+  from: string
+  to: string
+  cc: string
+  date: string
+  bodyText: string
+  bodyHtml: string
+  hasAttachments: boolean
+  attachments: OutlookAttachment[]
+  isRead: boolean
+  folderId: string
+  messageId: string
+  threadId: string
+}
+
+interface OutlookWebhookPayload {
+  email: SimplifiedOutlookEmail
+  timestamp: string
+  rawEmail?: OutlookEmail
+}
+
+function convertHtmlToPlainText(html: string): string {
+  if (!html) return ''
+  return htmlToText(html, {
+    wordwrap: false,
+    selectors: [
+      { selector: 'a', options: { hideLinkHrefIfSameAsText: true, noAnchorUrl: true } },
+      { selector: 'img', format: 'skip' },
+      { selector: 'script', format: 'skip' },
+      { selector: 'style', format: 'skip' },
+    ],
+    preserveNewlines: true,
+  })
+}
+
+export const outlookPollingHandler: PollingProviderHandler = {
+  provider: 'outlook',
+  label: 'Outlook',
+
+  async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
+    const { webhookData, workflowData, requestId, logger } = ctx
+    const webhookId = webhookData.id
+
+    try {
+      logger.info(`[${requestId}] Processing Outlook webhook: ${webhookId}`)
+
+      const accessToken = await resolveOAuthCredential(webhookData, 'outlook', requestId)
+      const config = getProviderConfig<OutlookWebhookConfig>(webhookData.providerConfig)
+      const now = new Date()
+
+      const { emails } = await fetchNewOutlookEmails(accessToken, config, requestId, logger)
+
+      if (!emails || !emails.length) {
+        await updateWebhookProviderConfig(
+          webhookId,
+          { lastCheckedTimestamp: now.toISOString() },
+          logger
+        )
+        await markWebhookSuccess(webhookId, logger)
+        logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`)
+        return 'success'
+      }
+
+      logger.info(`[${requestId}] Found ${emails.length} emails for webhook ${webhookId}`)
+
+      const { processedCount, failedCount } = await processOutlookEmails(
+        emails,
+        webhookData,
+        workflowData,
+        config,
+        accessToken,
+        requestId,
+        logger
+      )
+
+      await updateWebhookProviderConfig(
+        webhookId,
+        { lastCheckedTimestamp: now.toISOString() },
+        logger
+      )
+
+      if (failedCount > 0 && processedCount === 0) {
+        await markWebhookFailed(webhookId, logger)
+        logger.warn(
+          `[${requestId}] All ${failedCount} emails failed to process for webhook ${webhookId}`
+        )
+        return 'failure'
+      }
+
+      await markWebhookSuccess(webhookId, logger)
+      logger.info(
+        `[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
+      )
+      return 'success'
+    } catch (error) {
+      logger.error(`[${requestId}] Error processing Outlook webhook ${webhookId}:`, error)
+      await markWebhookFailed(webhookId, logger)
+      return 'failure'
+    }
+  },
+}
+
+/** Hard cap on total emails fetched per poll to prevent unbounded pagination loops. */
+const OUTLOOK_HARD_MAX_EMAILS = 200
+
+/** Number of items to request per Graph API page. Decoupled from the total cap so pagination actually runs. */
+const OUTLOOK_PAGE_SIZE = 50
+
+async function fetchNewOutlookEmails(
+  accessToken: string,
+  config: OutlookWebhookConfig,
+  requestId: string,
+  logger: Logger
+) {
+  try {
+    const apiUrl = 'https://graph.microsoft.com/v1.0/me/messages'
+    const params = new URLSearchParams()
+
+    params.append(
+      '$select',
+      'id,conversationId,subject,bodyPreview,body,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,isRead,parentFolderId'
+    )
+    params.append('$orderby', 'receivedDateTime desc')
+    const maxEmails = Math.min(config.maxEmailsPerPoll || 25, OUTLOOK_HARD_MAX_EMAILS)
+    params.append('$top', OUTLOOK_PAGE_SIZE.toString())
+
+    if (config.lastCheckedTimestamp) {
+      const lastChecked = new Date(config.lastCheckedTimestamp)
+      const bufferTime = new Date(lastChecked.getTime() - 60000)
+      params.append('$filter', `receivedDateTime gt ${bufferTime.toISOString()}`)
+    }
+    const allEmails: OutlookEmail[] = []
+    let nextUrl: string | undefined = `${apiUrl}?${params.toString()}`
+    logger.info(`[${requestId}] Fetching emails from: ${nextUrl}`)
+
+    while (nextUrl && allEmails.length < maxEmails) {
+      const response = await fetchWithRetry(nextUrl, {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+        },
+      })
+
+      if (!response.ok) {
+        const errorData = await response
+          .json()
+          .catch(() => ({ error: { message: 'Unknown error' } }))
+        logger.error(`[${requestId}] Microsoft Graph API error:`, {
+          status: response.status,
+          statusText: response.statusText,
+          error: errorData,
+        })
+        throw new Error(
+          `Microsoft Graph API error: ${response.status} ${response.statusText} - ${JSON.stringify(errorData)}`
+        )
+      }
+
+      const data = await response.json()
+      const pageEmails: OutlookEmail[] = data.value || []
+      const remaining = maxEmails - allEmails.length
+      allEmails.push(...pageEmails.slice(0, remaining))
+
+      nextUrl =
+        allEmails.length < maxEmails ? (data['@odata.nextLink'] as string | undefined) : undefined
+
+      if (pageEmails.length === 0) break
+    }
+
+    logger.info(`[${requestId}] Fetched ${allEmails.length} emails total`)
+
+    const emails = allEmails
+
+    let resolvedFolderIds: Map<string, string> | undefined
+    let skipFolderFilter = false
+    if (config.folderIds && config.folderIds.length > 0) {
+      const wellKnownFolders = config.folderIds.filter(isWellKnownFolderName)
+      if (wellKnownFolders.length > 0) {
+        resolvedFolderIds = await resolveWellKnownFolderIds(
+          accessToken,
+          config.folderIds,
+          requestId,
+          logger
+        )
+        if (resolvedFolderIds.size < wellKnownFolders.length) {
+          logger.warn(
+            `[${requestId}] Could not resolve all well-known folders (${resolvedFolderIds.size}/${wellKnownFolders.length}) — skipping folder filter to avoid incorrect results`
+          )
+          skipFolderFilter = true
+        }
+      }
+    }
+
+    const filteredEmails = skipFolderFilter
+      ? emails
+      : filterEmailsByFolder(emails, config, resolvedFolderIds)
+
+    logger.info(
+      `[${requestId}] Fetched ${emails.length} emails, ${filteredEmails.length} after filtering`
+    )
+
+    return { emails: filteredEmails }
+  } catch (error) {
+    const errorMessage = getErrorMessage(error, 'Unknown error')
+    logger.error(`[${requestId}] Error fetching new Outlook emails:`, errorMessage)
+    throw error
+  }
+}
+
+const OUTLOOK_WELL_KNOWN_FOLDERS = new Set([
+  'inbox',
+  'drafts',
+  'sentitems',
+  'deleteditems',
+  'junkemail',
+  'archive',
+  'outbox',
+])
+
+function isWellKnownFolderName(folderId: string): boolean {
+  return OUTLOOK_WELL_KNOWN_FOLDERS.has(folderId.toLowerCase())
+}
+
+async function resolveWellKnownFolderId(
+  accessToken: string,
+  folderName: string,
+  requestId: string,
+  logger: Logger
+): Promise<string | null> {
+  try {
+    const response = await fetchWithRetry(
+      `https://graph.microsoft.com/v1.0/me/mailFolders/${folderName}`,
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+        },
+      }
+    )
+
+    if (!response.ok) {
+      logger.warn(
+        `[${requestId}] Failed to resolve well-known folder '${folderName}': ${response.status}`
+      )
+      return null
+    }
+
+    const folder = await response.json()
+    return folder.id || null
+  } catch (error) {
+    logger.error(`[${requestId}] Error resolving well-known folder '${folderName}':`, error)
+    return null
+  }
+}
+
+async function resolveWellKnownFolderIds(
+  accessToken: string,
+  folderIds: string[],
+  requestId: string,
+  logger: Logger
+): Promise<Map<string, string>> {
+  const resolvedIds = new Map<string, string>()
+  const wellKnownFolders = folderIds.filter(isWellKnownFolderName)
+  if (wellKnownFolders.length === 0) return resolvedIds
+
+  const resolutions = await Promise.all(
+    wellKnownFolders.map(async (folderName) => {
+      const actualId = await resolveWellKnownFolderId(accessToken, folderName, requestId, logger)
+      return { folderName, actualId }
+    })
+  )
+
+  for (const { folderName, actualId } of resolutions) {
+    if (actualId) {
+      resolvedIds.set(folderName.toLowerCase(), actualId)
+    }
+  }
+
+  logger.info(
+    `[${requestId}] Resolved ${resolvedIds.size}/${wellKnownFolders.length} well-known folders`
+  )
+  return resolvedIds
+}
+
+function filterEmailsByFolder(
+  emails: OutlookEmail[],
+  config: OutlookWebhookConfig,
+  resolvedFolderIds?: Map<string, string>
+): OutlookEmail[] {
+  if (!config.folderIds || !config.folderIds.length) return emails
+
+  const actualFolderIds = config.folderIds.map((configFolder) => {
+    if (resolvedFolderIds && isWellKnownFolderName(configFolder)) {
+      const resolvedId = resolvedFolderIds.get(configFolder.toLowerCase())
+      if (resolvedId) return resolvedId
+    }
+    return configFolder
+  })
+
+  return emails.filter((email) => {
+    const emailFolderId = email.parentFolderId
+    const hasMatchingFolder = actualFolderIds.some(
+      (folderId) => emailFolderId.toLowerCase() === folderId.toLowerCase()
+    )
+    return config.folderFilterBehavior === 'INCLUDE' ? hasMatchingFolder : !hasMatchingFolder
+  })
+}
+
+async function processOutlookEmails(
+  emails: OutlookEmail[],
+  webhookData: PollWebhookContext['webhookData'],
+  workflowData: PollWebhookContext['workflowData'],
+  config: OutlookWebhookConfig,
+  accessToken: string,
+  requestId: string,
+  logger: Logger
+) {
+  let processedCount = 0
+  let failedCount = 0
+
+  for (const email of emails) {
+    try {
+      await pollingIdempotency.executeWithIdempotency(
+        'outlook',
+        `${webhookData.id}:${email.id}`,
+        async () => {
+          let attachments: OutlookAttachment[] = []
+          if (config.includeAttachments && email.hasAttachments) {
+            try {
+              attachments = await downloadOutlookAttachments(
+                accessToken,
+                email.id,
+                requestId,
+                logger
+              )
+            } catch (error) {
+              logger.error(
+                `[${requestId}] Error downloading attachments for email ${email.id}:`,
+                error
+              )
+            }
+          }
+
+          const simplifiedEmail: SimplifiedOutlookEmail = {
+            id: email.id,
+            conversationId: email.conversationId,
+            subject: email.subject || '',
+            from: email.from?.emailAddress?.address || '',
+            to: email.toRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
+            cc: email.ccRecipients?.map((r) => r.emailAddress.address).join(', ') || '',
+            date: email.receivedDateTime,
+            bodyText: (() => {
+              const content = email.body?.content || ''
+              const type = (email.body?.contentType || '').toLowerCase()
+              if (!content) return email.bodyPreview || ''
+              if (type === 'text' || type === 'text/plain') return content
+              return convertHtmlToPlainText(content)
+            })(),
+            bodyHtml: email.body?.content || '',
+            hasAttachments: email.hasAttachments,
+            attachments,
+            isRead: email.isRead,
+            folderId: email.parentFolderId,
+            messageId: email.id,
+            threadId: email.conversationId,
+          }
+
+          const payload: OutlookWebhookPayload = {
+            email: simplifiedEmail,
+            timestamp: new Date().toISOString(),
+          }
+
+          if (config.includeRawEmail) {
+            payload.rawEmail = email
+          }
+
+          logger.info(
+            `[${requestId}] Processing email: ${email.subject} from ${email.from?.emailAddress?.address}`
+          )
+
+          const result = await processPolledWebhookEvent(
+            webhookData,
+            workflowData,
+            payload,
+            requestId
+          )
+
+          if (!result.success) {
+            logger.error(
+              `[${requestId}] Failed to process webhook for email ${email.id}:`,
+              result.statusCode,
+              result.error
+            )
+            throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
+          }
+
+          if (config.markAsRead) {
+            await markOutlookEmailAsRead(accessToken, email.id, logger)
+          }
+
+          return { emailId: email.id, processed: true }
+        }
+      )
+
+      logger.info(
+        `[${requestId}] Successfully processed email ${email.id} for webhook ${webhookData.id}`
+      )
+      processedCount++
+    } catch (error) {
+      logger.error(`[${requestId}] Error processing email ${email.id}:`, error)
+      failedCount++
+    }
+  }
+
+  return { processedCount, failedCount }
+}
+
+async function downloadOutlookAttachments(
+  accessToken: string,
+  messageId: string,
+  requestId: string,
+  logger: Logger
+): Promise<OutlookAttachment[]> {
+  const attachments: OutlookAttachment[] = []
+
+  try {
+    const response = await fetchWithRetry(
+      `https://graph.microsoft.com/v1.0/me/messages/${messageId}/attachments`,
+      {
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+        },
+      }
+    )
+
+    if (!response.ok) {
+      logger.error(`[${requestId}] Failed to fetch attachments for message ${messageId}`)
+      return attachments
+    }
+
+    const data = await response.json()
+    const attachmentsList = data.value || []
+
+    for (const attachment of attachmentsList) {
+      try {
+        if (attachment['@odata.type'] === '#microsoft.graph.fileAttachment') {
+          const contentBytes = attachment.contentBytes
+          if (contentBytes) {
+            const buffer = Buffer.from(contentBytes, 'base64')
+            attachments.push({
+              name: attachment.name,
+              data: buffer,
+              contentType: attachment.contentType,
+              size: attachment.size,
+            })
+          }
+        }
+      } catch (error) {
+        logger.error(
+          `[${requestId}] Error processing attachment ${attachment.id} for message ${messageId}:`,
+          error
+        )
+      }
+    }
+
+    logger.info(
+      `[${requestId}] Downloaded ${attachments.length} attachments for message ${messageId}`
+    )
+  } catch (error) {
+    logger.error(`[${requestId}] Error downloading attachments for message ${messageId}:`, error)
+  }
+
+  return attachments
+}
+
+async function markOutlookEmailAsRead(accessToken: string, messageId: string, logger: Logger) {
+  try {
+    const response = await fetchWithRetry(
+      `https://graph.microsoft.com/v1.0/me/messages/${messageId}`,
+      {
+        method: 'PATCH',
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ isRead: true }),
+      }
+    )
+
+    if (!response.ok) {
+      logger.error(
+        `Failed to mark email ${messageId} as read:`,
+        response.status,
+        await response.text()
+      )
+    }
+  } catch (error) {
+    logger.error(`Error marking email ${messageId} as read:`, error)
+  }
+}