chore: import upstream snapshot with attribution

apps/sim/lib/webhooks/polling/rss.ts

@@ -0,0 +1,374 @@
+import type { Logger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import Parser from 'rss-parser'
+import { pollingIdempotency } from '@/lib/core/idempotency/service'
+import {
+  secureFetchWithPinnedIP,
+  validateUrlWithDNS,
+} from '@/lib/core/security/input-validation.server'
+import {
+  getProviderConfig,
+  type PollingProviderHandler,
+  type PollWebhookContext,
+} from '@/lib/webhooks/polling/types'
+import {
+  markWebhookFailed,
+  markWebhookSuccess,
+  updateWebhookProviderConfig,
+} from '@/lib/webhooks/polling/utils'
+import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
+
+const MAX_GUIDS_TO_TRACK = 500
+
+interface RssWebhookConfig {
+  feedUrl: string
+  lastCheckedTimestamp?: string
+  lastSeenGuids?: string[]
+  etag?: string
+  lastModified?: string
+}
+
+interface RssItem {
+  title?: string
+  link?: string
+  pubDate?: string
+  guid?: string
+  description?: string
+  content?: string
+  contentSnippet?: string
+  author?: string
+  creator?: string
+  categories?: string[]
+  enclosure?: {
+    url: string
+    type?: string
+    length?: string | number
+  }
+  isoDate?: string
+  [key: string]: unknown
+}
+
+interface RssFeed {
+  title?: string
+  link?: string
+  description?: string
+  items: RssItem[]
+}
+
+interface RssWebhookPayload {
+  title?: string
+  link?: string
+  pubDate?: string
+  item: RssItem
+  feed: {
+    title?: string
+    link?: string
+    description?: string
+  }
+  timestamp: string
+}
+
+const parser = new Parser({
+  timeout: 30000,
+  headers: {
+    'User-Agent': 'Sim/1.0 RSS Poller',
+  },
+})
+
+export const rssPollingHandler: PollingProviderHandler = {
+  provider: 'rss',
+  label: 'RSS',
+
+  async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
+    const { webhookData, workflowData, requestId, logger } = ctx
+    const webhookId = webhookData.id
+
+    try {
+      const config = getProviderConfig<RssWebhookConfig>(webhookData.providerConfig)
+
+      if (!config?.feedUrl) {
+        logger.error(`[${requestId}] Missing feedUrl for webhook ${webhookId}`)
+        await markWebhookFailed(webhookId, logger)
+        return 'failure'
+      }
+
+      const now = new Date()
+      const {
+        feed,
+        items: newItems,
+        etag,
+        lastModified,
+      } = await fetchNewRssItems(config, requestId, logger)
+
+      if (!newItems.length) {
+        await updateRssState(webhookId, now.toISOString(), [], config, logger, etag, lastModified)
+        await markWebhookSuccess(webhookId, logger)
+        logger.info(`[${requestId}] No new items found for webhook ${webhookId}`)
+        return 'success'
+      }
+
+      logger.info(`[${requestId}] Found ${newItems.length} new items for webhook ${webhookId}`)
+
+      const { processedCount, failedCount } = await processRssItems(
+        newItems,
+        feed,
+        webhookData,
+        workflowData,
+        requestId,
+        logger
+      )
+
+      const newGuids = newItems
+        .map(
+          (item) =>
+            item.guid ||
+            item.link ||
+            (item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
+        )
+        .filter((guid) => guid.length > 0)
+
+      await updateRssState(
+        webhookId,
+        now.toISOString(),
+        newGuids,
+        config,
+        logger,
+        etag,
+        lastModified
+      )
+
+      if (failedCount > 0 && processedCount === 0) {
+        await markWebhookFailed(webhookId, logger)
+        logger.warn(
+          `[${requestId}] All ${failedCount} items failed to process for webhook ${webhookId}`
+        )
+        return 'failure'
+      }
+
+      await markWebhookSuccess(webhookId, logger)
+      logger.info(
+        `[${requestId}] Successfully processed ${processedCount} items for webhook ${webhookId}${failedCount > 0 ? ` (${failedCount} failed)` : ''}`
+      )
+      return 'success'
+    } catch (error) {
+      logger.error(`[${requestId}] Error processing RSS webhook ${webhookId}:`, error)
+      await markWebhookFailed(webhookId, logger)
+      return 'failure'
+    }
+  },
+}
+
+async function updateRssState(
+  webhookId: string,
+  timestamp: string,
+  newGuids: string[],
+  config: RssWebhookConfig,
+  logger: Logger,
+  etag?: string,
+  lastModified?: string
+) {
+  const existingGuids = config.lastSeenGuids || []
+  const allGuids = [...newGuids, ...existingGuids].slice(0, MAX_GUIDS_TO_TRACK)
+
+  await updateWebhookProviderConfig(
+    webhookId,
+    {
+      lastCheckedTimestamp: timestamp,
+      lastSeenGuids: allGuids,
+      ...(etag !== undefined ? { etag } : {}),
+      ...(lastModified !== undefined ? { lastModified } : {}),
+    },
+    logger
+  )
+}
+
+async function fetchNewRssItems(
+  config: RssWebhookConfig,
+  requestId: string,
+  logger: Logger
+): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> {
+  try {
+    const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl')
+    if (!urlValidation.isValid) {
+      logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`)
+      throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`)
+    }
+
+    const headers: Record<string, string> = {
+      'User-Agent': 'Sim/1.0 RSS Poller',
+      Accept: 'application/rss+xml, application/xml, text/xml, */*',
+    }
+    if (config.etag) {
+      headers['If-None-Match'] = config.etag
+    }
+    if (config.lastModified) {
+      headers['If-Modified-Since'] = config.lastModified
+    }
+
+    const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, {
+      headers,
+      timeout: 30000,
+    })
+
+    if (response.status === 304) {
+      logger.info(`[${requestId}] RSS feed not modified (304) for ${config.feedUrl}`)
+      return {
+        feed: { items: [] } as RssFeed,
+        items: [],
+        etag: response.headers.get('etag') ?? config.etag,
+        lastModified: response.headers.get('last-modified') ?? config.lastModified,
+      }
+    }
+
+    if (!response.ok) {
+      await response.text().catch(() => {})
+      throw new Error(`Failed to fetch RSS feed: ${response.status} ${response.statusText}`)
+    }
+
+    const newEtag = response.headers.get('etag') ?? undefined
+    const newLastModified = response.headers.get('last-modified') ?? undefined
+
+    const xmlContent = await response.text()
+    const feed = await parser.parseString(xmlContent)
+
+    if (!feed.items || !feed.items.length) {
+      return { feed: feed as RssFeed, items: [], etag: newEtag, lastModified: newLastModified }
+    }
+
+    const lastCheckedTime = config.lastCheckedTimestamp
+      ? new Date(config.lastCheckedTimestamp)
+      : null
+    const lastSeenGuids = new Set(config.lastSeenGuids || [])
+
+    const newItems = feed.items.filter((item) => {
+      const itemGuid =
+        item.guid ||
+        item.link ||
+        (item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
+
+      if (itemGuid && lastSeenGuids.has(itemGuid)) {
+        return false
+      }
+
+      if (lastCheckedTime && item.isoDate) {
+        const itemDate = new Date(item.isoDate)
+        if (itemDate <= lastCheckedTime) {
+          return false
+        }
+      }
+
+      return true
+    })
+
+    newItems.sort((a, b) => {
+      const dateA = a.isoDate ? new Date(a.isoDate).getTime() : 0
+      const dateB = b.isoDate ? new Date(b.isoDate).getTime() : 0
+      return dateB - dateA
+    })
+
+    const limitedItems = newItems.slice(0, 25)
+
+    logger.info(
+      `[${requestId}] Found ${newItems.length} new items (processing ${limitedItems.length})`
+    )
+
+    return {
+      feed: feed as RssFeed,
+      items: limitedItems as RssItem[],
+      etag: newEtag,
+      lastModified: newLastModified,
+    }
+  } catch (error) {
+    const errorMessage = getErrorMessage(error, 'Unknown error')
+    logger.error(`[${requestId}] Error fetching RSS feed:`, errorMessage)
+    throw error
+  }
+}
+
+async function processRssItems(
+  items: RssItem[],
+  feed: RssFeed,
+  webhookData: PollWebhookContext['webhookData'],
+  workflowData: PollWebhookContext['workflowData'],
+  requestId: string,
+  logger: Logger
+): Promise<{ processedCount: number; failedCount: number }> {
+  let processedCount = 0
+  let failedCount = 0
+
+  for (const item of items) {
+    try {
+      const itemGuid =
+        item.guid ||
+        item.link ||
+        (item.title && item.pubDate ? `${item.title}-${item.pubDate}` : '')
+
+      if (!itemGuid) {
+        logger.warn(
+          `[${requestId}] Skipping RSS item with no identifiable GUID for webhook ${webhookData.id}`
+        )
+        continue
+      }
+
+      await pollingIdempotency.executeWithIdempotency(
+        'rss',
+        `${webhookData.id}:${itemGuid}`,
+        async () => {
+          const payload: RssWebhookPayload = {
+            title: item.title,
+            link: item.link,
+            pubDate: item.pubDate,
+            item: {
+              title: item.title,
+              link: item.link,
+              pubDate: item.pubDate,
+              guid: item.guid,
+              description: item.description,
+              content: item.content,
+              contentSnippet: item.contentSnippet,
+              author: item.author || item.creator,
+              categories: item.categories,
+              enclosure: item.enclosure,
+              isoDate: item.isoDate,
+            },
+            feed: {
+              title: feed.title,
+              link: feed.link,
+              description: feed.description,
+            },
+            timestamp: new Date().toISOString(),
+          }
+
+          const result = await processPolledWebhookEvent(
+            webhookData,
+            workflowData,
+            payload,
+            requestId
+          )
+
+          if (!result.success) {
+            logger.error(
+              `[${requestId}] Failed to process webhook for item ${itemGuid}:`,
+              result.statusCode,
+              result.error
+            )
+            throw new Error(`Webhook processing failed (${result.statusCode}): ${result.error}`)
+          }
+
+          return { itemGuid, processed: true }
+        }
+      )
+
+      logger.info(
+        `[${requestId}] Successfully processed item ${item.title || itemGuid} for webhook ${webhookData.id}`
+      )
+      processedCount++
+    } catch (error) {
+      const errorMessage = getErrorMessage(error, 'Unknown error')
+      logger.error(`[${requestId}] Error processing item:`, errorMessage)
+      failedCount++
+    }
+  }
+
+  return { processedCount, failedCount }
+}