chore: import upstream snapshot with attribution

apps/sim/connectors/utils.ts

@@ -0,0 +1,243 @@
+import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server'
+import { MAX_FILE_SIZE as KB_DOCUMENT_MAX_BYTES } from '@/lib/uploads/utils/validation'
+import type { ExternalDocument } from '@/connectors/types'
+
+/**
+ * Per-file size cap for knowledge base connector syncs. Aligned with the limit for
+ * manually uploaded KB documents (`MAX_FILE_SIZE` in `uploads/validation`) so a
+ * connector indexes the same files a user could add by hand — rather than the much
+ * lower proxy-derived 10 MB number that previously (and arbitrarily) applied here.
+ *
+ * Connector downloads are streamed against this cap via `readBodyWithLimit`, and
+ * files above it are surfaced as skipped (failed) documents instead of being dropped
+ * silently, so raising the limit stays memory-safe and visible.
+ */
+export const CONNECTOR_MAX_FILE_BYTES = KB_DOCUMENT_MAX_BYTES
+
+/**
+ * Strips HTML tags from content and decodes common HTML entities.
+ */
+export function htmlToPlainText(html: string): string {
+  let text = html.replace(/<[^>]*>/g, ' ')
+  text = text
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&amp;/g, '&')
+  return text.replace(/\s+/g, ' ').trim()
+}
+
+/**
+ * Computes a SHA-256 hash of the given content string.
+ * Used by connectors for change detection during sync.
+ */
+export async function computeContentHash(content: string): Promise<string> {
+  const encoder = new TextEncoder()
+  const data = encoder.encode(content)
+  const hashBuffer = await crypto.subtle.digest('SHA-256', data)
+  const hashArray = Array.from(new Uint8Array(hashBuffer))
+  return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
+}
+
+/**
+ * Parses a string metadata value as a Date for tag mapping.
+ * Returns the Date if valid, undefined otherwise.
+ */
+export function parseTagDate(value: unknown): Date | undefined {
+  if (typeof value !== 'string') return undefined
+  const date = new Date(value)
+  return Number.isNaN(date.getTime()) ? undefined : date
+}
+
+/**
+ * Joins an array metadata value into a comma-separated string for tag mapping.
+ * Returns the joined string if non-empty, undefined otherwise.
+ */
+export function joinTagArray(value: unknown): string | undefined {
+  const arr = Array.isArray(value) ? (value as string[]) : []
+  return arr.length > 0 ? arr.join(', ') : undefined
+}
+
+/**
+ * Normalizes a multi-value sourceConfig field into a trimmed, deduplicated string array.
+ *
+ * Accepts a string (CSV from advanced manual input or legacy single-value), an array
+ * of strings (from multi-select UI or new array storage), or undefined/null. Always
+ * returns a string[] — connectors call this once at the top of listDocuments to
+ * branch on `values.length` for single vs multi behavior.
+ */
+export function parseMultiValue(value: unknown): string[] {
+  if (Array.isArray(value)) {
+    const seen = new Set<string>()
+    const out: string[] = []
+    for (const item of value) {
+      if (typeof item !== 'string') continue
+      const trimmed = item.trim()
+      if (!trimmed || seen.has(trimmed)) continue
+      seen.add(trimmed)
+      out.push(trimmed)
+    }
+    return out
+  }
+  if (typeof value === 'string') {
+    const seen = new Set<string>()
+    const out: string[] = []
+    for (const part of value.split(',')) {
+      const trimmed = part.trim()
+      if (!trimmed || seen.has(trimmed)) continue
+      seen.add(trimmed)
+      out.push(trimmed)
+    }
+    return out
+  }
+  return []
+}
+
+/**
+ * Escapes a value for safe interpolation into a Google Drive `q` query string,
+ * neutralizing backslashes and single quotes to prevent query injection.
+ */
+export function escapeDriveQueryValue(value: string): string {
+  return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")
+}
+
+/**
+ * Builds a Drive `q` clause matching files parented by any of the given folder
+ * IDs — e.g. `('A' in parents or 'B' in parents)`. Returns null when no folder
+ * IDs are supplied so callers can omit the clause entirely. A single ID is
+ * emitted without wrapping parentheses to keep the query minimal.
+ */
+export function buildDriveParentsClause(folderIds: string[]): string | null {
+  if (folderIds.length === 0) return null
+  const clause = folderIds.map((id) => `'${escapeDriveQueryValue(id)}' in parents`).join(' or ')
+  return folderIds.length > 1 ? `(${clause})` : clause
+}
+
+/**
+ * Reads a response body into a Buffer while enforcing a hard byte cap. The
+ * declared `content-length` header cannot be trusted as the sole guard —
+ * chunked transfer encoding may omit it entirely — so bytes are accumulated
+ * from the stream and reading aborts as soon as the cap is exceeded, ensuring
+ * an oversized (or hostile) body is never fully buffered into memory.
+ * Returns null when the cap is exceeded.
+ */
+export async function readBodyWithLimit(
+  response: Response | SecureFetchResponse,
+  maxBytes: number
+): Promise<Buffer | null> {
+  if (!response.body) {
+    const buffer = Buffer.from(await response.arrayBuffer())
+    return buffer.byteLength > maxBytes ? null : buffer
+  }
+
+  const reader = response.body.getReader()
+  const chunks: Uint8Array[] = []
+  let total = 0
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    total += value.byteLength
+    if (total > maxBytes) {
+      await reader.cancel().catch(() => {})
+      return null
+    }
+    chunks.push(value)
+  }
+  return Buffer.concat(chunks)
+}
+
+/**
+ * Marks a listed document stub as intentionally skipped — for example because it
+ * exceeds the connector's size limit. The sync engine records these as `failed`
+ * documents carrying `skippedReason`, so oversized files stay visible in the
+ * knowledge base UI instead of vanishing from the index silently. Reuses the
+ * connector's own stub so the externalId, contentHash, sourceUrl, and metadata
+ * (including fileSize) are preserved.
+ */
+export function markSkipped(stub: ExternalDocument, reason: string): ExternalDocument {
+  return { ...stub, content: '', contentDeferred: false, skippedReason: reason }
+}
+
+/** Human-readable size-limit skip reason, e.g. "File exceeds the 10MB size limit". */
+export function sizeLimitSkipReason(maxBytes: number): string {
+  return `File exceeds the ${Math.round(maxBytes / (1024 * 1024))}MB size limit and was not indexed`
+}
+
+/**
+ * Returns the listing stub as-is, or a skipped marker when its size exceeds the cap.
+ * Lets each connector express the listing-time size decision once instead of
+ * repeating the `size > max ? markSkipped(...) : stub` ternary (and building the stub
+ * twice). A missing/zero size is treated as within the cap (oversize is then caught
+ * at fetch time via `ConnectorFileTooLargeError`).
+ */
+export function stubOrSkipBySize(
+  stub: ExternalDocument,
+  size: number | undefined,
+  maxBytes: number
+): ExternalDocument {
+  return size && size > maxBytes ? markSkipped(stub, sizeLimitSkipReason(maxBytes)) : stub
+}
+
+/** True when a stub has been flagged as skipped (e.g. oversized) via `markSkipped`. */
+export function isSkippedDocument(doc: ExternalDocument): boolean {
+  return doc.skippedReason !== undefined
+}
+
+/**
+ * Applies a document cap (`maxFiles`/`maxObjects`/`maxRecordings`) to a page of
+ * listing items so that only **indexable** items consume the cap. Skipped
+ * (oversized) items still ride along and surface as failed rows, but they no longer
+ * count toward the budget — otherwise a run of oversized files at the front of a
+ * listing could exhaust the cap before any indexable file is listed, silently
+ * shrinking real sync coverage.
+ *
+ * Items are walked in listing order: every item (indexable or skipped) is emitted
+ * until the indexable quota is reached, then iteration stops. A cap of `0` (or less)
+ * means unlimited and all items pass through.
+ *
+ * @param items page items in listing order
+ * @param isSkipped predicate identifying non-indexable (skipped) items
+ * @param max configured cap; `0` or less means no cap
+ * @param alreadyIndexed indexable items already counted on previous pages
+ * @returns the emitted items, the number of indexable items emitted (the only ones
+ *   that count toward the cap), and whether the cap is now reached
+ */
+export function takeIndexableWithinCap<T>(
+  items: T[],
+  isSkipped: (item: T) => boolean,
+  max: number,
+  alreadyIndexed: number
+): { documents: T[]; indexableCount: number; capReached: boolean } {
+  if (max <= 0) {
+    let indexableCount = 0
+    for (const item of items) {
+      if (!isSkipped(item)) indexableCount += 1
+    }
+    return { documents: items, indexableCount, capReached: false }
+  }
+
+  const remaining = max - alreadyIndexed
+  const documents: T[] = []
+  let indexableCount = 0
+  for (const item of items) {
+    if (indexableCount >= remaining) break
+    documents.push(item)
+    if (!isSkipped(item)) indexableCount += 1
+  }
+  return { documents, indexableCount, capReached: alreadyIndexed + indexableCount >= max }
+}
+
+/**
+ * Raised by a connector when a file exceeds its size cap mid-download — i.e. the
+ * listing did not report a size, so the limit is only discovered while streaming.
+ * `getDocument` catches it and returns a `markSkipped` document so the file surfaces
+ * as a failed row instead of being dropped silently.
+ */
+export class ConnectorFileTooLargeError extends Error {
+  constructor(readonly limitBytes: number) {
+    super(`File exceeds the ${Math.round(limitBytes / (1024 * 1024))}MB size limit`)
+    this.name = 'ConnectorFileTooLargeError'
+  }
+}