simstudioai--sim/apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts

import type { Buffer } from 'buffer'
import path from 'path'
import { createLogger } from '@sim/logger'
import {
  secureFetchWithPinnedIP,
  validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import {
  DEFAULT_MAX_ERROR_BODY_BYTES,
  readResponseTextWithLimit,
  readResponseToBufferWithLimit,
} from '@/lib/core/utils/stream-limits'
import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
import type { UserFile } from '@/executor/types'

const logger = createLogger('FetchExternalUrl')

const DEFAULT_TIMEOUT_MS = 30_000
const DEFAULT_MAX_DOWNLOAD_BYTES = 100 * 1024 * 1024

/**
 * Thrown when the URL fails SSRF/DNS validation. Callers should map this to a
 * user-facing 4xx-style response rather than a generic fetch failure.
 */
export class ExternalUrlValidationError extends Error {
  constructor(message: string) {
    super(message)
    this.name = 'ExternalUrlValidationError'
  }
}

export interface FetchExternalUrlOptions {
  url: string
  userId: string
  /** When provided alongside `saveToWorkspace: true`, the downloaded bytes are persisted as a workspace file. */
  workspaceId?: string
  /** Defaults to true when a `workspaceId` is provided. Set false when the URL already points at our own storage. */
  saveToWorkspace?: boolean
  headers?: Record<string, string>
  signal?: AbortSignal
  maxDownloadBytes?: number
  timeoutMs?: number
}

export interface FetchExternalUrlResult {
  /**
   * Filename derived from the URL path. NOT a content identity — distinct URLs
   * frequently share the same path tail (e.g. every Slack clipboard paste is
   * `image.png`). Never use this as a cache key.
   */
  filename: string
  buffer: Buffer
  /** Content-Type from the response, or inferred from the filename extension. */
  mimeType: string
  /**
   * Saved workspace file record. Undefined when the workspace save was skipped
   * (no workspaceId, `saveToWorkspace: false`, missing write permission, or a
   * save error — the last is logged, not thrown, so the parse path stays alive).
   */
  savedWorkspaceFile?: UserFile
}

/**
 * Fetch an external URL into memory and (optionally) save it as a fresh workspace file.
 *
 * URL fetches are NEVER deduplicated by filename. Two URLs whose paths end in
 * `image.png` are two different fetches that produce two different workspace
 * files; `uploadWorkspaceFile` allocates a unique on-disk name (`image.png`,
 * `image (1).png`, ...) on the save side. Keying a cache by path tail would
 * silently return stale bytes — that was the original bug this helper exists
 * to make unrepresentable.
 */
export async function fetchExternalUrlToWorkspace(
  options: FetchExternalUrlOptions
): Promise<FetchExternalUrlResult> {
  const {
    url,
    userId,
    workspaceId,
    saveToWorkspace = Boolean(workspaceId),
    headers,
    signal,
    maxDownloadBytes = DEFAULT_MAX_DOWNLOAD_BYTES,
    timeoutMs = DEFAULT_TIMEOUT_MS,
  } = options

  const urlValidation = await validateUrlWithDNS(url, 'fileUrl')
  if (!urlValidation.isValid || !urlValidation.resolvedIP) {
    throw new ExternalUrlValidationError(urlValidation.error || 'Invalid external URL')
  }

  const filename = new URL(url).pathname.split('/').pop() || 'download'
  const extension = path.extname(filename).toLowerCase().substring(1)

  const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, {
    timeout: timeoutMs,
    maxResponseBytes: maxDownloadBytes,
    signal,
    ...(headers && Object.keys(headers).length > 0 && { headers }),
  })

  if (!response.ok) {
    await readResponseTextWithLimit(response, {
      maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
      label: 'external url error body',
      signal,
    }).catch(() => '')
    throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`)
  }

  const buffer = await readResponseToBufferWithLimit(response, {
    maxBytes: maxDownloadBytes,
    label: 'external url download',
    signal,
  })

  const mimeType = response.headers.get('content-type') || getMimeTypeFromExtension(extension)

  let savedWorkspaceFile: UserFile | undefined
  if (workspaceId && saveToWorkspace) {
    const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
    if (permission === 'admin' || permission === 'write') {
      try {
        savedWorkspaceFile = await uploadWorkspaceFile(
          workspaceId,
          userId,
          buffer,
          filename,
          mimeType
        )
      } catch (saveError) {
        logger.warn('Failed to save fetched URL to workspace storage', {
          workspaceId,
          filename,
          saveError,
        })
      }
    } else if (permission === null) {
      logger.warn('Skipping workspace save: user is not a workspace member', {
        userId,
        workspaceId,
      })
    } else {
      logger.warn('Skipping workspace save: user lacks write permission', {
        userId,
        workspaceId,
        permission,
      })
    }
  }

  return { filename, buffer, mimeType, savedWorkspaceFile }
}