chore: import upstream snapshot with attribution

apps/sim/lib/uploads/core/setup.server.ts

@@ -0,0 +1,124 @@
+import { existsSync } from 'fs'
+import { mkdir } from 'fs/promises'
+import path, { join } from 'path'
+import { createLogger } from '@sim/logger'
+import { env } from '@/lib/core/config/env'
+import {
+  getStorageProvider,
+  S3_CONFIG,
+  USE_BLOB_STORAGE,
+  USE_S3_STORAGE,
+} from '@/lib/uploads/config'
+
+const logger = createLogger('UploadsSetup')
+
+// turbopackIgnore: an unscoped process.cwd() makes node-file-tracing sweep the whole
+// project (including next.config.ts) into every route graph that reaches this module.
+// Two routes doing so emit the swept config into same-named server chunks — when their
+// contents diverge, the build dies with "Two or more assets … same output path".
+const PROJECT_ROOT = path.resolve(/*turbopackIgnore: true*/ process.cwd())
+export const UPLOAD_DIR_SERVER = join(/*turbopackIgnore: true*/ PROJECT_ROOT, 'uploads')
+
+/**
+ * Server-only function to ensure uploads directory exists
+ */
+async function ensureUploadsDirectory() {
+  if (USE_S3_STORAGE) {
+    logger.info('Using S3 storage, skipping local uploads directory creation')
+    return true
+  }
+
+  if (USE_BLOB_STORAGE) {
+    logger.info('Using Azure Blob storage, skipping local uploads directory creation')
+    return true
+  }
+
+  try {
+    if (!existsSync(UPLOAD_DIR_SERVER)) {
+      await mkdir(UPLOAD_DIR_SERVER, { recursive: true })
+    } else {
+      logger.info(`Uploads directory already exists at ${UPLOAD_DIR_SERVER}`)
+    }
+    return true
+  } catch (error) {
+    logger.error('Failed to create uploads directory:', error)
+    return false
+  }
+}
+
+// Immediately invoke on server startup
+if (typeof process !== 'undefined') {
+  const storageProvider = getStorageProvider()
+
+  // Log storage mode
+  logger.info(`Storage provider: ${storageProvider}`)
+
+  if (USE_BLOB_STORAGE) {
+    // Verify Azure Blob credentials
+    if (!env.AZURE_STORAGE_CONTAINER_NAME) {
+      logger.warn('Azure Blob storage is enabled but AZURE_STORAGE_CONTAINER_NAME is not set')
+    } else if (!env.AZURE_ACCOUNT_NAME && !env.AZURE_CONNECTION_STRING) {
+      logger.warn(
+        'Azure Blob storage is enabled but neither AZURE_ACCOUNT_NAME nor AZURE_CONNECTION_STRING is set'
+      )
+      logger.warn(
+        'Set AZURE_ACCOUNT_NAME + AZURE_ACCOUNT_KEY or AZURE_CONNECTION_STRING for Azure Blob storage'
+      )
+    } else if (env.AZURE_ACCOUNT_NAME && !env.AZURE_ACCOUNT_KEY && !env.AZURE_CONNECTION_STRING) {
+      logger.warn(
+        'AZURE_ACCOUNT_NAME is set but AZURE_ACCOUNT_KEY is missing and no AZURE_CONNECTION_STRING provided'
+      )
+      logger.warn('Set AZURE_ACCOUNT_KEY or use AZURE_CONNECTION_STRING for authentication')
+    } else {
+      logger.info('Azure Blob storage credentials found in environment variables')
+      if (env.AZURE_CONNECTION_STRING) {
+        logger.info('Using Azure connection string for authentication')
+      } else {
+        logger.info('Using Azure account name and key for authentication')
+      }
+    }
+  } else if (USE_S3_STORAGE) {
+    // Verify AWS credentials
+    if (!env.S3_BUCKET_NAME || !env.AWS_REGION) {
+      logger.warn('S3 storage configuration is incomplete')
+      logger.warn('Set S3_BUCKET_NAME and AWS_REGION for S3 storage')
+    } else if (!env.AWS_ACCESS_KEY_ID || !env.AWS_SECRET_ACCESS_KEY) {
+      logger.warn('AWS credentials are not set in environment variables')
+      logger.warn('Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 storage')
+    } else {
+      logger.info('AWS S3 credentials found in environment variables')
+    }
+
+    if (env.S3_ENDPOINT) {
+      logger.info(
+        `Using S3-compatible endpoint: ${env.S3_ENDPOINT} (path-style: ${S3_CONFIG.forcePathStyle})`
+      )
+    }
+  } else {
+    // Local storage mode
+    logger.info('Using local file storage')
+
+    // Only initialize local uploads directory when using local storage
+    ensureUploadsDirectory().then((success) => {
+      if (success) {
+        logger.info('Local uploads directory initialized')
+      } else {
+        logger.error('Failed to initialize local uploads directory')
+      }
+    })
+  }
+
+  // Log additional configuration details
+  if (USE_BLOB_STORAGE && env.AZURE_STORAGE_KB_CONTAINER_NAME) {
+    logger.info(`Azure Blob knowledge base container: ${env.AZURE_STORAGE_KB_CONTAINER_NAME}`)
+  }
+  if (USE_BLOB_STORAGE && env.AZURE_STORAGE_COPILOT_CONTAINER_NAME) {
+    logger.info(`Azure Blob copilot container: ${env.AZURE_STORAGE_COPILOT_CONTAINER_NAME}`)
+  }
+  if (USE_S3_STORAGE && env.S3_KB_BUCKET_NAME) {
+    logger.info(`S3 knowledge base bucket: ${env.S3_KB_BUCKET_NAME}`)
+  }
+  if (USE_S3_STORAGE && env.S3_COPILOT_BUCKET_NAME) {
+    logger.info(`S3 copilot bucket: ${env.S3_COPILOT_BUCKET_NAME}`)
+  }
+}

apps/sim/lib/uploads/core/storage-client.ts

@@ -0,0 +1,97 @@
+import { USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/config'
+import type { StorageConfig } from '@/lib/uploads/shared/types'
+
+export type { StorageConfig } from '@/lib/uploads/shared/types'
+
+/**
+ * Get the current storage provider name
+ */
+export function getStorageProvider(): 'blob' | 's3' | 'local' {
+  if (USE_BLOB_STORAGE) return 'blob'
+  if (USE_S3_STORAGE) return 's3'
+  return 'local'
+}
+
+/**
+ * Get the serve path prefix (unified across all storage providers)
+ */
+export function getServePathPrefix(): string {
+  return '/api/files/serve/'
+}
+
+/**
+ * Get file metadata from storage provider
+ * @param key File key/name
+ * @param customConfig Optional custom storage configuration
+ * @returns File metadata object with userId, workspaceId, originalName, uploadedAt, etc.
+ */
+export async function getFileMetadata(
+  key: string,
+  customConfig?: StorageConfig
+): Promise<Record<string, string>> {
+  const { getFileMetadataByKey } = await import('../server/metadata')
+  const metadataRecord = await getFileMetadataByKey(key)
+
+  if (metadataRecord) {
+    return {
+      userId: metadataRecord.userId,
+      workspaceId: metadataRecord.workspaceId || '',
+      originalName: metadataRecord.originalName,
+      uploadedAt: metadataRecord.uploadedAt.toISOString(),
+      purpose: metadataRecord.context,
+    }
+  }
+
+  if (USE_BLOB_STORAGE) {
+    const { getBlobServiceClient } = await import('@/lib/uploads/providers/blob/client')
+    const { BLOB_CONFIG } = await import('@/lib/uploads/config')
+
+    let blobServiceClient = await getBlobServiceClient()
+    let containerName = BLOB_CONFIG.containerName
+
+    if (customConfig) {
+      const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob')
+      if (customConfig.connectionString) {
+        blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString)
+      } else if (customConfig.accountName && customConfig.accountKey) {
+        const credential = new StorageSharedKeyCredential(
+          customConfig.accountName,
+          customConfig.accountKey
+        )
+        blobServiceClient = new BlobServiceClient(
+          `https://${customConfig.accountName}.blob.core.windows.net`,
+          credential
+        )
+      }
+      containerName = customConfig.containerName || containerName
+    }
+
+    const containerClient = blobServiceClient.getContainerClient(containerName)
+    const blockBlobClient = containerClient.getBlockBlobClient(key)
+    const properties = await blockBlobClient.getProperties()
+    return properties.metadata || {}
+  }
+
+  if (USE_S3_STORAGE) {
+    const { getS3Client } = await import('@/lib/uploads/providers/s3/client')
+    const { HeadObjectCommand } = await import('@aws-sdk/client-s3')
+    const { S3_CONFIG } = await import('@/lib/uploads/config')
+
+    const s3Client = getS3Client()
+    const bucket = customConfig?.bucket || S3_CONFIG.bucket
+
+    if (!bucket) {
+      throw new Error('S3 bucket not configured')
+    }
+
+    const command = new HeadObjectCommand({
+      Bucket: bucket,
+      Key: key,
+    })
+
+    const response = await s3Client.send(command)
+    return response.Metadata || {}
+  }
+
+  return {}
+}

apps/sim/lib/uploads/core/storage-service.blob-connection-string.test.ts

@@ -0,0 +1,82 @@
+/**
+ * Regression tests: Azure Blob storage must be fully usable with ONLY
+ * AZURE_CONNECTION_STRING set (no AZURE_ACCOUNT_NAME/AZURE_ACCOUNT_KEY) — this
+ * is the connection-string auth mode documented as a standalone alternative
+ * across .env.example, helm/sim/values.yaml, and env.ts.
+ *
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const CONNECTION_STRING =
+  'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'
+
+const { mockHeadBlobObject, mockGetBlobServiceClient, mockGenerateBlobSASQueryParameters } =
+  vi.hoisted(() => ({
+    mockHeadBlobObject: vi.fn(),
+    mockGetBlobServiceClient: vi.fn(),
+    mockGenerateBlobSASQueryParameters: vi.fn(() => ({ toString: () => 'sig=fake' })),
+  }))
+
+vi.mock('@/lib/uploads/config', () => ({
+  USE_S3_STORAGE: false,
+  USE_BLOB_STORAGE: true,
+  // Connection-string-only: accountName/accountKey intentionally absent.
+  getStorageConfig: () => ({
+    containerName: 'workspace-files',
+    accountName: undefined,
+    accountKey: undefined,
+    connectionString: CONNECTION_STRING,
+  }),
+}))
+
+vi.mock('@/lib/uploads/providers/blob/client', () => ({
+  headBlobObject: mockHeadBlobObject,
+  getBlobServiceClient: mockGetBlobServiceClient,
+  parseConnectionString: (connectionString: string) => {
+    const accountName = connectionString.match(/AccountName=([^;]+)/)?.[1]
+    const accountKey = connectionString.match(/AccountKey=([^;]+)/)?.[1]
+    if (!accountName || !accountKey) throw new Error('cannot parse')
+    return { accountName, accountKey }
+  },
+}))
+
+vi.mock('@azure/storage-blob', () => ({
+  StorageSharedKeyCredential: vi.fn(),
+  BlobSASPermissions: { parse: vi.fn(() => 'w') },
+  generateBlobSASQueryParameters: mockGenerateBlobSASQueryParameters,
+}))
+
+import { generatePresignedUploadUrl, headObject } from '@/lib/uploads/core/storage-service'
+
+describe('Azure Blob storage — connection-string-only auth', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockHeadBlobObject.mockResolvedValue({ size: 42, contentType: 'text/plain' })
+    mockGetBlobServiceClient.mockResolvedValue({
+      getContainerClient: () => ({
+        getBlockBlobClient: () => ({ url: 'https://devstoreaccount1.blob.core.windows.net/c/k' }),
+      }),
+    })
+  })
+
+  it('headObject does not throw when only connectionString is configured', async () => {
+    await expect(headObject('some-key', 'workspace')).resolves.toEqual({
+      size: 42,
+      contentType: 'text/plain',
+    })
+    expect(mockHeadBlobObject).toHaveBeenCalled()
+  })
+
+  it('generatePresignedUploadUrl derives SAS credentials from connectionString when accountName/accountKey are absent', async () => {
+    const result = await generatePresignedUploadUrl({
+      fileName: 'report.csv',
+      contentType: 'text/csv',
+      context: 'workspace',
+      fileSize: 100,
+    })
+
+    expect(mockGenerateBlobSASQueryParameters).toHaveBeenCalled()
+    expect(result.url).toContain('sig=fake')
+  })
+})

apps/sim/lib/uploads/core/storage-service.test.ts

@@ -0,0 +1,102 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockInitiate, mockUploadPart, mockComplete, mockAbort, mockUploadToS3, partBodies } =
+  vi.hoisted(() => ({
+    mockInitiate: vi.fn(),
+    mockUploadPart: vi.fn(),
+    mockComplete: vi.fn(),
+    mockAbort: vi.fn(),
+    mockUploadToS3: vi.fn(),
+    partBodies: [] as Buffer[],
+  }))
+
+vi.mock('@/lib/uploads/config', () => ({
+  USE_S3_STORAGE: true,
+  USE_BLOB_STORAGE: false,
+  getStorageConfig: () => ({ bucket: 'b', region: 'r' }),
+}))
+
+vi.mock('@/lib/uploads/providers/s3/client', () => ({
+  initiateS3MultipartUpload: mockInitiate,
+  uploadS3Part: mockUploadPart,
+  completeS3MultipartUpload: mockComplete,
+  abortS3MultipartUpload: mockAbort,
+  uploadToS3: mockUploadToS3,
+}))
+
+import { createMultipartUpload } from '@/lib/uploads/core/storage-service'
+
+const PART_SIZE = 8 * 1024 * 1024
+
+describe('createMultipartUpload', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    partBodies.length = 0
+    mockInitiate.mockResolvedValue({ uploadId: 'up1', key: 'k' })
+    mockUploadPart.mockImplementation((_key, _uploadId, partNumber: number, body: Buffer) => {
+      partBodies.push(body)
+      return Promise.resolve({ PartNumber: partNumber, ETag: `etag-${partNumber}` })
+    })
+    mockComplete.mockResolvedValue({ location: 'l', path: 'p', key: 'k' })
+    mockAbort.mockResolvedValue(undefined)
+    mockUploadToS3.mockResolvedValue({ key: 'k', path: 'p', name: 'k', size: 0, type: 'text/csv' })
+  })
+
+  it('takes the single-shot PutObject path for a payload smaller than one part', async () => {
+    const handle = await createMultipartUpload({
+      key: 'k',
+      context: 'execution',
+      contentType: 'text/csv',
+    })
+    await handle.write('hello')
+    const result = await handle.complete()
+
+    expect(mockInitiate).not.toHaveBeenCalled()
+    expect(mockUploadPart).not.toHaveBeenCalled()
+    expect(mockUploadToS3).toHaveBeenCalledTimes(1)
+    expect((mockUploadToS3.mock.calls[0][0] as Buffer).toString('utf8')).toBe('hello')
+    expect(result).toEqual({ key: 'k', size: 5 })
+  })
+
+  it('splits into parts and reassembles byte-for-byte over one part boundary', async () => {
+    const a = Buffer.alloc(5 * 1024 * 1024, 1)
+    const b = Buffer.alloc(5 * 1024 * 1024, 2)
+
+    const handle = await createMultipartUpload({
+      key: 'k',
+      context: 'execution',
+      contentType: 'text/csv',
+    })
+    await handle.write(a)
+    await handle.write(b)
+    const result = await handle.complete()
+
+    expect(mockInitiate).toHaveBeenCalledTimes(1)
+    // 10MB → one full 8MB part + a 2MB remainder on complete.
+    expect(mockUploadPart).toHaveBeenCalledTimes(2)
+    expect(partBodies[0].length).toBe(PART_SIZE)
+    const reassembled = Buffer.concat(partBodies)
+    expect(reassembled.length).toBe(10 * 1024 * 1024)
+    expect(reassembled.equals(Buffer.concat([a, b]))).toBe(true)
+    expect(mockComplete).toHaveBeenCalledTimes(1)
+    expect(result.size).toBe(10 * 1024 * 1024)
+    expect(mockUploadToS3).not.toHaveBeenCalled()
+  })
+
+  it('aborts the multipart upload and leaves no object', async () => {
+    const handle = await createMultipartUpload({
+      key: 'k',
+      context: 'execution',
+      contentType: 'text/csv',
+    })
+    await handle.write(Buffer.alloc(9 * 1024 * 1024, 7)) // crosses one part → multipart started
+    await handle.abort()
+
+    expect(mockInitiate).toHaveBeenCalledTimes(1)
+    expect(mockAbort).toHaveBeenCalledTimes(1)
+    expect(mockComplete).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/uploads/core/storage-service.ts

@@ -0,0 +1,780 @@
+import type { Readable } from 'node:stream'
+import { randomBytes } from 'crypto'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits'
+import { getStorageConfig, USE_BLOB_STORAGE, USE_S3_STORAGE } from '@/lib/uploads/config'
+import type { AzureMultipartPart, BlobConfig } from '@/lib/uploads/providers/blob/types'
+import type { S3Config, S3MultipartPart } from '@/lib/uploads/providers/s3/types'
+import type {
+  DeleteFileOptions,
+  DownloadFileOptions,
+  FileInfo,
+  GeneratePresignedUrlOptions,
+  PresignedUrlResponse,
+  StorageConfig,
+  StorageContext,
+  UploadFileOptions,
+} from '@/lib/uploads/shared/types'
+import {
+  sanitizeFileKey,
+  sanitizeFilenameForMetadata,
+  sanitizeStorageMetadata,
+} from '@/lib/uploads/utils/file-utils'
+
+const logger = createLogger('StorageService')
+
+/**
+ * Create a Blob config from StorageConfig
+ * @throws Error if required properties are missing
+ */
+function createBlobConfig(config: StorageConfig): BlobConfig {
+  if (!config.containerName) {
+    throw new Error('Blob configuration missing required property: containerName')
+  }
+
+  if (!config.connectionString && !(config.accountName && config.accountKey)) {
+    throw new Error(
+      'Blob configuration missing authentication: either connectionString or both accountName and accountKey must be provided'
+    )
+  }
+
+  return {
+    containerName: config.containerName,
+    accountName: config.accountName,
+    accountKey: config.accountKey,
+    connectionString: config.connectionString,
+  }
+}
+
+/**
+ * Create an S3 config from StorageConfig
+ * @throws Error if required properties are missing
+ */
+function createS3Config(config: StorageConfig): S3Config {
+  if (!config.bucket || !config.region) {
+    throw new Error('S3 configuration missing required properties: bucket and region')
+  }
+
+  return {
+    bucket: config.bucket,
+    region: config.region,
+  }
+}
+
+/**
+ * Insert file metadata into the database
+ */
+async function insertFileMetadataHelper(
+  key: string,
+  metadata: Record<string, string>,
+  context: StorageContext,
+  fileName: string,
+  contentType: string,
+  fileSize: number
+): Promise<void> {
+  const { insertFileMetadata } = await import('../server/metadata')
+  await insertFileMetadata({
+    key,
+    userId: metadata.userId,
+    workspaceId: metadata.workspaceId || null,
+    folderId: metadata.folderId || null,
+    context,
+    originalName: metadata.originalName || fileName,
+    contentType,
+    size: fileSize,
+  })
+}
+
+/**
+ * Upload a file to the configured storage provider with context-aware configuration
+ */
+export async function uploadFile(options: UploadFileOptions): Promise<FileInfo> {
+  const { file, fileName, contentType, context, preserveKey, customKey, metadata } = options
+
+  logger.info(`Uploading file to ${context} storage: ${fileName}`)
+
+  const config = getStorageConfig(context)
+
+  const keyToUse = customKey || fileName
+
+  if (USE_BLOB_STORAGE) {
+    const { uploadToBlob } = await import('@/lib/uploads/providers/blob/client')
+    const uploadResult = await uploadToBlob(
+      file,
+      keyToUse,
+      contentType,
+      createBlobConfig(config),
+      file.length,
+      preserveKey,
+      metadata
+    )
+
+    if (metadata) {
+      await insertFileMetadataHelper(
+        uploadResult.key,
+        metadata,
+        context,
+        fileName,
+        contentType,
+        file.length
+      )
+    }
+
+    return uploadResult
+  }
+
+  if (USE_S3_STORAGE) {
+    const { uploadToS3 } = await import('@/lib/uploads/providers/s3/client')
+    const uploadResult = await uploadToS3(
+      file,
+      keyToUse,
+      contentType,
+      createS3Config(config),
+      file.length,
+      preserveKey,
+      metadata
+    )
+
+    if (metadata) {
+      await insertFileMetadataHelper(
+        uploadResult.key,
+        metadata,
+        context,
+        fileName,
+        contentType,
+        file.length
+      )
+    }
+
+    return uploadResult
+  }
+
+  const { writeFile, mkdir } = await import('fs/promises')
+  const { join, dirname } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('./setup.server')
+
+  const storageKey = keyToUse
+  const safeKey = sanitizeFileKey(keyToUse) // Validates and preserves path structure
+  const filesystemPath = join(UPLOAD_DIR_SERVER, safeKey)
+
+  await mkdir(dirname(filesystemPath), { recursive: true })
+
+  await writeFile(filesystemPath, file)
+
+  if (metadata) {
+    await insertFileMetadataHelper(
+      storageKey,
+      metadata,
+      context,
+      fileName,
+      contentType,
+      file.length
+    )
+  }
+
+  return {
+    path: `/api/files/serve/${storageKey}`,
+    key: storageKey,
+    name: fileName,
+    size: file.length,
+    type: contentType,
+  }
+}
+
+/** Part size for streaming multipart uploads. ≥ S3's 5MB minimum (all but the last part). */
+const MULTIPART_PART_SIZE = 8 * 1024 * 1024
+/** Max parts uploading concurrently — caps in-flight memory at ~`this × PART_SIZE`. */
+const MULTIPART_MAX_INFLIGHT = 4
+
+/**
+ * Streaming upload sink. The caller `write`s chunks (CSV rows, etc.) and `complete`s;
+ * the implementation buffers into ≥5MB parts and uploads them with bounded concurrency,
+ * so peak memory stays ~`MULTIPART_MAX_INFLIGHT × MULTIPART_PART_SIZE` regardless of total
+ * size. A payload that never crosses one part takes a plain single-shot PutObject.
+ */
+export interface MultipartUploadHandle {
+  write(chunk: Buffer | string): Promise<void>
+  complete(): Promise<{ key: string; size: number }>
+  abort(): Promise<void>
+}
+
+interface MultipartBackend {
+  uploadPart(partNumber: number, body: Buffer): Promise<void>
+  finish(): Promise<void>
+  abort(): Promise<void>
+}
+
+async function createS3Backend(
+  key: string,
+  config: S3Config,
+  contentType: string,
+  purpose: string
+): Promise<MultipartBackend> {
+  const {
+    initiateS3MultipartUpload,
+    uploadS3Part,
+    completeS3MultipartUpload,
+    abortS3MultipartUpload,
+  } = await import('@/lib/uploads/providers/s3/client')
+  const { uploadId } = await initiateS3MultipartUpload({
+    fileName: key,
+    contentType,
+    fileSize: 0,
+    customConfig: config,
+    customKey: key,
+    purpose,
+  })
+  const parts: S3MultipartPart[] = []
+  return {
+    async uploadPart(partNumber, body) {
+      parts.push(await uploadS3Part(key, uploadId, partNumber, body, config))
+    },
+    finish: () => completeS3MultipartUpload(key, uploadId, parts, config).then(() => undefined),
+    abort: () => abortS3MultipartUpload(key, uploadId, config),
+  }
+}
+
+async function createBlobBackend(
+  key: string,
+  config: BlobConfig,
+  contentType: string
+): Promise<MultipartBackend> {
+  const { stageBlobPart, commitBlobBlockList, abortMultipartUpload } = await import(
+    '@/lib/uploads/providers/blob/client'
+  )
+  const parts: AzureMultipartPart[] = []
+  return {
+    async uploadPart(partNumber, body) {
+      parts.push(await stageBlobPart(key, partNumber, body, config))
+    },
+    finish: () => commitBlobBlockList(key, parts, contentType, config),
+    abort: () => abortMultipartUpload(key, config),
+  }
+}
+
+/**
+ * Open a streaming multipart upload to the configured provider. On the local
+ * filesystem provider (and for any payload smaller than one part) the bytes are
+ * buffered and written via a single {@link uploadFile} on `complete`.
+ */
+export async function createMultipartUpload(options: {
+  key: string
+  context: StorageContext
+  contentType: string
+}): Promise<MultipartUploadHandle> {
+  const { key, context, contentType } = options
+  const config = getStorageConfig(context)
+  const cloud = hasCloudStorage()
+
+  let backend: MultipartBackend | null = null
+  // Accumulate writes as references, not a growing buffer — concatenating only when a part fills
+  // (or on complete) keeps total copying ~O(bytes) instead of O(bytes × writes).
+  let pendingChunks: Buffer[] = []
+  let pendingBytes = 0
+  let totalBytes = 0
+  let partNumber = 0
+  let aborted = false
+  let firstError: unknown
+  const inflight = new Set<Promise<void>>()
+
+  /** Merge the accumulated chunks into one ArrayBuffer-backed buffer (which `uploadFile` expects). */
+  const drainPending = (): Buffer<ArrayBuffer> => Buffer.concat(pendingChunks, pendingBytes)
+
+  const ensureBackend = async (): Promise<MultipartBackend> => {
+    if (!backend) {
+      backend = USE_BLOB_STORAGE
+        ? await createBlobBackend(key, createBlobConfig(config), contentType)
+        : await createS3Backend(key, createS3Config(config), contentType, context)
+    }
+    return backend
+  }
+
+  const dispatchPart = async (body: Buffer): Promise<void> => {
+    // Bound concurrency: wait for a free slot before starting another part.
+    while (inflight.size >= MULTIPART_MAX_INFLIGHT) await Promise.race(inflight)
+    if (firstError) throw firstError
+    const be = await ensureBackend()
+    const partNo = ++partNumber
+    const p = be
+      .uploadPart(partNo, body)
+      .catch((err) => {
+        firstError ??= err
+      })
+      .finally(() => {
+        inflight.delete(p)
+      })
+    inflight.add(p)
+  }
+
+  const abort = async (): Promise<void> => {
+    aborted = true
+    await Promise.allSettled(inflight)
+    if (backend) await backend.abort().catch(() => {})
+  }
+
+  return {
+    async write(chunk) {
+      if (aborted) throw new Error('Multipart upload already aborted')
+      if (firstError) throw firstError
+      const buf = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : chunk
+      totalBytes += buf.length
+      pendingChunks.push(buf)
+      pendingBytes += buf.length
+      // Local storage has no multipart concept — accumulate and write once on complete.
+      if (!cloud) return
+      while (pendingBytes >= MULTIPART_PART_SIZE) {
+        const merged = drainPending()
+        const part = merged.subarray(0, MULTIPART_PART_SIZE)
+        const rest = merged.subarray(MULTIPART_PART_SIZE)
+        pendingChunks = rest.length ? [rest] : []
+        pendingBytes = rest.length
+        await dispatchPart(part)
+      }
+    },
+    async complete() {
+      try {
+        if (!backend) {
+          // Never crossed one part (or local provider): single-shot upload.
+          await uploadFile({
+            file: drainPending(),
+            fileName: key,
+            contentType,
+            context,
+            preserveKey: true,
+            customKey: key,
+          })
+          return { key, size: totalBytes }
+        }
+        if (pendingBytes > 0) await dispatchPart(drainPending())
+        await Promise.all(inflight)
+        if (firstError) throw firstError
+        await backend.finish()
+        return { key, size: totalBytes }
+      } catch (err) {
+        await abort()
+        throw err
+      }
+    },
+    abort,
+  }
+}
+
+/**
+ * Download a file from the configured storage provider
+ */
+export async function downloadFile(options: DownloadFileOptions): Promise<Buffer> {
+  const { key, context, maxBytes } = options
+
+  if (context) {
+    const config = getStorageConfig(context)
+
+    if (USE_BLOB_STORAGE) {
+      const { downloadFromBlob } = await import('@/lib/uploads/providers/blob/client')
+      const blobConfig = createBlobConfig(config)
+      return maxBytes === undefined
+        ? downloadFromBlob(key, blobConfig)
+        : downloadFromBlob(key, blobConfig, maxBytes)
+    }
+
+    if (USE_S3_STORAGE) {
+      const { downloadFromS3 } = await import('@/lib/uploads/providers/s3/client')
+      const s3Config = createS3Config(config)
+      return maxBytes === undefined
+        ? downloadFromS3(key, s3Config)
+        : downloadFromS3(key, s3Config, maxBytes)
+    }
+  }
+
+  const { readFile, stat } = await import('fs/promises')
+  const { join } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('./setup.server')
+
+  const safeKey = sanitizeFileKey(key)
+  const filePath = join(UPLOAD_DIR_SERVER, safeKey)
+
+  if (maxBytes !== undefined) {
+    const fileStats = await stat(filePath)
+    assertKnownSizeWithinLimit(fileStats.size, maxBytes, 'storage download')
+  }
+
+  return readFile(filePath)
+}
+
+/**
+ * Stream a file out of the configured storage provider without buffering it in memory.
+ * The caller MUST fully consume or `destroy()` the returned stream. Used by the large-CSV
+ * import worker so a multi-hundred-MB file is never held resident.
+ */
+export async function downloadFileStream(options: {
+  key: string
+  context: StorageContext
+}): Promise<Readable> {
+  const { key, context } = options
+  const config = getStorageConfig(context)
+
+  if (USE_BLOB_STORAGE) {
+    const { downloadFromBlobStream } = await import('@/lib/uploads/providers/blob/client')
+    return downloadFromBlobStream(key, createBlobConfig(config))
+  }
+
+  if (USE_S3_STORAGE) {
+    const { downloadFromS3Stream } = await import('@/lib/uploads/providers/s3/client')
+    return downloadFromS3Stream(key, createS3Config(config))
+  }
+
+  const { createReadStream } = await import('fs')
+  const { join } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('./setup.server')
+  return createReadStream(join(UPLOAD_DIR_SERVER, sanitizeFileKey(key)))
+}
+
+/**
+ * Delete a file from the configured storage provider
+ */
+export async function deleteFile(options: DeleteFileOptions): Promise<void> {
+  const { key, context } = options
+
+  if (context) {
+    const config = getStorageConfig(context)
+
+    if (USE_BLOB_STORAGE) {
+      const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/client')
+      return deleteFromBlob(key, createBlobConfig(config))
+    }
+
+    if (USE_S3_STORAGE) {
+      const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/client')
+      return deleteFromS3(key, createS3Config(config))
+    }
+  }
+
+  const { unlink } = await import('fs/promises')
+  const { join } = await import('path')
+  const { UPLOAD_DIR_SERVER } = await import('./setup.server')
+
+  const safeKey = sanitizeFileKey(key)
+  const filePath = join(UPLOAD_DIR_SERVER, safeKey)
+
+  await unlink(filePath)
+}
+
+/** AWS SDK v3 silently caps HTTP connections at 50/endpoint — stay well under. */
+const PER_FILE_DELETE_CONCURRENCY = 25
+
+/**
+ * Bulk delete via the provider's native multi-object API when available
+ * (S3 `DeleteObjects`), else bounded-concurrency per-file. All keys must
+ * share `context`. Idempotent on missing keys.
+ */
+export async function deleteFiles(
+  keys: string[],
+  context: StorageContext
+): Promise<{ deleted: number; failed: Array<{ key: string; error: string }> }> {
+  if (keys.length === 0) return { deleted: 0, failed: [] }
+
+  const config = getStorageConfig(context)
+
+  if (USE_S3_STORAGE) {
+    const { deleteManyFromS3 } = await import('@/lib/uploads/providers/s3/client')
+    const { failed } = await deleteManyFromS3(keys, createS3Config(config))
+    return { deleted: keys.length - failed.length, failed }
+  }
+
+  const failed: Array<{ key: string; error: string }> = []
+  let cursor = 0
+  const runWorker = async (): Promise<void> => {
+    while (cursor < keys.length) {
+      const idx = cursor++
+      const key = keys[idx]
+      try {
+        await deleteFile({ key, context })
+      } catch (error) {
+        failed.push({ key, error: getErrorMessage(error) })
+      }
+    }
+  }
+
+  const workerCount = Math.min(PER_FILE_DELETE_CONCURRENCY, keys.length)
+  await Promise.all(Array.from({ length: workerCount }, runWorker))
+
+  return { deleted: keys.length - failed.length, failed }
+}
+
+/**
+ * Check whether an object exists in the configured cloud storage provider.
+ * Returns object size and content-type when present, or null when missing.
+ * Throws on errors other than "not found". For local filesystem, returns null.
+ */
+export async function headObject(
+  key: string,
+  context: StorageContext
+): Promise<{ size: number; contentType?: string } | null> {
+  const config = getStorageConfig(context)
+
+  if (USE_BLOB_STORAGE) {
+    const { headBlobObject } = await import('@/lib/uploads/providers/blob/client')
+    return headBlobObject(key, createBlobConfig(config))
+  }
+
+  if (USE_S3_STORAGE) {
+    const { headS3Object } = await import('@/lib/uploads/providers/s3/client')
+    return headS3Object(key, createS3Config(config))
+  }
+
+  return null
+}
+
+/**
+ * Generate a presigned URL for direct file upload
+ */
+export async function generatePresignedUploadUrl(
+  options: GeneratePresignedUrlOptions
+): Promise<PresignedUrlResponse> {
+  const {
+    fileName,
+    contentType,
+    fileSize,
+    context,
+    userId,
+    expirationSeconds = 3600,
+    metadata = {},
+    customKey,
+  } = options
+
+  const allMetadata = {
+    ...metadata,
+    originalName: fileName,
+    uploadedAt: new Date().toISOString(),
+    purpose: context,
+    ...(userId && { userId }),
+  }
+
+  const config = getStorageConfig(context)
+
+  let key: string
+  if (customKey) {
+    key = customKey
+  } else {
+    const timestamp = Date.now()
+    const uniqueId = randomBytes(8).toString('hex')
+    const safeFileName = fileName.replace(/[^a-zA-Z0-9.-]/g, '_')
+    key = `${context}/${timestamp}-${uniqueId}-${safeFileName}`
+  }
+
+  if (USE_S3_STORAGE) {
+    return generateS3PresignedUrl(
+      key,
+      contentType,
+      fileSize,
+      allMetadata,
+      config,
+      expirationSeconds
+    )
+  }
+
+  if (USE_BLOB_STORAGE) {
+    return generateBlobPresignedUrl(key, contentType, allMetadata, config, expirationSeconds)
+  }
+
+  throw new Error('Cloud storage not configured. Cannot generate presigned URL for local storage.')
+}
+
+/**
+ * Generate presigned URL for S3
+ */
+async function generateS3PresignedUrl(
+  key: string,
+  contentType: string,
+  fileSize: number,
+  metadata: Record<string, string>,
+  config: { bucket?: string; region?: string },
+  expirationSeconds: number
+): Promise<PresignedUrlResponse> {
+  const { getS3Client } = await import('@/lib/uploads/providers/s3/client')
+  const { PutObjectCommand } = await import('@aws-sdk/client-s3')
+  const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner')
+
+  if (!config.bucket || !config.region) {
+    throw new Error('S3 configuration missing bucket or region')
+  }
+
+  const sanitizedMetadata = sanitizeStorageMetadata(metadata, 2000)
+  if (sanitizedMetadata.originalName) {
+    sanitizedMetadata.originalName = sanitizeFilenameForMetadata(sanitizedMetadata.originalName)
+  }
+
+  const command = new PutObjectCommand({
+    Bucket: config.bucket,
+    Key: key,
+    ContentType: contentType,
+    ContentLength: fileSize,
+    Metadata: sanitizedMetadata,
+  })
+
+  const presignedUrl = await getSignedUrl(getS3Client(), command, { expiresIn: expirationSeconds })
+
+  return {
+    url: presignedUrl,
+    key,
+  }
+}
+
+/**
+ * Generate presigned URL for Azure Blob
+ */
+async function generateBlobPresignedUrl(
+  key: string,
+  contentType: string,
+  metadata: Record<string, string>,
+  config: {
+    containerName?: string
+    accountName?: string
+    accountKey?: string
+    connectionString?: string
+  },
+  expirationSeconds: number
+): Promise<PresignedUrlResponse> {
+  const { getBlobServiceClient, parseConnectionString } = await import(
+    '@/lib/uploads/providers/blob/client'
+  )
+  const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } =
+    await import('@azure/storage-blob')
+
+  if (!config.containerName) {
+    throw new Error('Blob configuration missing container name')
+  }
+
+  const blobServiceClient = await getBlobServiceClient()
+  const containerClient = blobServiceClient.getContainerClient(config.containerName)
+  const blobClient = containerClient.getBlockBlobClient(key)
+
+  const startsOn = new Date()
+  const expiresOn = new Date(startsOn.getTime() + expirationSeconds * 1000)
+
+  let accountName = config.accountName
+  let accountKey = config.accountKey
+  if ((!accountName || !accountKey) && config.connectionString) {
+    ;({ accountName, accountKey } = parseConnectionString(config.connectionString))
+  }
+
+  if (!accountName || !accountKey) {
+    throw new Error(
+      'Azure Blob SAS generation requires accountName/accountKey or a connectionString'
+    )
+  }
+
+  const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey)
+  const sasToken = generateBlobSASQueryParameters(
+    {
+      containerName: config.containerName,
+      blobName: key,
+      permissions: BlobSASPermissions.parse('w'), // write permission for upload
+      startsOn,
+      expiresOn,
+    },
+    sharedKeyCredential
+  ).toString()
+
+  return {
+    url: `${blobClient.url}?${sasToken}`,
+    key,
+    uploadHeaders: {
+      'x-ms-blob-type': 'BlockBlob',
+      'x-ms-blob-content-type': contentType,
+      ...Object.entries(metadata).reduce(
+        (acc, [k, v]) => {
+          acc[`x-ms-meta-${k}`] = encodeURIComponent(v)
+          return acc
+        },
+        {} as Record<string, string>
+      ),
+    },
+  }
+}
+
+/**
+ * Generate multiple presigned URLs at once (batch operation)
+ */
+export async function generateBatchPresignedUploadUrls(
+  files: Array<{
+    fileName: string
+    contentType: string
+    fileSize: number
+  }>,
+  context: StorageContext,
+  userId?: string,
+  expirationSeconds?: number
+): Promise<PresignedUrlResponse[]> {
+  const results: PresignedUrlResponse[] = []
+
+  for (const file of files) {
+    const result = await generatePresignedUploadUrl({
+      fileName: file.fileName,
+      contentType: file.contentType,
+      fileSize: file.fileSize,
+      context,
+      userId,
+      expirationSeconds,
+    })
+    results.push(result)
+  }
+
+  return results
+}
+
+/**
+ * Generate a presigned URL for downloading/accessing an existing file
+ */
+export async function generatePresignedDownloadUrl(
+  key: string,
+  context: StorageContext,
+  expirationSeconds = 3600
+): Promise<string> {
+  const config = getStorageConfig(context)
+
+  if (USE_S3_STORAGE) {
+    const { getPresignedUrlWithConfig } = await import('@/lib/uploads/providers/s3/client')
+    return getPresignedUrlWithConfig(key, createS3Config(config), expirationSeconds)
+  }
+
+  if (USE_BLOB_STORAGE) {
+    const { getPresignedUrlWithConfig } = await import('@/lib/uploads/providers/blob/client')
+    return getPresignedUrlWithConfig(key, createBlobConfig(config), expirationSeconds)
+  }
+
+  const { getBaseUrl } = await import('@/lib/core/utils/urls')
+  const baseUrl = getBaseUrl()
+  return `${baseUrl}/api/files/serve/${encodeURIComponent(key)}`
+}
+
+/**
+ * Check if cloud storage is available
+ */
+export function hasCloudStorage(): boolean {
+  return USE_BLOB_STORAGE || USE_S3_STORAGE
+}
+
+/**
+ * Get S3 bucket and key information for a storage key
+ * Useful for services that need direct S3 access (e.g., AWS Textract async)
+ */
+export function getS3InfoForKey(
+  key: string,
+  context: StorageContext
+): { bucket: string; key: string } {
+  if (!USE_S3_STORAGE) {
+    throw new Error('S3 storage is not configured. Cannot retrieve S3 info for key.')
+  }
+
+  const config = getStorageConfig(context)
+
+  if (!config.bucket) {
+    throw new Error(`S3 bucket not configured for context: ${context}`)
+  }
+
+  return {
+    bucket: config.bucket,
+    key,
+  }
+}

apps/sim/lib/uploads/core/upload-token.ts

@@ -0,0 +1,96 @@
+import { safeCompare } from '@sim/security/compare'
+import { hmacSha256Base64 } from '@sim/security/hmac'
+import { env } from '@/lib/core/config/env'
+import type { StorageContext } from '@/lib/uploads/shared/types'
+
+export interface UploadTokenPayload {
+  uploadId: string
+  key: string
+  userId: string
+  workspaceId: string
+  context: StorageContext
+  /** Original file name, carried so the completion handler can record ownership metadata. */
+  fileName?: string
+  /** File MIME type, carried for ownership metadata at completion. */
+  contentType?: string
+  /** File size in bytes, carried for ownership metadata at completion. */
+  fileSize?: number
+}
+
+interface SignedPayload extends UploadTokenPayload {
+  exp: number
+  v: 1
+}
+
+const toBase64Url = (input: string): string => Buffer.from(input, 'utf8').toString('base64url')
+
+const fromBase64Url = (input: string): string => Buffer.from(input, 'base64url').toString('utf8')
+
+const sign = (payload: string): string => hmacSha256Base64(payload, env.INTERNAL_API_SECRET)
+
+/**
+ * Sign an upload session token binding (uploadId, key, userId, workspaceId, context).
+ * Used to prevent IDOR on multipart upload follow-up calls (get-part-urls, complete, abort).
+ */
+export function signUploadToken(payload: UploadTokenPayload, expiresInSeconds = 60 * 60): string {
+  const signed: SignedPayload = {
+    ...payload,
+    exp: Math.floor(Date.now() / 1000) + expiresInSeconds,
+    v: 1,
+  }
+  const encoded = toBase64Url(JSON.stringify(signed))
+  return `${encoded}.${sign(encoded)}`
+}
+
+export type UploadTokenVerification =
+  | { valid: true; payload: UploadTokenPayload }
+  | { valid: false }
+
+export function verifyUploadToken(token: string): UploadTokenVerification {
+  if (typeof token !== 'string') {
+    return { valid: false }
+  }
+  const parts = token.split('.')
+  if (parts.length !== 2) return { valid: false }
+  const [encoded, signature] = parts
+  if (!encoded || !signature) return { valid: false }
+
+  const expected = sign(encoded)
+  if (!safeCompare(signature, expected)) {
+    return { valid: false }
+  }
+
+  let parsed: SignedPayload
+  try {
+    parsed = JSON.parse(fromBase64Url(encoded)) as SignedPayload
+  } catch {
+    return { valid: false }
+  }
+
+  if (
+    parsed.v !== 1 ||
+    typeof parsed.exp !== 'number' ||
+    parsed.exp < Math.floor(Date.now() / 1000) ||
+    typeof parsed.uploadId !== 'string' ||
+    typeof parsed.key !== 'string' ||
+    typeof parsed.userId !== 'string' ||
+    typeof parsed.workspaceId !== 'string' ||
+    typeof parsed.context !== 'string'
+  ) {
+    return { valid: false }
+  }
+
+  return {
+    valid: true,
+    payload: {
+      uploadId: parsed.uploadId,
+      key: parsed.key,
+      userId: parsed.userId,
+      workspaceId: parsed.workspaceId,
+      context: parsed.context as StorageContext,
+      ...(typeof parsed.fileName === 'string' ? { fileName: parsed.fileName } : {}),
+      ...(typeof parsed.contentType === 'string' ? { contentType: parsed.contentType } : {}),
+      ...(typeof parsed.fileSize === 'number' ? { fileSize: parsed.fileSize } : {}),
+    },
+  }
+}