simstudioai--sim/apps/sim/lib/execution/payloads/materialization.server.ts

import { createLogger, type Logger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
import {
  getLargeValueMaterializationError,
  isLargeValueRef,
  isLargeValueStorageKey,
  type LargeValueRef,
} from '@/lib/execution/payloads/large-value-ref'
import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors'
import type { StorageContext } from '@/lib/uploads'
import { bufferToBase64, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
import type { UserFile } from '@/executor/types'

const logger = createLogger('ExecutionPayloadMaterialization')

export const MAX_DURABLE_LARGE_VALUE_BYTES = 64 * 1024 * 1024
export const MAX_INLINE_MATERIALIZATION_BYTES = 16 * 1024 * 1024
export const MAX_FUNCTION_FILE_BYTES = 64 * 1024 * 1024
export const MAX_FUNCTION_INLINE_BYTES = 10 * 1024 * 1024

export interface ExecutionMaterializationContext {
  workflowId?: string
  workspaceId?: string
  executionId?: string
  largeValueExecutionIds?: string[]
  largeValueKeys?: string[]
  fileKeys?: string[]
  allowLargeValueWorkflowScope?: boolean
  userId?: string
  requestId?: string
  logger?: Logger
}

export interface MaterializeLargeValueOptions extends ExecutionMaterializationContext {
  maxBytes?: number
}

export interface ReadUserFileContentOptions extends ExecutionMaterializationContext {
  maxBytes?: number
  maxSourceBytes?: number
  offset?: number
  length?: number
  chunked?: boolean
  encoding: 'base64' | 'text'
}

function getLogger(options: ExecutionMaterializationContext): Logger {
  return options.logger ?? logger
}

export function assertDurableLargeValueSize(size: number): void {
  if (size > MAX_DURABLE_LARGE_VALUE_BYTES) {
    throw new ExecutionResourceLimitError({
      resource: 'execution_payload_bytes',
      attemptedBytes: size,
      limitBytes: MAX_DURABLE_LARGE_VALUE_BYTES,
    })
  }
}

export function assertInlineMaterializationSize(size: number, maxBytes?: number): void {
  const limit = maxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES
  if (size > limit) {
    throw new ExecutionResourceLimitError({
      resource: 'execution_payload_bytes',
      attemptedBytes: size,
      limitBytes: limit,
    })
  }
}

export function isValidLargeValueKey(ref: LargeValueRef): boolean {
  return Boolean(ref.key && isLargeValueStorageKey(ref.key, ref.id, ref.executionId))
}

export function assertLargeValueRefAccess(
  ref: LargeValueRef,
  context: ExecutionMaterializationContext
): void {
  if (!context.executionId) {
    throw new Error('Large execution value requires an execution context.')
  }
  const allowedExecutionIds = new Set([
    context.executionId,
    ...(context.largeValueExecutionIds ?? []),
  ])
  const allowedKeys = new Set(context.largeValueKeys ?? [])

  const parts = ref.key?.split('/') ?? []
  const [, workspaceId, workflowId, executionId] = parts

  if (!ref.key) {
    if (ref.executionId && !allowedExecutionIds.has(ref.executionId)) {
      throw new Error('Large execution value is not available in this execution.')
    }
    return
  }
  if (!context.workspaceId || !context.workflowId) {
    throw new Error('Large execution value requires workspace and workflow context.')
  }
  const workflowScopeAllowed =
    context.allowLargeValueWorkflowScope &&
    context.workspaceId === workspaceId &&
    context.workflowId === workflowId
  if (context.workspaceId && workspaceId !== context.workspaceId) {
    throw new Error('Large execution value is not available in this execution.')
  }
  if (context.workflowId && workflowId !== context.workflowId) {
    throw new Error('Large execution value is not available in this execution.')
  }
  if (allowedKeys.has(ref.key)) {
    return
  }
  if (ref.executionId && !allowedExecutionIds.has(ref.executionId) && !workflowScopeAllowed) {
    throw new Error('Large execution value is not available in this execution.')
  }
  if (!allowedExecutionIds.has(executionId) && !workflowScopeAllowed) {
    throw new Error('Large execution value is not available in this execution.')
  }
}

export async function readLargeValueRefFromStorage(
  ref: LargeValueRef,
  options: MaterializeLargeValueOptions = {}
): Promise<unknown | undefined> {
  const log = getLogger(options)
  if (!isLargeValueRef(ref) || !ref.key || !isValidLargeValueKey(ref)) {
    return undefined
  }

  assertLargeValueRefAccess(ref, options)
  assertInlineMaterializationSize(ref.size, options.maxBytes)
  const maxBytes = options.maxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES

  try {
    const { StorageService } = await import('@/lib/uploads')
    const buffer = await StorageService.downloadFile({
      key: ref.key,
      context: 'execution',
      maxBytes,
    })
    if (buffer.length > maxBytes) {
      throw new ExecutionResourceLimitError({
        resource: 'execution_payload_bytes',
        attemptedBytes: buffer.length,
        limitBytes: maxBytes,
      })
    }
    return JSON.parse(buffer.toString('utf8'))
  } catch (error) {
    if (isPayloadSizeLimitError(error)) {
      throw new ExecutionResourceLimitError({
        resource: 'execution_payload_bytes',
        attemptedBytes: error.observedBytes ?? maxBytes + 1,
        limitBytes: maxBytes,
      })
    }
    if (error instanceof ExecutionResourceLimitError) {
      throw error
    }
    log.warn('Failed to materialize persisted large execution value', {
      id: ref.id,
      key: ref.key,
      error: toError(error).message,
    })
    return undefined
  }
}

function normalizeRange(buffer: Buffer, options: ReadUserFileContentOptions): Buffer {
  const offset = Math.max(0, Math.floor(options.offset ?? 0))
  const maxLength = options.maxBytes ?? MAX_FUNCTION_INLINE_BYTES
  const requestedLength = options.length === undefined ? maxLength : Math.floor(options.length)
  const length = Math.max(0, Math.min(requestedLength, maxLength))
  return buffer.subarray(offset, offset + length)
}

function getExecutionKeyParts(key: string):
  | {
      workspaceId: string
      workflowId: string
      executionId: string
    }
  | undefined {
  const parts = key.split('/')
  if (parts[0] !== 'execution' || parts.length < 5) {
    return undefined
  }

  return {
    workspaceId: parts[1],
    workflowId: parts[2],
    executionId: parts[3],
  }
}

function assertExecutionFileScope(key: string, options: ExecutionMaterializationContext): void {
  const parts = getExecutionKeyParts(key)
  if (!parts) {
    throw new Error('File is not available in this execution.')
  }

  const allowedExecutionIds = new Set([
    options.executionId,
    ...(options.largeValueExecutionIds ?? []),
  ])
  const allowedFileKeys = new Set(options.fileKeys ?? [])
  const workflowScopeAllowed =
    options.allowLargeValueWorkflowScope &&
    options.workspaceId === parts.workspaceId &&
    options.workflowId === parts.workflowId

  if (options.workspaceId && parts.workspaceId !== options.workspaceId) {
    throw new Error('File is not available in this execution.')
  }

  if (options.workflowId && parts.workflowId !== options.workflowId) {
    throw new Error('File is not available in this execution.')
  }

  if (allowedFileKeys.has(key)) {
    return
  }

  if (
    !options.executionId ||
    (!allowedExecutionIds.has(parts.executionId) && !workflowScopeAllowed)
  ) {
    throw new Error('File is not available in this execution.')
  }
}

function getVerifiedStorageContext(file: UserFile): StorageContext {
  if (!file.key) {
    throw new Error('File content requires a storage key.')
  }

  const inferredContext = inferContextFromKey(file.key)
  if (file.context && file.context !== inferredContext) {
    throw new Error('File context does not match its storage key.')
  }

  return inferredContext
}

export async function assertUserFileContentAccess(
  file: UserFile,
  options: ExecutionMaterializationContext
): Promise<void> {
  const context = getVerifiedStorageContext(file)

  if (context === 'execution') {
    assertExecutionFileScope(file.key, options)
  }

  if (!options.userId) {
    throw new Error('File access requires an authenticated user.')
  }

  const { verifyFileAccess } = await import('@/app/api/files/authorization')
  const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false)
  if (!hasAccess) {
    throw new Error('File is not available in this execution.')
  }
}

export async function readUserFileContent(
  file: unknown,
  options: ReadUserFileContentOptions
): Promise<string> {
  if (!isUserFileWithMetadata(file)) {
    throw new Error('Expected a file object with metadata.')
  }

  await assertUserFileContentAccess(file, options)

  const maxSourceBytes = options.maxSourceBytes ?? MAX_FUNCTION_FILE_BYTES
  if (Number.isFinite(file.size) && file.size > maxSourceBytes) {
    throw new ExecutionResourceLimitError({
      resource: 'execution_payload_bytes',
      attemptedBytes: file.size,
      limitBytes: maxSourceBytes,
    })
  }

  let buffer: Buffer | null = null
  const log = getLogger(options)
  const requestId = options.requestId ?? 'unknown'

  try {
    buffer = await downloadFileFromStorage(file, requestId, log, { maxBytes: maxSourceBytes })
  } catch (error) {
    if (isPayloadSizeLimitError(error)) {
      throw new ExecutionResourceLimitError({
        resource: 'execution_payload_bytes',
        attemptedBytes: error.observedBytes ?? maxSourceBytes + 1,
        limitBytes: maxSourceBytes,
      })
    }
    throw error
  }

  if (!buffer) {
    throw new Error(`File content for ${file.name} is unavailable.`)
  }
  if (buffer.length > maxSourceBytes) {
    throw new ExecutionResourceLimitError({
      resource: 'execution_payload_bytes',
      attemptedBytes: buffer.length,
      limitBytes: maxSourceBytes,
    })
  }

  const shouldSlice =
    options.chunked || options.offset !== undefined || options.length !== undefined
  const selected = shouldSlice ? normalizeRange(buffer, options) : buffer
  assertInlineMaterializationSize(selected.length, options.maxBytes ?? MAX_FUNCTION_INLINE_BYTES)

  return options.encoding === 'base64' ? bufferToBase64(selected) : selected.toString('utf8')
}

export function unavailableLargeValueError(ref: LargeValueRef): Error {
  return getLargeValueMaterializationError(ref)
}