Files
simstudioai--sim/apps/sim/providers/attachments.ts
T
wehub-resource-sync d25d482dc2
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:20:55 +08:00

739 lines
24 KiB
TypeScript

import type Anthropic from '@anthropic-ai/sdk'
import type { ContentBlock } from '@aws-sdk/client-bedrock-runtime'
import type { Part } from '@google/genai'
import type OpenAI from 'openai'
import {
getContentType,
getExtensionFromMimeType,
getFileExtension,
getMimeTypeFromExtension,
MIME_TYPE_MAPPING,
MODEL_SUPPORTED_IMAGE_MIME_TYPES,
} from '@/lib/uploads/utils/file-utils'
import type { UserFile } from '@/executor/types'
import {
getProviderFileAttachment,
INLINE_ATTACHMENT_MAX_BYTES,
type ProviderFileAttachmentStrategy,
} from '@/providers/models'
import type { ProviderId } from '@/providers/types'
export type AttachmentProvider =
| 'openai'
| 'anthropic'
| 'google'
| 'bedrock'
| 'openrouter'
| 'mistral'
| 'groq'
| 'fireworks'
| 'together'
| 'baseten'
| 'ollama'
| 'vllm'
| 'litellm'
| 'xai'
| 'deepseek'
| 'cerebras'
| 'sakana'
| 'nvidia'
| 'meta'
| 'zai'
export interface PreparedProviderAttachment {
file: UserFile
filename: string
mimeType: string
providerMimeType: string
/** Base64 payload — present only for inlined files (≤ inline threshold). Absent for large uploaded files. */
base64?: string
/** `data:` URL — present only for inlined files. Absent for large uploaded files. */
dataUrl?: string
text?: string
extension: string
contentType: 'image' | 'document' | 'audio' | 'video'
/** Provider Files API id (OpenAI/Anthropic) when the file was uploaded instead of inlined. */
providerFileId?: string
/** Provider File API uri (Gemini) when the file was uploaded instead of inlined. */
providerFileUri?: string
/** Short-lived signed HTTPS URL for providers that fetch attachments by remote URL. */
remoteUrl?: string
}
type ProviderMessageInput = {
role: string
content?: string | null
files?: UserFile[]
}
type ProviderFormattedMessage = {
role: string
content?: string | null | Array<Record<string, unknown>>
files?: UserFile[]
[key: string]: unknown
}
/**
* Files at or below this size are inlined as base64, exactly as before. Larger files take
* the provider's large-file path. Keeping the threshold at the legacy 10 MB cap guarantees
* identical behaviour for existing attachments.
*/
export const INLINE_ATTACHMENT_THRESHOLD_BYTES = INLINE_ATTACHMENT_MAX_BYTES
export type ProviderFileStrategy = ProviderFileAttachmentStrategy
/** Large-file delivery strategy for a provider, sourced from its `models.ts` definition. */
export function getProviderFileStrategy(providerId: ProviderId | string): ProviderFileStrategy {
return getProviderFileAttachment(providerId).strategy
}
/** True when a file exceeds the inline threshold and the provider has a large-file path. */
export function shouldUseLargeFilePath(
file: Pick<UserFile, 'size'>,
providerId: ProviderId | string
): boolean {
if (getProviderFileAttachment(providerId).strategy === 'inline') return false
return Number.isFinite(file.size) && file.size > INLINE_ATTACHMENT_THRESHOLD_BYTES
}
const PDF_MIME_TYPE = 'application/pdf'
const DOCUMENT_MIME_TYPES = new Set(
Object.entries(MIME_TYPE_MAPPING)
.filter(([, contentType]) => contentType === 'document')
.map(([mimeType]) => mimeType)
)
const OPENAI_DOCUMENT_MIME_TYPES = new Set([...DOCUMENT_MIME_TYPES, 'application/x-yaml'])
const GEMINI_INLINE_MIME_TYPES = new Set([...Object.keys(MIME_TYPE_MAPPING), 'application/x-yaml'])
const BEDROCK_DOCUMENT_FORMATS = new Set([
'pdf',
'csv',
'doc',
'docx',
'xls',
'xlsx',
'html',
'txt',
'md',
])
const BEDROCK_IMAGE_FORMATS = new Set(['png', 'jpeg', 'jpg', 'gif', 'webp'])
const BEDROCK_VIDEO_FORMATS = new Set(['mp4', 'mov', 'mkv', 'webm'])
const UNSUPPORTED_FILE_PROVIDERS = new Set<AttachmentProvider>([
'deepseek',
'cerebras',
'sakana',
'nvidia',
'meta',
'zai',
])
const PROVIDER_SUPPORTED_LABELS: Record<AttachmentProvider, string> = {
openai: 'images and documents through the Responses API input_image/input_file parts',
anthropic: 'images, PDFs, and text documents through Claude content blocks',
google: 'images, audio, video, PDFs, and text documents through Gemini inlineData',
bedrock: 'Bedrock Converse image, document, and video content blocks',
openrouter: 'images and PDFs through OpenRouter multimodal message parts',
mistral: 'images through image_url message parts',
groq: 'images through image_url message parts on multimodal models',
fireworks: 'images through image_url message parts on vision models',
together: 'images through image_url message parts on vision models',
baseten: 'images through image_url message parts on vision models',
ollama: 'images through image_url message parts on vision models',
vllm: 'images through image_url message parts on multimodal models',
litellm: 'images through image_url message parts on multimodal models',
xai: 'images through image_url message parts on Grok vision models',
deepseek: 'no file attachments in the current API adapter',
cerebras: 'no file attachments in the current API adapter',
sakana: 'no file attachments in the current API adapter',
nvidia: 'no file attachments in the current API adapter',
meta: 'no file attachments in the current API adapter',
zai: 'no file attachments in the current API adapter',
}
export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null {
if (providerId === 'openai' || providerId === 'azure-openai') return 'openai'
if (providerId === 'anthropic' || providerId === 'azure-anthropic') return 'anthropic'
if (providerId === 'google' || providerId === 'vertex') return 'google'
if (providerId === 'bedrock') return 'bedrock'
if (providerId === 'openrouter') return 'openrouter'
if (providerId === 'mistral') return 'mistral'
if (providerId === 'groq') return 'groq'
if (providerId === 'fireworks') return 'fireworks'
if (providerId === 'together') return 'together'
if (providerId === 'baseten') return 'baseten'
if (providerId === 'ollama' || providerId === 'ollama-cloud') return 'ollama'
if (providerId === 'vllm') return 'vllm'
if (providerId === 'litellm') return 'litellm'
if (providerId === 'xai') return 'xai'
if (providerId === 'deepseek') return 'deepseek'
if (providerId === 'cerebras') return 'cerebras'
if (providerId === 'sakana') return 'sakana'
if (providerId === 'nvidia') return 'nvidia'
if (providerId === 'meta') return 'meta'
if (providerId === 'zai') return 'zai'
return null
}
export function supportsFileAttachments(providerId: ProviderId | string): boolean {
const provider = getAttachmentProvider(providerId)
return Boolean(provider && !UNSUPPORTED_FILE_PROVIDERS.has(provider))
}
/**
* Real maximum attachment size for a provider — its native ceiling when it has a large-file
* path, else the inline base64 threshold. Used for UI limits and validation, never as the
* base64 hydration cap (which stays at {@link INLINE_ATTACHMENT_THRESHOLD_BYTES}).
*/
export function getProviderAttachmentMaxBytes(providerId: ProviderId | string): number {
return getProviderFileAttachment(providerId).maxBytes
}
export function inferAttachmentMimeType(file: UserFile): string {
const explicitType = file.type?.trim().toLowerCase()
if (explicitType && explicitType !== 'application/octet-stream') {
return explicitType
}
const inferred = getMimeTypeFromExtension(getFileExtension(file.name))
return inferred.toLowerCase()
}
function isTextDocumentMimeType(mimeType: string): boolean {
return (
mimeType.startsWith('text/') ||
mimeType === 'application/json' ||
mimeType === 'application/xml' ||
mimeType === 'application/x-yaml'
)
}
function isImageMimeType(mimeType: string): boolean {
return MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType)
}
function isOpenAIDocumentMimeType(mimeType: string): boolean {
return OPENAI_DOCUMENT_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
}
function getAttachmentContentType(
mimeType: string
): PreparedProviderAttachment['contentType'] | null {
return getContentType(mimeType) || (isTextDocumentMimeType(mimeType) ? 'document' : null)
}
function sniffImageMimeType(base64: string): string {
let bytes: Buffer
try {
bytes = Buffer.from(base64, 'base64')
} catch {
return ''
}
if (
bytes.length >= 8 &&
bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))
) {
return 'image/png'
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return 'image/jpeg'
}
if (
bytes.length >= 6 &&
(bytes.subarray(0, 6).equals(Buffer.from('GIF87a')) ||
bytes.subarray(0, 6).equals(Buffer.from('GIF89a')))
) {
return 'image/gif'
}
if (
bytes.length >= 12 &&
bytes.subarray(0, 4).equals(Buffer.from('RIFF')) &&
bytes.subarray(8, 12).equals(Buffer.from('WEBP'))
) {
return 'image/webp'
}
return ''
}
function getAttachmentExtension(file: UserFile, mimeType: string): string {
if (mimeType === 'text/markdown') return 'md'
return getExtensionFromMimeType(mimeType) || getFileExtension(file.name)
}
function normalizeProviderMimeType(mimeType: string, provider: AttachmentProvider): string {
if ((provider === 'anthropic' || provider === 'google') && isTextDocumentMimeType(mimeType)) {
return 'text/plain'
}
return mimeType
}
function decodeBase64Text(base64: string, filename: string): string {
try {
return Buffer.from(base64, 'base64').toString('utf8')
} catch {
throw new Error(`File "${filename}" could not be decoded as UTF-8 text`)
}
}
function toDataUrl(mimeType: string, base64: string): string {
return `data:${mimeType};base64,${base64}`
}
function isMimeTypeSupportedByProvider(
provider: AttachmentProvider,
mimeType: string,
contentType: PreparedProviderAttachment['contentType'],
extension: string
): boolean {
switch (provider) {
case 'openai':
return isImageMimeType(mimeType) || isOpenAIDocumentMimeType(mimeType)
case 'anthropic':
return (
isImageMimeType(mimeType) || mimeType === PDF_MIME_TYPE || isTextDocumentMimeType(mimeType)
)
case 'google':
return GEMINI_INLINE_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
case 'bedrock':
return (
(contentType === 'image' && BEDROCK_IMAGE_FORMATS.has(extension)) ||
(contentType === 'document' && BEDROCK_DOCUMENT_FORMATS.has(extension)) ||
(contentType === 'video' && BEDROCK_VIDEO_FORMATS.has(extension))
)
case 'openrouter':
return isImageMimeType(mimeType) || mimeType === PDF_MIME_TYPE
case 'mistral':
case 'groq':
case 'fireworks':
case 'together':
case 'baseten':
case 'ollama':
case 'vllm':
case 'litellm':
case 'xai':
return isImageMimeType(mimeType)
case 'deepseek':
case 'cerebras':
case 'sakana':
case 'nvidia':
case 'meta':
case 'zai':
return false
default: {
const _exhaustive: never = provider
return _exhaustive
}
}
}
function validateProviderSupport(
attachment: Omit<PreparedProviderAttachment, 'providerMimeType' | 'dataUrl' | 'text'>,
provider: AttachmentProvider,
providerId: ProviderId | string
) {
const { filename, mimeType, contentType, extension } = attachment
const supportedLabel = PROVIDER_SUPPORTED_LABELS[provider]
const supported = isMimeTypeSupportedByProvider(provider, mimeType, contentType, extension)
if (!supported) {
throw new Error(
`File "${filename}" has MIME type "${mimeType}", which is not supported by provider "${providerId}". Supported attachments: ${supportedLabel}.`
)
}
}
export function prepareProviderAttachments(
files: UserFile[] | undefined,
providerId: ProviderId | string
): PreparedProviderAttachment[] {
if (!files || files.length === 0) return []
const provider = getAttachmentProvider(providerId)
if (!provider) {
throw new Error(`File attachments are not supported for provider "${providerId}"`)
}
if (UNSUPPORTED_FILE_PROVIDERS.has(provider)) {
throw new Error(
`File attachments are not supported for provider "${providerId}" in the current adapter. Supported attachments: ${PROVIDER_SUPPORTED_LABELS[provider]}.`
)
}
return files.map((file) => {
const declaredMimeType = inferAttachmentMimeType(file)
const contentType = getAttachmentContentType(declaredMimeType)
if (!contentType) {
throw new Error(
`File "${file.name}" has MIME type "${declaredMimeType}", which is not supported by provider "${providerId}". Supported attachments: ${PROVIDER_SUPPORTED_LABELS[provider]}.`
)
}
const maxBytes = getProviderAttachmentMaxBytes(providerId)
if (Number.isFinite(file.size) && file.size > maxBytes) {
const sizeMB = (file.size / (1024 * 1024)).toFixed(2)
const maxMB = (maxBytes / (1024 * 1024)).toFixed(0)
throw new Error(
`File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"`
)
}
const providerFileId = file.providerFileId
const providerFileUri = file.providerFileUri
const remoteUrl = file.remoteUrl
const hasHandle = Boolean(providerFileId || providerFileUri || remoteUrl)
if (!file.base64 && !hasHandle) {
throw new Error(`File "${file.name}" could not be read for provider "${providerId}"`)
}
const sniffedImageMimeType =
contentType === 'image' && file.base64 ? sniffImageMimeType(file.base64) : ''
if (contentType === 'image' && file.base64 && !sniffedImageMimeType) {
throw new Error(
`Image bytes in "${file.name}" are not a supported model image format (declared MIME type "${declaredMimeType}"). Supported image formats: image/jpeg, image/png, image/gif, image/webp.`
)
}
const mimeType = sniffedImageMimeType || declaredMimeType
const extension = getAttachmentExtension(file, mimeType)
const attachment = {
file,
filename: file.name,
mimeType,
base64: file.base64,
extension,
contentType,
}
validateProviderSupport(attachment, provider, providerId)
const providerMimeType = normalizeProviderMimeType(mimeType, provider)
return {
...attachment,
providerMimeType,
providerFileId,
providerFileUri,
remoteUrl,
...(file.base64 && { dataUrl: toDataUrl(providerMimeType, file.base64) }),
...(file.base64 &&
isTextDocumentMimeType(mimeType) && {
text: decodeBase64Text(file.base64, file.name),
}),
}
})
}
type OpenAIResponsesInputContent = OpenAI.Responses.ResponseInputContent
type OpenAIChatContentPart = OpenAI.Chat.Completions.ChatCompletionContentPart
type AnthropicImageMediaType = Anthropic.Messages.Base64ImageSource['media_type']
export function buildOpenAIMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | OpenAIResponsesInputContent[] {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: OpenAIResponsesInputContent[] = []
if (content) {
parts.push({ type: 'input_text', text: content } satisfies OpenAI.Responses.ResponseInputText)
}
for (const attachment of attachments) {
if (attachment.contentType === 'image') {
parts.push(
attachment.providerFileId
? ({
type: 'input_image',
file_id: attachment.providerFileId,
detail: 'auto',
} satisfies OpenAI.Responses.ResponseInputImage)
: ({
type: 'input_image',
image_url: attachment.dataUrl,
detail: 'auto',
} satisfies OpenAI.Responses.ResponseInputImage)
)
} else {
parts.push(
attachment.providerFileId
? ({
type: 'input_file',
file_id: attachment.providerFileId,
} satisfies OpenAI.Responses.ResponseInputFile)
: ({
type: 'input_file',
filename: attachment.filename,
file_data: attachment.dataUrl,
} satisfies OpenAI.Responses.ResponseInputFile)
)
}
}
return parts
}
export function buildAnthropicMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): Anthropic.Messages.ContentBlockParam[] {
const parts: Anthropic.Messages.ContentBlockParam[] = []
if (content) {
parts.push({ type: 'text', text: content } satisfies Anthropic.Messages.TextBlockParam)
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
if (attachment.contentType === 'image') {
parts.push({
type: 'image',
source: attachment.remoteUrl
? ({ type: 'url', url: attachment.remoteUrl } satisfies Anthropic.Messages.URLImageSource)
: ({
type: 'base64',
media_type: attachment.providerMimeType as AnthropicImageMediaType,
data: attachment.base64 ?? '',
} satisfies Anthropic.Messages.Base64ImageSource),
} satisfies Anthropic.Messages.ImageBlockParam)
} else if (attachment.remoteUrl) {
if (attachment.mimeType !== PDF_MIME_TYPE) {
throw new Error(
`Document "${attachment.filename}" (${attachment.mimeType}) is too large to send to provider "${providerId}". Only PDFs and images are supported above the inline limit — convert it to PDF or reduce its size.`
)
}
parts.push({
type: 'document',
source: { type: 'url', url: attachment.remoteUrl },
title: attachment.filename,
} satisfies Anthropic.Messages.DocumentBlockParam)
} else if (attachment.text) {
parts.push({
type: 'document',
source: {
type: 'text',
media_type: 'text/plain',
data: attachment.text,
},
title: attachment.filename,
} satisfies Anthropic.Messages.DocumentBlockParam)
} else {
parts.push({
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: attachment.base64 ?? '',
},
title: attachment.filename,
} satisfies Anthropic.Messages.DocumentBlockParam)
}
}
return parts
}
export function buildGeminiMessageParts(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): Part[] {
const parts: Part[] = []
if (content) {
parts.push({ text: content } satisfies Part)
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
parts.push(
attachment.providerFileUri
? ({
fileData: {
fileUri: attachment.providerFileUri,
mimeType: attachment.providerMimeType,
},
} satisfies Part)
: ({
inlineData: {
mimeType: attachment.providerMimeType,
data: attachment.base64 ?? '',
},
} satisfies Part)
)
}
return parts
}
export function buildOpenAICompatibleChatContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | OpenAIChatContentPart[] {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: OpenAIChatContentPart[] = []
if (content) {
parts.push({
type: 'text',
text: content,
} satisfies OpenAI.Chat.Completions.ChatCompletionContentPartText)
}
for (const attachment of attachments) {
parts.push({
type: 'image_url',
image_url: {
url: attachment.remoteUrl ?? attachment.dataUrl ?? '',
},
} satisfies OpenAI.Chat.Completions.ChatCompletionContentPartImage)
}
return parts
}
export function buildOpenRouterMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | OpenAIChatContentPart[] {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: OpenAIChatContentPart[] = []
if (content) {
parts.push({
type: 'text',
text: content,
} satisfies OpenAI.Chat.Completions.ChatCompletionContentPartText)
}
for (const attachment of attachments) {
if (attachment.contentType === 'image') {
parts.push({
type: 'image_url',
image_url: { url: attachment.remoteUrl ?? attachment.dataUrl ?? '' },
} satisfies OpenAI.Chat.Completions.ChatCompletionContentPartImage)
} else {
parts.push({
type: 'file',
file: {
filename: attachment.filename,
file_data: attachment.remoteUrl ?? attachment.dataUrl ?? '',
},
} satisfies OpenAI.Chat.Completions.ChatCompletionContentPart.File)
}
}
return parts
}
function sanitizeBedrockName(filename: string): string {
const baseName = filename.replace(/\.[^/.]+$/, '').replace(/[^a-zA-Z0-9\s()[\]-]/g, ' ')
const compacted = baseName.replace(/\s+/g, ' ').trim()
return compacted || 'Document'
}
function getBedrockDocumentFormat(attachment: PreparedProviderAttachment): string {
if (attachment.extension === 'md' || attachment.mimeType === 'text/markdown') return 'md'
if (attachment.extension === 'txt' || attachment.mimeType === 'text/plain') return 'txt'
return attachment.extension || 'txt'
}
function getBedrockImageFormat(attachment: PreparedProviderAttachment): string {
return attachment.extension === 'jpg' ? 'jpeg' : attachment.extension
}
export function buildBedrockMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): ContentBlock[] {
const parts: ContentBlock[] = []
if (content) {
parts.push({ text: content } as ContentBlock.TextMember)
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
const bytes = Buffer.from(attachment.base64 ?? '', 'base64')
if (attachment.contentType === 'image') {
parts.push({
image: {
format: getBedrockImageFormat(attachment) as ContentBlock.ImageMember['image']['format'],
source: { bytes },
},
} as ContentBlock.ImageMember)
} else if (attachment.contentType === 'video') {
parts.push({
video: {
format: attachment.extension as ContentBlock.VideoMember['video']['format'],
source: { bytes },
},
} as ContentBlock.VideoMember)
} else {
parts.push({
document: {
format: getBedrockDocumentFormat(
attachment
) as ContentBlock.DocumentMember['document']['format'],
name: sanitizeBedrockName(attachment.filename),
source: { bytes },
},
} as ContentBlock.DocumentMember)
}
}
return parts
}
const SDK_NATIVE_ATTACHMENT_PROVIDERS = new Set<AttachmentProvider>([
'openai',
'anthropic',
'google',
'bedrock',
])
export function formatMessagesForProvider(
messages: ProviderMessageInput[],
providerId: ProviderId | string
): ProviderFormattedMessage[] {
const provider = getAttachmentProvider(providerId)
if (provider && SDK_NATIVE_ATTACHMENT_PROVIDERS.has(provider)) {
return messages as ProviderFormattedMessage[]
}
return messages.map((message) => {
if (!message.files?.length || (message.role !== 'user' && message.role !== 'assistant')) {
return message as ProviderFormattedMessage
}
if (provider === 'openrouter') {
const { files: _omit, ...rest } = message
return {
...rest,
content: buildOpenRouterMessageContent(message.content, message.files, providerId) as
| string
| Array<Record<string, unknown>>,
}
}
const { files: _omit, ...rest } = message
return {
...rest,
content: buildOpenAICompatibleChatContent(message.content, message.files, providerId) as
| string
| Array<Record<string, unknown>>,
}
})
}