Files
simstudioai--sim/apps/sim/app/api/tools/image/route.ts
T
wehub-resource-sync d25d482dc2
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:20:55 +08:00

1046 lines
33 KiB
TypeScript

import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { isRecordLike } from '@sim/utils/object'
import { type NextRequest, NextResponse } from 'next/server'
import {
type ImageToolBody,
type imageProviders,
imageProxyQuerySchema,
imageToolContract,
} from '@/lib/api/contracts/tools/media/image'
import {
getValidationErrorMessage,
parseRequest,
searchParamsToObject,
validationErrorResponse,
} from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
import {
secureFetchWithPinnedIP,
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import {
assertKnownSizeWithinLimit,
consumeOrCancelBody,
DEFAULT_MAX_ERROR_BODY_BYTES,
isPayloadSizeLimitError,
readResponseJsonWithLimit,
readResponseTextWithLimit,
readResponseToBufferWithLimit,
} from '@/lib/core/utils/stream-limits'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing'
const logger = createLogger('ImageProxyAPI')
const MAX_IMAGE_BYTES = 25 * 1024 * 1024
const MAX_IMAGE_JSON_BYTES = Math.ceil((MAX_IMAGE_BYTES * 4) / 3) + 256 * 1024
export const dynamic = 'force-dynamic'
/**
* Mirrors the maximum plan execution timeout (enterprise async, 90 minutes) used by
* `getMaxExecutionTimeout()` for the provider polling loop below. Next.js requires a
* static literal for `maxDuration`, so this value must be kept in sync with that source.
*/
export const maxDuration = 5400
type ImageProvider = (typeof imageProviders)[number]
interface GeneratedImageResult {
buffer: Buffer
contentType: string
fileName: string
provider: ImageProvider
model: string
sourceUrl?: string
description?: string
revisedPrompt?: string
seed?: number
jobId?: string
falaiCost?: FalAICostMetadata
}
interface StoredImageResponse {
content: string
imageUrl: string
imageFile?: unknown
fileName: string
contentType: string
provider: ImageProvider
model: string
metadata: {
provider: ImageProvider
model: string
description?: string
revisedPrompt?: string
seed?: number
jobId?: string
contentType: string
}
__falaiCostDollars?: number
__falaiBilling?: FalAICostMetadata
}
export const POST = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
logger.info(`[${requestId}] Image generation request started`)
try {
const authResult = await checkInternalAuth(request, { requireWorkflowId: false })
if (!authResult.success || !authResult.userId) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}
const parsed = await parseRequest(
imageToolContract,
request,
{},
{
validationErrorResponse: (error) => {
logger.warn(`[${requestId}] Invalid image generation request:`, error.issues)
return validationErrorResponse(
error,
getValidationErrorMessage(error, 'Invalid request data')
)
},
}
)
if (!parsed.success) return parsed.response
const body = parsed.data.body
const provider = body.provider as ImageProvider
const { apiKey, model, prompt } = body
if (prompt.length < 3 || prompt.length > 4000) {
return NextResponse.json(
{ error: 'Prompt must be between 3 and 4000 characters' },
{ status: 400 }
)
}
logger.info(`[${requestId}] Generating image with ${provider}, model: ${model || 'default'}`)
let imageResult: GeneratedImageResult
try {
if (provider === 'openai') {
imageResult = await generateWithOpenAI(apiKey, body, requestId, logger)
} else if (provider === 'gemini') {
imageResult = await generateWithGemini(apiKey, body, requestId, logger)
} else if (provider === 'falai') {
imageResult = await generateWithFalAI(apiKey, body, requestId, logger)
} else {
return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
}
} catch (error) {
logger.error(`[${requestId}] Image generation failed:`, error)
const errorMessage = getErrorMessage(error, 'Image generation failed')
return NextResponse.json(
{ error: errorMessage },
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
const storedImage = await storeGeneratedImage(imageResult, body, authResult.userId, requestId)
logger.info(`[${requestId}] Image generation completed successfully`, {
provider,
model: storedImage.model,
contentType: storedImage.contentType,
})
return NextResponse.json(storedImage)
} catch (error) {
logger.error(`[${requestId}] Image generation route error:`, error)
const errorMessage = getErrorMessage(error, 'Unknown error')
return NextResponse.json(
{ error: errorMessage },
{ status: isPayloadSizeLimitError(error) ? 413 : 500 }
)
}
})
/**
* Proxy for fetching images
* This allows client-side requests to fetch images from various sources while avoiding CORS issues
*/
export const GET = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
const authResult = await checkInternalAuth(request, { requireWorkflowId: false })
if (!authResult.success) {
logger.error(`[${requestId}] Authentication failed for image proxy:`, authResult.error)
return new NextResponse('Unauthorized', { status: 401 })
}
const queryResult = imageProxyQuerySchema.safeParse(
searchParamsToObject(request.nextUrl.searchParams)
)
if (!queryResult.success) {
const error = getValidationErrorMessage(queryResult.error, 'Missing URL parameter')
logger.error(`[${requestId}] ${error}`)
return new NextResponse(error, { status: 400 })
}
const { url: imageUrl } = queryResult.data
const urlValidation = await validateUrlWithDNS(imageUrl, 'imageUrl')
if (!urlValidation.isValid) {
logger.warn(`[${requestId}] Blocked image proxy request`, {
url: imageUrl.substring(0, 100),
error: urlValidation.error,
})
return new NextResponse(urlValidation.error || 'Invalid image URL', { status: 403 })
}
logger.info(`[${requestId}] Proxying image request for: ${imageUrl}`)
try {
const imageResponse = await secureFetchWithPinnedIP(imageUrl, urlValidation.resolvedIP!, {
method: 'GET',
maxResponseBytes: MAX_IMAGE_BYTES,
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
Accept: 'image/webp,image/avif,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
Referer: 'https://sim.ai/',
'Sec-Fetch-Dest': 'image',
'Sec-Fetch-Mode': 'no-cors',
'Sec-Fetch-Site': 'cross-site',
},
})
if (!imageResponse.ok) {
await consumeOrCancelBody(imageResponse)
logger.error(`[${requestId}] Image fetch failed:`, {
status: imageResponse.status,
statusText: imageResponse.statusText,
})
return new NextResponse(`Failed to fetch image: ${imageResponse.statusText}`, {
status: imageResponse.status,
})
}
const contentType = imageResponse.headers.get('content-type') || 'image/jpeg'
const imageBuffer = await readResponseToBufferWithLimit(imageResponse, {
maxBytes: MAX_IMAGE_BYTES,
label: 'image proxy response',
})
if (imageBuffer.length === 0) {
logger.error(`[${requestId}] Empty image received`)
return new NextResponse('Empty image received', { status: 404 })
}
return new NextResponse(new Uint8Array(imageBuffer), {
headers: {
'Content-Type': contentType,
'Access-Control-Allow-Origin': '*',
'Cache-Control': 'public, max-age=86400', // Cache for 24 hours
},
})
} catch (error) {
const errorMessage = toError(error).message
logger.error(`[${requestId}] Image proxy error:`, { error: errorMessage })
return new NextResponse(`Failed to proxy image: ${errorMessage}`, {
status: isPayloadSizeLimitError(error) ? 413 : 500,
})
}
})
const OPENAI_IMAGE_MODELS = [
'gpt-image-2',
'gpt-image-1.5',
'gpt-image-1',
'gpt-image-1-mini',
] as const
const OPENAI_IMAGE_SIZES = ['auto', '1024x1024', '1536x1024', '1024x1536'] as const
const OPENAI_IMAGE_2_SIZES = [...OPENAI_IMAGE_SIZES, '2560x1440', '3840x2160'] as const
const OPENAI_IMAGE_QUALITIES = ['auto', 'low', 'medium', 'high'] as const
const OPENAI_IMAGE_BACKGROUNDS = ['auto', 'transparent', 'opaque'] as const
const IMAGE_OUTPUT_FORMATS = ['png', 'jpeg', 'webp'] as const
const OPENAI_MODERATION_LEVELS = ['auto', 'low'] as const
const GEMINI_IMAGE_MODELS = [
'gemini-3.1-flash-image-preview',
'gemini-3-pro-image-preview',
'gemini-2.5-flash-image',
] as const
const GEMINI_BASE_ASPECT_RATIOS = [
'1:1',
'2:3',
'3:2',
'3:4',
'4:3',
'4:5',
'5:4',
'9:16',
'16:9',
'21:9',
] as const
const GEMINI_EXTREME_ASPECT_RATIOS = ['1:4', '1:8', '4:1', '8:1'] as const
const GEMINI_IMAGE_SIZES = ['512', '1K', '2K', '4K'] as const
const GEMINI_PRO_IMAGE_SIZES = ['1K', '2K', '4K'] as const
interface FalAIImageModelConfig {
endpoint: string
defaultSize?: string
sizeOptions?: readonly string[]
defaultAspectRatio?: string
aspectRatios?: readonly string[]
defaultResolution?: string
resolutionOptions?: readonly string[]
defaultOutputFormat?: string
outputFormats?: readonly string[]
defaultQuality?: string
qualityOptions?: readonly string[]
defaultBackground?: string
backgroundOptions?: readonly string[]
defaultSafetyTolerance?: string
safetyToleranceOptions?: readonly string[]
maxNumImages?: number
supportsSeed?: boolean
supportsEnableSafetyChecker?: boolean
supportsEnableWebSearch?: boolean
supportsThinkingLevel?: boolean
}
const FALAI_NANO_BANANA_ASPECT_RATIOS = [
'auto',
'21:9',
'16:9',
'3:2',
'4:3',
'5:4',
'1:1',
'4:5',
'3:4',
'2:3',
'9:16',
] as const
const FALAI_EXTREME_ASPECT_RATIOS = ['4:1', '1:4', '8:1', '1:8'] as const
const FALAI_STANDARD_IMAGE_SIZES = [
'square_hd',
'square',
'portrait_4_3',
'portrait_16_9',
'landscape_4_3',
'landscape_16_9',
] as const
const FALAI_SEEDREAM_IMAGE_SIZES = [...FALAI_STANDARD_IMAGE_SIZES, 'auto_2K', 'auto_4K'] as const
const FALAI_IMAGE_MODEL_CONFIGS: Record<string, FalAIImageModelConfig> = {
'nano-banana-2': {
endpoint: 'fal-ai/nano-banana-2',
defaultAspectRatio: 'auto',
aspectRatios: [...FALAI_NANO_BANANA_ASPECT_RATIOS, ...FALAI_EXTREME_ASPECT_RATIOS],
defaultResolution: '1K',
resolutionOptions: ['0.5K', '1K', '2K', '4K'],
defaultOutputFormat: 'png',
outputFormats: IMAGE_OUTPUT_FORMATS,
defaultSafetyTolerance: '4',
safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'],
maxNumImages: 4,
supportsSeed: true,
supportsEnableWebSearch: true,
supportsThinkingLevel: true,
},
'nano-banana-pro': {
endpoint: 'fal-ai/nano-banana-pro',
defaultAspectRatio: '1:1',
aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS,
defaultResolution: '1K',
resolutionOptions: ['1K', '2K', '4K'],
defaultOutputFormat: 'png',
outputFormats: IMAGE_OUTPUT_FORMATS,
defaultSafetyTolerance: '4',
safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'],
maxNumImages: 4,
supportsSeed: true,
supportsEnableWebSearch: true,
},
'nano-banana': {
endpoint: 'fal-ai/nano-banana',
defaultAspectRatio: '1:1',
aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS.filter((ratio) => ratio !== 'auto'),
defaultOutputFormat: 'png',
outputFormats: IMAGE_OUTPUT_FORMATS,
defaultSafetyTolerance: '4',
safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'],
maxNumImages: 4,
supportsSeed: true,
},
'gpt-image-1.5': {
endpoint: 'fal-ai/gpt-image-1.5',
defaultSize: '1024x1024',
sizeOptions: ['1024x1024', '1536x1024', '1024x1536'],
defaultQuality: 'high',
qualityOptions: ['low', 'medium', 'high'],
defaultBackground: 'auto',
backgroundOptions: OPENAI_IMAGE_BACKGROUNDS,
defaultOutputFormat: 'png',
outputFormats: IMAGE_OUTPUT_FORMATS,
maxNumImages: 4,
},
'seedream-v4.5': {
endpoint: 'fal-ai/bytedance/seedream/v4.5/text-to-image',
defaultSize: 'auto_2K',
sizeOptions: FALAI_SEEDREAM_IMAGE_SIZES,
maxNumImages: 6,
supportsSeed: true,
supportsEnableSafetyChecker: true,
},
'flux-2-pro': {
endpoint: 'fal-ai/flux-2-pro',
defaultSize: 'landscape_4_3',
sizeOptions: FALAI_STANDARD_IMAGE_SIZES,
defaultOutputFormat: 'jpeg',
outputFormats: ['jpeg', 'png'],
defaultSafetyTolerance: '2',
safetyToleranceOptions: ['1', '2', '3', '4', '5'],
supportsSeed: true,
supportsEnableSafetyChecker: true,
},
'grok-imagine-image': {
endpoint: 'xai/grok-imagine-image',
defaultAspectRatio: '1:1',
aspectRatios: [
'2:1',
'20:9',
'19.5:9',
'16:9',
'4:3',
'3:2',
'1:1',
'2:3',
'3:4',
'9:16',
'9:19.5',
'9:20',
'1:2',
],
defaultResolution: '1k',
resolutionOptions: ['1k', '2k'],
defaultOutputFormat: 'jpeg',
outputFormats: IMAGE_OUTPUT_FORMATS,
maxNumImages: 4,
},
}
function getStringProperty(
record: Record<string, unknown> | undefined,
key: string
): string | undefined {
const value = record?.[key]
return typeof value === 'string' ? value : undefined
}
function getNumberProperty(
record: Record<string, unknown> | undefined,
key: string
): number | undefined {
const value = record?.[key]
return typeof value === 'number' ? value : undefined
}
function firstRecord(value: unknown): Record<string, unknown> | undefined {
return Array.isArray(value) ? value.find(isRecordLike) : undefined
}
function pickAllowed(
value: string | undefined,
allowed: readonly string[],
fallback: string
): string {
return value && allowed.includes(value) ? value : fallback
}
function clampInteger(
value: number | undefined,
min: number,
max: number,
fallback: number
): number {
if (typeof value !== 'number' || !Number.isInteger(value)) return fallback
return Math.min(Math.max(value, min), max)
}
function getContentTypeForFormat(format: string | undefined): string {
if (format === 'jpeg') return 'image/jpeg'
if (format === 'webp') return 'image/webp'
return 'image/png'
}
function extensionFromContentType(contentType: string): string {
if (contentType.includes('jpeg') || contentType.includes('jpg')) return 'jpg'
if (contentType.includes('webp')) return 'webp'
return 'png'
}
async function bufferFromImageUrl(url: string): Promise<{ buffer: Buffer; contentType: string }> {
if (url.startsWith('data:')) {
const match = /^data:([^;]+);base64,(.+)$/u.exec(url)
if (!match) throw new Error('Invalid data URI image response')
const buffer = Buffer.from(match[2], 'base64')
assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'inline image response')
return {
contentType: match[1],
buffer,
}
}
const urlValidation = await validateUrlWithDNS(url, 'imageUrl')
if (!urlValidation.isValid || !urlValidation.resolvedIP) {
throw new Error(urlValidation.error || 'Generated image URL failed validation')
}
const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, {
method: 'GET',
maxResponseBytes: MAX_IMAGE_BYTES,
})
if (!imageResponse.ok) {
await readResponseTextWithLimit(imageResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'generated image error response',
}).catch(() => '')
throw new Error(`Failed to download generated image: ${imageResponse.status}`)
}
const contentType = imageResponse.headers.get('content-type') || 'image/png'
const buffer = await readResponseToBufferWithLimit(imageResponse, {
maxBytes: MAX_IMAGE_BYTES,
label: 'generated image download',
})
return { buffer, contentType }
}
async function generateWithOpenAI(
apiKey: string,
body: ImageToolBody,
requestId: string,
logger: ReturnType<typeof createLogger>
): Promise<GeneratedImageResult> {
const model = pickAllowed(body.model, OPENAI_IMAGE_MODELS, 'gpt-image-1.5')
const size =
model === 'gpt-image-2'
? pickAllowed(body.size, OPENAI_IMAGE_2_SIZES, 'auto')
: pickAllowed(body.size, OPENAI_IMAGE_SIZES, 'auto')
const outputFormat = pickAllowed(body.outputFormat, IMAGE_OUTPUT_FORMATS, 'png')
const requestBody: Record<string, string | number> = {
model,
prompt: body.prompt,
size,
n: 1,
}
if (body.quality) {
requestBody.quality = pickAllowed(body.quality, OPENAI_IMAGE_QUALITIES, 'auto')
}
if (body.background) {
requestBody.background = pickAllowed(body.background, OPENAI_IMAGE_BACKGROUNDS, 'auto')
}
if (body.outputFormat) {
requestBody.output_format = outputFormat
}
if (body.moderation) {
requestBody.moderation = pickAllowed(body.moderation, OPENAI_MODERATION_LEVELS, 'auto')
}
const openaiResponse = await fetch('https://api.openai.com/v1/images/generations', {
method: 'POST',
headers: {
Authorization: `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
})
if (!openaiResponse.ok) {
const error = await readResponseTextWithLimit(openaiResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'OpenAI image error response',
})
throw new Error(`OpenAI API error: ${openaiResponse.status} - ${error}`)
}
const data = await readResponseJsonWithLimit(openaiResponse, {
maxBytes: MAX_IMAGE_JSON_BYTES,
label: 'OpenAI image response',
})
if (!isRecordLike(data)) {
throw new Error('Invalid OpenAI image response')
}
const firstImage = firstRecord(data.data)
const base64Image = getStringProperty(firstImage, 'b64_json')
const imageUrl = getStringProperty(firstImage, 'url')
const revisedPrompt = getStringProperty(firstImage, 'revised_prompt')
let buffer: Buffer
let contentType = getContentTypeForFormat(outputFormat)
if (base64Image) {
buffer = Buffer.from(base64Image, 'base64')
assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'OpenAI image response')
} else if (imageUrl) {
const downloaded = await bufferFromImageUrl(imageUrl)
buffer = downloaded.buffer
contentType = downloaded.contentType
} else {
logger.error(`[${requestId}] OpenAI response missing image payload`)
throw new Error('No image data found in OpenAI response')
}
return {
buffer,
contentType,
fileName: `openai-${model}.${extensionFromContentType(contentType)}`,
provider: 'openai',
model,
sourceUrl: imageUrl,
revisedPrompt,
}
}
async function generateWithGemini(
apiKey: string,
body: ImageToolBody,
requestId: string,
logger: ReturnType<typeof createLogger>
): Promise<GeneratedImageResult> {
const model = pickAllowed(body.model, GEMINI_IMAGE_MODELS, 'gemini-3.1-flash-image-preview')
const aspectRatios =
model === 'gemini-3.1-flash-image-preview'
? [...GEMINI_BASE_ASPECT_RATIOS, ...GEMINI_EXTREME_ASPECT_RATIOS]
: GEMINI_BASE_ASPECT_RATIOS
const imageConfig: Record<string, string> = {}
if (body.aspectRatio) {
imageConfig.aspectRatio = pickAllowed(body.aspectRatio, aspectRatios, '1:1')
}
if (model === 'gemini-3.1-flash-image-preview' && body.resolution) {
imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_IMAGE_SIZES, '1K')
} else if (model === 'gemini-3-pro-image-preview' && body.resolution) {
imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_PRO_IMAGE_SIZES, '1K')
}
const requestBody: Record<string, unknown> = {
contents: [
{
parts: [{ text: body.prompt }],
},
],
}
requestBody.generationConfig = {
responseModalities: ['TEXT', 'IMAGE'],
...(Object.keys(imageConfig).length > 0 && { imageConfig }),
}
const geminiResponse = await fetch(
`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
{
method: 'POST',
headers: {
'x-goog-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
}
)
if (!geminiResponse.ok) {
const error = await readResponseTextWithLimit(geminiResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'Gemini image error response',
})
throw new Error(`Gemini API error: ${geminiResponse.status} - ${error}`)
}
const data = await readResponseJsonWithLimit(geminiResponse, {
maxBytes: MAX_IMAGE_JSON_BYTES,
label: 'Gemini image response',
})
if (!isRecordLike(data)) {
throw new Error('Invalid Gemini image response')
}
const candidate = firstRecord(data.candidates)
const content = isRecordLike(candidate?.content) ? candidate.content : undefined
const parts = Array.isArray(content?.parts) ? content.parts : []
const textPart = parts.find((part) => isRecordLike(part) && typeof part.text === 'string')
const imagePart = parts.find((part) => {
if (!isRecordLike(part)) return false
return isRecordLike(part.inlineData) || isRecordLike(part.inline_data)
})
if (!isRecordLike(imagePart)) {
logger.error(`[${requestId}] Gemini response missing image part`)
throw new Error('No image data found in Gemini response')
}
const inlineData = isRecordLike(imagePart.inlineData)
? imagePart.inlineData
: isRecordLike(imagePart.inline_data)
? imagePart.inline_data
: undefined
const base64Image = getStringProperty(inlineData, 'data')
const contentType =
getStringProperty(inlineData, 'mimeType') ||
getStringProperty(inlineData, 'mime_type') ||
'image/png'
if (!base64Image) {
throw new Error('Gemini image response missing inline image data')
}
return {
buffer: (() => {
const buffer = Buffer.from(base64Image, 'base64')
assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'Gemini image response')
return buffer
})(),
contentType,
fileName: `gemini-${model}.${extensionFromContentType(contentType)}`,
provider: 'gemini',
model,
description: isRecordLike(textPart) ? getStringProperty(textPart, 'text') : undefined,
}
}
function buildFalAIQueueUrl(endpoint: string, requestId: string, path: 'status' | 'response') {
return `https://queue.fal.run/${endpoint}/requests/${requestId}/${path}`
}
function getFalAIErrorMessage(error: unknown): string {
if (typeof error === 'string') return error
if (isRecordLike(error)) {
return (
getStringProperty(error, 'message') ||
getStringProperty(error, 'detail') ||
JSON.stringify(error)
)
}
return 'Unknown Fal.ai error'
}
async function generateWithFalAI(
apiKey: string,
body: ImageToolBody,
requestId: string,
logger: ReturnType<typeof createLogger>
): Promise<GeneratedImageResult> {
const model = body.model || 'nano-banana-2'
const modelConfig = FALAI_IMAGE_MODEL_CONFIGS[model]
if (!modelConfig) {
throw new Error(`Unknown Fal.ai image model: ${model}`)
}
const requestBody: Record<string, string | number | boolean> = {
prompt: body.prompt,
sync_mode: false,
}
if (modelConfig.maxNumImages) {
requestBody.num_images = clampInteger(body.numImages, 1, modelConfig.maxNumImages, 1)
}
if (modelConfig.supportsSeed && body.seed !== undefined) {
requestBody.seed = body.seed
}
if (modelConfig.sizeOptions && modelConfig.defaultSize) {
requestBody.image_size = pickAllowed(
body.size,
modelConfig.sizeOptions,
modelConfig.defaultSize
)
}
if (modelConfig.aspectRatios && modelConfig.defaultAspectRatio) {
requestBody.aspect_ratio = pickAllowed(
body.aspectRatio,
modelConfig.aspectRatios,
modelConfig.defaultAspectRatio
)
}
if (modelConfig.resolutionOptions && modelConfig.defaultResolution) {
requestBody.resolution = pickAllowed(
body.resolution,
modelConfig.resolutionOptions,
modelConfig.defaultResolution
)
}
if (modelConfig.outputFormats && modelConfig.defaultOutputFormat) {
requestBody.output_format = pickAllowed(
body.outputFormat,
modelConfig.outputFormats,
modelConfig.defaultOutputFormat
)
}
if (modelConfig.qualityOptions && modelConfig.defaultQuality) {
requestBody.quality = pickAllowed(
body.quality,
modelConfig.qualityOptions,
modelConfig.defaultQuality
)
}
if (modelConfig.backgroundOptions && modelConfig.defaultBackground) {
requestBody.background = pickAllowed(
body.background,
modelConfig.backgroundOptions,
modelConfig.defaultBackground
)
}
if (modelConfig.safetyToleranceOptions && modelConfig.defaultSafetyTolerance) {
requestBody.safety_tolerance = pickAllowed(
body.safetyTolerance,
modelConfig.safetyToleranceOptions,
modelConfig.defaultSafetyTolerance
)
}
if (modelConfig.supportsEnableSafetyChecker && body.enableSafetyChecker !== undefined) {
requestBody.enable_safety_checker = body.enableSafetyChecker
}
if (modelConfig.supportsEnableWebSearch && body.enableWebSearch !== undefined) {
requestBody.enable_web_search = body.enableWebSearch
}
if (modelConfig.supportsThinkingLevel && body.thinkingLevel) {
requestBody.thinking_level = pickAllowed(body.thinkingLevel, ['minimal', 'high'], 'minimal')
}
const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, {
method: 'POST',
headers: {
Authorization: `Key ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
})
if (!createResponse.ok) {
const error = await readResponseTextWithLimit(createResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'Fal.ai create error response',
})
throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`)
}
const createData = await readResponseJsonWithLimit(createResponse, {
maxBytes: MAX_IMAGE_JSON_BYTES,
label: 'Fal.ai create response',
})
if (!isRecordLike(createData)) {
throw new Error('Invalid Fal.ai queue response')
}
const falRequestId = getStringProperty(createData, 'request_id')
if (!falRequestId) {
throw new Error('Fal.ai queue response missing request_id')
}
const statusUrl =
getStringProperty(createData, 'status_url') ||
buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'status')
const responseUrl =
getStringProperty(createData, 'response_url') ||
buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'response')
logger.info(`[${requestId}] Fal.ai image request created: ${falRequestId}`)
const pollIntervalMs = 3000
const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs)
let attempts = 0
while (attempts < maxAttempts) {
await sleep(pollIntervalMs)
const statusResponse = await fetch(statusUrl, {
headers: {
Authorization: `Key ${apiKey}`,
},
})
if (!statusResponse.ok) {
await readResponseTextWithLimit(statusResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'Fal.ai status error response',
}).catch(() => '')
throw new Error(`Fal.ai status check failed: ${statusResponse.status}`)
}
const statusData = await readResponseJsonWithLimit(statusResponse, {
maxBytes: MAX_IMAGE_JSON_BYTES,
label: 'Fal.ai status response',
})
if (!isRecordLike(statusData)) {
throw new Error('Invalid Fal.ai status response')
}
const status = getStringProperty(statusData, 'status')
if (status === 'COMPLETED') {
const statusError = statusData.error
if (statusError) {
throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`)
}
const resultResponse = await fetch(
getStringProperty(statusData, 'response_url') || responseUrl,
{
headers: {
Authorization: `Key ${apiKey}`,
},
}
)
if (!resultResponse.ok) {
await readResponseTextWithLimit(resultResponse, {
maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
label: 'Fal.ai result error response',
}).catch(() => '')
throw new Error(`Failed to fetch Fal.ai result: ${resultResponse.status}`)
}
const resultData = await readResponseJsonWithLimit(resultResponse, {
maxBytes: MAX_IMAGE_JSON_BYTES,
label: 'Fal.ai result response',
})
if (!isRecordLike(resultData)) {
throw new Error('Invalid Fal.ai result response')
}
const firstImage = firstRecord(resultData.images)
const imageUrl =
getStringProperty(firstImage, 'url') ||
getStringProperty(firstImage, 'data') ||
getStringProperty(firstImage, 'content')
if (!imageUrl) {
throw new Error('No image URL in Fal.ai response')
}
const downloaded = await bufferFromImageUrl(imageUrl)
const contentType =
getStringProperty(firstImage, 'content_type') ||
getStringProperty(firstImage, 'contentType') ||
downloaded.contentType
const fileName =
getStringProperty(firstImage, 'file_name') ||
getStringProperty(firstImage, 'fileName') ||
`falai-${model}.${extensionFromContentType(contentType)}`
return {
buffer: downloaded.buffer,
contentType,
fileName,
provider: 'falai',
model,
sourceUrl: imageUrl.startsWith('data:') ? undefined : imageUrl,
description: getStringProperty(resultData, 'description'),
revisedPrompt: getStringProperty(resultData, 'revised_prompt'),
seed: getNumberProperty(resultData, 'seed'),
jobId: falRequestId,
falaiCost: body.useHostedCostTracking
? await getFalAICostMetadata({
apiKey,
endpointId: modelConfig.endpoint,
requestId: falRequestId,
})
: undefined,
}
}
if (['ERROR', 'FAILED', 'CANCELLED'].includes(status || '')) {
throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusData.error)}`)
}
attempts += 1
}
throw new Error('Fal.ai image generation timed out')
}
async function storeGeneratedImage(
imageResult: GeneratedImageResult,
body: ImageToolBody,
userId: string,
requestId: string
): Promise<StoredImageResponse> {
const timestamp = Date.now()
const safeFileName = imageResult.fileName || `image-${imageResult.provider}-${timestamp}.png`
const executionContext =
body.workspaceId && body.workflowId && body.executionId
? {
workspaceId: body.workspaceId,
workflowId: body.workflowId,
executionId: body.executionId,
}
: null
if (executionContext) {
const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution')
const imageFile = await uploadExecutionFile(
executionContext,
imageResult.buffer,
safeFileName,
imageResult.contentType,
userId
)
return {
content: imageFile.url,
imageUrl: imageFile.url,
imageFile,
fileName: safeFileName,
contentType: imageResult.contentType,
provider: imageResult.provider,
model: imageResult.model,
metadata: {
provider: imageResult.provider,
model: imageResult.model,
description: imageResult.description,
revisedPrompt: imageResult.revisedPrompt,
seed: imageResult.seed,
jobId: imageResult.jobId,
contentType: imageResult.contentType,
},
__falaiCostDollars: imageResult.falaiCost?.costDollars,
__falaiBilling: imageResult.falaiCost,
}
}
const { StorageService } = await import('@/lib/uploads')
const fileInfo = await StorageService.uploadFile({
file: imageResult.buffer,
fileName: safeFileName,
contentType: imageResult.contentType,
context: 'copilot',
})
const imageUrl = `${getBaseUrl()}${fileInfo.path}`
logger.info(`[${requestId}] Stored generated image fallback`, {
fileName: safeFileName,
size: imageResult.buffer.length,
})
return {
content: imageUrl,
imageUrl,
fileName: safeFileName,
contentType: imageResult.contentType,
provider: imageResult.provider,
model: imageResult.model,
metadata: {
provider: imageResult.provider,
model: imageResult.model,
description: imageResult.description,
revisedPrompt: imageResult.revisedPrompt,
seed: imageResult.seed,
jobId: imageResult.jobId,
contentType: imageResult.contentType,
},
__falaiCostDollars: imageResult.falaiCost?.costDollars,
__falaiBilling: imageResult.falaiCost,
}
}