simstudioai--sim/apps/sim/app/api/tools/vision/analyze/route.ts

import { GoogleGenAI } from '@google/genai'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { type NextRequest, NextResponse } from 'next/server'
import { visionAnalyzeContract } from '@/lib/api/contracts/tools/media/vision'
import { parseRequest } from '@/lib/api/server'
import { checkInternalAuth } from '@/lib/auth/hybrid'
import {
  secureFetchWithPinnedIP,
  validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils'
import {
  downloadFileFromStorage,
  resolveInternalFileUrl,
} from '@/lib/uploads/utils/file-utils.server'
import { assertToolFileAccess } from '@/app/api/files/authorization'
import { convertUsageMetadata, extractTextContent } from '@/providers/google/utils'

export const dynamic = 'force-dynamic'

const logger = createLogger('VisionAnalyzeAPI')

export const POST = withRouteHandler(async (request: NextRequest) => {
  const requestId = generateRequestId()

  try {
    const authResult = await checkInternalAuth(request, { requireWorkflowId: false })

    if (!authResult.success || !authResult.userId) {
      logger.warn(`[${requestId}] Unauthorized Vision analyze attempt: ${authResult.error}`)
      return NextResponse.json(
        {
          success: false,
          error: authResult.error || 'Authentication required',
        },
        { status: 401 }
      )
    }

    logger.info(`[${requestId}] Authenticated Vision analyze request via ${authResult.authType}`, {
      userId: authResult.userId,
    })

    const userId = authResult.userId

    const parsed = await parseRequest(visionAnalyzeContract, request, {})
    if (!parsed.success) return parsed.response

    const validatedData = parsed.data.body

    if (!validatedData.imageUrl && !validatedData.imageFile) {
      return NextResponse.json(
        {
          success: false,
          error: 'Either imageUrl or imageFile is required',
        },
        { status: 400 }
      )
    }

    logger.info(`[${requestId}] Analyzing image`, {
      hasFile: !!validatedData.imageFile,
      hasUrl: !!validatedData.imageUrl,
      model: validatedData.model,
    })

    let imageSource: string = validatedData.imageUrl || ''

    if (validatedData.imageFile) {
      const rawFile = validatedData.imageFile
      logger.info(`[${requestId}] Processing image file: ${rawFile.name}`)

      let userFile
      try {
        userFile = processSingleFileToUserFile(rawFile, requestId, logger)
      } catch (error) {
        return NextResponse.json(
          {
            success: false,
            error: getErrorMessage(error, 'Failed to process image file'),
          },
          { status: 400 }
        )
      }

      let base64 = userFile.base64
      let bufferLength = 0
      if (!base64) {
        const denied = await assertToolFileAccess(
          userFile.key,
          authResult.userId,
          requestId,
          logger
        )
        if (denied) return denied
        const buffer = await downloadFileFromStorage(userFile, requestId, logger)
        base64 = buffer.toString('base64')
        bufferLength = buffer.length
      }
      const mimeType = userFile.type || 'image/jpeg'
      imageSource = `data:${mimeType};base64,${base64}`
      if (bufferLength > 0) {
        logger.info(`[${requestId}] Converted image to base64 (${bufferLength} bytes)`)
      }
    }

    let imageUrlValidation: Awaited<ReturnType<typeof validateUrlWithDNS>> | null = null
    if (imageSource && !imageSource.startsWith('data:')) {
      if (imageSource.startsWith('/') && !isInternalFileUrl(imageSource)) {
        return NextResponse.json(
          {
            success: false,
            error: 'Invalid file path. Only uploaded files are supported for internal paths.',
          },
          { status: 400 }
        )
      }

      if (isInternalFileUrl(imageSource)) {
        if (!userId) {
          return NextResponse.json(
            {
              success: false,
              error: 'Authentication required for internal file access',
            },
            { status: 401 }
          )
        }
        const resolution = await resolveInternalFileUrl(imageSource, userId, requestId, logger)
        if (resolution.error) {
          return NextResponse.json(
            {
              success: false,
              error: resolution.error.message,
            },
            { status: resolution.error.status }
          )
        }
        imageSource = resolution.fileUrl || imageSource
      }

      imageUrlValidation = await validateUrlWithDNS(imageSource, 'imageUrl')
      if (!imageUrlValidation.isValid) {
        return NextResponse.json(
          {
            success: false,
            error: imageUrlValidation.error,
          },
          { status: 400 }
        )
      }
    }

    const defaultPrompt = 'Please analyze this image and describe what you see in detail.'
    const prompt = validatedData.prompt || defaultPrompt

    const isClaude = validatedData.model.startsWith('claude-')
    const isGemini = validatedData.model.startsWith('gemini-')
    const apiUrl = isClaude
      ? 'https://api.anthropic.com/v1/messages'
      : 'https://api.openai.com/v1/chat/completions'

    const headers: Record<string, string> = {
      'Content-Type': 'application/json',
    }

    if (isClaude) {
      headers['x-api-key'] = validatedData.apiKey
      headers['anthropic-version'] = '2023-06-01'
    } else {
      headers.Authorization = `Bearer ${validatedData.apiKey}`
    }

    let requestBody: any

    if (isGemini) {
      let base64Payload = imageSource
      if (!base64Payload.startsWith('data:')) {
        const urlValidation =
          imageUrlValidation || (await validateUrlWithDNS(base64Payload, 'imageUrl'))
        if (!urlValidation.isValid) {
          return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 })
        }

        const response = await secureFetchWithPinnedIP(base64Payload, urlValidation.resolvedIP!, {
          method: 'GET',
        })
        if (!response.ok) {
          await response.text().catch(() => {})
          return NextResponse.json(
            { success: false, error: 'Failed to fetch image for Gemini' },
            { status: 400 }
          )
        }
        const contentType =
          response.headers.get('content-type') || validatedData.imageFile?.type || 'image/jpeg'
        const arrayBuffer = await response.arrayBuffer()
        const base64 = Buffer.from(arrayBuffer).toString('base64')
        base64Payload = `data:${contentType};base64,${base64}`
      }
      const base64Marker = ';base64,'
      const markerIndex = base64Payload.indexOf(base64Marker)
      if (!base64Payload.startsWith('data:') || markerIndex === -1) {
        return NextResponse.json(
          { success: false, error: 'Invalid base64 image format' },
          { status: 400 }
        )
      }
      const rawMimeType = base64Payload.slice('data:'.length, markerIndex)
      const mediaType = rawMimeType.split(';')[0] || 'image/jpeg'
      const base64Data = base64Payload.slice(markerIndex + base64Marker.length)
      if (!base64Data) {
        return NextResponse.json(
          { success: false, error: 'Invalid base64 image format' },
          { status: 400 }
        )
      }

      const ai = new GoogleGenAI({ apiKey: validatedData.apiKey })
      const geminiResponse = await ai.models.generateContent({
        model: validatedData.model,
        contents: [
          {
            role: 'user',
            parts: [{ text: prompt }, { inlineData: { mimeType: mediaType, data: base64Data } }],
          },
        ],
      })

      const content = extractTextContent(geminiResponse.candidates?.[0])
      const usage = convertUsageMetadata(geminiResponse.usageMetadata)

      return NextResponse.json({
        success: true,
        output: {
          content,
          model: validatedData.model,
          tokens: usage.totalTokenCount || undefined,
        },
      })
    }

    if (isClaude) {
      if (imageSource.startsWith('data:')) {
        const base64Match = imageSource.match(/^data:([^;]+);base64,(.+)$/)
        if (!base64Match) {
          return NextResponse.json(
            { success: false, error: 'Invalid base64 image format' },
            { status: 400 }
          )
        }
        const [, mediaType, base64Data] = base64Match

        requestBody = {
          model: validatedData.model,
          max_tokens: 1024,
          messages: [
            {
              role: 'user',
              content: [
                { type: 'text', text: prompt },
                {
                  type: 'image',
                  source: {
                    type: 'base64',
                    media_type: mediaType,
                    data: base64Data,
                  },
                },
              ],
            },
          ],
        }
      } else {
        requestBody = {
          model: validatedData.model,
          max_tokens: 1024,
          messages: [
            {
              role: 'user',
              content: [
                { type: 'text', text: prompt },
                {
                  type: 'image',
                  source: { type: 'url', url: imageSource },
                },
              ],
            },
          ],
        }
      }
    } else {
      requestBody = {
        model: validatedData.model,
        messages: [
          {
            role: 'user',
            content: [
              { type: 'text', text: prompt },
              {
                type: 'image_url',
                image_url: {
                  url: imageSource,
                },
              },
            ],
          },
        ],
        max_completion_tokens: 1000,
      }
    }

    logger.info(`[${requestId}] Sending request to ${isClaude ? 'Anthropic' : 'OpenAI'} API`)
    const response = await fetch(apiUrl, {
      method: 'POST',
      headers,
      body: JSON.stringify(requestBody),
    })

    if (!response.ok) {
      const errorData = await response.json().catch(() => ({}))
      logger.error(`[${requestId}] Vision API error:`, errorData)
      return NextResponse.json(
        {
          success: false,
          error: errorData.error?.message || errorData.message || 'Failed to analyze image',
        },
        { status: response.status }
      )
    }

    const data = await response.json()
    const result = data.content?.[0]?.text || data.choices?.[0]?.message?.content

    logger.info(`[${requestId}] Image analyzed successfully`)

    return NextResponse.json({
      success: true,
      output: {
        content: result,
        model: data.model,
        tokens: data.content
          ? (data.usage?.input_tokens || 0) + (data.usage?.output_tokens || 0)
          : data.usage?.total_tokens,
        usage: data.usage
          ? {
              input_tokens: data.usage.input_tokens,
              output_tokens: data.usage.output_tokens,
              total_tokens:
                data.usage.total_tokens ||
                (data.usage.input_tokens || 0) + (data.usage.output_tokens || 0),
            }
          : undefined,
      },
    })
  } catch (error) {
    logger.error(`[${requestId}] Error analyzing image:`, error)
    return NextResponse.json(
      {
        success: false,
        error: getErrorMessage(error, 'Unknown error occurred'),
      },
      { status: 500 }
    )
  }
})