simstudioai--sim/apps/sim/tools/jina/read_url.ts

import type { ReadUrlParams, ReadUrlResponse } from '@/tools/jina/types'
import type { ToolConfig } from '@/tools/types'

export const readUrlTool: ToolConfig<ReadUrlParams, ReadUrlResponse> = {
  id: 'jina_read_url',
  name: 'Jina Reader',
  description:
    'Extract and process web content into clean, LLM-friendly text using Jina AI Reader. Supports advanced content parsing, link gathering, and multiple output formats with configurable processing options.',
  version: '1.0.0',

  params: {
    url: {
      type: 'string',
      required: true,
      visibility: 'user-or-llm',
      description: 'The URL to read and convert to markdown (e.g., "https://example.com/page")',
    },
    useReaderLMv2: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Whether to use ReaderLM-v2 for better quality (3x token cost)',
    },
    gatherLinks: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Whether to gather all links at the end',
    },
    jsonResponse: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Whether to return response in JSON format',
    },
    apiKey: {
      type: 'string',
      required: true,
      visibility: 'user-only',
      description: 'Your Jina AI API key',
    },
    // Content extraction params
    withImagesummary: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Gather all images from the page with metadata',
    },
    retainImages: {
      type: 'string',
      required: false,
      visibility: 'user-only',
      description: 'Control image inclusion: "none" removes all, "all" keeps all',
    },
    returnFormat: {
      type: 'string',
      required: false,
      visibility: 'user-only',
      description: 'Output format: markdown, html, text, screenshot, or pageshot',
    },
    withIframe: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Include iframe content in extraction',
    },
    withShadowDom: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Extract Shadow DOM content',
    },
    // Performance & caching
    noCache: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Bypass cached content for real-time retrieval',
    },
    // Advanced options
    withGeneratedAlt: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Generate alt text for images using VLM',
    },
    robotsTxt: {
      type: 'string',
      required: false,
      visibility: 'user-only',
      description: 'Bot User-Agent for robots.txt checking',
    },
    dnt: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Do Not Track - prevents caching/tracking',
    },
    noGfm: {
      type: 'boolean',
      required: false,
      visibility: 'user-only',
      description: 'Disable GitHub Flavored Markdown',
    },
  },

  request: {
    url: (params: ReadUrlParams) => {
      return `https://r.jina.ai/https://${params.url.replace(/^https?:\/\//, '')}`
    },
    method: 'GET',
    headers: (params: ReadUrlParams) => {
      // Start with base headers
      const headers: Record<string, string> = {
        Accept: params.jsonResponse ? 'application/json' : 'text/plain',
        Authorization: `Bearer ${params.apiKey}`,
      }

      // Legacy params (backward compatible)
      if (params.useReaderLMv2 === true) {
        headers['X-Respond-With'] = 'readerlm-v2'
      }
      if (params.gatherLinks === true) {
        headers['X-With-Links-Summary'] = 'true'
      }

      // Content extraction headers
      if (params.withImagesummary === true) {
        headers['X-With-Images-Summary'] = 'true'
      }
      if (params.retainImages) {
        headers['X-Retain-Images'] = params.retainImages
      }
      if (params.returnFormat) {
        headers['X-Return-Format'] = params.returnFormat
      }
      if (params.withIframe === true) {
        headers['X-With-Iframe'] = 'true'
      }
      if (params.withShadowDom === true) {
        headers['X-With-Shadow-Dom'] = 'true'
      }

      // Advanced options
      if (params.withGeneratedAlt === true) {
        headers['X-With-Generated-Alt'] = 'true'
      }
      if (params.robotsTxt) {
        headers['X-Robots-Txt'] = params.robotsTxt
      }
      if (params.dnt === true) {
        headers.DNT = '1'
      }
      if (params.noGfm === true) {
        headers['X-No-Gfm'] = 'true'
      }

      return headers
    },
  },

  hosting: {
    envKeyPrefix: 'JINA_API_KEY',
    apiKeyParam: 'apiKey',
    byokProviderId: 'jina',
    pricing: {
      type: 'custom',
      getCost: (_params, output) => {
        if (output.tokensUsed == null) {
          throw new Error('Jina read_url response missing tokensUsed field')
        }
        // Jina bills per output token — $0.20 per 1M tokens
        // Source: https://cloud.jina.ai/pricing (token-based billing)
        const tokens = output.tokensUsed as number
        const cost = tokens * 0.0000002
        return { cost, metadata: { tokensUsed: tokens } }
      },
    },
    rateLimit: {
      mode: 'per_request',
      requestsPerMinute: 200,
    },
  },

  transformResponse: async (response: Response) => {
    let tokensUsed: number | undefined

    const tokensHeader = response.headers.get('x-tokens')
    if (tokensHeader) {
      const parsed = Number.parseInt(tokensHeader, 10)
      if (!Number.isNaN(parsed)) {
        tokensUsed = parsed
      }
    }

    const contentType = response.headers.get('content-type')

    if (contentType?.includes('application/json')) {
      const data = await response.json()
      tokensUsed ??= data.data?.usage?.tokens ?? data.usage?.tokens
      const content = data.data?.content || data.content || JSON.stringify(data)
      tokensUsed ??= Math.ceil(content.length / 4)
      return {
        success: response.ok,
        output: { content, tokensUsed },
      }
    }

    const content = await response.text()
    tokensUsed ??= Math.ceil(content.length / 4)
    return {
      success: response.ok,
      output: { content, tokensUsed },
    }
  },

  outputs: {
    content: {
      type: 'string',
      description: 'The extracted content from the URL, processed into clean, LLM-friendly text',
    },
    tokensUsed: {
      type: 'number',
      description: 'Number of Jina tokens consumed by this request',
      optional: true,
    },
  },
}