import type { ReadUrlParams, ReadUrlResponse } from '@/tools/jina/types' import type { ToolConfig } from '@/tools/types' export const readUrlTool: ToolConfig = { id: 'jina_read_url', name: 'Jina Reader', description: 'Extract and process web content into clean, LLM-friendly text using Jina AI Reader. Supports advanced content parsing, link gathering, and multiple output formats with configurable processing options.', version: '1.0.0', params: { url: { type: 'string', required: true, visibility: 'user-or-llm', description: 'The URL to read and convert to markdown (e.g., "https://example.com/page")', }, useReaderLMv2: { type: 'boolean', required: false, visibility: 'user-only', description: 'Whether to use ReaderLM-v2 for better quality (3x token cost)', }, gatherLinks: { type: 'boolean', required: false, visibility: 'user-only', description: 'Whether to gather all links at the end', }, jsonResponse: { type: 'boolean', required: false, visibility: 'user-only', description: 'Whether to return response in JSON format', }, apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Your Jina AI API key', }, // Content extraction params withImagesummary: { type: 'boolean', required: false, visibility: 'user-only', description: 'Gather all images from the page with metadata', }, retainImages: { type: 'string', required: false, visibility: 'user-only', description: 'Control image inclusion: "none" removes all, "all" keeps all', }, returnFormat: { type: 'string', required: false, visibility: 'user-only', description: 'Output format: markdown, html, text, screenshot, or pageshot', }, withIframe: { type: 'boolean', required: false, visibility: 'user-only', description: 'Include iframe content in extraction', }, withShadowDom: { type: 'boolean', required: false, visibility: 'user-only', description: 'Extract Shadow DOM content', }, // Performance & caching noCache: { type: 'boolean', required: false, visibility: 'user-only', description: 'Bypass cached content for real-time retrieval', }, // Advanced options withGeneratedAlt: { type: 'boolean', required: false, visibility: 'user-only', description: 'Generate alt text for images using VLM', }, robotsTxt: { type: 'string', required: false, visibility: 'user-only', description: 'Bot User-Agent for robots.txt checking', }, dnt: { type: 'boolean', required: false, visibility: 'user-only', description: 'Do Not Track - prevents caching/tracking', }, noGfm: { type: 'boolean', required: false, visibility: 'user-only', description: 'Disable GitHub Flavored Markdown', }, }, request: { url: (params: ReadUrlParams) => { return `https://r.jina.ai/https://${params.url.replace(/^https?:\/\//, '')}` }, method: 'GET', headers: (params: ReadUrlParams) => { // Start with base headers const headers: Record = { Accept: params.jsonResponse ? 'application/json' : 'text/plain', Authorization: `Bearer ${params.apiKey}`, } // Legacy params (backward compatible) if (params.useReaderLMv2 === true) { headers['X-Respond-With'] = 'readerlm-v2' } if (params.gatherLinks === true) { headers['X-With-Links-Summary'] = 'true' } // Content extraction headers if (params.withImagesummary === true) { headers['X-With-Images-Summary'] = 'true' } if (params.retainImages) { headers['X-Retain-Images'] = params.retainImages } if (params.returnFormat) { headers['X-Return-Format'] = params.returnFormat } if (params.withIframe === true) { headers['X-With-Iframe'] = 'true' } if (params.withShadowDom === true) { headers['X-With-Shadow-Dom'] = 'true' } // Advanced options if (params.withGeneratedAlt === true) { headers['X-With-Generated-Alt'] = 'true' } if (params.robotsTxt) { headers['X-Robots-Txt'] = params.robotsTxt } if (params.dnt === true) { headers.DNT = '1' } if (params.noGfm === true) { headers['X-No-Gfm'] = 'true' } return headers }, }, hosting: { envKeyPrefix: 'JINA_API_KEY', apiKeyParam: 'apiKey', byokProviderId: 'jina', pricing: { type: 'custom', getCost: (_params, output) => { if (output.tokensUsed == null) { throw new Error('Jina read_url response missing tokensUsed field') } // Jina bills per output token — $0.20 per 1M tokens // Source: https://cloud.jina.ai/pricing (token-based billing) const tokens = output.tokensUsed as number const cost = tokens * 0.0000002 return { cost, metadata: { tokensUsed: tokens } } }, }, rateLimit: { mode: 'per_request', requestsPerMinute: 200, }, }, transformResponse: async (response: Response) => { let tokensUsed: number | undefined const tokensHeader = response.headers.get('x-tokens') if (tokensHeader) { const parsed = Number.parseInt(tokensHeader, 10) if (!Number.isNaN(parsed)) { tokensUsed = parsed } } const contentType = response.headers.get('content-type') if (contentType?.includes('application/json')) { const data = await response.json() tokensUsed ??= data.data?.usage?.tokens ?? data.usage?.tokens const content = data.data?.content || data.content || JSON.stringify(data) tokensUsed ??= Math.ceil(content.length / 4) return { success: response.ok, output: { content, tokensUsed }, } } const content = await response.text() tokensUsed ??= Math.ceil(content.length / 4) return { success: response.ok, output: { content, tokensUsed }, } }, outputs: { content: { type: 'string', description: 'The extracted content from the URL, processed into clean, LLM-friendly text', }, tokensUsed: { type: 'number', description: 'Number of Jina tokens consumed by this request', optional: true, }, }, }