chore: import upstream snapshot with attribution

apps/sim/tools/jina/index.ts

@@ -0,0 +1,5 @@
+import { readUrlTool } from './read_url'
+import { searchTool } from './search'
+
+export const jinaReadUrlTool = readUrlTool
+export const jinaSearchTool = searchTool

apps/sim/tools/jina/read_url.ts

@@ -0,0 +1,228 @@
+import type { ReadUrlParams, ReadUrlResponse } from '@/tools/jina/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const readUrlTool: ToolConfig<ReadUrlParams, ReadUrlResponse> = {
+  id: 'jina_read_url',
+  name: 'Jina Reader',
+  description:
+    'Extract and process web content into clean, LLM-friendly text using Jina AI Reader. Supports advanced content parsing, link gathering, and multiple output formats with configurable processing options.',
+  version: '1.0.0',
+
+  params: {
+    url: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'The URL to read and convert to markdown (e.g., "https://example.com/page")',
+    },
+    useReaderLMv2: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Whether to use ReaderLM-v2 for better quality (3x token cost)',
+    },
+    gatherLinks: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Whether to gather all links at the end',
+    },
+    jsonResponse: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Whether to return response in JSON format',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'Your Jina AI API key',
+    },
+    // Content extraction params
+    withImagesummary: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Gather all images from the page with metadata',
+    },
+    retainImages: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Control image inclusion: "none" removes all, "all" keeps all',
+    },
+    returnFormat: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Output format: markdown, html, text, screenshot, or pageshot',
+    },
+    withIframe: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Include iframe content in extraction',
+    },
+    withShadowDom: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Extract Shadow DOM content',
+    },
+    // Performance & caching
+    noCache: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Bypass cached content for real-time retrieval',
+    },
+    // Advanced options
+    withGeneratedAlt: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Generate alt text for images using VLM',
+    },
+    robotsTxt: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Bot User-Agent for robots.txt checking',
+    },
+    dnt: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Do Not Track - prevents caching/tracking',
+    },
+    noGfm: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Disable GitHub Flavored Markdown',
+    },
+  },
+
+  request: {
+    url: (params: ReadUrlParams) => {
+      return `https://r.jina.ai/https://${params.url.replace(/^https?:\/\//, '')}`
+    },
+    method: 'GET',
+    headers: (params: ReadUrlParams) => {
+      // Start with base headers
+      const headers: Record<string, string> = {
+        Accept: params.jsonResponse ? 'application/json' : 'text/plain',
+        Authorization: `Bearer ${params.apiKey}`,
+      }
+
+      // Legacy params (backward compatible)
+      if (params.useReaderLMv2 === true) {
+        headers['X-Respond-With'] = 'readerlm-v2'
+      }
+      if (params.gatherLinks === true) {
+        headers['X-With-Links-Summary'] = 'true'
+      }
+
+      // Content extraction headers
+      if (params.withImagesummary === true) {
+        headers['X-With-Images-Summary'] = 'true'
+      }
+      if (params.retainImages) {
+        headers['X-Retain-Images'] = params.retainImages
+      }
+      if (params.returnFormat) {
+        headers['X-Return-Format'] = params.returnFormat
+      }
+      if (params.withIframe === true) {
+        headers['X-With-Iframe'] = 'true'
+      }
+      if (params.withShadowDom === true) {
+        headers['X-With-Shadow-Dom'] = 'true'
+      }
+
+      // Advanced options
+      if (params.withGeneratedAlt === true) {
+        headers['X-With-Generated-Alt'] = 'true'
+      }
+      if (params.robotsTxt) {
+        headers['X-Robots-Txt'] = params.robotsTxt
+      }
+      if (params.dnt === true) {
+        headers.DNT = '1'
+      }
+      if (params.noGfm === true) {
+        headers['X-No-Gfm'] = 'true'
+      }
+
+      return headers
+    },
+  },
+
+  hosting: {
+    envKeyPrefix: 'JINA_API_KEY',
+    apiKeyParam: 'apiKey',
+    byokProviderId: 'jina',
+    pricing: {
+      type: 'custom',
+      getCost: (_params, output) => {
+        if (output.tokensUsed == null) {
+          throw new Error('Jina read_url response missing tokensUsed field')
+        }
+        // Jina bills per output token — $0.20 per 1M tokens
+        // Source: https://cloud.jina.ai/pricing (token-based billing)
+        const tokens = output.tokensUsed as number
+        const cost = tokens * 0.0000002
+        return { cost, metadata: { tokensUsed: tokens } }
+      },
+    },
+    rateLimit: {
+      mode: 'per_request',
+      requestsPerMinute: 200,
+    },
+  },
+
+  transformResponse: async (response: Response) => {
+    let tokensUsed: number | undefined
+
+    const tokensHeader = response.headers.get('x-tokens')
+    if (tokensHeader) {
+      const parsed = Number.parseInt(tokensHeader, 10)
+      if (!Number.isNaN(parsed)) {
+        tokensUsed = parsed
+      }
+    }
+
+    const contentType = response.headers.get('content-type')
+
+    if (contentType?.includes('application/json')) {
+      const data = await response.json()
+      tokensUsed ??= data.data?.usage?.tokens ?? data.usage?.tokens
+      const content = data.data?.content || data.content || JSON.stringify(data)
+      tokensUsed ??= Math.ceil(content.length / 4)
+      return {
+        success: response.ok,
+        output: { content, tokensUsed },
+      }
+    }
+
+    const content = await response.text()
+    tokensUsed ??= Math.ceil(content.length / 4)
+    return {
+      success: response.ok,
+      output: { content, tokensUsed },
+    }
+  },
+
+  outputs: {
+    content: {
+      type: 'string',
+      description: 'The extracted content from the URL, processed into clean, LLM-friendly text',
+    },
+    tokensUsed: {
+      type: 'number',
+      description: 'Number of Jina tokens consumed by this request',
+      optional: true,
+    },
+  },
+}

apps/sim/tools/jina/search.ts

@@ -0,0 +1,239 @@
+import type { SearchParams, SearchResponse } from '@/tools/jina/types'
+import { JINA_SEARCH_RESULT_OUTPUT_PROPERTIES } from '@/tools/jina/types'
+import type { ToolConfig } from '@/tools/types'
+
+export const searchTool: ToolConfig<SearchParams, SearchResponse> = {
+  id: 'jina_search',
+  name: 'Jina Search',
+  description:
+    'Search the web and return top 5 results with LLM-friendly content. Each result is automatically processed through Jina Reader API. Supports geographic filtering, site restrictions, and pagination.',
+  version: '1.0.0',
+
+  params: {
+    q: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Search query string (e.g., "machine learning tutorials")',
+    },
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'Your Jina AI API key',
+    },
+    // Pagination
+    num: {
+      type: 'number',
+      required: false,
+      visibility: 'user-only',
+      description: 'Maximum number of results per page (default: 5)',
+    },
+    // Site restriction
+    site: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description:
+        'Restrict results to specific domain(s). Can be comma-separated for multiple sites (e.g., "jina.ai,github.com")',
+    },
+    // Content options
+    withFavicon: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Include website favicons in results',
+    },
+    withImagesummary: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Gather all images from result pages with metadata',
+    },
+    withLinksummary: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Gather all links from result pages',
+    },
+    retainImages: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Control image inclusion: "none" removes all, "all" keeps all',
+    },
+    noCache: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Bypass cached content for real-time retrieval',
+    },
+    withGeneratedAlt: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-only',
+      description: 'Generate alt text for images using VLM',
+    },
+    respondWith: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Set to "no-content" to get only metadata without page content',
+    },
+    returnFormat: {
+      type: 'string',
+      required: false,
+      visibility: 'user-only',
+      description: 'Output format: markdown, html, text, screenshot, or pageshot',
+    },
+  },
+
+  request: {
+    url: (params: SearchParams) => {
+      const baseUrl = 'https://s.jina.ai/'
+      const query = encodeURIComponent(params.q)
+
+      // Build query params
+      const queryParams: string[] = []
+
+      // Handle site parameter (can be string or array)
+      if (params.site) {
+        const sites = typeof params.site === 'string' ? params.site.split(',') : params.site
+        sites.forEach((s) => queryParams.push(`site=${encodeURIComponent(s.trim())}`))
+      }
+
+      const queryString = queryParams.length > 0 ? `?${queryParams.join('&')}` : ''
+      return `${baseUrl}${query}${queryString}`
+    },
+    method: 'GET',
+    headers: (params: SearchParams) => {
+      const headers: Record<string, string> = {
+        Accept: 'application/json',
+        Authorization: `Bearer ${params.apiKey}`,
+      }
+
+      // Content options
+      if (params.withFavicon === true) {
+        headers['X-With-Favicon'] = 'true'
+      }
+      if (params.withImagesummary === true) {
+        headers['X-With-Images-Summary'] = 'true'
+      }
+      if (params.withLinksummary === true) {
+        headers['X-With-Links-Summary'] = 'true'
+      }
+      if (params.retainImages) {
+        headers['X-Retain-Images'] = params.retainImages
+      }
+      if (params.noCache === true) {
+        headers['X-No-Cache'] = 'true'
+      }
+      if (params.withGeneratedAlt === true) {
+        headers['X-With-Generated-Alt'] = 'true'
+      }
+      if (params.respondWith) {
+        headers['X-Respond-With'] = params.respondWith
+      }
+      if (params.returnFormat) {
+        headers['X-Return-Format'] = params.returnFormat
+      }
+
+      // Pagination headers
+      if (params.num) {
+        headers['X-Num'] = params.num.toString()
+      }
+
+      return headers
+    },
+  },
+
+  hosting: {
+    envKeyPrefix: 'JINA_API_KEY',
+    apiKeyParam: 'apiKey',
+    byokProviderId: 'jina',
+    pricing: {
+      type: 'custom',
+      getCost: (_params, output) => {
+        if (output.tokensUsed == null) {
+          throw new Error('Jina search response missing tokensUsed field')
+        }
+        // Jina bills per output token — $0.20 per 1M tokens
+        // Search costs a fixed minimum of 10,000 tokens per request
+        // Source: https://cloud.jina.ai/pricing (token-based billing)
+        // x-tokens header is unreliable; falls back to content-length estimate (~4 chars/token)
+        const tokens = output.tokensUsed as number
+        const cost = tokens * 0.0000002
+        return { cost, metadata: { tokensUsed: tokens } }
+      },
+    },
+    rateLimit: {
+      mode: 'per_request',
+      requestsPerMinute: 40,
+    },
+  },
+
+  transformResponse: async (response: Response) => {
+    const data = await response.json()
+
+    let tokensUsed: number | undefined
+    const tokensHeader = response.headers.get('x-tokens')
+    if (tokensHeader) {
+      const parsed = Number.parseInt(tokensHeader, 10)
+      if (!Number.isNaN(parsed) && parsed > 0) {
+        tokensUsed = parsed
+      }
+    }
+
+    const results = Array.isArray(data) ? data : data.data || []
+
+    if (tokensUsed == null) {
+      let total = 0
+      for (const result of results) {
+        if (result.usage?.tokens) {
+          total += result.usage.tokens
+        }
+      }
+      if (total > 0) {
+        tokensUsed = total
+      }
+    }
+
+    if (tokensUsed == null) {
+      let totalChars = 0
+      for (const result of results) {
+        totalChars += (result.content?.length ?? 0) + (result.title?.length ?? 0)
+      }
+      tokensUsed = Math.max(Math.ceil(totalChars / 4), 10000)
+    }
+
+    return {
+      success: response.ok,
+      output: {
+        results: results.map((result: any) => ({
+          title: result.title || '',
+          description: result.description || '',
+          url: result.url || '',
+          content: result.content || '',
+        })),
+        tokensUsed,
+      },
+    }
+  },
+
+  outputs: {
+    results: {
+      type: 'array',
+      description:
+        'Array of search results, each containing title, description, url, and LLM-friendly content',
+      items: {
+        type: 'object',
+        properties: JINA_SEARCH_RESULT_OUTPUT_PROPERTIES,
+      },
+    },
+    tokensUsed: {
+      type: 'number',
+      description: 'Number of Jina tokens consumed by this request',
+      optional: true,
+    },
+  },
+}

apps/sim/tools/jina/types.ts

@@ -0,0 +1,153 @@
+import type { OutputProperty, ToolResponse } from '@/tools/types'
+
+/**
+ * Shared output property definitions for Jina AI API responses.
+ * Based on Jina AI documentation: https://jina.ai/reader/
+ */
+
+/**
+ * Output definition for usage/token information
+ * Based on Jina AI API usage object
+ */
+export const JINA_USAGE_OUTPUT_PROPERTIES = {
+  tokens: { type: 'number', description: 'Number of tokens consumed by this request' },
+} as const satisfies Record<string, OutputProperty>
+
+/**
+ * Complete usage output definition
+ */
+export const JINA_USAGE_OUTPUT: OutputProperty = {
+  type: 'object',
+  description: 'Token usage information for this request',
+  properties: JINA_USAGE_OUTPUT_PROPERTIES,
+}
+
+/**
+ * Core data properties for Reader API responses
+ * Based on Jina AI Reader API response structure
+ */
+export const JINA_READER_DATA_OUTPUT_PROPERTIES = {
+  title: { type: 'string', description: 'Page title' },
+  description: { type: 'string', description: 'Page meta description', optional: true },
+  url: { type: 'string', description: 'The URL that was processed' },
+  content: {
+    type: 'string',
+    description: 'Main content extracted from the page in markdown format',
+  },
+  images: {
+    type: 'json',
+    description: 'Dictionary of images found on the page (image caption/name to URL)',
+    optional: true,
+  },
+  links: {
+    type: 'json',
+    description: 'Dictionary of links found on the page (link text to URL)',
+    optional: true,
+  },
+  usage: {
+    type: 'object',
+    description: 'Token usage information',
+    optional: true,
+    properties: JINA_USAGE_OUTPUT_PROPERTIES,
+  },
+} as const satisfies Record<string, OutputProperty>
+
+/**
+ * Output definition for search result items
+ */
+export const JINA_SEARCH_RESULT_OUTPUT_PROPERTIES = {
+  title: { type: 'string', description: 'Page title' },
+  description: {
+    type: 'string',
+    description: 'Page description or meta description',
+    optional: true,
+  },
+  url: { type: 'string', description: 'Page URL' },
+  content: { type: 'string', description: 'LLM-friendly extracted content' },
+  usage: {
+    type: 'object',
+    description: 'Token usage information',
+    optional: true,
+    properties: JINA_USAGE_OUTPUT_PROPERTIES,
+  },
+} as const satisfies Record<string, OutputProperty>
+
+/**
+ * Complete search result output definition
+ */
+export const JINA_SEARCH_RESULT_OUTPUT: OutputProperty = {
+  type: 'object',
+  description: 'Search result with extracted content',
+  properties: JINA_SEARCH_RESULT_OUTPUT_PROPERTIES,
+}
+
+/**
+ * Search results array output definition
+ */
+export const JINA_SEARCH_RESULTS_OUTPUT: OutputProperty = {
+  type: 'array',
+  description: 'Array of search results with LLM-friendly content',
+  items: {
+    type: 'object',
+    properties: JINA_SEARCH_RESULT_OUTPUT_PROPERTIES,
+  },
+}
+
+export interface ReadUrlParams {
+  url: string
+  // Existing params (backward compatible)
+  useReaderLMv2?: boolean
+  gatherLinks?: boolean
+  jsonResponse?: boolean
+  apiKey?: string
+  // Content extraction params
+  withImagesummary?: boolean
+  retainImages?: 'none' | 'all'
+  returnFormat?: 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot'
+  withIframe?: boolean
+  withShadowDom?: boolean
+  // Performance & caching
+  noCache?: boolean
+  // Advanced options
+  withGeneratedAlt?: boolean
+  robotsTxt?: string
+  dnt?: boolean
+  noGfm?: boolean
+}
+
+export interface ReadUrlResponse extends ToolResponse {
+  output: {
+    content: string
+  }
+}
+
+export interface SearchParams {
+  q: string
+  apiKey?: string
+  // Pagination
+  num?: number
+  // Site restriction
+  site?: string | string[]
+  // Content options
+  withFavicon?: boolean
+  withImagesummary?: boolean
+  withLinksummary?: boolean
+  retainImages?: 'none' | 'all'
+  noCache?: boolean
+  withGeneratedAlt?: boolean
+  respondWith?: 'no-content'
+  returnFormat?: 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot'
+}
+
+interface SearchResult {
+  title: string
+  description: string
+  url: string
+  content: string
+}
+
+export interface SearchResponse extends ToolResponse {
+  output: {
+    results: SearchResult[]
+  }
+}