import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' const logger = createLogger('FirecrawlAgentTool') const POLL_INTERVAL_MS = 5000 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS export const agentTool: ToolConfig = { id: 'firecrawl_agent', name: 'Firecrawl Agent', description: 'Autonomous web data extraction agent. Searches and gathers information based on natural language prompts without requiring specific URLs.', version: '1.0.0', params: { prompt: { type: 'string', required: true, visibility: 'user-or-llm', description: 'Natural language description of the data to extract (max 10,000 characters)', }, urls: { type: 'json', required: false, visibility: 'user-or-llm', description: 'Optional array of URLs to focus the agent on (e.g., ["https://example.com", "https://docs.example.com"])', }, schema: { type: 'json', required: false, visibility: 'user-or-llm', description: 'JSON Schema defining the structure of data to extract', }, maxCredits: { type: 'number', required: false, visibility: 'user-only', description: 'Maximum credits to spend on this agent task', }, strictConstrainToURLs: { type: 'boolean', required: false, visibility: 'user-only', description: 'If true, agent will only visit URLs provided in the urls array', }, apiKey: { type: 'string', required: true, visibility: 'user-only', description: 'Firecrawl API key', }, }, request: { method: 'POST', url: 'https://api.firecrawl.dev/v2/agent', headers: (params) => ({ 'Content-Type': 'application/json', Authorization: `Bearer ${params.apiKey}`, }), body: (params) => { const body: Record = { prompt: params.prompt, } if (params.urls) { if (Array.isArray(params.urls)) { body.urls = params.urls } else if (typeof params.urls === 'string') { try { const parsed = JSON.parse(params.urls) body.urls = Array.isArray(parsed) ? parsed : [parsed] } catch { body.urls = [params.urls] } } } if (params.schema) body.schema = params.schema if (params.maxCredits) body.maxCredits = Number(params.maxCredits) if (typeof params.strictConstrainToURLs === 'boolean') body.strictConstrainToURLs = params.strictConstrainToURLs return body }, }, transformResponse: async (response: Response) => { const data = await response.json() return { success: true, output: { jobId: data.id, success: false, status: 'processing', data: {}, }, } }, postProcess: async (result, params) => { if (!result.success) { return result } const jobId = result.output.jobId logger.info(`Firecrawl agent job ${jobId} created, polling for completion...`) let elapsedTime = 0 while (elapsedTime < MAX_POLL_TIME_MS) { try { const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, 'Content-Type': 'application/json', }, }) if (!statusResponse.ok) { throw new Error(`Failed to get agent status: ${statusResponse.statusText}`) } const agentData = await statusResponse.json() logger.info(`Firecrawl agent job ${jobId} status: ${agentData.status}`) if (agentData.status === 'completed') { result.output = { jobId, success: true, status: 'completed', data: agentData.data || {}, creditsUsed: agentData.creditsUsed, expiresAt: agentData.expiresAt, sources: agentData.sources, } return result } if (agentData.status === 'failed') { return { ...result, success: false, error: `Agent job failed: ${agentData.error || 'Unknown error'}`, } } await sleep(POLL_INTERVAL_MS) elapsedTime += POLL_INTERVAL_MS } catch (error: any) { logger.error('Error polling for agent job status:', { message: error.message || 'Unknown error', jobId, }) return { ...result, success: false, error: `Error polling for agent job status: ${error.message || 'Unknown error'}`, } } } logger.warn( `Agent job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)` ) return { ...result, success: false, error: `Agent job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`, } }, outputs: { success: { type: 'boolean', description: 'Whether the agent operation was successful', }, status: { type: 'string', description: 'Current status of the agent job (processing, completed, failed)', }, data: { type: 'object', description: 'Extracted data from the agent', }, expiresAt: { type: 'string', description: 'Timestamp when the results expire (24 hours)', }, sources: { type: 'object', description: 'Array of source URLs used by the agent', }, }, }