chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
const logger = createLogger('FirecrawlAgentTool')
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
|
||||
|
||||
export const agentTool: ToolConfig<AgentParams, AgentResponse> = {
|
||||
id: 'firecrawl_agent',
|
||||
name: 'Firecrawl Agent',
|
||||
description:
|
||||
'Autonomous web data extraction agent. Searches and gathers information based on natural language prompts without requiring specific URLs.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Natural language description of the data to extract (max 10,000 characters)',
|
||||
},
|
||||
urls: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Optional array of URLs to focus the agent on (e.g., ["https://example.com", "https://docs.example.com"])',
|
||||
},
|
||||
schema: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'JSON Schema defining the structure of data to extract',
|
||||
},
|
||||
maxCredits: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Maximum credits to spend on this agent task',
|
||||
},
|
||||
strictConstrainToURLs: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'If true, agent will only visit URLs provided in the urls array',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/agent',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
prompt: params.prompt,
|
||||
}
|
||||
|
||||
if (params.urls) {
|
||||
if (Array.isArray(params.urls)) {
|
||||
body.urls = params.urls
|
||||
} else if (typeof params.urls === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(params.urls)
|
||||
body.urls = Array.isArray(parsed) ? parsed : [parsed]
|
||||
} catch {
|
||||
body.urls = [params.urls]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.schema) body.schema = params.schema
|
||||
if (params.maxCredits) body.maxCredits = Number(params.maxCredits)
|
||||
if (typeof params.strictConstrainToURLs === 'boolean')
|
||||
body.strictConstrainToURLs = params.strictConstrainToURLs
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
jobId: data.id,
|
||||
success: false,
|
||||
status: 'processing',
|
||||
data: {},
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
postProcess: async (result, params) => {
|
||||
if (!result.success) {
|
||||
return result
|
||||
}
|
||||
|
||||
const jobId = result.output.jobId
|
||||
logger.info(`Firecrawl agent job ${jobId} created, polling for completion...`)
|
||||
|
||||
let elapsedTime = 0
|
||||
|
||||
while (elapsedTime < MAX_POLL_TIME_MS) {
|
||||
try {
|
||||
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
throw new Error(`Failed to get agent status: ${statusResponse.statusText}`)
|
||||
}
|
||||
|
||||
const agentData = await statusResponse.json()
|
||||
logger.info(`Firecrawl agent job ${jobId} status: ${agentData.status}`)
|
||||
|
||||
if (agentData.status === 'completed') {
|
||||
result.output = {
|
||||
jobId,
|
||||
success: true,
|
||||
status: 'completed',
|
||||
data: agentData.data || {},
|
||||
creditsUsed: agentData.creditsUsed,
|
||||
expiresAt: agentData.expiresAt,
|
||||
sources: agentData.sources,
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
if (agentData.status === 'failed') {
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Agent job failed: ${agentData.error || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(POLL_INTERVAL_MS)
|
||||
elapsedTime += POLL_INTERVAL_MS
|
||||
} catch (error: any) {
|
||||
logger.error('Error polling for agent job status:', {
|
||||
message: error.message || 'Unknown error',
|
||||
jobId,
|
||||
})
|
||||
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Error polling for agent job status: ${error.message || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
`Agent job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
|
||||
)
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Agent job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
success: {
|
||||
type: 'boolean',
|
||||
description: 'Whether the agent operation was successful',
|
||||
},
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Current status of the agent job (processing, completed, failed)',
|
||||
},
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Extracted data from the agent',
|
||||
},
|
||||
expiresAt: {
|
||||
type: 'string',
|
||||
description: 'Timestamp when the results expire (24 hours)',
|
||||
},
|
||||
sources: {
|
||||
type: 'object',
|
||||
description: 'Array of source URLs used by the agent',
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import type {
|
||||
FirecrawlBatchScrapeStatusParams,
|
||||
FirecrawlBatchScrapeStatusResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const batchScrapeStatusTool: ToolConfig<
|
||||
FirecrawlBatchScrapeStatusParams,
|
||||
FirecrawlBatchScrapeStatusResponse
|
||||
> = {
|
||||
id: 'firecrawl_batch_scrape_status',
|
||||
name: 'Firecrawl Batch Scrape Status',
|
||||
description:
|
||||
'Check the status and retrieve results of a previously started Firecrawl batch scrape job by its job ID.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
jobId: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The ID of the batch scrape job to check',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'GET',
|
||||
url: (params) =>
|
||||
`https://api.firecrawl.dev/v2/batch/scrape/${encodeURIComponent(params.jobId.trim())}`,
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
status: data.status,
|
||||
total: data.total ?? 0,
|
||||
completed: data.completed ?? 0,
|
||||
creditsUsed: data.creditsUsed ?? 0,
|
||||
expiresAt: data.expiresAt ?? null,
|
||||
next: data.next ?? null,
|
||||
pages: data.data ?? [],
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Current batch scrape status (scraping, completed, or failed)',
|
||||
},
|
||||
total: { type: 'number', description: 'Total number of pages attempted' },
|
||||
completed: { type: 'number', description: 'Number of pages successfully scraped' },
|
||||
creditsUsed: { type: 'number', description: 'Credits consumed by the batch scrape' },
|
||||
expiresAt: {
|
||||
type: 'string',
|
||||
description: 'ISO timestamp when the batch scrape results expire',
|
||||
optional: true,
|
||||
},
|
||||
next: {
|
||||
type: 'string',
|
||||
description: 'URL to retrieve the next page of results when present',
|
||||
optional: true,
|
||||
},
|
||||
pages: {
|
||||
type: 'array',
|
||||
description: 'Array of scraped pages with their content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
import type {
|
||||
FirecrawlBatchScrapeParams,
|
||||
FirecrawlBatchScrapeResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
const logger = createLogger('FirecrawlBatchScrapeTool')
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
|
||||
|
||||
/**
|
||||
* Normalizes a list of URLs supplied as an array, a JSON-string array, or a
|
||||
* newline-separated string into a trimmed string array.
|
||||
*/
|
||||
function normalizeUrls(value: unknown): string[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((entry) => String(entry).trim()).filter((entry) => entry.length > 0)
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim()
|
||||
if (trimmed === '') return []
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed)
|
||||
if (Array.isArray(parsed)) {
|
||||
return parsed.map((entry) => String(entry).trim()).filter((entry) => entry.length > 0)
|
||||
}
|
||||
} catch {}
|
||||
return trimmed
|
||||
.split(/\r?\n/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0)
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
export const batchScrapeTool: ToolConfig<FirecrawlBatchScrapeParams, FirecrawlBatchScrapeResponse> =
|
||||
{
|
||||
id: 'firecrawl_batch_scrape',
|
||||
name: 'Firecrawl Batch Scrape',
|
||||
description:
|
||||
'Scrape multiple URLs in a single batch job and retrieve structured content from each page.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
urls: {
|
||||
type: 'json',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Array of URLs to scrape (e.g., ["https://example.com/page1", "https://example.com/page2"])',
|
||||
},
|
||||
formats: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Output formats for scraped content (e.g., ["markdown"], ["markdown", "html"])',
|
||||
},
|
||||
onlyMainContent: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Extract only main content from pages',
|
||||
},
|
||||
maxConcurrency: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Maximum number of concurrent scrapes',
|
||||
},
|
||||
ignoreInvalidURLs: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Skip invalid URLs instead of failing the batch (default: true)',
|
||||
},
|
||||
scrapeOptions: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'hidden',
|
||||
description: 'Advanced scraping configuration options',
|
||||
},
|
||||
zeroDataRetention: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Enable zero data retention',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
if (output.creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
const creditsUsed = Number(output.creditsUsed)
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/batch/scrape',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
urls: normalizeUrls(params.urls),
|
||||
}
|
||||
|
||||
const scrapeOptions: Record<string, any> = { ...(params.scrapeOptions ?? {}) }
|
||||
if (params.formats) scrapeOptions.formats = params.formats
|
||||
if (typeof params.onlyMainContent === 'boolean')
|
||||
scrapeOptions.onlyMainContent = params.onlyMainContent
|
||||
if (Object.keys(scrapeOptions).length > 0) {
|
||||
Object.assign(body, scrapeOptions)
|
||||
}
|
||||
|
||||
if (params.maxConcurrency != null) body.maxConcurrency = Number(params.maxConcurrency)
|
||||
if (typeof params.ignoreInvalidURLs === 'boolean')
|
||||
body.ignoreInvalidURLs = params.ignoreInvalidURLs
|
||||
if (typeof params.zeroDataRetention === 'boolean')
|
||||
body.zeroDataRetention = params.zeroDataRetention
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
if (!data.id) {
|
||||
throw new Error(data.error || 'Firecrawl did not return a batch scrape job id to poll')
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
jobId: data.id,
|
||||
invalidURLs: data.invalidURLs ?? [],
|
||||
pages: [],
|
||||
total: 0,
|
||||
completed: 0,
|
||||
creditsUsed: 0,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
postProcess: async (result, params) => {
|
||||
if (!result.success) {
|
||||
return result
|
||||
}
|
||||
|
||||
const jobId = result.output.jobId
|
||||
const invalidURLs = result.output.invalidURLs ?? []
|
||||
logger.info(`Firecrawl batch scrape job ${jobId} created, polling for completion...`)
|
||||
|
||||
let elapsedTime = 0
|
||||
|
||||
while (elapsedTime < MAX_POLL_TIME_MS) {
|
||||
try {
|
||||
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/batch/scrape/${jobId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
throw new Error(`Failed to get batch scrape status: ${statusResponse.statusText}`)
|
||||
}
|
||||
|
||||
const batchData = await statusResponse.json()
|
||||
logger.info(`Firecrawl batch scrape job ${jobId} status: ${batchData.status}`)
|
||||
|
||||
if (batchData.status === 'completed') {
|
||||
result.output = {
|
||||
jobId,
|
||||
invalidURLs,
|
||||
pages: batchData.data ?? [],
|
||||
total: batchData.total ?? 0,
|
||||
completed: batchData.completed ?? 0,
|
||||
creditsUsed: batchData.creditsUsed ?? 0,
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
if (batchData.status === 'failed') {
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Batch scrape job failed: ${batchData.error || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(POLL_INTERVAL_MS)
|
||||
elapsedTime += POLL_INTERVAL_MS
|
||||
} catch (error: any) {
|
||||
logger.error('Error polling for batch scrape job status:', {
|
||||
message: error.message || 'Unknown error',
|
||||
jobId,
|
||||
})
|
||||
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Error polling for batch scrape job status: ${error.message || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
`Batch scrape job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
|
||||
)
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Batch scrape job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
pages: {
|
||||
type: 'array',
|
||||
description: 'Array of scraped pages with their content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
total: { type: 'number', description: 'Total number of pages attempted' },
|
||||
completed: { type: 'number', description: 'Number of pages successfully scraped' },
|
||||
invalidURLs: {
|
||||
type: 'array',
|
||||
description: 'URLs that were skipped because they were invalid',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'Invalid URL' },
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import type {
|
||||
FirecrawlCancelCrawlParams,
|
||||
FirecrawlCancelCrawlResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const cancelCrawlTool: ToolConfig<FirecrawlCancelCrawlParams, FirecrawlCancelCrawlResponse> =
|
||||
{
|
||||
id: 'firecrawl_cancel_crawl',
|
||||
name: 'Firecrawl Cancel Crawl',
|
||||
description: 'Cancel an in-progress Firecrawl crawl job by its job ID.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
jobId: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The ID of the crawl job to cancel',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'DELETE',
|
||||
url: (params) =>
|
||||
`https://api.firecrawl.dev/v2/crawl/${encodeURIComponent(params.jobId.trim())}`,
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
status: data.status ?? 'cancelled',
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Status of the cancelled crawl job (e.g., "cancelled")',
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import type {
|
||||
FirecrawlCrawlStatusParams,
|
||||
FirecrawlCrawlStatusResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const crawlStatusTool: ToolConfig<FirecrawlCrawlStatusParams, FirecrawlCrawlStatusResponse> =
|
||||
{
|
||||
id: 'firecrawl_crawl_status',
|
||||
name: 'Firecrawl Crawl Status',
|
||||
description:
|
||||
'Check the status and retrieve results of a previously started Firecrawl crawl job by its job ID.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
jobId: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The ID of the crawl job to check',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'GET',
|
||||
url: (params) =>
|
||||
`https://api.firecrawl.dev/v2/crawl/${encodeURIComponent(params.jobId.trim())}`,
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
status: data.status,
|
||||
total: data.total ?? 0,
|
||||
completed: data.completed ?? 0,
|
||||
creditsUsed: data.creditsUsed ?? 0,
|
||||
expiresAt: data.expiresAt ?? null,
|
||||
next: data.next ?? null,
|
||||
pages: data.data ?? [],
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Current crawl status (scraping, completed, or failed)',
|
||||
},
|
||||
total: { type: 'number', description: 'Total number of pages attempted' },
|
||||
completed: { type: 'number', description: 'Number of pages successfully crawled' },
|
||||
creditsUsed: { type: 'number', description: 'Credits consumed by the crawl' },
|
||||
expiresAt: {
|
||||
type: 'string',
|
||||
description: 'ISO timestamp when the crawl results expire',
|
||||
optional: true,
|
||||
},
|
||||
next: {
|
||||
type: 'string',
|
||||
description: 'URL to retrieve the next page of results when present',
|
||||
optional: true,
|
||||
},
|
||||
pages: {
|
||||
type: 'array',
|
||||
description: 'Array of crawled pages with their content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
import type { FirecrawlCrawlParams, FirecrawlCrawlResponse } from '@/tools/firecrawl/types'
|
||||
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
const logger = createLogger('FirecrawlCrawlTool')
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
|
||||
|
||||
export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse> = {
|
||||
id: 'firecrawl_crawl',
|
||||
name: 'Firecrawl Crawl',
|
||||
description: 'Crawl entire websites and extract structured content from all accessible pages',
|
||||
version: '1.0.0',
|
||||
params: {
|
||||
url: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'The website URL to crawl (e.g., "https://example.com" or "https://docs.example.com/guide")',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Maximum number of pages to crawl (e.g., 50, 100, 500). Default: 100',
|
||||
},
|
||||
maxDepth: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Maximum depth to crawl from the starting URL (e.g., 1, 2, 3). Controls how many levels deep to follow links',
|
||||
},
|
||||
formats: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Output formats for scraped content (e.g., ["markdown"], ["markdown", "html"], ["markdown", "links"])',
|
||||
},
|
||||
excludePaths: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'URL paths to exclude from crawling (e.g., ["/blog/*", "/admin/*", "/*.pdf"])',
|
||||
},
|
||||
includePaths: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'URL paths to include in crawling (e.g., ["/docs/*", "/api/*"]). Only these paths will be crawled',
|
||||
},
|
||||
onlyMainContent: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Extract only main content from pages',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API Key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
if (output.creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
const creditsUsed = Number(output.creditsUsed)
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: 'https://api.firecrawl.dev/v2/crawl',
|
||||
method: 'POST',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
url: params.url,
|
||||
limit: Number(params.limit) || 100,
|
||||
scrapeOptions: params.scrapeOptions || {
|
||||
formats: params.formats || ['markdown'],
|
||||
onlyMainContent: params.onlyMainContent || false,
|
||||
},
|
||||
}
|
||||
|
||||
if (params.prompt) body.prompt = params.prompt
|
||||
if (params.maxDepth) body.maxDiscoveryDepth = Number(params.maxDepth)
|
||||
if (params.maxDiscoveryDepth) body.maxDiscoveryDepth = Number(params.maxDiscoveryDepth)
|
||||
if (params.sitemap) body.sitemap = params.sitemap
|
||||
if (typeof params.crawlEntireDomain === 'boolean')
|
||||
body.crawlEntireDomain = params.crawlEntireDomain
|
||||
if (typeof params.allowExternalLinks === 'boolean')
|
||||
body.allowExternalLinks = params.allowExternalLinks
|
||||
if (typeof params.allowSubdomains === 'boolean') body.allowSubdomains = params.allowSubdomains
|
||||
if (typeof params.ignoreQueryParameters === 'boolean')
|
||||
body.ignoreQueryParameters = params.ignoreQueryParameters
|
||||
if (params.delay) body.delay = Number(params.delay)
|
||||
if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency)
|
||||
if (params.excludePaths) body.excludePaths = params.excludePaths
|
||||
if (params.includePaths) body.includePaths = params.includePaths
|
||||
if (params.webhook) body.webhook = params.webhook
|
||||
if (typeof params.zeroDataRetention === 'boolean')
|
||||
body.zeroDataRetention = params.zeroDataRetention
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
jobId: data.jobId || data.id,
|
||||
pages: [],
|
||||
total: 0,
|
||||
creditsUsed: 0,
|
||||
},
|
||||
}
|
||||
},
|
||||
postProcess: async (result, params) => {
|
||||
if (!result.success) {
|
||||
return result
|
||||
}
|
||||
|
||||
const jobId = result.output.jobId
|
||||
logger.info(`Firecrawl crawl job ${jobId} created, polling for completion...`)
|
||||
|
||||
let elapsedTime = 0
|
||||
|
||||
while (elapsedTime < MAX_POLL_TIME_MS) {
|
||||
try {
|
||||
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
throw new Error(`Failed to get crawl status: ${statusResponse.statusText}`)
|
||||
}
|
||||
|
||||
const crawlData = await statusResponse.json()
|
||||
logger.info(`Firecrawl crawl job ${jobId} status: ${crawlData.status}`)
|
||||
|
||||
if (crawlData.status === 'completed') {
|
||||
result.output = {
|
||||
pages: crawlData.data || [],
|
||||
total: crawlData.total || 0,
|
||||
creditsUsed: crawlData.creditsUsed || 0,
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
if (crawlData.status === 'failed') {
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Crawl job failed: ${crawlData.error || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(POLL_INTERVAL_MS)
|
||||
elapsedTime += POLL_INTERVAL_MS
|
||||
} catch (error: any) {
|
||||
logger.error('Error polling for crawl job status:', {
|
||||
message: error.message || 'Unknown error',
|
||||
jobId,
|
||||
})
|
||||
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Error polling for crawl job status: ${error.message || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
`Crawl job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
|
||||
)
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Crawl job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
pages: {
|
||||
type: 'array',
|
||||
description: 'Array of crawled pages with their content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
total: { type: 'number', description: 'Total number of pages found during crawl' },
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import type {
|
||||
FirecrawlCreditUsageParams,
|
||||
FirecrawlCreditUsageResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const creditUsageTool: ToolConfig<FirecrawlCreditUsageParams, FirecrawlCreditUsageResponse> =
|
||||
{
|
||||
id: 'firecrawl_credit_usage',
|
||||
name: 'Firecrawl Credit Usage',
|
||||
description: 'Retrieve the remaining and allocated Firecrawl credits for the team.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'GET',
|
||||
url: 'https://api.firecrawl.dev/v2/team/credit-usage',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
const usage = data.data ?? {}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
remainingCredits: usage.remainingCredits ?? null,
|
||||
planCredits: usage.planCredits ?? null,
|
||||
billingPeriodStart: usage.billingPeriodStart ?? null,
|
||||
billingPeriodEnd: usage.billingPeriodEnd ?? null,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
remainingCredits: {
|
||||
type: 'number',
|
||||
description: 'Number of credits remaining for the team',
|
||||
},
|
||||
planCredits: {
|
||||
type: 'number',
|
||||
description: 'Credits allocated in the current plan',
|
||||
optional: true,
|
||||
},
|
||||
billingPeriodStart: {
|
||||
type: 'string',
|
||||
description: 'Start of the current billing period',
|
||||
optional: true,
|
||||
},
|
||||
billingPeriodEnd: {
|
||||
type: 'string',
|
||||
description: 'End of the current billing period',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import type {
|
||||
FirecrawlExtractStatusParams,
|
||||
FirecrawlExtractStatusResponse,
|
||||
} from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const extractStatusTool: ToolConfig<
|
||||
FirecrawlExtractStatusParams,
|
||||
FirecrawlExtractStatusResponse
|
||||
> = {
|
||||
id: 'firecrawl_extract_status',
|
||||
name: 'Firecrawl Extract Status',
|
||||
description:
|
||||
'Check the status and retrieve results of a previously started Firecrawl extract job by its job ID.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
jobId: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The ID of the extract job to check',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'GET',
|
||||
url: (params) =>
|
||||
`https://api.firecrawl.dev/v2/extract/${encodeURIComponent(params.jobId.trim())}`,
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
status: data.status,
|
||||
data: data.data ?? {},
|
||||
expiresAt: data.expiresAt ?? null,
|
||||
creditsUsed: data.creditsUsed ?? null,
|
||||
tokensUsed: data.tokensUsed ?? null,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Current extract status (processing, completed, failed, or cancelled)',
|
||||
},
|
||||
data: {
|
||||
type: 'json',
|
||||
description: 'Extracted structured data according to the schema or prompt',
|
||||
},
|
||||
expiresAt: {
|
||||
type: 'string',
|
||||
description: 'ISO timestamp when the extract results expire',
|
||||
optional: true,
|
||||
},
|
||||
creditsUsed: {
|
||||
type: 'number',
|
||||
description: 'Number of credits used by the extract job',
|
||||
optional: true,
|
||||
},
|
||||
tokensUsed: {
|
||||
type: 'number',
|
||||
description: 'Number of tokens used by the extract job',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
import type { ExtractParams, ExtractResponse } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
const logger = createLogger('FirecrawlExtractTool')
|
||||
|
||||
const POLL_INTERVAL_MS = 5000
|
||||
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
|
||||
|
||||
export const extractTool: ToolConfig<ExtractParams, ExtractResponse> = {
|
||||
id: 'firecrawl_extract',
|
||||
name: 'Firecrawl Extract',
|
||||
description:
|
||||
'Extract structured data from entire webpages using natural language prompts and JSON schema. Powerful agentic feature for intelligent data extraction.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
urls: {
|
||||
type: 'json',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Array of URLs to extract data from (e.g., ["https://example.com/page1", "https://example.com/page2"] or ["https://example.com/*"])',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Natural language guidance for the extraction process',
|
||||
},
|
||||
schema: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'JSON Schema defining the structure of data to extract',
|
||||
},
|
||||
enableWebSearch: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Enable web search to find supplementary information (default: false)',
|
||||
},
|
||||
ignoreSitemap: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Ignore sitemap.xml files during scanning (default: false)',
|
||||
},
|
||||
includeSubdomains: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Extend scanning to subdomains (default: true)',
|
||||
},
|
||||
showSources: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Return data sources in the response (default: false)',
|
||||
},
|
||||
ignoreInvalidURLs: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Skip invalid URLs in the array (default: true)',
|
||||
},
|
||||
scrapeOptions: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'hidden',
|
||||
description: 'Advanced scraping configuration options',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
if (output.creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
const creditsUsed = Number(output.creditsUsed)
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/extract',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
urls: params.urls,
|
||||
}
|
||||
|
||||
if (params.prompt) body.prompt = params.prompt
|
||||
if (params.schema) body.schema = params.schema
|
||||
if (typeof params.enableWebSearch === 'boolean') body.enableWebSearch = params.enableWebSearch
|
||||
if (typeof params.ignoreSitemap === 'boolean') body.ignoreSitemap = params.ignoreSitemap
|
||||
if (typeof params.includeSubdomains === 'boolean')
|
||||
body.includeSubdomains = params.includeSubdomains
|
||||
if (typeof params.showSources === 'boolean') body.showSources = params.showSources
|
||||
if (typeof params.ignoreInvalidURLs === 'boolean')
|
||||
body.ignoreInvalidURLs = params.ignoreInvalidURLs
|
||||
|
||||
if (params.scrapeOptions != null) {
|
||||
const cleanedScrapeOptions = Object.entries(params.scrapeOptions).reduce(
|
||||
(acc, [key, val]) => {
|
||||
if (val != null) {
|
||||
acc[key] = val
|
||||
}
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, any>
|
||||
)
|
||||
if (Object.keys(cleanedScrapeOptions).length > 0) {
|
||||
body.scrapeOptions = cleanedScrapeOptions
|
||||
}
|
||||
}
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
jobId: data.id,
|
||||
success: false,
|
||||
data: {},
|
||||
},
|
||||
}
|
||||
},
|
||||
postProcess: async (result, params) => {
|
||||
if (!result.success) {
|
||||
return result
|
||||
}
|
||||
|
||||
const jobId = result.output.jobId
|
||||
logger.info(`Firecrawl extract job ${jobId} created, polling for completion...`)
|
||||
|
||||
let elapsedTime = 0
|
||||
|
||||
while (elapsedTime < MAX_POLL_TIME_MS) {
|
||||
try {
|
||||
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/extract/${jobId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
throw new Error(`Failed to get extract status: ${statusResponse.statusText}`)
|
||||
}
|
||||
|
||||
const extractData = await statusResponse.json()
|
||||
logger.info(`Firecrawl extract job ${jobId} status: ${extractData.status}`)
|
||||
|
||||
if (extractData.status === 'completed') {
|
||||
result.output = {
|
||||
jobId,
|
||||
success: true,
|
||||
data: extractData.data || {},
|
||||
creditsUsed: extractData.creditsUsed,
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
if (extractData.status === 'failed') {
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Extract job failed: ${extractData.error || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(POLL_INTERVAL_MS)
|
||||
elapsedTime += POLL_INTERVAL_MS
|
||||
} catch (error: any) {
|
||||
logger.error('Error polling for extract job status:', {
|
||||
message: error.message || 'Unknown error',
|
||||
jobId,
|
||||
})
|
||||
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Error polling for extract job status: ${error.message || 'Unknown error'}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
`Extract job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
|
||||
)
|
||||
return {
|
||||
...result,
|
||||
success: false,
|
||||
error: `Extract job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
success: {
|
||||
type: 'boolean',
|
||||
description: 'Whether the extraction operation was successful',
|
||||
},
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Extracted structured data according to the schema or prompt',
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { agentTool } from '@/tools/firecrawl/agent'
|
||||
import { batchScrapeTool } from '@/tools/firecrawl/batch-scrape'
|
||||
import { batchScrapeStatusTool } from '@/tools/firecrawl/batch-scrape-status'
|
||||
import { cancelCrawlTool } from '@/tools/firecrawl/cancel-crawl'
|
||||
import { crawlTool } from '@/tools/firecrawl/crawl'
|
||||
import { crawlStatusTool } from '@/tools/firecrawl/crawl-status'
|
||||
import { creditUsageTool } from '@/tools/firecrawl/credit-usage'
|
||||
import { extractTool } from '@/tools/firecrawl/extract'
|
||||
import { extractStatusTool } from '@/tools/firecrawl/extract-status'
|
||||
import { mapTool } from '@/tools/firecrawl/map'
|
||||
import { parseTool } from '@/tools/firecrawl/parse'
|
||||
import { scrapeTool } from '@/tools/firecrawl/scrape'
|
||||
import { searchTool } from '@/tools/firecrawl/search'
|
||||
|
||||
export const firecrawlScrapeTool = scrapeTool
|
||||
export const firecrawlSearchTool = searchTool
|
||||
export const firecrawlCrawlTool = crawlTool
|
||||
export const firecrawlMapTool = mapTool
|
||||
export const firecrawlExtractTool = extractTool
|
||||
export const firecrawlAgentTool = agentTool
|
||||
export const firecrawlParseTool = parseTool
|
||||
export const firecrawlCrawlStatusTool = crawlStatusTool
|
||||
export const firecrawlCancelCrawlTool = cancelCrawlTool
|
||||
export const firecrawlBatchScrapeTool = batchScrapeTool
|
||||
export const firecrawlBatchScrapeStatusTool = batchScrapeStatusTool
|
||||
export const firecrawlExtractStatusTool = extractStatusTool
|
||||
export const firecrawlCreditUsageTool = creditUsageTool
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { MapParams, MapResponse } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const mapTool: ToolConfig<MapParams, MapResponse> = {
|
||||
id: 'firecrawl_map',
|
||||
name: 'Firecrawl Map',
|
||||
description:
|
||||
'Get a complete list of URLs from any website quickly and reliably. Useful for discovering all pages on a site without crawling them.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
url: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The base URL to map and discover links from (e.g., "https://example.com")',
|
||||
},
|
||||
search: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Filter results by relevance to a search term (e.g., "blog")',
|
||||
},
|
||||
sitemap: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Controls sitemap usage: "skip", "include" (default), or "only"',
|
||||
},
|
||||
includeSubdomains: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Whether to include URLs from subdomains (default: true)',
|
||||
},
|
||||
ignoreQueryParameters: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Exclude URLs containing query strings (default: true)',
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Maximum number of links to return (e.g., 100, 1000, 5000). Max: 100,000, default: 5,000',
|
||||
},
|
||||
timeout: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Request timeout in milliseconds',
|
||||
},
|
||||
location: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'hidden',
|
||||
description: 'Geographic context for proxying (country, languages)',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
if (output.creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
const creditsUsed = Number(output.creditsUsed)
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/map',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
url: params.url,
|
||||
}
|
||||
|
||||
if (params.search) body.search = params.search
|
||||
if (params.sitemap) body.sitemap = params.sitemap
|
||||
if (typeof params.includeSubdomains === 'boolean')
|
||||
body.includeSubdomains = params.includeSubdomains
|
||||
if (typeof params.ignoreQueryParameters === 'boolean')
|
||||
body.ignoreQueryParameters = params.ignoreQueryParameters
|
||||
if (params.limit) body.limit = Number(params.limit)
|
||||
if (params.timeout) body.timeout = Number(params.timeout)
|
||||
if (params.location) body.location = params.location
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: data.success,
|
||||
output: {
|
||||
success: data.success,
|
||||
links: data.links || [],
|
||||
creditsUsed: 1,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
success: {
|
||||
type: 'boolean',
|
||||
description: 'Whether the mapping operation was successful',
|
||||
},
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'Array of discovered URLs from the website',
|
||||
items: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import type { ParseParams, ParseResponse } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const parseTool: ToolConfig<ParseParams, ParseResponse> = {
|
||||
id: 'firecrawl_parse',
|
||||
name: 'Firecrawl Document Parser',
|
||||
description:
|
||||
'Parse uploaded documents (PDF, DOCX, HTML, etc.) into clean markdown using Firecrawl. Supports .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
file: {
|
||||
type: 'file',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Document file to be parsed',
|
||||
},
|
||||
formats: {
|
||||
type: 'array',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Output formats to return (e.g., ["markdown"]). Defaults to markdown.',
|
||||
},
|
||||
onlyMainContent: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Exclude headers, navs, footers. Defaults to true.',
|
||||
},
|
||||
includeTags: {
|
||||
type: 'array',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'HTML tags to include',
|
||||
},
|
||||
excludeTags: {
|
||||
type: 'array',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'HTML tags to exclude',
|
||||
},
|
||||
timeout: {
|
||||
type: 'number',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Timeout in milliseconds (max 300000). Defaults to 30000.',
|
||||
},
|
||||
parsers: {
|
||||
type: 'array',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Parser configuration (e.g., [{ "type": "pdf" }])',
|
||||
},
|
||||
removeBase64Images: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Remove base64 images, keep alt text. Defaults to true.',
|
||||
},
|
||||
blockAds: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Block ads and popups. Defaults to true.',
|
||||
},
|
||||
proxy: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Proxy mode: "basic" or "auto"',
|
||||
},
|
||||
zeroDataRetention: {
|
||||
type: 'boolean',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Enable zero data retention. Defaults to false.',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
const creditsUsed = (output.metadata as { creditsUsed?: number })?.creditsUsed
|
||||
if (creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: '/api/tools/firecrawl/parse',
|
||||
headers: () => ({
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
}),
|
||||
body: (params) => {
|
||||
if (!params.apiKey || typeof params.apiKey !== 'string' || params.apiKey.trim() === '') {
|
||||
throw new Error('Missing or invalid API key: A valid Firecrawl API key is required')
|
||||
}
|
||||
if (!params.file || typeof params.file !== 'object') {
|
||||
throw new Error('File input is required')
|
||||
}
|
||||
|
||||
const options: Record<string, unknown> = {}
|
||||
if (params.formats) options.formats = params.formats
|
||||
if (typeof params.onlyMainContent === 'boolean')
|
||||
options.onlyMainContent = params.onlyMainContent
|
||||
if (params.includeTags) options.includeTags = params.includeTags
|
||||
if (params.excludeTags) options.excludeTags = params.excludeTags
|
||||
if (params.timeout != null) options.timeout = Number(params.timeout)
|
||||
if (params.parsers) options.parsers = params.parsers
|
||||
if (typeof params.removeBase64Images === 'boolean')
|
||||
options.removeBase64Images = params.removeBase64Images
|
||||
if (typeof params.blockAds === 'boolean') options.blockAds = params.blockAds
|
||||
if (params.proxy) options.proxy = params.proxy
|
||||
if (typeof params.zeroDataRetention === 'boolean')
|
||||
options.zeroDataRetention = params.zeroDataRetention
|
||||
|
||||
return {
|
||||
apiKey: params.apiKey,
|
||||
file: params.file,
|
||||
options,
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
if (!data || typeof data !== 'object') {
|
||||
throw new Error('Invalid response format from Firecrawl parse API')
|
||||
}
|
||||
|
||||
const result = data.output ?? data.data ?? data
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
markdown: result.markdown ?? '',
|
||||
summary: result.summary ?? null,
|
||||
html: result.html ?? null,
|
||||
rawHtml: result.rawHtml ?? null,
|
||||
screenshot: result.screenshot ?? null,
|
||||
links: result.links ?? [],
|
||||
metadata: result.metadata ?? null,
|
||||
warning: result.warning ?? null,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
markdown: { type: 'string', description: 'Parsed document content in markdown format' },
|
||||
summary: {
|
||||
type: 'string',
|
||||
description: 'Generated summary of the document',
|
||||
optional: true,
|
||||
},
|
||||
html: {
|
||||
type: 'string',
|
||||
description: 'Processed HTML content',
|
||||
optional: true,
|
||||
},
|
||||
rawHtml: {
|
||||
type: 'string',
|
||||
description: 'Unprocessed raw HTML content',
|
||||
optional: true,
|
||||
},
|
||||
screenshot: {
|
||||
type: 'string',
|
||||
description: 'Screenshot URL or base64 (when requested)',
|
||||
optional: true,
|
||||
},
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'URLs discovered in the document',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'Discovered URL' },
|
||||
},
|
||||
metadata: {
|
||||
type: 'object',
|
||||
description: 'Document metadata',
|
||||
optional: true,
|
||||
properties: {
|
||||
title: { type: 'string', description: 'Document title', optional: true },
|
||||
description: { type: 'string', description: 'Document description', optional: true },
|
||||
language: { type: 'string', description: 'Document language code', optional: true },
|
||||
sourceURL: { type: 'string', description: 'Source URL', optional: true },
|
||||
url: { type: 'string', description: 'Final URL', optional: true },
|
||||
keywords: { type: 'string', description: 'Document keywords', optional: true },
|
||||
statusCode: { type: 'number', description: 'HTTP status code', optional: true },
|
||||
contentType: { type: 'string', description: 'Document content type', optional: true },
|
||||
error: { type: 'string', description: 'Error message if parse failed', optional: true },
|
||||
},
|
||||
},
|
||||
warning: {
|
||||
type: 'string',
|
||||
description: 'Warning message from the parse operation',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { ScrapeParams, ScrapeResponse } from '@/tools/firecrawl/types'
|
||||
import { PAGE_METADATA_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import { safeAssign } from '@/tools/safe-assign'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const scrapeTool: ToolConfig<ScrapeParams, ScrapeResponse> = {
|
||||
id: 'firecrawl_scrape',
|
||||
name: 'Firecrawl Website Scraper',
|
||||
description:
|
||||
'Extract structured content from web pages with comprehensive metadata support. Converts content to markdown or HTML while capturing SEO metadata, Open Graph tags, and page information.',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
url: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The URL to scrape content from (e.g., "https://example.com/page")',
|
||||
},
|
||||
scrapeOptions: {
|
||||
type: 'json',
|
||||
required: false,
|
||||
visibility: 'hidden',
|
||||
description: 'Options for content scraping',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
const creditsUsed = (output.metadata as { creditsUsed?: number })?.creditsUsed
|
||||
if (creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/scrape',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
url: params.url,
|
||||
formats: params.formats || params.scrapeOptions?.formats || ['markdown'],
|
||||
}
|
||||
|
||||
if (typeof params.onlyMainContent === 'boolean') body.onlyMainContent = params.onlyMainContent
|
||||
if (params.includeTags) body.includeTags = params.includeTags
|
||||
if (params.excludeTags) body.excludeTags = params.excludeTags
|
||||
if (params.maxAge) body.maxAge = Number(params.maxAge)
|
||||
if (params.headers) body.headers = params.headers
|
||||
if (params.waitFor) body.waitFor = Number(params.waitFor)
|
||||
if (typeof params.mobile === 'boolean') body.mobile = params.mobile
|
||||
if (typeof params.skipTlsVerification === 'boolean')
|
||||
body.skipTlsVerification = params.skipTlsVerification
|
||||
if (params.timeout) body.timeout = Number(params.timeout)
|
||||
if (params.parsers) body.parsers = params.parsers
|
||||
if (params.actions) body.actions = params.actions
|
||||
if (params.location) body.location = params.location
|
||||
if (typeof params.removeBase64Images === 'boolean')
|
||||
body.removeBase64Images = params.removeBase64Images
|
||||
if (typeof params.blockAds === 'boolean') body.blockAds = params.blockAds
|
||||
if (params.proxy) body.proxy = params.proxy
|
||||
if (typeof params.storeInCache === 'boolean') body.storeInCache = params.storeInCache
|
||||
if (typeof params.zeroDataRetention === 'boolean')
|
||||
body.zeroDataRetention = params.zeroDataRetention
|
||||
|
||||
if (params.scrapeOptions) {
|
||||
safeAssign(body, params.scrapeOptions as Record<string, unknown>)
|
||||
}
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
markdown: data.data.markdown,
|
||||
html: data.data.html,
|
||||
metadata: data.data.metadata,
|
||||
creditsUsed: data.creditsUsed,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
markdown: { type: 'string', description: 'Page content in markdown format' },
|
||||
html: { type: 'string', description: 'Raw HTML content of the page', optional: true },
|
||||
metadata: {
|
||||
type: 'object',
|
||||
description: 'Page metadata including SEO and Open Graph information',
|
||||
properties: PAGE_METADATA_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { SearchParams, SearchResponse } from '@/tools/firecrawl/types'
|
||||
import { SEARCH_RESULT_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
export const searchTool: ToolConfig<SearchParams, SearchResponse> = {
|
||||
id: 'firecrawl_search',
|
||||
name: 'Firecrawl Search',
|
||||
description: 'Search for information on the web using Firecrawl',
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
query: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'The search query to use',
|
||||
},
|
||||
apiKey: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'Firecrawl API key',
|
||||
},
|
||||
},
|
||||
|
||||
hosting: {
|
||||
envKeyPrefix: 'FIRECRAWL_API_KEY',
|
||||
apiKeyParam: 'apiKey',
|
||||
byokProviderId: 'firecrawl',
|
||||
pricing: {
|
||||
type: 'custom',
|
||||
getCost: (_params, output) => {
|
||||
if (output.creditsUsed == null) {
|
||||
throw new Error('Firecrawl response missing creditsUsed field')
|
||||
}
|
||||
|
||||
const creditsUsed = Number(output.creditsUsed)
|
||||
if (Number.isNaN(creditsUsed)) {
|
||||
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
|
||||
}
|
||||
|
||||
return {
|
||||
cost: creditsUsed * 0.001,
|
||||
metadata: { creditsUsed },
|
||||
}
|
||||
},
|
||||
},
|
||||
rateLimit: {
|
||||
mode: 'per_request',
|
||||
requestsPerMinute: 100,
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
method: 'POST',
|
||||
url: 'https://api.firecrawl.dev/v2/search',
|
||||
headers: (params) => ({
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
}),
|
||||
body: (params) => {
|
||||
const body: Record<string, any> = {
|
||||
query: params.query,
|
||||
}
|
||||
|
||||
// Add optional parameters if provided (truthy check filters empty strings, null, undefined)
|
||||
if (params.limit) body.limit = Number(params.limit)
|
||||
if (params.sources) body.sources = params.sources
|
||||
if (params.categories) body.categories = params.categories
|
||||
if (params.tbs) body.tbs = params.tbs
|
||||
if (params.location) body.location = params.location
|
||||
if (params.country) body.country = params.country
|
||||
if (params.timeout) body.timeout = Number(params.timeout)
|
||||
if (typeof params.ignoreInvalidURLs === 'boolean')
|
||||
body.ignoreInvalidURLs = params.ignoreInvalidURLs
|
||||
if (params.scrapeOptions) body.scrapeOptions = params.scrapeOptions
|
||||
|
||||
return body
|
||||
},
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
data: data.data,
|
||||
creditsUsed: data.creditsUsed,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
data: {
|
||||
type: 'array',
|
||||
description: 'Search results data with scraped content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,686 @@
|
||||
import type { OutputProperty, ToolResponse } from '@/tools/types'
|
||||
|
||||
/**
|
||||
* Shared output property definitions for Firecrawl API responses.
|
||||
* Based on Firecrawl API documentation: https://docs.firecrawl.dev/api-reference
|
||||
*
|
||||
* API Response Reference:
|
||||
* - Scrape: https://docs.firecrawl.dev/api-reference/endpoint/scrape
|
||||
* - Crawl: https://docs.firecrawl.dev/api-reference/endpoint/crawl-get
|
||||
* - Search: https://docs.firecrawl.dev/api-reference/endpoint/search
|
||||
* - Map: https://docs.firecrawl.dev/api-reference/endpoint/map
|
||||
* - Extract: https://docs.firecrawl.dev/api-reference/endpoint/extract
|
||||
* - Agent: https://docs.firecrawl.dev/api-reference/endpoint/agent
|
||||
*/
|
||||
|
||||
/**
|
||||
* Output definition for page metadata in scrape responses
|
||||
* Based on Firecrawl metadata object structure from POST /v2/scrape
|
||||
*/
|
||||
export const PAGE_METADATA_OUTPUT_PROPERTIES = {
|
||||
title: { type: 'string', description: 'Page title' },
|
||||
description: { type: 'string', description: 'Page meta description', optional: true },
|
||||
language: { type: 'string', description: 'Page language code (e.g., "en")', optional: true },
|
||||
sourceURL: { type: 'string', description: 'Original source URL that was scraped' },
|
||||
statusCode: { type: 'number', description: 'HTTP status code of the response' },
|
||||
keywords: { type: 'string', description: 'Page meta keywords', optional: true },
|
||||
robots: {
|
||||
type: 'string',
|
||||
description: 'Robots meta directive (e.g., "follow, index")',
|
||||
optional: true,
|
||||
},
|
||||
ogTitle: { type: 'string', description: 'Open Graph title', optional: true },
|
||||
ogDescription: { type: 'string', description: 'Open Graph description', optional: true },
|
||||
ogUrl: { type: 'string', description: 'Open Graph URL', optional: true },
|
||||
ogImage: { type: 'string', description: 'Open Graph image URL', optional: true },
|
||||
ogLocaleAlternate: {
|
||||
type: 'array',
|
||||
description: 'Alternate locale versions for Open Graph',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'Locale code' },
|
||||
},
|
||||
ogSiteName: { type: 'string', description: 'Open Graph site name', optional: true },
|
||||
error: { type: 'string', description: 'Error message if scrape failed', optional: true },
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Complete page metadata output definition
|
||||
*/
|
||||
export const PAGE_METADATA_OUTPUT: OutputProperty = {
|
||||
type: 'object',
|
||||
description: 'Page metadata including SEO and Open Graph information',
|
||||
properties: PAGE_METADATA_OUTPUT_PROPERTIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplified metadata for crawl responses (subset of full metadata)
|
||||
* Based on crawl data[].metadata structure from GET /v2/crawl/{id}
|
||||
*/
|
||||
export const CRAWL_METADATA_OUTPUT_PROPERTIES = {
|
||||
title: { type: 'string', description: 'Page title' },
|
||||
description: { type: 'string', description: 'Page meta description', optional: true },
|
||||
language: { type: 'string', description: 'Page language code', optional: true },
|
||||
sourceURL: { type: 'string', description: 'Original source URL' },
|
||||
statusCode: { type: 'number', description: 'HTTP status code' },
|
||||
ogLocaleAlternate: {
|
||||
type: 'array',
|
||||
description: 'Alternate locale versions',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'Locale code' },
|
||||
},
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Complete crawl metadata output definition
|
||||
*/
|
||||
export const CRAWL_METADATA_OUTPUT: OutputProperty = {
|
||||
type: 'object',
|
||||
description: 'Page metadata from crawl operation',
|
||||
properties: CRAWL_METADATA_OUTPUT_PROPERTIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* Search result metadata properties
|
||||
* Based on search data[].metadata structure from POST /v2/search
|
||||
*/
|
||||
export const SEARCH_METADATA_OUTPUT_PROPERTIES = {
|
||||
title: { type: 'string', description: 'Page title', optional: true },
|
||||
description: { type: 'string', description: 'Page meta description', optional: true },
|
||||
sourceURL: { type: 'string', description: 'Original source URL' },
|
||||
statusCode: { type: 'number', description: 'HTTP status code', optional: true },
|
||||
error: { type: 'string', description: 'Error message if scrape failed', optional: true },
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Complete search metadata output definition
|
||||
*/
|
||||
export const SEARCH_METADATA_OUTPUT: OutputProperty = {
|
||||
type: 'object',
|
||||
description: 'Metadata about the search result page',
|
||||
properties: SEARCH_METADATA_OUTPUT_PROPERTIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* Output properties for scrape tool response
|
||||
* Based on POST /v2/scrape response data object
|
||||
*/
|
||||
export const SCRAPE_OUTPUT_PROPERTIES = {
|
||||
markdown: { type: 'string', description: 'Page content converted to clean markdown format' },
|
||||
html: { type: 'string', description: 'Processed HTML content of the page', optional: true },
|
||||
rawHtml: { type: 'string', description: 'Unprocessed raw HTML content', optional: true },
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'Array of links found on the page',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'URL found on the page' },
|
||||
},
|
||||
screenshot: {
|
||||
type: 'string',
|
||||
description: 'Base64-encoded screenshot or URL (expires after 24 hours)',
|
||||
optional: true,
|
||||
},
|
||||
metadata: PAGE_METADATA_OUTPUT,
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Output properties for crawled page items
|
||||
* Based on GET /v2/crawl/{id} response data[] array items
|
||||
*/
|
||||
export const CRAWLED_PAGE_OUTPUT_PROPERTIES = {
|
||||
markdown: { type: 'string', description: 'Page content in markdown format' },
|
||||
html: { type: 'string', description: 'Processed HTML content of the page', optional: true },
|
||||
rawHtml: { type: 'string', description: 'Unprocessed raw HTML content', optional: true },
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'Array of links found on the page',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'URL found on the page' },
|
||||
},
|
||||
screenshot: {
|
||||
type: 'string',
|
||||
description: 'Screenshot URL (expires after 24 hours)',
|
||||
optional: true,
|
||||
},
|
||||
metadata: CRAWL_METADATA_OUTPUT,
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Complete crawled page output definition
|
||||
*/
|
||||
export const CRAWLED_PAGE_OUTPUT: OutputProperty = {
|
||||
type: 'object',
|
||||
description: 'Crawled page data with content and metadata',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* Output properties for crawl tool response
|
||||
* Based on GET /v2/crawl/{id} response (completed status)
|
||||
*/
|
||||
export const CRAWL_OUTPUT_PROPERTIES = {
|
||||
pages: {
|
||||
type: 'array',
|
||||
description: 'Array of crawled pages with their content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
total: { type: 'number', description: 'Total number of pages found during crawl' },
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Output properties for search result items
|
||||
* Based on POST /v2/search response data[] array items
|
||||
*/
|
||||
export const SEARCH_RESULT_OUTPUT_PROPERTIES = {
|
||||
title: { type: 'string', description: 'Search result title from search engine' },
|
||||
description: {
|
||||
type: 'string',
|
||||
description: 'Search result description/snippet from search engine',
|
||||
},
|
||||
url: { type: 'string', description: 'URL of the search result' },
|
||||
markdown: {
|
||||
type: 'string',
|
||||
description: 'Page content in markdown (when scrapeOptions.formats includes "markdown")',
|
||||
optional: true,
|
||||
},
|
||||
html: {
|
||||
type: 'string',
|
||||
description: 'Processed HTML content (when scrapeOptions.formats includes "html")',
|
||||
optional: true,
|
||||
},
|
||||
rawHtml: {
|
||||
type: 'string',
|
||||
description: 'Unprocessed raw HTML (when scrapeOptions.formats includes "rawHtml")',
|
||||
optional: true,
|
||||
},
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'Links found on the page (when scrapeOptions.formats includes "links")',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'URL found on the page' },
|
||||
},
|
||||
screenshot: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Screenshot URL (expires after 24 hours, when scrapeOptions.formats includes "screenshot")',
|
||||
optional: true,
|
||||
},
|
||||
metadata: SEARCH_METADATA_OUTPUT,
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Complete search result output definition
|
||||
*/
|
||||
export const SEARCH_RESULT_OUTPUT: OutputProperty = {
|
||||
type: 'object',
|
||||
description: 'Search result item with optional scraped content',
|
||||
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
|
||||
}
|
||||
|
||||
/**
|
||||
* Output properties for search tool response
|
||||
* Based on POST /v2/search response
|
||||
*/
|
||||
export const SEARCH_OUTPUT_PROPERTIES = {
|
||||
data: {
|
||||
type: 'array',
|
||||
description: 'Array of search results with scraped content and metadata',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
|
||||
},
|
||||
},
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Output properties for map tool response
|
||||
* Based on POST /v2/map response
|
||||
*/
|
||||
export const MAP_OUTPUT_PROPERTIES = {
|
||||
success: { type: 'boolean', description: 'Whether the mapping operation completed successfully' },
|
||||
links: {
|
||||
type: 'array',
|
||||
description: 'Array of discovered URLs from the website',
|
||||
items: { type: 'string', description: 'Discovered URL' },
|
||||
},
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Output properties for extract tool response
|
||||
* Based on GET /v2/extract/{id} response (completed status)
|
||||
*/
|
||||
export const EXTRACT_OUTPUT_PROPERTIES = {
|
||||
success: { type: 'boolean', description: 'Whether the extraction completed successfully' },
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Extracted structured data according to the provided schema or prompt',
|
||||
},
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
/**
|
||||
* Output properties for agent tool response
|
||||
* Based on GET /v2/agent/{id} response (completed status)
|
||||
*/
|
||||
export const AGENT_OUTPUT_PROPERTIES = {
|
||||
success: { type: 'boolean', description: 'Whether the agent task completed successfully' },
|
||||
status: {
|
||||
type: 'string',
|
||||
description: 'Current status of the agent job (processing, completed, failed)',
|
||||
},
|
||||
data: {
|
||||
type: 'object',
|
||||
description: 'Extracted data from the agent based on the prompt and schema',
|
||||
},
|
||||
expiresAt: {
|
||||
type: 'string',
|
||||
description: 'ISO timestamp when the results expire (24 hours after completion)',
|
||||
optional: true,
|
||||
},
|
||||
sources: {
|
||||
type: 'array',
|
||||
description: 'Array of source URLs visited and used by the agent',
|
||||
optional: true,
|
||||
items: { type: 'string', description: 'Source URL' },
|
||||
},
|
||||
} as const satisfies Record<string, OutputProperty>
|
||||
|
||||
// Common types
|
||||
interface LocationConfig {
|
||||
country?: string
|
||||
languages?: string[]
|
||||
}
|
||||
|
||||
interface ScrapeOptions {
|
||||
formats?: string[]
|
||||
onlyMainContent?: boolean
|
||||
includeTags?: string[]
|
||||
excludeTags?: string[]
|
||||
maxAge?: number
|
||||
headers?: Record<string, string>
|
||||
waitFor?: number
|
||||
mobile?: boolean
|
||||
skipTlsVerification?: boolean
|
||||
timeout?: number
|
||||
parsers?: string[]
|
||||
actions?: Array<{
|
||||
type: string
|
||||
[key: string]: any
|
||||
}>
|
||||
location?: LocationConfig
|
||||
removeBase64Images?: boolean
|
||||
blockAds?: boolean
|
||||
proxy?: 'basic' | 'stealth' | 'auto'
|
||||
storeInCache?: boolean
|
||||
}
|
||||
|
||||
export interface ScrapeParams {
|
||||
apiKey: string
|
||||
url: string
|
||||
scrapeOptions?: ScrapeOptions
|
||||
// Additional top-level scrape params
|
||||
onlyMainContent?: boolean
|
||||
formats?: string[]
|
||||
includeTags?: string[]
|
||||
excludeTags?: string[]
|
||||
maxAge?: number
|
||||
headers?: Record<string, string>
|
||||
waitFor?: number
|
||||
mobile?: boolean
|
||||
skipTlsVerification?: boolean
|
||||
timeout?: number
|
||||
parsers?: string[]
|
||||
actions?: Array<{
|
||||
type: string
|
||||
[key: string]: any
|
||||
}>
|
||||
location?: LocationConfig
|
||||
removeBase64Images?: boolean
|
||||
blockAds?: boolean
|
||||
proxy?: 'basic' | 'stealth' | 'auto'
|
||||
storeInCache?: boolean
|
||||
zeroDataRetention?: boolean
|
||||
}
|
||||
|
||||
export interface SearchParams {
|
||||
apiKey: string
|
||||
query: string
|
||||
limit?: number
|
||||
sources?: ('web' | 'images' | 'news')[]
|
||||
categories?: ('github' | 'research' | 'pdf')[]
|
||||
tbs?: string
|
||||
location?: string
|
||||
country?: string
|
||||
timeout?: number
|
||||
ignoreInvalidURLs?: boolean
|
||||
scrapeOptions?: ScrapeOptions
|
||||
}
|
||||
|
||||
export interface FirecrawlCrawlParams {
|
||||
apiKey: string
|
||||
url: string
|
||||
limit?: number
|
||||
maxDepth?: number
|
||||
formats?: string[]
|
||||
onlyMainContent?: boolean
|
||||
prompt?: string
|
||||
maxDiscoveryDepth?: number
|
||||
sitemap?: 'skip' | 'include'
|
||||
crawlEntireDomain?: boolean
|
||||
allowExternalLinks?: boolean
|
||||
allowSubdomains?: boolean
|
||||
ignoreQueryParameters?: boolean
|
||||
delay?: number
|
||||
maxConcurrency?: number
|
||||
excludePaths?: string[]
|
||||
includePaths?: string[]
|
||||
webhook?: {
|
||||
url: string
|
||||
headers?: Record<string, string>
|
||||
metadata?: Record<string, any>
|
||||
events?: ('completed' | 'page' | 'failed' | 'started')[]
|
||||
}
|
||||
scrapeOptions?: ScrapeOptions
|
||||
zeroDataRetention?: boolean
|
||||
}
|
||||
|
||||
export interface MapParams {
|
||||
apiKey: string
|
||||
url: string
|
||||
search?: string
|
||||
sitemap?: 'skip' | 'include' | 'only'
|
||||
includeSubdomains?: boolean
|
||||
ignoreQueryParameters?: boolean
|
||||
limit?: number
|
||||
timeout?: number
|
||||
location?: LocationConfig
|
||||
}
|
||||
|
||||
export interface ExtractParams {
|
||||
apiKey: string
|
||||
urls: string[]
|
||||
prompt?: string
|
||||
schema?: Record<string, any>
|
||||
enableWebSearch?: boolean
|
||||
ignoreSitemap?: boolean
|
||||
includeSubdomains?: boolean
|
||||
showSources?: boolean
|
||||
ignoreInvalidURLs?: boolean
|
||||
scrapeOptions?: ScrapeOptions
|
||||
}
|
||||
|
||||
export interface AgentParams {
|
||||
apiKey: string
|
||||
prompt: string
|
||||
urls?: string[]
|
||||
schema?: Record<string, any>
|
||||
maxCredits?: number
|
||||
strictConstrainToURLs?: boolean
|
||||
}
|
||||
|
||||
export interface ScrapeResponse extends ToolResponse {
|
||||
output: {
|
||||
markdown: string
|
||||
html?: string
|
||||
rawHtml?: string
|
||||
links?: string[]
|
||||
screenshot?: string
|
||||
metadata: {
|
||||
title: string
|
||||
description?: string
|
||||
language?: string
|
||||
keywords?: string
|
||||
robots?: string
|
||||
ogTitle?: string
|
||||
ogDescription?: string
|
||||
ogUrl?: string
|
||||
ogImage?: string
|
||||
ogLocaleAlternate?: string[]
|
||||
ogSiteName?: string
|
||||
sourceURL: string
|
||||
statusCode: number
|
||||
error?: string
|
||||
}
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface SearchResponse extends ToolResponse {
|
||||
output: {
|
||||
data: Array<{
|
||||
title: string
|
||||
description: string
|
||||
url: string
|
||||
markdown?: string
|
||||
html?: string
|
||||
rawHtml?: string
|
||||
links?: string[]
|
||||
screenshot?: string
|
||||
metadata: {
|
||||
title?: string
|
||||
description?: string
|
||||
sourceURL: string
|
||||
statusCode?: number
|
||||
error?: string
|
||||
}
|
||||
}>
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlCrawlResponse extends ToolResponse {
|
||||
output: {
|
||||
jobId?: string
|
||||
pages: Array<{
|
||||
markdown: string
|
||||
html?: string
|
||||
rawHtml?: string
|
||||
links?: string[]
|
||||
screenshot?: string
|
||||
metadata: {
|
||||
title: string
|
||||
description?: string
|
||||
language?: string
|
||||
sourceURL: string
|
||||
statusCode: number
|
||||
ogLocaleAlternate?: string[]
|
||||
}
|
||||
}>
|
||||
total: number
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface MapResponse extends ToolResponse {
|
||||
output: {
|
||||
success: boolean
|
||||
links: string[]
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExtractResponse extends ToolResponse {
|
||||
output: {
|
||||
jobId: string
|
||||
success: boolean
|
||||
data: Record<string, any>
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface AgentResponse extends ToolResponse {
|
||||
output: {
|
||||
jobId: string
|
||||
success: boolean
|
||||
status: string
|
||||
data: Record<string, any>
|
||||
creditsUsed?: number
|
||||
expiresAt?: string
|
||||
sources?: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface ParseParams {
|
||||
apiKey: string
|
||||
file: unknown
|
||||
formats?: Array<{ type: string } | string>
|
||||
onlyMainContent?: boolean
|
||||
includeTags?: string[]
|
||||
excludeTags?: string[]
|
||||
timeout?: number
|
||||
parsers?: Array<{ type: string; mode?: string } | string>
|
||||
removeBase64Images?: boolean
|
||||
blockAds?: boolean
|
||||
proxy?: 'basic' | 'auto'
|
||||
zeroDataRetention?: boolean
|
||||
}
|
||||
|
||||
export interface ParseResponse extends ToolResponse {
|
||||
output: {
|
||||
markdown: string
|
||||
summary?: string | null
|
||||
html?: string | null
|
||||
rawHtml?: string | null
|
||||
screenshot?: string | null
|
||||
links?: string[]
|
||||
metadata?: {
|
||||
title?: string | string[]
|
||||
description?: string | string[]
|
||||
language?: string | string[] | null
|
||||
sourceURL?: string
|
||||
url?: string
|
||||
keywords?: string | string[]
|
||||
statusCode?: number
|
||||
contentType?: string
|
||||
error?: string | null
|
||||
} | null
|
||||
warning?: string | null
|
||||
}
|
||||
}
|
||||
|
||||
interface CrawledPage {
|
||||
markdown: string
|
||||
html?: string
|
||||
rawHtml?: string
|
||||
links?: string[]
|
||||
screenshot?: string
|
||||
metadata: {
|
||||
title: string
|
||||
description?: string
|
||||
language?: string
|
||||
sourceURL: string
|
||||
statusCode: number
|
||||
ogLocaleAlternate?: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlCrawlStatusParams {
|
||||
apiKey: string
|
||||
jobId: string
|
||||
}
|
||||
|
||||
export interface FirecrawlCrawlStatusResponse extends ToolResponse {
|
||||
output: {
|
||||
status: string
|
||||
total: number
|
||||
completed: number
|
||||
creditsUsed: number
|
||||
expiresAt?: string | null
|
||||
next?: string | null
|
||||
pages: CrawledPage[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlCancelCrawlParams {
|
||||
apiKey: string
|
||||
jobId: string
|
||||
}
|
||||
|
||||
export interface FirecrawlCancelCrawlResponse extends ToolResponse {
|
||||
output: {
|
||||
status: string
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlBatchScrapeParams {
|
||||
apiKey: string
|
||||
urls: string[] | string
|
||||
formats?: string[]
|
||||
onlyMainContent?: boolean
|
||||
maxConcurrency?: number
|
||||
ignoreInvalidURLs?: boolean
|
||||
scrapeOptions?: ScrapeOptions
|
||||
zeroDataRetention?: boolean
|
||||
}
|
||||
|
||||
export interface FirecrawlBatchScrapeResponse extends ToolResponse {
|
||||
output: {
|
||||
jobId?: string
|
||||
invalidURLs?: string[]
|
||||
pages: CrawledPage[]
|
||||
total: number
|
||||
completed: number
|
||||
creditsUsed?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlBatchScrapeStatusParams {
|
||||
apiKey: string
|
||||
jobId: string
|
||||
}
|
||||
|
||||
export interface FirecrawlBatchScrapeStatusResponse extends ToolResponse {
|
||||
output: {
|
||||
status: string
|
||||
total: number
|
||||
completed: number
|
||||
creditsUsed: number
|
||||
expiresAt?: string | null
|
||||
next?: string | null
|
||||
pages: CrawledPage[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlExtractStatusParams {
|
||||
apiKey: string
|
||||
jobId: string
|
||||
}
|
||||
|
||||
export interface FirecrawlExtractStatusResponse extends ToolResponse {
|
||||
output: {
|
||||
status: string
|
||||
data: Record<string, any> | unknown[]
|
||||
expiresAt?: string | null
|
||||
creditsUsed?: number | null
|
||||
tokensUsed?: number | null
|
||||
}
|
||||
}
|
||||
|
||||
export interface FirecrawlCreditUsageParams {
|
||||
apiKey: string
|
||||
}
|
||||
|
||||
export interface FirecrawlCreditUsageResponse extends ToolResponse {
|
||||
output: {
|
||||
remainingCredits: number | null
|
||||
planCredits?: number | null
|
||||
billingPeriodStart?: string | null
|
||||
billingPeriodEnd?: string | null
|
||||
}
|
||||
}
|
||||
|
||||
export type FirecrawlResponse =
|
||||
| ScrapeResponse
|
||||
| SearchResponse
|
||||
| FirecrawlCrawlResponse
|
||||
| MapResponse
|
||||
| ExtractResponse
|
||||
| AgentResponse
|
||||
| ParseResponse
|
||||
| FirecrawlCrawlStatusResponse
|
||||
| FirecrawlCancelCrawlResponse
|
||||
| FirecrawlBatchScrapeResponse
|
||||
| FirecrawlBatchScrapeStatusResponse
|
||||
| FirecrawlExtractStatusResponse
|
||||
| FirecrawlCreditUsageResponse
|
||||
Reference in New Issue
Block a user