chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
+202
View File
@@ -0,0 +1,202 @@
import { createLogger } from '@sim/logger'
import { sleep } from '@sim/utils/helpers'
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('FirecrawlAgentTool')
const POLL_INTERVAL_MS = 5000
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
export const agentTool: ToolConfig<AgentParams, AgentResponse> = {
id: 'firecrawl_agent',
name: 'Firecrawl Agent',
description:
'Autonomous web data extraction agent. Searches and gathers information based on natural language prompts without requiring specific URLs.',
version: '1.0.0',
params: {
prompt: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'Natural language description of the data to extract (max 10,000 characters)',
},
urls: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description:
'Optional array of URLs to focus the agent on (e.g., ["https://example.com", "https://docs.example.com"])',
},
schema: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description: 'JSON Schema defining the structure of data to extract',
},
maxCredits: {
type: 'number',
required: false,
visibility: 'user-only',
description: 'Maximum credits to spend on this agent task',
},
strictConstrainToURLs: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'If true, agent will only visit URLs provided in the urls array',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/agent',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
prompt: params.prompt,
}
if (params.urls) {
if (Array.isArray(params.urls)) {
body.urls = params.urls
} else if (typeof params.urls === 'string') {
try {
const parsed = JSON.parse(params.urls)
body.urls = Array.isArray(parsed) ? parsed : [parsed]
} catch {
body.urls = [params.urls]
}
}
}
if (params.schema) body.schema = params.schema
if (params.maxCredits) body.maxCredits = Number(params.maxCredits)
if (typeof params.strictConstrainToURLs === 'boolean')
body.strictConstrainToURLs = params.strictConstrainToURLs
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
jobId: data.id,
success: false,
status: 'processing',
data: {},
},
}
},
postProcess: async (result, params) => {
if (!result.success) {
return result
}
const jobId = result.output.jobId
logger.info(`Firecrawl agent job ${jobId} created, polling for completion...`)
let elapsedTime = 0
while (elapsedTime < MAX_POLL_TIME_MS) {
try {
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${params.apiKey}`,
'Content-Type': 'application/json',
},
})
if (!statusResponse.ok) {
throw new Error(`Failed to get agent status: ${statusResponse.statusText}`)
}
const agentData = await statusResponse.json()
logger.info(`Firecrawl agent job ${jobId} status: ${agentData.status}`)
if (agentData.status === 'completed') {
result.output = {
jobId,
success: true,
status: 'completed',
data: agentData.data || {},
creditsUsed: agentData.creditsUsed,
expiresAt: agentData.expiresAt,
sources: agentData.sources,
}
return result
}
if (agentData.status === 'failed') {
return {
...result,
success: false,
error: `Agent job failed: ${agentData.error || 'Unknown error'}`,
}
}
await sleep(POLL_INTERVAL_MS)
elapsedTime += POLL_INTERVAL_MS
} catch (error: any) {
logger.error('Error polling for agent job status:', {
message: error.message || 'Unknown error',
jobId,
})
return {
...result,
success: false,
error: `Error polling for agent job status: ${error.message || 'Unknown error'}`,
}
}
}
logger.warn(
`Agent job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
)
return {
...result,
success: false,
error: `Agent job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
}
},
outputs: {
success: {
type: 'boolean',
description: 'Whether the agent operation was successful',
},
status: {
type: 'string',
description: 'Current status of the agent job (processing, completed, failed)',
},
data: {
type: 'object',
description: 'Extracted data from the agent',
},
expiresAt: {
type: 'string',
description: 'Timestamp when the results expire (24 hours)',
},
sources: {
type: 'object',
description: 'Array of source URLs used by the agent',
},
},
}
@@ -0,0 +1,87 @@
import type {
FirecrawlBatchScrapeStatusParams,
FirecrawlBatchScrapeStatusResponse,
} from '@/tools/firecrawl/types'
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const batchScrapeStatusTool: ToolConfig<
FirecrawlBatchScrapeStatusParams,
FirecrawlBatchScrapeStatusResponse
> = {
id: 'firecrawl_batch_scrape_status',
name: 'Firecrawl Batch Scrape Status',
description:
'Check the status and retrieve results of a previously started Firecrawl batch scrape job by its job ID.',
version: '1.0.0',
params: {
jobId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The ID of the batch scrape job to check',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'GET',
url: (params) =>
`https://api.firecrawl.dev/v2/batch/scrape/${encodeURIComponent(params.jobId.trim())}`,
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
status: data.status,
total: data.total ?? 0,
completed: data.completed ?? 0,
creditsUsed: data.creditsUsed ?? 0,
expiresAt: data.expiresAt ?? null,
next: data.next ?? null,
pages: data.data ?? [],
},
}
},
outputs: {
status: {
type: 'string',
description: 'Current batch scrape status (scraping, completed, or failed)',
},
total: { type: 'number', description: 'Total number of pages attempted' },
completed: { type: 'number', description: 'Number of pages successfully scraped' },
creditsUsed: { type: 'number', description: 'Credits consumed by the batch scrape' },
expiresAt: {
type: 'string',
description: 'ISO timestamp when the batch scrape results expire',
optional: true,
},
next: {
type: 'string',
description: 'URL to retrieve the next page of results when present',
optional: true,
},
pages: {
type: 'array',
description: 'Array of scraped pages with their content and metadata',
items: {
type: 'object',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
},
},
},
}
+274
View File
@@ -0,0 +1,274 @@
import { createLogger } from '@sim/logger'
import { sleep } from '@sim/utils/helpers'
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
import type {
FirecrawlBatchScrapeParams,
FirecrawlBatchScrapeResponse,
} from '@/tools/firecrawl/types'
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('FirecrawlBatchScrapeTool')
const POLL_INTERVAL_MS = 5000
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
/**
* Normalizes a list of URLs supplied as an array, a JSON-string array, or a
* newline-separated string into a trimmed string array.
*/
function normalizeUrls(value: unknown): string[] {
if (Array.isArray(value)) {
return value.map((entry) => String(entry).trim()).filter((entry) => entry.length > 0)
}
if (typeof value === 'string') {
const trimmed = value.trim()
if (trimmed === '') return []
try {
const parsed = JSON.parse(trimmed)
if (Array.isArray(parsed)) {
return parsed.map((entry) => String(entry).trim()).filter((entry) => entry.length > 0)
}
} catch {}
return trimmed
.split(/\r?\n/)
.map((entry) => entry.trim())
.filter((entry) => entry.length > 0)
}
return []
}
export const batchScrapeTool: ToolConfig<FirecrawlBatchScrapeParams, FirecrawlBatchScrapeResponse> =
{
id: 'firecrawl_batch_scrape',
name: 'Firecrawl Batch Scrape',
description:
'Scrape multiple URLs in a single batch job and retrieve structured content from each page.',
version: '1.0.0',
params: {
urls: {
type: 'json',
required: true,
visibility: 'user-or-llm',
description:
'Array of URLs to scrape (e.g., ["https://example.com/page1", "https://example.com/page2"])',
},
formats: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description:
'Output formats for scraped content (e.g., ["markdown"], ["markdown", "html"])',
},
onlyMainContent: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Extract only main content from pages',
},
maxConcurrency: {
type: 'number',
required: false,
visibility: 'user-only',
description: 'Maximum number of concurrent scrapes',
},
ignoreInvalidURLs: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Skip invalid URLs instead of failing the batch (default: true)',
},
scrapeOptions: {
type: 'json',
required: false,
visibility: 'hidden',
description: 'Advanced scraping configuration options',
},
zeroDataRetention: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Enable zero data retention',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
if (output.creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
const creditsUsed = Number(output.creditsUsed)
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/batch/scrape',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
urls: normalizeUrls(params.urls),
}
const scrapeOptions: Record<string, any> = { ...(params.scrapeOptions ?? {}) }
if (params.formats) scrapeOptions.formats = params.formats
if (typeof params.onlyMainContent === 'boolean')
scrapeOptions.onlyMainContent = params.onlyMainContent
if (Object.keys(scrapeOptions).length > 0) {
Object.assign(body, scrapeOptions)
}
if (params.maxConcurrency != null) body.maxConcurrency = Number(params.maxConcurrency)
if (typeof params.ignoreInvalidURLs === 'boolean')
body.ignoreInvalidURLs = params.ignoreInvalidURLs
if (typeof params.zeroDataRetention === 'boolean')
body.zeroDataRetention = params.zeroDataRetention
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
if (!data.id) {
throw new Error(data.error || 'Firecrawl did not return a batch scrape job id to poll')
}
return {
success: true,
output: {
jobId: data.id,
invalidURLs: data.invalidURLs ?? [],
pages: [],
total: 0,
completed: 0,
creditsUsed: 0,
},
}
},
postProcess: async (result, params) => {
if (!result.success) {
return result
}
const jobId = result.output.jobId
const invalidURLs = result.output.invalidURLs ?? []
logger.info(`Firecrawl batch scrape job ${jobId} created, polling for completion...`)
let elapsedTime = 0
while (elapsedTime < MAX_POLL_TIME_MS) {
try {
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/batch/scrape/${jobId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${params.apiKey}`,
'Content-Type': 'application/json',
},
})
if (!statusResponse.ok) {
throw new Error(`Failed to get batch scrape status: ${statusResponse.statusText}`)
}
const batchData = await statusResponse.json()
logger.info(`Firecrawl batch scrape job ${jobId} status: ${batchData.status}`)
if (batchData.status === 'completed') {
result.output = {
jobId,
invalidURLs,
pages: batchData.data ?? [],
total: batchData.total ?? 0,
completed: batchData.completed ?? 0,
creditsUsed: batchData.creditsUsed ?? 0,
}
return result
}
if (batchData.status === 'failed') {
return {
...result,
success: false,
error: `Batch scrape job failed: ${batchData.error || 'Unknown error'}`,
}
}
await sleep(POLL_INTERVAL_MS)
elapsedTime += POLL_INTERVAL_MS
} catch (error: any) {
logger.error('Error polling for batch scrape job status:', {
message: error.message || 'Unknown error',
jobId,
})
return {
...result,
success: false,
error: `Error polling for batch scrape job status: ${error.message || 'Unknown error'}`,
}
}
}
logger.warn(
`Batch scrape job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
)
return {
...result,
success: false,
error: `Batch scrape job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
}
},
outputs: {
pages: {
type: 'array',
description: 'Array of scraped pages with their content and metadata',
items: {
type: 'object',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
},
},
total: { type: 'number', description: 'Total number of pages attempted' },
completed: { type: 'number', description: 'Number of pages successfully scraped' },
invalidURLs: {
type: 'array',
description: 'URLs that were skipped because they were invalid',
optional: true,
items: { type: 'string', description: 'Invalid URL' },
},
},
}
+56
View File
@@ -0,0 +1,56 @@
import type {
FirecrawlCancelCrawlParams,
FirecrawlCancelCrawlResponse,
} from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const cancelCrawlTool: ToolConfig<FirecrawlCancelCrawlParams, FirecrawlCancelCrawlResponse> =
{
id: 'firecrawl_cancel_crawl',
name: 'Firecrawl Cancel Crawl',
description: 'Cancel an in-progress Firecrawl crawl job by its job ID.',
version: '1.0.0',
params: {
jobId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The ID of the crawl job to cancel',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'DELETE',
url: (params) =>
`https://api.firecrawl.dev/v2/crawl/${encodeURIComponent(params.jobId.trim())}`,
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
status: data.status ?? 'cancelled',
},
}
},
outputs: {
status: {
type: 'string',
description: 'Status of the cancelled crawl job (e.g., "cancelled")',
},
},
}
+85
View File
@@ -0,0 +1,85 @@
import type {
FirecrawlCrawlStatusParams,
FirecrawlCrawlStatusResponse,
} from '@/tools/firecrawl/types'
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const crawlStatusTool: ToolConfig<FirecrawlCrawlStatusParams, FirecrawlCrawlStatusResponse> =
{
id: 'firecrawl_crawl_status',
name: 'Firecrawl Crawl Status',
description:
'Check the status and retrieve results of a previously started Firecrawl crawl job by its job ID.',
version: '1.0.0',
params: {
jobId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The ID of the crawl job to check',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'GET',
url: (params) =>
`https://api.firecrawl.dev/v2/crawl/${encodeURIComponent(params.jobId.trim())}`,
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
status: data.status,
total: data.total ?? 0,
completed: data.completed ?? 0,
creditsUsed: data.creditsUsed ?? 0,
expiresAt: data.expiresAt ?? null,
next: data.next ?? null,
pages: data.data ?? [],
},
}
},
outputs: {
status: {
type: 'string',
description: 'Current crawl status (scraping, completed, or failed)',
},
total: { type: 'number', description: 'Total number of pages attempted' },
completed: { type: 'number', description: 'Number of pages successfully crawled' },
creditsUsed: { type: 'number', description: 'Credits consumed by the crawl' },
expiresAt: {
type: 'string',
description: 'ISO timestamp when the crawl results expire',
optional: true,
},
next: {
type: 'string',
description: 'URL to retrieve the next page of results when present',
optional: true,
},
pages: {
type: 'array',
description: 'Array of crawled pages with their content and metadata',
items: {
type: 'object',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
},
},
},
}
+234
View File
@@ -0,0 +1,234 @@
import { createLogger } from '@sim/logger'
import { sleep } from '@sim/utils/helpers'
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
import type { FirecrawlCrawlParams, FirecrawlCrawlResponse } from '@/tools/firecrawl/types'
import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('FirecrawlCrawlTool')
const POLL_INTERVAL_MS = 5000
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
export const crawlTool: ToolConfig<FirecrawlCrawlParams, FirecrawlCrawlResponse> = {
id: 'firecrawl_crawl',
name: 'Firecrawl Crawl',
description: 'Crawl entire websites and extract structured content from all accessible pages',
version: '1.0.0',
params: {
url: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'The website URL to crawl (e.g., "https://example.com" or "https://docs.example.com/guide")',
},
limit: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Maximum number of pages to crawl (e.g., 50, 100, 500). Default: 100',
},
maxDepth: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description:
'Maximum depth to crawl from the starting URL (e.g., 1, 2, 3). Controls how many levels deep to follow links',
},
formats: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description:
'Output formats for scraped content (e.g., ["markdown"], ["markdown", "html"], ["markdown", "links"])',
},
excludePaths: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description: 'URL paths to exclude from crawling (e.g., ["/blog/*", "/admin/*", "/*.pdf"])',
},
includePaths: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description:
'URL paths to include in crawling (e.g., ["/docs/*", "/api/*"]). Only these paths will be crawled',
},
onlyMainContent: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Extract only main content from pages',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API Key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
if (output.creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
const creditsUsed = Number(output.creditsUsed)
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
url: 'https://api.firecrawl.dev/v2/crawl',
method: 'POST',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
url: params.url,
limit: Number(params.limit) || 100,
scrapeOptions: params.scrapeOptions || {
formats: params.formats || ['markdown'],
onlyMainContent: params.onlyMainContent || false,
},
}
if (params.prompt) body.prompt = params.prompt
if (params.maxDepth) body.maxDiscoveryDepth = Number(params.maxDepth)
if (params.maxDiscoveryDepth) body.maxDiscoveryDepth = Number(params.maxDiscoveryDepth)
if (params.sitemap) body.sitemap = params.sitemap
if (typeof params.crawlEntireDomain === 'boolean')
body.crawlEntireDomain = params.crawlEntireDomain
if (typeof params.allowExternalLinks === 'boolean')
body.allowExternalLinks = params.allowExternalLinks
if (typeof params.allowSubdomains === 'boolean') body.allowSubdomains = params.allowSubdomains
if (typeof params.ignoreQueryParameters === 'boolean')
body.ignoreQueryParameters = params.ignoreQueryParameters
if (params.delay) body.delay = Number(params.delay)
if (params.maxConcurrency) body.maxConcurrency = Number(params.maxConcurrency)
if (params.excludePaths) body.excludePaths = params.excludePaths
if (params.includePaths) body.includePaths = params.includePaths
if (params.webhook) body.webhook = params.webhook
if (typeof params.zeroDataRetention === 'boolean')
body.zeroDataRetention = params.zeroDataRetention
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
jobId: data.jobId || data.id,
pages: [],
total: 0,
creditsUsed: 0,
},
}
},
postProcess: async (result, params) => {
if (!result.success) {
return result
}
const jobId = result.output.jobId
logger.info(`Firecrawl crawl job ${jobId} created, polling for completion...`)
let elapsedTime = 0
while (elapsedTime < MAX_POLL_TIME_MS) {
try {
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${params.apiKey}`,
'Content-Type': 'application/json',
},
})
if (!statusResponse.ok) {
throw new Error(`Failed to get crawl status: ${statusResponse.statusText}`)
}
const crawlData = await statusResponse.json()
logger.info(`Firecrawl crawl job ${jobId} status: ${crawlData.status}`)
if (crawlData.status === 'completed') {
result.output = {
pages: crawlData.data || [],
total: crawlData.total || 0,
creditsUsed: crawlData.creditsUsed || 0,
}
return result
}
if (crawlData.status === 'failed') {
return {
...result,
success: false,
error: `Crawl job failed: ${crawlData.error || 'Unknown error'}`,
}
}
await sleep(POLL_INTERVAL_MS)
elapsedTime += POLL_INTERVAL_MS
} catch (error: any) {
logger.error('Error polling for crawl job status:', {
message: error.message || 'Unknown error',
jobId,
})
return {
...result,
success: false,
error: `Error polling for crawl job status: ${error.message || 'Unknown error'}`,
}
}
}
logger.warn(
`Crawl job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
)
return {
...result,
success: false,
error: `Crawl job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
}
},
outputs: {
pages: {
type: 'array',
description: 'Array of crawled pages with their content and metadata',
items: {
type: 'object',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
},
},
total: { type: 'number', description: 'Total number of pages found during crawl' },
},
}
+68
View File
@@ -0,0 +1,68 @@
import type {
FirecrawlCreditUsageParams,
FirecrawlCreditUsageResponse,
} from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const creditUsageTool: ToolConfig<FirecrawlCreditUsageParams, FirecrawlCreditUsageResponse> =
{
id: 'firecrawl_credit_usage',
name: 'Firecrawl Credit Usage',
description: 'Retrieve the remaining and allocated Firecrawl credits for the team.',
version: '1.0.0',
params: {
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'GET',
url: 'https://api.firecrawl.dev/v2/team/credit-usage',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
const usage = data.data ?? {}
return {
success: true,
output: {
remainingCredits: usage.remainingCredits ?? null,
planCredits: usage.planCredits ?? null,
billingPeriodStart: usage.billingPeriodStart ?? null,
billingPeriodEnd: usage.billingPeriodEnd ?? null,
},
}
},
outputs: {
remainingCredits: {
type: 'number',
description: 'Number of credits remaining for the team',
},
planCredits: {
type: 'number',
description: 'Credits allocated in the current plan',
optional: true,
},
billingPeriodStart: {
type: 'string',
description: 'Start of the current billing period',
optional: true,
},
billingPeriodEnd: {
type: 'string',
description: 'End of the current billing period',
optional: true,
},
},
}
@@ -0,0 +1,82 @@
import type {
FirecrawlExtractStatusParams,
FirecrawlExtractStatusResponse,
} from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const extractStatusTool: ToolConfig<
FirecrawlExtractStatusParams,
FirecrawlExtractStatusResponse
> = {
id: 'firecrawl_extract_status',
name: 'Firecrawl Extract Status',
description:
'Check the status and retrieve results of a previously started Firecrawl extract job by its job ID.',
version: '1.0.0',
params: {
jobId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The ID of the extract job to check',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
request: {
method: 'GET',
url: (params) =>
`https://api.firecrawl.dev/v2/extract/${encodeURIComponent(params.jobId.trim())}`,
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
status: data.status,
data: data.data ?? {},
expiresAt: data.expiresAt ?? null,
creditsUsed: data.creditsUsed ?? null,
tokensUsed: data.tokensUsed ?? null,
},
}
},
outputs: {
status: {
type: 'string',
description: 'Current extract status (processing, completed, failed, or cancelled)',
},
data: {
type: 'json',
description: 'Extracted structured data according to the schema or prompt',
},
expiresAt: {
type: 'string',
description: 'ISO timestamp when the extract results expire',
optional: true,
},
creditsUsed: {
type: 'number',
description: 'Number of credits used by the extract job',
optional: true,
},
tokensUsed: {
type: 'number',
description: 'Number of tokens used by the extract job',
optional: true,
},
},
}
+245
View File
@@ -0,0 +1,245 @@
import { createLogger } from '@sim/logger'
import { sleep } from '@sim/utils/helpers'
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
import type { ExtractParams, ExtractResponse } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('FirecrawlExtractTool')
const POLL_INTERVAL_MS = 5000
const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
export const extractTool: ToolConfig<ExtractParams, ExtractResponse> = {
id: 'firecrawl_extract',
name: 'Firecrawl Extract',
description:
'Extract structured data from entire webpages using natural language prompts and JSON schema. Powerful agentic feature for intelligent data extraction.',
version: '1.0.0',
params: {
urls: {
type: 'json',
required: true,
visibility: 'user-or-llm',
description:
'Array of URLs to extract data from (e.g., ["https://example.com/page1", "https://example.com/page2"] or ["https://example.com/*"])',
},
prompt: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Natural language guidance for the extraction process',
},
schema: {
type: 'json',
required: false,
visibility: 'user-or-llm',
description: 'JSON Schema defining the structure of data to extract',
},
enableWebSearch: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Enable web search to find supplementary information (default: false)',
},
ignoreSitemap: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Ignore sitemap.xml files during scanning (default: false)',
},
includeSubdomains: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Extend scanning to subdomains (default: true)',
},
showSources: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Return data sources in the response (default: false)',
},
ignoreInvalidURLs: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Skip invalid URLs in the array (default: true)',
},
scrapeOptions: {
type: 'json',
required: false,
visibility: 'hidden',
description: 'Advanced scraping configuration options',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
if (output.creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
const creditsUsed = Number(output.creditsUsed)
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/extract',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
urls: params.urls,
}
if (params.prompt) body.prompt = params.prompt
if (params.schema) body.schema = params.schema
if (typeof params.enableWebSearch === 'boolean') body.enableWebSearch = params.enableWebSearch
if (typeof params.ignoreSitemap === 'boolean') body.ignoreSitemap = params.ignoreSitemap
if (typeof params.includeSubdomains === 'boolean')
body.includeSubdomains = params.includeSubdomains
if (typeof params.showSources === 'boolean') body.showSources = params.showSources
if (typeof params.ignoreInvalidURLs === 'boolean')
body.ignoreInvalidURLs = params.ignoreInvalidURLs
if (params.scrapeOptions != null) {
const cleanedScrapeOptions = Object.entries(params.scrapeOptions).reduce(
(acc, [key, val]) => {
if (val != null) {
acc[key] = val
}
return acc
},
{} as Record<string, any>
)
if (Object.keys(cleanedScrapeOptions).length > 0) {
body.scrapeOptions = cleanedScrapeOptions
}
}
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
jobId: data.id,
success: false,
data: {},
},
}
},
postProcess: async (result, params) => {
if (!result.success) {
return result
}
const jobId = result.output.jobId
logger.info(`Firecrawl extract job ${jobId} created, polling for completion...`)
let elapsedTime = 0
while (elapsedTime < MAX_POLL_TIME_MS) {
try {
const statusResponse = await fetch(`https://api.firecrawl.dev/v2/extract/${jobId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${params.apiKey}`,
'Content-Type': 'application/json',
},
})
if (!statusResponse.ok) {
throw new Error(`Failed to get extract status: ${statusResponse.statusText}`)
}
const extractData = await statusResponse.json()
logger.info(`Firecrawl extract job ${jobId} status: ${extractData.status}`)
if (extractData.status === 'completed') {
result.output = {
jobId,
success: true,
data: extractData.data || {},
creditsUsed: extractData.creditsUsed,
}
return result
}
if (extractData.status === 'failed') {
return {
...result,
success: false,
error: `Extract job failed: ${extractData.error || 'Unknown error'}`,
}
}
await sleep(POLL_INTERVAL_MS)
elapsedTime += POLL_INTERVAL_MS
} catch (error: any) {
logger.error('Error polling for extract job status:', {
message: error.message || 'Unknown error',
jobId,
})
return {
...result,
success: false,
error: `Error polling for extract job status: ${error.message || 'Unknown error'}`,
}
}
}
logger.warn(
`Extract job ${jobId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
)
return {
...result,
success: false,
error: `Extract job did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
}
},
outputs: {
success: {
type: 'boolean',
description: 'Whether the extraction operation was successful',
},
data: {
type: 'object',
description: 'Extracted structured data according to the schema or prompt',
},
},
}
+27
View File
@@ -0,0 +1,27 @@
import { agentTool } from '@/tools/firecrawl/agent'
import { batchScrapeTool } from '@/tools/firecrawl/batch-scrape'
import { batchScrapeStatusTool } from '@/tools/firecrawl/batch-scrape-status'
import { cancelCrawlTool } from '@/tools/firecrawl/cancel-crawl'
import { crawlTool } from '@/tools/firecrawl/crawl'
import { crawlStatusTool } from '@/tools/firecrawl/crawl-status'
import { creditUsageTool } from '@/tools/firecrawl/credit-usage'
import { extractTool } from '@/tools/firecrawl/extract'
import { extractStatusTool } from '@/tools/firecrawl/extract-status'
import { mapTool } from '@/tools/firecrawl/map'
import { parseTool } from '@/tools/firecrawl/parse'
import { scrapeTool } from '@/tools/firecrawl/scrape'
import { searchTool } from '@/tools/firecrawl/search'
export const firecrawlScrapeTool = scrapeTool
export const firecrawlSearchTool = searchTool
export const firecrawlCrawlTool = crawlTool
export const firecrawlMapTool = mapTool
export const firecrawlExtractTool = extractTool
export const firecrawlAgentTool = agentTool
export const firecrawlParseTool = parseTool
export const firecrawlCrawlStatusTool = crawlStatusTool
export const firecrawlCancelCrawlTool = cancelCrawlTool
export const firecrawlBatchScrapeTool = batchScrapeTool
export const firecrawlBatchScrapeStatusTool = batchScrapeStatusTool
export const firecrawlExtractStatusTool = extractStatusTool
export const firecrawlCreditUsageTool = creditUsageTool
+149
View File
@@ -0,0 +1,149 @@
import type { MapParams, MapResponse } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const mapTool: ToolConfig<MapParams, MapResponse> = {
id: 'firecrawl_map',
name: 'Firecrawl Map',
description:
'Get a complete list of URLs from any website quickly and reliably. Useful for discovering all pages on a site without crawling them.',
version: '1.0.0',
params: {
url: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The base URL to map and discover links from (e.g., "https://example.com")',
},
search: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Filter results by relevance to a search term (e.g., "blog")',
},
sitemap: {
type: 'string',
required: false,
visibility: 'user-only',
description: 'Controls sitemap usage: "skip", "include" (default), or "only"',
},
includeSubdomains: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Whether to include URLs from subdomains (default: true)',
},
ignoreQueryParameters: {
type: 'boolean',
required: false,
visibility: 'user-only',
description: 'Exclude URLs containing query strings (default: true)',
},
limit: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description:
'Maximum number of links to return (e.g., 100, 1000, 5000). Max: 100,000, default: 5,000',
},
timeout: {
type: 'number',
required: false,
visibility: 'user-only',
description: 'Request timeout in milliseconds',
},
location: {
type: 'json',
required: false,
visibility: 'hidden',
description: 'Geographic context for proxying (country, languages)',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
if (output.creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
const creditsUsed = Number(output.creditsUsed)
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/map',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
url: params.url,
}
if (params.search) body.search = params.search
if (params.sitemap) body.sitemap = params.sitemap
if (typeof params.includeSubdomains === 'boolean')
body.includeSubdomains = params.includeSubdomains
if (typeof params.ignoreQueryParameters === 'boolean')
body.ignoreQueryParameters = params.ignoreQueryParameters
if (params.limit) body.limit = Number(params.limit)
if (params.timeout) body.timeout = Number(params.timeout)
if (params.location) body.location = params.location
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: data.success,
output: {
success: data.success,
links: data.links || [],
creditsUsed: 1,
},
}
},
outputs: {
success: {
type: 'boolean',
description: 'Whether the mapping operation was successful',
},
links: {
type: 'array',
description: 'Array of discovered URLs from the website',
items: {
type: 'string',
},
},
},
}
+225
View File
@@ -0,0 +1,225 @@
import type { ParseParams, ParseResponse } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const parseTool: ToolConfig<ParseParams, ParseResponse> = {
id: 'firecrawl_parse',
name: 'Firecrawl Document Parser',
description:
'Parse uploaded documents (PDF, DOCX, HTML, etc.) into clean markdown using Firecrawl. Supports .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls.',
version: '1.0.0',
params: {
file: {
type: 'file',
required: true,
visibility: 'user-only',
description: 'Document file to be parsed',
},
formats: {
type: 'array',
required: false,
visibility: 'user-or-llm',
description: 'Output formats to return (e.g., ["markdown"]). Defaults to markdown.',
},
onlyMainContent: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Exclude headers, navs, footers. Defaults to true.',
},
includeTags: {
type: 'array',
required: false,
visibility: 'user-or-llm',
description: 'HTML tags to include',
},
excludeTags: {
type: 'array',
required: false,
visibility: 'user-or-llm',
description: 'HTML tags to exclude',
},
timeout: {
type: 'number',
required: false,
visibility: 'user-or-llm',
description: 'Timeout in milliseconds (max 300000). Defaults to 30000.',
},
parsers: {
type: 'array',
required: false,
visibility: 'user-or-llm',
description: 'Parser configuration (e.g., [{ "type": "pdf" }])',
},
removeBase64Images: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Remove base64 images, keep alt text. Defaults to true.',
},
blockAds: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Block ads and popups. Defaults to true.',
},
proxy: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Proxy mode: "basic" or "auto"',
},
zeroDataRetention: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Enable zero data retention. Defaults to false.',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
const creditsUsed = (output.metadata as { creditsUsed?: number })?.creditsUsed
if (creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: '/api/tools/firecrawl/parse',
headers: () => ({
'Content-Type': 'application/json',
Accept: 'application/json',
}),
body: (params) => {
if (!params.apiKey || typeof params.apiKey !== 'string' || params.apiKey.trim() === '') {
throw new Error('Missing or invalid API key: A valid Firecrawl API key is required')
}
if (!params.file || typeof params.file !== 'object') {
throw new Error('File input is required')
}
const options: Record<string, unknown> = {}
if (params.formats) options.formats = params.formats
if (typeof params.onlyMainContent === 'boolean')
options.onlyMainContent = params.onlyMainContent
if (params.includeTags) options.includeTags = params.includeTags
if (params.excludeTags) options.excludeTags = params.excludeTags
if (params.timeout != null) options.timeout = Number(params.timeout)
if (params.parsers) options.parsers = params.parsers
if (typeof params.removeBase64Images === 'boolean')
options.removeBase64Images = params.removeBase64Images
if (typeof params.blockAds === 'boolean') options.blockAds = params.blockAds
if (params.proxy) options.proxy = params.proxy
if (typeof params.zeroDataRetention === 'boolean')
options.zeroDataRetention = params.zeroDataRetention
return {
apiKey: params.apiKey,
file: params.file,
options,
}
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
if (!data || typeof data !== 'object') {
throw new Error('Invalid response format from Firecrawl parse API')
}
const result = data.output ?? data.data ?? data
return {
success: true,
output: {
markdown: result.markdown ?? '',
summary: result.summary ?? null,
html: result.html ?? null,
rawHtml: result.rawHtml ?? null,
screenshot: result.screenshot ?? null,
links: result.links ?? [],
metadata: result.metadata ?? null,
warning: result.warning ?? null,
},
}
},
outputs: {
markdown: { type: 'string', description: 'Parsed document content in markdown format' },
summary: {
type: 'string',
description: 'Generated summary of the document',
optional: true,
},
html: {
type: 'string',
description: 'Processed HTML content',
optional: true,
},
rawHtml: {
type: 'string',
description: 'Unprocessed raw HTML content',
optional: true,
},
screenshot: {
type: 'string',
description: 'Screenshot URL or base64 (when requested)',
optional: true,
},
links: {
type: 'array',
description: 'URLs discovered in the document',
optional: true,
items: { type: 'string', description: 'Discovered URL' },
},
metadata: {
type: 'object',
description: 'Document metadata',
optional: true,
properties: {
title: { type: 'string', description: 'Document title', optional: true },
description: { type: 'string', description: 'Document description', optional: true },
language: { type: 'string', description: 'Document language code', optional: true },
sourceURL: { type: 'string', description: 'Source URL', optional: true },
url: { type: 'string', description: 'Final URL', optional: true },
keywords: { type: 'string', description: 'Document keywords', optional: true },
statusCode: { type: 'number', description: 'HTTP status code', optional: true },
contentType: { type: 'string', description: 'Document content type', optional: true },
error: { type: 'string', description: 'Error message if parse failed', optional: true },
},
},
warning: {
type: 'string',
description: 'Warning message from the parse operation',
optional: true,
},
},
}
+127
View File
@@ -0,0 +1,127 @@
import type { ScrapeParams, ScrapeResponse } from '@/tools/firecrawl/types'
import { PAGE_METADATA_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import { safeAssign } from '@/tools/safe-assign'
import type { ToolConfig } from '@/tools/types'
export const scrapeTool: ToolConfig<ScrapeParams, ScrapeResponse> = {
id: 'firecrawl_scrape',
name: 'Firecrawl Website Scraper',
description:
'Extract structured content from web pages with comprehensive metadata support. Converts content to markdown or HTML while capturing SEO metadata, Open Graph tags, and page information.',
version: '1.0.0',
params: {
url: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The URL to scrape content from (e.g., "https://example.com/page")',
},
scrapeOptions: {
type: 'json',
required: false,
visibility: 'hidden',
description: 'Options for content scraping',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
const creditsUsed = (output.metadata as { creditsUsed?: number })?.creditsUsed
if (creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/scrape',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
url: params.url,
formats: params.formats || params.scrapeOptions?.formats || ['markdown'],
}
if (typeof params.onlyMainContent === 'boolean') body.onlyMainContent = params.onlyMainContent
if (params.includeTags) body.includeTags = params.includeTags
if (params.excludeTags) body.excludeTags = params.excludeTags
if (params.maxAge) body.maxAge = Number(params.maxAge)
if (params.headers) body.headers = params.headers
if (params.waitFor) body.waitFor = Number(params.waitFor)
if (typeof params.mobile === 'boolean') body.mobile = params.mobile
if (typeof params.skipTlsVerification === 'boolean')
body.skipTlsVerification = params.skipTlsVerification
if (params.timeout) body.timeout = Number(params.timeout)
if (params.parsers) body.parsers = params.parsers
if (params.actions) body.actions = params.actions
if (params.location) body.location = params.location
if (typeof params.removeBase64Images === 'boolean')
body.removeBase64Images = params.removeBase64Images
if (typeof params.blockAds === 'boolean') body.blockAds = params.blockAds
if (params.proxy) body.proxy = params.proxy
if (typeof params.storeInCache === 'boolean') body.storeInCache = params.storeInCache
if (typeof params.zeroDataRetention === 'boolean')
body.zeroDataRetention = params.zeroDataRetention
if (params.scrapeOptions) {
safeAssign(body, params.scrapeOptions as Record<string, unknown>)
}
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
markdown: data.data.markdown,
html: data.data.html,
metadata: data.data.metadata,
creditsUsed: data.creditsUsed,
},
}
},
outputs: {
markdown: { type: 'string', description: 'Page content in markdown format' },
html: { type: 'string', description: 'Raw HTML content of the page', optional: true },
metadata: {
type: 'object',
description: 'Page metadata including SEO and Open Graph information',
properties: PAGE_METADATA_OUTPUT_PROPERTIES,
},
},
}
+104
View File
@@ -0,0 +1,104 @@
import type { SearchParams, SearchResponse } from '@/tools/firecrawl/types'
import { SEARCH_RESULT_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types'
import type { ToolConfig } from '@/tools/types'
export const searchTool: ToolConfig<SearchParams, SearchResponse> = {
id: 'firecrawl_search',
name: 'Firecrawl Search',
description: 'Search for information on the web using Firecrawl',
version: '1.0.0',
params: {
query: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description: 'The search query to use',
},
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Firecrawl API key',
},
},
hosting: {
envKeyPrefix: 'FIRECRAWL_API_KEY',
apiKeyParam: 'apiKey',
byokProviderId: 'firecrawl',
pricing: {
type: 'custom',
getCost: (_params, output) => {
if (output.creditsUsed == null) {
throw new Error('Firecrawl response missing creditsUsed field')
}
const creditsUsed = Number(output.creditsUsed)
if (Number.isNaN(creditsUsed)) {
throw new Error('Firecrawl response returned a non-numeric creditsUsed field')
}
return {
cost: creditsUsed * 0.001,
metadata: { creditsUsed },
}
},
},
rateLimit: {
mode: 'per_request',
requestsPerMinute: 100,
},
},
request: {
method: 'POST',
url: 'https://api.firecrawl.dev/v2/search',
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
const body: Record<string, any> = {
query: params.query,
}
// Add optional parameters if provided (truthy check filters empty strings, null, undefined)
if (params.limit) body.limit = Number(params.limit)
if (params.sources) body.sources = params.sources
if (params.categories) body.categories = params.categories
if (params.tbs) body.tbs = params.tbs
if (params.location) body.location = params.location
if (params.country) body.country = params.country
if (params.timeout) body.timeout = Number(params.timeout)
if (typeof params.ignoreInvalidURLs === 'boolean')
body.ignoreInvalidURLs = params.ignoreInvalidURLs
if (params.scrapeOptions) body.scrapeOptions = params.scrapeOptions
return body
},
},
transformResponse: async (response: Response) => {
const data = await response.json()
return {
success: true,
output: {
data: data.data,
creditsUsed: data.creditsUsed,
},
}
},
outputs: {
data: {
type: 'array',
description: 'Search results data with scraped content and metadata',
items: {
type: 'object',
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
},
},
},
}
+686
View File
@@ -0,0 +1,686 @@
import type { OutputProperty, ToolResponse } from '@/tools/types'
/**
* Shared output property definitions for Firecrawl API responses.
* Based on Firecrawl API documentation: https://docs.firecrawl.dev/api-reference
*
* API Response Reference:
* - Scrape: https://docs.firecrawl.dev/api-reference/endpoint/scrape
* - Crawl: https://docs.firecrawl.dev/api-reference/endpoint/crawl-get
* - Search: https://docs.firecrawl.dev/api-reference/endpoint/search
* - Map: https://docs.firecrawl.dev/api-reference/endpoint/map
* - Extract: https://docs.firecrawl.dev/api-reference/endpoint/extract
* - Agent: https://docs.firecrawl.dev/api-reference/endpoint/agent
*/
/**
* Output definition for page metadata in scrape responses
* Based on Firecrawl metadata object structure from POST /v2/scrape
*/
export const PAGE_METADATA_OUTPUT_PROPERTIES = {
title: { type: 'string', description: 'Page title' },
description: { type: 'string', description: 'Page meta description', optional: true },
language: { type: 'string', description: 'Page language code (e.g., "en")', optional: true },
sourceURL: { type: 'string', description: 'Original source URL that was scraped' },
statusCode: { type: 'number', description: 'HTTP status code of the response' },
keywords: { type: 'string', description: 'Page meta keywords', optional: true },
robots: {
type: 'string',
description: 'Robots meta directive (e.g., "follow, index")',
optional: true,
},
ogTitle: { type: 'string', description: 'Open Graph title', optional: true },
ogDescription: { type: 'string', description: 'Open Graph description', optional: true },
ogUrl: { type: 'string', description: 'Open Graph URL', optional: true },
ogImage: { type: 'string', description: 'Open Graph image URL', optional: true },
ogLocaleAlternate: {
type: 'array',
description: 'Alternate locale versions for Open Graph',
optional: true,
items: { type: 'string', description: 'Locale code' },
},
ogSiteName: { type: 'string', description: 'Open Graph site name', optional: true },
error: { type: 'string', description: 'Error message if scrape failed', optional: true },
} as const satisfies Record<string, OutputProperty>
/**
* Complete page metadata output definition
*/
export const PAGE_METADATA_OUTPUT: OutputProperty = {
type: 'object',
description: 'Page metadata including SEO and Open Graph information',
properties: PAGE_METADATA_OUTPUT_PROPERTIES,
}
/**
* Simplified metadata for crawl responses (subset of full metadata)
* Based on crawl data[].metadata structure from GET /v2/crawl/{id}
*/
export const CRAWL_METADATA_OUTPUT_PROPERTIES = {
title: { type: 'string', description: 'Page title' },
description: { type: 'string', description: 'Page meta description', optional: true },
language: { type: 'string', description: 'Page language code', optional: true },
sourceURL: { type: 'string', description: 'Original source URL' },
statusCode: { type: 'number', description: 'HTTP status code' },
ogLocaleAlternate: {
type: 'array',
description: 'Alternate locale versions',
optional: true,
items: { type: 'string', description: 'Locale code' },
},
} as const satisfies Record<string, OutputProperty>
/**
* Complete crawl metadata output definition
*/
export const CRAWL_METADATA_OUTPUT: OutputProperty = {
type: 'object',
description: 'Page metadata from crawl operation',
properties: CRAWL_METADATA_OUTPUT_PROPERTIES,
}
/**
* Search result metadata properties
* Based on search data[].metadata structure from POST /v2/search
*/
export const SEARCH_METADATA_OUTPUT_PROPERTIES = {
title: { type: 'string', description: 'Page title', optional: true },
description: { type: 'string', description: 'Page meta description', optional: true },
sourceURL: { type: 'string', description: 'Original source URL' },
statusCode: { type: 'number', description: 'HTTP status code', optional: true },
error: { type: 'string', description: 'Error message if scrape failed', optional: true },
} as const satisfies Record<string, OutputProperty>
/**
* Complete search metadata output definition
*/
export const SEARCH_METADATA_OUTPUT: OutputProperty = {
type: 'object',
description: 'Metadata about the search result page',
properties: SEARCH_METADATA_OUTPUT_PROPERTIES,
}
/**
* Output properties for scrape tool response
* Based on POST /v2/scrape response data object
*/
export const SCRAPE_OUTPUT_PROPERTIES = {
markdown: { type: 'string', description: 'Page content converted to clean markdown format' },
html: { type: 'string', description: 'Processed HTML content of the page', optional: true },
rawHtml: { type: 'string', description: 'Unprocessed raw HTML content', optional: true },
links: {
type: 'array',
description: 'Array of links found on the page',
optional: true,
items: { type: 'string', description: 'URL found on the page' },
},
screenshot: {
type: 'string',
description: 'Base64-encoded screenshot or URL (expires after 24 hours)',
optional: true,
},
metadata: PAGE_METADATA_OUTPUT,
} as const satisfies Record<string, OutputProperty>
/**
* Output properties for crawled page items
* Based on GET /v2/crawl/{id} response data[] array items
*/
export const CRAWLED_PAGE_OUTPUT_PROPERTIES = {
markdown: { type: 'string', description: 'Page content in markdown format' },
html: { type: 'string', description: 'Processed HTML content of the page', optional: true },
rawHtml: { type: 'string', description: 'Unprocessed raw HTML content', optional: true },
links: {
type: 'array',
description: 'Array of links found on the page',
optional: true,
items: { type: 'string', description: 'URL found on the page' },
},
screenshot: {
type: 'string',
description: 'Screenshot URL (expires after 24 hours)',
optional: true,
},
metadata: CRAWL_METADATA_OUTPUT,
} as const satisfies Record<string, OutputProperty>
/**
* Complete crawled page output definition
*/
export const CRAWLED_PAGE_OUTPUT: OutputProperty = {
type: 'object',
description: 'Crawled page data with content and metadata',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
}
/**
* Output properties for crawl tool response
* Based on GET /v2/crawl/{id} response (completed status)
*/
export const CRAWL_OUTPUT_PROPERTIES = {
pages: {
type: 'array',
description: 'Array of crawled pages with their content and metadata',
items: {
type: 'object',
properties: CRAWLED_PAGE_OUTPUT_PROPERTIES,
},
},
total: { type: 'number', description: 'Total number of pages found during crawl' },
} as const satisfies Record<string, OutputProperty>
/**
* Output properties for search result items
* Based on POST /v2/search response data[] array items
*/
export const SEARCH_RESULT_OUTPUT_PROPERTIES = {
title: { type: 'string', description: 'Search result title from search engine' },
description: {
type: 'string',
description: 'Search result description/snippet from search engine',
},
url: { type: 'string', description: 'URL of the search result' },
markdown: {
type: 'string',
description: 'Page content in markdown (when scrapeOptions.formats includes "markdown")',
optional: true,
},
html: {
type: 'string',
description: 'Processed HTML content (when scrapeOptions.formats includes "html")',
optional: true,
},
rawHtml: {
type: 'string',
description: 'Unprocessed raw HTML (when scrapeOptions.formats includes "rawHtml")',
optional: true,
},
links: {
type: 'array',
description: 'Links found on the page (when scrapeOptions.formats includes "links")',
optional: true,
items: { type: 'string', description: 'URL found on the page' },
},
screenshot: {
type: 'string',
description:
'Screenshot URL (expires after 24 hours, when scrapeOptions.formats includes "screenshot")',
optional: true,
},
metadata: SEARCH_METADATA_OUTPUT,
} as const satisfies Record<string, OutputProperty>
/**
* Complete search result output definition
*/
export const SEARCH_RESULT_OUTPUT: OutputProperty = {
type: 'object',
description: 'Search result item with optional scraped content',
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
}
/**
* Output properties for search tool response
* Based on POST /v2/search response
*/
export const SEARCH_OUTPUT_PROPERTIES = {
data: {
type: 'array',
description: 'Array of search results with scraped content and metadata',
items: {
type: 'object',
properties: SEARCH_RESULT_OUTPUT_PROPERTIES,
},
},
} as const satisfies Record<string, OutputProperty>
/**
* Output properties for map tool response
* Based on POST /v2/map response
*/
export const MAP_OUTPUT_PROPERTIES = {
success: { type: 'boolean', description: 'Whether the mapping operation completed successfully' },
links: {
type: 'array',
description: 'Array of discovered URLs from the website',
items: { type: 'string', description: 'Discovered URL' },
},
} as const satisfies Record<string, OutputProperty>
/**
* Output properties for extract tool response
* Based on GET /v2/extract/{id} response (completed status)
*/
export const EXTRACT_OUTPUT_PROPERTIES = {
success: { type: 'boolean', description: 'Whether the extraction completed successfully' },
data: {
type: 'object',
description: 'Extracted structured data according to the provided schema or prompt',
},
} as const satisfies Record<string, OutputProperty>
/**
* Output properties for agent tool response
* Based on GET /v2/agent/{id} response (completed status)
*/
export const AGENT_OUTPUT_PROPERTIES = {
success: { type: 'boolean', description: 'Whether the agent task completed successfully' },
status: {
type: 'string',
description: 'Current status of the agent job (processing, completed, failed)',
},
data: {
type: 'object',
description: 'Extracted data from the agent based on the prompt and schema',
},
expiresAt: {
type: 'string',
description: 'ISO timestamp when the results expire (24 hours after completion)',
optional: true,
},
sources: {
type: 'array',
description: 'Array of source URLs visited and used by the agent',
optional: true,
items: { type: 'string', description: 'Source URL' },
},
} as const satisfies Record<string, OutputProperty>
// Common types
interface LocationConfig {
country?: string
languages?: string[]
}
interface ScrapeOptions {
formats?: string[]
onlyMainContent?: boolean
includeTags?: string[]
excludeTags?: string[]
maxAge?: number
headers?: Record<string, string>
waitFor?: number
mobile?: boolean
skipTlsVerification?: boolean
timeout?: number
parsers?: string[]
actions?: Array<{
type: string
[key: string]: any
}>
location?: LocationConfig
removeBase64Images?: boolean
blockAds?: boolean
proxy?: 'basic' | 'stealth' | 'auto'
storeInCache?: boolean
}
export interface ScrapeParams {
apiKey: string
url: string
scrapeOptions?: ScrapeOptions
// Additional top-level scrape params
onlyMainContent?: boolean
formats?: string[]
includeTags?: string[]
excludeTags?: string[]
maxAge?: number
headers?: Record<string, string>
waitFor?: number
mobile?: boolean
skipTlsVerification?: boolean
timeout?: number
parsers?: string[]
actions?: Array<{
type: string
[key: string]: any
}>
location?: LocationConfig
removeBase64Images?: boolean
blockAds?: boolean
proxy?: 'basic' | 'stealth' | 'auto'
storeInCache?: boolean
zeroDataRetention?: boolean
}
export interface SearchParams {
apiKey: string
query: string
limit?: number
sources?: ('web' | 'images' | 'news')[]
categories?: ('github' | 'research' | 'pdf')[]
tbs?: string
location?: string
country?: string
timeout?: number
ignoreInvalidURLs?: boolean
scrapeOptions?: ScrapeOptions
}
export interface FirecrawlCrawlParams {
apiKey: string
url: string
limit?: number
maxDepth?: number
formats?: string[]
onlyMainContent?: boolean
prompt?: string
maxDiscoveryDepth?: number
sitemap?: 'skip' | 'include'
crawlEntireDomain?: boolean
allowExternalLinks?: boolean
allowSubdomains?: boolean
ignoreQueryParameters?: boolean
delay?: number
maxConcurrency?: number
excludePaths?: string[]
includePaths?: string[]
webhook?: {
url: string
headers?: Record<string, string>
metadata?: Record<string, any>
events?: ('completed' | 'page' | 'failed' | 'started')[]
}
scrapeOptions?: ScrapeOptions
zeroDataRetention?: boolean
}
export interface MapParams {
apiKey: string
url: string
search?: string
sitemap?: 'skip' | 'include' | 'only'
includeSubdomains?: boolean
ignoreQueryParameters?: boolean
limit?: number
timeout?: number
location?: LocationConfig
}
export interface ExtractParams {
apiKey: string
urls: string[]
prompt?: string
schema?: Record<string, any>
enableWebSearch?: boolean
ignoreSitemap?: boolean
includeSubdomains?: boolean
showSources?: boolean
ignoreInvalidURLs?: boolean
scrapeOptions?: ScrapeOptions
}
export interface AgentParams {
apiKey: string
prompt: string
urls?: string[]
schema?: Record<string, any>
maxCredits?: number
strictConstrainToURLs?: boolean
}
export interface ScrapeResponse extends ToolResponse {
output: {
markdown: string
html?: string
rawHtml?: string
links?: string[]
screenshot?: string
metadata: {
title: string
description?: string
language?: string
keywords?: string
robots?: string
ogTitle?: string
ogDescription?: string
ogUrl?: string
ogImage?: string
ogLocaleAlternate?: string[]
ogSiteName?: string
sourceURL: string
statusCode: number
error?: string
}
creditsUsed?: number
}
}
export interface SearchResponse extends ToolResponse {
output: {
data: Array<{
title: string
description: string
url: string
markdown?: string
html?: string
rawHtml?: string
links?: string[]
screenshot?: string
metadata: {
title?: string
description?: string
sourceURL: string
statusCode?: number
error?: string
}
}>
creditsUsed?: number
}
}
export interface FirecrawlCrawlResponse extends ToolResponse {
output: {
jobId?: string
pages: Array<{
markdown: string
html?: string
rawHtml?: string
links?: string[]
screenshot?: string
metadata: {
title: string
description?: string
language?: string
sourceURL: string
statusCode: number
ogLocaleAlternate?: string[]
}
}>
total: number
creditsUsed?: number
}
}
export interface MapResponse extends ToolResponse {
output: {
success: boolean
links: string[]
creditsUsed?: number
}
}
export interface ExtractResponse extends ToolResponse {
output: {
jobId: string
success: boolean
data: Record<string, any>
creditsUsed?: number
}
}
export interface AgentResponse extends ToolResponse {
output: {
jobId: string
success: boolean
status: string
data: Record<string, any>
creditsUsed?: number
expiresAt?: string
sources?: string[]
}
}
export interface ParseParams {
apiKey: string
file: unknown
formats?: Array<{ type: string } | string>
onlyMainContent?: boolean
includeTags?: string[]
excludeTags?: string[]
timeout?: number
parsers?: Array<{ type: string; mode?: string } | string>
removeBase64Images?: boolean
blockAds?: boolean
proxy?: 'basic' | 'auto'
zeroDataRetention?: boolean
}
export interface ParseResponse extends ToolResponse {
output: {
markdown: string
summary?: string | null
html?: string | null
rawHtml?: string | null
screenshot?: string | null
links?: string[]
metadata?: {
title?: string | string[]
description?: string | string[]
language?: string | string[] | null
sourceURL?: string
url?: string
keywords?: string | string[]
statusCode?: number
contentType?: string
error?: string | null
} | null
warning?: string | null
}
}
interface CrawledPage {
markdown: string
html?: string
rawHtml?: string
links?: string[]
screenshot?: string
metadata: {
title: string
description?: string
language?: string
sourceURL: string
statusCode: number
ogLocaleAlternate?: string[]
}
}
export interface FirecrawlCrawlStatusParams {
apiKey: string
jobId: string
}
export interface FirecrawlCrawlStatusResponse extends ToolResponse {
output: {
status: string
total: number
completed: number
creditsUsed: number
expiresAt?: string | null
next?: string | null
pages: CrawledPage[]
}
}
export interface FirecrawlCancelCrawlParams {
apiKey: string
jobId: string
}
export interface FirecrawlCancelCrawlResponse extends ToolResponse {
output: {
status: string
}
}
export interface FirecrawlBatchScrapeParams {
apiKey: string
urls: string[] | string
formats?: string[]
onlyMainContent?: boolean
maxConcurrency?: number
ignoreInvalidURLs?: boolean
scrapeOptions?: ScrapeOptions
zeroDataRetention?: boolean
}
export interface FirecrawlBatchScrapeResponse extends ToolResponse {
output: {
jobId?: string
invalidURLs?: string[]
pages: CrawledPage[]
total: number
completed: number
creditsUsed?: number
}
}
export interface FirecrawlBatchScrapeStatusParams {
apiKey: string
jobId: string
}
export interface FirecrawlBatchScrapeStatusResponse extends ToolResponse {
output: {
status: string
total: number
completed: number
creditsUsed: number
expiresAt?: string | null
next?: string | null
pages: CrawledPage[]
}
}
export interface FirecrawlExtractStatusParams {
apiKey: string
jobId: string
}
export interface FirecrawlExtractStatusResponse extends ToolResponse {
output: {
status: string
data: Record<string, any> | unknown[]
expiresAt?: string | null
creditsUsed?: number | null
tokensUsed?: number | null
}
}
export interface FirecrawlCreditUsageParams {
apiKey: string
}
export interface FirecrawlCreditUsageResponse extends ToolResponse {
output: {
remainingCredits: number | null
planCredits?: number | null
billingPeriodStart?: string | null
billingPeriodEnd?: string | null
}
}
export type FirecrawlResponse =
| ScrapeResponse
| SearchResponse
| FirecrawlCrawlResponse
| MapResponse
| ExtractResponse
| AgentResponse
| ParseResponse
| FirecrawlCrawlStatusResponse
| FirecrawlCancelCrawlResponse
| FirecrawlBatchScrapeResponse
| FirecrawlBatchScrapeStatusResponse
| FirecrawlExtractStatusResponse
| FirecrawlCreditUsageResponse