chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
+104
View File
@@ -0,0 +1,104 @@
import type {
BrightDataScrapeDatasetParams,
BrightDataScrapeDatasetResponse,
} from '@/tools/brightdata/types'
import type { ToolConfig } from '@/tools/types'
export const brightDataScrapeDatasetTool: ToolConfig<
BrightDataScrapeDatasetParams,
BrightDataScrapeDatasetResponse
> = {
id: 'brightdata_scrape_dataset',
name: 'Bright Data Scrape Dataset',
description:
'Trigger a Bright Data pre-built scraper to extract structured data from URLs. Supports 660+ scrapers for platforms like Amazon, LinkedIn, Instagram, and more.',
version: '1.0.0',
params: {
apiKey: {
type: 'string',
required: true,
visibility: 'user-only',
description: 'Bright Data API token',
},
datasetId: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'Dataset scraper ID from your Bright Data dashboard (e.g., "gd_l1viktl72bvl7bjuj0")',
},
urls: {
type: 'string',
required: true,
visibility: 'user-or-llm',
description:
'JSON array of URL objects to scrape (e.g., [{"url": "https://example.com/product"}])',
},
format: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Output format: "json" or "csv". Defaults to "json"',
},
includeErrors: {
type: 'boolean',
required: false,
visibility: 'user-or-llm',
description: 'Whether to include a per-input error report in the results',
},
},
request: {
method: 'POST',
url: (params) => {
const queryParams = new URLSearchParams()
queryParams.set('dataset_id', params.datasetId.trim())
queryParams.set('format', params.format || 'json')
if (params.includeErrors) queryParams.set('include_errors', 'true')
return `https://api.brightdata.com/datasets/v3/trigger?${queryParams.toString()}`
},
headers: (params) => ({
'Content-Type': 'application/json',
Authorization: `Bearer ${params.apiKey}`,
}),
body: (params) => {
if (typeof params.urls === 'string') {
try {
return JSON.parse(params.urls)
} catch {
return [{ url: params.urls }]
}
}
return params.urls
},
},
transformResponse: async (response: Response) => {
if (!response.ok) {
const errorText = await response.text()
throw new Error(errorText || `Dataset trigger failed with status ${response.status}`)
}
const data = await response.json()
return {
success: true,
output: {
snapshotId: data.snapshot_id ?? data.snapshotId ?? '',
status: data.status ?? 'triggered',
},
}
},
outputs: {
snapshotId: {
type: 'string',
description: 'The snapshot ID to retrieve results later',
},
status: {
type: 'string',
description: 'Status of the scraping job (e.g., "triggered", "running")',
},
},
}