chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,13 @@
|
||||
import rootConfig from '../../eslint.config.mjs'
|
||||
|
||||
export default [
|
||||
...rootConfig,
|
||||
{
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
project: ['./tsconfig.json'],
|
||||
tsconfigRootDir: import.meta.dirname,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
This integration provides actions for web interaction, specifically for capturing screenshots and browsing web pages. The captureScreenshot action takes a URL as input and returns a URL pointing to the captured screenshot of the page. The browsePage action also takes a URL as input and returns the content of the web page along with metadata including the title, description, favicon, author, and date published. These actions facilitate automated web data extraction and visualization, enhancing the ability to interact with and analyze web content programmatically.
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-globe"><circle cx="12" cy="12" r="10"/><path d="M12 2a14.5 14.5 0 0 0 0 20 14.5 14.5 0 0 0 0-20"/><path d="M2 12h20"/></svg>
|
||||
|
After Width: | Height: | Size: 327 B |
@@ -0,0 +1,33 @@
|
||||
import { posthogHelper } from '@botpress/common'
|
||||
import { IntegrationDefinition } from '@botpress/sdk'
|
||||
import { actionDefinitions } from 'src/definitions/actions'
|
||||
|
||||
export const INTEGRATION_NAME = 'browser'
|
||||
export const INTEGRATION_VERSION = '0.8.8'
|
||||
|
||||
export default new IntegrationDefinition({
|
||||
name: INTEGRATION_NAME,
|
||||
title: 'Browser',
|
||||
version: INTEGRATION_VERSION,
|
||||
description:
|
||||
'Capture screenshots and retrieve web page content with metadata for automated browsing and data extraction.',
|
||||
readme: 'hub.md',
|
||||
icon: 'icon.svg',
|
||||
actions: actionDefinitions,
|
||||
secrets: {
|
||||
...posthogHelper.COMMON_SECRET_NAMES,
|
||||
SCREENSHOT_API_KEY: {
|
||||
description: 'ScreenShot key',
|
||||
},
|
||||
FIRECRAWL_API_KEY: {
|
||||
description: 'FireCrawl key',
|
||||
},
|
||||
LOGO_API_KEY: {
|
||||
description: 'Logo key',
|
||||
},
|
||||
},
|
||||
attributes: {
|
||||
category: 'Developer Tools',
|
||||
repo: 'botpress',
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "@botpresshub/browser",
|
||||
"scripts": {
|
||||
"check:type": "tsc --noEmit",
|
||||
"check:bplint": "bp lint",
|
||||
"build": "bp add -y && bp build",
|
||||
"test": "vitest --run"
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@botpress/sdk": "workspace:*",
|
||||
"@mendable/firecrawl-js": "^4.3.8",
|
||||
"axios": "^1.7.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@botpress/cli": "workspace:*",
|
||||
"@botpress/common": "workspace:*",
|
||||
"@botpress/sdk": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { IntegrationLogger, RuntimeError } from '@botpress/sdk'
|
||||
import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
|
||||
import { FullPage } from 'src/definitions/actions'
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_PAGE = 0.0015
|
||||
|
||||
const fixOutput = (val: unknown): string => {
|
||||
if (typeof val === 'string') {
|
||||
return val
|
||||
} else if (Array.isArray(val)) {
|
||||
return val.join(' ')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
const getPageContent = async (props: {
|
||||
url: string
|
||||
logger: IntegrationLogger
|
||||
waitFor?: number
|
||||
timeout?: number
|
||||
maxAge?: number
|
||||
}): Promise<FullPage> => {
|
||||
const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const result = await firecrawl.scrape(props.url, {
|
||||
onlyMainContent: true,
|
||||
maxAge: 60 * 60 * 24 * 7, // 1 week
|
||||
removeBase64Images: true,
|
||||
waitFor: props.waitFor,
|
||||
timeout: props.timeout,
|
||||
formats: ['markdown', 'rawHtml'],
|
||||
headers: { 'X-Botpress-Crawler': 'botpress' },
|
||||
storeInCache: true,
|
||||
})
|
||||
|
||||
props.logger.forBot().debug(`Firecrawl API call took ${Date.now() - startTime}ms for url: ${props.url}`)
|
||||
|
||||
const contentLength = result.markdown?.length || 0
|
||||
const isLargePage = contentLength > 50000
|
||||
|
||||
if (isLargePage) {
|
||||
await trackEvent('large_page_scraped', {
|
||||
url: props.url,
|
||||
contentLength,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
url: props.url,
|
||||
content: result.markdown!,
|
||||
raw: result.rawHtml!,
|
||||
favicon: fixOutput(result.metadata?.favicon),
|
||||
title: fixOutput(result.metadata?.title),
|
||||
description: fixOutput(result.metadata?.description),
|
||||
}
|
||||
} catch (err) {
|
||||
props.logger.error('There was an error while calling Firecrawl API.', err)
|
||||
|
||||
if (err instanceof SdkError) {
|
||||
const isRateLimit = err.status === 429 || err.message.includes('rate limit')
|
||||
await trackEvent('firecrawl_error', {
|
||||
url: props.url,
|
||||
errorType: isRateLimit ? 'rate_limited' : 'api_error',
|
||||
errorMessage: err.message,
|
||||
statusCode: err.status,
|
||||
errorCode: err.code,
|
||||
})
|
||||
}
|
||||
|
||||
throw new RuntimeError(`There was an error while browsing the page: ${props.url}`)
|
||||
}
|
||||
}
|
||||
|
||||
export const browsePages: bp.IntegrationProps['actions']['browsePages'] = async ({ input, logger, metadata }) => {
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const pageContentPromises = await Promise.allSettled(
|
||||
input.urls.map((url) => getPageContent({ url, logger, waitFor: input.waitFor, timeout: input.timeout }))
|
||||
)
|
||||
|
||||
const results = pageContentPromises
|
||||
.filter((promise): promise is PromiseFulfilledResult<FullPage> => promise.status === 'fulfilled')
|
||||
.map((result) => result.value)
|
||||
|
||||
// only charging for successful pages
|
||||
const cost = results.length * COST_PER_PAGE
|
||||
metadata.setCost(cost)
|
||||
|
||||
await trackEvent('pages_browsed', {
|
||||
urlCount: input.urls.length,
|
||||
successCount: results.length,
|
||||
failedCount: input.urls.length - results.length,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
|
||||
return {
|
||||
results,
|
||||
}
|
||||
} catch (err) {
|
||||
logger.forBot().error('There was an error while browsing the page.', err)
|
||||
throw err
|
||||
} finally {
|
||||
logger.forBot().info(`Browsing ${input.urls.length} urls took ${Date.now() - startTime}ms`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import axios, { isAxiosError } from 'axios'
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_PAGE = 0.0015
|
||||
|
||||
export const captureScreenshot: bp.IntegrationProps['actions']['captureScreenshot'] = async ({
|
||||
input,
|
||||
logger,
|
||||
metadata,
|
||||
client,
|
||||
}) => {
|
||||
logger.forBot().debug('Capturing Screenshot', { input })
|
||||
|
||||
const qs = new URLSearchParams({
|
||||
url: input.url,
|
||||
width: Math.max(input.width || 1080, 320).toString(),
|
||||
height: Math.max(input.height || 1920, 240).toString(),
|
||||
full_page: input.fullPage ? 'true' : 'false',
|
||||
fresh: 'true',
|
||||
output: 'json',
|
||||
file_type: 'png',
|
||||
wait_for_event: 'load',
|
||||
extract_html: 'true',
|
||||
lazy_load: 'true',
|
||||
delay: '500',
|
||||
})
|
||||
|
||||
if (input.javascriptToInject) {
|
||||
if (input.javascriptToInject.length > 100) {
|
||||
const { file: jsFile } = await client.uploadFile({
|
||||
key: `screenshot-js-${Date.now()}.js`,
|
||||
content: input.javascriptToInject,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
|
||||
index: false,
|
||||
})
|
||||
qs.append('js_url', jsFile.url)
|
||||
} else {
|
||||
qs.append('js', input.javascriptToInject)
|
||||
}
|
||||
}
|
||||
|
||||
if (input.cssToInject) {
|
||||
if (input.cssToInject.length > 100) {
|
||||
const { file: cssFile } = await client.uploadFile({
|
||||
key: `screenshot-css-${Date.now()}.css`,
|
||||
content: input.cssToInject,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
|
||||
index: false,
|
||||
})
|
||||
qs.append('css_url', cssFile.url)
|
||||
} else {
|
||||
qs.append('css', input.cssToInject)
|
||||
}
|
||||
}
|
||||
|
||||
const apiUrl = `https://shot.screenshotapi.net/screenshot?token=${bp.secrets.SCREENSHOT_API_KEY}&${qs.toString()}`
|
||||
|
||||
try {
|
||||
const { data } = await axios.get(apiUrl)
|
||||
|
||||
if (data.screenshot) {
|
||||
metadata.setCost(COST_PER_PAGE)
|
||||
|
||||
await trackEvent('screenshot_captured', {
|
||||
domain: new URL(input.url).hostname,
|
||||
width: input.width || 1080,
|
||||
height: input.height || 1920,
|
||||
fullPage: input.fullPage || false,
|
||||
hasJsInjection: !!input.javascriptToInject,
|
||||
hasCssInjection: !!input.cssToInject,
|
||||
})
|
||||
|
||||
return { imageUrl: data.screenshot, htmlUrl: data.extracted_html }
|
||||
} else {
|
||||
throw new Error('Screenshot not available')
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAxiosError(error)) {
|
||||
logger.forBot().error('There was an error while taking the screenshot', error.response?.data)
|
||||
await trackEvent('screenshot_error', {
|
||||
domain: new URL(input.url).hostname,
|
||||
statusCode: error.response?.status,
|
||||
errorMessage: error.message,
|
||||
})
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import { RuntimeError } from '@botpress/client'
|
||||
import { IntegrationLogger, z } from '@botpress/sdk'
|
||||
import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
|
||||
import { trackEvent } from '../tracking'
|
||||
import { isValidGlob, matchGlob } from '../utils/globs'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const LAMBDA_TIMEOUT = 55_000
|
||||
|
||||
const COST_PER_FIRECRAWL_MAP = 0.001
|
||||
|
||||
type StopReason = Awaited<ReturnType<bp.IntegrationProps['actions']['discoverUrls']>>['stopReason']
|
||||
|
||||
type ZodIssueCode = z.ZodIssue['code']
|
||||
|
||||
export const urlSchema = z.string().downstream((url) => {
|
||||
url = url.trim()
|
||||
if (!url.includes('://')) {
|
||||
url = `https://${url}`
|
||||
}
|
||||
try {
|
||||
const x = new URL(url)
|
||||
if (x.protocol !== 'http:' && x.protocol !== 'https:') {
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid protocol, only URLs starting with HTTP and HTTPS are supported',
|
||||
})
|
||||
}
|
||||
|
||||
if (!/.\.[a-zA-Z]{2,}$/.test(x.hostname)) {
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid TLD',
|
||||
})
|
||||
}
|
||||
const pathName = x.pathname.endsWith('/') ? x.pathname.slice(0, -1) : x.pathname
|
||||
return z.OK(`${x.origin}${pathName}${x.search ? x.search : ''}`)
|
||||
} catch (caught) {
|
||||
const err = caught instanceof Error ? caught : new Error('Unknown error while parsing URL')
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid URL: ' + err.message,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
class Accumulator {
|
||||
public processed = new Set<string>()
|
||||
public urls = new Set<string>()
|
||||
|
||||
public included: number = 0
|
||||
public excluded: number = 0
|
||||
|
||||
public cost: number = 0
|
||||
|
||||
public startedAt: number = Date.now()
|
||||
|
||||
public addUrls(urls: string[]): void {
|
||||
for (let url of urls) {
|
||||
if (this.stopReason) {
|
||||
return
|
||||
}
|
||||
|
||||
const parsed = urlSchema.safeParse(url)
|
||||
|
||||
if (!parsed.success) {
|
||||
continue
|
||||
}
|
||||
|
||||
url = parsed.data
|
||||
|
||||
if (this.onlyHttps && !url.toLowerCase().startsWith('https://')) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
if (this.processed.has(url)) {
|
||||
continue
|
||||
}
|
||||
|
||||
this.processed.add(url)
|
||||
|
||||
if (this.includedGlobs.length && this.includedGlobs.every((glob) => !matchGlob(url, glob))) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
this.included++
|
||||
|
||||
if (this.exclduedGlobs.some((glob) => matchGlob(url, glob))) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
this.urls.add(url)
|
||||
}
|
||||
}
|
||||
|
||||
public addCost(cost: number): void {
|
||||
this.cost += cost
|
||||
}
|
||||
|
||||
public constructor(
|
||||
public readonly includedGlobs: string[],
|
||||
public readonly exclduedGlobs: string[],
|
||||
public readonly timeout: number = 55_000,
|
||||
public readonly limit: number = 10_000,
|
||||
public readonly onlyHttps: boolean = true
|
||||
) {}
|
||||
|
||||
public get remainingTime(): number {
|
||||
return Math.max(0, this.startedAt + this.timeout - Date.now())
|
||||
}
|
||||
|
||||
public get stopReason(): StopReason | null {
|
||||
if (this.urls.size >= this.limit) {
|
||||
return 'urls_limit_reached'
|
||||
}
|
||||
|
||||
if (this.remainingTime <= 1000) {
|
||||
// if we have less than 1s left, we consider it a time limit reached
|
||||
return 'time_limit_reached'
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const firecrawlMap = async (props: { url: string; logger: IntegrationLogger; timeout: number }): Promise<string[]> => {
|
||||
const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
|
||||
|
||||
try {
|
||||
const result = await firecrawl.map(props.url, {
|
||||
sitemap: 'include',
|
||||
limit: 10_000,
|
||||
timeout: Math.max(1000, props.timeout - 2000),
|
||||
includeSubdomains: true,
|
||||
})
|
||||
|
||||
return result.links.map((x) => x.url)
|
||||
} catch (err) {
|
||||
props.logger.error('There was an error while calling Firecrawl map API.', err)
|
||||
|
||||
if (err instanceof SdkError) {
|
||||
const isRateLimit = err.status === 429 || err.message.includes('rate limit')
|
||||
await trackEvent('firecrawl_error', {
|
||||
url: props.url,
|
||||
operation: 'map',
|
||||
errorType: isRateLimit ? 'rate_limited' : 'api_error',
|
||||
errorMessage: err.message,
|
||||
statusCode: err.status,
|
||||
errorCode: err.code,
|
||||
})
|
||||
}
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export const discoverUrls: bp.IntegrationProps['actions']['discoverUrls'] = async ({ input, logger, metadata }) => {
|
||||
logger.forBot().debug('Discovering website URLs', { input })
|
||||
|
||||
for (const pattern of [...(input.include ?? []), ...(input.exclude ?? [])]) {
|
||||
if (!isValidGlob(pattern)) {
|
||||
throw new RuntimeError(`Forbidden characters in glob pattern: ${pattern}. Cannot contain "..", "{}", "[]"`)
|
||||
}
|
||||
}
|
||||
|
||||
const accumulator = new Accumulator(
|
||||
input.include ?? [],
|
||||
input.exclude ?? [],
|
||||
LAMBDA_TIMEOUT,
|
||||
input.count ?? 10_000,
|
||||
input.onlyHttps
|
||||
)
|
||||
|
||||
const firecrawl = firecrawlMap({ url: input.url, logger, timeout: accumulator.remainingTime }).then((links) => {
|
||||
accumulator.addUrls(links)
|
||||
accumulator.addCost(COST_PER_FIRECRAWL_MAP)
|
||||
})
|
||||
|
||||
await Promise.allSettled([firecrawl])
|
||||
|
||||
metadata.setCost(accumulator.cost)
|
||||
|
||||
await trackEvent('urls_discovered', {
|
||||
baseUrl: input.url,
|
||||
urlCount: accumulator.urls.size,
|
||||
excludedCount: accumulator.excluded,
|
||||
stopReason: accumulator.stopReason ?? 'end_of_results',
|
||||
hasIncludeFilters: (input.include?.length ?? 0) > 0,
|
||||
hasExcludeFilters: (input.exclude?.length ?? 0) > 0,
|
||||
})
|
||||
|
||||
return {
|
||||
excluded: accumulator.excluded,
|
||||
stopReason: accumulator.stopReason ?? 'end_of_results',
|
||||
urls: [...accumulator.urls],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_LOOKUP = 0.25
|
||||
|
||||
export const getWebsiteLogo: bp.IntegrationProps['actions']['getWebsiteLogo'] = async ({
|
||||
input,
|
||||
client,
|
||||
logger,
|
||||
metadata,
|
||||
}) => {
|
||||
logger.forBot().debug('Fetching logo for URL', { input })
|
||||
|
||||
const domain = input.domain.trim().replace(/^(https?:\/\/)?(www\.)?/, '')
|
||||
|
||||
const params = new URLSearchParams({
|
||||
format: 'png',
|
||||
size: (input.size || 128)?.toString(),
|
||||
token: bp.secrets.LOGO_API_KEY,
|
||||
})
|
||||
|
||||
if (input.greyscale) {
|
||||
params.append('greyscale', 'true')
|
||||
}
|
||||
|
||||
const url = `https://img.logo.dev/${domain}?${params.toString()}`
|
||||
|
||||
const buffer = (await fetch(url)).arrayBuffer()
|
||||
|
||||
const { file } = await client.uploadFile({
|
||||
key: `browser/logos/${domain}/logo.png`,
|
||||
content: await buffer,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
metadata: {
|
||||
domain,
|
||||
greyscale: input.greyscale || false,
|
||||
size: input.size || 128,
|
||||
},
|
||||
})
|
||||
|
||||
metadata.setCost(COST_PER_LOOKUP)
|
||||
|
||||
await trackEvent('logo_fetched', {
|
||||
domain,
|
||||
size: input.size || 128,
|
||||
greyscale: input.greyscale || false,
|
||||
})
|
||||
|
||||
return {
|
||||
logoUrl: file.url,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import axios, { isAxiosError } from 'axios'
|
||||
import { trackEvent } from '../tracking'
|
||||
import { browsePages } from './browse-pages'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type BingSearchResult = {
|
||||
name: string
|
||||
url: string
|
||||
snippet: string
|
||||
links?: {
|
||||
name: string
|
||||
url: string
|
||||
}[]
|
||||
}
|
||||
|
||||
export const webSearch: bp.IntegrationProps['actions']['webSearch'] = async ({
|
||||
client,
|
||||
input,
|
||||
logger,
|
||||
ctx,
|
||||
metadata,
|
||||
}) => {
|
||||
const clientConfig = client._inner.config
|
||||
|
||||
const axiosConfig = {
|
||||
...clientConfig,
|
||||
baseURL: `${clientConfig.apiUrl}/v1/cognitive`,
|
||||
}
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const { data: searchResults } = await axios.post<BingSearchResult[]>('bing/search', input, axiosConfig)
|
||||
logger.forBot().debug(`Search Web using Bing took ${Date.now() - startTime}ms`)
|
||||
|
||||
await trackEvent('web_search_performed', {
|
||||
queryLength: input.query.length,
|
||||
resultCount: searchResults.length,
|
||||
browsePages: input.browsePages || false,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
|
||||
if (!input.browsePages) {
|
||||
return { results: searchResults }
|
||||
}
|
||||
|
||||
const pageContent = await browsePages({
|
||||
input: { urls: searchResults.map((x) => x.url) },
|
||||
logger,
|
||||
client,
|
||||
type: 'browsePages',
|
||||
ctx,
|
||||
metadata,
|
||||
})
|
||||
|
||||
return {
|
||||
results: searchResults.map((searchResult) => ({
|
||||
...searchResult,
|
||||
page: pageContent.results?.find((page) => page.url === searchResult.url),
|
||||
})),
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAxiosError(err)) {
|
||||
logger.forBot().error('Error during web search', { error: err.response?.data })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import { ActionDefinition, z } from '@botpress/sdk'
|
||||
|
||||
const multiLineString = z.string().displayAs({ id: 'text', params: { multiLine: true, growVertically: true } })
|
||||
|
||||
const captureScreenshot: ActionDefinition = {
|
||||
title: 'Capture Screenshot',
|
||||
description: 'Capture a screenshot of the specified page.',
|
||||
input: {
|
||||
schema: z.object({
|
||||
url: z.string().describe('The url to screenshot').title('Url'),
|
||||
javascriptToInject: multiLineString
|
||||
.optional()
|
||||
.describe('JavaScript code to inject into the page before taking the screenshot')
|
||||
.title('Javascript to Inject'),
|
||||
cssToInject: multiLineString
|
||||
.optional()
|
||||
.describe('CSS code to inject into the page before taking the screenshot')
|
||||
.title('CSS To Inject'),
|
||||
width: z.number().default(1080).describe('The width of the screenshot').title('Width'),
|
||||
height: z.number().default(1920).describe('The height of the screenshot').title('Height'),
|
||||
fullPage: z.boolean().default(true).describe('Whether the screenshot is fullscreen or not').title('Full Page'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: z.object({
|
||||
imageUrl: z.string().describe('URL to the captured screenshot').title('Image Url'),
|
||||
htmlUrl: z.string().optional().describe('URL to the HTML page of the screenshot').title('Html Url'),
|
||||
}),
|
||||
},
|
||||
cacheable: true,
|
||||
billable: true,
|
||||
}
|
||||
|
||||
const fullPage = z.object({
|
||||
url: z.string(),
|
||||
content: z.string(),
|
||||
raw: z.string(),
|
||||
favicon: z.string().optional(),
|
||||
title: z.string().optional(),
|
||||
description: z.string().optional(),
|
||||
})
|
||||
|
||||
export type FullPage = z.infer<typeof fullPage>
|
||||
|
||||
const getWebsiteLogo: ActionDefinition = {
|
||||
title: 'Get Website Logo',
|
||||
description: 'Get the logo of the specified website.',
|
||||
input: {
|
||||
schema: z.object({
|
||||
domain: z.string().describe('The domain of the website to get the logo from (eg. "example.com")').title('Domain'),
|
||||
greyscale: z
|
||||
.boolean()
|
||||
.default(false)
|
||||
.describe('Whether to return the logo in grayscale (black & white)')
|
||||
.title('Grayscale'),
|
||||
size: z
|
||||
.enum(['64', '128', '256', '512'])
|
||||
.default('128')
|
||||
.describe('Size of the logo to return (64, 128 or 256, 512 pixels)')
|
||||
.title('Size'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: z.object({
|
||||
logoUrl: z.string().describe('URL to the website logo').title('Logo Url'),
|
||||
}),
|
||||
},
|
||||
cacheable: false,
|
||||
billable: true,
|
||||
}
|
||||
|
||||
const browsePages: ActionDefinition = {
|
||||
title: 'Browse Pages',
|
||||
description: 'Extract the full content & the metadata of the specified pages as markdown.',
|
||||
input: {
|
||||
schema: z.object({
|
||||
urls: z.array(z.string()).describe('The list of url to browse').title('Urls'),
|
||||
waitFor: z
|
||||
.number()
|
||||
.optional()
|
||||
.default(350)
|
||||
.describe(
|
||||
'Time to wait before extracting the content (in milliseconds). Set this value higher for dynamic pages.'
|
||||
)
|
||||
.title('Wait For'),
|
||||
timeout: z
|
||||
.number()
|
||||
.optional()
|
||||
.default(30000)
|
||||
.describe('Timeout for the request (in milliseconds)')
|
||||
.title('Time Out'),
|
||||
maxAge: z
|
||||
.number()
|
||||
.optional()
|
||||
.default(60 * 60 * 24 * 7)
|
||||
.describe('Maximum age of the cached page content (in seconds)')
|
||||
.title('Max Age'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: z.object({
|
||||
results: z.array(fullPage).describe('The list of pages browsed'),
|
||||
}),
|
||||
},
|
||||
cacheable: true,
|
||||
billable: true,
|
||||
}
|
||||
|
||||
const domainNameRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,}$/i
|
||||
const domainNameValidator = z
|
||||
.string()
|
||||
.regex(domainNameRegex, 'Invalid domain name')
|
||||
.min(3, 'Invalid URL')
|
||||
.max(50, 'Domain name is too long')
|
||||
|
||||
const webSearch: ActionDefinition = {
|
||||
title: 'Web Search',
|
||||
description: 'Search information on the web. You need to browse to that page to get the full content of the page.',
|
||||
input: {
|
||||
schema: z.object({
|
||||
query: z.string().min(1).max(1000).describe('What are we searching for?').title('Query'),
|
||||
includeSites: z
|
||||
.array(domainNameValidator)
|
||||
.max(20)
|
||||
.optional()
|
||||
.describe('Include only these domains in the search (max 20)')
|
||||
.title('Include Site'),
|
||||
excludeSites: z
|
||||
.array(domainNameValidator)
|
||||
.max(20)
|
||||
.optional()
|
||||
.describe('Exclude these domains from the search (max 20)')
|
||||
.title('Exclude Site'),
|
||||
count: z
|
||||
.number()
|
||||
.min(1)
|
||||
.max(20)
|
||||
.optional()
|
||||
.default(10)
|
||||
.describe('Number of search results to return (default: 10)')
|
||||
.title('Count'),
|
||||
freshness: z
|
||||
.enum(['Day', 'Week', 'Month'])
|
||||
.optional()
|
||||
.describe('Only consider results from the last day, week or month')
|
||||
.title('Freshness'),
|
||||
browsePages: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(false)
|
||||
.describe('Whether to browse to the pages to get the full content')
|
||||
.title('Browse Pages'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: z.object({
|
||||
results: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe('Title of the page'),
|
||||
url: z.string().describe('URL of the page'),
|
||||
snippet: z.string().describe('A short summary of the page'),
|
||||
links: z
|
||||
.array(z.object({ name: z.string(), url: z.string() }))
|
||||
.optional()
|
||||
.describe('Useful links on the page'),
|
||||
page: fullPage.optional().describe('The page itself'),
|
||||
})
|
||||
)
|
||||
.describe('Results of the search'),
|
||||
}),
|
||||
},
|
||||
billable: true,
|
||||
cacheable: true,
|
||||
}
|
||||
|
||||
export const globPattern = z
|
||||
.string()
|
||||
.min(1, 'Glob must be at least 1 char')
|
||||
.max(255, 'Glob must be at max 255 chars')
|
||||
.describe('Glob pattern to match URLs. Use * for wildcard matching')
|
||||
|
||||
const discoverUrls: ActionDefinition = {
|
||||
title: 'Discover Website URLs',
|
||||
description: 'Discovers the URLs of a website by finding links using sitemaps, robots.txt, and crawling.',
|
||||
input: {
|
||||
schema: z.object({
|
||||
url: z
|
||||
.string()
|
||||
.describe(
|
||||
'The URL of the website to discover URLs from. Can be a domain like example.com or a full URL like sub.example.com/page'
|
||||
)
|
||||
.title('Url'),
|
||||
onlyHttps: z.boolean().default(true).describe('Whether to only include HTTPS pages').title('Only HTTPS'),
|
||||
count: z.number().min(1).max(10_000).default(5_000).describe('The number of urls').title('Count'),
|
||||
include: z
|
||||
.array(globPattern)
|
||||
.max(100, 'You can include up to 100 URL patterns')
|
||||
.describe('List of glob patterns to include URLs from the discovery')
|
||||
.title('Include')
|
||||
.optional(),
|
||||
exclude: z
|
||||
.array(globPattern)
|
||||
.max(100, 'You can exclude up to 100 URL patterns')
|
||||
.optional()
|
||||
.describe(
|
||||
'List of glob patterns to exclude URLs from the discovery. All URLs matching these patterns will be excluded from the results, even if they are included in the "include" patterns.'
|
||||
)
|
||||
.title('Exclude'),
|
||||
}),
|
||||
},
|
||||
output: {
|
||||
schema: z.object({
|
||||
urls: z.array(z.string()).describe('List of discovered URLs').title('Urls'),
|
||||
excluded: z.number().describe('Number of URLs excluded due to robots.txt or filter').title('Excluded'),
|
||||
stopReason: z
|
||||
.enum(['urls_limit_reached', 'end_of_results', 'time_limit_reached'])
|
||||
.describe('Reason for stopping the URLs discovery. ')
|
||||
.title('Stop Reason'),
|
||||
}),
|
||||
},
|
||||
billable: true,
|
||||
cacheable: false,
|
||||
}
|
||||
|
||||
export const actionDefinitions = {
|
||||
captureScreenshot,
|
||||
browsePages,
|
||||
webSearch,
|
||||
discoverUrls,
|
||||
getWebsiteLogo,
|
||||
} satisfies Record<string, ActionDefinition>
|
||||
@@ -0,0 +1,40 @@
|
||||
import { posthogHelper } from '@botpress/common'
|
||||
import { INTEGRATION_NAME, INTEGRATION_VERSION } from 'integration.definition'
|
||||
import { browsePages } from './actions/browse-pages'
|
||||
import { captureScreenshot } from './actions/capture-screenshot'
|
||||
import { discoverUrls } from './actions/discover-urls'
|
||||
import { getWebsiteLogo } from './actions/get-website-logo'
|
||||
import { webSearch } from './actions/web-search'
|
||||
import { trackEvent } from './tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const integrationConfig: bp.IntegrationProps = {
|
||||
register: async ({ ctx }) => {
|
||||
await trackEvent('browser_registered', {
|
||||
integrationId: ctx.integrationId,
|
||||
})
|
||||
},
|
||||
unregister: async ({ ctx }) => {
|
||||
await trackEvent('browser_unregistered', {
|
||||
integrationId: ctx.integrationId,
|
||||
})
|
||||
},
|
||||
actions: {
|
||||
captureScreenshot,
|
||||
browsePages,
|
||||
webSearch,
|
||||
discoverUrls,
|
||||
getWebsiteLogo,
|
||||
},
|
||||
channels: {},
|
||||
handler: async () => {},
|
||||
}
|
||||
|
||||
export default posthogHelper.wrapIntegration(
|
||||
{
|
||||
integrationName: INTEGRATION_NAME,
|
||||
key: bp.secrets.POSTHOG_KEY,
|
||||
integrationVersion: INTEGRATION_VERSION,
|
||||
},
|
||||
integrationConfig
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
import { posthogHelper } from '@botpress/common'
|
||||
import { INTEGRATION_NAME, INTEGRATION_VERSION } from 'integration.definition'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type TrackingEvent =
|
||||
| 'browser_registered'
|
||||
| 'browser_unregistered'
|
||||
| 'screenshot_captured'
|
||||
| 'web_search_performed'
|
||||
| 'pages_browsed'
|
||||
| 'large_page_scraped'
|
||||
| 'urls_discovered'
|
||||
| 'logo_fetched'
|
||||
| 'firecrawl_error'
|
||||
| 'screenshot_error'
|
||||
|
||||
type EventProperties = Record<string, string | number | boolean | undefined>
|
||||
|
||||
const getPostHogConfig = () => ({
|
||||
key: bp.secrets.POSTHOG_KEY,
|
||||
integrationName: INTEGRATION_NAME,
|
||||
integrationVersion: INTEGRATION_VERSION,
|
||||
})
|
||||
|
||||
export const trackEvent = async (
|
||||
event: TrackingEvent,
|
||||
properties: EventProperties,
|
||||
distinctId?: string
|
||||
): Promise<void> => {
|
||||
try {
|
||||
await posthogHelper.sendPosthogEvent(
|
||||
{
|
||||
distinctId: distinctId ?? 'no id',
|
||||
event,
|
||||
properties,
|
||||
},
|
||||
getPostHogConfig()
|
||||
)
|
||||
} catch {
|
||||
// Silently fail
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { it, expect } from 'vitest'
|
||||
import { isValidGlob, matchGlob } from './globs'
|
||||
|
||||
it('isValidGlob', () => {
|
||||
expect(isValidGlob('*')).toBe(true)
|
||||
expect(isValidGlob('/admin/')).toBe(true)
|
||||
expect(isValidGlob('*/admin')).toBe(true)
|
||||
expect(isValidGlob('https://botpress.com/*')).toBe(true)
|
||||
expect(isValidGlob('https://botpress.com/*/blog')).toBe(true)
|
||||
|
||||
expect(isValidGlob('https://botpress.com/**/blog')).toBe(false)
|
||||
expect(isValidGlob('**')).toBe(false)
|
||||
expect(isValidGlob('**')).toBe(false)
|
||||
expect(isValidGlob('')).toBe(false)
|
||||
expect(isValidGlob(' ')).toBe(false)
|
||||
|
||||
expect(isValidGlob('https://*.botpress.com/*/blog')).toBe(false)
|
||||
})
|
||||
|
||||
it('matchGlob', () => {
|
||||
expect(matchGlob('https://example.com', '*')).toBe(true)
|
||||
expect(matchGlob('https://example.com/path', '*')).toBe(true)
|
||||
expect(matchGlob('http://sub.example.com', '*')).toBe(true)
|
||||
|
||||
expect(matchGlob('https://example.com/admin/dashboard', '/admin/')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'admin')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', '*/dashboard')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/admin/dashboard')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/admin/dashboard*')).toBe(true)
|
||||
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*/dashboard')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com')).toBe(true)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', '*example.com/admin/dashboard')).toBe(true)
|
||||
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'world')).toBe(false)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'http://example.com/*')).toBe(false)
|
||||
expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*/dashboard/hello')).toBe(false)
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
export const isValidGlob = (glob: string): boolean => {
|
||||
if (glob.split('*').length > 2) {
|
||||
// More than one * is not supported
|
||||
return false
|
||||
}
|
||||
|
||||
return glob.trim().length > 0 && glob.trim().length <= 255
|
||||
}
|
||||
|
||||
export const matchGlob = (url: string, glob: string): boolean => {
|
||||
url = url.toLowerCase().trim()
|
||||
glob = glob.toLowerCase().trim()
|
||||
|
||||
// If glob is just *, it matches everything
|
||||
if (glob === '*') {
|
||||
return true
|
||||
}
|
||||
|
||||
if (!glob.includes('*')) {
|
||||
// If glob doesn't contain any wildcard, we just check for presence
|
||||
return url.includes(glob)
|
||||
}
|
||||
|
||||
// if starts with *, we check that url ends with glob
|
||||
if (glob.startsWith('*')) {
|
||||
const trimmedGlob = glob.slice(1) // Remove the leading *
|
||||
return url.endsWith(trimmedGlob)
|
||||
}
|
||||
|
||||
// if ends with *, we check that url starts with glob
|
||||
if (glob.endsWith('*')) {
|
||||
const trimmedGlob = glob.slice(0, -1) // Remove the trailing *
|
||||
return url.startsWith(trimmedGlob)
|
||||
}
|
||||
|
||||
// if * in the middle, we check that starts with the part before * and ends with the part after *
|
||||
if (glob.includes('*')) {
|
||||
const [start, end] = glob.split('*')
|
||||
return url.startsWith(start!) && url.endsWith(end!)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": { "*": ["./*"] },
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import config from '../../vitest.config'
|
||||
export default config
|
||||
Reference in New Issue
Block a user