chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,112 @@
|
||||
import { IntegrationLogger, RuntimeError } from '@botpress/sdk'
|
||||
import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
|
||||
import { FullPage } from 'src/definitions/actions'
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_PAGE = 0.0015
|
||||
|
||||
const fixOutput = (val: unknown): string => {
|
||||
if (typeof val === 'string') {
|
||||
return val
|
||||
} else if (Array.isArray(val)) {
|
||||
return val.join(' ')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
const getPageContent = async (props: {
|
||||
url: string
|
||||
logger: IntegrationLogger
|
||||
waitFor?: number
|
||||
timeout?: number
|
||||
maxAge?: number
|
||||
}): Promise<FullPage> => {
|
||||
const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const result = await firecrawl.scrape(props.url, {
|
||||
onlyMainContent: true,
|
||||
maxAge: 60 * 60 * 24 * 7, // 1 week
|
||||
removeBase64Images: true,
|
||||
waitFor: props.waitFor,
|
||||
timeout: props.timeout,
|
||||
formats: ['markdown', 'rawHtml'],
|
||||
headers: { 'X-Botpress-Crawler': 'botpress' },
|
||||
storeInCache: true,
|
||||
})
|
||||
|
||||
props.logger.forBot().debug(`Firecrawl API call took ${Date.now() - startTime}ms for url: ${props.url}`)
|
||||
|
||||
const contentLength = result.markdown?.length || 0
|
||||
const isLargePage = contentLength > 50000
|
||||
|
||||
if (isLargePage) {
|
||||
await trackEvent('large_page_scraped', {
|
||||
url: props.url,
|
||||
contentLength,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
url: props.url,
|
||||
content: result.markdown!,
|
||||
raw: result.rawHtml!,
|
||||
favicon: fixOutput(result.metadata?.favicon),
|
||||
title: fixOutput(result.metadata?.title),
|
||||
description: fixOutput(result.metadata?.description),
|
||||
}
|
||||
} catch (err) {
|
||||
props.logger.error('There was an error while calling Firecrawl API.', err)
|
||||
|
||||
if (err instanceof SdkError) {
|
||||
const isRateLimit = err.status === 429 || err.message.includes('rate limit')
|
||||
await trackEvent('firecrawl_error', {
|
||||
url: props.url,
|
||||
errorType: isRateLimit ? 'rate_limited' : 'api_error',
|
||||
errorMessage: err.message,
|
||||
statusCode: err.status,
|
||||
errorCode: err.code,
|
||||
})
|
||||
}
|
||||
|
||||
throw new RuntimeError(`There was an error while browsing the page: ${props.url}`)
|
||||
}
|
||||
}
|
||||
|
||||
export const browsePages: bp.IntegrationProps['actions']['browsePages'] = async ({ input, logger, metadata }) => {
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const pageContentPromises = await Promise.allSettled(
|
||||
input.urls.map((url) => getPageContent({ url, logger, waitFor: input.waitFor, timeout: input.timeout }))
|
||||
)
|
||||
|
||||
const results = pageContentPromises
|
||||
.filter((promise): promise is PromiseFulfilledResult<FullPage> => promise.status === 'fulfilled')
|
||||
.map((result) => result.value)
|
||||
|
||||
// only charging for successful pages
|
||||
const cost = results.length * COST_PER_PAGE
|
||||
metadata.setCost(cost)
|
||||
|
||||
await trackEvent('pages_browsed', {
|
||||
urlCount: input.urls.length,
|
||||
successCount: results.length,
|
||||
failedCount: input.urls.length - results.length,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
|
||||
return {
|
||||
results,
|
||||
}
|
||||
} catch (err) {
|
||||
logger.forBot().error('There was an error while browsing the page.', err)
|
||||
throw err
|
||||
} finally {
|
||||
logger.forBot().info(`Browsing ${input.urls.length} urls took ${Date.now() - startTime}ms`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import axios, { isAxiosError } from 'axios'
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_PAGE = 0.0015
|
||||
|
||||
export const captureScreenshot: bp.IntegrationProps['actions']['captureScreenshot'] = async ({
|
||||
input,
|
||||
logger,
|
||||
metadata,
|
||||
client,
|
||||
}) => {
|
||||
logger.forBot().debug('Capturing Screenshot', { input })
|
||||
|
||||
const qs = new URLSearchParams({
|
||||
url: input.url,
|
||||
width: Math.max(input.width || 1080, 320).toString(),
|
||||
height: Math.max(input.height || 1920, 240).toString(),
|
||||
full_page: input.fullPage ? 'true' : 'false',
|
||||
fresh: 'true',
|
||||
output: 'json',
|
||||
file_type: 'png',
|
||||
wait_for_event: 'load',
|
||||
extract_html: 'true',
|
||||
lazy_load: 'true',
|
||||
delay: '500',
|
||||
})
|
||||
|
||||
if (input.javascriptToInject) {
|
||||
if (input.javascriptToInject.length > 100) {
|
||||
const { file: jsFile } = await client.uploadFile({
|
||||
key: `screenshot-js-${Date.now()}.js`,
|
||||
content: input.javascriptToInject,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
|
||||
index: false,
|
||||
})
|
||||
qs.append('js_url', jsFile.url)
|
||||
} else {
|
||||
qs.append('js', input.javascriptToInject)
|
||||
}
|
||||
}
|
||||
|
||||
if (input.cssToInject) {
|
||||
if (input.cssToInject.length > 100) {
|
||||
const { file: cssFile } = await client.uploadFile({
|
||||
key: `screenshot-css-${Date.now()}.css`,
|
||||
content: input.cssToInject,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
|
||||
index: false,
|
||||
})
|
||||
qs.append('css_url', cssFile.url)
|
||||
} else {
|
||||
qs.append('css', input.cssToInject)
|
||||
}
|
||||
}
|
||||
|
||||
const apiUrl = `https://shot.screenshotapi.net/screenshot?token=${bp.secrets.SCREENSHOT_API_KEY}&${qs.toString()}`
|
||||
|
||||
try {
|
||||
const { data } = await axios.get(apiUrl)
|
||||
|
||||
if (data.screenshot) {
|
||||
metadata.setCost(COST_PER_PAGE)
|
||||
|
||||
await trackEvent('screenshot_captured', {
|
||||
domain: new URL(input.url).hostname,
|
||||
width: input.width || 1080,
|
||||
height: input.height || 1920,
|
||||
fullPage: input.fullPage || false,
|
||||
hasJsInjection: !!input.javascriptToInject,
|
||||
hasCssInjection: !!input.cssToInject,
|
||||
})
|
||||
|
||||
return { imageUrl: data.screenshot, htmlUrl: data.extracted_html }
|
||||
} else {
|
||||
throw new Error('Screenshot not available')
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAxiosError(error)) {
|
||||
logger.forBot().error('There was an error while taking the screenshot', error.response?.data)
|
||||
await trackEvent('screenshot_error', {
|
||||
domain: new URL(input.url).hostname,
|
||||
statusCode: error.response?.status,
|
||||
errorMessage: error.message,
|
||||
})
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import { RuntimeError } from '@botpress/client'
|
||||
import { IntegrationLogger, z } from '@botpress/sdk'
|
||||
import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
|
||||
import { trackEvent } from '../tracking'
|
||||
import { isValidGlob, matchGlob } from '../utils/globs'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const LAMBDA_TIMEOUT = 55_000
|
||||
|
||||
const COST_PER_FIRECRAWL_MAP = 0.001
|
||||
|
||||
type StopReason = Awaited<ReturnType<bp.IntegrationProps['actions']['discoverUrls']>>['stopReason']
|
||||
|
||||
type ZodIssueCode = z.ZodIssue['code']
|
||||
|
||||
export const urlSchema = z.string().downstream((url) => {
|
||||
url = url.trim()
|
||||
if (!url.includes('://')) {
|
||||
url = `https://${url}`
|
||||
}
|
||||
try {
|
||||
const x = new URL(url)
|
||||
if (x.protocol !== 'http:' && x.protocol !== 'https:') {
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid protocol, only URLs starting with HTTP and HTTPS are supported',
|
||||
})
|
||||
}
|
||||
|
||||
if (!/.\.[a-zA-Z]{2,}$/.test(x.hostname)) {
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid TLD',
|
||||
})
|
||||
}
|
||||
const pathName = x.pathname.endsWith('/') ? x.pathname.slice(0, -1) : x.pathname
|
||||
return z.OK(`${x.origin}${pathName}${x.search ? x.search : ''}`)
|
||||
} catch (caught) {
|
||||
const err = caught instanceof Error ? caught : new Error('Unknown error while parsing URL')
|
||||
return z.ERR({
|
||||
code: 'custom' satisfies ZodIssueCode,
|
||||
message: 'Invalid URL: ' + err.message,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
class Accumulator {
|
||||
public processed = new Set<string>()
|
||||
public urls = new Set<string>()
|
||||
|
||||
public included: number = 0
|
||||
public excluded: number = 0
|
||||
|
||||
public cost: number = 0
|
||||
|
||||
public startedAt: number = Date.now()
|
||||
|
||||
public addUrls(urls: string[]): void {
|
||||
for (let url of urls) {
|
||||
if (this.stopReason) {
|
||||
return
|
||||
}
|
||||
|
||||
const parsed = urlSchema.safeParse(url)
|
||||
|
||||
if (!parsed.success) {
|
||||
continue
|
||||
}
|
||||
|
||||
url = parsed.data
|
||||
|
||||
if (this.onlyHttps && !url.toLowerCase().startsWith('https://')) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
if (this.processed.has(url)) {
|
||||
continue
|
||||
}
|
||||
|
||||
this.processed.add(url)
|
||||
|
||||
if (this.includedGlobs.length && this.includedGlobs.every((glob) => !matchGlob(url, glob))) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
this.included++
|
||||
|
||||
if (this.exclduedGlobs.some((glob) => matchGlob(url, glob))) {
|
||||
this.excluded++
|
||||
continue
|
||||
}
|
||||
|
||||
this.urls.add(url)
|
||||
}
|
||||
}
|
||||
|
||||
public addCost(cost: number): void {
|
||||
this.cost += cost
|
||||
}
|
||||
|
||||
public constructor(
|
||||
public readonly includedGlobs: string[],
|
||||
public readonly exclduedGlobs: string[],
|
||||
public readonly timeout: number = 55_000,
|
||||
public readonly limit: number = 10_000,
|
||||
public readonly onlyHttps: boolean = true
|
||||
) {}
|
||||
|
||||
public get remainingTime(): number {
|
||||
return Math.max(0, this.startedAt + this.timeout - Date.now())
|
||||
}
|
||||
|
||||
public get stopReason(): StopReason | null {
|
||||
if (this.urls.size >= this.limit) {
|
||||
return 'urls_limit_reached'
|
||||
}
|
||||
|
||||
if (this.remainingTime <= 1000) {
|
||||
// if we have less than 1s left, we consider it a time limit reached
|
||||
return 'time_limit_reached'
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const firecrawlMap = async (props: { url: string; logger: IntegrationLogger; timeout: number }): Promise<string[]> => {
|
||||
const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
|
||||
|
||||
try {
|
||||
const result = await firecrawl.map(props.url, {
|
||||
sitemap: 'include',
|
||||
limit: 10_000,
|
||||
timeout: Math.max(1000, props.timeout - 2000),
|
||||
includeSubdomains: true,
|
||||
})
|
||||
|
||||
return result.links.map((x) => x.url)
|
||||
} catch (err) {
|
||||
props.logger.error('There was an error while calling Firecrawl map API.', err)
|
||||
|
||||
if (err instanceof SdkError) {
|
||||
const isRateLimit = err.status === 429 || err.message.includes('rate limit')
|
||||
await trackEvent('firecrawl_error', {
|
||||
url: props.url,
|
||||
operation: 'map',
|
||||
errorType: isRateLimit ? 'rate_limited' : 'api_error',
|
||||
errorMessage: err.message,
|
||||
statusCode: err.status,
|
||||
errorCode: err.code,
|
||||
})
|
||||
}
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export const discoverUrls: bp.IntegrationProps['actions']['discoverUrls'] = async ({ input, logger, metadata }) => {
|
||||
logger.forBot().debug('Discovering website URLs', { input })
|
||||
|
||||
for (const pattern of [...(input.include ?? []), ...(input.exclude ?? [])]) {
|
||||
if (!isValidGlob(pattern)) {
|
||||
throw new RuntimeError(`Forbidden characters in glob pattern: ${pattern}. Cannot contain "..", "{}", "[]"`)
|
||||
}
|
||||
}
|
||||
|
||||
const accumulator = new Accumulator(
|
||||
input.include ?? [],
|
||||
input.exclude ?? [],
|
||||
LAMBDA_TIMEOUT,
|
||||
input.count ?? 10_000,
|
||||
input.onlyHttps
|
||||
)
|
||||
|
||||
const firecrawl = firecrawlMap({ url: input.url, logger, timeout: accumulator.remainingTime }).then((links) => {
|
||||
accumulator.addUrls(links)
|
||||
accumulator.addCost(COST_PER_FIRECRAWL_MAP)
|
||||
})
|
||||
|
||||
await Promise.allSettled([firecrawl])
|
||||
|
||||
metadata.setCost(accumulator.cost)
|
||||
|
||||
await trackEvent('urls_discovered', {
|
||||
baseUrl: input.url,
|
||||
urlCount: accumulator.urls.size,
|
||||
excludedCount: accumulator.excluded,
|
||||
stopReason: accumulator.stopReason ?? 'end_of_results',
|
||||
hasIncludeFilters: (input.include?.length ?? 0) > 0,
|
||||
hasExcludeFilters: (input.exclude?.length ?? 0) > 0,
|
||||
})
|
||||
|
||||
return {
|
||||
excluded: accumulator.excluded,
|
||||
stopReason: accumulator.stopReason ?? 'end_of_results',
|
||||
urls: [...accumulator.urls],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { trackEvent } from '../tracking'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const COST_PER_LOOKUP = 0.25
|
||||
|
||||
export const getWebsiteLogo: bp.IntegrationProps['actions']['getWebsiteLogo'] = async ({
|
||||
input,
|
||||
client,
|
||||
logger,
|
||||
metadata,
|
||||
}) => {
|
||||
logger.forBot().debug('Fetching logo for URL', { input })
|
||||
|
||||
const domain = input.domain.trim().replace(/^(https?:\/\/)?(www\.)?/, '')
|
||||
|
||||
const params = new URLSearchParams({
|
||||
format: 'png',
|
||||
size: (input.size || 128)?.toString(),
|
||||
token: bp.secrets.LOGO_API_KEY,
|
||||
})
|
||||
|
||||
if (input.greyscale) {
|
||||
params.append('greyscale', 'true')
|
||||
}
|
||||
|
||||
const url = `https://img.logo.dev/${domain}?${params.toString()}`
|
||||
|
||||
const buffer = (await fetch(url)).arrayBuffer()
|
||||
|
||||
const { file } = await client.uploadFile({
|
||||
key: `browser/logos/${domain}/logo.png`,
|
||||
content: await buffer,
|
||||
accessPolicies: ['public_content'],
|
||||
publicContentImmediatelyAccessible: true,
|
||||
metadata: {
|
||||
domain,
|
||||
greyscale: input.greyscale || false,
|
||||
size: input.size || 128,
|
||||
},
|
||||
})
|
||||
|
||||
metadata.setCost(COST_PER_LOOKUP)
|
||||
|
||||
await trackEvent('logo_fetched', {
|
||||
domain,
|
||||
size: input.size || 128,
|
||||
greyscale: input.greyscale || false,
|
||||
})
|
||||
|
||||
return {
|
||||
logoUrl: file.url,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import axios, { isAxiosError } from 'axios'
|
||||
import { trackEvent } from '../tracking'
|
||||
import { browsePages } from './browse-pages'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type BingSearchResult = {
|
||||
name: string
|
||||
url: string
|
||||
snippet: string
|
||||
links?: {
|
||||
name: string
|
||||
url: string
|
||||
}[]
|
||||
}
|
||||
|
||||
export const webSearch: bp.IntegrationProps['actions']['webSearch'] = async ({
|
||||
client,
|
||||
input,
|
||||
logger,
|
||||
ctx,
|
||||
metadata,
|
||||
}) => {
|
||||
const clientConfig = client._inner.config
|
||||
|
||||
const axiosConfig = {
|
||||
...clientConfig,
|
||||
baseURL: `${clientConfig.apiUrl}/v1/cognitive`,
|
||||
}
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const { data: searchResults } = await axios.post<BingSearchResult[]>('bing/search', input, axiosConfig)
|
||||
logger.forBot().debug(`Search Web using Bing took ${Date.now() - startTime}ms`)
|
||||
|
||||
await trackEvent('web_search_performed', {
|
||||
queryLength: input.query.length,
|
||||
resultCount: searchResults.length,
|
||||
browsePages: input.browsePages || false,
|
||||
durationMs: Date.now() - startTime,
|
||||
})
|
||||
|
||||
if (!input.browsePages) {
|
||||
return { results: searchResults }
|
||||
}
|
||||
|
||||
const pageContent = await browsePages({
|
||||
input: { urls: searchResults.map((x) => x.url) },
|
||||
logger,
|
||||
client,
|
||||
type: 'browsePages',
|
||||
ctx,
|
||||
metadata,
|
||||
})
|
||||
|
||||
return {
|
||||
results: searchResults.map((searchResult) => ({
|
||||
...searchResult,
|
||||
page: pageContent.results?.find((page) => page.url === searchResult.url),
|
||||
})),
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAxiosError(err)) {
|
||||
logger.forBot().error('Error during web search', { error: err.response?.data })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user