chore: import upstream snapshot with attribution

integrations/browser/src/actions/browse-pages.ts

@@ -0,0 +1,112 @@
+import { IntegrationLogger, RuntimeError } from '@botpress/sdk'
+import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
+import { FullPage } from 'src/definitions/actions'
+import { trackEvent } from '../tracking'
+import * as bp from '.botpress'
+
+const COST_PER_PAGE = 0.0015
+
+const fixOutput = (val: unknown): string => {
+  if (typeof val === 'string') {
+    return val
+  } else if (Array.isArray(val)) {
+    return val.join(' ')
+  }
+  return ''
+}
+
+const getPageContent = async (props: {
+  url: string
+  logger: IntegrationLogger
+  waitFor?: number
+  timeout?: number
+  maxAge?: number
+}): Promise<FullPage> => {
+  const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
+
+  const startTime = Date.now()
+
+  try {
+    const result = await firecrawl.scrape(props.url, {
+      onlyMainContent: true,
+      maxAge: 60 * 60 * 24 * 7, // 1 week
+      removeBase64Images: true,
+      waitFor: props.waitFor,
+      timeout: props.timeout,
+      formats: ['markdown', 'rawHtml'],
+      headers: { 'X-Botpress-Crawler': 'botpress' },
+      storeInCache: true,
+    })
+
+    props.logger.forBot().debug(`Firecrawl API call took ${Date.now() - startTime}ms for url: ${props.url}`)
+
+    const contentLength = result.markdown?.length || 0
+    const isLargePage = contentLength > 50000
+
+    if (isLargePage) {
+      await trackEvent('large_page_scraped', {
+        url: props.url,
+        contentLength,
+        durationMs: Date.now() - startTime,
+      })
+    }
+
+    return {
+      url: props.url,
+      content: result.markdown!,
+      raw: result.rawHtml!,
+      favicon: fixOutput(result.metadata?.favicon),
+      title: fixOutput(result.metadata?.title),
+      description: fixOutput(result.metadata?.description),
+    }
+  } catch (err) {
+    props.logger.error('There was an error while calling Firecrawl API.', err)
+
+    if (err instanceof SdkError) {
+      const isRateLimit = err.status === 429 || err.message.includes('rate limit')
+      await trackEvent('firecrawl_error', {
+        url: props.url,
+        errorType: isRateLimit ? 'rate_limited' : 'api_error',
+        errorMessage: err.message,
+        statusCode: err.status,
+        errorCode: err.code,
+      })
+    }
+
+    throw new RuntimeError(`There was an error while browsing the page: ${props.url}`)
+  }
+}
+
+export const browsePages: bp.IntegrationProps['actions']['browsePages'] = async ({ input, logger, metadata }) => {
+  const startTime = Date.now()
+
+  try {
+    const pageContentPromises = await Promise.allSettled(
+      input.urls.map((url) => getPageContent({ url, logger, waitFor: input.waitFor, timeout: input.timeout }))
+    )
+
+    const results = pageContentPromises
+      .filter((promise): promise is PromiseFulfilledResult<FullPage> => promise.status === 'fulfilled')
+      .map((result) => result.value)
+
+    // only charging for successful pages
+    const cost = results.length * COST_PER_PAGE
+    metadata.setCost(cost)
+
+    await trackEvent('pages_browsed', {
+      urlCount: input.urls.length,
+      successCount: results.length,
+      failedCount: input.urls.length - results.length,
+      durationMs: Date.now() - startTime,
+    })
+
+    return {
+      results,
+    }
+  } catch (err) {
+    logger.forBot().error('There was an error while browsing the page.', err)
+    throw err
+  } finally {
+    logger.forBot().info(`Browsing ${input.urls.length} urls took ${Date.now() - startTime}ms`)
+  }
+}

integrations/browser/src/actions/capture-screenshot.ts

@@ -0,0 +1,94 @@
+import axios, { isAxiosError } from 'axios'
+import { trackEvent } from '../tracking'
+import * as bp from '.botpress'
+
+const COST_PER_PAGE = 0.0015
+
+export const captureScreenshot: bp.IntegrationProps['actions']['captureScreenshot'] = async ({
+  input,
+  logger,
+  metadata,
+  client,
+}) => {
+  logger.forBot().debug('Capturing Screenshot', { input })
+
+  const qs = new URLSearchParams({
+    url: input.url,
+    width: Math.max(input.width || 1080, 320).toString(),
+    height: Math.max(input.height || 1920, 240).toString(),
+    full_page: input.fullPage ? 'true' : 'false',
+    fresh: 'true',
+    output: 'json',
+    file_type: 'png',
+    wait_for_event: 'load',
+    extract_html: 'true',
+    lazy_load: 'true',
+    delay: '500',
+  })
+
+  if (input.javascriptToInject) {
+    if (input.javascriptToInject.length > 100) {
+      const { file: jsFile } = await client.uploadFile({
+        key: `screenshot-js-${Date.now()}.js`,
+        content: input.javascriptToInject,
+        accessPolicies: ['public_content'],
+        publicContentImmediatelyAccessible: true,
+        expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
+        index: false,
+      })
+      qs.append('js_url', jsFile.url)
+    } else {
+      qs.append('js', input.javascriptToInject)
+    }
+  }
+
+  if (input.cssToInject) {
+    if (input.cssToInject.length > 100) {
+      const { file: cssFile } = await client.uploadFile({
+        key: `screenshot-css-${Date.now()}.css`,
+        content: input.cssToInject,
+        accessPolicies: ['public_content'],
+        publicContentImmediatelyAccessible: true,
+        expiresAt: new Date(Date.now() + 5 * 60 * 1000).toISOString(), // 5 minutes
+        index: false,
+      })
+      qs.append('css_url', cssFile.url)
+    } else {
+      qs.append('css', input.cssToInject)
+    }
+  }
+
+  const apiUrl = `https://shot.screenshotapi.net/screenshot?token=${bp.secrets.SCREENSHOT_API_KEY}&${qs.toString()}`
+
+  try {
+    const { data } = await axios.get(apiUrl)
+
+    if (data.screenshot) {
+      metadata.setCost(COST_PER_PAGE)
+
+      await trackEvent('screenshot_captured', {
+        domain: new URL(input.url).hostname,
+        width: input.width || 1080,
+        height: input.height || 1920,
+        fullPage: input.fullPage || false,
+        hasJsInjection: !!input.javascriptToInject,
+        hasCssInjection: !!input.cssToInject,
+      })
+
+      return { imageUrl: data.screenshot, htmlUrl: data.extracted_html }
+    } else {
+      throw new Error('Screenshot not available')
+    }
+  } catch (error) {
+    if (isAxiosError(error)) {
+      logger.forBot().error('There was an error while taking the screenshot', error.response?.data)
+      await trackEvent('screenshot_error', {
+        domain: new URL(input.url).hostname,
+        statusCode: error.response?.status,
+        errorMessage: error.message,
+      })
+    }
+
+    throw error
+  }
+}

integrations/browser/src/actions/discover-urls.ts

@@ -0,0 +1,200 @@
+import { RuntimeError } from '@botpress/client'
+import { IntegrationLogger, z } from '@botpress/sdk'
+import Firecrawl, { SdkError } from '@mendable/firecrawl-js'
+import { trackEvent } from '../tracking'
+import { isValidGlob, matchGlob } from '../utils/globs'
+import * as bp from '.botpress'
+
+const LAMBDA_TIMEOUT = 55_000
+
+const COST_PER_FIRECRAWL_MAP = 0.001
+
+type StopReason = Awaited<ReturnType<bp.IntegrationProps['actions']['discoverUrls']>>['stopReason']
+
+type ZodIssueCode = z.ZodIssue['code']
+
+export const urlSchema = z.string().downstream((url) => {
+  url = url.trim()
+  if (!url.includes('://')) {
+    url = `https://${url}`
+  }
+  try {
+    const x = new URL(url)
+    if (x.protocol !== 'http:' && x.protocol !== 'https:') {
+      return z.ERR({
+        code: 'custom' satisfies ZodIssueCode,
+        message: 'Invalid protocol, only URLs starting with HTTP and HTTPS are supported',
+      })
+    }
+
+    if (!/.\.[a-zA-Z]{2,}$/.test(x.hostname)) {
+      return z.ERR({
+        code: 'custom' satisfies ZodIssueCode,
+        message: 'Invalid TLD',
+      })
+    }
+    const pathName = x.pathname.endsWith('/') ? x.pathname.slice(0, -1) : x.pathname
+    return z.OK(`${x.origin}${pathName}${x.search ? x.search : ''}`)
+  } catch (caught) {
+    const err = caught instanceof Error ? caught : new Error('Unknown error while parsing URL')
+    return z.ERR({
+      code: 'custom' satisfies ZodIssueCode,
+      message: 'Invalid URL: ' + err.message,
+    })
+  }
+})
+
+class Accumulator {
+  public processed = new Set<string>()
+  public urls = new Set<string>()
+
+  public included: number = 0
+  public excluded: number = 0
+
+  public cost: number = 0
+
+  public startedAt: number = Date.now()
+
+  public addUrls(urls: string[]): void {
+    for (let url of urls) {
+      if (this.stopReason) {
+        return
+      }
+
+      const parsed = urlSchema.safeParse(url)
+
+      if (!parsed.success) {
+        continue
+      }
+
+      url = parsed.data
+
+      if (this.onlyHttps && !url.toLowerCase().startsWith('https://')) {
+        this.excluded++
+        continue
+      }
+
+      if (this.processed.has(url)) {
+        continue
+      }
+
+      this.processed.add(url)
+
+      if (this.includedGlobs.length && this.includedGlobs.every((glob) => !matchGlob(url, glob))) {
+        this.excluded++
+        continue
+      }
+
+      this.included++
+
+      if (this.exclduedGlobs.some((glob) => matchGlob(url, glob))) {
+        this.excluded++
+        continue
+      }
+
+      this.urls.add(url)
+    }
+  }
+
+  public addCost(cost: number): void {
+    this.cost += cost
+  }
+
+  public constructor(
+    public readonly includedGlobs: string[],
+    public readonly exclduedGlobs: string[],
+    public readonly timeout: number = 55_000,
+    public readonly limit: number = 10_000,
+    public readonly onlyHttps: boolean = true
+  ) {}
+
+  public get remainingTime(): number {
+    return Math.max(0, this.startedAt + this.timeout - Date.now())
+  }
+
+  public get stopReason(): StopReason | null {
+    if (this.urls.size >= this.limit) {
+      return 'urls_limit_reached'
+    }
+
+    if (this.remainingTime <= 1000) {
+      // if we have less than 1s left, we consider it a time limit reached
+      return 'time_limit_reached'
+    }
+
+    return null
+  }
+}
+
+const firecrawlMap = async (props: { url: string; logger: IntegrationLogger; timeout: number }): Promise<string[]> => {
+  const firecrawl = new Firecrawl({ apiKey: bp.secrets.FIRECRAWL_API_KEY })
+
+  try {
+    const result = await firecrawl.map(props.url, {
+      sitemap: 'include',
+      limit: 10_000,
+      timeout: Math.max(1000, props.timeout - 2000),
+      includeSubdomains: true,
+    })
+
+    return result.links.map((x) => x.url)
+  } catch (err) {
+    props.logger.error('There was an error while calling Firecrawl map API.', err)
+
+    if (err instanceof SdkError) {
+      const isRateLimit = err.status === 429 || err.message.includes('rate limit')
+      await trackEvent('firecrawl_error', {
+        url: props.url,
+        operation: 'map',
+        errorType: isRateLimit ? 'rate_limited' : 'api_error',
+        errorMessage: err.message,
+        statusCode: err.status,
+        errorCode: err.code,
+      })
+    }
+
+    throw err
+  }
+}
+
+export const discoverUrls: bp.IntegrationProps['actions']['discoverUrls'] = async ({ input, logger, metadata }) => {
+  logger.forBot().debug('Discovering website URLs', { input })
+
+  for (const pattern of [...(input.include ?? []), ...(input.exclude ?? [])]) {
+    if (!isValidGlob(pattern)) {
+      throw new RuntimeError(`Forbidden characters in glob pattern: ${pattern}. Cannot contain "..", "{}", "[]"`)
+    }
+  }
+
+  const accumulator = new Accumulator(
+    input.include ?? [],
+    input.exclude ?? [],
+    LAMBDA_TIMEOUT,
+    input.count ?? 10_000,
+    input.onlyHttps
+  )
+
+  const firecrawl = firecrawlMap({ url: input.url, logger, timeout: accumulator.remainingTime }).then((links) => {
+    accumulator.addUrls(links)
+    accumulator.addCost(COST_PER_FIRECRAWL_MAP)
+  })
+
+  await Promise.allSettled([firecrawl])
+
+  metadata.setCost(accumulator.cost)
+
+  await trackEvent('urls_discovered', {
+    baseUrl: input.url,
+    urlCount: accumulator.urls.size,
+    excludedCount: accumulator.excluded,
+    stopReason: accumulator.stopReason ?? 'end_of_results',
+    hasIncludeFilters: (input.include?.length ?? 0) > 0,
+    hasExcludeFilters: (input.exclude?.length ?? 0) > 0,
+  })
+
+  return {
+    excluded: accumulator.excluded,
+    stopReason: accumulator.stopReason ?? 'end_of_results',
+    urls: [...accumulator.urls],
+  }
+}

integrations/browser/src/actions/get-website-logo.ts

@@ -0,0 +1,53 @@
+import { trackEvent } from '../tracking'
+import * as bp from '.botpress'
+
+const COST_PER_LOOKUP = 0.25
+
+export const getWebsiteLogo: bp.IntegrationProps['actions']['getWebsiteLogo'] = async ({
+  input,
+  client,
+  logger,
+  metadata,
+}) => {
+  logger.forBot().debug('Fetching logo for URL', { input })
+
+  const domain = input.domain.trim().replace(/^(https?:\/\/)?(www\.)?/, '')
+
+  const params = new URLSearchParams({
+    format: 'png',
+    size: (input.size || 128)?.toString(),
+    token: bp.secrets.LOGO_API_KEY,
+  })
+
+  if (input.greyscale) {
+    params.append('greyscale', 'true')
+  }
+
+  const url = `https://img.logo.dev/${domain}?${params.toString()}`
+
+  const buffer = (await fetch(url)).arrayBuffer()
+
+  const { file } = await client.uploadFile({
+    key: `browser/logos/${domain}/logo.png`,
+    content: await buffer,
+    accessPolicies: ['public_content'],
+    publicContentImmediatelyAccessible: true,
+    metadata: {
+      domain,
+      greyscale: input.greyscale || false,
+      size: input.size || 128,
+    },
+  })
+
+  metadata.setCost(COST_PER_LOOKUP)
+
+  await trackEvent('logo_fetched', {
+    domain,
+    size: input.size || 128,
+    greyscale: input.greyscale || false,
+  })
+
+  return {
+    logoUrl: file.url,
+  }
+}

integrations/browser/src/actions/web-search.ts

@@ -0,0 +1,67 @@
+import axios, { isAxiosError } from 'axios'
+import { trackEvent } from '../tracking'
+import { browsePages } from './browse-pages'
+import * as bp from '.botpress'
+
+type BingSearchResult = {
+  name: string
+  url: string
+  snippet: string
+  links?: {
+    name: string
+    url: string
+  }[]
+}
+
+export const webSearch: bp.IntegrationProps['actions']['webSearch'] = async ({
+  client,
+  input,
+  logger,
+  ctx,
+  metadata,
+}) => {
+  const clientConfig = client._inner.config
+
+  const axiosConfig = {
+    ...clientConfig,
+    baseURL: `${clientConfig.apiUrl}/v1/cognitive`,
+  }
+  const startTime = Date.now()
+
+  try {
+    const { data: searchResults } = await axios.post<BingSearchResult[]>('bing/search', input, axiosConfig)
+    logger.forBot().debug(`Search Web using Bing took ${Date.now() - startTime}ms`)
+
+    await trackEvent('web_search_performed', {
+      queryLength: input.query.length,
+      resultCount: searchResults.length,
+      browsePages: input.browsePages || false,
+      durationMs: Date.now() - startTime,
+    })
+
+    if (!input.browsePages) {
+      return { results: searchResults }
+    }
+
+    const pageContent = await browsePages({
+      input: { urls: searchResults.map((x) => x.url) },
+      logger,
+      client,
+      type: 'browsePages',
+      ctx,
+      metadata,
+    })
+
+    return {
+      results: searchResults.map((searchResult) => ({
+        ...searchResult,
+        page: pageContent.results?.find((page) => page.url === searchResult.url),
+      })),
+    }
+  } catch (err) {
+    if (isAxiosError(err)) {
+      logger.forBot().error('Error during web search', { error: err.response?.data })
+    }
+    throw err
+  }
+}

integrations/browser/src/definitions/actions.ts

@@ -0,0 +1,232 @@
+import { ActionDefinition, z } from '@botpress/sdk'
+
+const multiLineString = z.string().displayAs({ id: 'text', params: { multiLine: true, growVertically: true } })
+
+const captureScreenshot: ActionDefinition = {
+  title: 'Capture Screenshot',
+  description: 'Capture a screenshot of the specified page.',
+  input: {
+    schema: z.object({
+      url: z.string().describe('The url to screenshot').title('Url'),
+      javascriptToInject: multiLineString
+        .optional()
+        .describe('JavaScript code to inject into the page before taking the screenshot')
+        .title('Javascript to Inject'),
+      cssToInject: multiLineString
+        .optional()
+        .describe('CSS code to inject into the page before taking the screenshot')
+        .title('CSS To Inject'),
+      width: z.number().default(1080).describe('The width of the screenshot').title('Width'),
+      height: z.number().default(1920).describe('The height of the screenshot').title('Height'),
+      fullPage: z.boolean().default(true).describe('Whether the screenshot is fullscreen or not').title('Full Page'),
+    }),
+  },
+  output: {
+    schema: z.object({
+      imageUrl: z.string().describe('URL to the captured screenshot').title('Image Url'),
+      htmlUrl: z.string().optional().describe('URL to the HTML page of the screenshot').title('Html Url'),
+    }),
+  },
+  cacheable: true,
+  billable: true,
+}
+
+const fullPage = z.object({
+  url: z.string(),
+  content: z.string(),
+  raw: z.string(),
+  favicon: z.string().optional(),
+  title: z.string().optional(),
+  description: z.string().optional(),
+})
+
+export type FullPage = z.infer<typeof fullPage>
+
+const getWebsiteLogo: ActionDefinition = {
+  title: 'Get Website Logo',
+  description: 'Get the logo of the specified website.',
+  input: {
+    schema: z.object({
+      domain: z.string().describe('The domain of the website to get the logo from (eg. "example.com")').title('Domain'),
+      greyscale: z
+        .boolean()
+        .default(false)
+        .describe('Whether to return the logo in grayscale (black & white)')
+        .title('Grayscale'),
+      size: z
+        .enum(['64', '128', '256', '512'])
+        .default('128')
+        .describe('Size of the logo to return (64, 128 or 256, 512 pixels)')
+        .title('Size'),
+    }),
+  },
+  output: {
+    schema: z.object({
+      logoUrl: z.string().describe('URL to the website logo').title('Logo Url'),
+    }),
+  },
+  cacheable: false,
+  billable: true,
+}
+
+const browsePages: ActionDefinition = {
+  title: 'Browse Pages',
+  description: 'Extract the full content & the metadata of the specified pages as markdown.',
+  input: {
+    schema: z.object({
+      urls: z.array(z.string()).describe('The list of url to browse').title('Urls'),
+      waitFor: z
+        .number()
+        .optional()
+        .default(350)
+        .describe(
+          'Time to wait before extracting the content (in milliseconds). Set this value higher for dynamic pages.'
+        )
+        .title('Wait For'),
+      timeout: z
+        .number()
+        .optional()
+        .default(30000)
+        .describe('Timeout for the request (in milliseconds)')
+        .title('Time Out'),
+      maxAge: z
+        .number()
+        .optional()
+        .default(60 * 60 * 24 * 7)
+        .describe('Maximum age of the cached page content (in seconds)')
+        .title('Max Age'),
+    }),
+  },
+  output: {
+    schema: z.object({
+      results: z.array(fullPage).describe('The list of pages browsed'),
+    }),
+  },
+  cacheable: true,
+  billable: true,
+}
+
+const domainNameRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,}$/i
+const domainNameValidator = z
+  .string()
+  .regex(domainNameRegex, 'Invalid domain name')
+  .min(3, 'Invalid URL')
+  .max(50, 'Domain name is too long')
+
+const webSearch: ActionDefinition = {
+  title: 'Web Search',
+  description: 'Search information on the web. You need to browse to that page to get the full content of the page.',
+  input: {
+    schema: z.object({
+      query: z.string().min(1).max(1000).describe('What are we searching for?').title('Query'),
+      includeSites: z
+        .array(domainNameValidator)
+        .max(20)
+        .optional()
+        .describe('Include only these domains in the search (max 20)')
+        .title('Include Site'),
+      excludeSites: z
+        .array(domainNameValidator)
+        .max(20)
+        .optional()
+        .describe('Exclude these domains from the search (max 20)')
+        .title('Exclude Site'),
+      count: z
+        .number()
+        .min(1)
+        .max(20)
+        .optional()
+        .default(10)
+        .describe('Number of search results to return (default: 10)')
+        .title('Count'),
+      freshness: z
+        .enum(['Day', 'Week', 'Month'])
+        .optional()
+        .describe('Only consider results from the last day, week or month')
+        .title('Freshness'),
+      browsePages: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe('Whether to browse to the pages to get the full content')
+        .title('Browse Pages'),
+    }),
+  },
+  output: {
+    schema: z.object({
+      results: z
+        .array(
+          z.object({
+            name: z.string().describe('Title of the page'),
+            url: z.string().describe('URL of the page'),
+            snippet: z.string().describe('A short summary of the page'),
+            links: z
+              .array(z.object({ name: z.string(), url: z.string() }))
+              .optional()
+              .describe('Useful links on the page'),
+            page: fullPage.optional().describe('The page itself'),
+          })
+        )
+        .describe('Results of the search'),
+    }),
+  },
+  billable: true,
+  cacheable: true,
+}
+
+export const globPattern = z
+  .string()
+  .min(1, 'Glob must be at least 1 char')
+  .max(255, 'Glob must be at max 255 chars')
+  .describe('Glob pattern to match URLs. Use * for wildcard matching')
+
+const discoverUrls: ActionDefinition = {
+  title: 'Discover Website URLs',
+  description: 'Discovers the URLs of a website by finding links using sitemaps, robots.txt, and crawling.',
+  input: {
+    schema: z.object({
+      url: z
+        .string()
+        .describe(
+          'The URL of the website to discover URLs from. Can be a domain like example.com or a full URL like sub.example.com/page'
+        )
+        .title('Url'),
+      onlyHttps: z.boolean().default(true).describe('Whether to only include HTTPS pages').title('Only HTTPS'),
+      count: z.number().min(1).max(10_000).default(5_000).describe('The number of urls').title('Count'),
+      include: z
+        .array(globPattern)
+        .max(100, 'You can include up to 100 URL patterns')
+        .describe('List of glob patterns to include URLs from the discovery')
+        .title('Include')
+        .optional(),
+      exclude: z
+        .array(globPattern)
+        .max(100, 'You can exclude up to 100 URL patterns')
+        .optional()
+        .describe(
+          'List of glob patterns to exclude URLs from the discovery. All URLs matching these patterns will be excluded from the results, even if they are included in the "include" patterns.'
+        )
+        .title('Exclude'),
+    }),
+  },
+  output: {
+    schema: z.object({
+      urls: z.array(z.string()).describe('List of discovered URLs').title('Urls'),
+      excluded: z.number().describe('Number of URLs excluded due to robots.txt or filter').title('Excluded'),
+      stopReason: z
+        .enum(['urls_limit_reached', 'end_of_results', 'time_limit_reached'])
+        .describe('Reason for stopping the URLs discovery. ')
+        .title('Stop Reason'),
+    }),
+  },
+  billable: true,
+  cacheable: false,
+}
+
+export const actionDefinitions = {
+  captureScreenshot,
+  browsePages,
+  webSearch,
+  discoverUrls,
+  getWebsiteLogo,
+} satisfies Record<string, ActionDefinition>

integrations/browser/src/index.ts

@@ -0,0 +1,40 @@
+import { posthogHelper } from '@botpress/common'
+import { INTEGRATION_NAME, INTEGRATION_VERSION } from 'integration.definition'
+import { browsePages } from './actions/browse-pages'
+import { captureScreenshot } from './actions/capture-screenshot'
+import { discoverUrls } from './actions/discover-urls'
+import { getWebsiteLogo } from './actions/get-website-logo'
+import { webSearch } from './actions/web-search'
+import { trackEvent } from './tracking'
+import * as bp from '.botpress'
+
+const integrationConfig: bp.IntegrationProps = {
+  register: async ({ ctx }) => {
+    await trackEvent('browser_registered', {
+      integrationId: ctx.integrationId,
+    })
+  },
+  unregister: async ({ ctx }) => {
+    await trackEvent('browser_unregistered', {
+      integrationId: ctx.integrationId,
+    })
+  },
+  actions: {
+    captureScreenshot,
+    browsePages,
+    webSearch,
+    discoverUrls,
+    getWebsiteLogo,
+  },
+  channels: {},
+  handler: async () => {},
+}
+
+export default posthogHelper.wrapIntegration(
+  {
+    integrationName: INTEGRATION_NAME,
+    key: bp.secrets.POSTHOG_KEY,
+    integrationVersion: INTEGRATION_VERSION,
+  },
+  integrationConfig
+)

integrations/browser/src/tracking.ts

@@ -0,0 +1,42 @@
+import { posthogHelper } from '@botpress/common'
+import { INTEGRATION_NAME, INTEGRATION_VERSION } from 'integration.definition'
+import * as bp from '.botpress'
+
+type TrackingEvent =
+  | 'browser_registered'
+  | 'browser_unregistered'
+  | 'screenshot_captured'
+  | 'web_search_performed'
+  | 'pages_browsed'
+  | 'large_page_scraped'
+  | 'urls_discovered'
+  | 'logo_fetched'
+  | 'firecrawl_error'
+  | 'screenshot_error'
+
+type EventProperties = Record<string, string | number | boolean | undefined>
+
+const getPostHogConfig = () => ({
+  key: bp.secrets.POSTHOG_KEY,
+  integrationName: INTEGRATION_NAME,
+  integrationVersion: INTEGRATION_VERSION,
+})
+
+export const trackEvent = async (
+  event: TrackingEvent,
+  properties: EventProperties,
+  distinctId?: string
+): Promise<void> => {
+  try {
+    await posthogHelper.sendPosthogEvent(
+      {
+        distinctId: distinctId ?? 'no id',
+        event,
+        properties,
+      },
+      getPostHogConfig()
+    )
+  } catch {
+    // Silently fail
+  }
+}

integrations/browser/src/utils/globs.test.ts

@@ -0,0 +1,39 @@
+import { it, expect } from 'vitest'
+import { isValidGlob, matchGlob } from './globs'
+
+it('isValidGlob', () => {
+  expect(isValidGlob('*')).toBe(true)
+  expect(isValidGlob('/admin/')).toBe(true)
+  expect(isValidGlob('*/admin')).toBe(true)
+  expect(isValidGlob('https://botpress.com/*')).toBe(true)
+  expect(isValidGlob('https://botpress.com/*/blog')).toBe(true)
+
+  expect(isValidGlob('https://botpress.com/**/blog')).toBe(false)
+  expect(isValidGlob('**')).toBe(false)
+  expect(isValidGlob('**')).toBe(false)
+  expect(isValidGlob('')).toBe(false)
+  expect(isValidGlob('          ')).toBe(false)
+
+  expect(isValidGlob('https://*.botpress.com/*/blog')).toBe(false)
+})
+
+it('matchGlob', () => {
+  expect(matchGlob('https://example.com', '*')).toBe(true)
+  expect(matchGlob('https://example.com/path', '*')).toBe(true)
+  expect(matchGlob('http://sub.example.com', '*')).toBe(true)
+
+  expect(matchGlob('https://example.com/admin/dashboard', '/admin/')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', 'admin')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', '*/dashboard')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/admin/dashboard')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/admin/dashboard*')).toBe(true)
+
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*/dashboard')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com')).toBe(true)
+  expect(matchGlob('https://example.com/admin/dashboard', '*example.com/admin/dashboard')).toBe(true)
+
+  expect(matchGlob('https://example.com/admin/dashboard', 'world')).toBe(false)
+  expect(matchGlob('https://example.com/admin/dashboard', 'http://example.com/*')).toBe(false)
+  expect(matchGlob('https://example.com/admin/dashboard', 'https://example.com/*/dashboard/hello')).toBe(false)
+})

integrations/browser/src/utils/globs.ts

@@ -0,0 +1,43 @@
+export const isValidGlob = (glob: string): boolean => {
+  if (glob.split('*').length > 2) {
+    // More than one * is not supported
+    return false
+  }
+
+  return glob.trim().length > 0 && glob.trim().length <= 255
+}
+
+export const matchGlob = (url: string, glob: string): boolean => {
+  url = url.toLowerCase().trim()
+  glob = glob.toLowerCase().trim()
+
+  // If glob is just *, it matches everything
+  if (glob === '*') {
+    return true
+  }
+
+  if (!glob.includes('*')) {
+    // If glob doesn't contain any wildcard, we just check for presence
+    return url.includes(glob)
+  }
+
+  // if starts with *, we check that url ends with glob
+  if (glob.startsWith('*')) {
+    const trimmedGlob = glob.slice(1) // Remove the leading *
+    return url.endsWith(trimmedGlob)
+  }
+
+  // if ends with *, we check that url starts with glob
+  if (glob.endsWith('*')) {
+    const trimmedGlob = glob.slice(0, -1) // Remove the trailing *
+    return url.startsWith(trimmedGlob)
+  }
+
+  // if * in the middle, we check that starts with the part before * and ends with the part after *
+  if (glob.includes('*')) {
+    const [start, end] = glob.split('*')
+    return url.startsWith(start!) && url.endsWith(end!)
+  }
+
+  return false
+}