chore: import upstream snapshot with attribution

packages/cli/src/utils/attribute-utils.ts

@@ -0,0 +1,30 @@
+import * as records from './record-utils'
+
+type ItemsWithAttributes = Record<
+  string,
+  {
+    attributes?: {
+      [key: string]: string | null
+    }
+  } | null
+>
+
+export const prepareAttributeUpdateBody = <TLocalItems extends ItemsWithAttributes>({
+  localItems,
+  remoteItems,
+}: {
+  localItems: TLocalItems
+  remoteItems: ItemsWithAttributes
+}): TLocalItems => {
+  const clonedLocalItems = structuredClone(localItems)
+
+  for (const [itemName, item] of Object.entries(clonedLocalItems)) {
+    if (!item || !remoteItems || !remoteItems[itemName]) {
+      continue
+    }
+
+    item.attributes = records.setNullOnMissingValues(item.attributes, remoteItems[itemName].attributes ?? {})
+  }
+
+  return clonedLocalItems
+}

packages/cli/src/utils/cache-utils.ts

@@ -0,0 +1,84 @@
+import fs from 'fs'
+import pathlib from 'path'
+import * as json from './json-utils'
+
+export class FSKeyValueCache<T extends Object> {
+  private _initialized = false
+
+  public constructor(private _filepath: string) {}
+
+  public async init(): Promise<void> {
+    if (this._initialized) {
+      return
+    }
+    const dirname = pathlib.dirname(this._filepath)
+    if (!fs.existsSync(dirname)) {
+      await fs.promises.mkdir(dirname, { recursive: true })
+    }
+    if (!fs.existsSync(this._filepath)) {
+      await this._writeJSON(this._filepath, {})
+    }
+    this._initialized = true
+  }
+
+  public async sync<K extends keyof T>(
+    key: K,
+    value: T[K] | undefined,
+    prompt: (initial?: T[K]) => Promise<T[K]>
+  ): Promise<T[K]> {
+    await this.init()
+    if (value) {
+      await this.set(key, value)
+      return value
+    }
+    const data = await this.get(key)
+    const newValue = await prompt(data)
+    await this.set(key, newValue)
+    return newValue
+  }
+
+  public async has<K extends keyof T>(key: K): Promise<boolean> {
+    await this.init()
+    const data = await this._readJSON(this._filepath)
+    return data[key] !== undefined
+  }
+
+  public async get<K extends keyof T>(key: K): Promise<T[K] | undefined> {
+    await this.init()
+    const data: T = await this._readJSON(this._filepath)
+    return data[key]
+  }
+
+  public async set<K extends keyof T>(key: K, value: T[K]): Promise<void> {
+    await this.init()
+    const data: T = await this._readJSON(this._filepath)
+    data[key] = value
+    return this._writeJSON(this._filepath, data)
+  }
+
+  public async rm<K extends keyof T>(key: K): Promise<void> {
+    await this.init()
+    const data: T = await this._readJSON(this._filepath)
+    delete data[key]
+    return this._writeJSON(this._filepath, data)
+  }
+
+  public async clear(): Promise<void> {
+    await this.init()
+    return this._writeJSON(this._filepath, {})
+  }
+
+  private _writeJSON = (filepath: string, data: any) => {
+    const fileContent = JSON.stringify(data, null, 2)
+    return fs.promises.writeFile(filepath, fileContent)
+  }
+
+  private _readJSON = async (filepath: string) => {
+    const fileContent = await fs.promises.readFile(filepath, 'utf8')
+    const parseResult = json.safeParseJson(fileContent)
+    if (!parseResult.success) {
+      throw new Error(`Failed to parse JSON file at ${filepath}: ${parseResult.error.message}`)
+    }
+    return parseResult.data as T
+  }
+}

packages/cli/src/utils/case-utils.test.ts

@@ -0,0 +1,59 @@
+import { test, expect } from 'vitest'
+import * as caseUtils from './case-utils'
+
+const pascalCase = 'HelloLittleWorld'
+const kebabCase = 'hello-little-world'
+const snakeCase = 'hello_little_world'
+const screamingSnakeCase = 'HELLO_LITTLE_WORLD'
+const camelCase = 'helloLittleWorld'
+
+test('case utils should convert from pascal case to all other cases', () => {
+  expect(caseUtils.to.kebabCase(pascalCase)).toBe(kebabCase)
+  expect(caseUtils.to.snakeCase(pascalCase)).toBe(snakeCase)
+  expect(caseUtils.to.screamingSnakeCase(pascalCase)).toBe(screamingSnakeCase)
+  expect(caseUtils.to.camelCase(pascalCase)).toBe(camelCase)
+})
+
+test('case utils should convert from kebab case to all other cases', () => {
+  expect(caseUtils.to.pascalCase(kebabCase)).toBe(pascalCase)
+  expect(caseUtils.to.snakeCase(kebabCase)).toBe(snakeCase)
+  expect(caseUtils.to.screamingSnakeCase(kebabCase)).toBe(screamingSnakeCase)
+  expect(caseUtils.to.camelCase(kebabCase)).toBe(camelCase)
+})
+
+test('case utils should convert from snake case to all other cases', () => {
+  expect(caseUtils.to.pascalCase(snakeCase)).toBe(pascalCase)
+  expect(caseUtils.to.kebabCase(snakeCase)).toBe(kebabCase)
+  expect(caseUtils.to.screamingSnakeCase(snakeCase)).toBe(screamingSnakeCase)
+  expect(caseUtils.to.camelCase(snakeCase)).toBe(camelCase)
+})
+
+test('case utils should convert from screaming snake case to all other cases', () => {
+  expect(caseUtils.to.pascalCase(screamingSnakeCase)).toBe(pascalCase)
+  expect(caseUtils.to.kebabCase(screamingSnakeCase)).toBe(kebabCase)
+  expect(caseUtils.to.snakeCase(screamingSnakeCase)).toBe(snakeCase)
+  expect(caseUtils.to.camelCase(screamingSnakeCase)).toBe(camelCase)
+})
+
+test('case utils should convert from camel case to all other cases', () => {
+  expect(caseUtils.to.pascalCase(camelCase)).toBe(pascalCase)
+  expect(caseUtils.to.kebabCase(camelCase)).toBe(kebabCase)
+  expect(caseUtils.to.snakeCase(camelCase)).toBe(snakeCase)
+  expect(caseUtils.to.screamingSnakeCase(camelCase)).toBe(screamingSnakeCase)
+})
+
+test('case utils should split special characters when converting', () => {
+  expect(caseUtils.to.pascalCase(`${pascalCase}.2`)).toBe(`${pascalCase}2`)
+  expect(caseUtils.to.kebabCase(`${kebabCase}.2`)).toBe(`${kebabCase}-2`)
+  expect(caseUtils.to.snakeCase(`${snakeCase}.2`)).toBe(`${snakeCase}_2`)
+  expect(caseUtils.to.screamingSnakeCase(`${screamingSnakeCase}.2`)).toBe(`${screamingSnakeCase}_2`)
+  expect(caseUtils.to.camelCase(`${camelCase}.2`)).toBe(`${camelCase}2`)
+})
+
+test('case utils should prevent special characters when checking', () => {
+  expect(caseUtils.is.pascalCase(`${pascalCase}.2`)).toBe(false)
+  expect(caseUtils.is.kebabCase(`${kebabCase}.2`)).toBe(false)
+  expect(caseUtils.is.snakeCase(`${snakeCase}.2`)).toBe(false)
+  expect(caseUtils.is.screamingSnakeCase(`${screamingSnakeCase}.2`)).toBe(false)
+  expect(caseUtils.is.camelCase(`${camelCase}.2`)).toBe(false)
+})

packages/cli/src/utils/case-utils.ts

@@ -0,0 +1,53 @@
+import _ from 'lodash'
+
+const specialChars = /[^a-zA-Z0-9_-]/g
+
+const capitalizeFirstLetter = (text: string) => text.charAt(0).toUpperCase() + text.slice(1).toLowerCase()
+
+const splitChar = (char: string) => (tokens: string[]) => tokens.flatMap((token) => token.split(char))
+const splitRegex = (regex: RegExp) => (tokens: string[]) => tokens.flatMap((token) => token.split(regex))
+const splitCaseChange = (tokens: string[]) => tokens.flatMap((token) => token.split(/(?<=[a-z])(?=[A-Z])/))
+const splitTokens = (tokens: string[]) => {
+  return [splitRegex(specialChars), splitChar('-'), splitChar('_'), splitCaseChange].reduce(
+    (acc, step) => step(acc),
+    tokens
+  )
+}
+
+const filterOutEmpty = (tokens: string[]) => tokens.filter((token) => token.length > 0)
+const filterOutRegex = (regex: RegExp) => (tokens: string[]) => tokens.filter((token) => !regex.test(token))
+const filterTokens = (tokens: string[]) => {
+  return [filterOutEmpty, filterOutRegex(specialChars)].reduce((acc, step) => step(acc), tokens)
+}
+
+type SupportedCase = `${'pascal' | 'kebab' | 'snake' | 'screamingSnake' | 'camel'}Case`
+
+const fromTokens = {
+  pascalCase: (tokens: string[]) => {
+    return tokens.map(capitalizeFirstLetter).join('')
+  },
+  kebabCase: (tokens: string[]) => {
+    return tokens.map((token) => token.toLowerCase()).join('-')
+  },
+  snakeCase: (tokens: string[]) => {
+    return tokens.map((token) => token.toLowerCase()).join('_')
+  },
+  screamingSnakeCase: (tokens: string[]) => {
+    return tokens.map((token) => token.toUpperCase()).join('_')
+  },
+  camelCase: (tokens: string[]) => {
+    const [first, ...others] = tokens
+    return [first!.toLowerCase(), ...others.map(capitalizeFirstLetter)].join('')
+  },
+} as Record<SupportedCase, (tokens: string[]) => string>
+
+export const to = _.mapValues(fromTokens, (fn) => (text: string) => {
+  let tokens = splitTokens([text])
+  tokens = filterTokens(tokens)
+  return fn(tokens)
+}) satisfies Record<SupportedCase, (text: string) => string>
+
+export const is = _.mapValues(to, (fn) => (text: string) => {
+  const result = fn(text)
+  return result === text
+}) satisfies Record<SupportedCase, (text: string) => boolean>

packages/cli/src/utils/concurrency-utils.test.ts

@@ -0,0 +1,73 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { debounceAsync } from './concurrency-utils'
+
+const TIMEOUT_MS = 100
+
+describe('debounceAsync', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(async () => {
+    vi.useRealTimers()
+    vi.restoreAllMocks()
+  })
+
+  it('should not execute function before delay', async () => {
+    // Arrange
+    const fn = vi.fn().mockResolvedValue('result')
+    const debouncedFn = debounceAsync(fn, TIMEOUT_MS)
+
+    // Act
+    debouncedFn('a')
+    await vi.advanceTimersByTimeAsync(TIMEOUT_MS - 1)
+
+    // Assert
+    expect(fn).not.toHaveBeenCalled()
+  })
+
+  it('should execute function only once after delay', async () => {
+    // Arrange
+    const fn = vi.fn().mockResolvedValue('result')
+    const debouncedFn = debounceAsync(fn, TIMEOUT_MS)
+
+    // Act
+    debouncedFn('a')
+    debouncedFn('b')
+    await vi.advanceTimersByTimeAsync(TIMEOUT_MS - 1)
+    debouncedFn('c')
+    await vi.advanceTimersByTimeAsync(TIMEOUT_MS)
+
+    // Assert
+    expect(fn).toHaveBeenCalledTimes(1)
+    expect(fn).toHaveBeenCalledWith('c')
+  })
+
+  it('should preserve `this` context', async () => {
+    // Arrange
+    const context = { value: 42 }
+    const fn = vi.fn(function (this: typeof context) {
+      return Promise.resolve(this.value)
+    })
+    const debouncedFn = debounceAsync(fn, TIMEOUT_MS)
+
+    // Act
+    const promise = debouncedFn.call(context)
+
+    await vi.advanceTimersByTimeAsync(TIMEOUT_MS)
+
+    // Assert
+    await expect(promise).resolves.toBe(42)
+  })
+
+  it('should handle rejections', async () => {
+    // Arrange
+    const error = new Error('test rejection')
+    const fn = vi.fn().mockRejectedValue(error)
+    const debouncedFn = debounceAsync(fn, 1)
+    vi.useRealTimers()
+
+    // Act & Assert
+    await expect(debouncedFn).rejects.toThrow(error)
+  })
+})

packages/cli/src/utils/concurrency-utils.ts

@@ -0,0 +1,17 @@
+export const debounceAsync = <TArgs extends unknown[], TReturn>(
+  fn: (...args: TArgs) => Promise<TReturn>,
+  ms: number
+): ((...args: TArgs) => Promise<TReturn>) => {
+  let timeout: NodeJS.Timeout | null = null
+
+  return async function (this: unknown, ...args: TArgs): Promise<TReturn> {
+    if (timeout) {
+      clearTimeout(timeout)
+    }
+    return new Promise<TReturn>((resolve, reject) => {
+      timeout = setTimeout(() => {
+        fn.apply(this, args).then(resolve, reject)
+      }, ms)
+    })
+  }
+}

packages/cli/src/utils/esbuild-utils.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it, vi } from 'vitest'
+import { BuildContext } from './esbuild-utils'
+
+class TestBuildContext extends BuildContext<{ value: string }> {
+  public contexts: Array<{
+    dispose: ReturnType<typeof vi.fn>
+    rebuild: ReturnType<typeof vi.fn>
+  }> = []
+
+  protected async _createContext() {
+    const context = {
+      dispose: vi.fn().mockResolvedValue(undefined),
+      rebuild: vi.fn().mockResolvedValue({ errors: [], warnings: [] }),
+    }
+    this.contexts.push(context)
+    return context as never
+  }
+}
+
+describe('BuildContext', () => {
+  it('recreates the underlying esbuild context after disposal', async () => {
+    const context = new TestBuildContext()
+
+    await context.rebuild({ value: 'first' })
+    await context.rebuild({ value: 'first' })
+
+    expect(context.contexts).toHaveLength(1)
+
+    await context.dispose()
+
+    expect(context.contexts[0]?.dispose).toHaveBeenCalledTimes(1)
+
+    await context.rebuild({ value: 'first' })
+
+    expect(context.contexts).toHaveLength(2)
+  })
+})

packages/cli/src/utils/esbuild-utils.ts

@@ -0,0 +1,128 @@
+import * as esb from 'esbuild'
+import _ from 'lodash'
+
+export * from 'esbuild'
+
+type BaseProps = {
+  absWorkingDir: string
+}
+
+export type BuildCodeProps = BaseProps & {
+  code: string
+  outfile: string
+}
+
+export type BuildEntrypointProps = BaseProps & {
+  entrypoint: string
+}
+
+const DEFAULT_OPTIONS: esb.BuildOptions = {
+  bundle: true,
+  sourcemap: false,
+  logLevel: 'silent',
+  platform: 'node',
+  target: 'es2020',
+  legalComments: 'none',
+  logOverride: { 'equals-negative-zero': 'silent' },
+  keepNames: true, // important : https://github.com/node-fetch/node-fetch/issues/784#issuecomment-1014768204
+  minify: false,
+}
+
+export abstract class BuildContext<T> {
+  private _context: esb.BuildContext | undefined
+  private _previousProps: T | undefined
+  private _previousOpts: esb.BuildOptions = {}
+
+  protected abstract _createContext(props: T, opts: esb.BuildOptions): Promise<esb.BuildContext>
+
+  public async rebuild(props: T, opts: esb.BuildOptions = {}) {
+    if (!this._context || !_.isEqual(props, this._previousProps) || !_.isEqual(opts, this._previousOpts)) {
+      if (this._context) {
+        await this._context.dispose()
+      }
+      this._context = await this._createContext(props, opts)
+      this._previousOpts = opts
+      this._previousProps = props
+    }
+    return await this._context?.rebuild()
+  }
+
+  public async dispose() {
+    if (!this._context) {
+      return
+    }
+
+    // Clear our handle in `finally` so a failed teardown can't leave a stale context behind
+    // that a later dispose would try to tear down a second time.
+    try {
+      await this._context.dispose()
+    } catch (thrown: unknown) {
+      throw thrown instanceof Error ? thrown : new Error(String(thrown))
+    } finally {
+      this._context = undefined
+      this._previousProps = undefined
+      this._previousOpts = {}
+    }
+  }
+}
+
+export class BuildCodeContext extends BuildContext<BuildCodeProps> {
+  protected _createContext(props: BuildCodeProps, opts: esb.BuildOptions = {}): Promise<esb.BuildContext> {
+    const { absWorkingDir, code, outfile } = props
+    return esb.context({
+      ...DEFAULT_OPTIONS,
+      ...opts,
+      absWorkingDir,
+      outfile,
+      stdin: { contents: code, resolveDir: absWorkingDir, loader: 'ts' },
+      write: true,
+    })
+  }
+}
+
+export class BuildEntrypointContext extends BuildContext<BuildEntrypointProps> {
+  protected _createContext(props: BuildEntrypointProps, opts: esb.BuildOptions = {}): Promise<esb.BuildContext> {
+    const { absWorkingDir, entrypoint } = props
+    return esb.context({
+      ...DEFAULT_OPTIONS,
+      ...opts,
+      absWorkingDir,
+      entryPoints: [entrypoint],
+      outfile: undefined,
+      write: false,
+    })
+  }
+}
+
+/**
+ * Bundles a string of typescript code and writes the output to a file
+ */
+export function buildCode(props: BuildCodeProps, opts: esb.BuildOptions = {}): Promise<esb.BuildResult> {
+  const { absWorkingDir, code, outfile } = props
+  return esb.build({
+    ...DEFAULT_OPTIONS,
+    ...opts,
+    absWorkingDir,
+    outfile,
+    stdin: { contents: code, resolveDir: absWorkingDir, loader: 'ts' },
+    write: true,
+  })
+}
+
+/**
+ * Bundles a typescript file and returns the output as a string
+ */
+export function buildEntrypoint(
+  props: BuildEntrypointProps,
+  opts: esb.BuildOptions = {}
+): Promise<esb.BuildResult & { outputFiles: esb.OutputFile[] }> {
+  const { absWorkingDir, entrypoint } = props
+  return esb.build({
+    ...DEFAULT_OPTIONS,
+    ...opts,
+    absWorkingDir,
+    entryPoints: [entrypoint],
+    outfile: undefined,
+    write: false,
+  })
+}

packages/cli/src/utils/event-emitter.ts

@@ -0,0 +1,41 @@
+export class EventEmitter<E extends object> {
+  private _listeners: {
+    [K in keyof E]?: ((event: E[K]) => void)[]
+  } = {}
+
+  public emit<K extends keyof E>(type: K, event: E[K]) {
+    const listeners = this._listeners[type]
+    if (!listeners) {
+      return
+    }
+    for (const listener of listeners) {
+      listener(event)
+    }
+  }
+
+  public once<K extends keyof E>(type: K, listener: (event: E[K]) => void) {
+    const wrapped = (event: E[K]) => {
+      this.off(type, wrapped)
+      listener(event)
+    }
+    this.on(type, wrapped)
+  }
+
+  public on<K extends keyof E>(type: K, listener: (event: E[K]) => void) {
+    if (!this._listeners[type]) {
+      this._listeners[type] = []
+    }
+    this._listeners[type]!.push(listener)
+  }
+
+  public off<K extends keyof E>(type: K, listener: (event: E[K]) => void) {
+    const listeners = this._listeners[type]
+    if (!listeners) {
+      return
+    }
+    const index = listeners.indexOf(listener)
+    if (index !== -1) {
+      listeners.splice(index, 1)
+    }
+  }
+}

packages/cli/src/utils/file-watcher.ts

@@ -0,0 +1,61 @@
+import type watcher from '@parcel/watcher'
+import { debounceAsync } from './concurrency-utils'
+import { EventEmitter } from './event-emitter'
+
+export type FileChangeHandler = (events: watcher.Event[]) => Promise<void>
+
+type EmptyObject = Record<never, never>
+type FileWatcherEvent = {
+  error: unknown
+  close: EmptyObject
+}
+
+type FileWatcherOptions = watcher.Options & {
+  debounceMs?: number
+}
+
+/**
+ * Wraps the Parcel file watcher to ensure errors can be catched in an async/await fashion
+ */
+export class FileWatcher {
+  public static async watch(dir: string, fn: FileChangeHandler, opt?: FileWatcherOptions) {
+    const eventEmitter = new EventEmitter<FileWatcherEvent>()
+
+    let subscriptionHandler = async (err: Error | null, events: watcher.Event[]) => {
+      if (err) {
+        eventEmitter.emit('error', err)
+        return
+      }
+
+      try {
+        await fn(events)
+      } catch (thrown) {
+        eventEmitter.emit('error', thrown)
+      }
+    }
+
+    if (opt?.debounceMs) {
+      subscriptionHandler = debounceAsync(subscriptionHandler, opt.debounceMs)
+    }
+    const watcher = await import('@parcel/watcher')
+
+    const subscription = await watcher.subscribe(dir, subscriptionHandler, opt)
+    return new FileWatcher(subscription, eventEmitter)
+  }
+
+  private constructor(
+    private _subscription: watcher.AsyncSubscription,
+    private _errorEmitter: EventEmitter<FileWatcherEvent>
+  ) {}
+
+  public async close() {
+    await this._subscription.unsubscribe()
+    this._errorEmitter.emit('close', {})
+  }
+
+  public wait = () =>
+    new Promise((resolve, reject) => {
+      this._errorEmitter.once('error', reject)
+      this._errorEmitter.once('close', resolve)
+    })
+}

packages/cli/src/utils/guard-utils.ts

@@ -0,0 +1,4 @@
+export const is = {
+  defined: <T>(value: T | undefined): value is T => value !== undefined,
+  notNull: <T>(value: T | null): value is T => value !== null,
+}

packages/cli/src/utils/id-utils.ts

@@ -0,0 +1,27 @@
+import { prefixToObjectMap } from '@bpinternal/const'
+import * as uuid from 'uuid'
+
+const ULID_LENGTH = 26 // Reference: https://github.com/ulid/spec#canonical-string-representation
+
+export function isValidID(id: string) {
+  // Note: UUIDs were used first and then prefixed ULIDs were introduced.
+  return isPrefixedULID(id) || uuid.validate(id)
+}
+
+export function isPrefixedULID(id: string) {
+  const [prefix, identifier] = id.split('_')
+
+  if (!(prefix && identifier)) {
+    return false
+  }
+
+  if (!Object.keys(prefixToObjectMap).includes(prefix)) {
+    return false
+  }
+
+  if (identifier.length < ULID_LENGTH) {
+    return false
+  }
+
+  return true
+}

packages/cli/src/utils/index.ts

@@ -0,0 +1,25 @@
+export * as attributes from './attribute-utils'
+export * as cache from './cache-utils'
+export * as casing from './case-utils'
+export * as emitter from './event-emitter'
+export * as esbuild from './esbuild-utils'
+export * as filewatcher from './file-watcher'
+export * as guards from './guard-utils'
+export * as id from './id-utils'
+export * as integrations from './integration-utils'
+export * as json from './json-utils'
+export * as object from './object-utils'
+export * as path from './path-utils'
+export * as pkgJson from './pkgjson-utils'
+export * as promises from './promise-utils'
+export * as prompt from './prompt-utils'
+export * as records from './record-utils'
+export * as require from './require-utils'
+export * as schema from './schema-utils'
+export * as semver from './semver-utils'
+export * as string from './string-utils'
+export * as template from './template-utils'
+export * as tunnel from './tunnel-utils'
+export * as types from './type-utils'
+export * as url from './url-utils'
+export * as vrl from './vrl-utils'

packages/cli/src/utils/integration-utils.test.ts

@@ -0,0 +1,17 @@
+import { test, expect } from 'vitest'
+import { prepareIntegrationsUpdate } from './integration-utils'
+
+test('new integration with no enabled field should default to enabled: false', () => {
+  const result = prepareIntegrationsUpdate({ telegram: {} }, {})
+  expect(result['telegram']).toMatchObject({ enabled: false })
+})
+
+test('new integration with enabled: true should remain enabled: true', () => {
+  const result = prepareIntegrationsUpdate({ telegram: { enabled: true } }, {})
+  expect(result['telegram']).toMatchObject({ enabled: true })
+})
+
+test('removed integration passed as null should remain null', () => {
+  const result = prepareIntegrationsUpdate({ telegram: null }, { telegram: { enabled: true } })
+  expect(result['telegram']).toBeNull()
+})

packages/cli/src/utils/integration-utils.ts

@@ -0,0 +1,15 @@
+import { mapValues } from './record-utils'
+
+type WithEnabled = { enabled?: boolean }
+
+export const prepareIntegrationsUpdate = <T extends WithEnabled>(
+  integrations: Record<string, T | null>,
+  remoteIntegrations: Record<string, unknown>
+): Record<string, T | null> =>
+  mapValues(integrations, (integration, key) => {
+    const isExistingInstance = Object.hasOwn(remoteIntegrations, key)
+    if (integration !== null && !isExistingInstance && integration.enabled === undefined) {
+      return { ...integration, enabled: false }
+    }
+    return integration
+  })

packages/cli/src/utils/json-utils.ts

@@ -0,0 +1,19 @@
+export type ParseJsonResult<T> =
+  | {
+      success: true
+      data: T
+    }
+  | {
+      success: false
+      error: Error
+    }
+
+export const safeParseJson = <T>(jsonStr: string): ParseJsonResult<T> => {
+  try {
+    const data: T = JSON.parse(jsonStr)
+    return { success: true, data }
+  } catch (thrown) {
+    const error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    return { success: false, error }
+  }
+}

packages/cli/src/utils/object-utils.ts

@@ -0,0 +1,4 @@
+export const omit = <O extends object, K extends keyof O>(obj: O, property: K): Omit<O, K> => {
+  const { [property]: _, ...rest } = obj
+  return rest
+}

packages/cli/src/utils/path-utils.test.ts

@@ -0,0 +1,56 @@
+import { test, expect, describe } from 'vitest'
+import * as pathUtils from './path-utils'
+
+describe('posix', () => {
+  test('isPath with valid relative path should return true', () => {
+    expect(pathUtils.posix.isPath('./src')).toBe(true)
+    expect(pathUtils.posix.isPath('../src')).toBe(true)
+    expect(pathUtils.posix.isPath('./src/services')).toBe(true)
+    expect(pathUtils.posix.isPath('./src/services/index.ts')).toBe(true)
+    expect(pathUtils.posix.isPath('./.././.././../lol.json')).toBe(true)
+  })
+
+  test('isPath with valid absolute path should return true', () => {
+    expect(pathUtils.posix.isPath('/src')).toBe(true)
+    expect(pathUtils.posix.isPath('/src')).toBe(true)
+    expect(pathUtils.posix.isPath('/src/services')).toBe(true)
+    expect(pathUtils.posix.isPath('/src/services/index.ts')).toBe(true)
+  })
+
+  test('isPath with invalid path should return false', () => {
+    // these are technically valid posix paths, but not for the bp cli
+    expect(pathUtils.posix.isPath('src')).toBe(false)
+    expect(pathUtils.posix.isPath('.src')).toBe(false)
+    expect(pathUtils.posix.isPath('..src')).toBe(false)
+    expect(pathUtils.posix.isPath('src/services')).toBe(false)
+    expect(pathUtils.posix.isPath('src/services/index.ts')).toBe(false)
+  })
+})
+
+describe('win32', () => {
+  test('isPath with valid relative path should return true', () => {
+    expect(pathUtils.win32.isPath('.\\src')).toBe(true)
+    expect(pathUtils.win32.isPath('..\\src')).toBe(true)
+    expect(pathUtils.win32.isPath('.\\src\\services')).toBe(true)
+    expect(pathUtils.win32.isPath('.\\src\\services\\index.ts')).toBe(true)
+    expect(pathUtils.win32.isPath('.\\..\\.\\..\\.\\..\\lol.json')).toBe(true)
+  })
+
+  test('isPath with valid absolute path should return true', () => {
+    expect(pathUtils.win32.isPath('C:\\src')).toBe(true)
+    expect(pathUtils.win32.isPath('C:\\src')).toBe(true)
+    expect(pathUtils.win32.isPath('C:\\src\\services')).toBe(true)
+    expect(pathUtils.win32.isPath('C:\\src\\services\\index.ts')).toBe(true)
+  })
+
+  test('isPath with invalid path should return false', () => {
+    // these are technically valid win32 paths, but not for the bp cli
+    expect(pathUtils.win32.isPath('src')).toBe(false)
+    expect(pathUtils.win32.isPath('.src')).toBe(false)
+    expect(pathUtils.win32.isPath('..src')).toBe(false)
+    expect(pathUtils.win32.isPath('src\\services')).toBe(false)
+    expect(pathUtils.win32.isPath('src\\services\\index.ts')).toBe(false)
+    expect(pathUtils.win32.isPath('\\services')).toBe(false)
+    expect(pathUtils.win32.isPath('\\services\\index.ts')).toBe(false)
+  })
+})

packages/cli/src/utils/path-utils.ts

@@ -0,0 +1,78 @@
+import _ from 'lodash'
+import oslib from 'os'
+import pathlib from 'path'
+
+export namespace posix {
+  export type AbsolutePath = `/${string}`
+  export const isPath = (path: string) => isAbsolutePath(path) || isRelativePath(path)
+  export const isRelativePath = (path: string) => path.startsWith('./') || path.startsWith('../')
+  export const isAbsolutePath = (path: string): path is AbsolutePath => pathlib.posix.isAbsolute(path)
+}
+
+export namespace win32 {
+  export type AbsolutePath = `${string}:\\${string}` // C:\path
+  export const isPath = (path: string) => isAbsolutePath(path) || isRelativePath(path)
+  export const isRelativePath = (path: string) => path.startsWith('.\\') || path.startsWith('..\\')
+  export const isAbsolutePath = (path: string): path is AbsolutePath => /^[a-zA-Z]:\\/.test(path) // bp cli does not allow omitting the drive letter
+}
+
+export type AbsolutePath = posix.AbsolutePath | win32.AbsolutePath
+export const isPath = (path: string) => win32.isPath(path) || posix.isPath(path)
+export const isPlatformSpecificPath = (path: string) =>
+  oslib.platform() === 'win32' ? win32.isPath(path) : posix.isPath(path)
+export const isPlatformSpecificAbsolutePath = (path: string): path is AbsolutePath =>
+  oslib.platform() === 'win32' ? win32.isAbsolutePath(path) : posix.isAbsolutePath(path)
+
+export const cwd = (): AbsolutePath => process.cwd() as AbsolutePath
+export const join = (abs: AbsolutePath, ...paths: string[]): AbsolutePath => {
+  const joined = pathlib.join(abs, ...paths)
+  return pathlib.normalize(joined) as AbsolutePath
+}
+
+export const rmExtension = (filename: string) => filename.replace(/\.[^/.]+$/, '')
+
+export const toUnix = (path: string) => path.split(pathlib.sep).join(pathlib.posix.sep)
+
+export const toNormalizedPosixPath = (path: string) =>
+  (path.startsWith('./') ? './' : '') + pathlib.posix.normalize(path.replaceAll(/\\/g, '/')).replace(/\/\.$/, '')
+
+export const absoluteFrom = (rootdir: AbsolutePath, target: string): AbsolutePath => {
+  if (isPlatformSpecificAbsolutePath(target)) {
+    return target
+  }
+  return pathlib.join(rootdir, target) as AbsolutePath
+}
+
+export const relativeFrom = (rootdir: AbsolutePath, target: string) => {
+  let absPath: string
+
+  if (isPlatformSpecificAbsolutePath(target)) {
+    absPath = target
+  } else {
+    absPath = pathlib.resolve(pathlib.join(rootdir, target))
+  }
+
+  return pathlib.relative(rootdir, absPath)
+}
+
+/**
+ * Same as relativeFrom but ensures the result starts with './' or '../',
+ * so it is recognized as a path ref by parsePackageRef.
+ * Falls back to the absolute target path when rootdir and target are on
+ * different drives (Windows), since no relative path is possible.
+ */
+export const relativePathFrom = (rootdir: AbsolutePath, target: string) => {
+  const rel = relativeFrom(rootdir, target)
+  if (isPlatformSpecificAbsolutePath(rel)) {
+    return rel // different drives on Windows — keep absolute
+  }
+  const unix = toUnix(rel)
+  return unix.startsWith('./') || unix.startsWith('../') ? unix : `./${unix}`
+}
+
+export class PathStore<P extends string> {
+  public constructor(public readonly abs: Record<P, AbsolutePath>) {}
+  public rel(from: Extract<P, `${string}Dir`>) {
+    return _.mapValues(this.abs, (to) => relativeFrom(this.abs[from], to))
+  }
+}

packages/cli/src/utils/pkgjson-utils.ts

@@ -0,0 +1,73 @@
+import fs from 'fs'
+import pathlib from 'path'
+import * as json from './json-utils'
+
+type JSON = string | number | boolean | null | JSON[] | { [key: string]: JSON }
+
+export type PackageJson = {
+  name: string
+  version?: string
+  description?: string
+  scripts?: Record<string, string>
+  dependencies?: Record<string, string>
+  devDependencies?: Record<string, string>
+  peerDependencies?: Record<string, string>
+} & {
+  [key: string]: JSON
+}
+
+export const PKGJSON_FILE_NAME = 'package.json'
+export const BP_DEPENDENCIES_KEY = 'bpDependencies'
+
+export const readPackageJson = async (path: string): Promise<PackageJson | undefined> => {
+  const filePath = _resolveFilePath(path)
+  if (!fs.existsSync(filePath)) {
+    return undefined
+  }
+
+  const strContent: string = await fs.promises.readFile(filePath, 'utf8')
+  const parseResult = json.safeParseJson(strContent)
+  if (!parseResult.success) {
+    throw new Error(`Failed to parse JSON at ${filePath}: ${parseResult.error.message}`)
+  }
+
+  return parseResult.data as PackageJson
+}
+
+export type ReadPackageJsonResult =
+  | {
+      success: true
+      pkgJson?: PackageJson
+    }
+  | {
+      success: false
+      error: Error
+    }
+
+export const safeReadPackageJson = async (path: string): Promise<ReadPackageJsonResult> => {
+  try {
+    const pkgJson = await readPackageJson(path)
+    if (!pkgJson) {
+      return { success: true }
+    }
+    return { success: true, pkgJson }
+  } catch (thrown) {
+    const error = thrown instanceof Error ? thrown : new Error(String(thrown))
+    return { success: false, error }
+  }
+}
+
+export const findDependency = (pkgJson: PackageJson, name: string): string | undefined => {
+  const { dependencies, devDependencies, peerDependencies } = pkgJson
+  const allDependencies = { ...(dependencies ?? {}), ...(devDependencies ?? {}), ...(peerDependencies ?? {}) }
+  return allDependencies[name]
+}
+
+export const writePackageJson = async (path: string, pkgJson: PackageJson) => {
+  const filePath = _resolveFilePath(path)
+  await fs.promises.writeFile(filePath, JSON.stringify(pkgJson, null, 2))
+}
+
+function _resolveFilePath(path: string) {
+  return pathlib.basename(path) === PKGJSON_FILE_NAME ? path : pathlib.join(path, PKGJSON_FILE_NAME)
+}

packages/cli/src/utils/promise-utils.test.ts

@@ -0,0 +1,13 @@
+import { test, expect } from 'vitest'
+import { awaitRecord } from './promise-utils'
+
+test('awaitRecord', async () => {
+  const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+  const record = {
+    a: sleep(30).then(() => 'a'),
+    b: sleep(20).then(() => 'b'),
+    c: sleep(10).then(() => 'c'),
+  }
+  const result = await awaitRecord(record)
+  expect(result).toEqual({ a: 'a', b: 'b', c: 'c' })
+})

packages/cli/src/utils/promise-utils.ts

@@ -0,0 +1,7 @@
+export const awaitRecord = async <T>(record: Record<string, Promise<T>>): Promise<Record<string, T>> => {
+  const keys = Object.keys(record)
+  const values = await Promise.all(Object.values(record))
+  return keys.reduce((acc, key, index) => ({ ...acc, [key]: values[index] }), {})
+}
+
+export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

packages/cli/src/utils/prompt-utils.ts

@@ -0,0 +1,120 @@
+import prompts from 'prompts'
+import type { Logger } from '../logger'
+
+export type CLIPromptsProps = {
+  confirm: boolean
+}
+
+type ChoiceValueType = string | number
+
+export type CLIPromptsChoice<V extends ChoiceValueType> = {
+  title: string
+  value: V
+}
+
+type PasswordOptions = Partial<{
+  default: string | undefined
+  initial: string
+}>
+type SelectOptions<V extends ChoiceValueType> = Partial<{
+  default: V | undefined
+  initial: CLIPromptsChoice<V>
+  choices: CLIPromptsChoice<V>[]
+}>
+type TextOptions = Partial<{
+  default: string | undefined
+  initial: string
+}>
+
+export class CLIPrompt {
+  public constructor(
+    private _props: CLIPromptsProps,
+    private _logger: Logger
+  ) {}
+
+  public async confirm(message: string): Promise<boolean> {
+    if (this._props.confirm) {
+      this._logger.debug(`Confirming automatically (non-interactive mode): ${message}`)
+      return true
+    }
+
+    const { confirm } = await this._prompts({
+      type: 'confirm',
+      name: 'confirm',
+      message,
+      initial: false,
+    })
+
+    if (!confirm) {
+      return false
+    }
+    return true
+  }
+
+  public async password(message: string, opts: PasswordOptions = {}): Promise<string | undefined> {
+    if (this._props.confirm) {
+      this._logger.debug(`Return default (non-interactive mode): ${message}`)
+      return opts?.default
+    }
+
+    const { prompted } = await this._prompts({
+      type: 'password',
+      name: 'prompted',
+      message,
+      initial: opts?.initial,
+    })
+
+    return prompted ? prompted : undefined
+  }
+
+  public async select<V extends ChoiceValueType = string>(
+    message: string,
+    opts: SelectOptions<V> = {}
+  ): Promise<V | undefined> {
+    if (this._props.confirm) {
+      this._logger.debug(`Return default (non-interactive mode): ${message}`)
+      return opts?.default
+    }
+
+    // NOTE: whilst Prompts supports string, number, date and boolean values for
+    //       choices, it behaves unexpectedly when the value is a number and is
+    //       equal to 0. To work around this, we convert the value to a symbol
+    //       if it is 0, and then convert it back to a number if it was 0.
+
+    const isNumber = typeof opts?.choices?.[0]?.value === 'number'
+    const transformedChoices = isNumber
+      ? opts?.choices?.map((c) => ({ ...c, value: c.value === 0 ? Symbol.for('0') : c.value }))
+      : opts?.choices
+
+    const { prompted } = await this._prompts({
+      type: 'autocomplete',
+      name: 'prompted',
+      message,
+      initial: opts?.initial?.value,
+      choices: transformedChoices as prompts.PromptObject['choices'],
+    })
+
+    return prompted !== undefined ? (isNumber && prompted === Symbol.for('0') ? 0 : prompted) : undefined
+  }
+
+  public async text(message: string, opts: TextOptions = {}): Promise<string | undefined> {
+    if (this._props.confirm) {
+      this._logger.debug(`Return default (non-interactive mode): ${message}`)
+      return opts?.default
+    }
+
+    const { prompted } = await this._prompts({
+      type: 'text',
+      name: 'prompted',
+      message,
+      initial: opts?.initial,
+    })
+
+    return prompted ? prompted : undefined
+  }
+
+  private _prompts = (...args: Parameters<typeof prompts>): ReturnType<typeof prompts> => {
+    this._logger.cleanup()
+    return prompts(...args)
+  }
+}

packages/cli/src/utils/record-utils.test.ts

@@ -0,0 +1,28 @@
+import { test, expect } from 'vitest'
+import * as recordUtils from './record-utils'
+
+test('zip objects should return both values only when key is defined in both objects', () => {
+  const objA = { a: 1, b: 2, c: 3 }
+  const objB = { c: 4, d: 5 }
+
+  const zipped = recordUtils.zipObjects(objA, objB)
+
+  const expected = {
+    a: [1, null],
+    b: [2, null],
+    c: [3, 4],
+    d: [null, 5],
+  }
+
+  expect(zipped).toEqual(expected)
+})
+
+test('mapValuesAsync should return a new object with the values mapped asynchronously', async () => {
+  const obj = { a: 1, b: 2, c: 3 }
+
+  const mapped = await recordUtils.mapValuesAsync(obj, async (v) => v * 2)
+
+  const expected = { a: 2, b: 4, c: 6 }
+
+  expect(mapped).toEqual(expected)
+})

packages/cli/src/utils/record-utils.ts

@@ -0,0 +1,111 @@
+import bluebird from 'bluebird'
+import _ from 'lodash'
+
+export const setNullOnMissingValues = <A, B>(
+  record: Record<string, A> = {},
+  oldRecord: Record<string, B> = {}
+): Record<string, A | null> => {
+  const newRecord: Record<string, A | null> = {}
+
+  for (const [key, value] of Object.entries(record)) {
+    newRecord[key] = value
+  }
+
+  for (const value of Object.keys(oldRecord)) {
+    if (!record[value]) {
+      newRecord[value] = null
+    }
+  }
+
+  return newRecord
+}
+
+export const zipObjects = <A, B>(
+  recordA: Record<string, A>,
+  recordB: Record<string, B>
+): Record<string, [A | null, B | null]> => {
+  const allKeys = new Set([...Object.keys(recordA), ...Object.keys(recordB)])
+  const newRecord: Record<string, [A | null, B | null]> = {}
+
+  for (const key of allKeys) {
+    newRecord[key] = [recordA[key] ?? null, recordB[key] ?? null]
+  }
+
+  return newRecord
+}
+
+export const mapValues = <A, B>(record: Record<string, A>, fn: (value: A, key: string) => B): Record<string, B> => {
+  const newRecord: Record<string, B> = {}
+
+  for (const [key, value] of Object.entries(record)) {
+    newRecord[key] = fn(value, key)
+  }
+
+  return newRecord
+}
+
+export const mapValuesAsync = async <A, B>(
+  record: Record<string, A>,
+  fn: (value: A, key: string) => Promise<B>
+): Promise<Record<string, B>> => {
+  const entries: [string, A][] = Object.entries(record)
+
+  const newEntries = await bluebird.map(
+    entries,
+    async ([key, value]): Promise<[string, B]> => [key, await fn(value, key)]
+  )
+
+  return Object.fromEntries(newEntries)
+}
+
+export const mapKeys = <A>(record: Record<string, A>, fn: (value: A, key: string) => string): Record<string, A> => {
+  const newRecord: Record<string, A> = {}
+
+  for (const [key, value] of Object.entries(record)) {
+    const newKey = fn(value, key) as string
+    newRecord[newKey] = value
+  }
+
+  return newRecord
+}
+
+export function filterValues<A, B extends A>(
+  record: Record<string, A>,
+  fn: (value: A, key: string) => value is B
+): Record<string, B>
+export function filterValues<A, _B extends A>(
+  record: Record<string, A>,
+  fn: (value: A, key: string) => boolean
+): Record<string, A>
+export function filterValues<A>(record: Record<string, A>, fn: (value: A, key: string) => boolean) {
+  const newRecord: Record<string, A> = {}
+
+  for (const [key, value] of Object.entries(record)) {
+    if (fn(value, key)) {
+      newRecord[key] = value
+    }
+  }
+
+  return newRecord
+}
+
+export const mergeRecords = <K extends string, V>(
+  a: Record<K, V>,
+  b: Record<K, V>,
+  merge: (v1: V, v2: V) => V
+): Record<K, V> => {
+  const keys = _.uniq([...Object.keys(a), ...Object.keys(b)]) as K[]
+  const result: Record<K, V> = {} as Record<K, V>
+  for (const key of keys) {
+    const aValue = a[key]
+    const bValue = b[key]
+    if (aValue && bValue) {
+      result[key] = merge(aValue, bValue)
+    } else if (aValue) {
+      result[key] = aValue
+    } else if (bValue) {
+      result[key] = bValue
+    }
+  }
+  return result
+}

packages/cli/src/utils/require-utils.test.ts

@@ -0,0 +1,37 @@
+import { test, expect } from 'vitest'
+import * as requireUtils from './require-utils'
+
+test('require js code should do as its name suggests', () => {
+  const code = `
+    module.exports = {
+      foo: 'bar'
+    }
+  `
+
+  const result = requireUtils.requireJsCode<{ foo: string }>(code)
+
+  expect(result.foo).toBe('bar')
+})
+
+const getError = (fn: () => void): Error | undefined => {
+  try {
+    fn()
+    return
+  } catch (thrown: unknown) {
+    return thrown instanceof Error ? thrown : new Error(`${thrown}`)
+  }
+}
+
+test('require js code should indicate issue in stack trace', () => {
+  const code = `
+    var foo = undefined
+    module.exports = {
+      bar: foo.bar
+    }
+  `
+
+  const error = getError(() => requireUtils.requireJsCode(code))
+
+  expect(error).toBeDefined()
+  expect(error?.message).toContain('bar: foo.bar')
+})

packages/cli/src/utils/require-utils.ts

@@ -0,0 +1,69 @@
+import * as sdk from '@botpress/sdk'
+import Module from 'module'
+import pathlib from 'path'
+
+export const requireJsFile = <T>(path: string): T => {
+  return require(path)
+}
+
+export const requireJsCode = <T>(code: string): T => {
+  const filedir = 'tmp'
+  const filename = `${Date.now()}.js`
+
+  const fileid = pathlib.join(filedir, filename)
+
+  const m = new Module(fileid)
+  m.filename = filename
+
+  try {
+    // @ts-ignore
+    m._compile(code, filename)
+    return m.exports
+  } catch (thrown: unknown) {
+    const error = thrown instanceof Error ? thrown : new Error(`${thrown}`)
+    // sdk.errors.isDefinitionError() handles cross-bundle detection: esbuild inlines a separate
+    // copy of @botpress/sdk into the compiled definition artifact, so instanceof alone is
+    // unreliable. See the isDefinitionError() docstring in the SDK for the full explanation.
+    if (sdk.errors.isDefinitionError(thrown)) {
+      throw error
+    }
+    throw _injectStackTrace(error, code, filename)
+  }
+}
+
+const STACK_TRACE_SURROUNDING_LINES = 3
+
+const _injectStackTrace = (compileError: Error, code: string, filename: string): Error => {
+  if (!compileError.stack || !compileError.stack.includes(`${filename}:`)) {
+    return compileError
+  }
+
+  // Extract line and column from the stack trace:
+  const [, locationInfo] = compileError.stack.split(`${filename}:`, 2)
+  if (!locationInfo) {
+    return compileError
+  }
+
+  const [lineNoStr, _rest] = locationInfo.split(':', 2)
+  if (!lineNoStr || !_rest) {
+    return compileError
+  }
+
+  const [columnStr] = _rest.split(')', 1)
+  if (!columnStr) {
+    return compileError
+  }
+
+  const lineNo = parseInt(lineNoStr)
+  const column = parseInt(columnStr)
+
+  // Build the stack trace:
+  const allLines = code.split('\n')
+  const linesBefore = allLines.slice(Math.max(0, lineNo - 1 - STACK_TRACE_SURROUNDING_LINES), lineNo - 1)
+  const offendingLine = allLines[lineNo - 1]
+  const caretLine = ' '.repeat(column - 1) + '^'
+  const linesAfter = allLines.slice(lineNo, Math.min(allLines.length, lineNo + STACK_TRACE_SURROUNDING_LINES))
+  const stackTrace = [...linesBefore, offendingLine, caretLine, ...linesAfter].join('\n')
+
+  return new Error(`${compileError.message}\n\nOffending code:\n\n${stackTrace}`, { cause: compileError })
+}

packages/cli/src/utils/schema-utils.test.ts

@@ -0,0 +1,52 @@
+import { it, expect, describe } from 'vitest'
+import { dereferenceSchema } from './schema-utils'
+import { JSONSchema7 } from 'json-schema'
+
+describe('dereferenceSchema', () => {
+  it('should do nothing if no $ref', async () => {
+    const schema: JSONSchema7 = { type: 'object' }
+    const result = await dereferenceSchema(schema)
+    expect(result).toEqual(schema)
+  })
+
+  it('should dereference local $ref', async () => {
+    const schema: JSONSchema7 = {
+      type: 'object',
+      properties: {
+        foo: { $ref: '#/$defs/foo' },
+        bar: { $ref: '#/definitions/bar' },
+      },
+      $defs: {
+        foo: { type: 'string' },
+      },
+      definitions: {
+        bar: { type: 'number' },
+      },
+    }
+    const result = await dereferenceSchema(schema)
+    expect(result).toEqual({
+      type: 'object',
+      properties: {
+        foo: { type: 'string' },
+        bar: { type: 'number' },
+      },
+      $defs: {
+        foo: { type: 'string' },
+      },
+      definitions: {
+        bar: { type: 'number' },
+      },
+    })
+  })
+
+  it('should ignore non-local $ref', async () => {
+    const schema: JSONSchema7 = {
+      type: 'object',
+      properties: {
+        foo: { $ref: 'TItem' },
+      },
+    }
+    const result = await dereferenceSchema(schema)
+    expect(result).toEqual(schema)
+  })
+})

packages/cli/src/utils/schema-utils.ts

@@ -0,0 +1,71 @@
+import { dereference } from '@apidevtools/json-schema-ref-parser'
+import * as sdk from '@botpress/sdk'
+import { JSONSchema7 } from 'json-schema'
+
+type ZuiToJsonSchema = typeof sdk.z.transforms.toJSONSchemaLegacy
+type JsonSchema = ReturnType<ZuiToJsonSchema>
+
+type SchemaOptions = {
+  title?: string
+  examples?: any[]
+}
+
+type SchemaDefinition = {
+  schema: sdk.z.ZuiObjectOrRefSchema
+  ui?: Record<string, SchemaOptions | undefined>
+}
+
+type MapSchemaOptions = {
+  useLegacyZuiTransformer?: boolean
+  toJSONSchemaOptions?: Partial<sdk.z.transforms.JSONSchemaGenerationOptions>
+}
+
+const isObjectSchema = (schema: JsonSchema): boolean => schema.type === 'object'
+
+export async function mapZodToJsonSchema(
+  definition: SchemaDefinition,
+  options: MapSchemaOptions
+): Promise<ReturnType<typeof sdk.z.transforms.toJSONSchemaLegacy>> {
+  let schema: JSONSchema7
+  if (options.useLegacyZuiTransformer) {
+    schema = sdk.z.transforms.toJSONSchemaLegacy(definition.schema, {
+      target: 'jsonSchema7',
+      ...options.toJSONSchemaOptions,
+    })
+  } else {
+    schema = sdk.z.transforms.toJSONSchema(definition.schema, options.toJSONSchemaOptions)
+  }
+  schema = (await dereferenceSchema(schema)) as typeof schema
+
+  if (!isObjectSchema(schema) || !definition.ui) {
+    return schema
+  }
+
+  for (const [key, value] of Object.entries(definition.ui ?? {})) {
+    const property = schema.properties?.[key]
+
+    if (!property) {
+      continue
+    }
+
+    if (!!value?.title) {
+      ;(property as any).title = value.title
+    }
+
+    if (!!value?.examples) {
+      ;(property as any).examples = value.examples
+    }
+  }
+
+  return schema
+}
+
+export const dereferenceSchema = async (schema: JSONSchema7): Promise<JSONSchema7> => {
+  return dereference(schema, {
+    resolve: {
+      external: false,
+      file: false,
+      http: false,
+    },
+  })
+}

packages/cli/src/utils/semver-utils.ts

@@ -0,0 +1,19 @@
+import semver from 'semver'
+
+const semverReleases: Record<semver.ReleaseType, number> = {
+  prerelease: 0,
+  prepatch: 1,
+  patch: 2,
+  preminor: 3,
+  minor: 4,
+  premajor: 5,
+  major: 6,
+}
+
+export namespace releases {
+  export const eq = (a: semver.ReleaseType, b: semver.ReleaseType) => semverReleases[a] === semverReleases[b]
+  export const gt = (a: semver.ReleaseType, b: semver.ReleaseType) => semverReleases[a] > semverReleases[b]
+  export const gte = (a: semver.ReleaseType, b: semver.ReleaseType) => semverReleases[a] >= semverReleases[b]
+  export const lt = (a: semver.ReleaseType, b: semver.ReleaseType) => semverReleases[a] < semverReleases[b]
+  export const lte = (a: semver.ReleaseType, b: semver.ReleaseType) => semverReleases[a] <= semverReleases[b]
+}

packages/cli/src/utils/string-utils.ts

@@ -0,0 +1,13 @@
+export const splitOnce = (text: string, separator: string): [string, string | undefined] => {
+  const index = text.indexOf(separator)
+  if (index === -1) {
+    return [text, undefined]
+  }
+  return [text.slice(0, index), text.slice(index + 1)]
+}
+
+export function* chunkString(input: string, chunkSize: number): Generator<string, void, void> {
+  for (let i = 0; i < input.length; i += chunkSize) {
+    yield input.slice(i, i + chunkSize)
+  }
+}

packages/cli/src/utils/template-utils.ts

@@ -0,0 +1,15 @@
+import hb from 'handlebars'
+import * as casing from './case-utils'
+
+hb.registerHelper('toUpperCase', (s) => s.toUpperCase())
+hb.registerHelper('toLowerCase', (s) => s.toLowerCase())
+hb.registerHelper('pascalCase', casing.to.pascalCase)
+hb.registerHelper('kebabCase', casing.to.kebabCase)
+hb.registerHelper('snakeCase', casing.to.snakeCase)
+hb.registerHelper('screamingSnakeCase', casing.to.screamingSnakeCase)
+hb.registerHelper('camelCase', casing.to.camelCase)
+
+export const formatHandleBars = (templateStr: string, data: Record<string, string>): string => {
+  const template = hb.compile(templateStr)
+  return template(data)
+}

packages/cli/src/utils/tunnel-utils.ts

@@ -0,0 +1,163 @@
+import { TunnelTail, ClientCloseEvent, ClientErrorEvent, errors } from '@bpinternal/tunnel'
+import { BotpressCLIError } from '../errors'
+import { Logger } from '../logger'
+import { EventEmitter } from './event-emitter'
+
+export type ReconnectionTriggerEvent =
+  | {
+      type: 'init'
+      ev: null
+    }
+  | {
+      type: 'error'
+      ev: ClientErrorEvent
+    }
+  | {
+      type: 'close'
+      ev: ClientCloseEvent
+    }
+
+export type ReconnectedEvent = {
+  tunnel: TunnelTail
+  ev: ReconnectionTriggerEvent
+}
+
+export class ReconnectionFailedError extends Error {
+  public constructor(
+    public readonly event: ReconnectionTriggerEvent,
+    cause?: Error
+  ) {
+    const reason = ReconnectionFailedError._reason(event)
+    const message = cause ? `Reconnection failed: ${reason}: ${cause.message}` : `Reconnection failed: ${reason}`
+    const options = cause ? { cause } : undefined
+    super(message, options)
+  }
+
+  private static _reason(event: ReconnectionTriggerEvent): string {
+    if (event.type === 'error') {
+      return 'error'
+    }
+
+    if (event.type === 'close') {
+      return `${event.ev.code} ${event.ev.reason}`
+    }
+
+    return 'init'
+  }
+}
+
+export class TunnelSupervisor {
+  private _tunnel?: TunnelTail
+  private _closed = false
+  private _started = false
+
+  public readonly events = new EventEmitter<{
+    connectionFailed: { ev: ReconnectionTriggerEvent; cause: Error }
+    manuallyClosed: null
+    connected: {
+      tunnel: TunnelTail
+      ev: ReconnectionTriggerEvent
+    }
+  }>()
+
+  public constructor(
+    private _tunnelUrl: string,
+    private _tunnelId: string,
+    private _logger: Logger
+  ) {}
+
+  public async start(): Promise<void> {
+    if (this._closed) {
+      throw new Error('Cannot start: Tunnel is closed')
+    }
+    if (this._started) {
+      throw new Error('Cannot start: Tunnel is already started')
+    }
+
+    this._started = true
+    const tunnel = await this._reconnect({ type: 'init', ev: null })
+    this._tunnel = tunnel
+  }
+
+  public get closed(): boolean {
+    return this._closed
+  }
+
+  /**
+   * @returns Promise that rejects when a reconnection attempt fails and resolves when the tunnel is closed manually
+   */
+  public async wait(): Promise<void> {
+    if (this._closed) {
+      throw new Error('Cannot wait: Tunnel is closed')
+    }
+
+    return new Promise((resolve, reject) => {
+      this.events.on('connectionFailed', ({ ev, cause }) => {
+        reject(new ReconnectionFailedError(ev, cause))
+      })
+
+      this.events.on('manuallyClosed', () => {
+        resolve()
+      })
+    })
+  }
+
+  public close(): void {
+    if (this._closed) {
+      return
+    }
+
+    this._closed = true
+    this._tunnel?.close()
+    this.events.emit('manuallyClosed', null)
+  }
+
+  private _reconnectSync(ev: ReconnectionTriggerEvent): void {
+    void this._reconnect(ev)
+      .then((t) => {
+        this._tunnel = t
+      })
+      .catch((thrown) => {
+        // carry the real failure as the cause; the dev server then tears down and the single
+        // "running the dev server" error surfaces this reason (avoids a duplicate log line here)
+        this.events.emit('connectionFailed', { ev, cause: BotpressCLIError.map(thrown) })
+      })
+  }
+
+  private async _reconnect(ev: ReconnectionTriggerEvent): Promise<TunnelTail> {
+    const newTunnel = async () => {
+      const tunnel = await TunnelTail.new(this._tunnelUrl, this._tunnelId)
+      this._registerListeners(tunnel)
+      this.events.emit('connected', { tunnel, ev })
+      return tunnel
+    }
+
+    if (ev.type === 'init') {
+      // skip logging on the first connection attempt
+      return newTunnel()
+    }
+
+    const line = this._logger.line()
+    line.started('Reconnecting tunnel...')
+    const tunnel = await newTunnel()
+    line.success('Reconnected')
+    line.commit()
+    return tunnel
+  }
+
+  private _registerListeners(tunnel: TunnelTail) {
+    tunnel.events.on('error', ({ target, type }) => {
+      this._logger.error(`Tunnel error: ${type}`)
+      this._reconnectSync({ type: 'error', ev: { target, type } })
+    })
+    tunnel.events.on('close', ({ code, reason, target, type, wasClean }) => {
+      this._logger.error(`Tunnel closed: ${code} ${reason}`)
+
+      if (code === errors.CLOSE_CODES.TUNNEL_ID_CONFLICT) {
+        throw new Error('Cannot start: Tunnel Id is already used, choose a different tunnel id.')
+      }
+
+      this._reconnectSync({ type: 'close', ev: { code, reason, target, type, wasClean } })
+    })
+  }
+}

packages/cli/src/utils/type-utils.ts

@@ -0,0 +1,29 @@
+export type Merge<A extends object, B extends object> = Omit<A, keyof B> & B
+export type SafeOmit<T, K extends keyof T> = Omit<T, K>
+export type Writable<T> = { -readonly [K in keyof T]: T[K] }
+
+export type AssertNever<_T extends never> = true
+export type AssertExtends<_A extends B, B> = true
+export type AssertKeyOf<_K extends keyof T, T> = true
+export type AssertTrue<_T extends true> = true
+export type AssertAll<_T extends true[]> = true
+
+export type IsExtend<X, Y> = X extends Y ? true : false
+export type IsEquivalent<X, Y> = IsExtend<X, Y> extends true ? IsExtend<Y, X> : false
+export type IsIdentical<X, Y> = (<T>() => T extends X ? 1 : 2) extends <T>() => T extends Y ? 1 : 2 ? true : false
+export type IsEqual<X, Y> = IsIdentical<Normalize<X>, Normalize<Y>>
+
+type NormalizeObject<T extends object> = T extends infer O ? { [K in keyof O]: Normalize<O[K]> } : never
+export type Normalize<T> = T extends (...args: infer A) => infer R
+  ? (...args: Normalize<A>) => Normalize<R>
+  : T extends Array<infer E>
+    ? Array<Normalize<E>>
+    : T extends ReadonlyArray<infer E>
+      ? ReadonlyArray<Normalize<E>>
+      : T extends Promise<infer R>
+        ? Promise<Normalize<R>>
+        : T extends Buffer
+          ? Buffer
+          : T extends object
+            ? NormalizeObject<T>
+            : T

packages/cli/src/utils/url-utils.ts

@@ -0,0 +1,61 @@
+const _PROTOCOLS = ['http', 'https', 'ws', 'wss'] as const
+
+export type Protocol = (typeof _PROTOCOLS)[number]
+
+export type Path = `/${string}`
+
+export type Url = {
+  protocol: Protocol
+  host: string
+  port?: number
+  path?: Path
+}
+
+export type UrlParseResult =
+  | {
+      status: 'success'
+      url: Url
+    }
+  | {
+      status: 'error'
+      error: string
+    }
+
+const toPath = (path: string): Path => {
+  if (!path.startsWith('/')) {
+    return `/${path}`
+  }
+  return path as Path
+}
+
+export function parse(hostOrUrl: string): UrlParseResult {
+  try {
+    const url = new URL(hostOrUrl)
+    return {
+      status: 'success',
+      url: {
+        protocol: url.protocol.replace(':', '') as Url['protocol'],
+        host: url.hostname,
+        port: url.port ? parseInt(url.port) : undefined,
+        path: toPath(url.pathname),
+      },
+    }
+  } catch (thrown) {
+    const message = thrown instanceof Error ? thrown.message : `${thrown}`
+    return {
+      status: 'error',
+      error: message,
+    }
+  }
+}
+
+export const format = (url: Url): string => {
+  let formatted = `${url.protocol}://${url.host}`
+  if (url.port) {
+    formatted += `:${url.port}`
+  }
+  if (url.path && url.path !== '/') {
+    formatted += url.path
+  }
+  return formatted
+}

packages/cli/src/utils/vrl-utils.ts

@@ -0,0 +1,12 @@
+import * as verel from '@bpinternal/verel'
+import * as errors from '../errors'
+
+export const getStringResult = ({ code, data }: { code: string; data: Record<string, any> }) => {
+  const { result } = verel.execute(code, data)
+
+  if (typeof result !== 'string') {
+    throw new errors.BotpressCLIError('VRL returned an invalid string result')
+  }
+
+  return result
+}