chore: import upstream snapshot with attribution

scripts/architectureUtils.ts

@@ -0,0 +1,889 @@
+import fs from 'node:fs';
+import { builtinModules } from 'node:module';
+import path from 'node:path';
+
+import { globSync } from 'glob';
+import ts from 'typescript';
+
+export interface LayerDefinition {
+  name: string;
+  roots: string[];
+  allowedDependencies: string[];
+  allowedImportPaths?: string[];
+  /**
+   * Layers this layer must NOT depend on, even transitionally. Used to lock a
+   * cross-layer pair once its cycle has been fully broken so it cannot regress.
+   * Entries must be known layer names and must not also appear in
+   * `allowedDependencies`.
+   */
+  forbiddenDependencies?: string[];
+  /**
+   * Bare module specifiers (npm packages or Node builtins) a LEAF layer is allowed to import.
+   * Only consulted for layers listed in {@link LayerConfig.leafLayers}: a leaf layer may import
+   * relative siblings plus these externals, and nothing else. `node:` prefixes are ignored when
+   * matching, so `"fs"` and `"node:fs"` are equivalent.
+   */
+  allowedExternal?: string[];
+}
+
+export interface LayerConfig {
+  publicFacade: string;
+  leafLayers?: string[];
+  aliases?: Record<string, string>;
+  ignoredRoots?: string[];
+  layers: LayerDefinition[];
+  /**
+   * Target topological order, listed most-depended-upon first (the intended
+   * DAG bottom-to-top). A cross-layer edge from an earlier entry to a later one
+   * is a cycle-causing "back edge". When defined, every configured layer must
+   * appear exactly once so the back-edge report cannot silently omit a layer.
+   */
+  tierOrder?: string[];
+  /**
+   * Ratchet for the largest strongly-connected component (dependency cycle) in
+   * the layer graph. The checker fails if the real graph exceeds this size, so
+   * the value can only be lowered over time (8 today, 1 == a DAG).
+   */
+  maxStronglyConnectedComponentSize?: number;
+}
+
+const TYPESCRIPT_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts'];
+const DIRECTORY_INDEXES = TYPESCRIPT_EXTENSIONS.map((extension) => `index${extension}`);
+const SOURCE_EXTENSIONS_BY_RUNTIME_EXTENSION: Record<string, string[]> = {
+  '.js': ['.ts', '.tsx'],
+  '.mjs': ['.mts'],
+  '.cjs': ['.cts'],
+};
+const BUILTIN_MODULES = new Set(
+  builtinModules.flatMap((moduleName) => [moduleName, moduleName.replace(/^node:/, '')]),
+);
+
+export function normalizePath(filePath: string): string {
+  return filePath.split(path.sep).join('/');
+}
+
+function validateLayerDefinition(
+  repoRoot: string,
+  configPath: string,
+  layer: LayerDefinition,
+): Array<{ layerName: string; root: string }> {
+  if (typeof layer.name !== 'string' || layer.name.length === 0) {
+    throw new Error(`${configPath} contains a layer without a name.`);
+  }
+  if (!Array.isArray(layer.roots) || layer.roots.length === 0) {
+    throw new Error(`Architecture layer "${layer.name}" must declare roots.`);
+  }
+  if (!Array.isArray(layer.allowedDependencies)) {
+    throw new Error(`Architecture layer "${layer.name}" must declare allowedDependencies.`);
+  }
+  if (
+    layer.allowedImportPaths &&
+    (!Array.isArray(layer.allowedImportPaths) ||
+      layer.allowedImportPaths.some(
+        (allowedImportPath) =>
+          typeof allowedImportPath !== 'string' ||
+          !fs.existsSync(path.join(repoRoot, allowedImportPath)) ||
+          !fs.statSync(path.join(repoRoot, allowedImportPath)).isFile(),
+      ))
+  ) {
+    throw new Error(
+      `Architecture layer "${layer.name}" contains an invalid allowedImportPaths entry.`,
+    );
+  }
+  if (
+    layer.allowedExternal &&
+    (!Array.isArray(layer.allowedExternal) ||
+      layer.allowedExternal.some((entry) => typeof entry !== 'string' || entry.length === 0))
+  ) {
+    throw new Error(
+      `Architecture layer "${layer.name}" contains an invalid allowedExternal entry.`,
+    );
+  }
+
+  return layer.roots.map((root) => {
+    if (typeof root !== 'string' || !fs.existsSync(path.join(repoRoot, root))) {
+      throw new Error(`Architecture layer "${layer.name}" root "${String(root)}" does not exist.`);
+    }
+    return { layerName: layer.name, root };
+  });
+}
+
+function validateDependencies(config: LayerConfig, layerNames: Set<string>): void {
+  for (const layer of config.layers) {
+    for (const allowedDependency of layer.allowedDependencies) {
+      if (typeof allowedDependency !== 'string' || !layerNames.has(allowedDependency)) {
+        throw new Error(
+          `Architecture layer "${layer.name}" allows unknown dependency "${String(allowedDependency)}".`,
+        );
+      }
+    }
+  }
+}
+
+function validateAliases(repoRoot: string, aliases: Record<string, string> = {}): void {
+  for (const [alias, target] of Object.entries(aliases)) {
+    if (typeof target !== 'string' || !fs.existsSync(path.join(repoRoot, target))) {
+      throw new Error(`Architecture alias "${alias}" points to missing path "${String(target)}".`);
+    }
+  }
+}
+
+function validateRootsDoNotOverlap(
+  configuredRoots: Array<{ layerName: string; root: string }>,
+): void {
+  for (let index = 0; index < configuredRoots.length; index++) {
+    const root = configuredRoots[index];
+    for (const otherRoot of configuredRoots.slice(index + 1)) {
+      if (isWithinRoot(root.root, otherRoot.root) || isWithinRoot(otherRoot.root, root.root)) {
+        throw new Error(
+          `Architecture roots "${root.root}" (${root.layerName}) and "${otherRoot.root}" (${otherRoot.layerName}) overlap.`,
+        );
+      }
+    }
+  }
+}
+
+function validateTierOrder(tierOrder: LayerConfig['tierOrder'], layerNames: Set<string>): void {
+  if (tierOrder === undefined) {
+    return;
+  }
+  if (!Array.isArray(tierOrder)) {
+    throw new Error('Architecture tierOrder must be an array of layer names.');
+  }
+  const seen = new Set<string>();
+  for (const layerName of tierOrder) {
+    if (typeof layerName !== 'string' || !layerNames.has(layerName)) {
+      throw new Error(`Architecture tierOrder contains unknown layer "${String(layerName)}".`);
+    }
+    if (seen.has(layerName)) {
+      throw new Error(`Architecture tierOrder lists layer "${layerName}" more than once.`);
+    }
+    seen.add(layerName);
+  }
+  if (seen.size !== layerNames.size) {
+    const missingLayers = [...layerNames].filter((layerName) => !seen.has(layerName)).sort();
+    throw new Error(
+      `Architecture tierOrder must list every layer. Missing: ${missingLayers.join(', ')}.`,
+    );
+  }
+}
+
+function validateMaxStronglyConnectedComponentSize(max: number | undefined): void {
+  if (max !== undefined && (typeof max !== 'number' || !Number.isInteger(max) || max < 1)) {
+    throw new Error('Architecture maxStronglyConnectedComponentSize must be a positive integer.');
+  }
+}
+
+function validateForbiddenDependencies(layers: LayerDefinition[], layerNames: Set<string>): void {
+  for (const layer of layers) {
+    if (layer.forbiddenDependencies !== undefined && !Array.isArray(layer.forbiddenDependencies)) {
+      throw new Error(
+        `Architecture layer "${layer.name}" forbiddenDependencies must be an array of layer names.`,
+      );
+    }
+    for (const forbidden of layer.forbiddenDependencies ?? []) {
+      if (typeof forbidden !== 'string' || !layerNames.has(forbidden)) {
+        throw new Error(
+          `Architecture layer "${layer.name}" forbids unknown dependency "${String(forbidden)}".`,
+        );
+      }
+      if (layer.allowedDependencies.includes(forbidden)) {
+        throw new Error(
+          `Architecture layer "${layer.name}" both allows and forbids dependency "${forbidden}".`,
+        );
+      }
+    }
+  }
+}
+
+function validateDagPolicy(config: LayerConfig, layerNames: Set<string>): void {
+  validateTierOrder(config.tierOrder, layerNames);
+  validateMaxStronglyConnectedComponentSize(config.maxStronglyConnectedComponentSize);
+  validateForbiddenDependencies(config.layers, layerNames);
+}
+
+export function readLayerConfig(repoRoot: string): LayerConfig {
+  const configPath = path.join(repoRoot, 'architecture/layers.json');
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf8')) as LayerConfig;
+
+  if (!config || typeof config !== 'object' || !Array.isArray(config.layers)) {
+    throw new Error(`${configPath} must define a layers array.`);
+  }
+  if (
+    typeof config.publicFacade !== 'string' ||
+    !fs.existsSync(path.join(repoRoot, config.publicFacade))
+  ) {
+    throw new Error(`${configPath} must define an existing publicFacade path.`);
+  }
+
+  const layerNames = new Set<string>();
+  const configuredRoots: Array<{ layerName: string; root: string }> = [];
+
+  for (const layer of config.layers) {
+    if (layerNames.has(layer.name)) {
+      throw new Error(`${configPath} contains duplicate layer "${layer.name}".`);
+    }
+    layerNames.add(layer.name);
+    configuredRoots.push(...validateLayerDefinition(repoRoot, configPath, layer));
+  }
+
+  validateDependencies(config, layerNames);
+  validateAliases(repoRoot, config.aliases);
+  validateRootsDoNotOverlap(configuredRoots);
+  validateDagPolicy(config, layerNames);
+
+  return config;
+}
+
+export function getSourceFiles(
+  repoRoot: string,
+  includeApp = true,
+  ignoredRoots: string[] = [],
+): string[] {
+  return globSync('src/**/*.{ts,tsx,mts,cts}', {
+    cwd: repoRoot,
+    ignore: [
+      'src/**/*.d.ts',
+      'src/**/node_modules/**',
+      ...(includeApp ? [] : ['src/app/**']),
+      ...ignoredRoots.map((root) => `${normalizePath(root)}/**`),
+    ],
+    nodir: true,
+  }).map(normalizePath);
+}
+
+function isWithinRoot(relativePath: string, root: string): boolean {
+  const normalizedPath = normalizePath(relativePath);
+  const normalizedRoot = normalizePath(root);
+  return normalizedPath === normalizedRoot || normalizedPath.startsWith(`${normalizedRoot}/`);
+}
+
+export function getLayerForFile(relativePath: string, config: LayerConfig): string {
+  const normalizedPath = normalizePath(relativePath);
+  for (const layer of config.layers) {
+    if (layer.roots.some((root) => isWithinRoot(normalizedPath, root))) {
+      return layer.name;
+    }
+  }
+  return 'unclassified';
+}
+
+export function extractModuleSpecifiers(sourceText: string, filePath: string): string[] {
+  const sourceFile = ts.createSourceFile(
+    filePath,
+    sourceText,
+    ts.ScriptTarget.Latest,
+    true,
+    filePath.endsWith('.tsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
+  );
+  const specifiers: string[] = [];
+
+  function addStaticCallSpecifier(node: ts.CallExpression): void {
+    if (node.arguments.length !== 1 || !ts.isStringLiteralLike(node.arguments[0])) {
+      return;
+    }
+
+    if (
+      node.expression.kind === ts.SyntaxKind.ImportKeyword ||
+      (ts.isIdentifier(node.expression) && node.expression.text === 'require') ||
+      (ts.isPropertyAccessExpression(node.expression) &&
+        ts.isIdentifier(node.expression.expression) &&
+        node.expression.expression.text === 'require' &&
+        node.expression.name.text === 'resolve')
+    ) {
+      specifiers.push(node.arguments[0].text);
+    }
+  }
+
+  function visit(node: ts.Node): void {
+    if (
+      (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) &&
+      node.moduleSpecifier &&
+      ts.isStringLiteralLike(node.moduleSpecifier)
+    ) {
+      specifiers.push(node.moduleSpecifier.text);
+    }
+
+    if (
+      ts.isImportEqualsDeclaration(node) &&
+      ts.isExternalModuleReference(node.moduleReference) &&
+      node.moduleReference.expression &&
+      ts.isStringLiteralLike(node.moduleReference.expression)
+    ) {
+      specifiers.push(node.moduleReference.expression.text);
+    }
+
+    if (
+      ts.isImportTypeNode(node) &&
+      ts.isLiteralTypeNode(node.argument) &&
+      ts.isStringLiteralLike(node.argument.literal)
+    ) {
+      specifiers.push(node.argument.literal.text);
+    }
+
+    if (ts.isCallExpression(node)) {
+      addStaticCallSpecifier(node);
+    }
+
+    ts.forEachChild(node, visit);
+  }
+
+  visit(sourceFile);
+  return specifiers;
+}
+
+export function resolveInternalModule(
+  repoRoot: string,
+  importerRelativePath: string,
+  specifier: string,
+  aliases: Record<string, string> = {},
+): string | undefined {
+  const matchingAlias = Object.keys(aliases)
+    .sort((left, right) => right.length - left.length)
+    .find((alias) => specifier === alias || specifier.startsWith(`${alias}/`));
+  const aliasedPath = matchingAlias
+    ? `${aliases[matchingAlias]}${specifier.slice(matchingAlias.length)}`
+    : undefined;
+
+  if (
+    !specifier.startsWith('.') &&
+    specifier !== 'src' &&
+    !specifier.startsWith('src/') &&
+    !aliasedPath
+  ) {
+    return undefined;
+  }
+
+  let unresolvedPath: string;
+  if (aliasedPath) {
+    unresolvedPath = path.resolve(repoRoot, aliasedPath);
+  } else if (specifier.startsWith('.')) {
+    unresolvedPath = path.resolve(
+      path.dirname(path.join(repoRoot, importerRelativePath)),
+      specifier,
+    );
+  } else {
+    unresolvedPath = path.resolve(repoRoot, specifier);
+  }
+
+  const runtimeExtension = path.extname(unresolvedPath);
+  const runtimeSourceCandidates = (
+    SOURCE_EXTENSIONS_BY_RUNTIME_EXTENSION[runtimeExtension] ?? []
+  ).map((extension) => `${unresolvedPath.slice(0, -runtimeExtension.length)}${extension}`);
+  const candidates = [
+    unresolvedPath,
+    ...runtimeSourceCandidates,
+    ...TYPESCRIPT_EXTENSIONS.map((extension) => `${unresolvedPath}${extension}`),
+    ...DIRECTORY_INDEXES.map((indexFile) => path.join(unresolvedPath, indexFile)),
+  ];
+
+  for (const candidate of candidates) {
+    if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
+      const relativeCandidate = normalizePath(path.relative(repoRoot, candidate));
+      if (
+        relativeCandidate.startsWith('src/') &&
+        TYPESCRIPT_EXTENSIONS.includes(path.extname(relativeCandidate))
+      ) {
+        return relativeCandidate;
+      }
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * The bare module name a specifier resolves to: its package root (`@scope/pkg`) or builtin name,
+ * with any `node:` prefix stripped. Returns undefined for relative/absolute/`#` subpath specifiers,
+ * which are not external dependencies. Unlike {@link getPackageName} this also names Node builtins,
+ * so a leaf-layer external check cannot let `node:fs` slip through.
+ */
+export function getExternalModuleName(specifier: string): string | undefined {
+  if (specifier.startsWith('.') || specifier.startsWith('/') || specifier.startsWith('#')) {
+    return undefined;
+  }
+
+  const withoutNodePrefix = specifier.replace(/^node:/, '');
+  const segments = withoutNodePrefix.split('/');
+  const moduleName = withoutNodePrefix.startsWith('@')
+    ? segments.slice(0, 2).join('/')
+    : segments[0];
+
+  return moduleName || undefined;
+}
+
+/** The npm package name a specifier imports, or undefined for relative imports and Node builtins. */
+export function getPackageName(specifier: string): string | undefined {
+  const moduleName = getExternalModuleName(specifier);
+  return moduleName && !BUILTIN_MODULES.has(moduleName) ? moduleName : undefined;
+}
+
+export type BoundaryViolationKind = 'facade' | 'layer' | 'leaf' | 'leaf-external' | 'path';
+
+export interface BoundaryViolation {
+  kind: BoundaryViolationKind;
+  importer: string;
+  importerLayer: string;
+  specifier: string;
+  /** Resolved import target. Set for every violation kind. */
+  imported: string;
+  /** Layer of the resolved import. Set for every violation kind. */
+  importedLayer: string;
+}
+
+export interface ArchitectureModuleReference {
+  importer: string;
+  importerLayer: string;
+  specifier: string;
+  resolvedImport?: string;
+  importedLayer?: string;
+}
+
+export interface ArchitectureSourceScan {
+  sourceFiles: string[];
+  references: ArchitectureModuleReference[];
+}
+
+/**
+ * Reads and parses the checked source tree once so callers that need multiple
+ * architecture views can reuse the same module-reference scan.
+ */
+export function scanArchitectureSources(
+  repoRoot: string,
+  config: LayerConfig,
+): ArchitectureSourceScan {
+  const publicFacade = normalizePath(config.publicFacade);
+  const sourceFiles = getSourceFiles(repoRoot, true, config.ignoredRoots);
+  const references: ArchitectureModuleReference[] = [];
+
+  for (const importer of sourceFiles) {
+    if (normalizePath(importer) === publicFacade) {
+      continue;
+    }
+
+    const importerLayer = getLayerForFile(importer, config);
+    const sourceText = fs.readFileSync(path.join(repoRoot, importer), 'utf8');
+    for (const specifier of extractModuleSpecifiers(sourceText, importer)) {
+      const resolvedImport = resolveInternalModule(repoRoot, importer, specifier, config.aliases);
+      references.push({
+        importer,
+        importerLayer,
+        specifier,
+        ...(resolvedImport
+          ? { resolvedImport, importedLayer: getLayerForFile(resolvedImport, config) }
+          : {}),
+      });
+    }
+  }
+
+  return { sourceFiles, references };
+}
+
+export function findUnclassifiedFiles(
+  repoRoot: string,
+  config: LayerConfig,
+  sourceFiles = getSourceFiles(repoRoot, true, config.ignoredRoots),
+): string[] {
+  return sourceFiles.filter((sourceFile) => getLayerForFile(sourceFile, config) === 'unclassified');
+}
+
+/**
+ * Walks the source tree under `repoRoot` and returns every layer-boundary
+ * violation, given a `LayerConfig`. Pure function: no I/O beyond the file
+ * reads required to scan source files; no `process.exit`. Exposed so the
+ * pipeline can be unit-tested against fixture trees.
+ */
+export function findViolations(
+  repoRoot: string,
+  config: LayerConfig,
+  sourceScan = scanArchitectureSources(repoRoot, config),
+): BoundaryViolation[] {
+  const publicFacade = normalizePath(config.publicFacade);
+  const leafLayers = new Set(config.leafLayers ?? []);
+  const layersByName = new Map(config.layers.map((layer) => [layer.name, layer]));
+  const violations: BoundaryViolation[] = [];
+
+  for (const {
+    importer,
+    importerLayer,
+    specifier,
+    resolvedImport,
+    importedLayer,
+  } of sourceScan.references) {
+    if (normalizePath(importer) === publicFacade) {
+      continue;
+    }
+
+    const importerIsLeaf = leafLayers.has(importerLayer);
+    const allowedExternal = importerIsLeaf
+      ? new Set(
+          (layersByName.get(importerLayer)?.allowedExternal ?? []).map((entry) =>
+            entry.replace(/^node:/, ''),
+          ),
+        )
+      : null;
+
+    if (!resolvedImport || !importedLayer) {
+      // Not an internal module (internal relative / src-rooted / aliased imports resolve above).
+      // A leaf layer may import only its allowlisted external packages and Node builtins; flag
+      // any other bare specifier.
+      if (importerIsLeaf) {
+        const externalName = getExternalModuleName(specifier);
+        if (externalName && !allowedExternal!.has(externalName)) {
+          violations.push({
+            kind: 'leaf-external',
+            importer,
+            importerLayer,
+            specifier,
+            imported: specifier,
+            importedLayer: 'external',
+          });
+        }
+      }
+      continue;
+    }
+
+    if (resolvedImport === publicFacade) {
+      violations.push({
+        kind: 'facade',
+        importer,
+        importerLayer,
+        specifier,
+        imported: resolvedImport,
+        importedLayer,
+      });
+    }
+
+    if (leafLayers.has(importerLayer)) {
+      if (importedLayer !== importerLayer) {
+        violations.push({
+          kind: 'leaf',
+          importer,
+          importerLayer,
+          specifier,
+          imported: resolvedImport,
+          importedLayer,
+        });
+      }
+    }
+
+    const importerConfig = layersByName.get(importerLayer);
+    const allowedLayerDependency =
+      importedLayer === importerLayer ||
+      (importerConfig?.allowedDependencies.includes(importedLayer) ?? false);
+
+    if (
+      importedLayer !== importerLayer &&
+      resolvedImport !== publicFacade &&
+      !leafLayers.has(importerLayer) &&
+      importerConfig &&
+      !allowedLayerDependency
+    ) {
+      violations.push({
+        kind: 'layer',
+        importer,
+        importerLayer,
+        specifier,
+        imported: resolvedImport,
+        importedLayer,
+      });
+    }
+
+    if (
+      importedLayer !== importerLayer &&
+      allowedLayerDependency &&
+      importerConfig?.allowedImportPaths &&
+      !importerConfig.allowedImportPaths.includes(resolvedImport)
+    ) {
+      violations.push({
+        kind: 'path',
+        importer,
+        importerLayer,
+        specifier,
+        imported: resolvedImport,
+        importedLayer,
+      });
+    }
+  }
+
+  return violations;
+}
+
+export interface CrossLayerEdge {
+  from: string;
+  to: string;
+  /** Number of module references crossing from `from` to `to`. */
+  count: number;
+  /** Distinct importer files behind the edge, sorted. */
+  files: string[];
+}
+
+/**
+ * Tallies the directed cross-layer module-reference edges across the source
+ * tree. Each edge is an (importerLayer -> importedLayer) pair with the number
+ * of module references and the distinct importer files behind it. The public
+ * facade file is skipped as an importer (it intentionally re-exports every
+ * layer); same-layer and unresolved/external references are ignored. Pure
+ * function aside from the source reads, so it can be unit-tested against
+ * fixture trees.
+ */
+export function computeCrossLayerEdges(
+  repoRoot: string,
+  config: LayerConfig,
+  sourceScan = scanArchitectureSources(repoRoot, config),
+): CrossLayerEdge[] {
+  const tallies = new Map<string, { count: number; files: Set<string> }>();
+
+  for (const { importer, importerLayer, resolvedImport, importedLayer } of sourceScan.references) {
+    if (!resolvedImport || !importedLayer || importedLayer === importerLayer) {
+      continue;
+    }
+
+    const key = edgeKey(importerLayer, importedLayer);
+    const tally = tallies.get(key) ?? { count: 0, files: new Set<string>() };
+    tally.count += 1;
+    tally.files.add(importer);
+    tallies.set(key, tally);
+  }
+
+  return [...tallies.entries()]
+    .map(([key, tally]) => {
+      const [from, to] = key.split(EDGE_SEPARATOR);
+      return { from, to, count: tally.count, files: [...tally.files].sort() };
+    })
+    .sort((left, right) => left.from.localeCompare(right.from) || left.to.localeCompare(right.to));
+}
+
+/**
+ * Computes the strongly-connected components of the layer dependency graph
+ * (Tarjan's algorithm). Every configured layer is a node, so singletons are
+ * included; a component with more than one layer is a dependency cycle.
+ * Components are returned with their layers sorted, ordered by descending size.
+ */
+export function computeStronglyConnectedComponents(
+  config: LayerConfig,
+  edges: CrossLayerEdge[],
+): string[][] {
+  const nodes = config.layers.map((layer) => layer.name);
+  const adjacency = new Map<string, string[]>(nodes.map((node) => [node, []]));
+  for (const edge of edges) {
+    if (adjacency.has(edge.from) && adjacency.has(edge.to)) {
+      adjacency.get(edge.from)!.push(edge.to);
+    }
+  }
+
+  let nextIndex = 0;
+  const indices = new Map<string, number>();
+  const lowLinks = new Map<string, number>();
+  const onStack = new Set<string>();
+  const stack: string[] = [];
+  const components: string[][] = [];
+
+  const strongConnect = (node: string): void => {
+    indices.set(node, nextIndex);
+    lowLinks.set(node, nextIndex);
+    nextIndex += 1;
+    stack.push(node);
+    onStack.add(node);
+
+    for (const next of adjacency.get(node) ?? []) {
+      if (!indices.has(next)) {
+        strongConnect(next);
+        lowLinks.set(node, Math.min(lowLinks.get(node)!, lowLinks.get(next)!));
+      } else if (onStack.has(next)) {
+        lowLinks.set(node, Math.min(lowLinks.get(node)!, indices.get(next)!));
+      }
+    }
+
+    if (lowLinks.get(node) === indices.get(node)) {
+      const component: string[] = [];
+      let member: string;
+      do {
+        member = stack.pop()!;
+        onStack.delete(member);
+        component.push(member);
+      } while (member !== node);
+      components.push(component.sort());
+    }
+  };
+
+  for (const node of nodes) {
+    if (!indices.has(node)) {
+      strongConnect(node);
+    }
+  }
+
+  return components.sort((left, right) => right.length - left.length);
+}
+
+/**
+ * Given a target topological order (`tierOrder`, most-depended-upon first),
+ * returns the cross-layer edges that violate it — a lower-tier layer importing
+ * a higher-tier layer. These are the cycle-causing "back edges" to eliminate on
+ * the way to a DAG. Edges touching a layer absent from `tierOrder` are ignored.
+ */
+export function findBackEdges(edges: CrossLayerEdge[], tierOrder: string[]): CrossLayerEdge[] {
+  const rank = new Map(tierOrder.map((layer, index) => [layer, index]));
+  return edges.filter((edge) => {
+    const fromRank = rank.get(edge.from);
+    const toRank = rank.get(edge.to);
+    return fromRank !== undefined && toRank !== undefined && fromRank < toRank;
+  });
+}
+
+export interface ForbiddenDependencyViolation {
+  from: string;
+  to: string;
+  path: CrossLayerEdge[];
+}
+
+function findDependencyPath(
+  edgesByFrom: Map<string, CrossLayerEdge[]>,
+  from: string,
+  to: string,
+): CrossLayerEdge[] | undefined {
+  const queue: Array<{ layer: string; path: CrossLayerEdge[] }> = [{ layer: from, path: [] }];
+  const visited = new Set([from]);
+
+  while (queue.length > 0) {
+    const current = queue.shift()!;
+    for (const edge of edgesByFrom.get(current.layer) ?? []) {
+      const path = [...current.path, edge];
+      if (edge.to === to) {
+        return path;
+      }
+      if (!visited.has(edge.to)) {
+        visited.add(edge.to);
+        queue.push({ layer: edge.to, path });
+      }
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Returns direct or transitive dependency paths that violate an explicit
+ * `forbiddenDependencies` lock.
+ */
+export function findForbiddenDependencyViolations(
+  config: LayerConfig,
+  edges: CrossLayerEdge[],
+): ForbiddenDependencyViolation[] {
+  const edgesByFrom = new Map<string, CrossLayerEdge[]>();
+  for (const edge of edges) {
+    const outgoing = edgesByFrom.get(edge.from) ?? [];
+    outgoing.push(edge);
+    edgesByFrom.set(edge.from, outgoing);
+  }
+  for (const outgoing of edgesByFrom.values()) {
+    outgoing.sort((left, right) => left.to.localeCompare(right.to));
+  }
+
+  const violations: ForbiddenDependencyViolation[] = [];
+  for (const layer of config.layers) {
+    for (const forbidden of layer.forbiddenDependencies ?? []) {
+      const path = findDependencyPath(edgesByFrom, layer.name, forbidden);
+      if (path !== undefined) {
+        violations.push({ from: layer.name, to: forbidden, path });
+      }
+    }
+  }
+  return violations;
+}
+
+export type EdgeBaseline = Record<string, number>;
+
+const EDGE_SEPARATOR = ' -> ';
+
+export function edgeKey(from: string, to: string): string {
+  return `${from}${EDGE_SEPARATOR}${to}`;
+}
+
+/** Builds a baseline mapping each cross-layer edge to its current module-reference count. */
+export function buildEdgeBaseline(edges: CrossLayerEdge[], config: LayerConfig): EdgeBaseline {
+  const layerNames = new Set(config.layers.map((layer) => layer.name));
+  const baseline: EdgeBaseline = {};
+  for (const edge of edges
+    .slice()
+    .sort((left, right) =>
+      edgeKey(left.from, left.to).localeCompare(edgeKey(right.from, right.to)),
+    )) {
+    const key = edgeKey(edge.from, edge.to);
+    if (!layerNames.has(edge.from) || !layerNames.has(edge.to)) {
+      throw new Error(`Cannot build edge baseline: edge "${key}" references an unknown layer.`);
+    }
+    baseline[key] = edge.count;
+  }
+  return baseline;
+}
+
+export interface EdgeBaselineComparison {
+  /** Edges whose import count exceeds the baseline (or are entirely new). */
+  regressions: Array<{ from: string; to: string; count: number; allowed: number }>;
+  /** Edges whose count dropped below the baseline (baseline can be lowered). */
+  improvements: Array<{ from: string; to: string; count: number; allowed: number }>;
+}
+
+/**
+ * Compares the live cross-layer edges against a committed baseline. Any edge
+ * whose count exceeds its baseline (including a brand-new edge, baseline 0) is
+ * a regression; any edge that shrank — or disappeared — is an improvement that
+ * the baseline can be lowered to capture.
+ */
+export function compareEdgesToBaseline(
+  edges: CrossLayerEdge[],
+  baseline: EdgeBaseline,
+): EdgeBaselineComparison {
+  const regressions: EdgeBaselineComparison['regressions'] = [];
+  const improvements: EdgeBaselineComparison['improvements'] = [];
+  const seen = new Set<string>();
+
+  for (const edge of edges) {
+    const key = edgeKey(edge.from, edge.to);
+    seen.add(key);
+    const allowed = baseline[key] ?? 0;
+    if (edge.count > allowed) {
+      regressions.push({ from: edge.from, to: edge.to, count: edge.count, allowed });
+    } else if (edge.count < allowed) {
+      improvements.push({ from: edge.from, to: edge.to, count: edge.count, allowed });
+    }
+  }
+
+  for (const key of Object.keys(baseline)) {
+    if (!seen.has(key) && baseline[key] > 0) {
+      const [from, to] = key.split(EDGE_SEPARATOR);
+      improvements.push({ from, to, count: 0, allowed: baseline[key] });
+    }
+  }
+
+  return { regressions, improvements };
+}
+
+/** Reads and validates the committed cross-layer edge baseline, or an empty baseline if absent. */
+export function readEdgeBaseline(repoRoot: string, config: LayerConfig): EdgeBaseline {
+  const baselinePath = path.join(repoRoot, 'architecture/edge-baseline.json');
+  if (!fs.existsSync(baselinePath)) {
+    return {};
+  }
+  const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf8')) as unknown;
+  if (!baseline || typeof baseline !== 'object' || Array.isArray(baseline)) {
+    throw new Error(`${baselinePath} must contain an object mapping layer edges to import counts.`);
+  }
+
+  const layerNames = new Set(config.layers.map((layer) => layer.name));
+  for (const [key, count] of Object.entries(baseline)) {
+    const parts = key.split(EDGE_SEPARATOR);
+    if (
+      parts.length !== 2 ||
+      parts.some((layerName) => layerName.length === 0 || !layerNames.has(layerName))
+    ) {
+      throw new Error(`${baselinePath} contains invalid edge "${key}".`);
+    }
+    if (typeof count !== 'number' || !Number.isInteger(count) || count < 1) {
+      throw new Error(`${baselinePath} contains invalid import count for edge "${key}".`);
+    }
+  }
+
+  return baseline as EdgeBaseline;
+}

scripts/checkArchitectureBoundaries.ts

@@ -0,0 +1,183 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import {
+  compareEdgesToBaseline,
+  computeCrossLayerEdges,
+  computeStronglyConnectedComponents,
+  findBackEdges,
+  findForbiddenDependencyViolations,
+  findUnclassifiedFiles,
+  findViolations,
+  readEdgeBaseline,
+  readLayerConfig,
+  scanArchitectureSources,
+} from './architectureUtils';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const config = readLayerConfig(repoRoot);
+const sourceScan = scanArchitectureSources(repoRoot, config);
+const violations = findViolations(repoRoot, config, sourceScan);
+
+const facadeViolations = violations.filter((v) => v.kind === 'facade');
+const leafViolations = violations.filter((v) => v.kind === 'leaf');
+const leafExternalViolations = violations.filter((v) => v.kind === 'leaf-external');
+const layerViolations = violations.filter((v) => v.kind === 'layer');
+const pathViolations = violations.filter((v) => v.kind === 'path');
+const unclassifiedFiles = findUnclassifiedFiles(repoRoot, config, sourceScan.sourceFiles);
+
+if (facadeViolations.length > 0) {
+  console.error('Architecture boundary violations found:');
+  for (const violation of facadeViolations) {
+    console.error(
+      `- ${violation.importer} (${violation.importerLayer}) imports public facade via "${violation.specifier}"`,
+    );
+  }
+  console.error(
+    '\nInternal modules must import a narrower internal surface instead of src/index.ts.',
+  );
+}
+
+if (leafViolations.length > 0) {
+  console.error('Leaf-layer boundary violations found:');
+  for (const violation of leafViolations) {
+    console.error(
+      `- ${violation.importer} (${violation.importerLayer}) imports ${violation.imported} (${violation.importedLayer}) via "${violation.specifier}"`,
+    );
+  }
+  console.error('\nLeaf layers must not import other product layers.');
+}
+
+if (leafExternalViolations.length > 0) {
+  console.error('Leaf-layer external dependency violations found:');
+  for (const violation of leafExternalViolations) {
+    console.error(
+      `- ${violation.importer} (${violation.importerLayer}) imports external "${violation.specifier}"`,
+    );
+  }
+  console.error(
+    '\nLeaf layers may import only their allowlisted external packages (allowedExternal).',
+  );
+}
+
+if (layerViolations.length > 0) {
+  console.error('Layer dependency violations found:');
+  for (const violation of layerViolations) {
+    console.error(
+      `- ${violation.importer} (${violation.importerLayer}) imports ${violation.imported} (${violation.importedLayer}) via "${violation.specifier}"`,
+    );
+  }
+  console.error('\nLayers may import only their explicitly allowed dependencies.');
+}
+
+if (pathViolations.length > 0) {
+  console.error('Restricted layer import violations found:');
+  for (const violation of pathViolations) {
+    console.error(
+      `- ${violation.importer} (${violation.importerLayer}) imports ${violation.imported} (${violation.importedLayer}) via "${violation.specifier}"`,
+    );
+  }
+  console.error('\nRestricted layers may import only their explicitly allowed internal paths.');
+}
+
+if (unclassifiedFiles.length > 0) {
+  console.error('Unclassified source files found:');
+  for (const sourceFile of unclassifiedFiles) {
+    console.error(`- ${sourceFile}`);
+  }
+  console.error('\nEvery checked source file must belong to an explicit architecture layer.');
+}
+
+// ---- DAG progress checks (layer dependency graph) ----
+const edges = computeCrossLayerEdges(repoRoot, config, sourceScan);
+
+// 1. Strongly-connected-component ratchet: the largest dependency cycle may
+//    only shrink over time (toward a DAG, where every component has size 1).
+const components = computeStronglyConnectedComponents(config, edges);
+const cycles = components.filter((component) => component.length > 1);
+const largestCycle = cycles[0]?.length ?? 1;
+let sccRatchetFailed = false;
+if (config.maxStronglyConnectedComponentSize !== undefined) {
+  if (largestCycle > config.maxStronglyConnectedComponentSize) {
+    sccRatchetFailed = true;
+    console.error(
+      `Layer dependency cycle grew: largest strongly-connected component has ${largestCycle} layers, ` +
+        `but maxStronglyConnectedComponentSize is ${config.maxStronglyConnectedComponentSize}.`,
+    );
+    for (const cycle of cycles) {
+      console.error(`- cycle: {${cycle.join(', ')}}`);
+    }
+    console.error(
+      '\nBreaking a cross-layer cycle should lower maxStronglyConnectedComponentSize in architecture/layers.json; new cycles are not allowed.',
+    );
+  } else if (largestCycle < config.maxStronglyConnectedComponentSize) {
+    console.log(
+      `Note: largest dependency cycle is now ${largestCycle} layers (ratchet is ${config.maxStronglyConnectedComponentSize}). ` +
+        'Lower maxStronglyConnectedComponentSize in architecture/layers.json to lock in the progress.',
+    );
+  }
+}
+
+// 2. Per-edge baseline ratchet: no cross-layer edge may grow, and no new
+//    cross-layer edge may appear, relative to architecture/edge-baseline.json.
+const baseline = readEdgeBaseline(repoRoot, config);
+const { regressions, improvements } = compareEdgesToBaseline(edges, baseline);
+if (regressions.length > 0) {
+  console.error('Cross-layer dependency baseline regressions found:');
+  for (const regression of regressions) {
+    const previously = regression.allowed === 0 ? 'new edge' : `was ${regression.allowed}`;
+    console.error(
+      `- ${regression.from} -> ${regression.to}: ${regression.count} imports (${previously})`,
+    );
+  }
+  console.error(
+    '\nNew or widened cross-layer dependencies are not allowed. Remove the import, or if it is intentional run `npm run architecture:baseline` and justify the change in review.',
+  );
+}
+if (improvements.length > 0) {
+  console.log(
+    `Note: ${improvements.length} cross-layer edge(s) shrank below baseline. Run \`npm run architecture:baseline\` to lock in the reduction.`,
+  );
+}
+
+// 3. Explicit forbidden-dependency locks (pairs whose cycle is fully broken).
+const forbiddenDependencyViolations = findForbiddenDependencyViolations(config, edges);
+if (forbiddenDependencyViolations.length > 0) {
+  console.error('Forbidden cross-layer dependencies found:');
+  for (const violation of forbiddenDependencyViolations) {
+    const layerPath = [violation.from, ...violation.path.map((edge) => edge.to)].join(' -> ');
+    console.error(`- ${violation.from} depends on ${violation.to} via ${layerPath} (forbidden)`);
+    for (const edge of violation.path) {
+      console.error(`    ${edge.from} -> ${edge.to}: ${edge.count} imports`);
+      for (const file of edge.files) {
+        console.error(`      ${file}`);
+      }
+    }
+  }
+  console.error(
+    '\nThese layer pairs are locked acyclic via forbiddenDependencies and must not regress directly or transitively.',
+  );
+}
+
+// 4. Informational: back-edges versus the target topological order.
+if (config.tierOrder !== undefined) {
+  const backEdges = findBackEdges(edges, config.tierOrder);
+  if (backEdges.length > 0) {
+    const totalBackImports = backEdges.reduce((sum, edge) => sum + edge.count, 0);
+    console.log(
+      `\nRemaining back-edges versus target order (${backEdges.length} edges / ${totalBackImports} imports to reach a DAG):`,
+    );
+    for (const edge of [...backEdges].sort((left, right) => left.count - right.count)) {
+      console.log(`- ${edge.from} -> ${edge.to}: ${edge.count} imports`);
+    }
+  }
+}
+
+const dagChecksFailed =
+  sccRatchetFailed || regressions.length > 0 || forbiddenDependencyViolations.length > 0;
+
+if (violations.length > 0 || unclassifiedFiles.length > 0 || dagChecksFailed) {
+  process.exitCode = 1;
+} else {
+  console.log('Architecture boundaries passed.');
+}

scripts/checkCoverageRatchets.ts

@@ -0,0 +1,553 @@
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export interface ChangedFile {
+  path: string;
+  status: string;
+}
+
+export interface CoverageThresholds {
+  branches: number;
+  functions: number;
+  lines: number;
+  statements: number;
+}
+
+interface Position {
+  line: number;
+}
+
+interface StatementLocation {
+  start: Position;
+}
+
+interface FileCoverage {
+  b?: Record<string, number[]>;
+  branchMap?: Record<string, unknown>;
+  f?: Record<string, number>;
+  fnMap?: Record<string, unknown>;
+  path?: string;
+  s?: Record<string, number>;
+  statementMap?: Record<string, StatementLocation>;
+}
+
+type CoverageMap = Record<string, FileCoverage>;
+
+export interface CoverageReportConfig {
+  coverageFile: string;
+  criticalFiles: string[];
+  criticalPrefixes: string[];
+  excludeFiles: string[];
+  excludePrefixes: string[];
+  name: string;
+  sourcePrefix: string;
+}
+
+interface CoverageTotals {
+  covered: number;
+  pct: number;
+  total: number;
+}
+
+export interface FileCoverageSummary {
+  branches: CoverageTotals;
+  functions: CoverageTotals;
+  lines: CoverageTotals;
+  statements: CoverageTotals;
+}
+
+interface CheckedFile {
+  file: string;
+  reason: string;
+  summary: FileCoverageSummary;
+}
+
+interface CoverageFailure {
+  file: string;
+  message: string;
+  reason: string;
+}
+
+interface CoverageRatchetResult {
+  checkedFiles: CheckedFile[];
+  failures: CoverageFailure[];
+  skippedFiles: string[];
+}
+
+interface CliOptions {
+  baseRef?: string;
+  reports: string[];
+}
+
+interface GithubPullRequestEvent {
+  pull_request?: {
+    base?: {
+      sha?: unknown;
+    };
+  };
+}
+
+export const DEFAULT_COVERAGE_THRESHOLDS: CoverageThresholds = {
+  branches: 70,
+  functions: 80,
+  lines: 80,
+  statements: 80,
+};
+
+export const COVERAGE_RATCHET_REPORTS: CoverageReportConfig[] = [
+  {
+    name: 'backend',
+    coverageFile: 'coverage/coverage-final.json',
+    sourcePrefix: 'src/',
+    excludePrefixes: ['src/app/', 'src/__mocks__/'],
+    excludeFiles: ['src/entrypoint.ts', 'src/main.ts', 'src/migrate.ts'],
+    criticalPrefixes: ['src/assertions/', 'src/matchers/', 'src/util/config/'],
+    criticalFiles: ['src/evaluator.ts', 'src/evaluatorHelpers.ts', 'src/prompts.ts'],
+  },
+  {
+    name: 'frontend',
+    coverageFile: 'src/app/coverage/coverage-final.json',
+    sourcePrefix: 'src/app/src/',
+    excludePrefixes: [],
+    excludeFiles: ['src/app/src/setupTests.ts'],
+    criticalPrefixes: ['src/app/src/store/', 'src/app/src/stores/', 'src/app/src/tests/'],
+    criticalFiles: ['src/app/src/utils/api.ts'],
+  },
+];
+
+function git(args: string[], cwd: string): string {
+  return execFileSync('git', args, {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+function hasGitRef(ref: string, cwd: string): boolean {
+  try {
+    git(['rev-parse', '--verify', '--quiet', ref], cwd);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function fetchGitRef(ref: string, cwd: string): void {
+  if (hasGitRef(ref, cwd)) {
+    return;
+  }
+
+  try {
+    git(['fetch', '--depth=1', 'origin', ref], cwd);
+  } catch {
+    // The ref may already be unavailable to this checkout; later diff candidates can still work.
+  }
+}
+
+function fetchGithubBaseRef(cwd: string): void {
+  const baseRef = process.env.GITHUB_BASE_REF;
+  if (!baseRef) {
+    return;
+  }
+
+  try {
+    git(['fetch', '--depth=1', 'origin', `${baseRef}:refs/remotes/origin/${baseRef}`], cwd);
+  } catch {
+    // The local checkout may already have enough history, and forks may not allow this fetch.
+  }
+}
+
+export function readGithubPullRequestBaseSha(
+  eventPath = process.env.GITHUB_EVENT_PATH,
+): string | undefined {
+  if (!eventPath) {
+    return undefined;
+  }
+
+  try {
+    const event = JSON.parse(fs.readFileSync(eventPath, 'utf8')) as GithubPullRequestEvent;
+    const baseSha = event.pull_request?.base?.sha;
+    return typeof baseSha === 'string' && baseSha.length > 0 ? baseSha : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function parseChangedFileList(output: string): ChangedFile[] {
+  return output
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [status, ...paths] = line.split('\t');
+      const normalizedStatus = status.charAt(0);
+      const changedPath =
+        normalizedStatus === 'R' || normalizedStatus === 'C' ? paths[1] : paths[0];
+
+      return {
+        path: normalizeSlashes(changedPath),
+        status: normalizedStatus,
+      };
+    });
+}
+
+export function getChangedFiles(cwd: string, baseRef?: string): ChangedFile[] {
+  fetchGithubBaseRef(cwd);
+
+  const diffCommands: string[][] = [];
+  const isGithubActions = process.env.GITHUB_ACTIONS === 'true';
+  const githubBaseSha = readGithubPullRequestBaseSha();
+
+  if (baseRef) {
+    fetchGitRef(baseRef, cwd);
+    diffCommands.push(['diff', '--name-status', '--diff-filter=ACMRTUXB', `${baseRef}...HEAD`]);
+  }
+
+  if (githubBaseSha) {
+    fetchGitRef(githubBaseSha, cwd);
+    diffCommands.push([
+      'diff',
+      '--name-status',
+      '--diff-filter=ACMRTUXB',
+      `${githubBaseSha}...HEAD`,
+    ]);
+  }
+
+  if (isGithubActions && hasGitRef('HEAD^1', cwd)) {
+    diffCommands.push(['diff', '--name-status', '--diff-filter=ACMRTUXB', 'HEAD^1', 'HEAD']);
+  }
+
+  const githubBaseRef = process.env.GITHUB_BASE_REF;
+  if (githubBaseRef) {
+    diffCommands.push([
+      'diff',
+      '--name-status',
+      '--diff-filter=ACMRTUXB',
+      `origin/${githubBaseRef}...HEAD`,
+    ]);
+  }
+
+  diffCommands.push(['diff', '--name-status', '--diff-filter=ACMRTUXB', 'origin/main...HEAD']);
+
+  if (!isGithubActions && hasGitRef('HEAD^', cwd)) {
+    diffCommands.push(['diff', '--name-status', '--diff-filter=ACMRTUXB', 'HEAD^', 'HEAD']);
+  }
+
+  for (const args of diffCommands) {
+    try {
+      return parseChangedFileList(git(args, cwd));
+    } catch {
+      // Try the next base candidate.
+    }
+  }
+
+  throw new Error('Unable to determine changed files for coverage ratchets');
+}
+
+function normalizeSlashes(filePath: string): string {
+  return filePath.replace(/\\/g, '/').replace(/^\.\//, '');
+}
+
+export function normalizeCoveragePath(filePath: string, repoRoot: string): string {
+  const normalizedRoot = normalizeSlashes(path.resolve(repoRoot));
+  const absolutePath = path.isAbsolute(filePath)
+    ? path.resolve(filePath)
+    : path.resolve(repoRoot, filePath);
+  const normalizedPath = normalizeSlashes(absolutePath);
+
+  if (normalizedPath.startsWith(`${normalizedRoot}/`)) {
+    return normalizedPath.slice(normalizedRoot.length + 1);
+  }
+
+  return normalizeSlashes(filePath);
+}
+
+function isSourceFile(filePath: string): boolean {
+  return (
+    (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) &&
+    !filePath.endsWith('.d.ts') &&
+    !filePath.endsWith('.test.ts') &&
+    !filePath.endsWith('.test.tsx') &&
+    !filePath.endsWith('.spec.ts') &&
+    !filePath.endsWith('.spec.tsx') &&
+    !filePath.endsWith('.stories.tsx')
+  );
+}
+
+function isReportSourceFile(report: CoverageReportConfig, filePath: string): boolean {
+  return (
+    isSourceFile(filePath) &&
+    filePath.startsWith(report.sourcePrefix) &&
+    !report.excludeFiles.includes(filePath) &&
+    !report.excludePrefixes.some((prefix) => filePath.startsWith(prefix))
+  );
+}
+
+function isCriticalPath(report: CoverageReportConfig, filePath: string): boolean {
+  return (
+    report.criticalFiles.includes(filePath) ||
+    report.criticalPrefixes.some((prefix) => filePath.startsWith(prefix))
+  );
+}
+
+function pct(covered: number, total: number): number {
+  return total === 0 ? 100 : (covered / total) * 100;
+}
+
+export function summarizeFileCoverage(fileCoverage: FileCoverage): FileCoverageSummary {
+  const statementMap = fileCoverage.statementMap ?? {};
+  const statements = Object.keys(statementMap);
+  const coveredStatements = statements.filter((id) => (fileCoverage.s?.[id] ?? 0) > 0).length;
+
+  const functions = Object.keys(fileCoverage.fnMap ?? {});
+  const coveredFunctions = functions.filter((id) => (fileCoverage.f?.[id] ?? 0) > 0).length;
+
+  const branches = Object.keys(fileCoverage.branchMap ?? {});
+  const branchHits = branches.flatMap((id) => fileCoverage.b?.[id] ?? []);
+  const coveredBranches = branchHits.filter((hit) => hit > 0).length;
+
+  const lineCoverage = new Map<number, boolean>();
+  for (const statementId of statements) {
+    const line = statementMap[statementId]?.start.line;
+    if (typeof line !== 'number') {
+      continue;
+    }
+    lineCoverage.set(
+      line,
+      lineCoverage.get(line) === true || (fileCoverage.s?.[statementId] ?? 0) > 0,
+    );
+  }
+
+  const coveredLines = [...lineCoverage.values()].filter(Boolean).length;
+
+  return {
+    branches: {
+      covered: coveredBranches,
+      total: branchHits.length,
+      pct: pct(coveredBranches, branchHits.length),
+    },
+    functions: {
+      covered: coveredFunctions,
+      total: functions.length,
+      pct: pct(coveredFunctions, functions.length),
+    },
+    lines: {
+      covered: coveredLines,
+      total: lineCoverage.size,
+      pct: pct(coveredLines, lineCoverage.size),
+    },
+    statements: {
+      covered: coveredStatements,
+      total: statements.length,
+      pct: pct(coveredStatements, statements.length),
+    },
+  };
+}
+
+function formatPct(value: number): string {
+  return value.toFixed(2);
+}
+
+function belowThresholds(summary: FileCoverageSummary, thresholds: CoverageThresholds): string[] {
+  const failures: string[] = [];
+
+  for (const metric of Object.keys(thresholds) as (keyof CoverageThresholds)[]) {
+    if (summary[metric].pct < thresholds[metric]) {
+      failures.push(
+        `${metric} ${formatPct(summary[metric].pct)}% < ${thresholds[metric]}% ` +
+          `(${summary[metric].covered}/${summary[metric].total})`,
+      );
+    }
+  }
+
+  return failures;
+}
+
+export function evaluateCoverageRatchets({
+  changedFiles,
+  coverageMap,
+  repoRoot,
+  report,
+  thresholds = DEFAULT_COVERAGE_THRESHOLDS,
+}: {
+  changedFiles: ChangedFile[];
+  coverageMap: CoverageMap;
+  repoRoot: string;
+  report: CoverageReportConfig;
+  thresholds?: CoverageThresholds;
+}): CoverageRatchetResult {
+  const coverageByFile = new Map<string, FileCoverage>();
+
+  for (const [coveragePath, fileCoverage] of Object.entries(coverageMap)) {
+    coverageByFile.set(
+      normalizeCoveragePath(fileCoverage.path ?? coveragePath, repoRoot),
+      fileCoverage,
+    );
+  }
+
+  const checkedFiles: CheckedFile[] = [];
+  const failures: CoverageFailure[] = [];
+  const skippedFiles: string[] = [];
+
+  for (const changedFile of changedFiles) {
+    const changedPath = normalizeCoveragePath(changedFile.path, repoRoot);
+    if (!isReportSourceFile(report, changedPath)) {
+      continue;
+    }
+
+    const reason =
+      changedFile.status === 'A'
+        ? 'new source file'
+        : isCriticalPath(report, changedPath)
+          ? 'critical path'
+          : undefined;
+
+    if (!reason) {
+      skippedFiles.push(changedPath);
+      continue;
+    }
+
+    const fileCoverage = coverageByFile.get(changedPath);
+    if (!fileCoverage) {
+      failures.push({
+        file: changedPath,
+        reason,
+        message: `No coverage entry found for ${changedPath}`,
+      });
+      continue;
+    }
+
+    const summary = summarizeFileCoverage(fileCoverage);
+    if (summary.statements.total === 0) {
+      skippedFiles.push(changedPath);
+      continue;
+    }
+
+    checkedFiles.push({ file: changedPath, reason, summary });
+
+    const coverageFailures = belowThresholds(summary, thresholds);
+    if (coverageFailures.length > 0) {
+      failures.push({
+        file: changedPath,
+        reason,
+        message: coverageFailures.join(', '),
+      });
+    }
+  }
+
+  return { checkedFiles, failures, skippedFiles };
+}
+
+function readCoverageMap(coverageFile: string): CoverageMap {
+  return JSON.parse(fs.readFileSync(coverageFile, 'utf8')) as CoverageMap;
+}
+
+function readFlagValue(argv: string[], index: number, flag: string): string {
+  const value = argv[index + 1];
+  if (!value || value.startsWith('--')) {
+    throw new Error(`Missing value for ${flag}`);
+  }
+  return value;
+}
+
+function parseArgs(argv: string[]): CliOptions {
+  const options: CliOptions = { reports: [] };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === '--base') {
+      options.baseRef = readFlagValue(argv, i, arg);
+      i += 1;
+    } else if (arg === '--report') {
+      options.reports.push(readFlagValue(argv, i, arg));
+      i += 1;
+    } else if (arg === '--help' || arg === '-h') {
+      console.log(
+        'Usage: tsx scripts/checkCoverageRatchets.ts [--base <ref>] [--report backend|frontend]',
+      );
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+
+  return options;
+}
+
+export function runCoverageRatchetCli(argv: string[], cwd = process.cwd()): number {
+  const options = parseArgs(argv);
+  const reportNames = new Set(COVERAGE_RATCHET_REPORTS.map((report) => report.name));
+  const unknownReports = options.reports.filter((report) => !reportNames.has(report));
+  if (unknownReports.length > 0) {
+    throw new Error(`Unknown coverage report(s): ${unknownReports.join(', ')}`);
+  }
+
+  const selectedReports =
+    options.reports.length > 0
+      ? COVERAGE_RATCHET_REPORTS.filter((report) => options.reports.includes(report.name))
+      : COVERAGE_RATCHET_REPORTS;
+
+  let failureCount = 0;
+  const availableReports: CoverageReportConfig[] = [];
+
+  for (const report of selectedReports) {
+    const coverageFile = path.resolve(cwd, report.coverageFile);
+    if (!fs.existsSync(coverageFile)) {
+      const message = `[coverage-ratchet] ${report.name}: missing ${report.coverageFile}`;
+      if (options.reports.length > 0) {
+        console.error(message);
+        failureCount += 1;
+      } else {
+        console.log(`${message} (skipped)`);
+      }
+      continue;
+    }
+
+    availableReports.push(report);
+  }
+
+  if (failureCount > 0 || availableReports.length === 0) {
+    return failureCount === 0 ? 0 : 1;
+  }
+
+  const changedFiles = getChangedFiles(cwd, options.baseRef);
+
+  for (const report of availableReports) {
+    const coverageFile = path.resolve(cwd, report.coverageFile);
+    const result = evaluateCoverageRatchets({
+      changedFiles,
+      coverageMap: readCoverageMap(coverageFile),
+      repoRoot: cwd,
+      report,
+    });
+
+    if (result.checkedFiles.length === 0) {
+      console.log(
+        `[coverage-ratchet] ${report.name}: no new or critical changed source files to check`,
+      );
+    } else {
+      console.log(
+        `[coverage-ratchet] ${report.name}: checked ${result.checkedFiles.length} file(s)`,
+      );
+    }
+
+    for (const failure of result.failures) {
+      failureCount += 1;
+      console.error(`[coverage-ratchet] ${failure.file} (${failure.reason}): ${failure.message}`);
+    }
+  }
+
+  return failureCount === 0 ? 0 : 1;
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  try {
+    process.exitCode = runCoverageRatchetCli(process.argv.slice(2));
+  } catch (error) {
+    console.error(`[coverage-ratchet] ${error instanceof Error ? error.message : String(error)}`);
+    process.exitCode = 1;
+  }
+}

scripts/checkTypeScriptCoverage.ts

@@ -0,0 +1,111 @@
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import ts from 'typescript';
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(scriptDir, '..');
+const rootOwnedPrefixes = ['src/', 'test/', 'scripts/'];
+const externalProjectPrefixes = ['src/app/', 'test/code-scan-action/'];
+
+function normalizePath(filePath: string): string {
+  return filePath.split(path.sep).join('/');
+}
+
+function isTypeScriptFile(filePath: string): boolean {
+  return (
+    filePath.endsWith('.ts') ||
+    filePath.endsWith('.tsx') ||
+    filePath.endsWith('.mts') ||
+    filePath.endsWith('.cts')
+  );
+}
+
+function hasPrefix(filePath: string, prefixes: string[]): boolean {
+  return prefixes.some((prefix) => filePath.startsWith(prefix));
+}
+
+function isRootOwnedTypeScriptFile(filePath: string): boolean {
+  return !filePath.includes('/') || hasPrefix(filePath, rootOwnedPrefixes);
+}
+
+export function getTrackedTypeScriptFiles(): string[] {
+  return execFileSync('git', ['ls-files'], {
+    cwd: repoRoot,
+    encoding: 'utf8',
+  })
+    .split('\n')
+    .map((filePath) => filePath.trim())
+    .filter(Boolean)
+    .map(normalizePath)
+    .filter(
+      (filePath) =>
+        isTypeScriptFile(filePath) &&
+        isRootOwnedTypeScriptFile(filePath) &&
+        !hasPrefix(filePath, externalProjectPrefixes),
+    )
+    .sort();
+}
+
+export function getRootProjectFiles(): Set<string> {
+  const configPath = ts.findConfigFile(repoRoot, ts.sys.fileExists, 'tsconfig.json');
+  if (!configPath) {
+    throw new Error('Could not find root tsconfig.json');
+  }
+
+  const configFile = ts.readConfigFile(configPath, ts.sys.readFile);
+  if (configFile.error) {
+    throw new Error(ts.flattenDiagnosticMessageText(configFile.error.messageText, '\n'));
+  }
+
+  const parsedConfig = ts.parseJsonConfigFileContent(
+    configFile.config,
+    ts.sys,
+    path.dirname(configPath),
+  );
+  if (parsedConfig.errors.length > 0) {
+    const message = parsedConfig.errors
+      .map((error) => ts.flattenDiagnosticMessageText(error.messageText, '\n'))
+      .join('\n');
+    throw new Error(message);
+  }
+
+  return new Set(
+    parsedConfig.fileNames.map((filePath) => normalizePath(path.relative(repoRoot, filePath))),
+  );
+}
+
+export function findMissingRootTypeScriptFiles(): string[] {
+  const projectFiles = getRootProjectFiles();
+  return getTrackedTypeScriptFiles().filter((filePath) => !projectFiles.has(filePath));
+}
+
+export function runTypeScriptCoverageCheck(): number {
+  const missingFiles = findMissingRootTypeScriptFiles();
+
+  if (missingFiles.length === 0) {
+    return 0;
+  }
+
+  console.error('Root tsconfig.json is not type-checking these tracked TypeScript files:');
+  for (const filePath of missingFiles) {
+    console.error(`- ${filePath}`);
+  }
+  console.error(
+    'Add them to the root project, or add the owning subtree to externalProjectPrefixes with a separate typecheck.',
+  );
+  return 1;
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  try {
+    process.exitCode = runTypeScriptCoverageCheck();
+  } catch (error) {
+    console.error(
+      `Failed to verify root TypeScript coverage: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    process.exitCode = 1;
+  }
+}

scripts/generate-blog-image.cjs

@@ -0,0 +1,164 @@
+#!/usr/bin/env node
+const fs = require('fs/promises');
+const path = require('path');
+const https = require('https');
+// Load environment variables from .env file
+require('dotenv').config({ path: path.join(__dirname, '..', '.env'), quiet: true });
+// Get OpenAI API key from environment
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+if (!OPENAI_API_KEY) {
+  console.error('Please set the OPENAI_API_KEY environment variable');
+  process.exit(1);
+}
+// Parse command line arguments
+const args = process.argv.slice(2);
+let filename = 'blog-image.png';
+let prompt = '';
+let size = '1024x1024';
+let outputDir = 'blog';
+// Parse arguments
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--filename' || args[i] === '-f') {
+    filename = args[i + 1];
+    i++;
+  } else if (args[i] === '--prompt' || args[i] === '-p') {
+    prompt = args[i + 1];
+    i++;
+  } else if (args[i] === '--size' || args[i] === '-s') {
+    size = args[i + 1];
+    i++;
+  } else if (args[i] === '--output-dir' || args[i] === '-o') {
+    outputDir = args[i + 1];
+    i++;
+  } else if (args[i] === '--help' || args[i] === '-h') {
+    console.log(`
+Usage: node generate-blog-image.js [options]
+Options:
+  -f, --filename <name>    Output filename (default: blog-image.png)
+  -p, --prompt <text>      Image generation prompt
+  -s, --size <size>        Image size: 1024x1024, 1536x1024, 1024x1536, auto (default: 1024x1024)
+  -o, --output-dir <dir>   Output subdirectory under site/static/img/ (default: blog)
+  -h, --help              Show this help message
+Examples:
+  node generate-blog-image.js -f system-cards-hero.png -p "A red panda reading documentation"
+  node generate-blog-image.js --filename hero.jpg --prompt "Conference banner" --size 1536x1024 --output-dir events
+`);
+    process.exit(0);
+  }
+}
+// Example prompt for system cards blog post:
+/*
+const systemCardsPrompt = `Create a professional hero image for a blog post about LLM System Cards and AI Safety Documentation. 
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in a cybersecurity/tech setting
+- The red panda should be examining or holding system documentation, security reports, or technical papers
+- Include visual elements that suggest security, transparency, and documentation (like shields, locks, checklists, or documents)
+- Modern tech aesthetic with a color palette that includes: deep purples, teals, and orange accents (matching Promptfoo's brand)
+- The style should be professional but approachable, similar to tech blog illustrations
+- Background could include subtle circuit patterns, document icons, or security symbols
+- The red panda should look intelligent and focused, perhaps wearing glasses or holding a magnifying glass
+- Overall mood: trustworthy, technical, but friendly
+Style: Modern tech illustration, clean lines, professional but approachable, suitable for a security-focused blog post`;
+*/
+if (!prompt) {
+  console.error('Please provide a prompt using --prompt or -p');
+  console.log('Use --help for usage information');
+  process.exit(1);
+}
+async function generateImage() {
+  const data = JSON.stringify({
+    model: 'gpt-image-1',
+    prompt: prompt,
+    size: size,
+    quality: 'high',
+    n: 1,
+  });
+  const options = {
+    hostname: 'api.openai.com',
+    port: 443,
+    path: '/v1/images/generations',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${OPENAI_API_KEY}`,
+      'Content-Length': data.length,
+    },
+  };
+  return new Promise((resolve, reject) => {
+    const req = https.request(options, (res) => {
+      let responseBody = '';
+      res.on('data', (chunk) => {
+        responseBody += chunk;
+      });
+      res.on('end', () => {
+        try {
+          const response = JSON.parse(responseBody);
+          if (response.error) {
+            reject(new Error(response.error.message));
+          } else {
+            resolve(response.data[0]);
+          }
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    req.on('error', (error) => {
+      reject(error);
+    });
+    req.write(data);
+    req.end();
+  });
+}
+async function downloadImage(url, filepath) {
+  const buffer = await new Promise((resolve, reject) => {
+    https
+      .get(url, (response) => {
+        if (response.statusCode && response.statusCode >= 400) {
+          response.resume();
+          reject(new Error(`Failed to download image: ${response.statusCode}`));
+          return;
+        }
+        const chunks = [];
+        response.on('data', (chunk) => {
+          chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        });
+        response.on('end', () => {
+          resolve(Buffer.concat(chunks));
+        });
+        response.on('error', reject);
+      })
+      .on('error', reject);
+  });
+  try {
+    await fs.writeFile(filepath, buffer);
+  } catch (error) {
+    await fs.unlink(filepath).catch(() => {}); // Delete the file on error
+    throw error;
+  }
+}
+async function main() {
+  try {
+    console.log('Generating image with gpt-image-1...');
+    const imageData = await generateImage();
+    if (imageData.b64_json) {
+      // If we get base64 data, save it directly
+      const outputPath = path.join(__dirname, '..', 'site', 'static', 'img', outputDir, filename);
+      const buffer = Buffer.from(imageData.b64_json, 'base64');
+      await fs.writeFile(outputPath, buffer);
+      console.log(`Image saved to: ${outputPath}`);
+    } else if (imageData.url) {
+      // If we get a URL, download the image
+      const outputPath = path.join(__dirname, '..', 'site', 'static', 'img', outputDir, filename);
+      await downloadImage(imageData.url, outputPath);
+      console.log(`Image downloaded and saved to: ${outputPath}`);
+    } else {
+      console.error('Unexpected response format:', imageData);
+    }
+  } catch (error) {
+    console.error('Error generating image:', error);
+    process.exit(1);
+  }
+}
+// biome-ignore lint/nursery/noFloatingPromises: convert this to ESM
+main();

scripts/generate-event-images.cjs

@@ -0,0 +1,551 @@
+#!/usr/bin/env node
+const fs = require('fs/promises');
+const path = require('path');
+const https = require('https');
+const { execSync } = require('child_process');
+
+// Load environment variables from .env file
+require('dotenv').config({ path: path.join(__dirname, '..', '.env'), quiet: true });
+
+// Get OpenAI API key from environment
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+if (!OPENAI_API_KEY) {
+  console.error('Please set the OPENAI_API_KEY environment variable');
+  process.exit(1);
+}
+
+const OUTPUT_DIR = path.join(__dirname, '..', 'site', 'static', 'img', 'events');
+
+// Event image configurations
+const events = [
+  {
+    filename: 'bsides-seattle-2025',
+    prompt: `Create a cute, friendly hero image for BSides Seattle 2025 security conference.
+The image should feature:
+- An adorable, expressive red panda character (the Promptfoo mascot) looking intelligent and friendly
+- The red panda should be cozy, holding a warm coffee cup, with a cheerful expression
+- Pacific Northwest setting: gentle rain, evergreen tree silhouettes, misty mountain backdrop
+- Subtle security touches: a small laptop or tablet with code visible
+- Modern tech illustration style with clean lines, professional but approachable
+- Color palette: forest green, warm coffee browns, soft gray tones, cream accents
+- Overall mood: trustworthy, technical, but friendly and welcoming
+Style: Modern tech illustration, clean lines, cute mascot character, cozy PNW coffee shop vibes, professional but approachable like a friendly tech blog`,
+  },
+  {
+    filename: 'ai-security-summit-2025',
+    prompt: `Create a clean, modern flat graphic illustration for AI Security Summit 2025 in San Francisco.
+
+STYLE (match RSA Conference and Black Hat poster style):
+- Clean, polished flat illustration - NOT painterly, NOT watercolor
+- Smooth gradient background filling entire image
+- Limited color palette: deep PURPLE (#7C3AED) and BLUE (#3B82F6) gradients, with pink/magenta accents
+- Clean digital look, professional and sleek
+
+BACKGROUND (must fill entire image):
+- Deep purple to blue gradient filling the entire background
+- San Francisco skyline silhouette with Golden Gate Bridge in darker purple shade
+- Floating neural network nodes and connection lines as subtle decorative elements
+- Abstract AI brain or circuit pattern overlay
+
+RED PANDA CHARACTER:
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES
+- Small black nose
+- Friendly OPEN MOUTH smile showing warmth
+- Wearing cute round GLASSES (this is an AI summit - smart look)
+- Purple or blue blazer/jacket - professional but techy
+- Standing at left, gesturing toward presentation screen
+
+COMPOSITION:
+- Red panda on left at a sleek podium
+- Large screen on right showing: AI brain icon, neural network diagram, security shield with AI chip
+- SOLID dark silhouettes of 3-4 attendees in foreground (clean shapes, not sketchy)
+- Laptop on podium
+- Everything should feel COMPLETE and POLISHED
+
+TEXT: "AI SECURITY SUMMIT 2025" at top, "San Francisco, CA" below
+
+Style: Premium tech conference, clean flat illustration with rich purple/blue filled background`,
+  },
+  {
+    filename: 'sector-2025',
+    prompt: `Create a professional hero image for SecTor 2025, Canada's largest IT security conference in Toronto.
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in a Canadian security setting
+- The red panda could be wearing a subtle Canadian element (red scarf) while examining security tools
+- Toronto skyline with CN Tower prominently featured in background
+- Subtle maple leaf patterns or motifs incorporated into the design
+- Include security elements: shields, locks, code snippets, Arsenal badge
+- Text overlay area should be considered (space for "SecTor 2025" and "Sep 30 - Oct 2, 2025 • Toronto, Canada")
+- Color palette: deep crimson red (#B22234), white, dark slate backgrounds, warm accents
+- Professional, internationally respected, but with Canadian warmth
+- Northern lights inspired subtle gradient in sky
+Style: Canadian tech aesthetic, professional security conference, warm and welcoming despite dark theme`,
+  },
+  {
+    filename: 'telecom-talks-2025',
+    prompt: `Create a clean, modern flat graphic illustration for Telecom Talks 2025 - a panel discussion with Swisscom.
+
+STYLE (match RSA Conference poster style):
+- Clean, polished flat illustration - NOT vintage, NOT heavy texture
+- Smooth subtle gradients in background
+- The red panda should have a subtle dark outline/stroke to pop from background
+- Limited color palette: Swiss-inspired - deep blue, white, red accents, with teal telecom accent
+- Clean digital look, minimal texture
+
+RED PANDA CHARACTER (critical details):
+- Round face with cream/white patches on cheeks and forehead
+- Small simple black dot eyes
+- Small black nose
+- Friendly OPEN MOUTH smile (like RSA image - warm, welcoming)
+- Dark reddish-brown and orange fur as flat color shapes
+- Subtle dark outline around the character
+- Wearing a casual polo shirt or blazer
+
+SCENE - PANEL DISCUSSION (this is key):
+- Show a PANEL setup: long table/desk with 3 panelists seated behind it
+- Red panda in the CENTER as a panelist, with 2 stylized human panelists on either side
+- Each panelist has a small microphone in front of them
+- Name placards on the table in front of each panelist
+- Behind them: large screen showing smartphone icons, 5G signal waves, network diagrams
+- Swiss Alps silhouette subtly visible through/behind the screen
+- Small Swiss cross (white cross on red) as a subtle badge or logo element on the screen
+- Audience silhouettes in foreground watching the panel
+
+TELECOM ELEMENTS:
+- Smartphone icons, cell signal bars, 5G text
+- Network tower silhouette
+- Signal wave arcs
+
+TEXT AREA: Space at top for "TELECOM TALKS 2025" and "Panel with Swisscom • SRI International"
+
+Style: Clean modern tech illustration like RSA Conference poster, polished and professional`,
+  },
+  {
+    filename: 'rsa-2025',
+    prompt: `Create a professional hero image for RSA Conference 2025 in San Francisco.
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in an enterprise security setting
+- The red panda should be presenting or demonstrating security tools to an audience
+- Corporate security elements: shields, locks, security badges, enterprise architecture diagrams
+- Moscone Center or San Francisco financial district skyline in background
+- Text overlay area should be considered (space for "RSA Conference 2025" and "April 28 - May 1, 2025 • San Francisco, CA")
+- Color palette: RSA blue (#0066CC), professional silver (#94A3B8), navy (#1e3a5f)
+- Professional, enterprise-grade, corporate but approachable atmosphere
+- Clean geometric patterns, subtle grid overlays
+Style: Enterprise security aesthetic, corporate tech conference, professional and trustworthy imagery`,
+  },
+  {
+    filename: 'bsides-sf-2025',
+    prompt: `Create a professional hero image for BSides San Francisco 2025.
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in a San Francisco community setting
+- The red panda should be engaging with other hackers/community members in discussion
+- Golden Gate Bridge silhouette prominently in background with SF fog rolling in
+- Grassroots hacker community elements: laptops, coffee cups, stickers, casual setting
+- Text overlay area should be considered (space for "BSides SF 2025" and "April 26-27, 2025 • San Francisco, CA")
+- Color palette: BSides orange (#F97316), purple (#8B5CF6), fog gray tones
+- Community-driven, accessible, inclusive, hacker culture atmosphere
+- Warm community vibes with SF fog aesthetic
+Style: Grassroots hacker conference, San Francisco community, warm and inclusive imagery`,
+  },
+  {
+    filename: 'rsa-2026',
+    prompt: `Create a clean, modern flat graphic illustration for RSA Conference 2026 - a "Save the Date" announcement. The ENTIRE IMAGE must have a BLUE background - NO WHITE.
+
+CRITICAL - BACKGROUND COLOR:
+- The ENTIRE background must be filled with RSA BLUE gradient - absolutely NO white or empty space
+- Rich blue (#0066CC) at top fading to deep navy (#1a365d) at bottom
+- This is the most important requirement - the background must be COMPLETELY FILLED with blue
+
+BACKGROUND ELEMENTS:
+- San Francisco skyline silhouette in darker blue shade (Golden Gate Bridge, Transamerica Pyramid, Moscone Center)
+- Subtle grid pattern or circuit lines overlay in lighter blue
+- Floating security shield icons scattered around
+
+RED PANDA CHARACTER:
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES
+- Small black nose
+- Friendly OPEN MOUTH smile - excited expression
+- Wearing blue blazer over white shirt
+- Standing at left side, pointing excitedly at a calendar/sign
+
+COMPOSITION:
+- Red panda on left pointing at a "SAVE THE DATE" sign/calendar in center-right
+- The sign shows "March 23-26, 2026" and "San Francisco"
+- 2-3 excited colleagues as solid darker blue silhouettes
+- Shield icons with checkmarks floating
+- Everything on the BLUE background - no white areas
+
+TEXT: "RSA CONFERENCE 2026" at top in white, "Save the Date" below
+
+Style: Must look like RSA 2025 poster - rich blue filled background, clean flat illustration, professional but exciting`,
+  },
+  {
+    filename: 'bsides-sf-2026',
+    prompt: `Create a professional hero image for BSides San Francisco 2026.
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in a futuristic San Francisco setting
+- The red panda should be inviting viewers to join, looking welcoming and excited
+- Golden Gate Bridge with futuristic overlay, SF fog with neon accents
+- Forward-looking community elements: save the date energy, excitement for upcoming event
+- Text overlay area should be considered (space for "BSides SF 2026" and "March 21-22, 2026 • San Francisco, CA")
+- Color palette: BSides orange (#F97316), neon purple (#8B5CF6), electric accents
+- Exciting, community anticipation, grassroots energy for upcoming event
+- Warm SF community vibes with futuristic accents
+Style: Future hacker community gathering, San Francisco, anticipation and welcoming imagery`,
+  },
+  // ===== 2026 EVENTS (NEW) =====
+  {
+    filename: 'bsides-seattle-2026',
+    prompt: `Create a clean, modern FLAT VECTOR illustration for BSides Seattle 2026 security conference. Match the exact style of BSides SF 2025 poster.
+
+CRITICAL STYLE REQUIREMENTS:
+- FLAT VECTOR illustration - NO 3D, NO realistic rendering, NO painterly texture
+- Simple geometric shapes with clean edges
+- Limited color palette with solid fills
+
+BACKGROUND (must fill entire image):
+- Forest green (#2D5A27) gradient background filling entire image
+- Seattle skyline silhouette in darker green (Space Needle prominently featured, pine trees)
+- Subtle rain drops or mist effect
+- Gentle mountains in distance
+
+RED PANDA CHARACTER (EXACT SPECIFICATIONS - CRITICAL):
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES (NOT realistic eyes - just black circles)
+- Small black nose
+- Friendly OPEN MOUTH smile showing warmth (like BSides SF 2025)
+- Dark reddish-brown and bright orange fur as FLAT color shapes (NO texture, NO gradients in fur)
+- Ears pointing up with cream inside
+- Wearing cozy dark green hoodie
+- Holding a coffee cup with steam rising
+- Sitting at table with open laptop
+
+COMPOSITION:
+- Red panda at center with laptop and coffee
+- 2-3 other hackers as FLAT stylized characters (not silhouettes) around table
+- Everyone has laptops with security stickers
+- Warm coffee shop community gathering feel
+- Pine tree and rain elements in background
+
+TEXT (REQUIRED): "BSIDES SEATTLE 2026" in bold at top, "February 27-28, 2026 • Redmond, WA" below
+
+Style: MUST match BSides SF 2025 - warm community gathering, flat vector, cozy PNW coffee vibes`,
+  },
+  {
+    filename: 'humanx-2026',
+    prompt: `Create a clean, modern FLAT VECTOR illustration for HumanX 2026 AI conference. Match the style of RSA Conference 2025 poster.
+
+CRITICAL STYLE REQUIREMENTS:
+- FLAT VECTOR illustration - NO 3D, NO realistic rendering, NO painterly texture
+- Simple geometric shapes with clean edges
+- Limited color palette with solid fills
+
+BACKGROUND (must fill entire image):
+- Deep purple (#7c3aed) to cyan (#06b6d4) gradient background filling entire image
+- San Francisco skyline silhouette in darker purple (Golden Gate Bridge, Transamerica Pyramid)
+- Floating neural network nodes and connection lines as subtle pattern
+- Abstract AI/brain circuit overlay
+
+RED PANDA CHARACTER (EXACT SPECIFICATIONS - CRITICAL):
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES (NOT realistic eyes - just black circles)
+- Small black nose
+- Friendly OPEN MOUTH smile showing warmth
+- Dark reddish-brown and bright orange fur as FLAT color shapes (NO texture, NO gradients in fur)
+- Ears pointing up with cream inside
+- Wearing purple blazer over white shirt - professional but modern
+- Standing at left, gesturing toward presentation screen
+
+COMPOSITION:
+- Red panda on left at sleek podium
+- Large presentation screen on right showing: AI brain icon, neural network diagram, "AI" text
+- SOLID dark purple silhouettes of 3-4 attendees in foreground
+- Clean, futuristic conference feel
+
+TEXT (REQUIRED): "HUMANX 2026" in bold white at top, "April 6-9, 2026 • San Francisco, CA" below
+
+Style: MUST match RSA 2025 - clean flat vector, professional conference, purple/cyan AI theme`,
+  },
+  {
+    filename: 'gartner-security-2026',
+    prompt: `A dark navy blue conference poster illustration. The entire image is filled with a rich navy blue (#1e3a5f) gradient background from edge to edge.
+
+Scene: A flat vector illustration showing a cute red panda mascot presenting at an enterprise security summit. The entire composition sits against the deep navy blue backdrop.
+
+Background: Deep navy blue gradient fills every pixel of the image. Washington DC skyline silhouette (Capitol dome, monuments) appears in slightly darker navy. Gold grid lines create subtle pattern overlay.
+
+Red panda character: Cute cartoon style with round face, cream cheek patches, simple black dot eyes, open mouth smile, flat orange-brown fur. Wearing navy blazer with gold tie. Standing at podium gesturing to screen.
+
+Composition: Red panda at podium on left. Large presentation screen on right showing risk charts, shield icons, governance flowchart. Dark navy silhouettes of 3-4 executives in foreground watching presentation.
+
+Text at top: "GARTNER SECURITY 2026" in white. Below: "June 1-3, 2026 • National Harbor, MD"
+
+Style: Clean flat vector like RSA Conference 2025 poster. Professional enterprise aesthetic. Navy blue background fills entire image.`,
+  },
+  {
+    filename: 'blackhat-2026',
+    prompt: `A dark charcoal gray conference poster illustration. The entire image is filled with a deep charcoal (#1a1a2e) to near-black gradient background from edge to edge.
+
+Scene: A flat vector illustration showing a cute red panda mascot at a security conference booth. The entire composition sits against the dark charcoal backdrop.
+
+Background: Dark charcoal gray gradient fills every pixel of the image. Las Vegas skyline silhouette (Stratosphere tower, casino hotels, desert mountains) appears in slightly lighter dark gray. Subtle circuit board pattern overlays the dark background.
+
+Red panda character: Cute cartoon style with round face, cream cheek patches, simple black dot eyes, open mouth smile, flat orange-brown fur. Wearing dark gray blazer over black shirt. Red conference lanyard with badge. Standing at booth on left, gesturing toward screen.
+
+Composition: Red panda at demo booth with laptop on table. Large presentation screen on right showing: red security shield with lock icon, red line graph trending up, identity credential icon. Dark gray silhouettes of 3-4 attendees in foreground watching demo.
+
+Text at top: "BLACK HAT USA 2026" in light gray. Below: "August 1-6, 2026 • Las Vegas, NV"
+
+Style: Clean flat vector exactly like Black Hat USA 2025 poster. Professional hacker conference aesthetic. Dark charcoal background fills entire image. Crimson red accent colors only.`,
+  },
+  {
+    filename: 'scaleup-ai-2025',
+    prompt: `Create a professional hero image for ScaleUp:AI 2025, an Insight Partners venture capital thought leadership series.
+The image should feature:
+- A cute red panda character (the Promptfoo mascot) in a sophisticated, professional setting
+- The red panda should appear confident and visionary, perhaps at a podium or in a modern office with city views
+- Venture capital / investment aesthetic: sleek modern architecture, glass buildings, upward growth charts
+- Abstract elements suggesting scale and growth: ascending lines, geometric shapes, network nodes
+- Text overlay area should be considered (space for "ScaleUp:AI 2025" and "Insight Partners")
+- Color palette: deep indigo (#4F46E5), professional slate (#475569), clean whites
+- Sophisticated, authoritative, thought leadership atmosphere
+- Clean geometric patterns, subtle gradient overlays
+Style: Venture capital thought leadership, professional media feature, sophisticated tech investment aesthetic`,
+  },
+  {
+    filename: 'defcon-2025',
+    prompt: `Create a clean, modern flat graphic illustration for DEF CON 33 (2025) hacker conference. The ENTIRE IMAGE must have a DARK background - NO WHITE.
+
+CRITICAL - BACKGROUND COLOR:
+- The ENTIRE background must be filled with DARK CHARCOAL to BLACK gradient - absolutely NO white or light areas
+- Dark charcoal (#1a1a2e) at top fading to near-black (#0d0d0d) at bottom
+- This is the most important requirement - the background must be COMPLETELY DARK like Black Hat poster
+
+BACKGROUND ELEMENTS:
+- Las Vegas skyline silhouette in slightly lighter dark gray (Stratosphere tower, casino hotels)
+- Green (#00FF41) circuit traces and network lines as pattern overlay
+- Matrix-style subtle code/text falling in background
+- Terminal window shapes floating
+
+RED PANDA CHARACTER:
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES
+- Small black nose
+- Friendly OPEN MOUTH smile - playful mischievous grin
+- Wearing dark gray HOODIE
+- Green DEF CON badge on lanyard
+- Sitting at center with laptop
+
+COMPOSITION:
+- Red panda at center-left at table with open laptop showing green terminal text
+- 2-3 hacker friends as stylized flat characters (not just silhouettes) sitting around with laptops
+- Everyone wearing hoodies, has badges
+- Skull stickers on laptops
+- SOLID dark silhouettes of crowd in far background
+- Warm community gathering feel despite dark colors
+
+TEXT: "DEF CON 33" in neon green at top, "August 7-10, 2025 • Las Vegas" below
+
+Style: Must look like Black Hat poster - dark filled background, clean flat illustration, hacker aesthetic with green accents`,
+  },
+  {
+    filename: 'blackhat-2025',
+    prompt: `Create a clean, modern flat graphic illustration for Black Hat USA 2025 in Las Vegas. Match the polished style of RSA Conference posters.
+
+BACKGROUND (critical - must fill entire image):
+- DARK gradient background filling the ENTIRE image - deep charcoal gray at top fading to dark gray
+- Las Vegas skyline silhouette across the bottom/middle in darker shade (Stratosphere tower, casino hotel shapes, desert mountains)
+- The background must be FILLED with color, not white/empty
+- Subtle grid lines or circuit pattern overlay in the dark background
+
+COLOR PALETTE (strict):
+- Background: dark charcoal grays and blacks
+- Accents: CRIMSON RED (#DC143C) for highlights and icons
+- White/light gray for text and secondary elements
+- The red panda's orange fur should POP against the dark background
+
+RED PANDA CHARACTER:
+- Round face with cream/white patches on cheeks and forehead
+- Small simple BLACK DOT EYES
+- Small black nose
+- Friendly OPEN MOUTH smile showing warmth
+- Dark reddish-brown and bright orange fur as flat color shapes
+- Wearing dark blazer over black shirt - very professional
+- Red lanyard with badge visible
+- Standing at left-center, gesturing toward a presentation screen
+
+COMPOSITION:
+- Red panda on left side at a sleek demo podium/booth
+- Large presentation SCREEN on the right showing: security shield icon, lock icon, threat graph with red data points, profile/identity card icon
+- SOLID dark silhouettes of 3-4 attendees in foreground watching (not sketchy outlines - clean solid shapes like RSA image)
+- Laptop on podium showing dashboard
+- Everything should feel COMPLETE and POLISHED, not sparse
+
+TEXT: "BLACK HAT USA 2025" at top, "August 2-7, 2025 • Las Vegas, NV" below
+
+Style: Premium, dark, professional - like a high-end security conference. Clean flat illustration with rich dark filled background like RSA poster style.`,
+  },
+];
+
+async function generateImage(prompt) {
+  const data = JSON.stringify({
+    model: 'gpt-image-1',
+    prompt: prompt,
+    size: '1536x1024', // Wider aspect ratio for hero images
+    quality: 'high',
+    n: 1,
+  });
+
+  const options = {
+    hostname: 'api.openai.com',
+    port: 443,
+    path: '/v1/images/generations',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${OPENAI_API_KEY}`,
+      'Content-Length': Buffer.byteLength(data),
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    const req = https.request(options, (res) => {
+      let responseBody = '';
+      res.on('data', (chunk) => {
+        responseBody += chunk;
+      });
+      res.on('end', () => {
+        try {
+          const response = JSON.parse(responseBody);
+          if (response.error) {
+            reject(new Error(response.error.message));
+          } else {
+            resolve(response.data[0]);
+          }
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    req.on('error', (error) => {
+      reject(error);
+    });
+    req.write(data);
+    req.end();
+  });
+}
+
+async function downloadImage(url, filepath) {
+  const buffer = await new Promise((resolve, reject) => {
+    https
+      .get(url, (response) => {
+        if (response.statusCode && response.statusCode >= 400) {
+          response.resume();
+          reject(new Error(`Failed to download image: ${response.statusCode}`));
+          return;
+        }
+        const chunks = [];
+        response.on('data', (chunk) => {
+          chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        });
+        response.on('end', () => {
+          resolve(Buffer.concat(chunks));
+        });
+        response.on('error', reject);
+      })
+      .on('error', reject);
+  });
+  try {
+    await fs.writeFile(filepath, buffer);
+  } catch (error) {
+    await fs.unlink(filepath).catch(() => {});
+    throw error;
+  }
+}
+
+async function processEvent(event) {
+  const pngPath = path.join(OUTPUT_DIR, `${event.filename}.png`);
+  const jpgPath = path.join(OUTPUT_DIR, `${event.filename}.jpg`);
+
+  console.log(`[${event.filename}] Generating image...`);
+
+  try {
+    const imageData = await generateImage(event.prompt);
+
+    if (imageData.b64_json) {
+      const buffer = Buffer.from(imageData.b64_json, 'base64');
+      await fs.writeFile(pngPath, buffer);
+      console.log(`[${event.filename}] PNG saved`);
+    } else if (imageData.url) {
+      await downloadImage(imageData.url, pngPath);
+      console.log(`[${event.filename}] PNG downloaded`);
+    }
+
+    // Convert PNG to JPEG using sips (macOS) or ImageMagick
+    try {
+      execSync(`sips -s format jpeg -s formatOptions 85 "${pngPath}" --out "${jpgPath}"`, {
+        stdio: 'pipe',
+      });
+      console.log(`[${event.filename}] Converted to JPEG`);
+
+      // Remove PNG after successful conversion
+      await fs.unlink(pngPath);
+      console.log(`[${event.filename}] Cleaned up PNG`);
+    } catch (_convertError) {
+      // Try ImageMagick as fallback
+      try {
+        execSync(`convert "${pngPath}" -quality 85 "${jpgPath}"`, { stdio: 'pipe' });
+        console.log(`[${event.filename}] Converted to JPEG (ImageMagick)`);
+        await fs.unlink(pngPath);
+      } catch {
+        console.log(`[${event.filename}] Could not convert to JPEG, keeping PNG`);
+      }
+    }
+
+    return { success: true, filename: event.filename };
+  } catch (error) {
+    console.error(`[${event.filename}] Error:`, error.message);
+    return { success: false, filename: event.filename, error: error.message };
+  }
+}
+
+async function main() {
+  // Parse command line for specific event
+  const args = process.argv.slice(2);
+  let eventsToProcess = events;
+
+  if (args.length > 0 && args[0] !== '--all') {
+    const eventName = args[0];
+    const found = events.find((e) => e.filename === eventName || e.filename.includes(eventName));
+    if (found) {
+      eventsToProcess = [found];
+    } else {
+      console.error(`Event not found: ${eventName}`);
+      console.log('Available events:', events.map((e) => e.filename).join(', '));
+      process.exit(1);
+    }
+  }
+
+  console.log(`Generating ${eventsToProcess.length} event image(s) in parallel...`);
+  console.log('Output directory:', OUTPUT_DIR);
+  console.log('');
+
+  // Run all in parallel
+  const results = await Promise.all(eventsToProcess.map(processEvent));
+
+  console.log('');
+  console.log('=== Results ===');
+  for (const result of results) {
+    if (result.success) {
+      console.log(`✓ ${result.filename}.jpg`);
+    } else {
+      console.log(`✗ ${result.filename}: ${result.error}`);
+    }
+  }
+}
+
+// biome-ignore lint/nursery/noFloatingPromises: should probably migrate this to ESM
+main();

scripts/generateCitation.ts

@@ -0,0 +1,121 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import * as yaml from 'js-yaml';
+import { loadYaml } from '../src/util/yamlLoad';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Represents the structure of package.json file
+ */
+interface PackageJson {
+  license: string;
+  version: string;
+  description: string;
+}
+
+/**
+ * Represents the structure of CITATION.cff file
+ */
+interface Citation {
+  'cff-version': string;
+  message: string;
+  authors: Array<{
+    'family-names': string;
+    'given-names': string;
+  }>;
+  title: string;
+  version: string;
+  'date-released': string;
+  url: string;
+  'repository-code': string;
+  license: string;
+  type: string;
+  description: string;
+  keywords: string[];
+}
+
+/**
+ * Creates a default Citation object with information from package.json
+ * @param packageJson - The parsed package.json file
+ * @returns A default Citation object
+ */
+const createDefaultCitation = (packageJson: PackageJson): Citation => ({
+  'cff-version': '1.2.0',
+  message: 'If you use this software, please cite it as below.',
+  authors: [
+    {
+      'family-names': 'Webster',
+      'given-names': 'Ian',
+    },
+  ],
+  title: 'promptfoo',
+  version: packageJson.version,
+  'date-released': new Date().toISOString().slice(0, 10),
+  url: 'https://promptfoo.dev',
+  'repository-code': 'https://github.com/promptfoo/promptfoo',
+  license: packageJson.license,
+  type: 'software',
+  description: packageJson.description,
+  keywords: ['llm', 'evaluation', 'evals', 'testing', 'prompt-engineering', 'red-team'],
+});
+
+/**
+ * Fetches the release date for a specific version from GitHub
+ * @param version The version to fetch the release date for
+ * @returns Promise<string> The release date in ISO format, or null if not found
+ */
+async function getReleaseDate(version: string): Promise<string | null> {
+  try {
+    // biome-ignore lint/style/noRestrictedGlobals: Standalone script, fetchWithProxy not needed
+    const response = await fetch(
+      `https://api.github.com/repos/promptfoo/promptfoo/releases/tags/${version}`,
+    );
+    if (!response.ok) {
+      if (response.status === 404) {
+        console.warn(`No release found for version ${version}`);
+        return null;
+      }
+      throw new Error(`GitHub API request failed: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return data.published_at ? new Date(data.published_at).toISOString().slice(0, 10) : null;
+  } catch (error) {
+    console.error(`Error fetching release date for version ${version}:`, error);
+    return null;
+  }
+}
+
+/**
+ * Updates the CITATION.cff file with the latest information from package.json and GitHub
+ * @throws {Error} If there's an issue reading or writing files
+ */
+export const updateCitation = async (): Promise<void> => {
+  const packageJsonPath: string = path.join(__dirname, '../package.json');
+  const citationPath: string = path.join(__dirname, '../CITATION.cff');
+
+  const packageJson: PackageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+
+  let citation: Citation;
+  try {
+    citation = loadYaml(await fs.readFile(citationPath, 'utf8')) as Citation;
+  } catch {
+    citation = createDefaultCitation(packageJson);
+  }
+  citation['version'] = packageJson.version;
+
+  const releaseDate = await getReleaseDate(packageJson.version);
+  citation['date-released'] = releaseDate || new Date().toISOString().slice(0, 10);
+
+  await fs.writeFile(citationPath, yaml.dump(citation, { lineWidth: -1 }));
+
+  console.log('CITATION.cff file has been updated.');
+};
+
+updateCitation().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/generateJsonSchema.ts

@@ -0,0 +1,157 @@
+import { type ZodType, z } from 'zod';
+import { UnifiedConfigSchema } from '../src/types';
+import { TRANSFORM_KEYS } from '../src/util/transform';
+import { StringOrFunctionSchema } from '../src/validators/shared';
+
+// NOTE: This script accesses Zod's internal _def property to extract the input schema
+// from pipe/transform wrappers. This is necessary because UnifiedConfigSchema uses
+// .pipe() for transforms, and we need the input schema for JSON Schema generation.
+// This may need updating if Zod's internal structure changes.
+
+// Recursively unwrap to get the base schema (unwrapping optional, nullable, pipe, transform)
+function getBaseSchema(schema: ZodType): ZodType {
+  const def = schema._def as { type?: string; in?: ZodType; innerType?: ZodType };
+
+  if (def?.type === 'optional' && def?.innerType) {
+    return getBaseSchema(def.innerType);
+  }
+  if (def?.type === 'nullable' && def?.innerType) {
+    return getBaseSchema(def.innerType);
+  }
+  if (def?.type === 'pipe' && def?.in) {
+    return getBaseSchema(def.in);
+  }
+  if (def?.type === 'transform' && def?.in) {
+    return getBaseSchema(def.in);
+  }
+
+  return schema;
+}
+
+// UnifiedConfigSchema is wrapped in a .pipe() transform, so generate JSON Schema from its input.
+const innerSchema = getBaseSchema(UnifiedConfigSchema);
+
+const transformSchemaKeys: Set<string> = new Set(TRANSFORM_KEYS);
+
+/**
+ * Strips every key from `target` and reassigns it to `{ type: 'string', description? }`,
+ * preserving an optional description so generated docs stay useful. Used by both the
+ * `override` hook (for inline `StringOrFunctionSchema` nodes) and the post-pass walker
+ * (for nodes Zod rewrites after `override` runs).
+ */
+function rewriteNodeToStringSchema(target: Record<string, unknown>): void {
+  const description = target.description;
+  for (const key of Object.keys(target)) {
+    delete target[key];
+  }
+  if (typeof description === 'string') {
+    target.description = description;
+  }
+  target.type = 'string';
+}
+
+function forceStringTransformSchemas(node: unknown): void {
+  if (!node || typeof node !== 'object') {
+    return;
+  }
+
+  if (Array.isArray(node)) {
+    for (const item of node) {
+      forceStringTransformSchemas(item);
+    }
+    return;
+  }
+
+  const schemaObject = node as Record<string, unknown>;
+
+  for (const [key, value] of Object.entries(schemaObject)) {
+    if (
+      transformSchemaKeys.has(key) &&
+      value &&
+      typeof value === 'object' &&
+      !Array.isArray(value)
+    ) {
+      rewriteNodeToStringSchema(value as Record<string, unknown>);
+      // Recursing into the just-rewritten `{type, description}` stub is pointless
+      // and could accidentally re-match if a future rewrite leaves nested junk.
+      continue;
+    }
+
+    forceStringTransformSchemas(value);
+  }
+}
+
+// Options for nested toJSONSchema calls (without reused:'ref' to avoid orphaned definitions)
+const nestedOptions = {
+  target: 'draft-07' as const,
+  unrepresentable: 'any' as const,
+};
+
+// Generate JSON Schema using Zod v4's native toJSONSchema
+// - target: 'draft-07' for compatibility with most tooling
+// - unrepresentable: 'any' converts z.custom() functions to {} (accepts any)
+// - reused: 'ref' extracts duplicate schemas to definitions, reducing size significantly
+// - override: handles nested pipe/transform schemas that Zod doesn't convert automatically
+const schemaContent = z.toJSONSchema(innerSchema, {
+  ...nestedOptions,
+  reused: 'ref' as const,
+  // biome-ignore lint/suspicious/noExplicitAny: Zod v4 internal types are not publicly exported
+  override: (ctx: any) => {
+    const zodSchema = ctx.zodSchema as ZodType;
+    const def = zodSchema._def as { type?: string; in?: ZodType; innerType?: ZodType };
+    const baseSchema = getBaseSchema(zodSchema);
+
+    // Config files can only represent string transforms. Preserve runtime support for function
+    // transforms in the Zod schema, but keep generated JSON Schema string-only for editor/Ajv use.
+    if (baseSchema === StringOrFunctionSchema) {
+      rewriteNodeToStringSchema(ctx.jsonSchema as Record<string, unknown>);
+      return;
+    }
+
+    // Handle optional/nullable wrapping a pipe/transform
+    if ((def?.type === 'optional' || def?.type === 'nullable') && def?.innerType) {
+      const innerDef = def.innerType._def as { type?: string };
+      if (innerDef?.type === 'pipe' || innerDef?.type === 'transform') {
+        if (Object.keys(ctx.jsonSchema).length === 0) {
+          const result = z.toJSONSchema(baseSchema, nestedOptions);
+          const { $schema: _, ...rest } = result as Record<string, unknown>;
+          Object.assign(ctx.jsonSchema, rest);
+        }
+      }
+    }
+
+    // Handle pipe/transform directly
+    if ((def?.type === 'pipe' || def?.type === 'transform') && def?.in) {
+      if (Object.keys(ctx.jsonSchema).length === 0) {
+        const result = z.toJSONSchema(baseSchema, nestedOptions);
+        const { $schema: _, ...rest } = result as Record<string, unknown>;
+        Object.assign(ctx.jsonSchema, rest);
+      }
+    }
+  },
+});
+
+// Extract the main schema parts
+const {
+  $schema: _,
+  definitions: zodDefinitions,
+  ...mainSchema
+} = schemaContent as Record<string, unknown>;
+
+// Build final schema with proper structure and metadata
+const jsonSchema = {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  $id: 'https://promptfoo.dev/config-schema.json',
+  title: 'Promptfoo Configuration Schema',
+  $ref: '#/definitions/PromptfooConfigSchema',
+  definitions: {
+    PromptfooConfigSchema: mainSchema,
+    ...(zodDefinitions as Record<string, unknown>),
+  },
+};
+
+// Zod may rewrite reused StringOrFunctionSchema nodes after `override` runs. Do a final pass to keep
+// transform-like fields string-only in JSON Schema while runtime Zod still accepts functions.
+forceStringTransformSchemas(jsonSchema);
+
+console.log(JSON.stringify(jsonSchema, null, 2));

scripts/generateOpenApi.ts

@@ -0,0 +1,46 @@
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+
+import { createServerOpenApiDocument } from '../src/openapi/server';
+
+const DEFAULT_OUTPUT_PATH = 'site/static/openapi.json';
+const require = createRequire(import.meta.url);
+const args = process.argv.slice(2);
+const checkOnly = args.includes('--check');
+const outputPath = args.find((arg) => arg !== '--check') ?? DEFAULT_OUTPUT_PATH;
+const absoluteOutputPath = path.resolve(outputPath);
+const biomePath = require.resolve('@biomejs/biome/bin/biome');
+const unformattedSpec = `${JSON.stringify(createServerOpenApiDocument(), null, 2)}\n`;
+const renderedSpec = execFileSync(
+  process.execPath,
+  [biomePath, 'format', '--stdin-file-path', absoluteOutputPath],
+  {
+    encoding: 'utf8',
+    input: unformattedSpec,
+  },
+);
+
+if (checkOnly) {
+  if (!fs.existsSync(absoluteOutputPath)) {
+    console.error(`OpenAPI spec is missing at ${absoluteOutputPath}`);
+    console.error(`Run "npm run openapi:generate" and commit the generated file.`);
+    process.exit(1);
+  }
+
+  const currentSpec = fs.readFileSync(absoluteOutputPath, 'utf8');
+  if (currentSpec !== renderedSpec) {
+    console.error(`OpenAPI spec is stale at ${absoluteOutputPath}`);
+    console.error(`Run "npm run openapi:generate" and commit the generated file.`);
+    process.exit(1);
+  }
+
+  console.log(`OpenAPI spec is current at ${absoluteOutputPath}`);
+  process.exit(0);
+}
+
+fs.mkdirSync(path.dirname(absoluteOutputPath), { recursive: true });
+fs.writeFileSync(absoluteOutputPath, renderedSpec, 'utf8');
+
+console.log(`Wrote OpenAPI spec to ${absoluteOutputPath}`);

scripts/generate_modelaudit_schema.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Generate checked-in JSON Schema artifacts for the current ModelAudit release.
+
+Use Python 3.10-3.13 and install the pinned release before running this script:
+
+    npm ci
+    python3 -m pip install --no-deps -r scripts/modelaudit_schema_requirements.txt
+    npm run modelAuditSchema:generate
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as distribution_version
+from pathlib import Path
+from typing import Any
+
+SCHEMA_FILENAME = "modelaudit-scan-result.schema.json"
+EXAMPLE_FILENAME = "modelaudit-scan-result.example.json"
+SCHEMA_DRAFT_URI = "https://json-schema.org/draft/2020-12/schema"
+HAS_ERRORS_DESCRIPTION = "Whether operational errors occurred during scanning"
+REPO_ROOT = Path(__file__).resolve().parents[1]
+REQUIREMENTS_PATH = REPO_ROOT / "scripts" / "modelaudit_schema_requirements.txt"
+MIN_PYTHON_VERSION = (3, 10)
+MAX_PYTHON_VERSION = (3, 14)
+SCHEMA_PATH = REPO_ROOT / "site" / "static" / "schemas" / "modelaudit" / SCHEMA_FILENAME
+EXAMPLE_PATH = (
+    REPO_ROOT / "site" / "static" / "examples" / "modelaudit" / EXAMPLE_FILENAME
+)
+
+
+def build_schema(result_model: Any) -> dict[str, Any]:
+    """Build the public schema with stable metadata and corrected field semantics."""
+    schema = result_model.model_json_schema()
+    has_errors = schema.get("properties", {}).get("has_errors")
+    if not isinstance(has_errors, dict):
+        raise ValueError("Generated ModelAudit schema is missing properties.has_errors")
+    has_errors["description"] = HAS_ERRORS_DESCRIPTION
+
+    schema.pop("$schema", None)
+    schema.pop("$id", None)
+    return {
+        "$schema": SCHEMA_DRAFT_URI,
+        "$id": f"https://www.promptfoo.dev/schemas/modelaudit/{SCHEMA_FILENAME}",
+        **schema,
+    }
+
+
+def expected_modelaudit_version() -> str:
+    """Read the pinned ModelAudit version from the schema requirements file."""
+    for line in REQUIREMENTS_PATH.read_text(encoding="utf-8").splitlines():
+        requirement = line.strip()
+        if not requirement or requirement.startswith("#"):
+            continue
+        if requirement.startswith("modelaudit=="):
+            return requirement.removeprefix("modelaudit==").split()[0]
+    raise RuntimeError(
+        f"Missing modelaudit pin in {REQUIREMENTS_PATH.relative_to(REPO_ROOT)}"
+    )
+
+
+def install_hint() -> str:
+    """Return the local install command for this generator's Python contract."""
+    return (
+        "python3 -m pip install --no-deps -r "
+        f"{REQUIREMENTS_PATH.relative_to(REPO_ROOT)}"
+    )
+
+
+def validate_python_version() -> None:
+    """Fail with a direct message before importing an unsupported ModelAudit runtime."""
+    version_info = sys.version_info
+    if not (MIN_PYTHON_VERSION <= version_info[:2] < MAX_PYTHON_VERSION):
+        version = f"{version_info.major}.{version_info.minor}"
+        raise RuntimeError(
+            "ModelAudit schema generation requires Python 3.10-3.13; "
+            f"found Python {version}. Use a supported Python and run: {install_hint()}"
+        )
+
+
+def load_result_model(expected_version: str) -> Any:
+    """Load the schema source from the expected installed ModelAudit release."""
+    try:
+        installed_version = distribution_version("modelaudit")
+    except PackageNotFoundError as exc:
+        raise RuntimeError(
+            f"Expected modelaudit=={expected_version}. Run: {install_hint()}"
+        ) from exc
+    if installed_version != expected_version:
+        raise RuntimeError(
+            f"Expected modelaudit=={expected_version}, found {installed_version}. "
+            f"Run: {install_hint()}"
+        )
+
+    from modelaudit.models import ModelAuditResultModel
+
+    return ModelAuditResultModel
+
+
+def render_json(data: Any) -> str:
+    """Render deterministic JSON before repository formatters normalize style."""
+    return f"{json.dumps(data, indent=2)}\n"
+
+
+def write_text_atomic(path: Path, contents: str) -> None:
+    """Write a file without leaving a partially-written artifact on failure."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        dir=path.parent,
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as tmp_file:
+            tmp_file.write(contents)
+        os.replace(tmp_name, path)
+    except Exception:
+        Path(tmp_name).unlink(missing_ok=True)
+        raise
+
+
+def main() -> None:
+    """Generate the ModelAudit schema and example artifacts."""
+    validate_python_version()
+    result_model = load_result_model(expected_modelaudit_version())
+
+    example = json.loads(EXAMPLE_PATH.read_text(encoding="utf-8"))
+    result_model.model_validate(example)
+
+    write_text_atomic(SCHEMA_PATH, render_json(build_schema(result_model)))
+    write_text_atomic(EXAMPLE_PATH, render_json(example))
+    print(f"Wrote {SCHEMA_PATH.relative_to(REPO_ROOT)}")
+    print(f"Wrote {EXAMPLE_PATH.relative_to(REPO_ROOT)}")
+
+
+if __name__ == "__main__":
+    main()

scripts/modelaudit_schema_requirements.txt

@@ -0,0 +1,14 @@
+# Keep this file in sync with the current ModelAudit JSON output contract.
+# Install with --no-deps so CI uses only the transitive pins listed here.
+# Renovate updates `modelaudit` here; after a bump, run
+# `npm run modelAuditSchema:generate` and re-pin the transitive deps below to
+# whatever the new modelaudit pulls in. pydantic, pydantic-core and the typing
+# helpers are a matched set: pydantic enforces an exact pydantic-core pair at
+# import time, so they must move together. Renovate is configured to skip
+# standalone updates of these transitive pins to avoid incompatible pairs.
+modelaudit==0.2.49
+pydantic==2.13.4
+pydantic-core==2.46.4
+annotated-types==0.7.0
+typing-extensions==4.15.0
+typing-inspection==0.4.2

scripts/postbuild.ts

@@ -0,0 +1,320 @@
+/**
+ * Post-build script that copies non-TypeScript assets to the dist directory.
+ *
+ * This script runs automatically after the TypeScript build (tsdown) completes.
+ * It handles:
+ * - HTML template files (all *.html in src/)
+ * - Python/Go/Ruby wrapper scripts for custom providers
+ * - Drizzle ORM migration files
+ * - ESM package.json marker
+ * - CLI executable permissions
+ *
+ * @module scripts/postbuild
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const ROOT = path.join(__dirname, '..');
+const DIST = path.join(ROOT, 'dist');
+const SRC = path.join(ROOT, 'src');
+
+/**
+ * Wrapper types supported by the build.
+ * IMPORTANT: Must match WrapperType in src/esm.ts (used by getWrapperDir()).
+ * If you add a new wrapper type, update both files.
+ */
+const WRAPPER_TYPES = ['python', 'ruby', 'golang'] as const;
+type WrapperType = (typeof WRAPPER_TYPES)[number];
+
+/**
+ * Wrapper files for each language type.
+ * Maps wrapper type to the list of files that should be copied.
+ */
+const WRAPPER_FILES: Record<WrapperType, string[]> = {
+  python: ['wrapper.py', 'persistent_wrapper.py'],
+  ruby: ['wrapper.rb'],
+  golang: ['wrapper.go'],
+};
+
+/**
+ * Files/patterns to exclude when copying the drizzle directory.
+ */
+const DRIZZLE_EXCLUDE_PATTERNS = ['.md', 'CLAUDE', 'AGENTS'];
+
+export function shouldCopyDrizzlePath(src: string): boolean {
+  const basename = path.basename(src);
+  return !DRIZZLE_EXCLUDE_PATTERNS.some(
+    (pattern) => basename.includes(pattern) || basename.endsWith(pattern),
+  );
+}
+
+/**
+ * Critical build outputs that must exist for the build to be valid.
+ */
+const REQUIRED_BUILD_OUTPUTS = [
+  'dist/src/entrypoint.js', // CLI entry (Node version check wrapper)
+  'dist/src/main.js', // CLI main module
+  'dist/src/contracts.js', // ESM contracts subpath
+  'dist/src/contracts.cjs', // CJS contracts subpath
+  'dist/src/contracts.d.ts', // ESM contracts declarations
+  'dist/src/contracts.d.cts', // CJS contracts declarations
+  'dist/src/index.js', // ESM library entry
+  'dist/src/index.cjs', // CJS library entry
+  'dist/src/server/index.js', // Server entry
+];
+
+interface CopyTask {
+  src: string;
+  dest: string;
+  recursive?: boolean;
+  filter?: (src: string) => boolean;
+}
+
+interface PostbuildResult {
+  success: boolean;
+  copied: string[];
+  errors: string[];
+}
+
+/**
+ * Logs a message to stdout with consistent formatting.
+ */
+function log(message: string): void {
+  console.log(`[postbuild] ${message}`);
+}
+
+/**
+ * Logs an error message to stderr with consistent formatting.
+ */
+function logError(message: string): void {
+  console.error(`[postbuild] ERROR: ${message}`);
+}
+
+/**
+ * Find all HTML files in src/ directory (non-recursive).
+ */
+function getHtmlFiles(): CopyTask[] {
+  try {
+    return fs
+      .readdirSync(SRC)
+      .filter((file) => file.endsWith('.html'))
+      .map((file) => ({
+        src: path.join(SRC, file),
+        dest: path.join(DIST, 'src', file),
+      }));
+  } catch (error) {
+    logError(`Failed to read src/ directory: ${error}`);
+    return [];
+  }
+}
+
+/**
+ * Generate copy tasks for all wrapper scripts.
+ * Uses WRAPPER_TYPES and WRAPPER_FILES to ensure consistency with src/esm.ts
+ *
+ * Wrapper files are copied to two locations:
+ * 1. dist/src/{python,ruby,golang}/ - for CLI builds (entrypoint.js, main.js)
+ * 2. dist/src/server/{python,ruby,golang}/ - for bundled server build (server/index.js)
+ *
+ * This is necessary because getWrapperDir() uses import.meta.url to determine
+ * the base directory. In the bundled server, import.meta.url points to
+ * dist/src/server/index.js, so wrapper files need to be at dist/src/server/{type}/.
+ */
+function getWrapperTasks(): CopyTask[] {
+  const tasks: CopyTask[] = [];
+
+  // Destinations for wrapper files:
+  // - dist/src/ for CLI (entrypoint.js, main.js use import.meta.url → dist/src/)
+  // - dist/src/server/ for bundled server (server/index.js uses import.meta.url → dist/src/server/)
+  const destBases = [path.join(DIST, 'src'), path.join(DIST, 'src', 'server')];
+
+  for (const wrapperType of WRAPPER_TYPES) {
+    const files = WRAPPER_FILES[wrapperType];
+    for (const file of files) {
+      for (const destBase of destBases) {
+        tasks.push({
+          src: path.join(SRC, wrapperType, file),
+          dest: path.join(destBase, wrapperType, file),
+        });
+      }
+    }
+  }
+
+  return tasks;
+}
+
+/**
+ * Get the drizzle migration copy task with exclusion filter.
+ */
+function getDrizzleTask(): CopyTask {
+  return {
+    src: path.join(ROOT, 'drizzle'),
+    dest: path.join(DIST, 'drizzle'),
+    recursive: true,
+    filter: shouldCopyDrizzlePath,
+  };
+}
+
+/**
+ * Get the proto files copy task for OTLP protobuf support.
+ */
+function getProtoTask(): CopyTask {
+  return {
+    src: path.join(SRC, 'tracing', 'proto'),
+    dest: path.join(DIST, 'src', 'tracing', 'proto'),
+    recursive: true,
+  };
+}
+
+/**
+ * Verify that all critical build outputs exist.
+ */
+function verifyBuildOutputs(): string[] {
+  const missing: string[] = [];
+
+  for (const outputPath of REQUIRED_BUILD_OUTPUTS) {
+    const fullPath = path.join(ROOT, outputPath);
+    if (!fs.existsSync(fullPath)) {
+      missing.push(outputPath);
+    }
+  }
+
+  return missing;
+}
+
+/**
+ * Clean destination directories before copying to prevent stale files.
+ * Only cleans specific subdirectories, not all of dist/.
+ */
+function cleanDestinations(_tasks: CopyTask[]): void {
+  // Clean wrapper directories (both at dist/src/ and dist/src/server/)
+  const wrapperBases = [path.join(DIST, 'src'), path.join(DIST, 'src', 'server')];
+  for (const base of wrapperBases) {
+    for (const wrapperType of WRAPPER_TYPES) {
+      const wrapperDest = path.join(base, wrapperType);
+      if (fs.existsSync(wrapperDest)) {
+        fs.rmSync(wrapperDest, { recursive: true, force: true });
+      }
+    }
+  }
+
+  // Clean drizzle directory
+  const drizzleDest = path.join(DIST, 'drizzle');
+  if (fs.existsSync(drizzleDest)) {
+    fs.rmSync(drizzleDest, { recursive: true, force: true });
+  }
+}
+
+/**
+ * Execute a single copy task.
+ */
+function executeCopyTask(task: CopyTask): { success: boolean; error?: string } {
+  try {
+    if (!fs.existsSync(task.src)) {
+      return { success: false, error: `Source not found: ${task.src.replace(ROOT, '.')}` };
+    }
+
+    fs.mkdirSync(path.dirname(task.dest), { recursive: true });
+
+    fs.cpSync(task.src, task.dest, {
+      recursive: task.recursive ?? false,
+      filter: task.filter,
+    });
+
+    return { success: true };
+  } catch (error) {
+    return { success: false, error: `Copy failed: ${error}` };
+  }
+}
+
+/**
+ * Main postbuild function.
+ */
+export function postbuild(): PostbuildResult {
+  const result: PostbuildResult = {
+    success: true,
+    copied: [],
+    errors: [],
+  };
+
+  log('Starting postbuild...');
+
+  // Verify tsdown produced the expected outputs first
+  const missingOutputs = verifyBuildOutputs();
+  if (missingOutputs.length > 0) {
+    for (const missing of missingOutputs) {
+      result.errors.push(`Missing build output: ${missing}`);
+    }
+    logError('tsdown build appears to have failed. Missing outputs:');
+    for (const missing of missingOutputs) {
+      logError(`  - ${missing}`);
+    }
+    result.success = false;
+    return result;
+  }
+
+  // Gather all copy tasks
+  const copyTasks = [...getHtmlFiles(), ...getWrapperTasks(), getDrizzleTask(), getProtoTask()];
+
+  // Clean destinations to prevent stale files
+  cleanDestinations(copyTasks);
+
+  // Execute copy tasks
+  for (const task of copyTasks) {
+    const copyResult = executeCopyTask(task);
+    if (copyResult.success) {
+      const relativePath = task.dest.replace(ROOT, '.');
+      result.copied.push(relativePath);
+      log(`Copied: ${task.src.replace(ROOT, '.')} -> ${relativePath}`);
+    } else {
+      result.errors.push(copyResult.error!);
+      logError(copyResult.error!);
+      result.success = false;
+    }
+  }
+
+  // Create ESM package.json marker for dist/src
+  const distSrcPackageJson = path.join(DIST, 'src', 'package.json');
+  try {
+    fs.writeFileSync(distSrcPackageJson, JSON.stringify({ type: 'module' }, null, 2) + '\n');
+    log('Created: ./dist/src/package.json');
+  } catch (error) {
+    result.errors.push(`Failed to create ESM marker: ${error}`);
+    logError(`Failed to create ESM marker: ${error}`);
+    result.success = false;
+  }
+
+  // Make CLI executables (no-op on Windows, but doesn't hurt)
+  const cliExecutables = ['entrypoint.js', 'main.js'];
+  for (const executable of cliExecutables) {
+    const execPath = path.join(DIST, 'src', executable);
+    try {
+      fs.chmodSync(execPath, 0o755);
+      log(`Made executable: ./dist/src/${executable}`);
+    } catch (error) {
+      // chmod may fail on Windows - this is acceptable
+      log(`Note: chmod failed (expected on Windows): ${error}`);
+    }
+  }
+
+  if (result.success) {
+    log(`Postbuild complete. Copied ${result.copied.length} items.`);
+  } else {
+    logError(`Postbuild failed with ${result.errors.length} error(s).`);
+  }
+
+  return result;
+}
+
+// Run if executed directly (not when imported for testing)
+const isMainModule = process.argv[1] === fileURLToPath(import.meta.url);
+if (isMainModule) {
+  const result = postbuild();
+  if (!result.success) {
+    process.exit(1);
+  }
+}

scripts/regenerateEdgeBaseline.ts

@@ -0,0 +1,33 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import {
+  buildEdgeBaseline,
+  computeCrossLayerEdges,
+  readLayerConfig,
+  scanArchitectureSources,
+} from './architectureUtils';
+
+/**
+ * Regenerates architecture/edge-baseline.json from the current source tree.
+ * The baseline records the import count for every cross-layer edge; the
+ * architecture checker fails if any edge grows above its baseline or a new
+ * cross-layer edge appears. Run this after intentionally reducing (or, with
+ * justification, changing) cross-layer coupling so the ratchet captures the
+ * new, lower numbers.
+ */
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const config = readLayerConfig(repoRoot);
+const sourceScan = scanArchitectureSources(repoRoot, config);
+const edges = computeCrossLayerEdges(repoRoot, config, sourceScan);
+const baseline = buildEdgeBaseline(edges, config);
+
+const baselinePath = path.join(repoRoot, 'architecture/edge-baseline.json');
+fs.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`);
+
+const totalEdges = Object.keys(baseline).length;
+const totalImports = Object.values(baseline).reduce((sum, count) => sum + count, 0);
+console.log(
+  `Wrote ${path.relative(repoRoot, baselinePath)} with ${totalEdges} cross-layer edges / ${totalImports} imports.`,
+);

scripts/reportDependencyOwnership.ts

@@ -0,0 +1,95 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import {
+  extractModuleSpecifiers,
+  getLayerForFile,
+  getPackageName,
+  getSourceFiles,
+  readLayerConfig,
+} from './architectureUtils';
+
+interface DependencyUsage {
+  files: Set<string>;
+  layers: Set<string>;
+}
+
+interface PackageJson {
+  dependencies?: Record<string, string>;
+  optionalDependencies?: Record<string, string>;
+}
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const config = readLayerConfig(repoRoot);
+const packageJson = JSON.parse(
+  fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'),
+) as PackageJson;
+
+const runtimeDependencies = new Map<string, Set<'dependency' | 'optional'>>();
+for (const dependency of Object.keys(packageJson.dependencies ?? {})) {
+  runtimeDependencies.set(dependency, new Set(['dependency']));
+}
+for (const dependency of Object.keys(packageJson.optionalDependencies ?? {})) {
+  const kinds = runtimeDependencies.get(dependency) ?? new Set<'dependency' | 'optional'>();
+  kinds.add('optional');
+  runtimeDependencies.set(dependency, kinds);
+}
+
+const usages = new Map<string, DependencyUsage>();
+const undeclaredUsages = new Map<string, DependencyUsage>();
+
+for (const sourceFile of getSourceFiles(repoRoot, false, config.ignoredRoots)) {
+  const sourceText = fs.readFileSync(path.join(repoRoot, sourceFile), 'utf8');
+  const layer = getLayerForFile(sourceFile, config);
+
+  for (const specifier of extractModuleSpecifiers(sourceText, sourceFile)) {
+    const packageName = getPackageName(specifier);
+    if (!packageName) {
+      continue;
+    }
+
+    const targetMap = runtimeDependencies.has(packageName) ? usages : undeclaredUsages;
+    const usage = targetMap.get(packageName) ?? {
+      files: new Set<string>(),
+      layers: new Set<string>(),
+    };
+    usage.files.add(sourceFile);
+    usage.layers.add(layer);
+    targetMap.set(packageName, usage);
+  }
+}
+
+const rows = [...runtimeDependencies.entries()]
+  .map(([dependency, kinds]) => {
+    const usage = usages.get(dependency);
+    const layers = usage ? [...usage.layers].sort() : [];
+    return {
+      dependency,
+      kind: [...kinds].sort().join('+'),
+      owner: layers.length === 0 ? 'unreferenced' : layers.length === 1 ? layers[0] : 'shared',
+      layers: layers.join(', ') || '-',
+      files: usage?.files.size ?? 0,
+    };
+  })
+  .sort((left, right) => left.dependency.localeCompare(right.dependency));
+
+if (process.argv.includes('--json')) {
+  console.log(JSON.stringify(rows, null, 2));
+  process.exit(0);
+}
+
+console.log('| Dependency | Kind | Candidate owner | Direct layers | Source files |');
+console.log('| --- | --- | --- | --- | ---: |');
+for (const row of rows) {
+  console.log(`| ${row.dependency} | ${row.kind} | ${row.owner} | ${row.layers} | ${row.files} |`);
+}
+
+if (undeclaredUsages.size > 0) {
+  console.log('\nImported packages outside root runtime dependencies:');
+  for (const [dependency, usage] of [...undeclaredUsages.entries()].sort(([left], [right]) =>
+    left.localeCompare(right),
+  )) {
+    console.log(`- ${dependency}: ${[...usage.layers].sort().join(', ')}`);
+  }
+}

scripts/testPackageArtifact.ts

@@ -0,0 +1,658 @@
+import assert from 'node:assert/strict';
+import { execFile, execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import { createServer } from 'node:http';
+import os from 'node:os';
+import path from 'node:path';
+import { brotliCompressSync, gzipSync } from 'node:zlib';
+
+import { shouldCopyDrizzlePath } from './postbuild';
+
+type PackFile = {
+  mode: number;
+  path: string;
+  size: number;
+};
+
+type PackResult = {
+  files: PackFile[];
+  filename: string;
+  name: string;
+  version: string;
+};
+
+type ArtifactEvalOutput = {
+  results?: {
+    results?: Array<{
+      error?: string;
+      response?: {
+        error?: string;
+        output?: unknown;
+      };
+      success?: boolean;
+    }>;
+  };
+};
+
+const ROOT = path.resolve(import.meta.dirname, '..');
+const drizzleDir = path.join(ROOT, 'drizzle');
+const requiredPackagedPaths = [
+  'dist/drizzle/meta/_journal.json',
+  'dist/src/app/index.html',
+  'dist/src/entrypoint.js',
+  'dist/src/golang/wrapper.go',
+  'dist/src/contracts.cjs',
+  'dist/src/contracts.d.cts',
+  'dist/src/contracts.d.ts',
+  'dist/src/contracts.js',
+  'dist/src/index.cjs',
+  'dist/src/index.d.ts',
+  'dist/src/index.js',
+  'dist/src/main.js',
+  'dist/src/package.json',
+  'dist/src/python/persistent_wrapper.py',
+  'dist/src/python/wrapper.py',
+  'dist/src/ruby/wrapper.rb',
+  'dist/src/server/golang/wrapper.go',
+  'dist/src/server/index.js',
+  'dist/src/server/python/persistent_wrapper.py',
+  'dist/src/server/python/wrapper.py',
+  'dist/src/server/ruby/wrapper.rb',
+  'dist/src/tableOutput.html',
+  'dist/src/tracing/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto',
+  'dist/src/tracing/proto/opentelemetry/proto/common/v1/common.proto',
+  'dist/src/tracing/proto/opentelemetry/proto/resource/v1/resource.proto',
+  'dist/src/tracing/proto/opentelemetry/proto/trace/v1/trace.proto',
+];
+
+function listFiles(rootDir: string): string[] {
+  return fs.readdirSync(rootDir, { withFileTypes: true }).flatMap((entry) => {
+    const fullPath = path.join(rootDir, entry.name);
+
+    if (entry.isDirectory()) {
+      return listFiles(fullPath);
+    }
+
+    return [path.relative(ROOT, fullPath).split(path.sep).join('/')];
+  });
+}
+
+function run(
+  command: string,
+  args: string[],
+  cwd: string,
+  envOverrides: NodeJS.ProcessEnv = {},
+): string {
+  try {
+    return execFileSync(command, args, {
+      cwd,
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        npm_config_audit: 'false',
+        npm_config_fund: 'false',
+        ...envOverrides,
+      },
+      maxBuffer: 128 * 1024 * 1024,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      const details = [error.message];
+      if ('stdout' in error && typeof error.stdout === 'string' && error.stdout.length > 0) {
+        details.push(`stdout:\n${error.stdout}`);
+      }
+      if (
+        'stderr' in error &&
+        typeof error.stderr === 'string' &&
+        error.stderr.length > 0 &&
+        !error.message.includes(error.stderr)
+      ) {
+        details.push(`stderr:\n${error.stderr}`);
+      }
+      throw new Error(details.join('\n'), { cause: error });
+    }
+    throw error;
+  }
+}
+
+async function runAsync(
+  command: string,
+  args: string[],
+  cwd: string,
+  envOverrides: NodeJS.ProcessEnv = {},
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    execFile(
+      command,
+      args,
+      {
+        cwd,
+        encoding: 'utf8',
+        env: {
+          ...process.env,
+          npm_config_audit: 'false',
+          npm_config_fund: 'false',
+          ...envOverrides,
+        },
+        maxBuffer: 128 * 1024 * 1024,
+        timeout: 60_000,
+      },
+      (error, stdout, stderr) => {
+        if (!error) {
+          resolve(stdout);
+          return;
+        }
+
+        const details = [error.message];
+        if (stdout.length > 0) {
+          details.push(`stdout:\n${stdout}`);
+        }
+        if (stderr.length > 0 && !error.message.includes(stderr)) {
+          details.push(`stderr:\n${stderr}`);
+        }
+        reject(new Error(details.join('\n'), { cause: error }));
+      },
+    );
+  });
+}
+
+function runNpm(args: string[], cwd: string, envOverrides: NodeJS.ProcessEnv = {}): string {
+  assert(process.env.npm_execpath, 'Expected npm_execpath when running package artifact test');
+  return run(process.execPath, [process.env.npm_execpath, ...args], cwd, envOverrides);
+}
+
+function assertPackagedFiles(packResult: PackResult): void {
+  const packagedPaths = new Set(packResult.files.map((file) => file.path));
+  const missingPaths = requiredPackagedPaths.filter((file) => !packagedPaths.has(file));
+  const missingDrizzleFiles = listFiles(drizzleDir)
+    .filter(shouldCopyDrizzlePath)
+    .map((file) => `dist/${file}`)
+    .filter((file) => !packagedPaths.has(file));
+  const missingWebAppFiles = listFiles(path.join(ROOT, 'dist', 'src', 'app'))
+    .filter((file) => !file.endsWith('.map'))
+    .filter((file) => !packagedPaths.has(file));
+
+  assert.deepEqual(missingPaths, [], `Missing packaged runtime assets: ${missingPaths.join(', ')}`);
+  assert.deepEqual(
+    missingDrizzleFiles,
+    [],
+    `Missing packaged Drizzle files: ${missingDrizzleFiles.join(', ')}`,
+  );
+  assert.deepEqual(
+    missingWebAppFiles,
+    [],
+    `Missing packaged web app files: ${missingWebAppFiles.join(', ')}`,
+  );
+  assert(
+    packResult.files.every((file) => !file.path.endsWith('.map')),
+    'Source maps should be excluded from the package',
+  );
+  assert(
+    packResult.files.every((file) => !file.path.startsWith('dist/test/')),
+    'Compiled tests should be excluded from the package',
+  );
+  assert(
+    packResult.files.every((file) => !file.path.startsWith('dist/src/__mocks__/')),
+    'Compiled mocks should be excluded from the package',
+  );
+
+  for (const executablePath of ['dist/src/entrypoint.js', 'dist/src/main.js']) {
+    const executable = packResult.files.find((file) => file.path === executablePath);
+    assert(executable, `Missing packaged CLI executable: ${executablePath}`);
+    assert(
+      executable.mode & 0o111,
+      `Packaged CLI executable should be executable: ${executablePath}`,
+    );
+  }
+}
+
+function assertInstalledWebApp(installedPackageDir: string): void {
+  const webAppDir = path.join(installedPackageDir, 'dist', 'src', 'app');
+  const indexHtml = fs.readFileSync(path.join(webAppDir, 'index.html'), 'utf8');
+  const localAssetPaths = [...indexHtml.matchAll(/\b(?:href|src)="\/([^"]+)"/g)].map(
+    ([, assetPath]) => assetPath.split(/[?#]/, 1)[0],
+  );
+
+  assert(localAssetPaths.length > 0, 'Expected packaged web app index to reference local assets');
+  assert(
+    localAssetPaths.some((assetPath) => assetPath.endsWith('.js')),
+    'Expected packaged web app index to reference a JavaScript entrypoint',
+  );
+  assert(
+    localAssetPaths.some((assetPath) => assetPath.endsWith('.css')),
+    'Expected packaged web app index to reference a stylesheet',
+  );
+
+  const missingAssets = localAssetPaths.filter(
+    (assetPath) => !fs.existsSync(path.join(webAppDir, assetPath)),
+  );
+  assert.deepEqual(
+    missingAssets,
+    [],
+    `Missing packaged web app assets: ${missingAssets.join(', ')}`,
+  );
+}
+
+/**
+ * Asserts every file path declared in the installed package's `exports` and `typesVersions`
+ * resolves to a real file. The consumer `tsc` checks can't catch a wrong declared `types` path on
+ * their own — TypeScript falls through to the `default` condition and auto-discovers the sibling
+ * `.d.ts` — so validate the declared paths directly here.
+ */
+function assertExportsResolve(
+  installedPackageDir: string,
+  manifest: {
+    exports?: unknown;
+    typesVersions?: Record<string, Record<string, string[]>>;
+  },
+): void {
+  const referencedPaths = new Set<string>();
+  const collect = (value: unknown): void => {
+    if (typeof value === 'string') {
+      if (value.startsWith('.')) {
+        referencedPaths.add(value);
+      }
+      return;
+    }
+    if (value && typeof value === 'object') {
+      for (const nested of Object.values(value)) {
+        collect(nested);
+      }
+    }
+  };
+  collect(manifest.exports);
+  for (const subpaths of Object.values(manifest.typesVersions ?? {})) {
+    for (const candidatePaths of Object.values(subpaths)) {
+      for (const candidatePath of candidatePaths) {
+        referencedPaths.add(candidatePath);
+      }
+    }
+  }
+
+  assert(
+    referencedPaths.size > 0,
+    'Expected the installed package manifest to declare export paths',
+  );
+  const missing = [...referencedPaths].filter(
+    (relativePath) => !fs.existsSync(path.join(installedPackageDir, relativePath)),
+  );
+  assert.deepEqual(
+    missing,
+    [],
+    `Installed package exports reference missing files: ${missing.join(', ')}`,
+  );
+}
+
+function runInstalledBinVersion(consumerDir: string, configDir: string, binName: string): string {
+  const binPath = path.join(
+    consumerDir,
+    'node_modules',
+    '.bin',
+    process.platform === 'win32' ? `${binName}.cmd` : binName,
+  );
+  assert(fs.existsSync(binPath), `Missing installed ${binName} bin: ${binPath}`);
+  const envOverrides = {
+    PROMPTFOO_CONFIG_DIR: configDir,
+    PROMPTFOO_DISABLE_TELEMETRY: '1',
+    PROMPTFOO_DISABLE_UPDATE: 'true',
+  };
+
+  if (process.platform === 'win32') {
+    return run(
+      process.env.ComSpec || 'cmd.exe',
+      ['/d', '/s', '/c', `"${binPath}" --version`],
+      consumerDir,
+      envOverrides,
+    );
+  }
+
+  return run(binPath, ['--version'], consumerDir, envOverrides);
+}
+
+function writeConsumerScripts(consumerDir: string): void {
+  fs.writeFileSync(
+    path.join(consumerDir, 'import-package.mjs'),
+    [
+      "import { AssertionSchema, AtomicTestCaseSchema, TestSuiteSchema } from 'promptfoo';",
+      "import { EmailSchema, GetUserResponseSchema, InputsSchema, PromptSchema, hasFunctionToolCallValidator } from 'promptfoo/contracts';",
+      '',
+      'for (const value of [AssertionSchema, AtomicTestCaseSchema, EmailSchema, GetUserResponseSchema, InputsSchema, PromptSchema, TestSuiteSchema]) {',
+      "  if (!value || typeof value.safeParse !== 'function') {",
+      "    throw new Error('Missing expected ESM schema export');",
+      '  }',
+      '}',
+      'if (!hasFunctionToolCallValidator({ validateFunctionToolCall() {} })) {',
+      "  throw new Error('Missing expected ESM provider capability export');",
+      '}',
+      '',
+    ].join('\n'),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'require-package.cjs'),
+    [
+      "const { AssertionSchema, AtomicTestCaseSchema, TestSuiteSchema } = require('promptfoo');",
+      "const { EmailSchema, GetUserResponseSchema, InputsSchema, PromptSchema, hasFunctionToolCallValidator } = require('promptfoo/contracts');",
+      '',
+      'for (const value of [AssertionSchema, AtomicTestCaseSchema, EmailSchema, GetUserResponseSchema, InputsSchema, PromptSchema, TestSuiteSchema]) {',
+      "  if (!value || typeof value.safeParse !== 'function') {",
+      "    throw new Error('Missing expected CJS schema export');",
+      '  }',
+      '}',
+      'if (!hasFunctionToolCallValidator({ validateFunctionToolCall() {} })) {',
+      "  throw new Error('Missing expected CJS provider capability export');",
+      '}',
+      '',
+    ].join('\n'),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'import-contracts.ts'),
+    [
+      "import { GetUserResponseSchema, PromptSchema, hasFunctionToolCallValidator, isTransformFunction } from 'promptfoo/contracts';",
+      "import type { BlobRef, FunctionToolCallValidator, GetUserResponse, Prompt, ProviderResponse, TransformFunction } from 'promptfoo/contracts';",
+      '',
+      "const prompt: Prompt = { label: 'Greeting', raw: 'Hello, world!' };",
+      'const transform: TransformFunction<string, string> = (output) => output;',
+      "const user: GetUserResponse = { email: 'user@example.com' };",
+      'const validator: FunctionToolCallValidator = { validateFunctionToolCall() {} };',
+      "const blobRef: BlobRef = { hash: 'abc123', mimeType: 'image/png', provider: 'filesystem', sizeBytes: 3, uri: 'promptfoo://blob/abc123' };",
+      "const response: ProviderResponse = { images: [{ blobRef }], output: 'ok' };",
+      '',
+      'GetUserResponseSchema.parse(user);',
+      'PromptSchema.parse(prompt);',
+      'void response;',
+      'if (!isTransformFunction(transform) || !hasFunctionToolCallValidator(validator)) {',
+      "  throw new Error('Missing expected TypeScript contracts export');",
+      '}',
+      '',
+    ].join('\n'),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'tsconfig.json'),
+    JSON.stringify({
+      compilerOptions: {
+        module: 'NodeNext',
+        moduleResolution: 'NodeNext',
+        noEmit: true,
+        strict: true,
+      },
+      include: ['import-contracts.ts'],
+    }),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'tsconfig.legacy.json'),
+    JSON.stringify({
+      compilerOptions: {
+        ignoreDeprecations: '6.0',
+        module: 'CommonJS',
+        moduleResolution: 'node',
+        noEmit: true,
+        strict: true,
+      },
+      include: ['import-contracts.ts'],
+    }),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'require-contracts.cts'),
+    [
+      "import contracts = require('promptfoo/contracts');",
+      '',
+      "const prompt: contracts.Prompt = { label: 'Greeting', raw: 'Hello, world!' };",
+      "const user: contracts.GetUserResponse = { email: 'user@example.com' };",
+      'const validator: contracts.FunctionToolCallValidator = { validateFunctionToolCall() {} };',
+      "const blobRef: contracts.BlobRef = { hash: 'abc123', mimeType: 'image/png', provider: 'filesystem', sizeBytes: 3, uri: 'promptfoo://blob/abc123' };",
+      "const response: contracts.ProviderResponse = { images: [{ blobRef }], output: 'ok' };",
+      'contracts.GetUserResponseSchema.parse(user);',
+      'contracts.PromptSchema.parse(prompt);',
+      'void response;',
+      'if (!contracts.hasFunctionToolCallValidator(validator)) {',
+      "  throw new Error('Missing expected CommonJS TypeScript contracts export');",
+      '}',
+      '',
+    ].join('\n'),
+  );
+  fs.writeFileSync(
+    path.join(consumerDir, 'tsconfig.node16-cjs.json'),
+    JSON.stringify({
+      compilerOptions: {
+        module: 'Node16',
+        moduleResolution: 'Node16',
+        noEmit: true,
+        strict: true,
+      },
+      include: ['require-contracts.cts'],
+    }),
+  );
+}
+
+async function runInstalledCompressionEval(consumerDir: string, configDir: string): Promise<void> {
+  const expectedOutput = 'compressed response';
+  const requestedPaths: string[] = [];
+  const server = createServer((request, response) => {
+    const requestPath = request.url;
+    if (requestPath !== '/gzip' && requestPath !== '/br') {
+      response.writeHead(404).end();
+      return;
+    }
+
+    const encoding = requestPath === '/gzip' ? 'gzip' : 'br';
+    requestedPaths.push(requestPath);
+    const rawBody = Buffer.from(JSON.stringify({ output: expectedOutput }));
+    const body = encoding === 'gzip' ? gzipSync(rawBody) : brotliCompressSync(rawBody);
+    response.writeHead(200, {
+      'content-encoding': encoding,
+      'content-length': String(body.length),
+      'content-type': 'application/json',
+    });
+    response.end(body);
+  });
+
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(0, '127.0.0.1', () => {
+        server.off('error', reject);
+        resolve();
+      });
+    });
+
+    const address = server.address();
+    assert(address && typeof address !== 'string', 'Failed to bind compressed response server');
+
+    const configPath = path.join(consumerDir, 'promptfooconfig.json');
+    const outputPath = path.join(consumerDir, 'compression-results.json');
+    const baseUrl = `http://127.0.0.1:${address.port}`;
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify(
+        {
+          prompts: ['compression artifact test'],
+          providers: ['gzip', 'br'].map((encoding) => ({
+            id: `${baseUrl}/${encoding}`,
+            config: {
+              maxRetries: 0,
+              method: 'GET',
+              transformResponse: 'json.output',
+            },
+          })),
+          tests: [
+            {
+              assert: [{ type: 'equals', value: expectedOutput }],
+            },
+          ],
+        },
+        null,
+        2,
+      ),
+    );
+
+    const entrypointPath = path.join(
+      consumerDir,
+      'node_modules',
+      'promptfoo',
+      'dist',
+      'src',
+      'entrypoint.js',
+    );
+    assert(
+      fs.existsSync(entrypointPath),
+      `Missing installed promptfoo entrypoint: ${entrypointPath}`,
+    );
+
+    await runAsync(
+      process.execPath,
+      [
+        entrypointPath,
+        'eval',
+        '--config',
+        configPath,
+        '--output',
+        outputPath,
+        '--no-cache',
+        '--no-write',
+        '--no-table',
+        '--no-progress-bar',
+        '--max-concurrency',
+        '1',
+      ],
+      consumerDir,
+      {
+        ALL_PROXY: '',
+        HTTP_PROXY: '',
+        HTTPS_PROXY: '',
+        NO_PROXY: '127.0.0.1,localhost',
+        PROMPTFOO_CONFIG_DIR: configDir,
+        PROMPTFOO_DISABLE_REMOTE_GENERATION: 'true',
+        PROMPTFOO_DISABLE_TELEMETRY: '1',
+        PROMPTFOO_DISABLE_UPDATE: 'true',
+        all_proxy: '',
+        http_proxy: '',
+        https_proxy: '',
+        no_proxy: '127.0.0.1,localhost',
+      },
+    );
+
+    assert(fs.existsSync(outputPath), `Installed promptfoo did not write results: ${outputPath}`);
+    const evalOutput = JSON.parse(fs.readFileSync(outputPath, 'utf8')) as ArtifactEvalOutput;
+    const results = evalOutput.results?.results;
+    assert(Array.isArray(results), 'Installed promptfoo output is missing evaluation results');
+    assert.equal(results.length, 2, 'Expected one result for each compressed provider');
+    for (const result of results) {
+      assert.equal(result.success, true, `Compressed provider failed: ${JSON.stringify(result)}`);
+      assert.equal(
+        result.error,
+        undefined,
+        `Compressed provider errored: ${JSON.stringify(result)}`,
+      );
+      assert.equal(
+        result.response?.error,
+        undefined,
+        `Compressed provider response errored: ${JSON.stringify(result)}`,
+      );
+      assert.equal(result.response?.output, expectedOutput);
+    }
+    assert.deepEqual(requestedPaths.sort(), ['/br', '/gzip']);
+  } finally {
+    if (server.listening) {
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => (error ? reject(error) : resolve()));
+      });
+    }
+  }
+}
+
+async function main(): Promise<void> {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'promptfoo-package-artifact-'));
+  const artifactsDir = path.join(tempDir, 'artifacts');
+  const configDir = path.join(tempDir, 'config');
+  const consumerDir = path.join(tempDir, 'consumer');
+  const consumerNpmrc = path.join(tempDir, 'consumer.npmrc');
+
+  try {
+    fs.mkdirSync(artifactsDir);
+    fs.mkdirSync(configDir);
+    fs.mkdirSync(consumerDir);
+    fs.writeFileSync(consumerNpmrc, '');
+
+    const packOutput = runNpm(
+      ['pack', '--ignore-scripts', '--json', '--pack-destination', artifactsDir],
+      ROOT,
+    );
+    let packResults: PackResult[];
+    try {
+      packResults = JSON.parse(packOutput) as PackResult[];
+    } catch (error) {
+      throw new Error(`Failed to parse npm pack JSON output:\n${packOutput}`, { cause: error });
+    }
+    assert.equal(packResults.length, 1, 'Expected npm pack to produce one artifact');
+
+    const [packResult] = packResults;
+    assert.equal(packResult.name, 'promptfoo');
+    assertPackagedFiles(packResult);
+
+    const tarballPath = path.join(artifactsDir, packResult.filename);
+    assert(fs.existsSync(tarballPath), `Missing tarball: ${tarballPath}`);
+
+    fs.writeFileSync(
+      path.join(consumerDir, 'package.json'),
+      JSON.stringify({
+        name: 'promptfoo-package-artifact-consumer',
+        private: true,
+        type: 'module',
+      }),
+    );
+    runNpm(
+      [
+        'install',
+        '--ignore-scripts',
+        '--no-audit',
+        '--no-fund',
+        '--no-package-lock',
+        '--registry=https://registry.npmjs.org/',
+        tarballPath,
+      ],
+      consumerDir,
+      {
+        npm_config_engine_strict: 'false',
+        npm_config_userconfig: consumerNpmrc,
+      },
+    );
+
+    const installedPackageDir = path.join(consumerDir, 'node_modules', 'promptfoo');
+    const installedPackageJson = JSON.parse(
+      fs.readFileSync(path.join(installedPackageDir, 'package.json'), 'utf8'),
+    ) as {
+      version: string;
+      exports?: unknown;
+      typesVersions?: Record<string, Record<string, string[]>>;
+    };
+    assert.equal(installedPackageJson.version, packResult.version);
+    assertExportsResolve(installedPackageDir, installedPackageJson);
+
+    writeConsumerScripts(consumerDir);
+    run(process.execPath, ['import-package.mjs'], consumerDir);
+    run(process.execPath, ['require-package.cjs'], consumerDir);
+    const tscPath = path.join(ROOT, 'node_modules', 'typescript', 'bin', 'tsc');
+    for (const tsconfig of ['tsconfig.json', 'tsconfig.legacy.json', 'tsconfig.node16-cjs.json']) {
+      run(process.execPath, [tscPath, '--project', tsconfig], consumerDir);
+    }
+    assertInstalledWebApp(installedPackageDir);
+
+    for (const binName of ['promptfoo', 'pf']) {
+      assert.equal(
+        runInstalledBinVersion(consumerDir, configDir, binName).trim(),
+        packResult.version,
+      );
+    }
+    await runInstalledCompressionEval(consumerDir, configDir);
+
+    console.log(`Verified installed package artifact: ${packResult.filename}`);
+  } finally {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+}
+
+main().catch((error: unknown) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/update-changelog-version.cjs

@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+/**
+ * Updates CHANGELOG.md when bumping version:
+ * - Moves Unreleased entries to new versioned section
+ * - Creates fresh Unreleased section
+ *
+ * Run automatically via `npm version` postversion hook
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const CHANGELOG_PATH = path.join(__dirname, '..', 'CHANGELOG.md');
+
+/**
+ * Empty template for the Unreleased section after release
+ */
+const EMPTY_UNRELEASED = `## [Unreleased]
+
+### Added
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+### Dependencies
+
+### Documentation
+
+### Tests
+`;
+
+/**
+ * All valid Keep a Changelog category headers
+ * https://keepachangelog.com/en/1.1.0/
+ */
+const CHANGELOG_CATEGORIES = [
+  'Added',
+  'Changed',
+  'Deprecated',
+  'Removed',
+  'Fixed',
+  'Security',
+  'Dependencies',
+  'Documentation',
+  'Tests',
+];
+
+/**
+ * Check if a section contains only headers and whitespace (no actual entries)
+ * @param {string} content - The section content to check
+ * @returns {boolean} True if section has no actual entries
+ */
+function isEmptySection(content) {
+  // Build regex pattern for all category headers
+  const headerPattern = new RegExp(`### (${CHANGELOG_CATEGORIES.join('|')})`, 'g');
+  const withoutHeaders = content.replace(headerPattern, '').trim();
+  return withoutHeaders.length === 0;
+}
+
+/**
+ * Extract content from the Unreleased section
+ * @param {string} changelog - Full changelog content
+ * @returns {{content: string, match: RegExpMatchArray} | null} Extracted content or null
+ */
+function extractUnreleasedContent(changelog) {
+  // Match content between ## [Unreleased] and next ## [x.y.z] version header
+  const unreleasedMatch = changelog.match(/## \[Unreleased\]\n([\s\S]*?)(?=\n## \[\d)/);
+
+  if (!unreleasedMatch) {
+    return null;
+  }
+
+  return {
+    content: unreleasedMatch[1].trim(),
+    match: unreleasedMatch,
+  };
+}
+
+/**
+ * Build the updated changelog with new version section
+ * @param {string} changelog - Original changelog content
+ * @param {string} version - New version number
+ * @param {string} unreleasedContent - Content to move to versioned section
+ * @param {string} date - Release date in YYYY-MM-DD format
+ * @returns {string} Updated changelog content
+ */
+function buildUpdatedChangelog(changelog, version, unreleasedContent, date) {
+  // Build new versioned section
+  const newVersionSection = `## [${version}] - ${date}\n\n${unreleasedContent}`;
+
+  // Replace Unreleased content with empty template and insert new version
+  const updatedChangelog = changelog.replace(
+    /## \[Unreleased\]\n[\s\S]*?(?=\n## \[\d)/,
+    `${EMPTY_UNRELEASED}\n\n${newVersionSection}\n\n`,
+  );
+
+  // Clean up any triple+ newlines
+  return updatedChangelog.replace(/\n{3,}/g, '\n\n');
+}
+
+/**
+ * @typedef {Object} UpdateChangelogOptions
+ * @property {string} [changelogPath] - Path to CHANGELOG.md
+ * @property {string} [packageJsonPath] - Path to package.json
+ * @property {string} [date] - Override date (for testing)
+ */
+
+/**
+ * @typedef {Object} UpdateChangelogResult
+ * @property {boolean} success - Whether update succeeded
+ * @property {string} [error] - Error message if failed
+ * @property {string} [version] - Version that was released
+ * @property {string} [changelog] - Updated changelog content
+ */
+
+/**
+ * Update changelog for a new version release
+ * @param {UpdateChangelogOptions} [options] - Configuration options
+ * @returns {UpdateChangelogResult} Result of the update operation
+ */
+function updateChangelog(options = {}) {
+  const changelogPath = options.changelogPath || CHANGELOG_PATH;
+  const packageJsonPath = options.packageJsonPath || path.join(__dirname, '..', 'package.json');
+  const date = options.date || new Date().toISOString().split('T')[0];
+
+  try {
+    // Read package.json to get new version
+    let packageJson;
+    try {
+      packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+    } catch (err) {
+      return {
+        success: false,
+        error: `Failed to read package.json: ${err.message}`,
+      };
+    }
+
+    const version = packageJson.version;
+    if (!version) {
+      return {
+        success: false,
+        error: 'No version found in package.json',
+      };
+    }
+
+    // Read current changelog
+    let changelog;
+    try {
+      changelog = fs.readFileSync(changelogPath, 'utf8');
+    } catch (err) {
+      return {
+        success: false,
+        error: `Failed to read CHANGELOG.md: ${err.message}`,
+      };
+    }
+
+    // Extract Unreleased content
+    const extracted = extractUnreleasedContent(changelog);
+    if (!extracted) {
+      return {
+        success: false,
+        error: 'Could not find Unreleased section in CHANGELOG.md',
+      };
+    }
+
+    const unreleasedContent = extracted.content;
+
+    // Check if Unreleased has actual content
+    if (!unreleasedContent || isEmptySection(unreleasedContent)) {
+      return {
+        success: false,
+        error:
+          'No entries in Unreleased section to release. Add changelog entries before bumping version.',
+      };
+    }
+
+    // Build updated changelog
+    const updatedChangelog = buildUpdatedChangelog(changelog, version, unreleasedContent, date);
+
+    // Write back
+    try {
+      fs.writeFileSync(changelogPath, updatedChangelog);
+    } catch (err) {
+      return {
+        success: false,
+        error: `Failed to write CHANGELOG.md: ${err.message}`,
+      };
+    }
+
+    return {
+      success: true,
+      version,
+      changelog: updatedChangelog,
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: `Unexpected error: ${err.message}`,
+    };
+  }
+}
+
+/**
+ * Main entry point when run from command line
+ */
+function main() {
+  const result = updateChangelog();
+
+  if (!result.success) {
+    console.error(`Error: ${result.error}`);
+    process.exit(1);
+  }
+
+  console.log(`Updated CHANGELOG.md for version ${result.version}`);
+}
+
+// Export for testing
+module.exports = {
+  updateChangelog,
+  isEmptySection,
+  extractUnreleasedContent,
+  buildUpdatedChangelog,
+  EMPTY_UNRELEASED,
+  CHANGELOG_CATEGORIES,
+};
+
+// Run if called directly
+if (require.main === module) {
+  main();
+}

scripts/validate-changelog.cjs

@@ -0,0 +1,312 @@
+#!/usr/bin/env node
+/**
+ * Validates CHANGELOG.md entries for PR requirements:
+ * - All entries must have PR numbers (#1234)
+ * - No commit hashes used instead of PR numbers
+ * - All entries must be in Unreleased section
+ *
+ * Used by GitHub Actions workflow for changelog validation
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @typedef {Object} ValidationResult
+ * @property {boolean} valid - Whether validation passed
+ * @property {string} [error] - Error type if validation failed
+ * @property {string} [message] - Human-readable error message
+ * @property {string[]} [entries] - Problematic entries
+ */
+
+/**
+ * Check if entries are missing PR numbers
+ * @param {string[]} entries - Array of changelog entry lines
+ * @returns {string[]} Entries missing PR numbers
+ */
+function findEntriesMissingPrNumber(entries) {
+  const prNumberPattern = /\(#\d+\)/;
+  return entries.filter((entry) => !prNumberPattern.test(entry));
+}
+
+/**
+ * Check if entries use commit hashes instead of PR numbers
+ * @param {string[]} entries - Array of changelog entry lines
+ * @returns {string[]} Entries with commit hashes
+ */
+function findEntriesWithCommitHash(entries) {
+  // Match (abc1234) pattern - a hash in parentheses without # prefix
+  const commitHashPattern = /\([a-f0-9]{7,40}\)/;
+  // Valid PR number pattern
+  const prNumberPattern = /\(#\d+\)/;
+
+  return entries.filter((entry) => {
+    const hasCommitHash = commitHashPattern.test(entry);
+    if (!hasCommitHash) {
+      return false;
+    }
+
+    // If entry has a valid PR number, check if the hash pattern is separate from it
+    // e.g., "feature (abc1234) (#1234)" has both - that's still wrong
+    // Remove valid PR references and check if commit hash pattern still exists
+    const withoutPrRefs = entry.replace(prNumberPattern, '');
+    return commitHashPattern.test(withoutPrRefs);
+  });
+}
+
+/**
+ * Parse git diff to extract added changelog entries
+ * @param {string} diff - Git diff output
+ * @returns {string[]} Added entry lines (without leading +)
+ */
+function parseAddedEntries(diff) {
+  const lines = diff.split('\n');
+  const entries = [];
+
+  for (const line of lines) {
+    // Match lines that start with +- (added entry lines)
+    if (line.match(/^\+- /)) {
+      entries.push(line.substring(1)); // Remove leading +
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Find line number where Unreleased section ends
+ * @param {string} changelogContent - Full changelog content
+ * @returns {number} Line number of first versioned section header
+ */
+function findUnreleasedEnd(changelogContent) {
+  const lines = changelogContent.split('\n');
+  const versionPattern = /^## \[\d+\.\d+\.\d+\]/;
+
+  for (let i = 0; i < lines.length; i++) {
+    if (versionPattern.test(lines[i])) {
+      return i + 1; // Convert to 1-based line number
+    }
+  }
+
+  return 999999; // No versioned sections yet
+}
+
+/**
+ * Parse diff hunks to find entries added in versioned sections
+ * @param {string} diff - Git diff output
+ * @param {number} unreleasedEnd - Line number where versioned sections begin
+ * @returns {Array<{line: number, content: string}>} Entries in versioned sections
+ */
+function findEntriesInVersionedSections(diff, unreleasedEnd) {
+  const lines = diff.split('\n');
+  const entriesInVersioned = [];
+  let currentLine = 0;
+
+  for (const line of lines) {
+    // Parse hunk header: @@ -old_start,old_count +new_start,new_count @@
+    const hunkMatch = line.match(/^@@ .* \+(\d+)/);
+    if (hunkMatch) {
+      currentLine = parseInt(hunkMatch[1], 10) - 1;
+      continue;
+    }
+
+    // Skip removed lines
+    if (line.startsWith('-')) {
+      continue;
+    }
+
+    // Track added lines
+    if (line.startsWith('+')) {
+      currentLine++;
+      // Check if this is a content line (starts with +- )
+      if (line.match(/^\+- /) && currentLine >= unreleasedEnd) {
+        entriesInVersioned.push({
+          line: currentLine,
+          content: line.substring(1), // Remove leading +
+        });
+      }
+      continue;
+    }
+
+    // Context lines
+    currentLine++;
+  }
+
+  return entriesInVersioned;
+}
+
+/**
+ * Validate changelog entries in a git diff
+ * @param {Object} options - Validation options
+ * @param {string} options.diff - Git diff of CHANGELOG.md
+ * @param {string} options.changelogContent - Current changelog file content
+ * @param {string} options.branchName - Current branch name
+ * @param {number} options.prNumber - PR number for error messages
+ * @returns {ValidationResult} Validation result
+ */
+function validateChangelogDiff({ diff, changelogContent, branchName, prNumber }) {
+  // Skip validation for version bump PRs
+  if (branchName && branchName.startsWith('chore/bump-version-')) {
+    return { valid: true };
+  }
+
+  const addedEntries = parseAddedEntries(diff);
+
+  // No entries to validate
+  if (addedEntries.length === 0) {
+    return { valid: true };
+  }
+
+  // Rule 1: Check for PR number requirement
+  const missingPrNumber = findEntriesMissingPrNumber(addedEntries);
+  if (missingPrNumber.length > 0) {
+    return {
+      valid: false,
+      error: 'missing_pr_number',
+      message: `Changelog entry missing PR number
+
+Every changelog entry must include a PR number in (#1234) format.
+
+Expected format: - feat(scope): description (#${prNumber || '1234'})
+
+Entries missing PR numbers:`,
+      entries: missingPrNumber,
+    };
+  }
+
+  // Rule 2: Check for commit hashes instead of PR numbers
+  const withCommitHash = findEntriesWithCommitHash(addedEntries);
+  if (withCommitHash.length > 0) {
+    return {
+      valid: false,
+      error: 'commit_hash_instead_of_pr',
+      message: `Commit hash detected instead of PR number
+
+Use PR numbers (#1234) instead of commit hashes (abc1234).
+
+Entries with commit hashes:`,
+      entries: withCommitHash,
+    };
+  }
+
+  // Rule 3: Check entries are in Unreleased section
+  const unreleasedEnd = findUnreleasedEnd(changelogContent);
+  const entriesInVersioned = findEntriesInVersionedSections(diff, unreleasedEnd);
+
+  if (entriesInVersioned.length > 0) {
+    return {
+      valid: false,
+      error: 'entries_in_versioned_section',
+      message: `Changelog entries must be in the Unreleased section
+
+The following entries were added to versioned sections:`,
+      entries: entriesInVersioned.map((e) => `Line ${e.line}: ${e.content}`),
+      footer: `
+Add your entries under '## [Unreleased]', not in versioned sections.
+Versioned sections are only updated during release.`,
+    };
+  }
+
+  return { valid: true };
+}
+
+/**
+ * Run validation from command line
+ */
+function main() {
+  const baseRef = process.env.BASE_REF || 'main';
+  const prNumber = process.env.PR_NUMBER || '';
+  const branchName = process.env.BRANCH_NAME || '';
+
+  try {
+    // Fetch base branch
+    execSync(`git fetch origin "${baseRef}"`, { stdio: 'pipe' });
+
+    // Check if CHANGELOG.md was modified
+    const modifiedFiles = execSync(`git diff --name-only origin/${baseRef}...HEAD`, {
+      encoding: 'utf8',
+    });
+
+    const changelogModified = modifiedFiles.split('\n').includes('CHANGELOG.md');
+
+    if (!changelogModified) {
+      // Check for bypass labels
+      const hasNoChangelogLabel = process.env.HAS_NO_CHANGELOG_LABEL === 'true';
+      const hasDependenciesLabel = process.env.HAS_DEPENDENCIES_LABEL === 'true';
+
+      if (hasNoChangelogLabel) {
+        console.log("PR has 'no-changelog' label - bypassing check");
+        process.exit(0);
+      }
+
+      if (hasDependenciesLabel) {
+        console.log("PR has 'dependencies' label - bypassing check");
+        process.exit(0);
+      }
+
+      console.log('Changelog update required');
+      console.log('');
+      console.log(
+        "PRs that modify source code must update CHANGELOG.md under the 'Unreleased' section.",
+      );
+      console.log('');
+      console.log('This ensures complete traceability of all changes to the codebase.');
+      console.log('');
+      console.log('To bypass this check, add one of these labels:');
+      console.log("  - 'no-changelog' (exceptional cases only)");
+      console.log("  - 'dependencies' (automated dependency updates)");
+      process.exit(1);
+    }
+
+    console.log('CHANGELOG.md has been updated');
+
+    // Get diff and changelog content
+    const diff = execSync(`git diff origin/${baseRef}...HEAD -- CHANGELOG.md`, {
+      encoding: 'utf8',
+    });
+
+    const changelogPath = path.join(process.cwd(), 'CHANGELOG.md');
+    const changelogContent = fs.readFileSync(changelogPath, 'utf8');
+
+    // Validate
+    const result = validateChangelogDiff({
+      diff,
+      changelogContent,
+      branchName,
+      prNumber,
+    });
+
+    if (!result.valid) {
+      console.log(result.message);
+      if (result.entries) {
+        result.entries.forEach((entry) => console.log(`  ${entry}`));
+      }
+      if (result.footer) {
+        console.log(result.footer);
+      }
+      process.exit(1);
+    }
+
+    console.log('Changelog validation passed');
+    process.exit(0);
+  } catch (error) {
+    console.error('Error during changelog validation:', error.message);
+    process.exit(1);
+  }
+}
+
+// Export for testing
+module.exports = {
+  findEntriesMissingPrNumber,
+  findEntriesWithCommitHash,
+  parseAddedEntries,
+  findUnreleasedEnd,
+  findEntriesInVersionedSections,
+  validateChangelogDiff,
+};
+
+// Run if called directly
+if (require.main === module) {
+  main();
+}