Files
elizaos--eliza/packages/scripts/e2e-coverage/inventory.ts
T
wehub-resource-sync 426e9eeabd
Voice Workbench / headless workbench (mocked backends) (push) Has been cancelled
Voice Workbench / real acoustic lane (nightly, provisioned only) (push) Has been cancelled
ci / test (push) Has been cancelled
ci / lint-and-format (push) Has been cancelled
ci / build (push) Has been cancelled
ci / dev-startup (push) Has been cancelled
gitleaks / gitleaks (push) Has been cancelled
Markdown Links / Relative Markdown Links (push) Has been cancelled
Quality (Extended) / Homepage Build (PR smoke) (push) Has been cancelled
Quality (Extended) / Comment-only diff guard (push) Has been cancelled
Quality (Extended) / Format + Type Safety Ratchet (push) Has been cancelled
Quality (Extended) / Develop Gate (secret scan + UI determinism) (push) Has been cancelled
Quality (Extended) / Develop Gate (lint) (push) Has been cancelled
Chat shell gestures / Chat shell gesture + parity e2e (push) Has been cancelled
Cloud Gateway Discord / Test (push) Has been cancelled
Benchmark Bridge Tests / benchmark (bunx @biomejs/biome check packages/lifeops-bench/src, benchmark-lint) (push) Has been cancelled
Benchmark Bridge Tests / benchmark (bunx vitest run --config packages/lifeops-bench/vitest.config.ts --root packages/lifeops-bench --passWithNoTests, benchmark-tests) (push) Has been cancelled
Build Agent Image / build-and-push (push) Has been cancelled
Dev Smoke / bun run dev onboarding chat (push) Has been cancelled
Dev Smoke / Vite HMR dependency-level smoke (push) Has been cancelled
Electrobun Submodule Guard / electrobun gitlink is fetchable (push) Has been cancelled
Publish @elizaos/example-code / check_npm (push) Has been cancelled
Publish @elizaos/example-code / publish_npm (push) Has been cancelled
Publish @elizaos/plugin-elizacloud / verify_version (push) Has been cancelled
Publish @elizaos/plugin-elizacloud / publish_npm (push) Has been cancelled
Sandbox Live Smoke / Sandbox live smoke (push) Has been cancelled
Snap Build & Test / Build Snap (amd64) (push) Has been cancelled
Snap Build & Test / Build Snap (arm64) (push) Has been cancelled
Test Packaging / elizaos CLI global-install smoke (node + bun) (push) Has been cancelled
Cloud Gateway Webhook / Test (push) Has been cancelled
Cloud Tests / lint-and-types (push) Has been cancelled
Cloud Tests / unit-tests (push) Has been cancelled
Cloud Tests / integration-tests (push) Has been cancelled
Cloud Tests / e2e-tests (push) Has been cancelled
CodeQL Advanced / Analyze (javascript-typescript) (push) Has been cancelled
Deploy Apps Worker (Product 2) / Determine environment (push) Has been cancelled
Deploy Apps Worker (Product 2) / Deploy apps worker to apps-control host (${{ needs.determine-env.outputs.environment }}) (push) Has been cancelled
Deploy Eliza Provisioning Worker / Determine environment (push) Has been cancelled
Deploy Eliza Provisioning Worker / Deploy worker to Hetzner host (${{ needs.determine-env.outputs.environment }} @ ${{ needs.determine-env.outputs.deployment_sha }}) (push) Has been cancelled
Dev Smoke / Classify changed paths (push) Has been cancelled
supply-chain / sbom (push) Has been cancelled
supply-chain / vulnerability-scan (push) Has been cancelled
Build, Push & Deploy to Phala Cloud / build-and-push (push) Has been cancelled
Test Packaging / Validate Packaging Configs (push) Has been cancelled
Test Packaging / Build & Test PyPI Package (push) Has been cancelled
Test Packaging / PyPI on Python ${{ matrix.python }} (push) Has been cancelled
Test Packaging / Pack & Test JS Tarballs (push) Has been cancelled
UI Fixture E2E / ui-fixture-e2e (push) Has been cancelled
UI Fixture E2E / fixture-e2e (push) Has been cancelled
UI Story Gate / story-gate (push) Has been cancelled
vault-ci / test (macos-latest) (push) Has been cancelled
vault-ci / test (ubuntu-latest) (push) Has been cancelled
vault-ci / test (windows-latest) (push) Has been cancelled
vault-ci / app-core wiring tests (push) Has been cancelled
verify-patches / verify patches/CHECKSUMS.sha256 (push) Has been cancelled
Voice Benchmark Smoke / voice-emotion fixture smoke (push) Has been cancelled
Voice Benchmark Smoke / voiceagentbench fixture smoke (push) Has been cancelled
Voice Benchmark Smoke / voicebench-quality unit smoke (push) Has been cancelled
Voice Benchmark Smoke / voicebench TypeScript unit (no audio) (push) Has been cancelled
Voice Benchmark Smoke / voice bench smoke summary (push) Has been cancelled
Windows CI / windows ([bun run --cwd packages/app-core test bun run --cwd packages/elizaos test bun run --cwd packages/cloud/shared test], app-and-cli) (push) Has been cancelled
Windows CI / windows ([bun run --cwd packages/scenario-runner test bun run --cwd packages/vault test bun run --cwd packages/security test bun run --cwd plugins/plugin-coding-tools test], framework-packages) (push) Has been cancelled
Windows CI / windows ([bun run --cwd plugins/plugin-elizacloud test bun run --cwd plugins/plugin-discord test bun run --cwd plugins/plugin-anthropic test bun run --cwd plugins/plugin-openai test bun run --cwd plugins/plugin-app-control test bun run --cwd plugins/pl… (push) Has been cancelled
Windows CI / windows ([node packages/scripts/run-turbo.mjs run build --filter=@elizaos/core --filter=@elizaos/shared --filter=@elizaos/agent --concurrency=4 node packages/scripts/run-bash-linux-only.mjs scripts/verify-riscv64-buildpaths.sh node packages/scripts/run… (push) Has been cancelled
Windows CI / windows ([node packages/scripts/run-turbo.mjs run typecheck --filter=@elizaos/core --filter=@elizaos/shared --filter=@elizaos/cloud-shared --concurrency=4 bun run --cwd packages/core test bun run --cwd packages/shared test], core-runtime, 75) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:43:05 +08:00

736 lines
24 KiB
TypeScript

/**
* Canonical e2e coverage inventory.
*
* Two complementary surface inventories live here:
*
* 1. **Surface coverage matrix (issue #8802).** Enumerates, from real source,
* every surface that ships a behavioural effect a user can trigger — slash
* commands, pre-LLM shortcuts (#8791), plugin-declared HTTP routes, and
* views — then cross-checks each against the committed coverage manifest
* (`./manifest.ts`). A surface item is "covered" only when a real test
* artifact exists AND contains a declared signal string (the anti-larp
* check: a shape-only unit test that never names the real handler does not
* count). Items may instead be `exempt` with a written justification. This is
* the single source of truth for the ship-gate
* (`packages/scripts/__tests__/e2e-coverage.test.ts`) and the report CLI
* (`packages/scripts/e2e-coverage/write-coverage-matrix-report.ts`).
*
* 2. **Per-plugin keyless-e2e coverage (issue #8801).** For every checked-out
* plugin under `plugins/`, what agent surface it exposes (actions /
* connectors) and whether any keyless ("pr-deterministic") scenario
* exercises it. Consumed by the per-plugin coverage gate
* (`./check-e2e-coverage.ts`).
*
* Both inventories perform no network or runtime boot — they statically scan
* the plugin tree and (for the matrix) import only the dependency-light
* `getConnectorCommands` projection.
*/
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { commandShortcuts } from "../../../plugins/plugin-commands/src/actions/shortcuts.ts";
// Dependency-light: connector-catalog only imports ./registry + ./settings-sections
// + ./types (a type-only `@elizaos/core` import that erases at compile), so this
// pulls no runtime framework code.
import { getConnectorCommands } from "../../../plugins/plugin-commands/src/connector-catalog.ts";
import type { ManifestEntry } from "./manifest.ts";
import {
COMMAND_COVERAGE,
LARP_TEST_ARTIFACTS,
PLUGIN_ROUTE_COVERAGE,
SHORTCUT_COVERAGE,
SHORTCUT_REGISTRY_HINTS,
VIEW_COVERAGE_GATES,
} from "./manifest.ts";
export const REPO_ROOT = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"../../..",
);
const SKIP_DIRS = new Set([
"node_modules",
"dist",
".turbo",
"__tests__",
"__mocks__",
"test",
"tests",
"fixtures",
]);
function walkTsFiles(dir: string, out: string[] = []): string[] {
let entries: string[];
try {
entries = readdirSync(dir);
} catch {
return out;
}
for (const entry of entries) {
if (SKIP_DIRS.has(entry)) continue;
const full = path.join(dir, entry);
let st: ReturnType<typeof statSync>;
try {
st = statSync(full);
} catch {
continue;
}
if (st.isDirectory()) {
walkTsFiles(full, out);
} else if (
entry.endsWith(".ts") &&
!entry.endsWith(".d.ts") &&
!entry.endsWith(".test.ts")
) {
out.push(full);
}
}
return out;
}
/**
* A `routes:` property value is real route-wiring unless it is an empty array
* (`routes: []`) or a bare type annotation (`routes: Route[]`, used in
* interfaces/config types). Identifiers, spreads, array literals and factory
* calls are real wiring.
*/
function isRealRoutesWiring(rawValue: string): boolean {
const value = rawValue.trim().replace(/,$/, "").trim();
if (value === "") return false;
if (/^\[\s*\]$/.test(value)) return false; // routes: []
// routes: Route[] / routes: readonly LinearRoute[] — a type annotation.
if (/^(readonly\s+)?[A-Za-z_][A-Za-z0-9_]*\[\]$/.test(value)) return false;
return true;
}
export interface RoutePluginInfo {
plugin: string;
/** The matched `routes:` wiring value (for diagnostics). */
wiring: string;
}
/**
* Plugins whose exported `Plugin` object wires a non-empty `routes` array — the
* surfaces served in prod via `tryHandleRuntimePluginRoute`.
*/
export function discoverRoutePlugins(root = REPO_ROOT): RoutePluginInfo[] {
const pluginsDir = path.join(root, "plugins");
let dirs: string[];
try {
dirs = readdirSync(pluginsDir);
} catch {
return [];
}
const found: RoutePluginInfo[] = [];
for (const plugin of dirs) {
if (!plugin.startsWith("plugin-") && !plugin.startsWith("app-")) continue;
const src = path.join(pluginsDir, plugin, "src");
if (!existsSync(src)) continue;
let wiring: string | null = null;
for (const file of walkTsFiles(src)) {
const text = readFileSync(file, "utf8");
for (const line of text.split(/\r?\n/)) {
const match = line.match(/^\s*routes:\s*(.+?)\s*$/);
if (!match) continue;
if (isRealRoutesWiring(match[1])) {
wiring = match[1].trim();
break;
}
}
if (wiring) break;
}
if (wiring) found.push({ plugin, wiring });
}
return found.sort((a, b) => a.plugin.localeCompare(b.plugin));
}
/**
* Plugins under `plugins/` with no test file at all (the issue's "zero-test"
* list). Used to report them in the matrix; coverage/exemption is owned by the
* manifest.
*/
export function discoverZeroTestPlugins(root = REPO_ROOT): string[] {
const pluginsDir = path.join(root, "plugins");
let dirs: string[];
try {
dirs = readdirSync(pluginsDir);
} catch {
return [];
}
const zero: string[] = [];
for (const plugin of dirs) {
if (!plugin.startsWith("plugin-") && !plugin.startsWith("app-")) continue;
const dir = path.join(pluginsDir, plugin);
if (!statSafe(dir)?.isDirectory()) continue;
if (!isPluginInventoryDir(dir)) continue;
if (!hasAnyTestFile(dir)) zero.push(plugin);
}
return zero.sort();
}
function isPluginInventoryDir(dir: string): boolean {
return (
existsSync(path.join(dir, "package.json")) ||
existsSync(path.join(dir, "src")) ||
existsSync(path.join(dir, "bun.lock"))
);
}
function statSafe(p: string) {
try {
return statSync(p);
} catch {
return null;
}
}
function hasAnyTestFile(dir: string): boolean {
let entries: string[];
try {
entries = readdirSync(dir);
} catch {
return false;
}
for (const entry of entries) {
if (entry === "node_modules" || entry === "dist" || entry === ".turbo") {
continue;
}
const full = path.join(dir, entry);
const st = statSafe(full);
if (!st) continue;
if (st.isDirectory()) {
if (hasAnyTestFile(full)) return true;
} else if (/\.(test|spec|scenario)\.[cm]?tsx?$/.test(entry)) {
return true;
}
}
return false;
}
/** True when the #8791 pre-LLM shortcut registry exists in source yet. */
export function discoverShortcutRegistry(root = REPO_ROOT): string[] {
const hits: string[] = [];
for (const rel of SHORTCUT_REGISTRY_HINTS) {
if (existsSync(path.join(root, rel))) hits.push(rel);
}
return hits;
}
export interface ShortcutSurfaceInfo {
shortcutId: string;
alias: string;
targetKind: string;
targetName: string;
signal: string;
}
export function discoverCommandShortcutSurfaces(): ShortcutSurfaceInfo[] {
const surfaces: ShortcutSurfaceInfo[] = [];
for (const shortcut of commandShortcuts) {
const target =
shortcut.target.kind === "action"
? shortcut.target.name
: "path" in shortcut.target
? shortcut.target.path
: shortcut.target.kind;
for (const alias of shortcut.aliases ?? []) {
surfaces.push({
shortcutId: shortcut.id,
alias,
targetKind: shortcut.target.kind,
targetName: target,
signal: `${shortcut.id}:${alias}->${target}`,
});
}
}
return surfaces.sort((a, b) => a.signal.localeCompare(b.signal));
}
export interface CoverageResolution {
status: "covered" | "exempt" | "missing";
detail: string;
artifacts: string[];
/** Signals that were required but not found in any artifact (larp risk). */
missingSignals: string[];
}
/** Resolve a manifest entry against the filesystem with anti-larp signal checks. */
export function resolveCoverage(
entry: ManifestEntry | undefined,
root = REPO_ROOT,
): CoverageResolution {
if (!entry) {
return {
status: "missing",
detail: "no manifest entry",
artifacts: [],
missingSignals: [],
};
}
if (entry.status === "exempt") {
return {
status: "exempt",
detail: entry.reason,
artifacts: entry.artifacts ?? [],
missingSignals: [],
};
}
// covered — every artifact must exist; signals must each appear in ≥1 artifact.
const sources: Array<{ rel: string; text: string }> = [];
const missingFiles: string[] = [];
for (const rel of entry.artifacts) {
const full = path.join(root, rel);
if (!existsSync(full)) {
missingFiles.push(rel);
continue;
}
sources.push({ rel, text: readFileSync(full, "utf8") });
}
if (missingFiles.length > 0) {
return {
status: "missing",
detail: `covering artifact(s) not found: ${missingFiles.join(", ")}`,
artifacts: entry.artifacts,
missingSignals: [],
};
}
// Anti-larp: a covering artifact must not be a known shape-only unit test.
const larp = entry.artifacts.filter((rel) => LARP_TEST_ARTIFACTS.has(rel));
if (larp.length > 0) {
return {
status: "missing",
detail: `larp artifact(s) do not count as coverage: ${larp.join(", ")}`,
artifacts: entry.artifacts,
missingSignals: [],
};
}
const missingSignals = entry.signals.filter(
(signal) => !sources.some((s) => s.text.includes(signal)),
);
if (missingSignals.length > 0) {
return {
status: "missing",
detail: `required signal(s) absent from every artifact: ${missingSignals.join(", ")}`,
artifacts: entry.artifacts,
missingSignals,
};
}
return {
status: "covered",
detail: entry.note ?? `covered by ${entry.artifacts.length} artifact(s)`,
artifacts: entry.artifacts,
missingSignals: [],
};
}
export interface SurfaceItem {
id: string;
kind: "command" | "shortcut" | "view" | "plugin-route";
status: "covered" | "exempt" | "missing";
detail: string;
artifacts: string[];
/** Whether a gap on this item blocks CI (false = advisory, e.g. shortcuts). */
blocking: boolean;
meta?: Record<string, unknown>;
}
export interface CoverageMatrix {
schema: "eliza_e2e_coverage_matrix_v1";
generatedAt: string;
summary: {
commands: { total: number; covered: number };
shortcuts: { total: number; covered: number; gated: boolean };
pluginRoutes: { total: number; covered: number; exempt: number };
views: { gates: number };
blockingGaps: number;
advisoryGaps: number;
};
items: SurfaceItem[];
blockingGaps: SurfaceItem[];
advisoryGaps: SurfaceItem[];
}
/**
* Build the full coverage matrix. `generatedAt` is injected (not read from the
* clock) so callers control determinism of the emitted report.
*/
export function buildCoverageMatrix(options?: {
root?: string;
generatedAt?: string;
}): CoverageMatrix {
const root = options?.root ?? REPO_ROOT;
const generatedAt = options?.generatedAt ?? "1970-01-01T00:00:00.000Z";
const items: SurfaceItem[] = [];
// ── Slash commands ──────────────────────────────────────────────────────
// The served catalog is the source of truth; coverage is satisfied
// collectively by the full-catalog contract artifacts (which assert the exact
// served set == getConnectorCommands), plus the navigate/client/agent dispatch
// specs. We list each command for visibility but resolve them as one surface.
const commands = getConnectorCommands("gui");
const commandCoverage = resolveCoverage(COMMAND_COVERAGE, root);
let commandsCovered = 0;
for (const command of commands) {
const covered = commandCoverage.status === "covered";
if (covered) commandsCovered += 1;
items.push({
id: `command:${command.name}`,
kind: "command",
status: commandCoverage.status,
detail:
commandCoverage.status === "covered"
? `target=${command.target.kind}; ${commandCoverage.detail}`
: commandCoverage.detail,
artifacts: commandCoverage.artifacts,
blocking: true,
meta: { targetKind: command.target.kind },
});
}
// ── Shortcuts (#8791 — pre-LLM shortcut registry) ───────────────────────
// While the registry is absent the surface is gated (empty + advisory). Once
// #8791 lands at one of SHORTCUT_REGISTRY_HINTS it lights up and the gate
// requires shortcut coverage, resolved from SHORTCUT_COVERAGE against the real
// shortcut-gate e2e (runShortcutGate driving a real AgentRuntime).
const shortcutRegistry = discoverShortcutRegistry(root);
const shortcutSurfaces =
shortcutRegistry.length === 0 ? [] : discoverCommandShortcutSurfaces();
const shortcutsGated = shortcutRegistry.length === 0;
let shortcutsCovered = 0;
if (!shortcutsGated) {
for (const shortcut of shortcutSurfaces) {
const resolution = resolveCoverage(
SHORTCUT_COVERAGE.status === "covered"
? {
...SHORTCUT_COVERAGE,
signals: [...SHORTCUT_COVERAGE.signals, shortcut.signal],
}
: SHORTCUT_COVERAGE,
root,
);
if (resolution.status === "covered") shortcutsCovered += 1;
items.push({
id: `shortcut:${shortcut.signal}`,
kind: "shortcut",
status: resolution.status,
detail: `#8791 shortcut registry present (${shortcutRegistry.join(", ")}); alias=${shortcut.alias}; target=${shortcut.targetKind}:${shortcut.targetName}; ${resolution.detail}`,
artifacts: resolution.artifacts,
// A landed registry with no real e2e is a blocking gap (the contract:
// every shortcut alias/target has deterministic e2e evidence).
blocking: resolution.status !== "covered",
meta: {
registry: shortcutRegistry,
shortcutId: shortcut.shortcutId,
alias: shortcut.alias,
targetKind: shortcut.targetKind,
targetName: shortcut.targetName,
},
});
}
}
// ── Plugin routes ───────────────────────────────────────────────────────
const routePlugins = discoverRoutePlugins(root);
let routesCovered = 0;
let routesExempt = 0;
for (const { plugin, wiring } of routePlugins) {
const resolution = resolveCoverage(PLUGIN_ROUTE_COVERAGE[plugin], root);
if (resolution.status === "covered") routesCovered += 1;
if (resolution.status === "exempt") routesExempt += 1;
items.push({
id: `plugin-route:${plugin}`,
kind: "plugin-route",
status: resolution.status,
detail: resolution.detail,
artifacts: resolution.artifacts,
blocking: true,
meta: { wiring },
});
}
// ── Views (delegated to the existing view gates — not re-implemented here) ─
for (const gate of VIEW_COVERAGE_GATES) {
const exists = existsSync(path.join(root, gate));
items.push({
id: `view-gate:${gate}`,
kind: "view",
status: exists ? "covered" : "missing",
detail: exists
? "views covered by the existing view ship-gate (referenced, not re-implemented per #8796/#8797/#8798)"
: `expected view gate file is missing: ${gate}`,
artifacts: exists ? [gate] : [],
blocking: true,
});
}
const blockingGaps = items.filter(
(item) => item.blocking && item.status === "missing",
);
const advisoryGaps = items.filter(
(item) => !item.blocking && item.status === "missing",
);
return {
schema: "eliza_e2e_coverage_matrix_v1",
generatedAt,
summary: {
commands: { total: commands.length, covered: commandsCovered },
shortcuts: {
total: shortcutSurfaces.length,
covered: shortcutsCovered,
gated: shortcutsGated,
},
pluginRoutes: {
total: routePlugins.length,
covered: routesCovered,
exempt: routesExempt,
},
views: { gates: VIEW_COVERAGE_GATES.length },
blockingGaps: blockingGaps.length,
advisoryGaps: advisoryGaps.length,
},
items,
blockingGaps,
advisoryGaps,
};
}
// ───────────────────────────────────────────────────────────────────────────
// Per-plugin keyless-e2e coverage (issue #8801)
//
// "Keyless e2e" means a scenario that runs on a PR under the deterministic LLM
// proxy with zero credentials — i.e. a scenario in the
// `packages/scenario-runner/test/scenarios` deterministic corpus, or one in the
// big `packages/test/scenarios` corpus tagged `lane: "pr-deterministic"`. A
// plugin "has keyless e2e" when at least one such scenario names it in its
// `requires.plugins`. Detection is static (source read, no plugin import) so the
// inventory stays cheap and works even for plugins that cannot be imported under
// Node.
// ───────────────────────────────────────────────────────────────────────────
const PLUGINS_DIR = path.join(REPO_ROOT, "plugins");
/** Scenario corpora that run keyless on a PR. */
const KEYLESS_SCENARIO_ROOTS = [
path.join(REPO_ROOT, "packages", "scenario-runner", "test", "scenarios"),
path.join(REPO_ROOT, "packages", "test", "scenarios"),
];
/** The lane string a corpus scenario must declare to count as keyless. */
const KEYLESS_LANE = "pr-deterministic";
export interface PluginSurface {
/** Directory name, e.g. `plugin-discord`. */
dir: string;
/** Package name from package.json, e.g. `@elizaos/plugin-discord`. */
packageName: string;
/** True when the plugin wires an agent action surface. */
hasActions: boolean;
/** True when the plugin implements/registers a message connector. */
hasConnector: boolean;
}
export interface PluginCoverage extends PluginSurface {
/** Scenario ids (keyless) that name this plugin in `requires.plugins`. */
keylessScenarioIds: string[];
/** True when the plugin exposes an action/connector surface. */
hasSurface: boolean;
/** True when the plugin has a keyless scenario for its surface. */
hasKeylessE2e: boolean;
}
function listDirs(root: string): string[] {
if (!existsSync(root)) return [];
return readdirSync(root)
.filter((entry) => {
const full = path.join(root, entry);
return statSync(full).isDirectory();
})
.sort();
}
function readPackageName(pluginDir: string): string | null {
const pkgPath = path.join(pluginDir, "package.json");
if (!existsSync(pkgPath)) return null;
const parsed = JSON.parse(readFileSync(pkgPath, "utf8")) as {
name?: unknown;
};
return typeof parsed.name === "string" ? parsed.name : null;
}
function readSourceFiles(srcDir: string): string[] {
const out: string[] = [];
const walk = (dir: string): void => {
for (const entry of readdirSync(dir)) {
if (
entry === "node_modules" ||
entry === "dist" ||
entry === "__tests__" ||
entry === "test" ||
entry === "tests"
) {
continue;
}
const full = path.join(dir, entry);
const st = statSync(full);
if (st.isDirectory()) {
walk(full);
} else if (
entry.endsWith(".ts") &&
!entry.endsWith(".test.ts") &&
!entry.endsWith(".d.ts")
) {
out.push(full);
}
}
};
walk(srcDir);
return out;
}
/** A plugin wires actions when its barrel declares a non-empty `actions:`. */
function detectActions(srcDir: string): boolean {
if (existsSync(path.join(srcDir, "actions"))) return true;
for (const indexName of ["index.ts", "plugin.ts"]) {
const indexPath = path.join(srcDir, indexName);
if (!existsSync(indexPath)) continue;
const source = readFileSync(indexPath, "utf8");
// Match `actions: [ ... ]` with at least one entry inside the brackets.
if (/\bactions\s*:\s*\[\s*[^\]\s]/.test(source)) return true;
}
return false;
}
/** A plugin is a connector when it implements/registers a message connector. */
function detectConnector(srcFiles: string[]): boolean {
const markers = [
/\bimplements\s+MessageConnector\b/,
/\bsatisfies\s+MessageConnector\b/,
/:\s*MessageConnector\b/,
/\bregisterConnector\s*\(/,
/\bregisterMessageConnector\s*\(/,
];
for (const file of srcFiles) {
const source = readFileSync(file, "utf8");
if (markers.some((marker) => marker.test(source))) return true;
}
return false;
}
export function inventoryPluginSurfaces(): PluginSurface[] {
const surfaces: PluginSurface[] = [];
for (const dir of listDirs(PLUGINS_DIR)) {
if (!dir.startsWith("plugin-")) continue;
const pluginDir = path.join(PLUGINS_DIR, dir);
const srcDir = path.join(pluginDir, "src");
// Submodules that are not checked out have no `src/` — skip them; the gate
// can only reason about plugins whose source is present in this tree.
if (!existsSync(srcDir)) continue;
const packageName = readPackageName(pluginDir);
if (!packageName) continue;
const srcFiles = readSourceFiles(srcDir);
surfaces.push({
dir,
packageName,
hasActions: detectActions(srcDir),
hasConnector: detectConnector(srcFiles),
});
}
return surfaces;
}
interface ScenarioRequire {
id: string;
requiredPlugins: string[];
}
function readStaticScenario(file: string): ScenarioRequire | null {
const source = readFileSync(file, "utf8");
const idMatch = source.match(/\bid\s*:\s*["'`]([^"'`]+)["'`]/);
if (!idMatch) return null;
const id = idMatch[1];
// A scenario counts as keyless if it declares the keyless lane OR it lives in
// the deterministic corpus (those run keyless by construction).
const declaresKeylessLane = new RegExp(
`\\blane\\s*:\\s*["'\`]${KEYLESS_LANE}["'\`]`,
).test(source);
const isDeterministicCorpus = file.includes(
`${path.sep}scenario-runner${path.sep}test${path.sep}scenarios${path.sep}`,
);
if (!declaresKeylessLane && !isDeterministicCorpus) return null;
const requiredPlugins: string[] = [];
const requiresMatch = source.match(
/requires\s*:\s*{[\s\S]*?plugins\s*:\s*\[([\s\S]*?)\]/,
);
if (requiresMatch) {
for (const m of requiresMatch[1].matchAll(/["'`]([^"'`]+)["'`]/g)) {
requiredPlugins.push(m[1]);
}
}
return { id, requiredPlugins };
}
function discoverScenarioFiles(root: string): string[] {
if (!existsSync(root)) return [];
const out: string[] = [];
const walk = (dir: string): void => {
for (const entry of readdirSync(dir)) {
if (entry.startsWith("_") || entry === "node_modules") continue;
const full = path.join(dir, entry);
const st = statSync(full);
if (st.isDirectory()) {
walk(full);
} else if (entry.endsWith(".scenario.ts")) {
out.push(full);
}
}
};
walk(root);
return out;
}
/** Map of package-name -> keyless scenario ids that require it. */
export function keylessScenariosByPlugin(): Map<string, string[]> {
const byPlugin = new Map<string, string[]>();
for (const root of KEYLESS_SCENARIO_ROOTS) {
for (const file of discoverScenarioFiles(root)) {
const scenario = readStaticScenario(file);
if (!scenario) continue;
for (const pluginRef of scenario.requiredPlugins) {
const existing = byPlugin.get(pluginRef) ?? [];
existing.push(scenario.id);
byPlugin.set(pluginRef, existing);
}
}
}
return byPlugin;
}
export function buildPluginCoverage(): PluginCoverage[] {
const surfaces = inventoryPluginSurfaces();
const byPlugin = keylessScenariosByPlugin();
return surfaces.map((surface) => {
// Scenarios may reference a plugin by package name or by short name (the
// `requires.plugins` field accepts both styles across the corpus).
const shortName = surface.dir;
const altNames = [
surface.packageName,
shortName,
shortName.replace(/^plugin-/, ""),
];
const keylessScenarioIds = [
...new Set(altNames.flatMap((name) => byPlugin.get(name) ?? [])),
].sort();
const hasSurface = surface.hasActions || surface.hasConnector;
return {
...surface,
keylessScenarioIds,
hasSurface,
hasKeylessE2e: keylessScenarioIds.length > 0,
};
});
}