chore: import upstream snapshot with attribution

showcase/harness/scripts/d6-capture-references.ts

@@ -0,0 +1,297 @@
+#!/usr/bin/env tsx
+/**
+ * D6 — reference-snapshot capture CLI.
+ *
+ * Captures `ParitySnapshot` JSON files for every `D5FeatureType` against
+ * the LangGraph-Python (LGP) showcase — the canonical implementation the
+ * D6 parity probe grades every other showcase against. Snapshots land
+ * under `showcase/harness/fixtures/d6-reference/<featureType>.json`.
+ *
+ * When to run:
+ *   - Weekly cadence (matches the D6 probe's Monday rotation),
+ *   - After an LGP showcase redeploy,
+ *   - After a D5 fixture update (`fixtures/d5/*.json`),
+ *   - After a parity-tolerances change in `parity-compare.ts`,
+ *   - After an ag-ui protocol bump.
+ *
+ * Usage:
+ *   tsx scripts/d6-capture-references.ts
+ *   tsx scripts/d6-capture-references.ts --integration langgraph-python
+ *   tsx scripts/d6-capture-references.ts --base-url https://langgraph-python.up.railway.app
+ *   tsx scripts/d6-capture-references.ts --feature agentic-chat
+ *
+ * Env overrides (`--flag` wins over env when both are set):
+ *   LGP_BASE_URL          base URL of the LGP showcase (required if
+ *                         `--base-url` not passed)
+ *   D6_REFERENCE_DIR      override output directory (defaults to
+ *                         `<package>/fixtures/d6-reference`)
+ *
+ * Exit codes:
+ *   0   every result was `captured` or `skipped` (no failures)
+ *   1   any result was `failed`, OR a required argument was missing,
+ *       OR an unexpected error occurred during capture orchestration.
+ */
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import type { Page as PlaywrightPage } from "playwright";
+import { findPackageRoot } from "../src/probes/helpers/package-root.js";
+import {
+  captureAllReferences,
+  captureReferenceForFeature,
+  defaultWriteSnapshot,
+  serializeRelevantDom,
+} from "../src/probes/helpers/reference-capture.js";
+import type {
+  ReferenceCaptureBrowserHandle,
+  ReferenceCaptureContext,
+  ReferenceCaptureDeps,
+  ReferenceCapturePage,
+  ReferenceCaptureResult,
+} from "../src/probes/helpers/reference-capture.js";
+import { attachSseInterceptor } from "../src/probes/helpers/sse-interceptor.js";
+import type { SseInterceptorHandle } from "../src/probes/helpers/sse-interceptor.js";
+import { runConversation } from "../src/probes/helpers/conversation-runner.js";
+import type {
+  ConversationTurn,
+  Page as RunnerPage,
+} from "../src/probes/helpers/conversation-runner.js";
+import {
+  D5_REGISTRY,
+  isD5FeatureType,
+} from "../src/probes/helpers/d5-registry.js";
+import type { D5FeatureType } from "../src/probes/helpers/d5-registry.js";
+import { defaultScriptLoader } from "../src/probes/drivers/d6-all-pills.js";
+import type { ProbeContext } from "../src/types/index.js";
+
+interface Args {
+  integration: string;
+  baseUrl: string;
+  feature?: D5FeatureType;
+  outputDir: string;
+}
+
+function parseArgs(argv: string[]): Args {
+  const flag = (name: string): string | undefined => {
+    const idx = argv.indexOf(`--${name}`);
+    if (idx === -1) return undefined;
+    const next = argv[idx + 1];
+    if (next === undefined || next.startsWith("--")) {
+      throw new Error(`flag --${name} requires a value`);
+    }
+    return next;
+  };
+
+  const integration = flag("integration") ?? "langgraph-python";
+  const baseUrl = flag("base-url") ?? process.env.LGP_BASE_URL;
+  if (!baseUrl) {
+    throw new Error(
+      "missing --base-url (or set LGP_BASE_URL env). Example: " +
+        "--base-url https://langgraph-python.up.railway.app",
+    );
+  }
+  const featureRaw = flag("feature");
+  let feature: D5FeatureType | undefined;
+  if (featureRaw !== undefined) {
+    if (!isD5FeatureType(featureRaw)) {
+      const known = [...D5_REGISTRY.keys()].sort().join(", ");
+      throw new Error(
+        `unknown --feature "${featureRaw}". Known: ${known || "(registry empty)"}`,
+      );
+    }
+    if (!D5_REGISTRY.has(featureRaw)) {
+      const known = [...D5_REGISTRY.keys()].sort().join(", ");
+      throw new Error(
+        `unknown --feature "${featureRaw}". Known: ${known || "(registry empty)"}`,
+      );
+    }
+    feature = featureRaw;
+  }
+  const outputDir =
+    process.env.D6_REFERENCE_DIR ??
+    path.resolve(resolvePackageRoot(), "fixtures", "d6-reference");
+
+  return { integration, baseUrl, feature, outputDir };
+}
+
+/**
+ * Resolve the showcase-harness package root via `findPackageRoot`. Handles
+ * both source and compiled locations:
+ *
+ *   - source:    `<package>/scripts/d6-capture-references.ts`  → `<package>`
+ *   - compiled:  `<package>/dist/scripts/d6-capture-references.js` → `<package>`
+ *
+ * Without this, `path.dirname(import.meta.url) + ".."` resolves to
+ * `<package>` from source but `<package>/dist` after build, so the
+ * default `outputDir` becomes `<package>/dist/fixtures/d6-reference`
+ * — wrong destination. The shared helper is source/dist-symmetric.
+ */
+function resolvePackageRoot(): string {
+  return findPackageRoot(path.dirname(fileURLToPath(import.meta.url)));
+}
+
+/**
+ * Build the production dependency surface. Mirrors the wiring the D6
+ * driver uses — same launcher args, same interceptor, same DOM
+ * serializer, same disk writer. Kept inline (not factored) because the
+ * driver and the CLI have slightly different page-shape needs (the
+ * driver carries a `asPlaywrightPage()` shim; the CLI hands the raw
+ * Playwright page straight through), and a shared helper would need
+ * conditional branches that obscure each call site.
+ */
+async function buildDeps(): Promise<ReferenceCaptureDeps> {
+  const mod = (await import("playwright")) as typeof import("playwright");
+
+  return {
+    launchBrowser: async (): Promise<ReferenceCaptureBrowserHandle> => {
+      const browser = await mod.chromium.launch({
+        headless: true,
+        args: ["--no-sandbox", "--disable-dev-shm-usage"],
+      });
+      const ctx = await browser.newContext();
+      const page = await ctx.newPage();
+      // Playwright's Page satisfies ReferenceCapturePage structurally;
+      // cast through unknown because the structural compat isn't visible
+      // through Playwright's overloads.
+      const wrapped = page as unknown as ReferenceCapturePage;
+      return {
+        page: wrapped,
+        close: async () => {
+          await ctx.close();
+          await browser.close();
+        },
+      };
+    },
+    attachSseInterceptor: (page) =>
+      attachSseInterceptor(page as unknown as PlaywrightPage),
+    runConversation: (page: ReferenceCapturePage, turns: ConversationTurn[]) =>
+      runConversation(page as RunnerPage, turns),
+    serializeDom: (page) => serializeRelevantDom(page),
+    writeSnapshot: defaultWriteSnapshot,
+    warn: (message, extra) => {
+      const line = `[d6-capture] WARN ${message}`;
+      if (extra) {
+        // eslint-disable-next-line no-console
+        console.warn(line, extra);
+      } else {
+        // eslint-disable-next-line no-console
+        console.warn(line);
+      }
+    },
+  };
+}
+
+function summaryLine(r: ReferenceCaptureResult): string {
+  const tail =
+    r.status === "captured"
+      ? (r.snapshotPath ?? "(no snapshotPath)")
+      : (r.reason ?? "(no reason)");
+  return `[d6-capture] ${r.featureType} → ${r.status} (${tail})`;
+}
+
+/**
+ * Build a minimal `ProbeContext` for the script loader. The CLI does
+ * not have an invoker-level logger / writer / abort signal — it just
+ * needs `logger` for the loader's warn/error reporting.
+ */
+function buildLoaderCtx(): ProbeContext {
+  return {
+    now: () => new Date(),
+    env: process.env,
+    logger: {
+      // eslint-disable-next-line no-console
+      info: (msg, extra) => console.log(`[d6-capture] ${msg}`, extra ?? ""),
+      // eslint-disable-next-line no-console
+      warn: (msg, extra) =>
+        console.warn(`[d6-capture] WARN ${msg}`, extra ?? ""),
+      // eslint-disable-next-line no-console
+      error: (msg, extra) =>
+        console.error(`[d6-capture] ERROR ${msg}`, extra ?? ""),
+      // eslint-disable-next-line no-console
+      debug: (msg, extra) =>
+        console.log(`[d6-capture] DEBUG ${msg}`, extra ?? ""),
+    },
+  };
+}
+
+async function main(): Promise<number> {
+  // Populate the D5 registry by scanning `src/probes/scripts/` and
+  // importing each `d5-*.ts` file (each one calls `registerD5Script(...)`
+  // as a top-level side effect). Without this, `D5_REGISTRY` would be
+  // empty when `parseArgs` validates `--feature` and when
+  // `captureAllReferences` enumerates known features — silently producing
+  // a "no features captured" run that looks like success.
+  try {
+    await defaultScriptLoader(buildLoaderCtx());
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error(
+      `[d6-capture] script loader failed: ${(err as Error).message}`,
+    );
+    return 1;
+  }
+  if (D5_REGISTRY.size === 0) {
+    // eslint-disable-next-line no-console
+    console.error(
+      "[d6-capture] D5_REGISTRY is empty after script loader ran — " +
+        "check that `src/probes/scripts/d5-*.ts` exist and call " +
+        "`registerD5Script(...)` at module top-level.",
+    );
+    return 1;
+  }
+
+  let args: Args;
+  try {
+    args = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error(`[d6-capture] ${(err as Error).message}`);
+    return 1;
+  }
+
+  // eslint-disable-next-line no-console
+  console.log(
+    `[d6-capture] integration=${args.integration} baseUrl=${args.baseUrl} ` +
+      `feature=${args.feature ?? "ALL"} outputDir=${args.outputDir}`,
+  );
+
+  const ctx: ReferenceCaptureContext = {
+    baseUrl: args.baseUrl,
+    integrationSlug: args.integration,
+    outputDir: args.outputDir,
+  };
+
+  const deps = await buildDeps();
+
+  let results: ReferenceCaptureResult[];
+  if (args.feature) {
+    results = [await captureReferenceForFeature(args.feature, ctx, deps)];
+  } else {
+    results = await captureAllReferences(ctx, deps);
+  }
+
+  for (const r of results) {
+    // eslint-disable-next-line no-console
+    console.log(summaryLine(r));
+  }
+
+  const failed = results.filter((r) => r.status === "failed");
+  const captured = results.filter((r) => r.status === "captured").length;
+  const skipped = results.filter((r) => r.status === "skipped").length;
+  // eslint-disable-next-line no-console
+  console.log(
+    `[d6-capture] done: captured=${captured} skipped=${skipped} ` +
+      `failed=${failed.length}`,
+  );
+  return failed.length === 0 ? 0 : 1;
+}
+
+main()
+  .then((code) => {
+    process.exit(code);
+  })
+  .catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error("[d6-capture] orchestration error:", err);
+    process.exit(1);
+  });

showcase/harness/scripts/load-test.ts

@@ -0,0 +1,128 @@
+#!/usr/bin/env tsx
+/**
+ * Load test simulating 50 keys × 5-minute cadence against a local or
+ * deployed showcase-harness (spec §9 Phase 5).
+ *
+ * For each iteration (default: 3), the script fires a burst of 50
+ * requests to each probed endpoint, records per-request latency, and
+ * prints a summary table of p50/p95/p99 per endpoint. Exit code 0 on
+ * success; non-zero when any percentile breaches a configurable
+ * threshold (see LOAD_TEST_MAX_MS, default 5000).
+ *
+ * Usage:
+ *   tsx scripts/load-test.ts --url https://showcase-harness.railway.app
+ *
+ * Env overrides:
+ *   LOAD_TEST_URL        (alias for --url)
+ *   LOAD_TEST_KEYS       number of simulated keys per burst (default 50)
+ *   LOAD_TEST_ITERATIONS number of bursts (default 3)
+ *   LOAD_TEST_MAX_MS     fail if p99 exceeds this (default 5000)
+ */
+
+interface EndpointSpec {
+  label: string;
+  path: string;
+  method?: "GET" | "POST";
+  body?: () => string;
+  headers?: Record<string, string>;
+}
+
+const ENDPOINTS: EndpointSpec[] = [
+  { label: "GET /health", path: "/health" },
+  { label: "GET /metrics", path: "/metrics" },
+];
+
+const args = process.argv.slice(2);
+const urlFlagIdx = args.indexOf("--url");
+const url =
+  (urlFlagIdx !== -1 ? args[urlFlagIdx + 1] : undefined) ??
+  process.env.LOAD_TEST_URL ??
+  "http://localhost:8080";
+const keys = Number(process.env.LOAD_TEST_KEYS ?? "50");
+const iterations = Number(process.env.LOAD_TEST_ITERATIONS ?? "3");
+const maxMs = Number(process.env.LOAD_TEST_MAX_MS ?? "5000");
+
+/**
+ * Nearest-rank percentile. Note `p=1.0` returns the last element (max), which
+ * is expected behavior for small n: with the default 50 requests/iteration,
+ * p99 effectively degenerates to the max. If that ambiguity matters, pass a
+ * larger LOAD_TEST_KEYS to get a stable p99.
+ */
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
+  return sorted[idx]!;
+}
+
+async function measure(spec: EndpointSpec): Promise<number> {
+  const start = Date.now();
+  const res = await fetch(`${url}${spec.path}`, {
+    method: spec.method ?? "GET",
+    body: spec.body?.(),
+    headers: spec.headers,
+  });
+  // Drain response so the measurement includes body transfer.
+  await res.text();
+  if (!res.ok && res.status !== 404) {
+    throw new Error(`${spec.label} → HTTP ${res.status}`);
+  }
+  // 404 on /metrics is not fatal (some deploys disable the endpoint) but it
+  // IS operationally visible — warn so operators notice if they expected
+  // metrics to be enabled.
+  if (res.status === 404) {
+    console.warn(
+      `WARN: ${spec.label} returned 404 — endpoint disabled on this deploy?`,
+    );
+  }
+  return Date.now() - start;
+}
+
+async function runBurst(spec: EndpointSpec, count: number): Promise<number[]> {
+  const tasks: Promise<number>[] = [];
+  for (let i = 0; i < count; i++) {
+    tasks.push(measure(spec));
+  }
+  return Promise.all(tasks);
+}
+
+async function main(): Promise<void> {
+  console.log(
+    `load-test against ${url}: ${iterations} iterations × ${keys} keys`,
+  );
+  const perEndpoint = new Map<string, number[]>();
+  for (const ep of ENDPOINTS) perEndpoint.set(ep.label, []);
+
+  for (let i = 0; i < iterations; i++) {
+    for (const ep of ENDPOINTS) {
+      const timings = await runBurst(ep, keys);
+      perEndpoint.get(ep.label)!.push(...timings);
+      console.log(
+        `  iter ${i + 1}/${iterations} ${ep.label}: ${timings.length} requests, min=${Math.min(...timings)}ms max=${Math.max(...timings)}ms`,
+      );
+    }
+  }
+
+  let failed = false;
+  console.log("\nper-endpoint latency percentiles (ms):");
+  console.log("endpoint                         p50    p95    p99    n");
+  console.log("-------------------------------- ------ ------ ------ -----");
+  for (const [label, timings] of perEndpoint) {
+    const sorted = [...timings].sort((a, b) => a - b);
+    const p50 = percentile(sorted, 0.5);
+    const p95 = percentile(sorted, 0.95);
+    const p99 = percentile(sorted, 0.99);
+    console.log(
+      `${label.padEnd(32)} ${String(p50).padStart(6)} ${String(p95).padStart(6)} ${String(p99).padStart(6)} ${String(sorted.length).padStart(5)}`,
+    );
+    if (p99 > maxMs) {
+      console.error(`FAIL: ${label} p99=${p99}ms exceeds threshold ${maxMs}ms`);
+      failed = true;
+    }
+  }
+  if (failed) process.exit(1);
+}
+
+main().catch((err) => {
+  console.error("load-test crashed:", err);
+  process.exit(2);
+});

showcase/harness/scripts/test-notify-harness-jq.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Guards the notify-harness per-job partition jq expression against
+# service-name collisions.
+#
+# The showcase_deploy.yml `notify-harness` step maps each service name in
+# $SERVICES to the matrix build job that ran for it, then partitions into
+# FAILED / SUCCEEDED. Matrix job names render as
+# "build (<dispatch_name>, <context>, <image>, ...)". The previous
+# `contains($svc)` matcher produced false positives when one service name
+# was a substring of another. The fixed matcher uses a token-bounded
+# prefix `startswith("build (" + $svc + ",")`.
+#
+# This script replays a scenario where `agno` (failed) and `ag2`
+# (succeeded) coexist — verifying that the partition matcher attributes
+# each correctly without substring confusion.
+#
+# Usage: bash showcase/harness/scripts/test-notify-harness-jq.sh
+# Exits non-zero on regression.
+
+set -euo pipefail
+
+# Fixture matching the real GitHub Actions jobs API shape: a single build
+# job per matrix leg, named `build (<all object fields, comma-separated>)`.
+BUILD_JOBS='[
+  {"name":"build (agno, showcase/integrations/agno, showcase-agno, 32cab80b, 15, false, , , /api/health)","conclusion":"failure"},
+  {"name":"build (ag2, showcase/integrations/ag2, showcase-ag2, 4a37481b, 15, false, , , /api/health)","conclusion":"success"},
+  {"name":"build (mastra, showcase/integrations/mastra, showcase-mastra, d7979eb7, 15, false, , , /api/health)","conclusion":"success"}
+]'
+
+SERVICES='["agno","ag2","mastra"]'
+
+# Fixed matcher — mirrors the jq inside showcase_deploy.yml.
+FAILED=$(echo "$SERVICES" | jq -c --argjson jobs "$BUILD_JOBS" '
+  [
+    .[] as $svc
+    | $jobs[]
+    | select((.name // "") as $n | ($n | startswith("build (" + $svc + ",")) or $n == ("build (" + $svc + ")"))
+    | select(.conclusion == "failure")
+    | $svc
+  ] | unique
+')
+
+SUCCEEDED=$(echo "$SERVICES" | jq -c --argjson jobs "$BUILD_JOBS" '
+  [
+    .[] as $svc
+    | $jobs[]
+    | select((.name // "") as $n | ($n | startswith("build (" + $svc + ",")) or $n == ("build (" + $svc + ")"))
+    | select(.conclusion == "success")
+    | $svc
+  ] | unique
+')
+
+EXPECTED_FAILED='["agno"]'
+EXPECTED_SUCCEEDED='["ag2","mastra"]'
+
+fail=0
+if [ "$FAILED" != "$EXPECTED_FAILED" ]; then
+  echo "FAIL: FAILED mismatch"
+  echo "  expected: $EXPECTED_FAILED"
+  echo "  got:      $FAILED"
+  fail=1
+fi
+if [ "$SUCCEEDED" != "$EXPECTED_SUCCEEDED" ]; then
+  echo "FAIL: SUCCEEDED mismatch"
+  echo "  expected: $EXPECTED_SUCCEEDED"
+  echo "  got:      $SUCCEEDED"
+  fail=1
+fi
+
+if [ $fail -ne 0 ]; then
+  exit 1
+fi
+
+echo "PASS: notify-harness jq partition is collision-free"
+echo "  FAILED:    $FAILED"
+echo "  SUCCEEDED: $SUCCEEDED"