elizaos--eliza/plugins/plugin-browser/scripts/capture-web-grounding-evidence.mjs

#!/usr/bin/env node
/**
 * Evidence capture for the web-element grounding lane (#10333).
 *
 * Renders each ScreenSpot-Web-style task in a real Chromium, overlays the
 * target's TRUE on-screen bbox (orange outline) and the grounder's predicted
 * click point (cyan dot), screenshots it, and writes a scorecard. Artifacts land
 * under `test-results/evidence/10333-web-grounding/`.
 *
 * Usage (from repo root):  bun plugins/plugin-browser/scripts/capture-web-grounding-evidence.mjs
 */
import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(here, "..", "..", "..");
const OUT =
  process.env.OUT ||
  join(repoRoot, "test-results", "evidence", "10333-web-grounding");
mkdirSync(OUT, { recursive: true });

const { createChromiumBenchmarkEngine } = await import(
  "../src/benchmark/chromium-executor.ts"
);
const { scoreWebGrounding, centerGrounder, WEB_GROUNDING_TASKS } = await import(
  "../src/benchmark/web-grounding.ts"
);

const engine = await createChromiumBenchmarkEngine({ headless: true });
console.log(`[grounding] engine: ${engine.executablePath}`);

let i = 0;
const captured = [];
const score = await scoreWebGrounding(
  engine,
  WEB_GROUNDING_TASKS,
  centerGrounder,
  async ({ task, box, predicted, correct }) => {
    i += 1;
    const page = engine.currentPage();
    if (!page) return;
    const tag = `${String(i).padStart(2, "0")}-${task.id}`;
    console.log(`[grounding] ${tag}: overlay`);
    // Overlay the ground-truth bbox + the predicted point, then screenshot.
    await page.evaluate(
      ({ box, predicted }) => {
        const mk = (style) => {
          const d = document.createElement("div");
          d.style.cssText = style;
          d.style.position = "fixed";
          d.style.zIndex = "999999";
          d.style.pointerEvents = "none";
          document.body.appendChild(d);
        };
        mk(
          `left:${box.x}px;top:${box.y}px;width:${box.width}px;height:${box.height}px;` +
            `border:3px solid #ff5800;border-radius:6px;box-shadow:0 0 0 2px rgba(255,88,0,.3)`,
        );
        if (predicted) {
          mk(
            `left:${predicted.x - 7}px;top:${predicted.y - 7}px;width:14px;height:14px;` +
              `background:#34d6ff;border:2px solid #001018;border-radius:50%`,
          );
        }
      },
      { box, predicted },
    );
    console.log(`[grounding] ${tag}: screenshot`);
    // Puppeteer's screenshot path can stall when request interception is still
    // enabled on some Chromium builds. The page is already fully rendered and
    // overlaid, and this executor page is disposed immediately after the sample.
    await page.setRequestInterception(false).catch(() => {});
    await page.screenshot({ path: join(OUT, `${tag}.png`) });
    captured.push({
      tag,
      task: task.id,
      group: task.group,
      correct,
      box,
      predicted,
    });
    console.log(
      `[grounding] ${tag}: ${correct ? "HIT" : "MISS"} — point (${Math.round(predicted.x)},${Math.round(predicted.y)}) in bbox [${Math.round(box.x)},${Math.round(box.y)},${Math.round(box.width)},${Math.round(box.height)}]`,
    );
  },
);

await engine.close();

const report = {
  benchmark: "web-element-grounding (ScreenSpot-Web style)",
  engine: "chromium",
  executablePath: engine.executablePath,
  grounder: "center",
  total: score.total,
  correct: score.correct,
  accuracy: score.accuracy,
  byGroup: score.byGroup,
  samples: captured,
};
writeFileSync(
  join(OUT, "scorecard.json"),
  `${JSON.stringify(report, null, 2)}\n`,
);
console.log(
  `\n[grounding] accuracy ${score.correct}/${score.total} (${score.accuracy}) → ${OUT}`,
);
process.exit(score.accuracy === 1 ? 0 : 1);