elizaos--eliza/plugins/plugin-benchmarks/__tests__/plugin.test.ts

/**
 * Deterministic vitest suite over the assembled `benchmarksPlugin`: asserts the
 * default/named export identity, the total action count (umbrellas plus
 * promoted virtuals), and the expected umbrella names. No runtime or model.
 */
import { describe, expect, it } from "vitest";

import benchmarksPlugin, {
  benchmarksPlugin as namedPlugin,
  osworldAction,
  tauBenchToolAction,
  vendingMachineAction,
  visualWebBenchTaskAction,
  webshopAction,
} from "../src/index";

describe("@elizaos/plugin-benchmarks", () => {
  it("exports the same plugin via default and named export", () => {
    expect(benchmarksPlugin).toBe(namedPlugin);
    expect(benchmarksPlugin.name).toBe("benchmarks");
  });

  it("registers every promoted virtual action plus the umbrellas", () => {
    // 1+9 vending + 1+5 webshop + 1+11 osworld + 1 tau-bench + 1+7 visualwebbench = 37
    expect(benchmarksPlugin.actions).toBeDefined();
    expect(benchmarksPlugin.actions?.length).toBe(37);
  });

  it("includes the expected umbrella names", () => {
    const names = (benchmarksPlugin.actions ?? []).map((action) => action.name);
    expect(names).toContain("VENDING_MACHINE");
    expect(names).toContain("WEBSHOP");
    expect(names).toContain("OSWORLD");
    expect(names).toContain("TAU_BENCH_TOOL");
    expect(names).toContain("VISUALWEBBENCH_TASK");
  });

  it("includes representative promoted virtuals", () => {
    const names = new Set((benchmarksPlugin.actions ?? []).map((action) => action.name));
    expect(names.has("VENDING_MACHINE_VIEW_STATE")).toBe(true);
    expect(names.has("VENDING_MACHINE_PLACE_ORDER")).toBe(true);
    expect(names.has("VENDING_MACHINE_ADVANCE_DAY")).toBe(true);
    expect(names.has("WEBSHOP_SEARCH")).toBe(true);
    expect(names.has("WEBSHOP_BUY")).toBe(true);
    expect(names.has("OSWORLD_CLICK")).toBe(true);
    expect(names.has("OSWORLD_SCREENSHOT")).toBe(true);
    expect(names.has("OSWORLD_DONE")).toBe(true);
    expect(names.has("VISUALWEBBENCH_TASK_ACTION_PREDICTION")).toBe(true);
    expect(names.has("VISUALWEBBENCH_TASK_ACTION_GROUND")).toBe(true);
  });

  it("does not promote virtuals for tau-bench (no enum on tool_name)", () => {
    const names = (benchmarksPlugin.actions ?? []).map((action) => action.name);
    const tauVirtuals = names.filter(
      (name) => name.startsWith("TAU_BENCH_TOOL_") && name !== "TAU_BENCH_TOOL"
    );
    expect(tauVirtuals).toHaveLength(0);
  });

  it("attaches subActions to umbrellas after promotion", () => {
    expect(vendingMachineAction.subActions?.length).toBe(9);
    expect(webshopAction.subActions?.length).toBe(5);
    expect(osworldAction.subActions?.length).toBe(11);
    expect(visualWebBenchTaskAction.subActions?.length).toBe(7);
    expect(tauBenchToolAction.subActions ?? []).toHaveLength(0);
  });

  it("each registered action has a non-empty description and a handler function", () => {
    for (const action of benchmarksPlugin.actions ?? []) {
      expect(typeof action.description).toBe("string");
      expect(action.description.length).toBeGreaterThan(0);
      expect(typeof action.handler).toBe("function");
      expect(typeof action.validate).toBe("function");
    }
  });
});