elizaos--eliza/packages/scripts/__tests__/trajectory-validate.test.ts

// Exercises tests trajectory validate.test automation behavior with deterministic script fixtures.
import { describe, expect, test } from "bun:test";
import { computeCallCostUsd } from "../lib/cost-table";
import {
  compareTrajectories,
  deriveTrajectoryEvidenceGrade,
  type RecordedStage,
  type RecordedTrajectory,
  validateTrajectory,
  validateTrajectoryJsonReport,
  validateTrajectoryMarkdownReport,
} from "../lib/trajectory-validate";

function modelStage(args: {
  stageId: string;
  kind: "messageHandler" | "planner" | "evaluation";
  startedAt: number;
  endedAt: number;
  prompt: string;
  response: string;
  tools?: Array<{ name: string; description: string }>;
  toolCalls?: Array<{
    id: string;
    name: string;
    args: Record<string, unknown>;
  }>;
  usage: {
    promptTokens: number;
    completionTokens: number;
    cacheReadInputTokens?: number;
    cacheCreationInputTokens?: number;
    totalTokens: number;
  };
  evaluation?: { success: boolean; decision: string; thought?: string };
  iteration?: number;
  cache?: RecordedStage["cache"];
}): RecordedStage {
  return {
    stageId: args.stageId,
    kind: args.kind,
    iteration: args.iteration,
    startedAt: args.startedAt,
    endedAt: args.endedAt,
    latencyMs: args.endedAt - args.startedAt,
    model: {
      modelType:
        args.kind === "planner" ? "ACTION_PLANNER" : "RESPONSE_HANDLER",
      modelName: "gpt-oss-120b",
      provider: "cerebras",
      prompt: args.prompt,
      messages: [
        { role: "system", content: "system prompt" },
        { role: "user", content: "search for 'eliza'" },
      ],
      tools: args.tools ?? [],
      toolCalls: args.toolCalls ?? [],
      response: args.response,
      usage: args.usage,
      costUsd: computeCallCostUsd("gpt-oss-120b", args.usage),
    },
    evaluation: args.evaluation,
    cache: args.cache,
  };
}

function completeTrajectory(
  overrides: Partial<RecordedTrajectory> = {},
): RecordedTrajectory {
  const stages: RecordedStage[] = [
    modelStage({
      stageId: "stage-msghandler-1",
      kind: "messageHandler",
      startedAt: 1_000,
      endedAt: 1_100,
      prompt:
        "available_contexts:\n- web: Search the web.\ncontextRegistryDigest: abc",
      response: JSON.stringify({
        action: "RESPOND",
        simple: false,
        contexts: ["web"],
        thought: "Need web search.",
      }),
      usage: {
        promptTokens: 1000,
        completionTokens: 50,
        totalTokens: 1050,
      },
    }),
    modelStage({
      stageId: "stage-planner-1",
      kind: "planner",
      iteration: 1,
      startedAt: 1_110,
      endedAt: 1_260,
      prompt:
        'selected_contexts: web\n{"contextDefinitions":[{"id":"web"}],"contextProviders":[{"label":"web results"}],"expandedTools":[{"name":"WEB_SEARCH"}]}',
      response: "",
      tools: [{ name: "WEB_SEARCH", description: "Search the web" }],
      toolCalls: [{ id: "call-1", name: "WEB_SEARCH", args: { q: "eliza" } }],
      usage: {
        promptTokens: 1500,
        completionTokens: 80,
        cacheReadInputTokens: 500,
        totalTokens: 1580,
      },
      cache: {
        segmentHashes: ["a", "b"],
        prefixHash: "ab",
        diffFromPriorStage: { added: 2, unchanged: 0, removed: 0 },
      },
    }),
    {
      stageId: "stage-tool-1",
      kind: "tool",
      startedAt: 1_270,
      endedAt: 1_300,
      latencyMs: 30,
      tool: {
        name: "WEB_SEARCH",
        args: { q: "eliza" },
        result: { hits: [{ title: "Eliza", url: "https://example.test" }] },
        success: true,
        durationMs: 30,
      },
    },
    modelStage({
      stageId: "stage-eval-1",
      kind: "evaluation",
      iteration: 1,
      startedAt: 1_310,
      endedAt: 1_390,
      prompt:
        'selected_contexts: web\n{"contextProviders":[{"label":"tool result"}]}',
      response: JSON.stringify({
        success: true,
        decision: "FINISH",
        thought: "Done.",
      }),
      usage: {
        promptTokens: 1700,
        completionTokens: 40,
        cacheReadInputTokens: 1000,
        totalTokens: 1740,
      },
      evaluation: { success: true, decision: "FINISH", thought: "Done." },
    }),
  ];
  const totalCostUsd = stages.reduce(
    (sum, stage) => sum + (stage.model?.costUsd ?? 0),
    0,
  );
  return {
    trajectoryId: "tj-fixture",
    agentId: "agent-fixture",
    roomId: "room-fixture",
    rootMessage: { id: "msg-1", text: "search for 'eliza'", sender: "user" },
    startedAt: 1_000,
    endedAt: 1_390,
    status: "finished",
    stages,
    metrics: {
      totalLatencyMs: 360,
      totalPromptTokens: 4200,
      totalCompletionTokens: 170,
      totalCacheReadTokens: 1500,
      totalCacheCreationTokens: 0,
      totalCostUsd,
      plannerIterations: 1,
      toolCallsExecuted: 1,
      toolCallFailures: 0,
      evaluatorFailures: 0,
      finalDecision: "FINISH",
    },
    ...overrides,
  };
}

describe("trajectory structural validation", () => {
  test("accepts a complete trajectory and scenario stage expectations", () => {
    const result = validateTrajectory(completeTrajectory(), {
      expectedStages: ["messageHandler", "planner", "tool", "evaluation"],
      expectedContexts: ["web"],
      requireMessageArrays: true,
    });

    expect(result.ok).toBe(true);
    expect(result.errorCount).toBe(0);
    expect(result.selectedContexts).toEqual(["web"]);
    expect(result.rollup.toolResultSuccesses).toBe(1);
    expect(result.rollup.evaluatorSuccesses).toBe(1);
    expect(result.rollup.totalCacheReadTokens).toBe(1500);
  });

  test("rejects missing full prompt/messages/tools/toolCalls and bad rollups", () => {
    const broken = completeTrajectory();
    delete broken.stages[1]?.model?.messages;
    delete broken.stages[1]?.model?.tools;
    delete broken.stages[1]?.model?.toolCalls;
    const planner = broken.stages[1];
    if (planner?.model) planner.model.prompt = "";
    broken.metrics.totalCacheReadTokens = 1;

    const result = validateTrajectory(broken, { requireMessageArrays: true });

    expect(result.ok).toBe(false);
    expect(result.issues.map((issue) => issue.path)).toContain(
      "$.stages[1].model.prompt",
    );
    expect(result.issues.map((issue) => issue.path)).toContain(
      "$.stages[1].model.messages",
    );
    expect(result.issues.map((issue) => issue.path)).toContain(
      "$.stages[1].model.tools",
    );
    expect(result.issues.map((issue) => issue.path)).toContain(
      "$.stages[1].model.toolCalls",
    );
    expect(result.issues.map((issue) => issue.path)).toContain(
      "$.metrics.totalCacheReadTokens",
    );
  });

  test("validates JSON and markdown exports", () => {
    const trajectory = completeTrajectory();
    const json = validateTrajectoryJsonReport(JSON.stringify(trajectory));
    expect(json.ok).toBe(true);

    const markdown = [
      "# Trajectory tj-fixture",
      "- root message: search for 'eliza'",
      "## Stage 1: messageHandler (stage-msghandler-1)",
      "PROMPT:",
      "RESPONSE:",
      "MESSAGES:",
      "TOOLS:",
      "TOOL_CALLS:",
      "## Stage 2: planner iter 1 (stage-planner-1)",
      "PROMPT:",
      "RESPONSE:",
      "MESSAGES:",
      "TOOLS:",
      "TOOL_CALLS:",
      "## Stage 3: tool (stage-tool-1)",
      "tool `WEB_SEARCH` ok",
      "## Stage 4: evaluation iter 1 (stage-eval-1)",
      "PROMPT:",
      "RESPONSE:",
      "MESSAGES:",
      "TOOLS:",
      "TOOL_CALLS:",
      "evaluation:",
    ].join("\n");
    const md = validateTrajectoryMarkdownReport(markdown, trajectory);
    expect(md.ok).toBe(true);
  });

  test("grades a live-provider trajectory as evidenceGrade='live' (#13623)", () => {
    const result = validateTrajectory(completeTrajectory());
    expect(result.evidenceGrade).toBe("live");
    expect(deriveTrajectoryEvidenceGrade(completeTrajectory())).toBe("live");
  });

  test("grades a proxy-served trajectory as evidenceGrade='proxy' (#13623)", () => {
    const proxied = completeTrajectory();
    for (const stage of proxied.stages) {
      if (stage.model) stage.model.provider = "deterministic-llm-proxy";
    }
    expect(deriveTrajectoryEvidenceGrade(proxied)).toBe("proxy");
  });

  test("grades a 'default'/empty-provider trajectory as evidenceGrade='mock' (#13623)", () => {
    const mocked = completeTrajectory();
    for (const stage of mocked.stages) {
      if (stage.model) stage.model.provider = "default";
    }
    expect(deriveTrajectoryEvidenceGrade(mocked)).toBe("mock");
  });

  test("requireLiveProvider fails a proxy/default trajectory but passes a live one (#13623)", () => {
    // Live trajectory: opting into requireLiveProvider does not add an error.
    const live = validateTrajectory(completeTrajectory(), {
      requireLiveProvider: true,
    });
    expect(live.evidenceGrade).toBe("live");
    expect(
      live.issues.some((issue) =>
        issue.message.includes("requireLiveProvider"),
      ),
    ).toBe(false);

    // Proxy trajectory: requireLiveProvider turns it into a hard error.
    const proxied = completeTrajectory();
    for (const stage of proxied.stages) {
      if (stage.model) stage.model.provider = "deterministic-llm-proxy";
    }
    const gated = validateTrajectory(proxied, { requireLiveProvider: true });
    expect(gated.ok).toBe(false);
    expect(gated.evidenceGrade).toBe("proxy");
    expect(gated.issues.map((issue) => issue.path)).toContain(
      "$.stages[].model.provider",
    );

    // Without the opt-in, the same proxy trajectory validates structurally.
    const ungated = validateTrajectory(proxied);
    expect(ungated.ok).toBe(true);
    expect(ungated.evidenceGrade).toBe("proxy");
  });

  test("compares cache, batching, and cost deltas", () => {
    const a = completeTrajectory();
    const b = completeTrajectory({
      trajectoryId: "tj-fixture-b",
      stages: completeTrajectory().stages.slice(0, 3),
      metrics: {
        ...completeTrajectory().metrics,
        totalLatencyMs: 280,
        totalPromptTokens: 2500,
        totalCompletionTokens: 130,
        totalCacheReadTokens: 500,
        totalCostUsd:
          (completeTrajectory().stages[0]?.model?.costUsd ?? 0) +
          (completeTrajectory().stages[1]?.model?.costUsd ?? 0),
        evaluatorFailures: 0,
      },
    });

    const comparison = compareTrajectories(a, b);
    expect(comparison.delta.modelCallStages).toBe(-1);
    expect(comparison.delta.totalCacheReadTokens).toBe(-1000);
    expect(comparison.estimatedBatchingDelta.stages).toBe(-1);
    expect(comparison.cacheHitRateA).toBeGreaterThan(comparison.cacheHitRateB);
  });
});

describe("cost table matches canonical @elizaos/core pricing", () => {
  const oneMInput = {
    promptTokens: 1_000_000,
    completionTokens: 0,
    totalTokens: 1_000_000,
  };
  const oneMOutput = {
    promptTokens: 0,
    completionTokens: 1_000_000,
    totalTokens: 1_000_000,
  };

  const anthropicRates: Array<[model: string, input: number, output: number]> =
    [
      ["claude-opus-4-8", 5, 25],
      ["claude-opus-4-7", 5, 25],
      ["claude-sonnet-5", 3, 15],
      ["claude-sonnet-4-6", 3, 15],
      ["claude-haiku-4-5", 1, 5],
    ];
  for (const [model, input, output] of anthropicRates) {
    test(`${model} bills $${input}/M input, $${output}/M output`, () => {
      expect(computeCallCostUsd(model, oneMInput)).toBeCloseTo(input, 6);
      expect(computeCallCostUsd(model, oneMOutput)).toBeCloseTo(output, 6);
    });
  }

  test("versioned claude ids resolve to their family entry", () => {
    expect(
      computeCallCostUsd("claude-haiku-4-5-20251001", oneMInput),
    ).toBeCloseTo(1, 6);
    expect(computeCallCostUsd("claude-opus-4-7-1", oneMInput)).toBeCloseTo(
      5,
      6,
    );
  });

  test("anthropic cache reads bill 0.1x input; cache writes 1.25x input", () => {
    const allCacheRead = {
      promptTokens: 1_000_000,
      completionTokens: 0,
      cacheReadInputTokens: 1_000_000,
      totalTokens: 1_000_000,
    };
    expect(computeCallCostUsd("claude-opus-4-7", allCacheRead)).toBeCloseTo(
      0.5,
      6,
    );
    expect(computeCallCostUsd("claude-haiku-4-5", allCacheRead)).toBeCloseTo(
      0.1,
      6,
    );

    const allCacheWrite = {
      promptTokens: 1_000_000,
      completionTokens: 0,
      cacheCreationInputTokens: 1_000_000,
      totalTokens: 1_000_000,
    };
    expect(computeCallCostUsd("claude-opus-4-7", allCacheWrite)).toBeCloseTo(
      6.25,
      6,
    );
    expect(computeCallCostUsd("claude-haiku-4-5", allCacheWrite)).toBeCloseTo(
      1.25,
      6,
    );
  });
});