diegosouzapw--omniroute/tests/unit/inspector-llm-metadata.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import { extractLlmMetadata } from "../../src/mitm/inspector/llmMetadataExtractor.ts";
import type { InterceptedRequest } from "../../src/mitm/inspector/types.ts";

function makeReq(overrides: Partial<InterceptedRequest> = {}): InterceptedRequest {
  return {
    id: "test",
    source: "agent-bridge",
    timestamp: new Date().toISOString(),
    method: "POST",
    host: "api.openai.com",
    path: "/v1/chat/completions",
    requestHeaders: {},
    requestBody: null,
    requestSize: 0,
    responseHeaders: {},
    responseBody: null,
    responseSize: 0,
    status: 200,
    detectedKind: "llm",
    ...overrides,
  };
}

test("returns null for non-llm requests", () => {
  const req = makeReq({ detectedKind: "app" });
  assert.equal(extractLlmMetadata(req), null);
});

test("infers provider=openai from host", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ model: "gpt-4", messages: [] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.provider, "openai");
  assert.equal(meta.apiKind, "chat.completions");
  assert.equal(meta.model, "gpt-4");
});

test("infers provider=anthropic + apiKind=messages", () => {
  const req = makeReq({
    host: "api.anthropic.com",
    path: "/v1/messages",
    requestBody: JSON.stringify({ model: "claude-3", messages: [{}, {}] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.provider, "anthropic");
  assert.equal(meta.apiKind, "messages");
  assert.equal(meta.messages, 2);
});

test("infers provider=gemini", () => {
  const req = makeReq({
    host: "generativelanguage.googleapis.com",
    path: "/v1beta/models/gemini-pro:generateContent",
    requestBody: JSON.stringify({ contents: [{}, {}, {}] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.provider, "gemini");
  assert.equal(meta.messages, 3);
});

test("extracts model from response body when missing in request", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ messages: [] }),
    responseBody: JSON.stringify({ model: "gpt-4-turbo", choices: [] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.model, "gpt-4-turbo");
});

test("extracts tokensIn/tokensOut from prompt_tokens/completion_tokens", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ model: "gpt-4", messages: [] }),
    responseBody: JSON.stringify({
      usage: { prompt_tokens: 10, completion_tokens: 25 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.tokensIn, 10);
  assert.equal(meta.tokensOut, 25);
});

test("computes costEstimateUsd for gpt-4o with token counts", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ model: "gpt-4o", messages: [] }),
    responseBody: JSON.stringify({
      usage: { prompt_tokens: 1_000_000, completion_tokens: 100_000 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.tokensIn, 1_000_000);
  assert.equal(meta.tokensOut, 100_000);
  // 1M*2.50/1M + 100k*10.00/1M = 2.50 + 1.00 = 3.50
  assert.equal(meta.costEstimateUsd, 3.50);
});

test("computes costEstimateUsd for claude-3-5-sonnet with token counts", () => {
  const req = makeReq({
    host: "api.anthropic.com",
    path: "/v1/messages",
    requestBody: JSON.stringify({ model: "claude-3-5-sonnet-20240620", messages: [] }),
    responseBody: JSON.stringify({
      usage: { input_tokens: 500_000, output_tokens: 200_000 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  // 500k*3.00/1M + 200k*15.00/1M = 1.50 + 3.00 = 4.50
  assert.equal(meta.costEstimateUsd, 4.50);
});

test("costEstimateUsd is null for unknown model", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ model: "unknown-model-xyz", messages: [] }),
    responseBody: JSON.stringify({
      usage: { prompt_tokens: 100, completion_tokens: 50 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.costEstimateUsd, null);
});

test("extracts tokensIn/tokensOut from input_tokens/output_tokens (Anthropic)", () => {
  const req = makeReq({
    host: "api.anthropic.com",
    path: "/v1/messages",
    requestBody: JSON.stringify({ model: "claude-3", messages: [] }),
    responseBody: JSON.stringify({
      usage: { input_tokens: 50, output_tokens: 100 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.tokensIn, 50);
  assert.equal(meta.tokensOut, 100);
});

test("extracts tokens from Gemini usageMetadata", () => {
  const req = makeReq({
    host: "generativelanguage.googleapis.com",
    path: "/v1beta/models/gemini-pro:generateContent",
    requestBody: JSON.stringify({ contents: [] }),
    responseBody: JSON.stringify({
      usageMetadata: { promptTokenCount: 7, candidatesTokenCount: 14 },
    }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.tokensIn, 7);
  assert.equal(meta.tokensOut, 14);
});

test("flags streamed=true on SSE content-type", () => {
  const req = makeReq({
    requestBody: JSON.stringify({ model: "gpt-4", messages: [] }),
    responseHeaders: { "content-type": "text/event-stream" },
    responseBody: "data: {}\n",
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.streamed, true);
});

test("returns null fields when no info available", () => {
  const req = makeReq({
    host: "unknown.example.com",
    path: "/v1/messages",
    requestBody: JSON.stringify({ messages: [] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.provider, null);
  assert.equal(meta.tokensIn, null);
  assert.equal(meta.tokensOut, null);
  assert.equal(meta.costEstimateUsd, null);
});

test("captures mappedTo from request override", () => {
  const req = makeReq({
    mappedModel: "gpt-4o",
    requestBody: JSON.stringify({ model: "gpt-3.5", messages: [] }),
  });
  const meta = extractLlmMetadata(req);
  assert.ok(meta);
  assert.equal(meta.mappedTo, "gpt-4o");
});