/**
* Batch B — Final Tasks Tests
*
* Tests for: evalRunner, a11yAudit, responsiveSpecs, compliance (noLog)
*/
import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
// ──────────────── T-42: Eval Runner ────────────────
import {
registerSuite,
getSuite,
listSuites,
evaluateCase,
runSuite,
createScorecard,
resetSuites,
} from "../../src/lib/evals/evalRunner.ts";
describe("evalRunner", () => {
after(() => {
// Remove test-registered suites while preserving built-ins.
resetSuites();
});
it("should have golden-set suite pre-registered", () => {
const suite = getSuite("golden-set");
assert.ok(suite);
assert.equal(suite.name, "OmniRoute Golden Set");
assert.ok(suite.cases.length >= 10);
});
it("should list registered suites", () => {
const suites = listSuites();
assert.ok(suites.length >= 1);
assert.ok(suites.some((s) => s.id === "golden-set"));
});
it("should evaluate exact match", () => {
const result = evaluateCase(
{
id: "t1",
name: "test",
model: "test",
input: {},
expected: { strategy: "exact", value: "hello" },
},
"hello"
);
assert.equal(result.passed, true);
});
it("should fail exact match on mismatch", () => {
const result = evaluateCase(
{
id: "t2",
name: "test",
model: "test",
input: {},
expected: { strategy: "exact", value: "hello" },
},
"world"
);
assert.equal(result.passed, false);
});
it("should evaluate contains (case-insensitive)", () => {
const result = evaluateCase(
{
id: "t3",
name: "test",
model: "test",
input: {},
expected: { strategy: "contains", value: "paris" },
},
"The capital is Paris."
);
assert.equal(result.passed, true);
});
it("should evaluate regex", () => {
const result = evaluateCase(
{
id: "t4",
name: "test",
model: "test",
input: {},
expected: { strategy: "regex", value: "\\d+" },
},
"The answer is 42."
);
assert.equal(result.passed, true);
});
it("should fail regex when expected value is missing", () => {
const result = evaluateCase(
{
id: "t4-missing",
name: "test",
model: "test",
input: {},
expected: { strategy: "regex" },
},
"The answer is 42."
);
assert.equal(result.passed, false);
assert.equal(result.details.error, "No regex value provided for evaluation.");
});
it("should reject oversized regex patterns", () => {
const result = evaluateCase(
{
id: "t4-large",
name: "test",
model: "test",
input: {},
expected: { strategy: "regex", value: "a".repeat(513) },
},
"test"
);
assert.equal(result.passed, false);
assert.equal(result.details.error, "Regex pattern too large for safe evaluation.");
});
it("should evaluate stateful RegExp values consistently", () => {
const regex = /answer/g;
const evalCase = {
id: "t4-stateful",
name: "test",
model: "test",
input: {},
expected: { strategy: "regex", value: regex },
};
assert.equal(evaluateCase(evalCase, "answer").passed, true);
assert.equal(evaluateCase(evalCase, "answer").passed, true);
});
it("should evaluate custom function", () => {
const result = evaluateCase(
{
id: "t5",
name: "test",
model: "test",
input: {},
expected: { strategy: "custom", fn: (output) => output.length > 5 },
},
"this is long enough"
);
assert.equal(result.passed, true);
});
it("should handle unknown strategy gracefully", () => {
const result = evaluateCase(
{ id: "t6", name: "test", model: "test", input: {}, expected: { strategy: "unknown" } },
"test"
);
assert.equal(result.passed, false);
assert.ok(result.error.includes("Unknown strategy"));
});
it("should normalize non-Error thrown values", () => {
const result = evaluateCase(
{
id: "t7",
name: "test",
model: "test",
input: {},
expected: {
strategy: "custom",
fn: () => {
throw "custom failure";
},
},
},
"test"
);
assert.equal(result.passed, false);
assert.equal(result.error, "custom failure");
});
it("should run suite and produce summary", () => {
registerSuite({
id: "test-suite",
name: "Test Suite",
cases: [
{
id: "c1",
name: "pass",
model: "m",
input: {},
expected: { strategy: "contains", value: "yes" },
},
{
id: "c2",
name: "fail",
model: "m",
input: {},
expected: { strategy: "contains", value: "no" },
},
],
});
const result = runSuite("test-suite", { c1: "yes it works", c2: "yes it works" });
assert.equal(result.summary.total, 2);
assert.equal(result.summary.passed, 1);
assert.equal(result.summary.failed, 1);
assert.equal(result.summary.passRate, 50);
});
it("should create scorecard from runs", () => {
const run1 = runSuite("test-suite", { c1: "yes", c2: "no" });
const scorecard = createScorecard([run1]);
assert.equal(scorecard.suites, 1);
assert.equal(scorecard.totalCases, 2);
});
it("should throw on unknown suite", () => {
assert.throws(() => runSuite("nonexistent", {}), { message: /not found/ });
});
it("should restore built-in suites when resetting test suites", () => {
registerSuite({ id: "temporary-suite", name: "Temporary", cases: [] });
resetSuites();
assert.equal(getSuite("temporary-suite"), null);
assert.ok(getSuite("golden-set"));
});
});
// ──────────────── T-35: a11y Audit ────────────────
import { auditHTML, generateReport, WCAG_RULES } from "../../src/shared/utils/a11yAudit.ts";
describe("a11yAudit", () => {
it("should pass for compliant HTML", () => {
const html = '
';
const violations = auditHTML(html);
assert.equal(violations.length, 0);
});
it("should detect images without alt text", () => {
const html = '
';
const violations = auditHTML(html);
assert.ok(violations.some((v) => v.id === WCAG_RULES.IMAGE_ALT));
});
it("should detect dialogs without role", () => {
const html = '
Content