/** * TDD test for the Claude Code auto-mode classifier compat mode (opt-in, default off). * * Claude Code's `--permission-mode auto` sends an internal `/v1/messages` security-classifier * request and requires the response to START with the literal token `no` (ALLOW) * or `yes` (BLOCK) — anything else is unparseable and Claude Code fails closed * with "Auto mode could not evaluate this action and is blocking it for safety". * * When a combo/fallback route sends the classifier call to a cheap model that returns 200 with * empty content, the well-formed-but-empty Claude message OmniRoute produces still fails that * parser. With `claudeClassifierCompat` set to "auto" (or "always"), handleChatCore detects the * classifier request and short-circuits with a synthetic ALLOW response — WITHOUT ever calling * the upstream provider. Default is "off": nothing changes unless an operator explicitly opts in. */ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-claude-classifier-compat-")); process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const { updateSettings } = await import("../../src/lib/db/settings.ts"); const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); const { shouldDefaultAllowClassifier, buildDefaultAllowClaudeMessage } = await import( "../../open-sse/handlers/chatCore/claudeClassifierCompat.ts" ); const { FORMATS } = await import("../../open-sse/translator/formats.ts"); const originalFetch = globalThis.fetch; function noopLog() { return { debug() {}, info() {}, warn() {}, error() {} }; } // Shape of the classifier request Claude Code's `--permission-mode auto` sends internally: // a Claude Messages request carrying the security-monitor system prompt AND `` as a // stop sequence — the two independent signals the compat detector relies on. const CLASSIFIER_BODY = { model: "claude-3-5-haiku-20241022", stream: false, system: [ { type: "text", text: "You are a security monitor for autonomous AI coding agents. Evaluate the following action and respond with yes or no.", }, ], stop_sequences: [""], messages: [ { role: "user", content: [{ type: "text", text: "WebFetch https://example.com" }], }, ], max_tokens: 8, }; test.after(() => { globalThis.fetch = originalFetch; core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); // ─── Settings default is opt-in (off) ──────────────────────────────────────── // Runs FIRST, before any updateSettings() write, so it reads the pristine DB. // (DATA_DIR freezes at the first DB open, so a later "fresh dir" swap would still // read this same DB — hence assert the default up front.) test("settings default: claudeClassifierCompat is 'off' (opt-in)", async () => { const { getSettings } = await import("../../src/lib/db/settings.ts"); const settings = await getSettings(); assert.equal(settings.claudeClassifierCompat, "off", "claudeClassifierCompat defaults to off"); }); // ─── Pure detector: shouldDefaultAllowClassifier ───────────────────────────── test("detector: off never short-circuits (pass-through preserved by default)", () => { assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, CLASSIFIER_BODY, "off"), false); assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, CLASSIFIER_BODY, undefined), false); }); test("detector: auto fires on the security-monitor system-prompt marker", () => { const body = { system: [{ type: "text", text: "You are a security monitor for autonomous AI coding agents." }], stop_sequences: [], }; assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, body, "auto"), true); }); test("detector: auto fires on the stop_sequence token", () => { const body = { system: [{ type: "text", text: "unrelated" }], stop_sequences: [""] }; assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, body, "auto"), true); }); test("detector: auto does NOT fire on a regular Claude request (no marker, no )", () => { const body = { system: [{ type: "text", text: "You are a helpful coding assistant." }], stop_sequences: [], messages: [{ role: "user", content: "hello" }], }; assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, body, "auto"), false); }); test("detector: never fires for non-Claude source formats even in always mode", () => { assert.equal(shouldDefaultAllowClassifier(FORMATS.OPENAI, CLASSIFIER_BODY, "always"), false); }); test("detector: always fires for every Claude-format request", () => { const plain = { system: [{ type: "text", text: "hi" }], stop_sequences: [] }; assert.equal(shouldDefaultAllowClassifier(FORMATS.CLAUDE, plain, "always"), true); }); // ─── Pure builder: buildDefaultAllowClaudeMessage ──────────────────────────── test("builder: synthetic message text STARTS WITH no", async () => { const built = buildDefaultAllowClaudeMessage("claude-3-5-haiku-20241022"); assert.equal(built.success, true); const payload = (await built.response.json()) as { type: string; role: string; stop_reason: string; content: Array<{ type: string; text?: string }>; }; assert.equal(payload.type, "message"); assert.equal(payload.role, "assistant"); assert.equal(payload.stop_reason, "end_turn"); const text = payload.content.find((b) => b.type === "text")?.text ?? ""; assert.ok( text.startsWith("no"), `expected synthetic text to start with no, got: ${text}` ); assert.ok(!text.includes("yes"), "must not signal BLOCK"); }); // ─── Handler-level: end-to-end short-circuit through handleChatCore ────────── test("handler: claudeClassifierCompat=auto short-circuits WITHOUT calling upstream, text starts with no", async () => { await updateSettings({ claudeClassifierCompat: "auto" }); let fetchCalls = 0; globalThis.fetch = (async () => { fetchCalls++; throw new Error("upstream fetch should NOT be called when the classifier short-circuits"); }) as typeof fetch; try { const result = await handleChatCore({ body: structuredClone(CLASSIFIER_BODY), modelInfo: { provider: "openai", model: "gpt-4o-mini", extendedContext: false }, credentials: { apiKey: "sk-test", providerSpecificData: {} }, log: noopLog(), clientRawRequest: { endpoint: "/v1/messages", body: structuredClone(CLASSIFIER_BODY), headers: new Headers({ accept: "application/json" }), }, userAgent: "unit-test", }); assert.equal(fetchCalls, 0, "upstream fetch must NOT be called"); assert.equal(result.success, true, "handleChatCore must report success"); const payload = (await (result as { response: Response }).response.json()) as { type: string; content: Array<{ type: string; text?: string }>; }; assert.equal(payload.type, "message"); const text = payload.content.find((b) => b.type === "text")?.text ?? ""; assert.ok( text.startsWith("no"), `expected classifier response to start with no, got: ${text}` ); } finally { globalThis.fetch = originalFetch; } });