ruvnet--ruflo/v3/@claude-flow/cli/__tests__/services/fable-harness.test.ts

/**
 * fable-harness.test.ts — unit tests for the cost-disciplined Fable judge.
 *
 * Every test MOCKS the `claude -p` spawn via the injectable `spawnClaude` — no
 * test ever launches the real CLI or spends a cent. Assertions cover:
 *   • command construction (argv: -p / --model / --output-format json /
 *     --append-system-prompt / --max-budget-usd)
 *   • the fresh-empty-temp-cwd requirement (never the project dir)
 *   • batching (ceil(n / batchSize) calls)
 *   • budget-cap enforcement (spend stops launching batches)
 *   • the OFF-BY-DEFAULT guarantee — no budget cap ⇒ ZERO spawns
 *   • verdict extraction from the JSON envelope
 *
 * One optional live smoke (RUFLO_FABLE_LIVE=1) a human can run is at the end.
 */

import { describe, it, expect, vi } from 'vitest';
import { tmpdir } from 'os';
import {
  FableHarness,
  estimateFableCostUsd,
  extractVerdictArray,
  parseCostFromEnvelope,
  FABLE_COST_MODEL,
  type ClaudeSpawnFn,
  type JudgeItem,
} from '../../src/services/fable-harness.js';

/** Build a fake spawnClaude that records every call and returns a canned envelope. */
function fakeSpawn(makeResult: (batchIndex: number, argv: string[], prompt: string, cwd: string) => {
  stdout?: string;
  stderr?: string;
  code?: number | null;
  costUsd?: number;
}) {
  const calls: Array<{ argv: string[]; prompt: string; cwd: string; timeoutMs: number }> = [];
  const fn: ClaudeSpawnFn = async (argv, prompt, cwd, opts) => {
    const idx = calls.length;
    calls.push({ argv, prompt, cwd, timeoutMs: opts.timeoutMs });
    const r = makeResult(idx, argv, prompt, cwd);
    return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', code: r.code ?? 0, costUsd: r.costUsd };
  };
  return { fn, calls };
}

/** Envelope shaped like `claude -p --output-format json` output. */
function envelope(verdicts: unknown[], costUsd = 0.4): string {
  return JSON.stringify({ type: 'result', result: JSON.stringify(verdicts), total_cost_usd: costUsd });
}

const items = (n: number): JudgeItem[] =>
  Array.from({ length: n }, (_, i) => ({ id: `t${i}`, task: `task ${i}`, output: `output ${i}` }));

describe('FableHarness — command construction', () => {
  it('builds argv with -p, model, json output, append-system-prompt and budget cap', () => {
    const h = new FableHarness({ maxBudgetUsd: 5, spawnClaude: async () => ({ stdout: '', stderr: '', code: 0 }) });
    const argv = h.buildArgv('JUDGE ROLE');
    expect(argv).toContain('-p');
    expect(argv).toEqual(expect.arrayContaining(['--model', 'claude-fable-5']));
    expect(argv).toEqual(expect.arrayContaining(['--output-format', 'json']));
    expect(argv).toEqual(expect.arrayContaining(['--append-system-prompt', 'JUDGE ROLE']));
    const capIdx = argv.indexOf('--max-budget-usd');
    expect(capIdx).toBeGreaterThan(-1);
    expect(Number(argv[capIdx + 1])).toBeCloseTo(5, 2);
  });

  it('runs claude from a FRESH TEMP cwd, never the project dir', async () => {
    const { fn, calls } = fakeSpawn(() => ({ stdout: envelope([{ id: 't0', resolved: true, confidence: 0.9, reason: 'ok' }]) }));
    const h = new FableHarness({ maxBudgetUsd: 5, spawnClaude: fn });
    await h.judgeBatch(items(1));
    expect(calls).toHaveLength(1);
    expect(calls[0].cwd).toContain('ruflo-fable');
    expect(calls[0].cwd).not.toBe(process.cwd());
    // Temp dir lives under the OS temp root (allow macOS /private symlink drift).
    expect(calls[0].cwd.includes(tmpdir()) || calls[0].cwd.includes('ruflo-fable')).toBe(true);
    // The prompt piped to stdin is the batch JSON, not an argv positional.
    expect(calls[0].prompt).toContain('task 0');
    expect(calls[0].argv).not.toContain('task 0');
  });
});

describe('FableHarness — off by default (zero spend)', () => {
  it('makes ZERO spawns when no budget cap is set', async () => {
    const { fn, calls } = fakeSpawn(() => ({ stdout: envelope([]) }));
    const h = new FableHarness({ spawnClaude: fn }); // no maxBudgetUsd
    expect(h.isEnabled()).toBe(false);
    const out = await h.judgeBatch(items(10));
    expect(out).toEqual([]);
    expect(calls).toHaveLength(0);
    expect(h.getSpentUsd()).toBe(0);
  });

  it('reflectFailures also spends nothing without a budget cap', async () => {
    const { fn, calls } = fakeSpawn(() => ({ stdout: envelope([]) }));
    const h = new FableHarness({ spawnClaude: fn });
    const out = await h.reflectFailures([{ id: 'a', task: 't', output: 'o' }]);
    expect(out).toEqual([]);
    expect(calls).toHaveLength(0);
  });
});

describe('FableHarness — batching & judging', () => {
  it('batches items into ceil(n / batchSize) calls', async () => {
    const { fn, calls } = fakeSpawn((idx, _argv, prompt) => {
      const batch = JSON.parse(prompt) as JudgeItem[];
      return { stdout: envelope(batch.map((b) => ({ id: b.id, resolved: true, confidence: 0.8, reason: 'ok' }))) };
    });
    const h = new FableHarness({ maxBudgetUsd: 100, batchSize: 4, spawnClaude: fn });
    const out = await h.judgeBatch(items(10)); // 10 / 4 = 3 batches
    expect(calls).toHaveLength(3);
    expect(out).toHaveLength(10);
    expect(out[0]).toMatchObject({ id: 't0', resolved: true });
  });

  it('normalizes verdicts (resolved boolean, clamped confidence)', async () => {
    const { fn } = fakeSpawn(() => ({
      stdout: envelope([
        { id: 't0', resolved: 'true', confidence: 1.7, reason: 'x' },
        { id: 't1', resolved: false, confidence: -3, reason: 'y' },
      ]),
    }));
    const h = new FableHarness({ maxBudgetUsd: 10, spawnClaude: fn });
    const out = await h.judgeBatch(items(2));
    expect(out[0]).toEqual({ id: 't0', resolved: true, confidence: 1, reason: 'x' });
    expect(out[1]).toEqual({ id: 't1', resolved: false, confidence: 0, reason: 'y' });
  });

  it('stops launching batches once measured spend reaches the cap', async () => {
    const { fn, calls } = fakeSpawn((idx, _argv, prompt) => {
      const batch = JSON.parse(prompt) as JudgeItem[];
      // Each batch reports $3 spend; cap is $5 ⇒ only the first batch should run.
      return { stdout: envelope(batch.map((b) => ({ id: b.id, resolved: true, confidence: 0.5, reason: 'ok' }))), costUsd: 3 };
    });
    const h = new FableHarness({ maxBudgetUsd: 5, batchSize: 2, spawnClaude: fn });
    const out = await h.judgeBatch(items(10)); // 5 batches if unbounded
    // batch1: spent 0<5 → run (spent 3); batch2: spent 3<5 → run (spent 6);
    // batch3: spent 6>=5 → stop. So exactly 2 batches run, covering 4 items.
    expect(calls.length).toBe(2);
    expect(out.length).toBe(4);
    expect(h.getSpentUsd()).toBeCloseTo(6, 2);
  });
});

describe('FableHarness — reflectFailures', () => {
  it('returns parsed reflection results', async () => {
    const { fn } = fakeSpawn(() => ({
      stdout: envelope([
        { id: 'a', failureClass: 'wrong-file', diagnosis: 'edited the wrong module', mutationHint: 'target foo.ts' },
      ]),
    }));
    const h = new FableHarness({ maxBudgetUsd: 10, spawnClaude: fn });
    const out = await h.reflectFailures([{ id: 'a', task: 't', output: 'o', failureHint: 'tests red' }]);
    expect(out).toEqual([
      { id: 'a', failureClass: 'wrong-file', diagnosis: 'edited the wrong module', mutationHint: 'target foo.ts' },
    ]);
  });
});

describe('extractVerdictArray', () => {
  it('parses an enveloped result string', () => {
    const arr = extractVerdictArray(envelope([{ id: 'x', resolved: true }]));
    expect(arr).toEqual([{ id: 'x', resolved: true }]);
  });
  it('parses a bare JSON array', () => {
    expect(extractVerdictArray('[{"id":"x"}]')).toEqual([{ id: 'x' }]);
  });
  it('parses a fenced ```json array inside a result string', () => {
    const env = JSON.stringify({ result: '```json\n[{"id":"y"}]\n```' });
    expect(extractVerdictArray(env)).toEqual([{ id: 'y' }]);
  });
  it('returns [] on garbage', () => {
    expect(extractVerdictArray('not json at all')).toEqual([]);
    expect(extractVerdictArray('')).toEqual([]);
  });
});

describe('cost model', () => {
  it('estimateFableCostUsd amortizes per item', () => {
    expect(estimateFableCostUsd(0)).toBe(0);
    expect(estimateFableCostUsd(20)).toBeCloseTo(20 * FABLE_COST_MODEL.perItemBatchedUsd, 4);
  });
  it('parseCostFromEnvelope reads total_cost_usd', () => {
    expect(parseCostFromEnvelope(envelope([], 1.23)) ?? 0).toBeCloseTo(1.23, 2);
    expect(parseCostFromEnvelope('garbage')).toBeUndefined();
  });
  it('exposes the measured cost anchors', () => {
    expect(FABLE_COST_MODEL.perCallProjectCwdUsd).toBeGreaterThan(FABLE_COST_MODEL.perCallCleanCwdUsd);
    expect(FABLE_COST_MODEL.model).toBe('claude-fable-5');
  });
});

// ── Optional live smoke — human-run only, never in CI ─────────────────────
const LIVE = process.env.RUFLO_FABLE_LIVE === '1';
(LIVE ? describe : describe.skip)('FableHarness — LIVE smoke (RUFLO_FABLE_LIVE=1)', () => {
  it('judges a trivial item against the real Fable CLI', async () => {
    const h = new FableHarness({ maxBudgetUsd: 0.5 }); // real spawnClaude
    const out = await h.judgeBatch([
      { id: 'live-1', task: 'Return the number 4', output: '4' },
    ]);
    expect(Array.isArray(out)).toBe(true);
    if (out.length > 0) expect(typeof out[0].resolved).toBe('boolean');
  }, 120_000);
});