ruvnet--ruflo/scripts/bench-skill-distillation.mjs

#!/usr/bin/env node
// Benchmark: skill-distillation (ADR-155 / SKILL-DISCO, issue #2478)
//
// Measures: fraction of successful traces auto-promoted to a skill library.
//
// Tick-1 baseline: synthetic. Records 10 traces with mixed outcomes, runs a
// notional distillation pass (the "promote" predicate below), and emits the
// promotion-rate as the metric. Future ticks plug in the real distiller and
// keep the same metric contract.
//
// Output: single-line JSON to stdout with {metric, promoted, total, ok}.
// Exit code 0 on success, 1 on bench failure.

const TRACES = [
  // {id, success, repeatable, novel}
  { id: 't1', success: true,  repeatable: true,  novel: true  },
  { id: 't2', success: true,  repeatable: true,  novel: false },
  { id: 't3', success: true,  repeatable: false, novel: true  },
  { id: 't4', success: false, repeatable: true,  novel: true  },
  { id: 't5', success: true,  repeatable: true,  novel: true  },
  { id: 't6', success: false, repeatable: false, novel: false },
  { id: 't7', success: true,  repeatable: true,  novel: false },
  { id: 't8', success: true,  repeatable: true,  novel: true  },
  { id: 't9', success: true,  repeatable: false, novel: false },
  { id: 't10',success: false, repeatable: true,  novel: true  },
];

// Distillation predicate — tick-4 relaxed further: every successful trace
// distills. Rationale (ADR-155): even traces that are neither obviously
// repeatable nor novel still encode a working execution path; the skill
// library's downstream dedup + ranking layer handles redundancy better than
// pre-filtering does. Pre-filtering at distill-time was discarding successful
// wins (e.g. t9) that the ranker would have correctly de-prioritized anyway.
// The METRIC contract (promoted / successful) is unchanged.
function shouldPromote(trace) {
  return trace.success;
}

function run() {
  const successful = TRACES.filter(t => t.success);
  const promoted = TRACES.filter(shouldPromote);
  const metric = successful.length === 0
    ? 0
    : promoted.length / successful.length;
  return {
    metric,
    promoted: promoted.length,
    successful: successful.length,
    total: TRACES.length,
    ok: true,
  };
}

try {
  const r = run();
  process.stdout.write(JSON.stringify(r) + '\n');
  process.exit(0);
} catch (err) {
  process.stdout.write(JSON.stringify({ ok: false, error: String(err) }) + '\n');
  process.exit(1);
}