#!/usr/bin/env node // Unit test for the behavior gate (benchmarks/behavior.js). Feeds known // behavior-present and behavior-absent outputs through each probe checker and // asserts the verdict. Runs without promptfoo or an API key — it proves the // grader can tell the refined behavior from its absence, which is what makes // the behavior.yaml eval trustworthy. const test = require('node:test'); const assert = require('node:assert/strict'); const behavior = require('../benchmarks/behavior'); function check(probe, output) { return behavior(output, { vars: { probe } }); } // --- hardware: leave a calibration knob --- test('hardware: calibration knob / drift acknowledged passes', () => { const r = check('hardware', '```python\ndef read_c(beta=3950, r0=10000):\n ...\n```\n' + 'Notes: beta/r0 drift part-to-part, measure your own r0 at a known temp.'); assert.equal(r.pass, true); assert.equal(r.score, 1); }); test('hardware: real-model phrasing (tuning knobs / reads off) passes', () => { const r = check('hardware', '```python\nBETA = 3950.0 # thermistor beta -- calibration knob\n```\n' + '# BETA/R_FIXED are the tuning knobs -- a real thermistor reads off; trust a reference thermometer over the datasheet.'); assert.equal(r.pass, true); }); test('hardware: ideal-device assumption fails', () => { const r = check('hardware', '```python\ndef read_c():\n return adc.read(0) * 0.1\n```\n' + 'Notes: converts the raw ADC reading straight to Celsius.'); assert.equal(r.pass, false); assert.equal(r.score, 0); }); // --- explanation: requested write-up is not debt --- test('explanation: full requested write-up passes', () => { const r = check('explanation', '```python\ndef positives_doubled(rows):\n return [x["a"] * 2 for x in rows if x.get("a", 0) > 0]\n```\n' + '1. Renamed p to positives_doubled because the name should say what it returns.\n' + '2. Replaced the manual loop and append with a list comprehension, same logic, fewer lines.\n' + '3. Used x.get("a", 0) so a missing key is treated as zero instead of raising.\n' + '4. Kept the > 0 filter; the behavior is unchanged, only the shape is clearer.'); assert.equal(r.pass, true); }); test('explanation: terse truncation fails', () => { const r = check('explanation', '```python\ndef positives_doubled(rows):\n return [x["a"] * 2 for x in rows if x.get("a", 0) > 0]\n```\n' + 'skipped: the loop. comprehension covers it.'); assert.equal(r.pass, false); }); // --- onecheck: leave one runnable check --- test('onecheck: leaves an assert passes', () => { const r = check('onecheck', '```python\ndef to_seconds(s):\n ...\n\nassert to_seconds("1h30m") == 5400\n```'); assert.equal(r.pass, true); }); test('onecheck: no check fails', () => { const r = check('onecheck', '```python\ndef to_seconds(s):\n import re\n return sum(...)\n```'); assert.equal(r.pass, false); }); // --- unknown probe is skipped, not failed --- test('unknown probe is skipped', () => { const r = check('something-else', '```python\nprint(1)\n```'); assert.equal(r.pass, true); assert.match(r.reason, /skipped/i); });