wehub-resource-sync
609 lines
22 KiB
TypeScript
609 lines
22 KiB
TypeScript
/**
|
|
* Unit tests for the pure `compareReports` function in
|
|
* `e2e/scripts/benchmark-compare.ts`. Mirrors the pattern of
|
|
* `scripts/__tests__/memory-compare.test.ts` — the script lives under `e2e/`
|
|
* but its regression-diffing core is plain TypeScript with no Appium deps.
|
|
*
|
|
* Behaviour under test:
|
|
* - identical reports pass, no flags;
|
|
* - >15 pp% delta (in the regression direction) is flagged;
|
|
* - >15 tg% delta (in the regression direction) is flagged;
|
|
* - effective_backend mismatch is flagged independently of numeric deltas;
|
|
* - missing rows are surfaced via `missing_in_current` / `missing_in_baseline`.
|
|
*/
|
|
|
|
import {
|
|
BenchmarkMatrixReport,
|
|
BenchmarkRunReport,
|
|
compareReports,
|
|
} from '../../e2e/scripts/benchmark-compare';
|
|
|
|
function makeRun(
|
|
overrides: Partial<BenchmarkRunReport> = {},
|
|
): BenchmarkRunReport {
|
|
return {
|
|
model_id: 'qwen3-1.7b',
|
|
quant: 'q4_0',
|
|
requested_backend: 'gpu',
|
|
effective_backend: 'opencl',
|
|
pp_avg: 300,
|
|
tg_avg: 24,
|
|
wall_ms: 20_000,
|
|
peak_memory_mb: 2048,
|
|
status: 'ok',
|
|
timestamp: '2026-04-21T00:00:00Z',
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function makeReport(
|
|
runs: BenchmarkRunReport[],
|
|
overrides: Partial<BenchmarkMatrixReport> = {},
|
|
): BenchmarkMatrixReport {
|
|
return {
|
|
version: '1.0',
|
|
device: 'S26 Ultra',
|
|
soc: 'SM8875',
|
|
commit: 'abc123',
|
|
llama_rn_version: '0.12.0',
|
|
platform: 'android',
|
|
os_version: '16',
|
|
timestamp: '2026-04-21T00:00:00Z',
|
|
preseeded: true,
|
|
runs,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
// Suppress the WARN line `compareReports` emits when one or both sides omit
|
|
// `bench` — every test in this file uses the legacy fixture (no bench field)
|
|
// except the protocol-mismatch suite below, which sets it explicitly.
|
|
const originalConsoleError = console.error;
|
|
beforeAll(() => {
|
|
jest.spyOn(console, 'error').mockImplementation((...args: any[]) => {
|
|
if (
|
|
typeof args[0] === 'string' &&
|
|
args[0].includes('protocol comparison disabled')
|
|
) {
|
|
return;
|
|
}
|
|
originalConsoleError(...args);
|
|
});
|
|
});
|
|
afterAll(() => {
|
|
(console.error as jest.Mock).mockRestore?.();
|
|
});
|
|
|
|
describe('compareReports (benchmark-compare)', () => {
|
|
// ---------------------------------------------------------------------------
|
|
// Happy path
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('passes with no flags when both reports are identical', () => {
|
|
const runs = [
|
|
makeRun({quant: 'q4_0'}),
|
|
makeRun({quant: 'q5_k_m', pp_avg: 280, tg_avg: 22}),
|
|
];
|
|
const result = compareReports(makeReport(runs), makeReport(runs));
|
|
|
|
expect(result.pass).toBe(true);
|
|
expect(result.rows).toHaveLength(2);
|
|
expect(result.rows.every(r => !r.flagged)).toBe(true);
|
|
expect(result.rows.every(r => r.flags.length === 0)).toBe(true);
|
|
expect(result.missing_in_current).toHaveLength(0);
|
|
expect(result.missing_in_baseline).toHaveLength(0);
|
|
expect(result.threshold_pct).toBe(15); // default
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Numeric-regression flags
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('flags a >15% pp regression', () => {
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
// -20% pp, tg unchanged
|
|
const current = makeReport([makeRun({pp_avg: 240, tg_avg: 24})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(result.rows[0].delta_pp_pct).toBeCloseTo(-20, 1);
|
|
expect(result.rows[0].flags.some(f => f.startsWith('pp_regression'))).toBe(
|
|
true,
|
|
);
|
|
expect(result.rows[0].flags.some(f => f.startsWith('tg_regression'))).toBe(
|
|
false,
|
|
);
|
|
});
|
|
|
|
it('flags a >15% tg regression', () => {
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
// pp unchanged, -25% tg
|
|
const current = makeReport([makeRun({pp_avg: 300, tg_avg: 18})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(result.rows[0].delta_tg_pct).toBeCloseTo(-25, 1);
|
|
expect(result.rows[0].flags.some(f => f.startsWith('tg_regression'))).toBe(
|
|
true,
|
|
);
|
|
expect(result.rows[0].flags.some(f => f.startsWith('pp_regression'))).toBe(
|
|
false,
|
|
);
|
|
});
|
|
|
|
it('does NOT flag when both deltas are inside the threshold', () => {
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
// -10% pp, -10% tg; both inside 15%
|
|
const current = makeReport([makeRun({pp_avg: 270, tg_avg: 21.6})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(true);
|
|
expect(result.rows[0].flagged).toBe(false);
|
|
});
|
|
|
|
it('does NOT flag improvements (positive deltas past threshold)', () => {
|
|
// The compare script reports regressions only: a +20% pp jump is an
|
|
// improvement and must NOT be flagged.
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
const current = makeReport([makeRun({pp_avg: 360, tg_avg: 30})]); // +20%, +25%
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(true);
|
|
expect(result.rows[0].flagged).toBe(false);
|
|
expect(result.rows[0].delta_pp_pct).toBeCloseTo(20, 1);
|
|
expect(result.rows[0].delta_tg_pct).toBeCloseTo(25, 1);
|
|
});
|
|
|
|
it('uses OR semantics: flags when pp OR tg regresses (not AND)', () => {
|
|
// Contrast with memory-compare which requires BOTH thresholds.
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
// -20% pp but +5% tg — one dim regresses, the other improves.
|
|
const current = makeReport([makeRun({pp_avg: 240, tg_avg: 25.2})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
});
|
|
|
|
it('respects a custom --pct threshold', () => {
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
const current = makeReport([makeRun({pp_avg: 276, tg_avg: 24})]); // -8% pp
|
|
|
|
// Default (15%): passes
|
|
expect(compareReports(baseline, current).pass).toBe(true);
|
|
|
|
// Custom 5%: fails
|
|
const strict = compareReports(baseline, current, {pct: 5});
|
|
expect(strict.pass).toBe(false);
|
|
expect(strict.threshold_pct).toBe(5);
|
|
expect(strict.rows[0].flags.some(f => f.startsWith('pp_regression'))).toBe(
|
|
true,
|
|
);
|
|
});
|
|
|
|
it('flags status regression when current row is failed (null metrics, ok->failed)', () => {
|
|
// A failed run typically has pp_avg=null, tg_avg=null. The numeric
|
|
// deltas remain null (un-comparable), but the status flip from `ok` to
|
|
// `failed` MUST be flagged so a regression is not silenced.
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
const current = makeReport([
|
|
makeRun({pp_avg: null, tg_avg: null, status: 'failed'}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.rows[0].delta_pp_pct).toBeNull();
|
|
expect(result.rows[0].delta_tg_pct).toBeNull();
|
|
// No numeric-regression flag because both deltas are null...
|
|
expect(
|
|
result.rows[0].flags.filter(
|
|
f => f.startsWith('pp_regression') || f.startsWith('tg_regression'),
|
|
),
|
|
).toHaveLength(0);
|
|
// ...but the status flip is the regression signal.
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(result.rows[0].flags).toContain('status_regression(failed)');
|
|
// Mutual exclusivity invariant: status_regression and *_null_regression
|
|
// never both fire on the same row. The null-regression flags are gated
|
|
// on `curRow.status === 'ok'`, which cannot coexist with a status flip.
|
|
expect(result.rows[0].flags).not.toContain('pp_null_regression');
|
|
expect(result.rows[0].flags).not.toContain('tg_null_regression');
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
it('does NOT flag when baseline failed and current ok (a previously-broken cell now works)', () => {
|
|
// The inverse of an `ok->failed` flip: if baseline was already broken
|
|
// (status:'failed' with null metrics) and the current run recovers
|
|
// (status:'ok' with valid metrics), the row has improved and must NOT
|
|
// be flagged. status_regression is one-way.
|
|
const baseline = makeReport([
|
|
makeRun({status: 'failed', pp_avg: null, tg_avg: null}),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({status: 'ok', pp_avg: 300, tg_avg: 24}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.rows[0].flagged).toBe(false);
|
|
expect(result.rows[0].flags).toHaveLength(0);
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('treats zero baseline as un-comparable (avoids division by zero)', () => {
|
|
const baseline = makeReport([makeRun({pp_avg: 0, tg_avg: 0})]);
|
|
const current = makeReport([makeRun({pp_avg: 100, tg_avg: 10})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.rows[0].delta_pp_pct).toBeNull();
|
|
expect(result.rows[0].delta_tg_pct).toBeNull();
|
|
expect(result.rows[0].flagged).toBe(false);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// effective_backend mismatch (independent of numeric deltas)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('flags an effective_backend mismatch even when pp/tg are stable', () => {
|
|
// Same numbers, but GPU run silently fell back to CPU in current.
|
|
const baseline = makeReport([
|
|
makeRun({effective_backend: 'opencl', pp_avg: 300, tg_avg: 24}),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({
|
|
effective_backend: 'cpu+opencl-partial',
|
|
pp_avg: 300,
|
|
tg_avg: 24,
|
|
}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(
|
|
result.rows[0].flags.some(f => f.startsWith('effective_backend:')),
|
|
).toBe(true);
|
|
expect(
|
|
result.rows[0].flags.find(f => f.startsWith('effective_backend:')),
|
|
).toBe('effective_backend:opencl->cpu+opencl-partial');
|
|
// Numeric regressions NOT present.
|
|
expect(
|
|
result.rows[0].flags.some(
|
|
f => f.startsWith('pp_regression') || f.startsWith('tg_regression'),
|
|
),
|
|
).toBe(false);
|
|
});
|
|
|
|
it('stacks effective_backend mismatch with numeric regressions', () => {
|
|
const baseline = makeReport([
|
|
makeRun({effective_backend: 'opencl', pp_avg: 300, tg_avg: 24}),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({effective_backend: 'cpu', pp_avg: 100, tg_avg: 10}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
expect(result.rows[0].flags.length).toBeGreaterThanOrEqual(3);
|
|
expect(result.rows[0].flags).toEqual(
|
|
expect.arrayContaining([
|
|
expect.stringMatching(/^pp_regression/),
|
|
expect.stringMatching(/^tg_regression/),
|
|
'effective_backend:opencl->cpu',
|
|
]),
|
|
);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Row-level set semantics
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('surfaces rows missing in current under missing_in_current AND fails the comparison (B3)', () => {
|
|
// A truncated current report (missing rows the baseline has) is a
|
|
// regression-class failure: the comparison cannot prove no regression
|
|
// for rows the current run did not exercise. `pass` must be false.
|
|
const baseline = makeReport([
|
|
makeRun({quant: 'q4_0'}),
|
|
makeRun({quant: 'q5_k_m', pp_avg: 280, tg_avg: 22}),
|
|
]);
|
|
const current = makeReport([makeRun({quant: 'q4_0'})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
// 4-tuple key format with the defensive 'app-default' fallback for
|
|
// legacy v1.0 runs that lack settings_fingerprint (WHAT 4g.1).
|
|
expect(result.missing_in_current).toEqual([
|
|
'qwen3-1.7b::q5_k_m::gpu::app-default',
|
|
]);
|
|
expect(result.missing_in_baseline).toEqual([]);
|
|
expect(result.rows).toHaveLength(1);
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
it('surfaces rows missing in current AND passes when intersecting rows are clean and no rows are missing (control)', () => {
|
|
// Sanity: make sure the missing-row gate does not over-fire when the
|
|
// report is fully covered. Both sides have q4_0 only.
|
|
const baseline = makeReport([makeRun({quant: 'q4_0'})]);
|
|
const current = makeReport([makeRun({quant: 'q4_0'})]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.missing_in_current).toEqual([]);
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('surfaces new rows under missing_in_baseline', () => {
|
|
const baseline = makeReport([makeRun({quant: 'q4_0'})]);
|
|
const current = makeReport([
|
|
makeRun({quant: 'q4_0'}),
|
|
makeRun({quant: 'q8_0', pp_avg: 200, tg_avg: 16}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.missing_in_baseline).toEqual([
|
|
'qwen3-1.7b::q8_0::gpu::app-default',
|
|
]);
|
|
expect(result.missing_in_current).toEqual([]);
|
|
});
|
|
|
|
it('keys rows by model::quant::requested_backend::fingerprint (4-tuple per WHAT 4g.1)', () => {
|
|
const runs = [
|
|
makeRun({requested_backend: 'cpu', effective_backend: 'cpu'}),
|
|
makeRun({requested_backend: 'gpu', effective_backend: 'opencl'}),
|
|
];
|
|
const result = compareReports(makeReport(runs), makeReport(runs));
|
|
|
|
expect(result.rows).toHaveLength(2);
|
|
const keys = result.rows.map(r => r.key).sort();
|
|
// Legacy v1.0 inputs lack fingerprint; rowKey falls back to
|
|
// 'app-default' (defensive default for direct-baseline use).
|
|
expect(keys).toEqual([
|
|
'qwen3-1.7b::q4_0::cpu::app-default',
|
|
'qwen3-1.7b::q4_0::gpu::app-default',
|
|
]);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Metadata passthrough
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('copies baseline/current commits and devices into the result', () => {
|
|
const baseline = makeReport([makeRun()], {
|
|
commit: 'base-sha',
|
|
device: 'BaselineDevice',
|
|
});
|
|
const current = makeReport([makeRun()], {
|
|
commit: 'cur-sha',
|
|
device: 'CurrentDevice',
|
|
});
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.baseline_commit).toBe('base-sha');
|
|
expect(result.current_commit).toBe('cur-sha');
|
|
expect(result.baseline_device).toBe('BaselineDevice');
|
|
expect(result.current_device).toBe('CurrentDevice');
|
|
});
|
|
|
|
it('round-trips 2-dp precision on the delta percentages', () => {
|
|
// 300 -> 289.88 is a -3.3733...% delta; rounded to 2 dp is -3.37.
|
|
const baseline = makeReport([makeRun({pp_avg: 300, tg_avg: 24})]);
|
|
const current = makeReport([makeRun({pp_avg: 289.88, tg_avg: 24})]);
|
|
const result = compareReports(baseline, current);
|
|
expect(result.rows[0].delta_pp_pct).toBe(-3.37);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// C1 defense-in-depth: null-metric regression flags. The screen-side
|
|
// invariant (status:'ok' rows always carry non-null pp/tg) is the primary
|
|
// guard; these flags fire only if that invariant ever regresses.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('flags pp_null_regression when both rows claim ok but current pp_avg is null', () => {
|
|
const baseline = makeReport([
|
|
makeRun({status: 'ok', pp_avg: 200, tg_avg: 20}),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({status: 'ok', pp_avg: null, tg_avg: 20}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(result.rows[0].flags).toContain('pp_null_regression');
|
|
expect(result.rows[0].flags).not.toContain('tg_null_regression');
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
it('flags tg_null_regression when both rows claim ok but current tg_avg is null', () => {
|
|
const baseline = makeReport([
|
|
makeRun({status: 'ok', pp_avg: 200, tg_avg: 20}),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({status: 'ok', pp_avg: 200, tg_avg: null}),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.rows[0].flagged).toBe(true);
|
|
expect(result.rows[0].flags).toContain('tg_null_regression');
|
|
expect(result.rows[0].flags).not.toContain('pp_null_regression');
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// C2: bench protocol mismatch (pp/tg/pl/nr drift between baseline and
|
|
// current). When detected, `pass` is false and the CLI exits 2.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('passes when both reports have matching bench params', () => {
|
|
const bench = {pp: 512, tg: 128, pl: 1, nr: 3};
|
|
const baseline = makeReport([makeRun()], {bench});
|
|
const current = makeReport([makeRun()], {bench});
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(true);
|
|
expect(result.bench_protocol_mismatch).toBeUndefined();
|
|
});
|
|
|
|
it('flags bench_protocol_mismatch when nr differs (1 vs 3)', () => {
|
|
const baseline = makeReport([makeRun()], {
|
|
bench: {pp: 512, tg: 128, pl: 1, nr: 1},
|
|
});
|
|
const current = makeReport([makeRun()], {
|
|
bench: {pp: 512, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.bench_protocol_mismatch).toEqual({
|
|
baseline: {pp: 512, tg: 128, pl: 1, nr: 1},
|
|
current: {pp: 512, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
});
|
|
|
|
it('flags bench_protocol_mismatch when pp differs', () => {
|
|
const baseline = makeReport([makeRun()], {
|
|
bench: {pp: 256, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
const current = makeReport([makeRun()], {
|
|
bench: {pp: 512, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.bench_protocol_mismatch?.baseline.pp).toBe(256);
|
|
expect(result.bench_protocol_mismatch?.current.pp).toBe(512);
|
|
});
|
|
|
|
it('gracefully degrades (no mismatch, pass=true) when baseline.bench is undefined', () => {
|
|
// The committed POCO baseline pre-dates the unification; it has no
|
|
// `bench` field. The compare script must keep loading it (with a
|
|
// stderr warning) rather than failing closed.
|
|
const baseline = makeReport([makeRun()]); // no bench
|
|
const current = makeReport([makeRun()], {
|
|
bench: {pp: 512, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.bench_protocol_mismatch).toBeUndefined();
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
it('gracefully degrades when current.bench is undefined', () => {
|
|
const baseline = makeReport([makeRun()], {
|
|
bench: {pp: 512, tg: 128, pl: 1, nr: 3},
|
|
});
|
|
const current = makeReport([makeRun()]); // no bench
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.bench_protocol_mismatch).toBeUndefined();
|
|
expect(result.pass).toBe(true);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// v1.1 — settings_fingerprint axis + Hexagon effective_backend arms
|
|
// (WHAT 4g.1, 4g.4)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
it('keys rows by 4-tuple including settings_fingerprint when both sides carry v1.1 fields', () => {
|
|
const runs = [
|
|
makeRun({
|
|
settings_fingerprint: 'cache_type_k=f16;...',
|
|
} as Partial<BenchmarkRunReport>),
|
|
makeRun({
|
|
settings_fingerprint: 'cache_type_k=q8_0;...',
|
|
} as Partial<BenchmarkRunReport>),
|
|
];
|
|
const result = compareReports(makeReport(runs), makeReport(runs));
|
|
expect(result.rows).toHaveLength(2);
|
|
const keys = result.rows.map(r => r.key).sort();
|
|
expect(keys).toEqual([
|
|
'qwen3-1.7b::q4_0::gpu::cache_type_k=f16;...',
|
|
'qwen3-1.7b::q4_0::gpu::cache_type_k=q8_0;...',
|
|
]);
|
|
});
|
|
|
|
it('reports a baseline app-default row missing in a current sweep run as missing_in_current (WHAT 4g.3)', () => {
|
|
// Operator-facing signal: current run swept cache_type_k but did
|
|
// not also include the app-default cell.
|
|
const baseline = makeReport([
|
|
makeRun({
|
|
settings_fingerprint: 'app-default',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({
|
|
settings_fingerprint: 'cache_type_k=q8_0;...',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.missing_in_current).toEqual([
|
|
'qwen3-1.7b::q4_0::gpu::app-default',
|
|
]);
|
|
expect(result.missing_in_baseline).toEqual([
|
|
'qwen3-1.7b::q4_0::gpu::cache_type_k=q8_0;...',
|
|
]);
|
|
expect(result.pass).toBe(false);
|
|
});
|
|
|
|
it('flags effective_backend mismatch on hexagon arms (WHAT 4g.4)', () => {
|
|
// Same numeric performance, but the run silently fell back from
|
|
// hexagon to cpu+hexagon-partial. Existing string-equality check
|
|
// covers the new arm without code changes.
|
|
const baseline = makeReport([
|
|
makeRun({
|
|
effective_backend: 'hexagon',
|
|
requested_backend: 'hexagon',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({
|
|
effective_backend: 'cpu+hexagon-partial',
|
|
requested_backend: 'hexagon',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
|
|
const result = compareReports(baseline, current);
|
|
|
|
expect(result.pass).toBe(false);
|
|
expect(result.rows[0].flags).toContain(
|
|
'effective_backend:hexagon->cpu+hexagon-partial',
|
|
);
|
|
});
|
|
|
|
it('passes when hexagon arms match (regression of cpu+hexagon-partial -> cpu+hexagon-partial is not a flag)', () => {
|
|
const baseline = makeReport([
|
|
makeRun({
|
|
effective_backend: 'cpu+hexagon-partial',
|
|
requested_backend: 'hexagon',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
const current = makeReport([
|
|
makeRun({
|
|
effective_backend: 'cpu+hexagon-partial',
|
|
requested_backend: 'hexagon',
|
|
} as Partial<BenchmarkRunReport>),
|
|
]);
|
|
const result = compareReports(baseline, current);
|
|
expect(result.pass).toBe(true);
|
|
expect(result.rows[0].flagged).toBe(false);
|
|
});
|
|
});
|