kenn-io--agentsview/cmd/benchgate/main_test.go

package main

import (
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"golang.org/x/perf/benchfmt"
)

func parseString(t *testing.T, input string) (benchSamples, []string) {
	t.Helper()
	got, syntaxErrs, err := parseBench(
		benchfmt.NewReader(strings.NewReader(input), "test"),
	)
	require.NoError(t, err)
	return got, syntaxErrs
}

func TestParseBench(t *testing.T) {
	tests := []struct {
		name       string
		input      string
		want       benchSamples
		wantSyntax int
	}{
		{
			name: "full benchmem line with tidied units",
			input: "goos: linux\n" +
				"pkg: example.com/x\n" +
				"BenchmarkFoo-8   \t 100\t 1234567 ns/op\t 2345 B/op\t 67 allocs/op\n" +
				"PASS\nok  \texample.com/x\t1.2s\n",
			want: benchSamples{
				"example.com/x.Foo-8": {
					"sec/op":    {1234567e-9},
					"B/op":      {2345},
					"allocs/op": {67},
				},
			},
		},
		{
			name: "multiple counts collect all samples",
			input: "BenchmarkFoo-8 100 200 ns/op 9 allocs/op\n" +
				"BenchmarkFoo-8 100 150 ns/op 8 allocs/op\n" +
				"BenchmarkFoo-8 100 180 ns/op 10 allocs/op\n",
			want: benchSamples{
				"Foo-8": {
					"sec/op":    {200e-9, 150e-9, 180e-9},
					"allocs/op": {9, 8, 10},
				},
			},
		},
		{
			name: "same benchmark name in different packages stays separate",
			input: "pkg: example.com/a\n" +
				"BenchmarkScan-4 10 100 ns/op\n" +
				"pkg: example.com/b\n" +
				"BenchmarkScan-4 10 900 ns/op\n",
			want: benchSamples{
				"example.com/a.Scan-4": {"sec/op": {100e-9}},
				"example.com/b.Scan-4": {"sec/op": {900e-9}},
			},
		},
		{
			name: "log lines and headers are ignored",
			input: "2026/07/03 10:20:36 discovered 40 files in 0s\n" +
				"cpu: Apple M5 Max\n",
			want: benchSamples{},
		},
		{
			name: "result line corrupted by interleaved log output is a syntax error",
			input: "BenchmarkSyncAllWarmNoop-18   \t2026/07/03 15:39:07 discovered 40 files in 0s\n" +
				"       5\t   3581975 ns/op\t  212433 B/op\t    2760 allocs/op\n",
			want:       benchSamples{},
			wantSyntax: 1,
		},
		{
			name:  "custom ReportMetric units are kept",
			input: "BenchmarkBaz-2 10 900 ns/op 3 things/op\n",
			want: benchSamples{
				"Baz-2": {
					"sec/op":    {900e-9},
					"things/op": {3},
				},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, syntaxErrs := parseString(t, tt.input)
			assert.Len(t, syntaxErrs, tt.wantSyntax)
			require.Len(t, got, len(tt.want))
			for name, wantUnits := range tt.want {
				gotUnits, ok := got[name]
				require.True(t, ok, "missing benchmark %s", name)
				for unit, wantVals := range wantUnits {
					assert.InDeltaSlice(
						t, wantVals, gotUnits[unit], 1e-15,
						"%s %s", name, unit,
					)
				}
			}
		})
	}
}

func testGates() []gate {
	return []gate{
		{unit: "allocs/op", maxRatio: 1.25, floor: 64, worstCase: true},
		{unit: "B/op", maxRatio: 1.35, floor: 16_384, worstCase: true},
		{
			unit: "sec/op", maxRatio: 2.0, floor: 100_000e-9,
			needSignificance: true,
		},
	}
}

// noisy fabricates a benchmark sample of n values spread ±2% around
// center, so significance tests have a realistic distribution.
func noisy(center float64, n int) []float64 {
	out := make([]float64, n)
	for i := range out {
		out[i] = center * (0.98 + 0.01*float64(i%5))
	}
	return out
}

func TestCompare(t *testing.T) {
	tests := []struct {
		name       string
		old, new   benchSamples
		wantUnits  []string // units of expected violations, in order
		wantReport []string // substrings that must appear in the report
	}{
		{
			name: "within thresholds passes",
			old: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1e-3, 6),
					"B/op":      {100_000},
					"allocs/op": {1000},
				},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1.5e-3, 6),
					"B/op":      {120_000},
					"allocs/op": {1100},
				},
			},
			wantUnits: nil,
		},
		{
			name: "alloc regression fails even with a single run",
			old: benchSamples{
				"BenchmarkFoo-8": {"allocs/op": {1000}},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {"allocs/op": {2000}},
			},
			wantUnits: []string{"allocs/op"},
		},
		{
			name: "significant time blowup fails",
			old: benchSamples{
				"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 6)},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {"sec/op": noisy(5e-3, 6)},
			},
			wantUnits: []string{"sec/op"},
		},
		{
			name: "time blowup without significance is reported, not gated",
			old: benchSamples{
				"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 5)},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op": append(noisy(1e-3, 4), 5e-3),
				},
			},
			wantUnits:  nil,
			wantReport: []string{"not significant, not gated"},
		},
		{
			name: "tiny baseline below floor is not gated",
			old: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    {500e-9},
					"B/op":      {128},
					"allocs/op": {3},
				},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    {5000e-9},
					"B/op":      {1280},
					"allocs/op": {30},
				},
			},
			wantUnits:  nil,
			wantReport: []string{"not gated"},
		},
		{
			name: "new benchmark without baseline is reported, not gated",
			old:  benchSamples{},
			new: benchSamples{
				"BenchmarkNew-8": {"allocs/op": {99999}},
			},
			wantUnits:  nil,
			wantReport: []string{"no baseline to compare"},
		},
		{
			name: "removed benchmark is reported, not gated",
			old: benchSamples{
				"BenchmarkGone-8": {"sec/op": {1e-3}},
			},
			new:        benchSamples{},
			wantUnits:  nil,
			wantReport: []string{"missing from candidate"},
		},
		{
			name: "gated unit missing from the baseline is reported, not gated",
			old: benchSamples{
				"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 6)},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1e-3, 6),
					"allocs/op": {90000},
				},
			},
			wantUnits:  nil,
			wantReport: []string{"allocs/op missing from baseline, not gated"},
		},
		{
			name: "custom ReportMetric unit is reported as ungated",
			old: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1e-3, 6),
					"things/op": {3},
				},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1e-3, 6),
					"things/op": {900},
				},
			},
			wantUnits:  nil,
			wantReport: []string{"things/op has no gate, not gated"},
		},
		{
			name: "multiple regressions are all reported",
			old: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(1e-3, 6),
					"B/op":      {100_000},
					"allocs/op": {1000},
				},
			},
			new: benchSamples{
				"BenchmarkFoo-8": {
					"sec/op":    noisy(9e-3, 6),
					"B/op":      {900_000},
					"allocs/op": {9000},
				},
			},
			wantUnits: []string{"allocs/op", "B/op", "sec/op"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			report, violations, issues := compare(tt.old, tt.new, testGates())
			units := make([]string, 0, len(violations))
			for _, v := range violations {
				units = append(units, v.unit)
			}
			assert.Empty(t, issues)
			if len(tt.wantUnits) == 0 {
				assert.Empty(t, violations)
			} else {
				assert.Equal(t, tt.wantUnits, units)
			}
			joined := strings.Join(report, "\n")
			for _, want := range tt.wantReport {
				assert.Contains(t, joined, want)
			}
		})
	}
}

// TestCompareOutlierRunPolicy pins the split policy for a single
// outlier among repeated -count runs of one benchmark. allocs/op is
// deterministic for identical code and iteration counts, so one
// outlier run means a real intermittent allocation path and must
// fail even though the median is unchanged. Wall-clock time is
// summarized by its median, so one slow run on a noisy runner cannot
// fail the gate on its own.
func TestCompareOutlierRunPolicy(t *testing.T) {
	t.Run("alloc outlier run fails", func(t *testing.T) {
		old := benchSamples{
			"BenchmarkFoo-8": {"allocs/op": {1000, 1000, 1000, 1000, 1000}},
		}
		next := benchSamples{
			"BenchmarkFoo-8": {"allocs/op": {1000, 1000, 9000, 1000, 1000}},
		}
		_, violations, issues := compare(old, next, testGates())
		require.Len(t, violations, 1)
		assert.Empty(t, issues)
		assert.Equal(t, "allocs/op", violations[0].unit)
		assert.InDelta(t, 9000, violations[0].new, 1e-9,
			"the worst run is what gets gated")
	})

	t.Run("time outlier run does not fail", func(t *testing.T) {
		old := benchSamples{
			"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 6)},
		}
		next := benchSamples{
			"BenchmarkFoo-8": {"sec/op": append(noisy(1e-3, 5), 9e-3)},
		}
		_, violations, issues := compare(old, next, testGates())
		assert.Empty(t, issues)
		assert.Empty(t, violations)
	})
}

// TestCompareMissingCandidateUnit pins the asymmetric missing-unit
// policy: a gated unit the baseline has but the candidate lost
// (e.g. -benchmem dropped from the candidate run) is a config error
// so the gate exits 2 instead of silently disabling that metric.
func TestCompareMissingCandidateUnit(t *testing.T) {
	old := benchSamples{
		"BenchmarkFoo-8": {
			"sec/op":    noisy(1e-3, 6),
			"allocs/op": {1000},
		},
	}
	next := benchSamples{
		"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 6)},
	}
	report, violations, issues := compare(old, next, testGates())
	assert.Empty(t, violations)
	require.Len(t, issues, 1)
	assert.Contains(t, issues[0].msg,
		"allocs/op present in baseline but missing from candidate")
	assert.Contains(t, strings.Join(report, "\n"),
		"allocs/op missing from candidate")
}

// TestCompareTimeGateSampleCounts pins the asymmetric sample-count
// policy: too few candidate samples is a configuration error (the
// candidate run is under the workflow's control), while a short
// baseline is merely reported and not gated (the base run may
// legitimately be partial).
func TestCompareTimeGateSampleCounts(t *testing.T) {
	t.Run("too few candidate samples is a config issue", func(t *testing.T) {
		old := benchSamples{"BenchmarkFoo-8": {"sec/op": noisy(1e-3, 6)}}
		next := benchSamples{"BenchmarkFoo-8": {"sec/op": {2e-3}}}
		_, violations, issues := compare(old, next, testGates())
		assert.Empty(t, violations)
		require.Len(t, issues, 1)
		assert.Contains(t, issues[0].msg, "at least 5 candidate samples")
	})

	t.Run("short baseline is reported, not a config issue", func(t *testing.T) {
		old := benchSamples{"BenchmarkFoo-8": {"sec/op": {1e-3, 1e-3}}}
		next := benchSamples{"BenchmarkFoo-8": {"sec/op": noisy(9e-3, 6)}}
		report, violations, issues := compare(old, next, testGates())
		assert.Empty(t, violations)
		assert.Empty(t, issues)
		assert.Contains(t, strings.Join(report, "\n"),
			"baseline has only 2 sample(s)")
	})
}

func TestRender(t *testing.T) {
	sampleViolation := violation{
		name: "BenchmarkFoo-8", unit: "allocs/op",
		old: 1000, new: 2000, ratio: 2.0, maxRatio: 1.25,
	}
	tests := []struct {
		name     string
		r        results
		wantCode int
		wantOut  []string
	}{
		{
			name:     "clean run passes",
			r:        results{report: []string{"BenchmarkFoo-8: ok"}, newCount: 1},
			wantCode: 0,
			wantOut:  []string{"no regressions beyond thresholds"},
		},
		{
			name: "violations exit 1",
			r: results{
				violations: []violation{sampleViolation},
				newCount:   1,
			},
			wantCode: 1,
			wantOut:  []string{"1 regression(s)", "allocs/op regressed 2.00x"},
		},
		{
			name: "violations still print when a config issue forces exit 2",
			r: results{
				violations: []violation{sampleViolation},
				issues:     []configIssue{{name: "BenchmarkFoo-8", msg: "too few"}},
				newCount:   1,
			},
			wantCode: 2,
			wantOut: []string{
				"allocs/op regressed 2.00x",
				"invalid benchmark configuration",
			},
		},
		{
			name: "candidate corrupted capture exits 2 and is described",
			r: results{
				newCount:  1,
				newSyntax: []string{"test:3: no iteration count"},
			},
			wantCode: 2,
			wantOut: []string{
				"candidate capture is corrupted",
				"no iteration count",
			},
		},
		{
			name: "baseline corrupted capture is reported but does not fail",
			r: results{
				newCount:  1,
				oldSyntax: []string{"test:3: no iteration count"},
			},
			wantCode: 0,
			wantOut: []string{
				"baseline capture is corrupted",
				"no iteration count",
				"no regressions beyond thresholds",
			},
		},
		{
			name: "baseline corruption still allows regressions to fail",
			r: results{
				violations: []violation{sampleViolation},
				newCount:   1,
				oldSyntax:  []string{"test:3: no iteration count"},
			},
			wantCode: 1,
			wantOut: []string{
				"baseline capture is corrupted",
				"1 regression(s)",
			},
		},
		{
			name:     "empty candidate exits 2",
			r:        results{newCount: 0},
			wantCode: 2,
			wantOut:  []string{"candidate output contains no benchmarks"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			out, code := render(tt.r)
			assert.Equal(t, tt.wantCode, code)
			for _, want := range tt.wantOut {
				assert.Contains(t, out, want)
			}
		})
	}
}

func TestViolationString(t *testing.T) {
	v := violation{
		name: "BenchmarkFoo-8", unit: "allocs/op",
		old: 1000, new: 2000, ratio: 2.0, maxRatio: 1.25,
	}
	got := v.String()
	assert.Contains(t, got, "BenchmarkFoo-8")
	assert.Contains(t, got, "allocs/op regressed 2.00x")
	assert.Contains(t, got, "limit 1.25x")
}