zzet--gortex/internal/eval/recall/judge_test.go

package recall

import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestVerdictKey_OrderIndependent(t *testing.T) {
	k1 := verdictKey("claude", "find X", []string{"a/b.go::Foo", "c/d.go::Bar"})
	k2 := verdictKey("claude", "find X", []string{"c/d.go::Bar", "a/b.go::Foo"})
	assert.Equal(t, k1, k2, "verdict key must be independent of top-K order")
}

func TestVerdictKey_ChangesWithModel(t *testing.T) {
	k1 := verdictKey("claude-haiku-4-5", "q", []string{"a"})
	k2 := verdictKey("claude-opus-4", "q", []string{"a"})
	assert.NotEqual(t, k1, k2, "verdict key must differ when the model differs")
}

func TestJudgeCacheRoundTrip(t *testing.T) {
	tmp := t.TempDir()
	path := filepath.Join(tmp, "judge_cache.json")
	j := &Judge{
		Model:     "claude-haiku-4-5",
		APIKey:    "test-key",
		CachePath: path,
	}
	j.loadCache()
	j.cacheMu.Lock()
	j.cache["abc"] = true
	j.cache["def"] = false
	j.cacheMu.Unlock()
	require.NoError(t, j.saveCache())

	// Reload from disk.
	j2 := &Judge{CachePath: path}
	j2.loadCache()
	assert.Equal(t, true, j2.cache["abc"])
	assert.Equal(t, false, j2.cache["def"])

	// Sanity: on-disk format is valid JSON.
	raw, err := os.ReadFile(path)
	require.NoError(t, err)
	var decoded map[string]bool
	require.NoError(t, json.Unmarshal(raw, &decoded))
	assert.Len(t, decoded, 2)
}

func TestNewJudge_NoAPIKeyReturnsNil(t *testing.T) {
	t.Setenv("ANTHROPIC_API_KEY", "")
	assert.Nil(t, NewJudge("claude-haiku-4-5"))
}

func TestNewJudge_EmptyModelReturnsNil(t *testing.T) {
	t.Setenv("ANTHROPIC_API_KEY", "fake")
	assert.Nil(t, NewJudge(""))
}

func TestApplyJudge_NilIsNoop(t *testing.T) {
	r := Report{Rankers: []RankerResult{
		{Name: "x", Hits: map[int]int{20: 5}, Recall: map[int]float64{20: 0.5}, Cases: 10},
	}}
	before := r.Rankers[0].Hits[20]
	rescued, errs := ApplyJudge(&r, nil)
	assert.Equal(t, 0, rescued)
	assert.Nil(t, errs)
	assert.Equal(t, before, r.Rankers[0].Hits[20])
}

func TestRun_CapturesMisses(t *testing.T) {
	fixture := Fixture{
		Name: "t",
		Cases: []Case{
			{ID: "hit", Query: "q1", Tier: TierExact, Expected: []string{"a"}},
			{ID: "miss", Query: "q2", Tier: TierExact, Expected: []string{"z"}},
		},
	}
	r := staticRanker("x", []string{"a", "b", "c"})
	report := Run(fixture, []Ranker{r}, nil)
	require.Len(t, report.Rankers, 1)
	assert.Len(t, report.Rankers[0].Misses, 1)
	miss := report.Rankers[0].Misses[0]
	assert.Equal(t, "miss", miss.CaseID)
	assert.Equal(t, []string{"z"}, miss.Expected)
	assert.Equal(t, []string{"a", "b", "c"}, miss.Top)
	assert.Nil(t, miss.JudgedHit)
}

func TestApplyJudge_RescuesMissWithStubbedVerdict(t *testing.T) {
	// Shape-level test: we can't hit the real API, so pre-seed the
	// judge cache with a "yes" verdict for a known query+top.
	tmp := t.TempDir()
	cachePath := filepath.Join(tmp, "cache.json")
	query := "find me a writer"
	top := []string{"pkg/x.go::WriteThing"}

	judge := &Judge{
		Model:     "claude-stub",
		APIKey:    "stub",
		CachePath: cachePath,
	}
	judge.loadCache()
	judge.cacheMu.Lock()
	judge.cache[verdictKey("claude-stub", query, top)] = true
	judge.cacheMu.Unlock()

	report := Report{
		Rankers: []RankerResult{{
			Name:   "r",
			Cases:  1,
			Hits:   map[int]int{1: 0, 5: 0, 20: 0},
			Recall: map[int]float64{1: 0, 5: 0, 20: 0},
			Misses: []Miss{{
				CaseID:   "c1",
				Query:    query,
				Expected: []string{"pkg/x.go::OtherWriter"},
				Top:      top,
			}},
		}},
	}
	rescued, errs := ApplyJudge(&report, judge)
	assert.Empty(t, errs)
	assert.Equal(t, 1, rescued)
	// Judged hits bump the biggest-K bucket (20).
	assert.Equal(t, 1, report.Rankers[0].Hits[20])
	assert.Equal(t, 1.0, report.Rankers[0].Recall[20])
	// R@1 / R@5 are NOT credited by the judge — we don't know the
	// judged rank, so the conservative call is max-K only.
	assert.Equal(t, 0, report.Rankers[0].Hits[1])
	require.NotNil(t, report.Rankers[0].Misses[0].JudgedHit)
	assert.True(t, *report.Rankers[0].Misses[0].JudgedHit)
}