zzet--gortex/internal/eval/quality/confidence.go

package quality

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"path/filepath"
	"sort"
	"time"

	"github.com/zzet/gortex/internal/platform"
)

// ConfidenceRecord captures the candidate-score distribution for
// one search call. The distribution shape (top-1 vs top-2 gap,
// std-dev, ratio) tells you how confident the ranker was — a sharp
// top-1 with a long tail is a high-confidence answer; a flat
// distribution is the ranker shrugging.
type ConfidenceRecord struct {
	TS       time.Time `json:"ts"`
	Query    string    `json:"query"`
	Top1     float64   `json:"top1"`
	Top2     float64   `json:"top2,omitempty"`
	Mean     float64   `json:"mean"`
	StdDev   float64   `json:"std_dev"`
	Ratio12  float64   `json:"ratio_top1_top2,omitempty"` // top1 / top2; >>1 = confident
	K        int       `json:"k"`                          // number of scored candidates
}

// ConfidenceFromScores derives a ConfidenceRecord from a slice of
// per-candidate scores. Negative / zero K returns a zero record so
// the caller can opt out by passing an empty slice.
func ConfidenceFromScores(query string, scores []float64) ConfidenceRecord {
	r := ConfidenceRecord{
		TS:    time.Now().UTC(),
		Query: query,
		K:     len(scores),
	}
	if len(scores) == 0 {
		return r
	}
	sorted := make([]float64, len(scores))
	copy(sorted, scores)
	sort.Sort(sort.Reverse(sort.Float64Slice(sorted)))
	r.Top1 = sorted[0]
	if len(sorted) > 1 {
		r.Top2 = sorted[1]
		if r.Top2 != 0 {
			r.Ratio12 = r.Top1 / r.Top2
		}
	}
	var sum float64
	for _, v := range scores {
		sum += v
	}
	r.Mean = sum / float64(len(scores))
	var ss float64
	for _, v := range scores {
		ss += (v - r.Mean) * (v - r.Mean)
	}
	r.StdDev = math.Sqrt(ss / float64(len(scores)))
	return r
}

// ConfidenceTracker appends records to a JSONL log on disk. Safe
// for in-process concurrent appends because the file is opened
// with O_APPEND.
type ConfidenceTracker struct {
	Path string
}

// NewConfidenceTracker returns a tracker bound to a log path.
// Empty path is allowed — Record then no-ops without erroring.
func NewConfidenceTracker(path string) *ConfidenceTracker {
	return &ConfidenceTracker{Path: path}
}

// DefaultConfidenceLogPath is the canonical persistence location.
// An absolute $XDG_CACHE_HOME is honoured; otherwise the log stays
// under os.UserCacheDir() as before. Returns empty when no cache dir
// can be resolved.
func DefaultConfidenceLogPath() string {
	if v := os.Getenv("XDG_CACHE_HOME"); v == "" || !filepath.IsAbs(v) {
		if base, err := os.UserCacheDir(); err != nil || base == "" {
			return ""
		}
	}
	return filepath.Join(platform.OSCacheDir(), "confidence.jsonl")
}

// Record appends one record to the log. No-op when Path is empty.
func (t *ConfidenceTracker) Record(r ConfidenceRecord) error {
	if t.Path == "" {
		return nil
	}
	if err := os.MkdirAll(filepath.Dir(t.Path), 0o755); err != nil {
		return err
	}
	f, err := os.OpenFile(t.Path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
	if err != nil {
		return err
	}
	defer f.Close()
	body, err := json.Marshal(r)
	if err != nil {
		return err
	}
	body = append(body, '\n')
	_, err = f.Write(body)
	return err
}

// LoadConfidenceLog reads the log at path and returns records with
// ts >= since (zero since returns all). Malformed lines are
// skipped so a previous crash mid-write doesn't break readers.
func LoadConfidenceLog(path string, since time.Time) ([]ConfidenceRecord, error) {
	if path == "" {
		return nil, nil
	}
	f, err := os.Open(path)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, nil
		}
		return nil, fmt.Errorf("open confidence log: %w", err)
	}
	defer f.Close()
	r := bufio.NewReaderSize(f, 64*1024)
	out := []ConfidenceRecord{}
	for {
		line, err := r.ReadBytes('\n')
		if len(line) > 0 {
			if n := len(line); n > 0 && line[n-1] == '\n' {
				line = line[:n-1]
			}
			if len(line) > 0 {
				var rec ConfidenceRecord
				if json.Unmarshal(line, &rec) == nil {
					if since.IsZero() || !rec.TS.Before(since) {
						out = append(out, rec)
					}
				}
			}
		}
		if err != nil {
			if errors.Is(err, io.EOF) {
				break
			}
			return out, fmt.Errorf("read confidence log: %w", err)
		}
	}
	return out, nil
}

// ConfidenceSummary aggregates a slice of records into the headline
// statistics the `gortex eval quality confidence` subcommand emits.
type ConfidenceSummary struct {
	Count            int     `json:"count"`
	MedianTop1       float64 `json:"median_top1"`
	MedianRatio12    float64 `json:"median_ratio_top1_top2"`
	MedianStdDev     float64 `json:"median_std_dev"`
	LowConfidenceCount int   `json:"low_confidence_count"` // records with Ratio12 < 1.25
}

// SummarizeConfidence reduces a slice of records into the summary.
// Median across records; "low confidence" = Ratio12 < 1.25 (the
// ranker barely separated top-1 from top-2).
func SummarizeConfidence(records []ConfidenceRecord) ConfidenceSummary {
	s := ConfidenceSummary{Count: len(records)}
	if len(records) == 0 {
		return s
	}
	top1s := make([]float64, 0, len(records))
	ratios := make([]float64, 0, len(records))
	stds := make([]float64, 0, len(records))
	for _, r := range records {
		top1s = append(top1s, r.Top1)
		stds = append(stds, r.StdDev)
		if r.Ratio12 > 0 {
			ratios = append(ratios, r.Ratio12)
		}
		if r.Ratio12 > 0 && r.Ratio12 < 1.25 {
			s.LowConfidenceCount++
		}
	}
	s.MedianTop1 = medianFloats(top1s)
	s.MedianRatio12 = medianFloats(ratios)
	s.MedianStdDev = medianFloats(stds)
	return s
}

func medianFloats(vs []float64) float64 {
	if len(vs) == 0 {
		return 0
	}
	c := make([]float64, len(vs))
	copy(c, vs)
	sort.Float64s(c)
	return c[len(c)/2]
}