package main import ( "context" "fmt" "os" "path/filepath" "github.com/spf13/cobra" "go.uber.org/zap" "github.com/zzet/gortex/internal/eval/parity" ) var ( parityCacheDir string parityUpdate bool parityLang string parityEpsilon float64 ) var evalParityCmd = &cobra.Command{ Use: "parity", Short: "Measure per-language cross-file coverage on benchmark repos and check it against the committed baseline", Long: `Clone (and cache) the per-language benchmark corpus, index each repo, and measure the share of symbol-bearing source files that have at least one resolved cross-file dependent. Default (assert) mode compares the measured coverage against the committed baseline (internal/eval/parity/baseline.json) and exits non-zero if any language regressed. --update prints the measured coverage as a baseline document instead; redirect it into baseline.json and commit to capture or refresh the baseline. gortex eval parity # assert against the committed baseline gortex eval parity --lang go # one language only gortex eval parity --update > internal/eval/parity/baseline.json `, RunE: runEvalParity, } func init() { evalParityCmd.Flags().StringVar(&parityCacheDir, "cache-dir", "", "where benchmark repos are cloned/cached (default: a parity-cache dir under the OS temp dir)") evalParityCmd.Flags().BoolVar(&parityUpdate, "update", false, "print measured coverage as a baseline JSON document instead of asserting") evalParityCmd.Flags().StringVar(&parityLang, "lang", "", "only run the benchmark repo for this language") evalParityCmd.Flags().Float64Var(&parityEpsilon, "epsilon", 0.005, "tolerance below baseline before a language counts as a regression") evalCmd.AddCommand(evalParityCmd) } func runEvalParity(cmd *cobra.Command, args []string) error { cacheDir := parityCacheDir if cacheDir == "" { cacheDir = filepath.Join(os.TempDir(), "gortex-parity-cache") } ctx := context.Background() var measured []parity.LanguageCoverage for _, repo := range parity.BenchRepos() { if parityLang != "" && repo.Language != parityLang { continue } fmt.Fprintf(cmd.ErrOrStderr(), "[gortex eval parity] %-10s %s\n", repo.Language, repo.URL) dir, err := parity.EnsureRepo(cacheDir, repo) if err != nil { fmt.Fprintf(cmd.ErrOrStderr(), " clone failed: %v — skipping\n", err) continue } g, cleanup, err := indexRepoForInit(ctx, dir, zap.NewNop()) if err != nil { fmt.Fprintf(cmd.ErrOrStderr(), " index failed: %v — skipping\n", err) continue } covs := parity.CoverageOf(g) cleanup() // release the temp store before the next repo for _, c := range covs { if c.Language != repo.Language { continue // a Go repo may carry a few yaml/json files; measure its own language } measured = append(measured, c) fmt.Fprintf(cmd.ErrOrStderr(), " coverage %.1f%% (%d/%d files)\n", c.Coverage*100, c.CoveredFiles, c.SymbolFiles) } } if parityUpdate { out, err := parity.MarshalBaseline(measured) if err != nil { return err } fmt.Fprintln(cmd.OutOrStdout(), string(out)) return nil } baseline, err := parity.LoadBaseline() if err != nil { return err } if len(baseline) == 0 { fmt.Fprintln(cmd.ErrOrStderr(), "[gortex eval parity] no committed baseline yet — run with --update and commit the output to internal/eval/parity/baseline.json") return nil } regs := baseline.Check(measured, parityEpsilon) for _, r := range regs { fmt.Fprintf(cmd.ErrOrStderr(), "[gortex eval parity] REGRESSION %s: %.1f%% < baseline %.1f%%\n", r.Language, r.Measured*100, r.Baseline*100) } if len(regs) > 0 { return fmt.Errorf("%d language(s) regressed below the parity baseline", len(regs)) } fmt.Fprintln(cmd.ErrOrStderr(), "[gortex eval parity] all measured languages hold or beat the baseline") return nil }