wehub-resource-sync
a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
63 lines
1.9 KiB
Go
63 lines
1.9 KiB
Go
package stdbench
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func queriesByID(ds Dataset) map[string]Query {
|
|
out := make(map[string]Query, len(ds.Queries))
|
|
for _, q := range ds.Queries {
|
|
out[q.ID] = q
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestLoadCoIR(t *testing.T) {
|
|
ds, err := LoadCoIR("testdata/coir")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "CoIR", ds.Name)
|
|
require.Len(t, ds.Corpus, 3)
|
|
// Only queries present in qrels are kept — q3 has no judgement.
|
|
require.Len(t, ds.Queries, 2)
|
|
|
|
q := queriesByID(ds)
|
|
require.Equal(t, "search a sorted array for an element", q["q1"].Text)
|
|
require.Equal(t, map[string]int{"d1": 2}, q["q1"].Relevant)
|
|
require.Equal(t, map[string]int{"d3": 1}, q["q2"].Relevant)
|
|
// Corpus text folds the title into the body.
|
|
require.Contains(t, ds.Corpus[0].Text, "binary search")
|
|
}
|
|
|
|
func TestLoadContextBench(t *testing.T) {
|
|
ds, err := LoadContextBench("testdata/contextbench.jsonl")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "ContextBench", ds.Name)
|
|
// Both tasks carry the same 2-candidate pool — deduplicated to 2.
|
|
require.Len(t, ds.Corpus, 2)
|
|
require.Len(t, ds.Queries, 2)
|
|
require.Equal(t, 2, ds.RelevantCount())
|
|
|
|
q := queriesByID(ds)
|
|
// A bare string relevance entry defaults to grade 1.
|
|
require.Equal(t, map[string]int{"c1": 1}, q["t1"].Relevant)
|
|
// An {id,score} relevance object keeps its graded score.
|
|
require.Equal(t, map[string]int{"c2": 3}, q["t2"].Relevant)
|
|
}
|
|
|
|
func TestLoadSWEContextBench_SharesJSONLLoader(t *testing.T) {
|
|
// SWE-ContextBench reuses the JSONL task loader — only the dataset
|
|
// name differs from ContextBench.
|
|
ds, err := LoadSWEContextBench("testdata/contextbench.jsonl")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "SWE-ContextBench", ds.Name)
|
|
require.Len(t, ds.Queries, 2)
|
|
require.Len(t, ds.Corpus, 2)
|
|
}
|
|
|
|
func TestLoadCoIR_MissingDirectory(t *testing.T) {
|
|
_, err := LoadCoIR("testdata/no-such-dir")
|
|
require.Error(t, err)
|
|
}
|