wehub-resource-sync
7a0da7932b
Backwards Compatibility / Verify Encryption Constants (push) Waiting to run
Backwards Compatibility / PyPI Version Compatibility (push) Waiting to run
Backwards Compatibility / Database Migration Tests (push) Waiting to run
CodeQL Advanced / Analyze (javascript-typescript) (push) Waiting to run
CodeQL Advanced / Analyze (python) (push) Waiting to run
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-form] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-metrics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [research-workflow] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [settings-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [history-news] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [library] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [link-analytics] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [mobile] (push) Blocked by required conditions
Docker Tests (Consolidated) / detect-changes (push) Waiting to run
Docker Tests (Consolidated) / Build Test Image (push) Waiting to run
Docker Tests (Consolidated) / All Pytest Tests + Coverage (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [accessibility] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [api-crud] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-login] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-pages] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [auth-register] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-core] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [chat-lifecycle] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) [error-benchmark] (push) Blocked by required conditions
Docker Tests (Consolidated) / UI Tests (Puppeteer) (push) Blocked by required conditions
Docker Tests (Consolidated) / Accessibility Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Unit Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / LLM Example Tests (push) Blocked by required conditions
Docker Tests (Consolidated) / Production Image Smoke Test (push) Blocked by required conditions
Docker Tests (Consolidated) / Infrastructure Tests (push) Blocked by required conditions
OSSF Scorecard / OSSF Security Scorecard Analysis (push) Waiting to run
OSV-Scanner (Scheduled) / scan-scheduled (push) Failing after 0s
Create Release / test-gate (push) Has been cancelled
Create Release / release-gate (push) Has been cancelled
Create Release / ci-gate (push) Has been cancelled
Create Release / version-check (push) Has been cancelled
Create Release / e2e-test-gate (push) Has been cancelled
Create Release / responsive-test-gate (push) Has been cancelled
Create Release / compat-test-gate (push) Has been cancelled
Create Release / compose-integration-gate (push) Has been cancelled
Create Release / vulture-gate (push) Has been cancelled
Create Release / build (push) Has been cancelled
Create Release / provenance (push) Has been cancelled
Create Release / prerelease-docker (push) Has been cancelled
Create Release / publish-docker (push) Has been cancelled
Create Release / create-release (push) Has been cancelled
Create Release / cleanup-changelog (push) Has been cancelled
Create Release / trigger-pypi (push) Has been cancelled
Create Release / monitor-pypi (push) Has been cancelled
Create Release / Clean up orphan prerelease tags and signatures (push) Has been cancelled
365 lines
12 KiB
Python
365 lines
12 KiB
Python
"""
|
|
Tests for benchmarks/metrics/statistics.py
|
|
|
|
Tests cover:
|
|
- normal_quantile inverse CDF
|
|
- wilson_score_interval confidence intervals
|
|
- proportion_std_error
|
|
- sample_size_for_difference power analysis
|
|
- Integration: calculate_metrics returns accuracy_ci
|
|
"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
|
|
class TestNormalQuantile:
|
|
"""Tests for the normal_quantile function."""
|
|
|
|
def test_median(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
assert abs(normal_quantile(0.5) - 0.0) < 1e-6
|
|
|
|
def test_975_gives_196(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
assert abs(normal_quantile(0.975) - 1.96) < 0.001
|
|
|
|
def test_025_gives_negative_196(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
assert abs(normal_quantile(0.025) - (-1.96)) < 0.001
|
|
|
|
def test_symmetry(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
for p in [0.01, 0.05, 0.1, 0.25]:
|
|
assert abs(normal_quantile(p) + normal_quantile(1 - p)) < 1e-6
|
|
|
|
def test_raises_on_zero(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
with pytest.raises(ValueError):
|
|
normal_quantile(0.0)
|
|
|
|
def test_raises_on_one(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
with pytest.raises(ValueError):
|
|
normal_quantile(1.0)
|
|
|
|
def test_raises_on_negative(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
with pytest.raises(ValueError):
|
|
normal_quantile(-0.5)
|
|
|
|
def test_extreme_tails(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
normal_quantile,
|
|
)
|
|
|
|
# p=0.001 -> z ~ -3.09
|
|
assert abs(normal_quantile(0.001) - (-3.09)) < 0.01
|
|
# p=0.999 -> z ~ 3.09
|
|
assert abs(normal_quantile(0.999) - 3.09) < 0.01
|
|
|
|
|
|
class TestWilsonScoreInterval:
|
|
"""Tests for the wilson_score_interval function."""
|
|
|
|
def test_91_of_100(self):
|
|
"""91/100 at 95% should give approximately [0.837, 0.955]."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci = wilson_score_interval(91, 100)
|
|
assert abs(ci["lower"] - 0.837) < 0.005
|
|
assert abs(ci["upper"] - 0.955) < 0.005
|
|
assert ci["sample_size"] == 100
|
|
|
|
def test_perfect_score(self):
|
|
"""100/100 should NOT give [1.0, 1.0] (unlike normal approx)."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci = wilson_score_interval(100, 100)
|
|
assert ci["upper"] == 1.0
|
|
assert ci["lower"] < 1.0 # Wilson correctly shows uncertainty
|
|
assert ci["lower"] > 0.95
|
|
|
|
def test_zero_score(self):
|
|
"""0/100 should NOT give [0.0, 0.0] (unlike normal approx)."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci = wilson_score_interval(0, 100)
|
|
assert ci["lower"] == 0.0
|
|
assert ci["upper"] > 0.0 # Wilson correctly shows uncertainty
|
|
assert ci["upper"] < 0.05
|
|
|
|
def test_zero_total(self):
|
|
"""0/0 should return zeros without error."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci = wilson_score_interval(0, 0)
|
|
assert ci["lower"] == 0.0
|
|
assert ci["upper"] == 0.0
|
|
assert ci["margin_of_error"] == 0.0
|
|
assert ci["sample_size"] == 0
|
|
|
|
def test_raises_on_successes_greater_than_total(self):
|
|
"""successes > total should raise ValueError."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="successes must be in"):
|
|
wilson_score_interval(110, 100)
|
|
|
|
def test_raises_on_negative_successes(self):
|
|
"""Negative successes should raise ValueError."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="successes must be in"):
|
|
wilson_score_interval(-1, 100)
|
|
|
|
def test_single_observation(self):
|
|
"""1/1 should give a wide interval."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci = wilson_score_interval(1, 1)
|
|
assert ci["lower"] < 0.5
|
|
assert ci["upper"] == 1.0
|
|
|
|
def test_bounds_within_0_1(self):
|
|
"""Bounds should always be in [0, 1]."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
for successes, total in [
|
|
(0, 10),
|
|
(10, 10),
|
|
(5, 10),
|
|
(1, 100),
|
|
(99, 100),
|
|
]:
|
|
ci = wilson_score_interval(successes, total)
|
|
assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0
|
|
|
|
def test_higher_n_narrower_interval(self):
|
|
"""Larger sample size should give narrower CI at same proportion."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci_small = wilson_score_interval(9, 10)
|
|
ci_large = wilson_score_interval(90, 100)
|
|
assert ci_large["margin_of_error"] < ci_small["margin_of_error"]
|
|
|
|
def test_90_confidence_narrower_than_99(self):
|
|
"""90% CI should be narrower than 99% CI."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
wilson_score_interval,
|
|
)
|
|
|
|
ci_90 = wilson_score_interval(50, 100, confidence=0.90)
|
|
ci_99 = wilson_score_interval(50, 100, confidence=0.99)
|
|
assert ci_90["margin_of_error"] < ci_99["margin_of_error"]
|
|
|
|
|
|
class TestProportionStdError:
|
|
"""Tests for the proportion_std_error function."""
|
|
|
|
def test_basic(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
proportion_std_error,
|
|
)
|
|
|
|
# p=0.5, n=100 -> sqrt(0.25/100) = 0.05
|
|
assert abs(proportion_std_error(0.5, 100) - 0.05) < 1e-10
|
|
|
|
def test_zero_n(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
proportion_std_error,
|
|
)
|
|
|
|
assert proportion_std_error(0.5, 0) == 0.0
|
|
|
|
def test_extreme_proportions(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
proportion_std_error,
|
|
)
|
|
|
|
assert proportion_std_error(0.0, 100) == 0.0
|
|
assert proportion_std_error(1.0, 100) == 0.0
|
|
|
|
def test_raises_on_invalid_p(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
proportion_std_error,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="p must be in"):
|
|
proportion_std_error(-0.1, 100)
|
|
with pytest.raises(ValueError, match="p must be in"):
|
|
proportion_std_error(1.5, 100)
|
|
|
|
|
|
class TestSampleSizeForDifference:
|
|
"""Tests for the sample_size_for_difference function."""
|
|
|
|
def test_5pp_difference(self):
|
|
"""5pp difference (85% vs 90%) should need ~680 per group."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
n = sample_size_for_difference(0.85, 0.90)
|
|
assert 600 <= n <= 760
|
|
|
|
def test_10pp_difference(self):
|
|
"""10pp difference (80% vs 90%) should need ~200 per group."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
n = sample_size_for_difference(0.80, 0.90)
|
|
assert 170 <= n <= 230
|
|
|
|
def test_15pp_difference(self):
|
|
"""15pp difference (75% vs 90%) should need ~90 per group."""
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
n = sample_size_for_difference(0.75, 0.90)
|
|
assert 75 <= n <= 105
|
|
|
|
def test_raises_on_equal_proportions(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
with pytest.raises(ValueError):
|
|
sample_size_for_difference(0.9, 0.9)
|
|
|
|
def test_raises_on_invalid_proportions(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="p1 must be in"):
|
|
sample_size_for_difference(-0.1, 0.9)
|
|
with pytest.raises(ValueError, match="p2 must be in"):
|
|
sample_size_for_difference(0.8, 1.5)
|
|
|
|
def test_raises_on_invalid_power_or_alpha(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="power must be in"):
|
|
sample_size_for_difference(0.8, 0.9, power=0.0)
|
|
with pytest.raises(ValueError, match="alpha must be in"):
|
|
sample_size_for_difference(0.8, 0.9, alpha=1.0)
|
|
|
|
def test_higher_power_needs_more_samples(self):
|
|
from local_deep_research.benchmarks.metrics.statistics import (
|
|
sample_size_for_difference,
|
|
)
|
|
|
|
n_80 = sample_size_for_difference(0.80, 0.90, power=0.80)
|
|
n_95 = sample_size_for_difference(0.80, 0.90, power=0.95)
|
|
assert n_95 > n_80
|
|
|
|
|
|
class TestCalculateMetricsIntegration:
|
|
"""Test that calculate_metrics() returns accuracy_ci (backward compat)."""
|
|
|
|
def test_accuracy_ci_present(self, tmp_path):
|
|
from local_deep_research.benchmarks.metrics.calculation import (
|
|
calculate_metrics,
|
|
)
|
|
|
|
results_file = tmp_path / "results.jsonl"
|
|
results = [
|
|
{"is_correct": True, "processing_time": 1.5},
|
|
{"is_correct": True, "processing_time": 2.0},
|
|
{"is_correct": False, "processing_time": 1.0},
|
|
]
|
|
with open(results_file, "w") as f:
|
|
for r in results:
|
|
f.write(json.dumps(r) + "\n")
|
|
|
|
metrics = calculate_metrics(str(results_file))
|
|
|
|
# Existing keys still present
|
|
assert metrics["accuracy"] == 2 / 3
|
|
assert metrics["total_examples"] == 3
|
|
|
|
# New CI key present
|
|
assert "accuracy_ci" in metrics
|
|
ci = metrics["accuracy_ci"]
|
|
assert ci["sample_size"] == 3
|
|
assert 0.0 <= ci["lower"] <= ci["upper"] <= 1.0
|
|
# CI should contain the point estimate
|
|
assert ci["lower"] <= metrics["accuracy"] <= ci["upper"]
|
|
|
|
def test_category_ci_present(self, tmp_path):
|
|
from local_deep_research.benchmarks.metrics.calculation import (
|
|
calculate_metrics,
|
|
)
|
|
|
|
results_file = tmp_path / "results.jsonl"
|
|
results = [
|
|
{"is_correct": True, "category": "science"},
|
|
{"is_correct": False, "category": "science"},
|
|
{"is_correct": True, "category": "history"},
|
|
]
|
|
with open(results_file, "w") as f:
|
|
for r in results:
|
|
f.write(json.dumps(r) + "\n")
|
|
|
|
metrics = calculate_metrics(str(results_file))
|
|
|
|
assert "accuracy_ci" in metrics["categories"]["science"]
|
|
assert "accuracy_ci" in metrics["categories"]["history"]
|
|
|
|
def test_empty_results_no_ci_error(self, tmp_path):
|
|
"""Empty file should return error dict, not crash on CI."""
|
|
from local_deep_research.benchmarks.metrics.calculation import (
|
|
calculate_metrics,
|
|
)
|
|
|
|
results_file = tmp_path / "empty.jsonl"
|
|
results_file.write_text("")
|
|
|
|
metrics = calculate_metrics(str(results_file))
|
|
assert "error" in metrics
|