learningcircuit--local-deep-research/tests/benchmarks/test_calculation_high_value.py

"""
High-value edge case tests for benchmarks/metrics/calculation.py.

Complements test_metrics_calculation.py with additional edge cases:
- JSONL with blank lines and whitespace
- Mixed graded/ungraded results
- Confidence with falsy values (0, empty string)
- calculate_combined_score with partial metrics
- calculate_resource_metrics with extreme values
"""

import json


from local_deep_research.benchmarks.metrics.calculation import (
    calculate_combined_score,
    calculate_metrics,
    calculate_resource_metrics,
)


# ---------------------------------------------------------------------------
# calculate_metrics edge cases
# ---------------------------------------------------------------------------


class TestCalculateMetricsEdgeCases:
    """Additional edge case tests for calculate_metrics."""

    def test_blank_lines_in_jsonl_skipped(self, tmp_path):
        """Blank lines between JSON entries are ignored."""
        results_file = tmp_path / "results.jsonl"
        results_file.write_text(
            json.dumps({"is_correct": True}) + "\n"
            "\n"
            "   \n" + json.dumps({"is_correct": False}) + "\n"
        )
        metrics = calculate_metrics(str(results_file))
        assert metrics["total_examples"] == 2
        assert metrics["graded_examples"] == 2

    def test_ungraded_results_not_counted_for_accuracy(self, tmp_path):
        """Results without 'is_correct' are excluded from accuracy calculation."""
        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": True},
            {"question": "What is X?"},  # ungraded
            {"is_correct": False},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["total_examples"] == 3
        assert metrics["graded_examples"] == 2
        assert metrics["accuracy"] == 0.5

    def test_confidence_zero_string_is_included(self, tmp_path):
        """String '0' is truthy so it passes the 'if r.get(confidence)' check.
        int('0') = 0, so both values are included: (0 + 80) / 2 = 40."""
        results_file = tmp_path / "results.jsonl"
        results = [
            {"confidence": "0"},
            {"confidence": "80"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        # "0" is truthy string, int("0")=0, so (0+80)/2 = 40
        assert metrics["average_confidence"] == 40

    def test_confidence_empty_string_skipped(self, tmp_path):
        """Empty string confidence is falsy and skipped."""
        results_file = tmp_path / "results.jsonl"
        results = [
            {"confidence": ""},
            {"confidence": "90"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["average_confidence"] == 90

    def test_processing_time_only_from_entries_that_have_it(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results = [
            {"processing_time": 2.0},
            {"question": "no time"},
            {"processing_time": 4.0},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["average_processing_time"] == 3.0

    def test_all_results_are_errors(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results = [
            {"error": "timeout"},
            {"error": "connection failed"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["error_count"] == 2
        assert metrics["error_rate"] == 1.0
        assert metrics["graded_examples"] == 0
        assert metrics["accuracy"] == 0

    def test_category_with_zero_correct(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results = [
            {"is_correct": False, "category": "hard"},
            {"is_correct": False, "category": "hard"},
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["categories"]["hard"]["accuracy"] == 0.0

    def test_single_result(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results_file.write_text(
            json.dumps({"is_correct": True, "confidence": "95"}) + "\n"
        )

        metrics = calculate_metrics(str(results_file))
        assert metrics["total_examples"] == 1
        assert metrics["accuracy"] == 1.0
        assert metrics["average_confidence"] == 95

    def test_is_correct_false_counted_in_graded(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results = [{"is_correct": False}]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert metrics["graded_examples"] == 1
        assert metrics["correct"] == 0
        assert metrics["accuracy"] == 0.0

    def test_timestamp_present_in_metrics(self, tmp_path):
        results_file = tmp_path / "results.jsonl"
        results_file.write_text(json.dumps({"is_correct": True}) + "\n")

        metrics = calculate_metrics(str(results_file))
        assert "timestamp" in metrics
        assert "T" in metrics["timestamp"]  # ISO format


# ---------------------------------------------------------------------------
# calculate_combined_score edge cases
# ---------------------------------------------------------------------------


class TestCalculateCombinedScoreEdgeCases:
    """Additional edge cases for combined score calculation."""

    def test_empty_metrics_dict(self):
        score = calculate_combined_score({})
        assert score == 0.0

    def test_only_quality_metric(self):
        metrics = {"quality": {"quality_score": 0.9}}
        score = calculate_combined_score(metrics)
        # Only quality contributes: 0.9 * (0.6/1.0) = 0.54
        expected = 0.9 * 0.6
        assert abs(score - expected) < 0.001

    def test_only_speed_metric(self):
        metrics = {"speed": {"speed_score": 0.8}}
        score = calculate_combined_score(metrics)
        expected = 0.8 * 0.3
        assert abs(score - expected) < 0.001

    def test_only_resource_metric(self):
        metrics = {"resource": {"resource_score": 0.5}}
        score = calculate_combined_score(metrics)
        expected = 0.5 * 0.1
        assert abs(score - expected) < 0.001

    def test_unrecognized_metric_keys_ignored(self):
        metrics = {
            "quality": {"quality_score": 1.0},
            "custom_metric": {"score": 0.5},
        }
        score = calculate_combined_score(metrics)
        # custom_metric has no weight, so ignored
        expected = 1.0 * 0.6
        assert abs(score - expected) < 0.001

    def test_all_scores_zero(self):
        metrics = {
            "quality": {"quality_score": 0.0},
            "speed": {"speed_score": 0.0},
            "resource": {"resource_score": 0.0},
        }
        assert calculate_combined_score(metrics) == 0.0

    def test_negative_weights_still_normalize(self):
        """Negative weights are technically allowed by the function."""
        metrics = {
            "quality": {"quality_score": 1.0},
            "speed": {"speed_score": 1.0},
        }
        weights = {"quality": 1.0, "speed": -0.5, "resource": 0.0}
        score = calculate_combined_score(metrics, weights)
        # total_weight = 0.5, norm: quality=2.0, speed=-1.0
        # score = 1.0 * 2.0 + 1.0 * (-1.0) = 1.0
        assert abs(score - 1.0) < 0.001

    def test_missing_score_key_defaults_to_zero(self):
        metrics = {
            "quality": {},  # no quality_score key
        }
        score = calculate_combined_score(metrics)
        assert score == 0.0


# ---------------------------------------------------------------------------
# calculate_resource_metrics edge cases
# ---------------------------------------------------------------------------


class TestCalculateResourceMetricsEdgeCases:
    """Additional edge cases for resource metrics."""

    def test_very_high_complexity(self):
        config = {
            "iterations": 10,
            "questions_per_iteration": 10,
            "max_results": 200,
        }
        metrics = calculate_resource_metrics(config)
        assert (
            metrics["resource_score"] < 0.1
        )  # Very high complexity -> low score

    def test_minimal_complexity(self):
        config = {
            "iterations": 1,
            "questions_per_iteration": 1,
            "max_results": 10,
        }
        metrics = calculate_resource_metrics(config)
        assert (
            metrics["resource_score"] > 0.8
        )  # Very low complexity -> high score

    def test_zero_iterations(self):
        config = {"iterations": 0}
        metrics = calculate_resource_metrics(config)
        assert metrics["estimated_complexity"] == 0
        assert metrics["resource_score"] == 1.0  # 1/(1+0) = 1.0

    def test_complexity_formula(self):
        """Verify the exact formula: iterations * questions * (max_results/50)."""
        config = {
            "iterations": 3,
            "questions_per_iteration": 4,
            "max_results": 100,
        }
        metrics = calculate_resource_metrics(config)
        expected_complexity = 3 * 4 * (100 / 50)
        assert metrics["estimated_complexity"] == expected_complexity

    def test_resource_score_formula(self):
        """Verify: resource_score = 1/(1 + complexity/4)."""
        config = {
            "iterations": 2,
            "questions_per_iteration": 2,
            "max_results": 50,
        }
        metrics = calculate_resource_metrics(config)
        # complexity = 2 * 2 * (50/50) = 4.0
        expected_score = 1.0 / (1.0 + 4.0 / 4.0)
        assert abs(metrics["resource_score"] - expected_score) < 0.001