learningcircuit--local-deep-research/tests/benchmarks/test_reporting_high_value.py

"""High-value tests for benchmarks/metrics/reporting.py.

Covers report generation, empty/null data handling, category metrics,
config info inclusion, and file I/O error paths.
"""

import json

from unittest.mock import patch


class TestGenerateReport:
    """Tests for generate_report function."""

    def test_basic_report_generation(self, tmp_path):
        """Basic report generates with minimal metrics."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        output_file = str(tmp_path / "report.md")
        metrics = {
            "total_examples": 10,
            "graded_examples": 10,
            "correct": 8,
            "accuracy": 0.8,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            result = generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=output_file,
                dataset_name="TestDataset",
            )

            assert result == output_file
            mock_write.assert_called_once()
            content = mock_write.call_args[0][1]
            assert "TestDataset" in content
            assert "0.800" in content
            assert "8" in content

    def test_report_with_processing_time(self, tmp_path):
        """Report includes average processing time when available."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 5,
            "graded_examples": 5,
            "correct": 3,
            "accuracy": 0.6,
            "average_processing_time": 12.345,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            assert "12.35" in content

    def test_report_with_error_count(self, tmp_path):
        """Report includes error stats when error_count > 0."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 10,
            "graded_examples": 8,
            "correct": 6,
            "accuracy": 0.75,
            "error_count": 2,
            "error_rate": 0.2,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            assert "Error Count" in content
            assert "Error Rate" in content

    def test_report_with_categories(self, tmp_path):
        """Report includes per-category metrics."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 10,
            "graded_examples": 10,
            "correct": 7,
            "accuracy": 0.7,
            "categories": {
                "Science": {"total": 5, "correct": 4, "accuracy": 0.8},
                "History": {"total": 5, "correct": 3, "accuracy": 0.6},
            },
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            assert "Science" in content
            assert "History" in content
            assert "0.800" in content

    def test_report_with_config_info(self, tmp_path):
        """Report includes configuration info when provided."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
                config_info={"model": "gpt-4", "strategy": "rapid"},
            )
            content = mock_write.call_args[0][1]
            assert "Configuration" in content
            assert "gpt-4" in content
            assert "rapid" in content

    def test_report_with_correct_examples(self, tmp_path):
        """Report includes correct answer examples from results file."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "problem": "What is 2+2?",
                "correct_answer": "4",
                "extracted_answer": "4",
                "is_correct": True,
            },
        ]
        results_file.write_text("\n".join(json.dumps(r) for r in results))

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            assert "Example Correct Answers" in content
            assert "What is 2+2?" in content

    def test_report_with_incorrect_examples(self, tmp_path):
        """Report includes incorrect answer examples."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "problem": "Capital of France?",
                "correct_answer": "Paris",
                "extracted_answer": "London",
                "is_correct": False,
            },
        ]
        results_file.write_text("\n".join(json.dumps(r) for r in results))

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            assert "Example Incorrect Answers" in content
            assert "Capital of France?" in content

    def test_report_handles_missing_results_file(self, tmp_path):
        """Missing results file doesn't crash report generation."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        metrics = {
            "total_examples": 0,
            "graded_examples": 0,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            result = generate_report(
                metrics=metrics,
                results_file="/nonexistent/path/results.jsonl",
                output_file=str(tmp_path / "report.md"),
            )
            assert result == str(tmp_path / "report.md")
            mock_write.assert_called_once()

    def test_report_limits_examples_to_five(self, tmp_path):
        """Report limits correct/incorrect examples to 5 each."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "problem": f"Q{i}?",
                "correct_answer": f"A{i}",
                "extracted_answer": f"A{i}",
                "is_correct": True,
            }
            for i in range(10)
        ]
        results_file.write_text("\n".join(json.dumps(r) for r in results))

        metrics = {
            "total_examples": 10,
            "graded_examples": 10,
            "correct": 10,
            "accuracy": 1.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )
            content = mock_write.call_args[0][1]
            # Should have exactly 5 "Example N" entries
            assert content.count("### Example") == 5

    def test_report_includes_metadata_timestamp(self, tmp_path):
        """Report includes metadata section with timestamp."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 0,
            "graded_examples": 0,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
                dataset_name="TestDS",
            )
            content = mock_write.call_args[0][1]
            assert "Metadata" in content
            assert "Generated" in content
            assert "TestDS" in content