learningcircuit--local-deep-research/tests/benchmarks/test_metrics_reporting.py

"""
Tests for benchmarks/metrics/reporting.py

Tests cover:
- generate_report function
- Report formatting and structure
"""

import json
from unittest.mock import patch


class TestGenerateReport:
    """Tests for the generate_report function."""

    def test_generates_markdown_report(self, tmp_path):
        """Test that a markdown report is generated."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        # Create a results file
        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "is_correct": True,
                "problem": "What is 2+2?",
                "correct_answer": "4",
                "extracted_answer": "4",
                "reasoning": "Basic math",
            },
            {
                "is_correct": False,
                "problem": "What is the capital?",
                "correct_answer": "Paris",
                "extracted_answer": "London",
                "reasoning": "Wrong answer",
            },
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        output_file = tmp_path / "report.md"
        metrics = {
            "total_examples": 2,
            "graded_examples": 2,
            "correct": 1,
            "accuracy": 0.5,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            mock_write.return_value = None

            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(output_file),
                dataset_name="TestDataset",
            )

            # Check write was called
            mock_write.assert_called_once()
            written_content = mock_write.call_args[0][1]

            # Check report structure
            assert "# Evaluation Report: TestDataset" in written_content
            assert "## Summary" in written_content
            assert "**Total Examples**: 2" in written_content
            assert "**Accuracy**: 0.500" in written_content

    def test_includes_processing_time_when_available(self, tmp_path):
        """Test that processing time is included when available."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
            "average_processing_time": 5.25,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert (
                "**Average Processing Time**: 5.25 seconds" in written_content
            )

    def test_includes_confidence_when_available(self, tmp_path):
        """Test that confidence is included when available."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
            "average_confidence": 85.5,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert "**Average Confidence**: 85.50%" in written_content

    def test_includes_error_info_when_present(self, tmp_path):
        """Test that error info is included when present."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 10,
            "graded_examples": 8,
            "correct": 6,
            "accuracy": 0.75,
            "error_count": 2,
            "error_rate": 0.2,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert "**Error Count**: 2" in written_content
            assert "**Error Rate**: 0.200" in written_content

    def test_includes_category_performance(self, tmp_path):
        """Test that category performance is included."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 4,
            "graded_examples": 4,
            "correct": 3,
            "accuracy": 0.75,
            "categories": {
                "science": {"total": 2, "correct": 2, "accuracy": 1.0},
                "history": {"total": 2, "correct": 1, "accuracy": 0.5},
            },
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert "## Category Performance" in written_content
            assert "### science" in written_content
            assert "### history" in written_content

    def test_includes_config_info(self, tmp_path):
        """Test that config info is included when provided."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
        }

        config_info = {
            "model": "gpt-4",
            "temperature": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
                config_info=config_info,
            )

            written_content = mock_write.call_args[0][1]
            assert "## Configuration" in written_content
            assert "**model**: gpt-4" in written_content
            assert "**temperature**: 0.0" in written_content

    def test_includes_correct_examples(self, tmp_path):
        """Test that correct examples are included."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "is_correct": True,
                "problem": "Test question?",
                "correct_answer": "Answer",
                "extracted_answer": "Answer",
                "reasoning": "Correct reasoning",
            }
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert "## Example Correct Answers" in written_content
            assert "Test question?" in written_content

    def test_includes_incorrect_examples(self, tmp_path):
        """Test that incorrect examples are included."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results = [
            {
                "is_correct": False,
                "problem": "Wrong question?",
                "correct_answer": "Right",
                "extracted_answer": "Wrong",
                "reasoning": "Bad reasoning",
            }
        ]
        with open(results_file, "w") as f:
            for r in results:
                f.write(json.dumps(r) + "\n")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
            )

            written_content = mock_write.call_args[0][1]
            assert "## Example Incorrect Answers" in written_content
            assert "Wrong question?" in written_content

    def test_handles_missing_results_file(self, tmp_path):
        """Test handling of missing results file."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        metrics = {
            "total_examples": 0,
            "graded_examples": 0,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            # Should not raise even with missing file
            generate_report(
                metrics=metrics,
                results_file=str(tmp_path / "nonexistent.jsonl"),
                output_file=str(tmp_path / "report.md"),
            )

            mock_write.assert_called_once()

    def test_includes_metadata_section(self, tmp_path):
        """Test that metadata section is included."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")

        metrics = {
            "total_examples": 1,
            "graded_examples": 1,
            "correct": 1,
            "accuracy": 1.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ) as mock_write:
            generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=str(tmp_path / "report.md"),
                dataset_name="TestDataset",
            )

            written_content = mock_write.call_args[0][1]
            assert "## Metadata" in written_content
            assert "**Generated**:" in written_content
            assert "**Dataset**: TestDataset" in written_content

    def test_returns_output_file_path(self, tmp_path):
        """Test that output file path is returned."""
        from local_deep_research.benchmarks.metrics.reporting import (
            generate_report,
        )

        results_file = tmp_path / "results.jsonl"
        results_file.write_text("")
        output_file = str(tmp_path / "report.md")

        metrics = {
            "total_examples": 0,
            "graded_examples": 0,
            "correct": 0,
            "accuracy": 0.0,
        }

        with patch(
            "local_deep_research.security.file_write_verifier.write_file_verified"
        ):
            result = generate_report(
                metrics=metrics,
                results_file=str(results_file),
                output_file=output_file,
            )

            assert result == output_file