learningcircuit--local-deep-research/tests/benchmarks/test_benchmark_service_snapshot.py

"""Tests for benchmark service persistence of ldr_version + settings_snapshot.

Pins the contract added by the migration-0014 PR:
- start_benchmark writes ldr_version (== current __version__) on the row.
- start_benchmark writes a redacted settings_snapshot.
- A LookupError raised by SettingsManager.get_all_settings (as happens with
  legacy CHAT-typed enum rows) is swallowed; benchmark proceeds with empty
  snapshot and ldr_version still recorded.

Patterned after test_benchmark_service.test_start_benchmark_creates_thread.
The thread is stubbed so we only exercise the synchronous prelude where the
new fields are written.
"""

from unittest.mock import Mock, patch


def _make_run(**overrides):
    mock_run = Mock()
    mock_run.id = 1
    mock_run.config_hash = "abc12345"
    mock_run.datasets_config = {"simpleqa": {"count": 1}}
    mock_run.search_config = {}
    mock_run.evaluation_config = {}
    # Start with no provenance set; assertions read the post-call value.
    mock_run.ldr_version = None
    mock_run.settings_snapshot = None
    for k, v in overrides.items():
        setattr(mock_run, k, v)
    return mock_run


def _patched_start_benchmark(service, mock_run, snapshot):
    """Run service.start_benchmark with all collaborators stubbed."""
    with patch(
        "local_deep_research.database.session_context.get_user_db_session"
    ) as mock_get_session:
        mock_session = Mock()
        mock_session.__enter__ = Mock(return_value=mock_session)
        mock_session.__exit__ = Mock(return_value=False)
        mock_get_session.return_value = mock_session
        mock_session.query.return_value.filter.return_value.first.return_value = mock_run

        with patch(
            "local_deep_research.settings.SettingsManager"
        ) as mock_settings_mgr:
            if isinstance(snapshot, Exception):
                mock_settings_mgr.return_value.get_all_settings.side_effect = (
                    snapshot
                )
            else:
                mock_settings_mgr.return_value.get_all_settings.return_value = (
                    snapshot
                )

            with patch("flask.session", {"session_id": "s"}):
                with patch(
                    "local_deep_research.database.session_passwords.session_password_store"
                ):
                    with patch.object(
                        service, "_run_benchmark_thread", return_value=None
                    ):
                        return service.start_benchmark(
                            mock_run.id, username="testuser"
                        )


def test_start_benchmark_persists_ldr_version():
    from local_deep_research import __version__
    from local_deep_research.benchmarks.web_api.benchmark_service import (
        BenchmarkService,
    )

    service = BenchmarkService(socket_service=Mock())
    mock_run = _make_run()
    result = _patched_start_benchmark(service, mock_run, snapshot={})
    assert result is True
    assert mock_run.ldr_version == __version__


def test_start_benchmark_persists_redacted_snapshot():
    """The persisted snapshot must redact secrets via DataSanitizer.redact_settings_snapshot."""
    from local_deep_research.benchmarks.web_api.benchmark_service import (
        BenchmarkService,
    )

    service = BenchmarkService(socket_service=Mock())
    mock_run = _make_run()
    snapshot = {
        "llm.openai.api_key": {
            "value": "sk-real-secret",
            "ui_element": "password",
            "type": "LLM",
        },
        "search.fetch.mode": {
            "value": "summary_focus_query",
            "ui_element": "select",
            "type": "SEARCH",
        },
    }
    _patched_start_benchmark(service, mock_run, snapshot=snapshot)

    persisted = mock_run.settings_snapshot
    assert isinstance(persisted, dict)
    # Secret value redacted, metadata intact.
    assert persisted["llm.openai.api_key"]["value"] == "[REDACTED]"
    assert persisted["llm.openai.api_key"]["ui_element"] == "password"
    # Non-secret values pass through.
    assert persisted["search.fetch.mode"]["value"] == "summary_focus_query"


def test_start_benchmark_does_not_mutate_in_memory_snapshot():
    """The unredacted snapshot the background thread uses to authenticate
    against providers must NOT be mutated by the redaction step."""
    from local_deep_research.benchmarks.web_api.benchmark_service import (
        BenchmarkService,
    )

    service = BenchmarkService(socket_service=Mock())
    mock_run = _make_run()
    snapshot = {
        "llm.openai.api_key": {
            "value": "sk-real-secret",
            "ui_element": "password",
        }
    }
    _patched_start_benchmark(service, mock_run, snapshot=snapshot)
    # Original dict still has the real secret.
    assert snapshot["llm.openai.api_key"]["value"] == "sk-real-secret"
    # The active_runs in-memory copy keeps the unredacted snapshot for the
    # background thread (which needs the real keys to call providers).
    assert (
        service.active_runs[1]["data"]["settings_snapshot"][
            "llm.openai.api_key"
        ]["value"]
        == "sk-real-secret"
    )


def test_start_benchmark_survives_settings_lookup_error():
    """A LookupError from get_all_settings (e.g. stale CHAT enum row) must
    not crash benchmark start. Snapshot becomes empty; ldr_version is still
    recorded so users can tell which version produced the empty snapshot."""
    from local_deep_research import __version__
    from local_deep_research.benchmarks.web_api.benchmark_service import (
        BenchmarkService,
    )

    service = BenchmarkService(socket_service=Mock())
    mock_run = _make_run()
    err = LookupError("'CHAT' is not among the defined enum values")
    result = _patched_start_benchmark(service, mock_run, snapshot=err)

    assert result is True
    assert mock_run.ldr_version == __version__
    assert mock_run.settings_snapshot == {}