comet-ml--opik/sdks/python/tests/unit/plugins/pytest/test_hooks.py

from types import SimpleNamespace
from unittest import mock

# TODO(#5219): move these tests to a public pytest plugin test harness/API.
from opik.plugins.pytest import hooks, test_runs_storage


def _config(
    *, opik_option: bool = False, ini_enabled: bool = False, auto_active: bool = False
) -> SimpleNamespace:
    config = SimpleNamespace(
        option=SimpleNamespace(opik=opik_option),
        getini=lambda _: ini_enabled,
    )
    if auto_active:
        setattr(config, "_opik_pytest_active", True)
    return config


def _session(
    *, opik_option: bool = False, ini_enabled: bool = False, auto_active: bool = False
) -> SimpleNamespace:
    return SimpleNamespace(
        config=_config(
            opik_option=opik_option, ini_enabled=ini_enabled, auto_active=auto_active
        )
    )


def _terminal_reporter(
    *, opik_option: bool = False, ini_enabled: bool = False, auto_active: bool = False
) -> SimpleNamespace:
    return SimpleNamespace(
        config=_config(
            opik_option=opik_option, ini_enabled=ini_enabled, auto_active=auto_active
        ),
        stats={"passed": [], "failed": []},
    )


def setup_function(function: object) -> None:
    del function
    test_runs_storage.LLM_UNIT_TEST_RUNS.clear()
    test_runs_storage.TEST_RUNS_TO_TRACE_DATA.clear()
    test_runs_storage.TEST_RUNS_CONTENTS.clear()


def test_pytest_sessionfinish__plugin_disabled__no_client_calls(monkeypatch):
    session = _session(opik_option=False, ini_enabled=False)
    session.items = [SimpleNamespace(nodeid="a")]
    test_runs_storage.LLM_UNIT_TEST_RUNS.add("a")

    get_client_mock = mock.Mock()
    monkeypatch.setattr(hooks.opik_client, "get_global_client", get_client_mock)

    hooks.pytest_sessionfinish(session=session, exitstatus=0)

    get_client_mock.assert_not_called()


def test_pytest_sessionfinish__missing_session_items__returns_without_error(
    monkeypatch,
):
    session = _session(opik_option=True)
    get_client_mock = mock.Mock()
    monkeypatch.setattr(hooks.opik_client, "get_global_client", get_client_mock)

    hooks.pytest_sessionfinish(session=session, exitstatus=0)

    get_client_mock.assert_not_called()


def test_pytest_sessionfinish__item_without_report__is_ignored(monkeypatch):
    session = _session(opik_option=True)
    session.items = [SimpleNamespace(nodeid="case-1")]
    test_runs_storage.LLM_UNIT_TEST_RUNS.add("case-1")
    test_runs_storage.TEST_RUNS_TO_TRACE_DATA["case-1"] = SimpleNamespace(id="trace-1")

    get_client_mock = mock.Mock()
    monkeypatch.setattr(hooks.opik_client, "get_global_client", get_client_mock)

    hooks.pytest_sessionfinish(session=session, exitstatus=0)

    get_client_mock.assert_not_called()


def test_pytest_sessionfinish__valid_item__logs_scores_and_runs_experiment__happyflow(
    monkeypatch,
):
    report = SimpleNamespace(passed=True)
    item = SimpleNamespace(nodeid="case-1", report=report)
    session = _session(auto_active=True)
    session.items = [item]

    test_runs_storage.LLM_UNIT_TEST_RUNS.add("case-1")
    test_runs_storage.TEST_RUNS_TO_TRACE_DATA["case-1"] = SimpleNamespace(id="trace-1")

    client = mock.Mock()
    monkeypatch.setattr(
        hooks.opik_client, "get_global_client", mock.Mock(return_value=client)
    )
    run_mock = mock.Mock()
    monkeypatch.setattr(hooks.experiment_runner, "run", run_mock)

    hooks.pytest_sessionfinish(session=session, exitstatus=0)

    client.log_traces_feedback_scores.assert_called_once()
    payload = client.log_traces_feedback_scores.call_args.args[0]
    assert len(payload) == 1
    score = payload[0]
    assert score["id"] == "trace-1"
    assert score["name"] == "Passed"
    assert score["value"] == 1.0
    run_mock.assert_called_once_with(client=client, test_items=[item])
    client.flush.assert_called_once()


def test_pytest_sessionfinish__runner_error__flushes_client(monkeypatch):
    report = SimpleNamespace(passed=True)
    item = SimpleNamespace(nodeid="case-1", report=report)
    session = _session(auto_active=True)
    session.items = [item]

    test_runs_storage.LLM_UNIT_TEST_RUNS.add("case-1")
    test_runs_storage.TEST_RUNS_TO_TRACE_DATA["case-1"] = SimpleNamespace(id="trace-1")

    client = mock.Mock()
    monkeypatch.setattr(
        hooks.opik_client, "get_global_client", mock.Mock(return_value=client)
    )
    monkeypatch.setattr(
        hooks.experiment_runner, "run", mock.Mock(side_effect=RuntimeError("boom"))
    )

    hooks.pytest_sessionfinish(session=session, exitstatus=0)

    client.log_traces_feedback_scores.assert_called_once()
    client.flush.assert_called_once()


def test_pytest_terminal_summary__plugin_disabled__no_summary_output(monkeypatch):
    terminal_reporter = _terminal_reporter(opik_option=False, ini_enabled=False)
    print_mock = mock.Mock()
    monkeypatch.setattr(hooks.summary, "print", print_mock)

    hooks.pytest_terminal_summary(terminalreporter=terminal_reporter)

    print_mock.assert_not_called()


def test_pytest_collection_modifyitems__activates_plugin_for_llm_unit_tests():
    config = _config()
    llm_item = SimpleNamespace(obj=SimpleNamespace(_opik_llm_unit=True))

    hooks.pytest_collection_modifyitems(
        session=SimpleNamespace(), config=config, items=[llm_item]
    )

    assert getattr(config, "_opik_pytest_active", False) is True