learningcircuit--local-deep-research/tests/config/test_llm_config.py

"""Tests for llm_config module."""

from unittest.mock import AsyncMock, MagicMock, patch


from local_deep_research.config.llm_config import (
    get_selected_llm_provider,
    wrap_llm_without_think_tags,
    get_llm,
)


class TestGetSelectedLlmProvider:
    """Tests for get_selected_llm_provider function."""

    def test_returns_provider_from_settings(self):
        """Should return provider from settings."""
        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value="anthropic",
        ):
            result = get_selected_llm_provider()
            assert result == "anthropic"

    def test_returns_lowercase(self):
        """Should return lowercase provider."""
        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value="OPENAI",
        ):
            result = get_selected_llm_provider()
            assert result == "openai"

    def test_defaults_to_ollama(self):
        """Should default to ollama."""
        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value="ollama",
        ) as mock:
            get_selected_llm_provider()
            # Check default is ollama
            mock.assert_called_with(
                "llm.provider", "ollama", settings_snapshot=None
            )


class TestWrapLlmWithoutThinkTags:
    """Tests for wrap_llm_without_think_tags function."""

    def test_returns_wrapper_instance(self):
        """Should return a wrapper instance."""
        mock_llm = MagicMock()
        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            result = wrap_llm_without_think_tags(mock_llm)
            assert hasattr(result, "invoke")
            assert hasattr(result, "base_llm")

    def test_wrapper_invoke_calls_base_llm(self):
        """Should call base LLM on invoke."""
        mock_llm = MagicMock()
        mock_response = MagicMock()
        mock_response.content = "test response"
        mock_llm.invoke.return_value = mock_response

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            wrapper.invoke("test prompt")
            mock_llm.invoke.assert_called_with("test prompt")

    def test_wrapper_removes_think_tags(self):
        """Should remove think tags from response."""
        mock_llm = MagicMock()
        mock_response = MagicMock()
        mock_response.content = "<think>internal</think>visible"
        mock_llm.invoke.return_value = mock_response

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.remove_think_tags",
                return_value="visible",
            ) as mock_remove:
                wrapper = wrap_llm_without_think_tags(mock_llm)
                wrapper.invoke("test")
                mock_remove.assert_called_with("<think>internal</think>visible")

    def test_wrapper_preserves_nonstring_content(self):
        """Non-string content (e.g. provider content-block lists) must pass
        through unchanged instead of raising TypeError in remove_think_tags."""
        mock_llm = MagicMock()
        mock_response = MagicMock()
        blocks = [{"type": "text", "text": "visible"}]
        mock_response.content = blocks
        mock_llm.invoke.return_value = mock_response

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            result = wrapper.invoke("test")
            assert result.content == blocks

    def test_wrapper_preserves_none_content(self):
        """None content must pass through unchanged rather than raising."""
        mock_llm = MagicMock()
        mock_response = MagicMock()
        mock_response.content = None
        mock_llm.invoke.return_value = mock_response

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            result = wrapper.invoke("test")
            assert result.content is None

    def test_wrapper_delegates_attributes(self):
        """Should delegate attribute access to base LLM."""
        mock_llm = MagicMock()
        mock_llm.model_name = "gpt-4"

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            assert wrapper.model_name == "gpt-4"

    def test_applies_rate_limiting_when_enabled(self):
        """Should apply rate limiting when enabled in settings."""
        mock_llm = MagicMock()
        mock_wrapped = MagicMock()

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=True,
        ):
            # Patch at source location since it's imported inside the function
            with patch(
                "local_deep_research.web_search_engines.rate_limiting.llm.create_rate_limited_llm_wrapper",
                return_value=mock_wrapped,
            ) as mock_create:
                wrap_llm_without_think_tags(mock_llm, provider="openai")
                mock_create.assert_called_with(mock_llm, "openai")

    # -- bind_tools regression tests (issue #4804) -------------------------
    #
    # ``create_agent()`` calls ``model.bind_tools(tools)`` and uses the
    # return value as the model that runs inside the agent loop. Without
    # a ``bind_tools`` override on ``ProcessingLLMWrapper``, Python falls
    # through ``__getattr__`` to ``self.base_llm.bind_tools(...)`` and the
    # agent loop runs on an unwrapped ``BaseChatModel`` — which means
    # ``_normalize_response`` never strips ``<think>…`` and reasoning-mode
    # models (Qwen 3.x, deepseek-r1, etc.) leak raw ``<think>`` text into
    # the conversation history. On the next LLM turn, strict
    # OpenAI-compatible providers reject the request with
    # ``Failed to parse input at pos 0: <think>…``. The override keeps
    # the bound model wrapped, so the stripper stays in the loop.

    def test_bind_tools_returns_wrapper_not_raw_base_llm(self):
        """``bind_tools`` must return a ``ProcessingLLMWrapper`` instance,
        not the unwrapped bound model. Guards against future regressions
        that reintroduce the bypass by deleting this override (issue
        #4804)."""
        mock_llm = MagicMock()
        mock_bound = MagicMock()
        mock_llm.bind_tools.return_value = mock_bound

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            bound_wrapper = wrapper.bind_tools([lambda: None])

            assert hasattr(bound_wrapper, "base_llm")
            assert bound_wrapper.base_llm is mock_bound
            # The wrapped object must NOT be the raw base LLM (or its
            # bound child) — it must be the wrapper that runs
            # _normalize_response.
            assert bound_wrapper is not mock_bound
            assert bound_wrapper is not mock_llm

    def test_bind_tools_forwards_tools_and_kwargs_to_base_llm(self):
        """Tools and provider-specific kwargs must be forwarded verbatim
        to ``base_llm.bind_tools``."""
        mock_llm = MagicMock()
        mock_bound = MagicMock()
        mock_llm.bind_tools.return_value = mock_bound

        def sentinel_tool():
            return None

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            wrapper.bind_tools(sentinel_tool, tool_choice="auto", parallel=True)

            mock_llm.bind_tools.assert_called_once_with(
                sentinel_tool, tool_choice="auto", parallel=True
            )

    def test_bound_wrapper_strips_think_tags_from_invoke(self):
        """After ``bind_tools``, ``invoke()`` on the bound wrapper must
        still strip ``<think>…</think>`` from the model response. This is
        the exact path ``create_agent()`` uses — if it strips, the
        ``BadRequestError`` from issue #4804 cannot recur."""
        mock_llm = MagicMock()
        mock_bound = MagicMock()
        mock_llm.bind_tools.return_value = mock_bound

        mock_response = MagicMock()
        mock_response.content = "<think>reasoning trace…</think>visible answer"
        mock_bound.invoke.return_value = mock_response

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.remove_think_tags",
                wraps=lambda text: text.replace(
                    "<think>reasoning trace…</think>", ""
                ),
            ) as mock_remove:
                wrapper = wrap_llm_without_think_tags(mock_llm)
                bound_wrapper = wrapper.bind_tools([lambda: None])
                bound_wrapper.invoke("test")

                mock_bound.invoke.assert_called_once_with("test")
                mock_remove.assert_called_once_with(
                    "<think>reasoning trace…</think>visible answer"
                )
                # The bound model is what the agent calls — the original
                # base LLM must NOT receive the invoke (otherwise the
                # bypass is back).
                assert not mock_llm.invoke.called

    def test_bound_wrapper_delegates_attributes_to_bound_model(self):
        """Attributes on the bound model (e.g. ``model_name``) must still
        be reachable through the wrapped binding, so LangChain internals
        that introspect the model see the bound model's metadata."""
        mock_llm = MagicMock()
        mock_bound = MagicMock()
        mock_bound.model_name = "qwen3:8b"
        mock_llm.bind_tools.return_value = mock_bound

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            bound_wrapper = wrapper.bind_tools([lambda: None])
            assert bound_wrapper.model_name == "qwen3:8b"

    def test_bound_wrapper_ainvoke_strips_think_tags(self):
        """Async counterpart of ``test_bound_wrapper_strips_think_tags_from_invoke``.
        ``ainvoke`` is the path async LangGraph nodes use; it must also
        pass through ``_normalize_response`` after the re-wrap."""
        import asyncio

        mock_llm = MagicMock()
        mock_bound = MagicMock()
        mock_llm.bind_tools.return_value = mock_bound

        mock_response = MagicMock()
        mock_response.content = "<think>…</think>ok"
        mock_bound.ainvoke = AsyncMock(return_value=mock_response)

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.remove_think_tags",
                wraps=lambda text: text.replace("<think>…</think>", ""),
            ):
                wrapper = wrap_llm_without_think_tags(mock_llm)
                bound_wrapper = wrapper.bind_tools([lambda: None])
                result = asyncio.run(bound_wrapper.ainvoke("test"))

                mock_bound.ainvoke.assert_called_once_with("test")
                # Must actually strip on the async path. Without the bind_tools
                # override this fails: the raw bound model is returned and its
                # ainvoke never runs _normalize_response, so content keeps the
                # <think> block. Regression guard for #4804.
                assert result is mock_response
                assert result.content == "ok"


class TestBindToolsCreateAgentIntegration:
    """End-to-end guard for #4804.

    The unit tests above exercise ``ProcessingLLMWrapper.bind_tools`` in
    isolation. This class drives a REAL ``langchain.agents.create_agent`` over
    the think-stripping wrapper and asserts the agent's final message has
    ``<think>…</think>`` stripped — the actual path that regressed. Without the
    ``bind_tools`` override, ``create_agent`` binds tools to the unwrapped base
    model and the ``<think>`` block leaks into the agent's output.
    """

    @staticmethod
    def _build_agent():
        from langchain.agents import create_agent
        from langchain_core.language_models.chat_models import BaseChatModel
        from langchain_core.messages import AIMessage
        from langchain_core.outputs import ChatGeneration, ChatResult
        from langchain_core.tools import tool

        class _FakeThinkModel(BaseChatModel):
            """Reasoning-style model: emits a ``<think>`` block and no tool
            calls, so ``create_agent`` runs exactly one model turn and stops."""

            @property
            def _llm_type(self) -> str:
                return "fake-think"

            def _generate(
                self, messages, stop=None, run_manager=None, **kwargs
            ):
                message = AIMessage(
                    content="<think>internal reasoning</think>FINAL_ANSWER"
                )
                return ChatResult(generations=[ChatGeneration(message=message)])

            def bind_tools(self, tools, **kwargs):
                # The fake never emits tool_calls, so returning self is enough
                # for create_agent to bind tools and run a single turn.
                return self

        @tool
        def noop(query: str) -> str:
            """No-op tool so the agent has a non-empty tool list."""
            return query

        # Patch the settings lookup so wrap_llm_without_think_tags doesn't try
        # to read rate-limit settings from a (nonexistent) settings context.
        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapped = wrap_llm_without_think_tags(_FakeThinkModel())
        return create_agent(model=wrapped, tools=[noop])

    def test_create_agent_invoke_strips_think_tags(self):
        agent = self._build_agent()
        result = agent.invoke({"messages": [{"role": "user", "content": "hi"}]})
        final = result["messages"][-1]
        assert "<think>" not in final.content
        assert final.content == "FINAL_ANSWER"

    def test_create_agent_ainvoke_strips_think_tags(self):
        import asyncio

        agent = self._build_agent()
        result = asyncio.run(
            agent.ainvoke({"messages": [{"role": "user", "content": "hi"}]})
        )
        final = result["messages"][-1]
        assert "<think>" not in final.content
        assert final.content == "FINAL_ANSWER"


class TestGetLlm:
    """Tests for get_llm function.

    Many historical tests in this class mocked ``is_llm_registered=False``
    to force the procedural ``if/elif`` chain in ``get_llm`` (lines
    ~405-707 prior to the dead-code deletion). That chain has been
    removed; equivalent live-path coverage now lives in
    ``tests/llm_providers/implementations/test_*_provider.py``. The
    remaining tests below exercise the registered-LLM branch and the
    coordinator surface (provider normalization, model-name validation,
    final guards). The deleted tests are replaced 1:1 by their class-path
    equivalents — see commit history for the mapping.
    """

    def test_uses_custom_registered_llm(self):
        """Should use custom LLM when registered."""
        # Import BaseChatModel for proper spec
        from langchain_core.language_models import BaseChatModel

        mock_llm = MagicMock(spec=BaseChatModel)

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=True,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_llm_from_registry",
                return_value=mock_llm,
            ):
                with patch(
                    "local_deep_research.config.llm_config.wrap_llm_without_think_tags",
                    return_value=mock_llm,
                ):
                    with patch(
                        "local_deep_research.config.llm_config.get_setting_from_snapshot",
                        return_value="custom_provider",
                    ):
                        result = get_llm(
                            provider="custom_provider",
                            settings_snapshot={"search.tool": "searxng"},
                        )
                        assert result is mock_llm

    def test_invalid_provider_raises_error(self):
        """Should raise ValueError for invalid provider."""
        import pytest

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_setting_from_snapshot"
            ) as mock_get:
                mock_get.side_effect = lambda key, default=None, **kwargs: {
                    "llm.model": "test-model",
                    "llm.temperature": 0.7,
                    "llm.provider": "invalid_provider",
                }.get(key, default)

                with pytest.raises(ValueError, match="Invalid provider"):
                    get_llm(settings_snapshot={"search.tool": "searxng"})

    def test_raises_when_model_setting_empty(self):
        """get_llm() must raise ValueError when llm.model is empty string."""
        import pytest

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_setting_from_snapshot"
            ) as mock_get:
                mock_get.side_effect = lambda key, default=None, **kwargs: {
                    "llm.model": "",
                    "llm.temperature": 0.7,
                    "llm.provider": "ollama",
                }.get(key, default)

                with pytest.raises(
                    ValueError, match="LLM model not configured"
                ):
                    get_llm()

    def test_raises_when_model_setting_whitespace_only(self):
        """get_llm() must raise ValueError when llm.model is whitespace."""
        import pytest

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_setting_from_snapshot"
            ) as mock_get:
                mock_get.side_effect = lambda key, default=None, **kwargs: {
                    "llm.model": "   ",
                    "llm.temperature": 0.7,
                    "llm.provider": "ollama",
                }.get(key, default)

                with pytest.raises(
                    ValueError, match="LLM model not configured"
                ):
                    get_llm()

    def test_raises_when_model_setting_missing_returns_empty_default(self):
        """get_llm() must raise when snapshot returns empty default."""
        import pytest

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=False,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_setting_from_snapshot"
            ) as mock_get:
                # Snapshot has neither llm.model nor a custom default;
                # function falls back to its own "" default.
                mock_get.side_effect = lambda key, default=None, **kwargs: {
                    "llm.temperature": 0.7,
                    "llm.provider": "ollama",
                }.get(key, default)

                with pytest.raises(
                    ValueError, match="LLM model not configured"
                ):
                    get_llm()

    def test_custom_factory_function_is_called(self):
        """Should call factory function for custom registered LLM."""
        from langchain_core.language_models import BaseChatModel

        mock_llm = MagicMock(spec=BaseChatModel)
        mock_factory = MagicMock(return_value=mock_llm)

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=True,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_llm_from_registry",
                return_value=mock_factory,
            ):
                with patch(
                    "local_deep_research.config.llm_config.get_setting_from_snapshot"
                ) as mock_get:
                    mock_get.side_effect = lambda key, default=None, **kwargs: {
                        "llm.model": "custom-model",
                        "llm.temperature": 0.5,
                        "llm.provider": "custom_provider",
                        "rate_limiting.llm_enabled": False,
                    }.get(key, default)

                    get_llm(
                        model_name="custom-model",
                        temperature=0.5,
                        provider="custom_provider",
                        settings_snapshot={"search.tool": "searxng"},
                    )

                    mock_factory.assert_called_once()
                    call_kwargs = mock_factory.call_args.kwargs
                    assert call_kwargs["model_name"] == "custom-model"
                    assert call_kwargs["temperature"] == 0.5

    def test_custom_factory_with_invalid_signature_raises(self):
        """Should raise TypeError when factory has invalid signature."""
        import pytest

        def bad_factory():
            return MagicMock()

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=True,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_llm_from_registry",
                return_value=bad_factory,
            ):
                with patch(
                    "local_deep_research.config.llm_config.get_setting_from_snapshot"
                ) as mock_get:
                    mock_get.side_effect = lambda key, default=None, **kwargs: {
                        "llm.model": "model",
                        "llm.temperature": 0.7,
                        "llm.provider": "bad_factory",
                        "rate_limiting.llm_enabled": False,
                    }.get(key, default)

                    with pytest.raises(TypeError, match="invalid signature"):
                        get_llm(
                            provider="bad_factory",
                            settings_snapshot={"search.tool": "searxng"},
                        )

    def test_custom_factory_returning_non_basechatmodel_raises(self):
        """Should raise ValueError when factory returns non-BaseChatModel."""
        import pytest

        def bad_factory(
            model_name=None, temperature=None, settings_snapshot=None
        ):
            return "not a model"

        with patch(
            "local_deep_research.config.llm_config.is_llm_registered",
            return_value=True,
        ):
            with patch(
                "local_deep_research.config.llm_config.get_llm_from_registry",
                return_value=bad_factory,
            ):
                with patch(
                    "local_deep_research.config.llm_config.get_setting_from_snapshot"
                ) as mock_get:
                    mock_get.side_effect = lambda key, default=None, **kwargs: {
                        "llm.model": "model",
                        "llm.temperature": 0.7,
                        "llm.provider": "bad_factory",
                        "rate_limiting.llm_enabled": False,
                    }.get(key, default)

                    with pytest.raises(
                        ValueError, match="must return a BaseChatModel"
                    ):
                        get_llm(
                            provider="bad_factory",
                            settings_snapshot={"search.tool": "searxng"},
                        )


class TestWrapperStringResponse:
    """Tests for wrapper handling string responses."""

    def test_wrapper_handles_string_response(self):
        """Should handle string response from LLM."""
        mock_llm = MagicMock()
        mock_llm.invoke.return_value = "<think>thought</think>answer"

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            result = wrapper.invoke("test")
            # A bare-string return is normalized into a message (so callers can
            # always use .content); think tags are still removed.
            assert not isinstance(result, str)
            assert "answer" in result.content
            assert "<think>" not in result.content

    def test_wrapper_handles_invoke_exception(self):
        """Should propagate exceptions from LLM invoke."""
        import pytest

        mock_llm = MagicMock()
        mock_llm.invoke.side_effect = RuntimeError("LLM error")

        with patch(
            "local_deep_research.config.llm_config.get_setting_from_snapshot",
            return_value=False,
        ):
            wrapper = wrap_llm_without_think_tags(mock_llm)
            with pytest.raises(RuntimeError, match="LLM error"):
                wrapper.invoke("test")


class TestImportTimeAutoDiscovery:
    """Guard the import-time provider auto-discovery contract.

    ``llm_config`` imports ``discover_providers`` (``# noqa: F401``) purely
    for its import-time side effect: registering every built-in provider.
    Since get_llm() has no fallback construction path, removing that import
    would leave the registry empty and break every dispatch. This runs in a
    fresh interpreter so the assertion genuinely exercises import-time
    registration rather than registrations left over from earlier tests.
    """

    def test_importing_llm_config_registers_builtin_providers(self):
        import subprocess
        import sys

        code = (
            "from local_deep_research.config import llm_config  # noqa: F401\n"
            "from local_deep_research.llm import is_llm_registered\n"
            "assert is_llm_registered('openai'), 'openai not registered'\n"
            "assert is_llm_registered('anthropic'), 'anthropic not registered'\n"
            "print('OK')\n"
        )
        result = subprocess.run(
            [sys.executable, "-c", code],
            capture_output=True,
            text=True,
        )
        assert result.returncode == 0, (
            f"import-time registration failed:\n"
            f"stdout={result.stdout}\nstderr={result.stderr}"
        )
        assert "OK" in result.stdout


class TestDiscoveredProviderOptions:
    """The live UI provider-dropdown path (replaces the removed
    get_available_providers()): get_discovered_provider_options() enumerates
    all discovered provider classes; get_available_discovered_provider_options()
    filters that set by ProviderClass.is_available(settings_snapshot)."""

    def test_discovered_options_shape_and_core_providers(self):
        from local_deep_research.llm.providers import (
            get_discovered_provider_options,
        )

        options = get_discovered_provider_options()
        assert isinstance(options, list) and options
        for opt in options:
            assert "value" in opt and "label" in opt
        values = {opt["value"].lower() for opt in options}
        # Core built-in providers must always be discovered. The local
        # providers (llamacpp, lmstudio) are included so they can't silently
        # drop out of auto-discovery: the model-provider dropdown is derived
        # from this set, and #4594 removed the hardcoded LLAMACPP fallback that
        # would otherwise have masked such a regression.
        for provider in (
            "openai",
            "anthropic",
            "ollama",
            "llamacpp",
            "lmstudio",
        ):
            assert provider in values

    def test_available_options_is_filtered_subset(self):
        from local_deep_research.llm.providers import (
            get_discovered_provider_options,
            get_available_discovered_provider_options,
        )

        all_values = {
            o["value"].lower() for o in get_discovered_provider_options()
        }
        # With no settings snapshot, no API keys / reachable local servers
        # are configured, so the filtered set is a (here empty) subset.
        available = get_available_discovered_provider_options(None)
        assert isinstance(available, list)
        available_values = {o["value"].lower() for o in available}
        assert available_values <= all_values