chore: import upstream snapshot with attribution

sdks/python/tests/unit/evaluation/models/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for evaluation models."""

sdks/python/tests/unit/evaluation/models/test_anthropic_chat_model.py

@@ -0,0 +1,944 @@
+import json
+import sys
+import types
+from contextlib import asynccontextmanager, contextmanager
+from types import SimpleNamespace
+from unittest.mock import MagicMock, AsyncMock
+
+import pydantic
+import pytest
+
+from opik.evaluation.models import models_factory
+from opik.evaluation.models import base_model
+from opik.evaluation.models.anthropic import anthropic_chat_model
+from opik.evaluation.models.anthropic import message_adapter, response_parser
+
+
+class SampleFormat(pydantic.BaseModel):
+    score: int
+    reason: str
+
+
+def _make_text_response(text="ok"):
+    block = SimpleNamespace(type="text", text=text)
+    return SimpleNamespace(content=[block])
+
+
+def _make_tool_use_response(data, *, block_id="call_1", name="json_tool_call"):
+    block = SimpleNamespace(type="tool_use", id=block_id, name=name, input=data)
+    return SimpleNamespace(content=[block])
+
+
+def _install_anthropic_stub(monkeypatch):
+    stub = types.ModuleType("anthropic")
+
+    mock_messages = MagicMock()
+    mock_messages.create = MagicMock(return_value=_make_text_response())
+
+    mock_client = MagicMock()
+    mock_client.messages = mock_messages
+
+    async_mock_messages = MagicMock()
+    async_mock_messages.create = AsyncMock(return_value=_make_text_response())
+
+    async_mock_client = MagicMock()
+    async_mock_client.messages = async_mock_messages
+
+    stub.Anthropic = MagicMock(return_value=mock_client)
+    stub.AsyncAnthropic = MagicMock(return_value=async_mock_client)
+
+    monkeypatch.setitem(sys.modules, "anthropic", stub)
+    return stub, mock_client, async_mock_client
+
+
+@pytest.fixture(autouse=True)
+def _clear_model_cache():
+    models_factory._MODEL_CACHE.clear()
+    yield
+    models_factory._MODEL_CACHE.clear()
+
+
+class TestResponseParser:
+    def test_parses_text_response(self):
+        message = response_parser.parse_assistant_message(
+            _make_text_response("hello world")
+        )
+        assert message == {"role": "assistant", "content": "hello world"}
+
+    def test_concatenates_multiple_text_blocks(self):
+        response = SimpleNamespace(
+            content=[
+                SimpleNamespace(type="text", text="hello "),
+                SimpleNamespace(type="text", text="world"),
+            ]
+        )
+        message = response_parser.parse_assistant_message(response)
+        assert message["content"] == "hello world"
+
+    def test_promotes_single_tool_use_arguments_into_content(self):
+        data = {"score": 10, "reason": "good"}
+        message = response_parser.parse_assistant_message(_make_tool_use_response(data))
+        assert message["role"] == "assistant"
+        assert "tool_calls" not in message
+        assert json.loads(message["content"]) == data
+
+    def test_emits_tool_calls_when_text_and_tool_use_coexist(self):
+        response = SimpleNamespace(
+            content=[
+                SimpleNamespace(type="text", text="picking a tool"),
+                SimpleNamespace(
+                    type="tool_use",
+                    id="call_42",
+                    name="web_search",
+                    input={"query": "capital of France"},
+                ),
+            ]
+        )
+        message = response_parser.parse_assistant_message(response)
+        assert message["content"] == "picking a tool"
+        assert message["tool_calls"] == [
+            {
+                "id": "call_42",
+                "type": "function",
+                "function": {
+                    "name": "web_search",
+                    "arguments": json.dumps({"query": "capital of France"}),
+                },
+            }
+        ]
+
+    def test_raises_when_no_text_and_no_tool_use(self):
+        from opik import exceptions
+
+        response = SimpleNamespace(content=[])
+        with pytest.raises(exceptions.BaseLLMError):
+            response_parser.parse_assistant_message(response)
+
+    def test_keeps_registered_tool_use_as_tool_call(self):
+        """Regression: with `output_format` set, Anthropic emits the
+        structured-output finalizer as a `tool_use` block too. Without
+        disambiguation we'd misclassify a *real* registered-tool call
+        (e.g. `read`) as the finalizer and promote its arguments to
+        `content`, leaving the agentic loop with nothing to execute.
+        Passing `registered_tool_names` lets the parser tell them apart.
+        """
+        response = SimpleNamespace(
+            content=[
+                SimpleNamespace(
+                    type="tool_use",
+                    id="call_42",
+                    name="read",
+                    input={"type": "trace", "id": "t-1"},
+                ),
+            ]
+        )
+        message = response_parser.parse_assistant_message(
+            response, registered_tool_names=["read", "scan", "search"]
+        )
+        assert message["role"] == "assistant"
+        assert "content" not in message
+        assert message["tool_calls"] == [
+            {
+                "id": "call_42",
+                "type": "function",
+                "function": {
+                    "name": "read",
+                    "arguments": json.dumps({"type": "trace", "id": "t-1"}),
+                },
+            }
+        ]
+
+    def test_promotes_unknown_tool_use_when_tools_registered(self):
+        """Counterpart to the previous test: when the single tool_use's
+        name is NOT in the registered set, treat it as the structured-
+        output finalizer (Anthropic's name for it varies by SDK version,
+        but it's always not one of the user's tools).
+        """
+        data = {"score": 10, "reason": "good"}
+        message = response_parser.parse_assistant_message(
+            _make_tool_use_response(data),
+            registered_tool_names=["read", "scan", "search"],
+        )
+        assert "tool_calls" not in message
+        assert json.loads(message["content"]) == data
+
+
+class TestMessageAdapter:
+    def test_extracts_system_messages(self):
+        messages = [
+            {"role": "system", "content": "You are helpful."},
+            {"role": "user", "content": "Hello"},
+        ]
+        system_text, non_system = message_adapter.extract_system_messages(messages)
+        assert system_text == "You are helpful."
+        assert len(non_system) == 1
+        assert non_system[0]["role"] == "user"
+
+    def test_multiple_system_messages(self):
+        messages = [
+            {"role": "system", "content": "Part 1"},
+            {"role": "system", "content": "Part 2"},
+            {"role": "user", "content": "Hello"},
+        ]
+        system_text, non_system = message_adapter.extract_system_messages(messages)
+        assert system_text == "Part 1\n\nPart 2"
+        assert len(non_system) == 1
+
+    def test_no_system_messages(self):
+        messages = [{"role": "user", "content": "Hello"}]
+        system_text, non_system = message_adapter.extract_system_messages(messages)
+        assert system_text is None
+        assert len(non_system) == 1
+
+    def test_converts_pydantic_model_to_output_config(self):
+        config = message_adapter.pydantic_to_output_config(SampleFormat)
+        assert config["format"]["type"] == "json_schema"
+        schema = config["format"]["schema"]
+        assert "score" in schema["properties"]
+        assert "reason" in schema["properties"]
+        assert "title" not in schema
+
+    def test_strips_prefix(self):
+        assert (
+            message_adapter.strip_anthropic_prefix("anthropic/claude-sonnet-4-20250514")
+            == "claude-sonnet-4-20250514"
+        )
+
+    def test_no_prefix(self):
+        assert (
+            message_adapter.strip_anthropic_prefix("claude-sonnet-4-20250514")
+            == "claude-sonnet-4-20250514"
+        )
+
+    def test_filter_unsupported_params_drops_openai_specific(self):
+        warned: set = set()
+        result = message_adapter.filter_unsupported_params(
+            {"temperature": 0.5, "logprobs": True, "top_logprobs": 20, "top_p": 0.9},
+            warned,
+        )
+        assert result == {"temperature": 0.5, "top_p": 0.9}
+        assert "logprobs" in warned
+        assert "top_logprobs" in warned
+
+    def test_filter_unsupported_params_warns_once(self):
+        warned: set = set()
+        message_adapter.filter_unsupported_params({"logprobs": True}, warned)
+        message_adapter.filter_unsupported_params({"logprobs": True}, warned)
+        assert warned == {"logprobs"}
+
+    def test_normalize_tool_choice_translates_openai_strings(self):
+        # OpenAI-style string forms map to Anthropic's object form.
+        assert message_adapter.normalize_tool_choice("auto") == {"type": "auto"}
+        assert message_adapter.normalize_tool_choice("none") == {"type": "none"}
+        # "required" → "any" (Anthropic's name for "force *some* tool").
+        assert message_adapter.normalize_tool_choice("required") == {"type": "any"}
+
+    def test_normalize_tool_choice_translates_openai_function_object(self):
+        # OpenAI "force this specific function" → Anthropic "force this tool".
+        translated = message_adapter.normalize_tool_choice(
+            {"type": "function", "function": {"name": "read"}}
+        )
+        assert translated == {"type": "tool", "name": "read"}
+
+    def test_normalize_tool_choice_passes_through_anthropic_native_shape(self):
+        # Already-correct Anthropic forms shouldn't be touched.
+        assert message_adapter.normalize_tool_choice({"type": "auto"}) == {
+            "type": "auto"
+        }
+        assert message_adapter.normalize_tool_choice(
+            {"type": "tool", "name": "read"}
+        ) == {"type": "tool", "name": "read"}
+
+    def test_normalize_tool_choice_passes_through_unknown_values(self):
+        # Unrecognized strings or shapes pass through unchanged so the
+        # Anthropic SDK can surface the error rather than us silently
+        # dropping the field.
+        assert message_adapter.normalize_tool_choice("bogus") == "bogus"
+        assert message_adapter.normalize_tool_choice(
+            {"type": "function"}  # missing function.name
+        ) == {"type": "function"}
+
+    def test_normalize_tools_translates_openai_function_specs(self):
+        openai_spec = {
+            "type": "function",
+            "function": {
+                "name": "read",
+                "description": "Fetch a trace by id.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"id": {"type": "string"}},
+                    "required": ["id"],
+                },
+            },
+        }
+        translated = message_adapter.normalize_tools([openai_spec])
+        assert translated == [
+            {
+                "type": "custom",
+                "name": "read",
+                "description": "Fetch a trace by id.",
+                "input_schema": {
+                    "type": "object",
+                    "properties": {"id": {"type": "string"}},
+                    "required": ["id"],
+                },
+            }
+        ]
+
+    def test_normalize_tools_passes_through_native_anthropic_specs(self):
+        # Hand-rolled Anthropic-native specs (no `type=function` wrapper)
+        # must not be rewritten — we have no information to safely map
+        # them, and rewriting would corrupt a working spec.
+        native = {
+            "type": "custom",
+            "name": "scan",
+            "description": "Evaluate a jq path.",
+            "input_schema": {"type": "object"},
+        }
+        assert message_adapter.normalize_tools([native]) == [native]
+
+    def test_normalize_tools_passes_through_malformed_specs(self):
+        # No function.name → unrecognizable; pass through so the SDK
+        # surfaces the error rather than us masking it.
+        malformed = {"type": "function", "function": {"description": "x"}}
+        assert message_adapter.normalize_tools([malformed]) == [malformed]
+
+    def test_normalize_tools_passes_through_non_list_input(self):
+        # `None` (or any non-list sentinel) should not blow up — just
+        # return it so the SDK's own validation handles it.
+        assert message_adapter.normalize_tools(None) is None
+
+    def test_extract_tool_names_handles_openai_shape(self):
+        tools = [
+            {"type": "function", "function": {"name": "read"}},
+            {"type": "function", "function": {"name": "scan"}},
+        ]
+        assert message_adapter.extract_tool_names(tools) == ["read", "scan"]
+
+    def test_extract_tool_names_handles_anthropic_native_shape(self):
+        # After `normalize_tools` runs, names live at the top level.
+        # `extract_tool_names` must handle both shapes so it stays
+        # usable on either side of normalization.
+        tools = [
+            {"type": "custom", "name": "read"},
+            {"type": "custom", "name": "search"},
+        ]
+        assert message_adapter.extract_tool_names(tools) == ["read", "search"]
+
+    def test_extract_tool_names_skips_malformed_entries(self):
+        tools = [
+            {"type": "function"},  # missing function dict
+            {"type": "function", "function": {"description": "x"}},  # no name
+            {"type": "custom", "name": "read"},  # well-formed
+            "not a dict",  # ignored
+        ]
+        assert message_adapter.extract_tool_names(tools) == ["read"]
+
+    def test_extract_tool_names_returns_empty_for_non_list(self):
+        assert message_adapter.extract_tool_names(None) == []
+        assert message_adapter.extract_tool_names("nope") == []
+
+    def test_normalize_messages_passes_through_plain_history(self):
+        # User + plain assistant text → no shape change beyond the
+        # `tool_calls=None` cleanup that pop'd through the loop.
+        messages = [
+            {"role": "user", "content": "hi"},
+            {"role": "assistant", "content": "hello"},
+        ]
+        assert message_adapter.normalize_messages(messages) == messages
+
+    def test_normalize_messages_converts_assistant_tool_calls_to_blocks(self):
+        # Assistant message with one tool_call → assistant message
+        # whose content is a list of blocks (no leading text block
+        # when `content` is empty/None).
+        messages = [
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {
+                            "name": "read",
+                            "arguments": json.dumps({"type": "trace", "id": "t-1"}),
+                        },
+                    }
+                ],
+            }
+        ]
+        assert message_adapter.normalize_messages(messages) == [
+            {
+                "role": "assistant",
+                "content": [
+                    {
+                        "type": "tool_use",
+                        "id": "call_1",
+                        "name": "read",
+                        "input": {"type": "trace", "id": "t-1"},
+                    }
+                ],
+            }
+        ]
+
+    def test_normalize_messages_keeps_leading_text_block(self):
+        # Assistant emits text alongside the tool_use — both blocks
+        # must appear, text first.
+        messages = [
+            {
+                "role": "assistant",
+                "content": "let me check",
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {
+                            "name": "read",
+                            "arguments": "{}",
+                        },
+                    }
+                ],
+            }
+        ]
+        normalized = message_adapter.normalize_messages(messages)
+        assert normalized[0]["content"] == [
+            {"type": "text", "text": "let me check"},
+            {"type": "tool_use", "id": "call_1", "name": "read", "input": {}},
+        ]
+
+    def test_normalize_messages_translates_tool_role_to_user_tool_result(self):
+        messages = [
+            {
+                "role": "tool",
+                "tool_call_id": "call_1",
+                "content": "{'data': 'value'}",
+            }
+        ]
+        assert message_adapter.normalize_messages(messages) == [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "call_1",
+                        "content": "{'data': 'value'}",
+                    }
+                ],
+            }
+        ]
+
+    def test_normalize_messages_coalesces_consecutive_tool_messages(self):
+        # Two tool replies in a row must end up as a single user
+        # message with two tool_result blocks — Anthropic rejects
+        # split tool_result responses when the prior assistant turn
+        # emitted multiple tool_use blocks.
+        messages = [
+            {"role": "tool", "tool_call_id": "call_1", "content": "result-1"},
+            {"role": "tool", "tool_call_id": "call_2", "content": "result-2"},
+        ]
+        assert message_adapter.normalize_messages(messages) == [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "call_1",
+                        "content": "result-1",
+                    },
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "call_2",
+                        "content": "result-2",
+                    },
+                ],
+            }
+        ]
+
+    def test_normalize_messages_coerces_non_object_arguments_to_empty_dict(self):
+        """Anthropic's `tool_use.input` is specified as a JSON object.
+        OpenAI's `arguments` is *almost* always a stringified dict, but
+        a malformed model output (top-level array, scalar, null, or
+        non-JSON text) could leak a non-dict value through. The
+        translator must coerce those to `{}` so we hit the SDK's own
+        schema validation instead of a generic 400 from the API.
+        """
+        # Top-level JSON list → empty dict.
+        messages = [
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {
+                            "name": "read",
+                            "arguments": json.dumps([1, 2, 3]),
+                        },
+                    }
+                ],
+            }
+        ]
+        normalized = message_adapter.normalize_messages(messages)
+        assert normalized[0]["content"][0]["input"] == {}
+
+        # Top-level scalar → empty dict.
+        messages[0]["tool_calls"][0]["function"]["arguments"] = json.dumps(42)
+        assert (
+            message_adapter.normalize_messages(messages)[0]["content"][0]["input"] == {}
+        )
+
+        # Null → empty dict.
+        messages[0]["tool_calls"][0]["function"]["arguments"] = "null"
+        assert (
+            message_adapter.normalize_messages(messages)[0]["content"][0]["input"] == {}
+        )
+
+        # Non-JSON text → empty dict.
+        messages[0]["tool_calls"][0]["function"]["arguments"] = "not json"
+        assert (
+            message_adapter.normalize_messages(messages)[0]["content"][0]["input"] == {}
+        )
+
+        # Already-a-list (not a string) → empty dict too — we only
+        # forward dict-shaped values.
+        messages[0]["tool_calls"][0]["function"]["arguments"] = [1, 2]
+        assert (
+            message_adapter.normalize_messages(messages)[0]["content"][0]["input"] == {}
+        )
+
+    def test_normalize_messages_full_round_trip(self):
+        # End-to-end: a typical agentic-loop history with one
+        # round-trip should land in valid Anthropic shape.
+        messages = [
+            {"role": "user", "content": "find the marker"},
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {
+                            "name": "read",
+                            "arguments": json.dumps({"type": "trace", "id": "t-1"}),
+                        },
+                    }
+                ],
+            },
+            {"role": "tool", "tool_call_id": "call_1", "content": "MARKER-XYZ-987"},
+        ]
+        normalized = message_adapter.normalize_messages(messages)
+        # User → unchanged.
+        assert normalized[0] == {"role": "user", "content": "find the marker"}
+        # Assistant → tool_use content block.
+        assert normalized[1]["role"] == "assistant"
+        assert normalized[1]["content"][0]["type"] == "tool_use"
+        # Tool result → user message with tool_result block.
+        assert normalized[2]["role"] == "user"
+        assert normalized[2]["content"][0]["type"] == "tool_result"
+        assert normalized[2]["content"][0]["tool_use_id"] == "call_1"
+
+
+class TestAnthropicChatModelGenerateString:
+    def test_generate_string_text(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        @contextmanager
+        def fake_provider_response(model_provider, messages, **kwargs):
+            yield _make_text_response("test output")
+
+        monkeypatch.setattr(base_model, "get_provider_response", fake_provider_response)
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+        result = model.generate_string("hello")
+        assert result == "test output"
+
+    def test_generate_string_with_response_format(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        json_text = json.dumps({"score": 10, "reason": "good"})
+
+        @contextmanager
+        def fake_provider_response(model_provider, messages, **kwargs):
+            yield _make_text_response(json_text)
+
+        monkeypatch.setattr(base_model, "get_provider_response", fake_provider_response)
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+        result = model.generate_string("hello", response_format=SampleFormat)
+        parsed = json.loads(result)
+        assert parsed["score"] == 10
+        assert parsed["reason"] == "good"
+
+
+class TestAnthropicChatModelProviderResponse:
+    def test_passes_system_as_top_level_param(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        messages = [
+            {"role": "system", "content": "Be helpful"},
+            {"role": "user", "content": "Hello"},
+        ]
+        model.generate_provider_response(messages)
+
+        call_kwargs = mock_client.messages.create.call_args
+        assert call_kwargs.kwargs["system"] == "Be helpful"
+        assert all(m["role"] != "system" for m in call_kwargs.kwargs["messages"])
+
+    def test_response_format_uses_parse(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        mock_client.messages.parse = MagicMock(return_value=_make_text_response())
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        messages = [{"role": "user", "content": "Score this"}]
+        model.generate_provider_response(messages, response_format=SampleFormat)
+
+        mock_client.messages.parse.assert_called_once()
+        call_kwargs = mock_client.messages.parse.call_args.kwargs
+        assert call_kwargs["output_format"] is SampleFormat
+        assert "tools" not in call_kwargs
+        assert "tool_choice" not in call_kwargs
+
+    def test_default_max_tokens(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        model.generate_provider_response([{"role": "user", "content": "hi"}])
+        call_kwargs = mock_client.messages.create.call_args.kwargs
+        assert call_kwargs["max_tokens"] == 4096
+
+    def test_strips_anthropic_prefix_in_api_call(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        model.generate_provider_response([{"role": "user", "content": "hi"}])
+        call_kwargs = mock_client.messages.create.call_args.kwargs
+        assert call_kwargs["model"] == "claude-sonnet-4-20250514"
+
+    def test_constructor_tools_feed_response_parser_disambiguation(self, monkeypatch):
+        """Regression: when `tools` is supplied only at construction time
+        (the agentic loop's default path through the factory), the
+        response parser still needs the registered names to tell a real
+        `read` tool call from the structured-output finalizer. Looking
+        only at the per-call `kwargs` (as the original code did) would
+        miss constructor-time tools and leave the parser blind.
+        """
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        mock_client.messages.parse = MagicMock(
+            return_value=SimpleNamespace(
+                content=[
+                    SimpleNamespace(
+                        type="tool_use",
+                        id="call_42",
+                        name="read",
+                        input={"type": "trace", "id": "t-1"},
+                    )
+                ]
+            )
+        )
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514",
+            track=False,
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "read",
+                        "description": "Fetch a trace.",
+                        "parameters": {"type": "object", "properties": {}},
+                    },
+                },
+            ],
+        )
+
+        # Per-call `tools` omitted on purpose — the constructor-time
+        # list must still reach the parser.
+        message = model.generate_chat_completion(
+            messages=[{"role": "user", "content": "go"}],
+            response_format=SampleFormat,
+        )
+
+        # If the parser got the registered names, the `read` tool_use
+        # block stays as a tool_call. Without them, it would have been
+        # promoted to `content` and the test would see `tool_calls`
+        # missing.
+        assert message["tool_calls"] == [
+            {
+                "id": "call_42",
+                "type": "function",
+                "function": {
+                    "name": "read",
+                    "arguments": json.dumps({"type": "trace", "id": "t-1"}),
+                },
+            }
+        ]
+
+    def test_per_call_tools_override_constructor_tools_for_parser(self, monkeypatch):
+        """Per-call `tools` replace constructor-time `tools` in
+        `_build_call_kwargs` (last-write-wins merge). The names the
+        parser sees must follow the same precedence — otherwise a
+        caller who narrows tools per-call could still get a tool_use
+        block misclassified because the parser was keyed on the wider
+        constructor set, or vice versa.
+        """
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        # Response uses the per-call tool name (`scan`), not the
+        # constructor name (`read`) — proves the per-call list wins.
+        mock_client.messages.create = MagicMock(
+            return_value=SimpleNamespace(
+                content=[
+                    SimpleNamespace(
+                        type="tool_use",
+                        id="call_99",
+                        name="scan",
+                        input={"path": "$.trace"},
+                    )
+                ]
+            )
+        )
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514",
+            track=False,
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "read",
+                        "description": "Fetch a trace.",
+                        "parameters": {"type": "object"},
+                    },
+                },
+            ],
+        )
+
+        message = model.generate_chat_completion(
+            messages=[{"role": "user", "content": "go"}],
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "scan",
+                        "description": "Evaluate a jq path.",
+                        "parameters": {"type": "object"},
+                    },
+                },
+            ],
+        )
+
+        # `scan` (the per-call tool) is in the registered set the
+        # parser saw, so the response's `scan` tool_use stays as a
+        # tool call. If the precedence was wrong, the parser would have
+        # seen only `read` and promoted `scan` to content.
+        assert message.get("tool_calls", [])[0]["function"]["name"] == "scan"
+
+
+class TestParamFiltering:
+    def test_filters_unsupported_constructor_kwargs(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514",
+            track=False,
+            temperature=0.5,
+            logprobs=True,
+            top_logprobs=20,
+            frequency_penalty=0.1,
+        )
+
+        assert model._completion_kwargs["temperature"] == 0.5
+        assert "logprobs" not in model._completion_kwargs
+        assert "top_logprobs" not in model._completion_kwargs
+        assert "frequency_penalty" not in model._completion_kwargs
+
+    def test_normalizes_constructor_tools_into_anthropic_shape(self, monkeypatch):
+        """Regression: OpenAI-shape `tools` passed at construction time
+        (e.g. via `AnthropicChatModel(tools=[...])` or the factory)
+        must be normalized before they're merged into per-call kwargs.
+        Without this, the per-call normalization in `_build_call_kwargs`
+        only catches `tools` arriving as call-time kwargs, and the
+        constructor-time list reaches the Anthropic SDK unchanged.
+        """
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514",
+            track=False,
+            tools=[
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "read",
+                        "description": "Fetch a trace.",
+                        "parameters": {"type": "object", "properties": {}},
+                    },
+                },
+            ],
+        )
+
+        model.generate_provider_response([{"role": "user", "content": "hi"}])
+
+        call_kwargs = mock_client.messages.create.call_args.kwargs
+        assert call_kwargs["tools"] == [
+            {
+                "type": "custom",
+                "name": "read",
+                "description": "Fetch a trace.",
+                "input_schema": {"type": "object", "properties": {}},
+            }
+        ]
+
+    def test_normalizes_constructor_tool_choice_into_anthropic_shape(self, monkeypatch):
+        # Companion to the `tools` test: this is the existing
+        # constructor-side normalization for `tool_choice`. Pinning it
+        # here so future refactors that reshuffle the __init__ order
+        # don't silently regress the pairing.
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514",
+            track=False,
+            tool_choice="auto",
+        )
+        assert model._completion_kwargs["tool_choice"] == {"type": "auto"}
+
+    def test_filters_unsupported_per_call_kwargs(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        model.generate_provider_response(
+            [{"role": "user", "content": "hi"}],
+            logprobs=True,
+            top_logprobs=20,
+            temperature=0.7,
+        )
+
+        call_kwargs = mock_client.messages.create.call_args.kwargs
+        assert call_kwargs["temperature"] == 0.7
+        assert "logprobs" not in call_kwargs
+        assert "top_logprobs" not in call_kwargs
+
+    def test_keeps_all_valid_anthropic_params(self, monkeypatch):
+        _, mock_client, _ = _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+
+        model.generate_provider_response(
+            [{"role": "user", "content": "hi"}],
+            temperature=0.5,
+            top_p=0.9,
+            top_k=40,
+            stop_sequences=["END"],
+        )
+
+        call_kwargs = mock_client.messages.create.call_args.kwargs
+        assert call_kwargs["temperature"] == 0.5
+        assert call_kwargs["top_p"] == 0.9
+        assert call_kwargs["top_k"] == 40
+        assert call_kwargs["stop_sequences"] == ["END"]
+
+
+class TestFactoryRouting:
+    def test_factory_routes_anthropic_prefix(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = models_factory.get("anthropic/claude-sonnet-4-20250514", track=False)
+        assert isinstance(model, anthropic_chat_model.AnthropicChatModel)
+
+    def test_factory_routes_bare_claude_name(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        model = models_factory.get("claude-sonnet-4-20250514", track=False)
+        assert isinstance(model, anthropic_chat_model.AnthropicChatModel)
+
+    def test_factory_does_not_route_non_anthropic(self, monkeypatch):
+        litellm_stub = types.ModuleType("litellm")
+        litellm_stub.suppress_debug_info = False
+
+        def completion(model, messages, **kwargs):
+            return SimpleNamespace(
+                choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))]
+            )
+
+        litellm_stub.completion = completion
+        litellm_stub.acompletion = completion
+        litellm_stub.get_supported_openai_params = lambda model: [
+            "temperature",
+            "response_format",
+        ]
+        litellm_stub.get_llm_provider = lambda model: ("openai", "openai")
+        litellm_stub.utils = SimpleNamespace(UnsupportedParamsError=Exception)
+        litellm_stub.exceptions = SimpleNamespace(BadRequestError=Exception)
+        litellm_stub.callbacks = []
+        monkeypatch.setitem(sys.modules, "litellm", litellm_stub)
+
+        litellm_integration_stub = types.ModuleType("opik.integrations.litellm")
+        litellm_integration_stub.track_completion = lambda **kw: (lambda f: f)
+        monkeypatch.setitem(
+            sys.modules, "opik.integrations.litellm", litellm_integration_stub
+        )
+
+        model = models_factory.get("gpt-4o", track=False)
+        from opik.evaluation.models.litellm.litellm_chat_model import LiteLLMChatModel
+
+        assert isinstance(model, LiteLLMChatModel)
+
+
+@pytest.mark.asyncio
+class TestAnthropicChatModelAsync:
+    async def test_agenerate_string(self, monkeypatch):
+        _install_anthropic_stub(monkeypatch)
+        monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "false")
+
+        @asynccontextmanager
+        async def fake_aget_provider_response(model_provider, messages, **kwargs):
+            yield _make_text_response("async result")
+
+        monkeypatch.setattr(
+            base_model, "aget_provider_response", fake_aget_provider_response
+        )
+
+        model = anthropic_chat_model.AnthropicChatModel(
+            model_name="anthropic/claude-sonnet-4-20250514", track=False
+        )
+        result = await model.agenerate_string("hello async")
+        assert result == "async result"

sdks/python/tests/unit/evaluation/models/test_litellm_chat_model.py

@@ -0,0 +1,878 @@
+import json
+import logging
+import sys
+import types
+from contextlib import asynccontextmanager, contextmanager
+from types import SimpleNamespace
+
+import pytest
+
+from opik.evaluation.metrics.llm_judges import g_eval
+from opik.evaluation.models import models_factory
+from opik.evaluation.models.litellm import litellm_chat_model, response_parser
+from opik.evaluation.models import base_model
+
+
+def _install_litellm_stub(monkeypatch, *, supported_params=None):
+    stub_module = types.ModuleType("litellm")
+    stub_module.suppress_debug_info = False
+    stub_module._calls = []
+
+    if supported_params is None:
+        supported_params = ["temperature", "response_format"]
+
+    def completion(model, messages, **kwargs):
+        stub_module._calls.append((model, messages, kwargs))
+        return SimpleNamespace(
+            choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))]
+        )
+
+    async def acompletion(model, messages, **kwargs):
+        # Async version for testing
+        return await completion(model, messages, **kwargs)
+
+    def get_supported_openai_params(model):
+        return list(supported_params)
+
+    def get_llm_provider(model):
+        return ("openai", "openai")
+
+    stub_module.completion = completion
+    stub_module.acompletion = acompletion
+    stub_module.get_supported_openai_params = get_supported_openai_params
+    stub_module.get_llm_provider = get_llm_provider
+    stub_module.utils = SimpleNamespace(UnsupportedParamsError=Exception)
+    stub_module.exceptions = SimpleNamespace(BadRequestError=Exception)
+    stub_module.callbacks = []
+
+    monkeypatch.setitem(sys.modules, "litellm", stub_module)
+
+    # Mock the track_completion decorator to be a no-op for unit tests
+    def mock_track_completion(project_name=None):
+        def decorator(func):
+            # Mark as tracked to prevent actual tracking
+            func.opik_tracked = True
+            return func
+
+        return decorator
+
+    litellm_integration_stub = types.ModuleType("opik.integrations.litellm")
+    litellm_integration_stub.track_completion = mock_track_completion
+    monkeypatch.setitem(
+        sys.modules, "opik.integrations.litellm", litellm_integration_stub
+    )
+
+    return stub_module
+
+
+@pytest.fixture(autouse=True)
+def _clear_model_cache():
+    models_factory._MODEL_CACHE.clear()
+    yield
+    models_factory._MODEL_CACHE.clear()
+
+
+def test_models_factory_reuses_cached_instance(monkeypatch):
+    _install_litellm_stub(monkeypatch)
+
+    first_gpt5 = models_factory.get("gpt-5-nano")
+    second_gpt5 = models_factory.get("gpt-5-nano")
+
+    first_gpt4 = models_factory.get("gpt-4o")
+    second_gpt4 = models_factory.get("gpt-4o")
+
+    assert first_gpt5 is second_gpt5
+    assert first_gpt4 is second_gpt4
+    assert first_gpt5 is not first_gpt4
+
+
+def test_models_factory_cache_freezes_unhashable(monkeypatch):
+    _install_litellm_stub(monkeypatch)
+
+    params = {"metadata": {"labels": ["a", "b"], "nested": {"c"}}}
+    first = models_factory.get("gpt-4o", **params)
+    second = models_factory.get("gpt-4o", **params)
+
+    assert first is second
+
+
+def test_models_factory_default_model(monkeypatch):
+    _install_litellm_stub(monkeypatch)
+
+    default_instance = models_factory.get(None)
+
+    assert default_instance.model_name == "openai/gpt-5-nano"
+
+
+def test_models_factory_default_model_from_env(monkeypatch):
+    _install_litellm_stub(monkeypatch)
+    monkeypatch.setenv("OPIK_DEFAULT_LLM", "gpt-4o-mini")
+
+    default_instance = models_factory.get(None)
+
+    assert default_instance.model_name == "gpt-4o-mini"
+
+
+class TestCoerceTemperatureToFloat:
+    """The helper underpins every "is temperature 1?" check on the
+    LiteLLM path (GPT-5 filter, Anthropic reasoning_effort conflict).
+    Pin its semantics directly so the indirect tests above don't
+    over-couple to incidental call-site behavior.
+    """
+
+    def test_returns_float_for_int(self):
+        from opik.evaluation.models.litellm import util
+
+        assert util.coerce_temperature_to_float(1) == 1.0
+
+    def test_returns_float_for_float(self):
+        from opik.evaluation.models.litellm import util
+
+        assert util.coerce_temperature_to_float(0.5) == 0.5
+
+    def test_returns_float_for_numeric_string(self):
+        from opik.evaluation.models.litellm import util
+
+        # The bug the reviewer flagged: `temperature="1"` would compare
+        # unequal to `1` and trigger an unwanted drop. Pinning the
+        # coercion path directly here so regressions surface fast.
+        assert util.coerce_temperature_to_float("1") == 1.0
+        assert util.coerce_temperature_to_float("1.0") == 1.0
+        assert util.coerce_temperature_to_float(" 0.7 ") == 0.7
+
+    def test_returns_none_for_non_numeric_string(self):
+        from opik.evaluation.models.litellm import util
+
+        assert util.coerce_temperature_to_float("not-a-number") is None
+
+    def test_returns_none_for_none(self):
+        from opik.evaluation.models.litellm import util
+
+        assert util.coerce_temperature_to_float(None) is None
+
+    def test_returns_none_for_arbitrary_object(self):
+        from opik.evaluation.models.litellm import util
+
+        assert util.coerce_temperature_to_float(object()) is None
+
+
+def test_litellm_chat_model_drops_temperature_for_gpt5(monkeypatch, caplog):
+    stub = _install_litellm_stub(monkeypatch)
+
+    caplog.set_level(logging.WARNING)
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="gpt-5-nano",
+        temperature=0.5,
+    )
+
+    assert any(
+        "temperature" in record.message and "Dropping" in record.message
+        for record in caplog.records
+    )
+
+    caplog.clear()
+    model.generate_string("hello")
+
+    # Assert against the actual outbound call, not on the private
+    # `_completion_kwargs` constructor state — the contract callers
+    # care about is "what gets sent to litellm.completion".
+    assert stub._calls, "Expected completion to be invoked"
+    _, _, kwargs = stub._calls[-1]
+    assert "temperature" not in kwargs
+    assert not caplog.records
+
+
+def test_litellm_chat_model_drops_temperature_for_provider_prefixed_gpt5(
+    monkeypatch, caplog
+):
+    stub = _install_litellm_stub(monkeypatch)
+
+    caplog.set_level(logging.WARNING)
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="openai/gpt-5-nano",
+        temperature=1e-8,
+    )
+
+    assert any(
+        "temperature" in record.message and "Dropping" in record.message
+        for record in caplog.records
+    )
+
+    caplog.clear()
+    model.generate_string("hello")
+
+    assert stub._calls, "Expected completion to be invoked"
+    _, _, kwargs = stub._calls[-1]
+    assert "temperature" not in kwargs
+    assert not caplog.records
+
+
+def test_litellm_chat_model_drops_seed_when_provider_does_not_support(
+    monkeypatch, caplog
+):
+    """`seed` is an OpenAI-shape param; Anthropic (and a handful of
+    other providers) reject it with `UnsupportedParamsError` rather
+    than ignoring it. The native `AnthropicChatModel` silently filters
+    it; this test pins the same behavior on the LiteLLM path so callers
+    that pass `seed` (e.g. the agentic judge integration tests for
+    reproducibility) don't blow up when the underlying provider is
+    Anthropic.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        # Mirror Anthropic's litellm-reported support: no `seed`.
+        supported_params=["temperature", "response_format", "tools", "tool_choice"],
+    )
+
+    caplog.set_level(
+        logging.DEBUG, logger="opik.evaluation.models.litellm.litellm_chat_model"
+    )
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        seed=42,
+        temperature=0.0,
+    )
+
+    model.generate_string("hello")
+
+    # Public observable: the outbound litellm.completion call. `seed`
+    # must be filtered out; `temperature` must survive.
+    assert stub._calls, "Expected completion to be invoked"
+    _, _, kwargs = stub._calls[-1]
+    assert "seed" not in kwargs
+    assert kwargs.get("temperature") == 0.0
+
+
+def test_litellm_chat_model_drops_reasoning_effort_for_anthropic_when_temperature_conflicts(
+    monkeypatch, caplog
+):
+    """LiteLLM translates OpenAI-shape `reasoning_effort` into the
+    Anthropic-specific `thinking` parameter; with thinking enabled,
+    Anthropic requires `temperature == 1`. Callers that set a
+    deterministic `temperature` (the agentic loop's default, plus
+    most reproducibility-sensitive callers) hit a 400 from the
+    provider. The drop only fires when the conflict is real — i.e.
+    when temperature is explicitly set to a non-1 value — so callers
+    who opt into both keep extended thinking. See the
+    `_keeps_reasoning_effort_when_temperature_is_one` counterpart.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        # LiteLLM reports `reasoning_effort` as supported for
+        # Anthropic — that's exactly why the generic
+        # supported_params filter doesn't catch it.
+        supported_params=[
+            "temperature",
+            "response_format",
+            "reasoning_effort",
+            "tools",
+            "tool_choice",
+        ],
+    )
+
+    caplog.set_level(
+        logging.DEBUG, logger="opik.evaluation.models.litellm.litellm_chat_model"
+    )
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="low",
+        temperature=0.0,
+    )
+
+    model.generate_string("hello")
+
+    _, _, kwargs = stub._calls[-1]
+    # The actual call to `litellm.completion` is what matters: the
+    # conflict-resolution pass must strip `reasoning_effort` from the
+    # outbound request even though it was set at construction time.
+    # `temperature` is preserved because it's the half of the conflict
+    # that survives, and the provider needs it to know the request is
+    # in deterministic mode.
+    assert "reasoning_effort" not in kwargs
+    assert kwargs.get("temperature") == 0.0
+
+
+def test_litellm_chat_model_drops_reasoning_effort_for_bare_claude_when_temperature_conflicts(
+    monkeypatch,
+):
+    # Same drop must trigger for the bare `claude-...` form too,
+    # matching `model_name_helper.is_anthropic_model`'s predicate.
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "response_format", "reasoning_effort"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="claude-sonnet-4-6",
+        reasoning_effort="low",
+        temperature=0.5,
+    )
+
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    assert "reasoning_effort" not in kwargs
+
+
+def test_litellm_chat_model_drops_reasoning_effort_when_temperature_is_per_call_only(
+    monkeypatch,
+):
+    """Cross-source conflict — `reasoning_effort` from constructor,
+    `temperature=0` from the per-call kwargs (the agentic judge loop's
+    pattern). The conflict-resolution pass must run on the merged
+    effective dict so it catches conflicts whose two halves come from
+    different sources. The per-source `_remove_unnecessary_not_supported_params`
+    by itself couldn't see this — at constructor time it never saw the
+    per-call temperature, and at call time it never sees the
+    constructor-time reasoning_effort.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "response_format", "reasoning_effort"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="low",
+    )
+
+    # Per-call temperature=0 (the agentic judge loop's hardcoded pin)
+    # is the half of the conflict that lives outside `_completion_kwargs`.
+    model.generate_string("hello", temperature=0)
+    _, _, kwargs = stub._calls[-1]
+    assert "reasoning_effort" not in kwargs
+
+
+def test_litellm_chat_model_keeps_reasoning_effort_when_temperature_is_string_one(
+    monkeypatch,
+):
+    """`temperature` can arrive as a string ("1", "1.0") and LiteLLM
+    coerces it before the API call. The conflict-resolution pass must
+    coerce the same way before comparing, otherwise it'd drop
+    `reasoning_effort` on a caller who legitimately opted into
+    thinking via a stringy temperature.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    for stringy_one in ("1", "1.0", " 1 "):
+        model = litellm_chat_model.LiteLLMChatModel(
+            model_name="anthropic/claude-haiku-4-5",
+            reasoning_effort="low",
+            temperature=stringy_one,
+        )
+        model.generate_string("hello")
+        _, _, kwargs = stub._calls[-1]
+        assert kwargs.get("reasoning_effort") == "low", (
+            f"temperature={stringy_one!r} should coerce to 1.0 and "
+            f"preserve reasoning_effort; got kwargs={kwargs}"
+        )
+
+
+def test_litellm_chat_model_drops_reasoning_effort_when_temperature_is_string_non_one(
+    monkeypatch,
+):
+    # Counterpart: stringy non-1 values must still trigger the drop.
+    # Otherwise stringy callers would bypass conflict detection
+    # entirely.
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="low",
+        temperature="0.5",
+    )
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    assert "reasoning_effort" not in kwargs
+
+
+def test_litellm_chat_model_keeps_reasoning_effort_when_temperature_is_non_numeric(
+    monkeypatch,
+):
+    """Coercion failure ("unknown" type) is treated as "don't drop" —
+    a non-numeric temperature is the provider's problem to surface,
+    and we shouldn't compound the error by also silently dropping
+    `reasoning_effort`.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="low",
+        temperature="not-a-number",
+    )
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    # `reasoning_effort` stays — the provider will surface the bad
+    # temperature shape on its own.
+    assert kwargs.get("reasoning_effort") == "low"
+
+
+def test_litellm_chat_model_drops_reasoning_effort_when_reasoning_effort_is_per_call_only(
+    monkeypatch,
+):
+    """Mirror of the previous test — the other cross-source ordering:
+    `temperature` from the constructor, `reasoning_effort` arriving
+    per-call. Without the merged-dict check this would slip past:
+    constructor `_remove_unnecessary_not_supported_params` saw only
+    `temperature`, per-call `_remove_unnecessary_not_supported_params`
+    saw only `reasoning_effort`, neither was the conflict pair.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "response_format", "reasoning_effort"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        temperature=0,
+    )
+
+    model.generate_string("hello", reasoning_effort="low")
+    _, _, kwargs = stub._calls[-1]
+    assert "reasoning_effort" not in kwargs
+
+
+def test_litellm_chat_model_keeps_reasoning_effort_for_anthropic_when_temperature_is_one(
+    monkeypatch,
+):
+    """Opt-in path for Anthropic extended thinking: explicit
+    `temperature=1` plus `reasoning_effort=...` keeps both. The
+    `thinking` mode LiteLLM enables under the hood is compatible with
+    `temperature=1`, so the drop must not fire.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="medium",
+        temperature=1,
+    )
+
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    # Both must reach the outbound call — that's the opt-in shape for
+    # Anthropic extended thinking.
+    assert kwargs.get("reasoning_effort") == "medium"
+    assert kwargs.get("temperature") == 1
+
+
+def test_litellm_chat_model_keeps_reasoning_effort_for_anthropic_when_temperature_omitted(
+    monkeypatch,
+):
+    """When `temperature` isn't set explicitly, Anthropic defaults to
+    1 server-side, which doesn't conflict with thinking mode. The
+    drop must not fire under that signal-absent state.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="anthropic/claude-haiku-4-5",
+        reasoning_effort="low",
+    )
+
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    # `reasoning_effort` must reach the outbound call; `temperature`
+    # was never set, so it must not appear either (Anthropic defaults
+    # to 1 server-side).
+    assert kwargs.get("reasoning_effort") == "low"
+    assert "temperature" not in kwargs
+
+
+def test_litellm_chat_model_keeps_reasoning_effort_for_openai(monkeypatch):
+    """Counterpart to the Anthropic drop tests: for OpenAI (and any
+    other provider whose litellm support doesn't translate
+    `reasoning_effort` into a conflicting param), the value must
+    round-trip so determinism + reasoning callers keep working.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "reasoning_effort", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="gpt-5-mini",
+        reasoning_effort="low",
+    )
+
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    assert kwargs.get("reasoning_effort") == "low"
+
+
+def test_litellm_chat_model_keeps_seed_when_provider_supports_it(monkeypatch):
+    """Counterpart to the drop test: when `seed` IS in the provider's
+    supported set (e.g. OpenAI), it must round-trip through to the
+    completion call so determinism still works on providers that
+    honor it.
+    """
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["temperature", "seed", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="gpt-4o-mini",
+        seed=42,
+    )
+
+    model.generate_string("hello")
+    _, _, kwargs = stub._calls[-1]
+    assert kwargs.get("seed") == 42
+
+
+def test_litellm_chat_model_drops_top_logprobs_for_dashscope(
+    monkeypatch,
+):
+    stub = _install_litellm_stub(
+        monkeypatch,
+        supported_params=["logprobs", "top_logprobs", "response_format"],
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="dashscope/qwen-flash",
+        logprobs=True,
+        top_logprobs=10,
+    )
+
+    model.generate_string("hello")
+
+    # top_logprobs should not be forwarded to the provider (public
+    # observable — what `litellm.completion` actually receives).
+    assert stub._calls, "Expected completion to be invoked"
+    _, _, kwargs = stub._calls[-1]
+    assert "top_logprobs" not in kwargs
+
+
+def test_geval_passes_logprobs_only_when_supported(monkeypatch):
+    _install_litellm_stub(
+        monkeypatch, supported_params=["logprobs", "top_logprobs", "response_format"]
+    )
+
+    captured = {}
+
+    @contextmanager
+    def fake_get_provider_response(model_provider, messages, **kwargs):
+        captured["kwargs"] = kwargs
+        yield SimpleNamespace(
+            choices=[
+                {
+                    "message": {"content": json.dumps({"score": 10, "reason": "ok"})},
+                    "logprobs": {
+                        "content": [
+                            {},
+                            {},
+                            {},
+                            {
+                                "top_logprobs": [
+                                    {"token": "10", "logprob": 0.0},
+                                ],
+                                "token": "10",
+                            },
+                        ]
+                    },
+                }
+            ]
+        )
+
+    monkeypatch.setattr(base_model, "get_provider_response", fake_get_provider_response)
+
+    metric = g_eval.GEval(
+        task_introduction="intro",
+        evaluation_criteria="criteria",
+        model="gpt-4o",
+    )
+    metric.score("{}")
+
+    assert captured["kwargs"]["logprobs"] is True
+    assert captured["kwargs"]["top_logprobs"] == 20
+
+    # Now simulate model without logprob support
+    _install_litellm_stub(monkeypatch, supported_params=["response_format"])
+    captured.clear()
+
+    @contextmanager
+    def fake_response_no_logprobs(model_provider, messages, **kwargs):
+        captured["kwargs"] = kwargs
+        yield SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(
+                        content=json.dumps({"score": 10, "reason": "ok"})
+                    )
+                )
+            ]
+        )
+
+    monkeypatch.setattr(base_model, "get_provider_response", fake_response_no_logprobs)
+
+    metric = g_eval.GEval(
+        task_introduction="intro",
+        evaluation_criteria="criteria",
+        model="gpt-5-nano",
+    )
+
+    # Even if litellm claims logprob support, gpt-5 should drop them
+    _install_litellm_stub(
+        monkeypatch, supported_params=["logprobs", "top_logprobs", "response_format"]
+    )
+    captured.clear()
+
+    monkeypatch.setattr(base_model, "get_provider_response", fake_response_no_logprobs)
+
+    metric = g_eval.GEval(
+        task_introduction="intro",
+        evaluation_criteria="criteria",
+        model="gpt-5-nano",
+    )
+    metric.score("{}")
+
+    assert "logprobs" not in captured["kwargs"]
+    assert "top_logprobs" not in captured["kwargs"]
+
+
+@pytest.mark.asyncio
+async def test_litellm_chat_model_agenerate_string_supports_dict_choices(monkeypatch):
+    _install_litellm_stub(monkeypatch)
+
+    captured_kwargs = {}
+
+    @asynccontextmanager
+    async def fake_aget_provider_response(model_provider, messages, **kwargs):
+        captured_kwargs["messages"] = messages
+        yield SimpleNamespace(
+            choices=[
+                {
+                    "message": {"content": "async-ok"},
+                    "logprobs": None,
+                }
+            ]
+        )
+
+    monkeypatch.setattr(
+        base_model, "aget_provider_response", fake_aget_provider_response
+    )
+
+    model = litellm_chat_model.LiteLLMChatModel(model_name="gpt-4o")
+
+    result = await model.agenerate_string(input="hello async")
+
+    assert result == "async-ok"
+    assert captured_kwargs["messages"][0]["content"] == "hello async"
+
+
+def test_models_factory_track_parameter_creates_separate_instances(monkeypatch):
+    """Test that track parameter creates separate cached instances."""
+    _install_litellm_stub(monkeypatch)
+
+    # Get model with track=True
+    model_tracked = models_factory.get("gpt-4o", track=True)
+    # Get model with track=False
+    model_untracked = models_factory.get("gpt-4o", track=False)
+    # Get another model with track=True (should reuse first)
+    model_tracked_2 = models_factory.get("gpt-4o", track=True)
+
+    # track=True and track=False should create separate instances
+    assert model_tracked is not model_untracked
+    # Same track value should reuse cached instance
+    assert model_tracked is model_tracked_2
+
+
+class TestParseAssistantMessage:
+    def _wrap(self, message):
+        return SimpleNamespace(choices=[{"message": message}])
+
+    def test_returns_content_when_present(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap({"content": "hello"})
+        )
+        assert message == {"role": "assistant", "content": "hello"}
+
+    def test_raises_when_no_content_and_no_tool_calls(self):
+        from opik import exceptions
+
+        with pytest.raises(exceptions.BaseLLMError):
+            response_parser.parse_assistant_message(self._wrap({"content": None}))
+
+    def test_falls_back_to_structured_output_tool_call(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap(
+                {
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "call_synthetic",
+                            "function": {
+                                "name": "json_tool_call",
+                                "arguments": '{"assertion_1": {"score": true, "reason": "ok", "confidence": 0.9}}',
+                            },
+                        }
+                    ],
+                }
+            )
+        )
+        assert '"assertion_1"' in message["content"]
+        assert "tool_calls" not in message
+
+    def test_falls_back_to_structured_output_tool_call_from_object_message(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap(
+                SimpleNamespace(
+                    content=None,
+                    tool_calls=[
+                        SimpleNamespace(
+                            id="call_synthetic",
+                            function=SimpleNamespace(
+                                name="json_tool_call",
+                                arguments='{"score": true}',
+                            ),
+                        )
+                    ],
+                )
+            )
+        )
+        assert message["content"] == '{"score": true}'
+        assert "tool_calls" not in message
+
+    def test_surfaces_real_tool_calls_alongside_text(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap(
+                {
+                    "content": "looking that up",
+                    "tool_calls": [
+                        {
+                            "id": "call_123",
+                            "function": {
+                                "name": "web_search",
+                                "arguments": '{"query": "capital of France"}',
+                            },
+                        }
+                    ],
+                }
+            )
+        )
+        assert message["content"] == "looking that up"
+        assert message["tool_calls"] == [
+            {
+                "id": "call_123",
+                "type": "function",
+                "function": {
+                    "name": "web_search",
+                    "arguments": '{"query": "capital of France"}',
+                },
+            }
+        ]
+
+    def test_skips_synthetic_tool_call_when_listed_alongside_real_ones(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap(
+                {
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "synth",
+                            "function": {
+                                "name": "json_tool_call",
+                                "arguments": '{"score": 5}',
+                            },
+                        },
+                        {
+                            "id": "call_real",
+                            "function": {
+                                "name": "get_weather",
+                                "arguments": '{"city": "Paris"}',
+                            },
+                        },
+                    ],
+                }
+            )
+        )
+        assert message["content"] == '{"score": 5}'
+        assert message["tool_calls"] == [
+            {
+                "id": "call_real",
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "arguments": '{"city": "Paris"}',
+                },
+            }
+        ]
+
+    def test_prefers_content_over_synthetic_tool_call_arguments(self):
+        message = response_parser.parse_assistant_message(
+            self._wrap(
+                {
+                    "content": "direct content",
+                    "tool_calls": [
+                        {
+                            "id": "synth",
+                            "function": {
+                                "name": "json_tool_call",
+                                "arguments": "should not use this",
+                            },
+                        }
+                    ],
+                }
+            )
+        )
+        assert message["content"] == "direct content"
+        assert "tool_calls" not in message
+
+
+@pytest.mark.parametrize(
+    "track,expected_calls",
+    [
+        (False, 0),
+        (True, 2),  # Once for completion, once for acompletion
+    ],
+)
+def test_litellm_chat_model_track_parameter_controls_monitoring(
+    monkeypatch, track, expected_calls
+):
+    """Test that track parameter controls LiteLLM monitoring when globally enabled."""
+    _install_litellm_stub(monkeypatch)
+    monkeypatch.setenv("OPIK_ENABLE_LITELLM_MODELS_MONITORING", "true")
+
+    # Track which decorator was used
+    decorator_calls = 0
+
+    def mock_track_completion(project_name=None):
+        def decorator(func):
+            nonlocal decorator_calls
+            decorator_calls += 1
+            return func
+
+        return decorator
+
+    # Patch the function on the actual module. Replacing the `sys.modules`
+    # entry isn't enough because `import opik.integrations.litellm as X`
+    # resolves via the `opik.integrations` package attribute, which still
+    # points at the real module once it has been imported earlier in the
+    # suite (e.g. by any LLM-judge metric default-model instantiation).
+    import opik.integrations.litellm as _real_litellm_integration
+
+    monkeypatch.setattr(
+        _real_litellm_integration, "track_completion", mock_track_completion
+    )
+
+    # Create model with specified track value
+    litellm_chat_model.LiteLLMChatModel(model_name="gpt-4o", track=track)
+
+    # Verify that track_completion decorator was applied the expected number of times
+    assert decorator_calls == expected_calls

sdks/python/tests/unit/evaluation/models/test_message_renderer.py

@@ -0,0 +1,255 @@
+import random
+import string
+from typing import Any, Dict, Optional
+
+import pytest
+
+from opik.api_objects.prompt.chat import chat_prompt_template
+from opik.api_objects.prompt.chat.chat_prompt_template import ChatPromptTemplate
+from opik.api_objects.prompt.chat.content_renderer_registry import (
+    ChatContentRendererRegistry,
+)
+
+
+def _render_content(
+    content: Any,
+    *,
+    variables: Optional[Dict[str, Any]] = None,
+    supported_modalities: Optional[Dict[str, bool]] = None,
+    registry: Optional[ChatContentRendererRegistry] = None,
+) -> Any:
+    template = ChatPromptTemplate(
+        messages=[{"role": "user", "content": content}],
+        registry=registry,
+    )
+    rendered = template.format(
+        variables=variables or {},
+        supported_modalities=supported_modalities,
+    )
+    assert len(rendered) == 1
+    return rendered[0]["content"]
+
+
+class TestChatPromptTemplate:
+    def test_renders_plain_text(self) -> None:
+        rendered = _render_content(
+            "Hello {{name}}",
+            variables={"name": "Opik"},
+            supported_modalities={"vision": False},
+        )
+        assert rendered == "Hello Opik"
+
+    def test_preserves_structured_content_for_vision_models(self) -> None:
+        content = [
+            {"type": "text", "text": "Describe this image"},
+            {"type": "image_url", "image_url": {"url": "{{image_url}}"}},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={"image_url": "https://example.com/cat.jpg"},
+            supported_modalities={"vision": True, "video": True},
+        )
+
+        assert isinstance(rendered, list)
+        assert rendered[0]["text"] == "Describe this image"
+        assert rendered[1]["image_url"]["url"] == "https://example.com/cat.jpg"
+
+    def test_preserves_structured_content_for_video_models(self) -> None:
+        content = [
+            {"type": "text", "text": "Watch this video"},
+            {"type": "video_url", "video_url": {"url": "{{video_url}}"}},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={"video_url": "https://example.com/clip.mp4"},
+            supported_modalities={"vision": True, "video": True},
+        )
+
+        assert isinstance(rendered, list)
+        assert rendered[0]["text"] == "Watch this video"
+        assert rendered[1]["video_url"]["url"] == "https://example.com/clip.mp4"
+
+    @pytest.mark.parametrize("detail", ["low", "high"])
+    def test_includes_detail_field_when_present(self, detail: str) -> None:
+        content = [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "https://example.com/image.png",
+                    "detail": detail,
+                },
+            }
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": True},
+        )
+
+        assert rendered[0]["image_url"]["detail"] == detail
+
+    @pytest.mark.parametrize(
+        "data_url_prefix",
+        ["data:image/png;base64,", "data:image/jpeg;base64,"],
+    )
+    def test_supports_base64_image_urls(self, data_url_prefix: str) -> None:
+        data_url = f"{data_url_prefix}iVBORw0KGgoAAAANSUhEUgAAAAUA"
+        content = [
+            {"type": "text", "text": "Inline data"},
+            {"type": "image_url", "image_url": {"url": data_url}},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": True},
+        )
+
+        assert rendered[1]["image_url"]["url"] == data_url
+
+    def test_flattens_structured_content_when_vision_disabled(self) -> None:
+        content = [
+            {"type": "text", "text": "First"},
+            {"type": "image_url", "image_url": {"url": "https://example.com/one.png"}},
+            {"type": "text", "text": "Second"},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": False, "video": False},
+        )
+
+        assert isinstance(rendered, str)
+        assert "First" in rendered
+        assert "Second" in rendered
+        assert "https://example.com/one.png" in rendered
+        assert rendered.count("<<<image>>>") == 1
+
+    def test_flattens_structured_video_when_video_disabled(self) -> None:
+        content = [
+            {"type": "text", "text": "Context"},
+            {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": True, "video": False},
+        )
+
+        assert isinstance(rendered, str)
+        assert "Context" in rendered
+        assert "<<<video>>>" in rendered
+
+    def test_flattened_placeholder_truncates_large_base64(self) -> None:
+        random_payload = "".join(
+            random.choices(string.ascii_letters + string.digits + "+/", k=700)
+        )
+        data_url = f"data:image/png;base64,{random_payload}"
+        content = [
+            {"type": "image_url", "image_url": {"url": data_url}},
+        ]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": False},
+        )
+
+        assert isinstance(rendered, str)
+        assert "<<<image>>>" in rendered
+        inner = rendered.split("<<<image>>>")[1].split("<<</image>>>")[0]
+        assert len(inner) <= 500
+        assert inner.endswith("...")
+
+    def test_skips_invalid_parts(self) -> None:
+        content = [{"type": "text", "text": "ok"}, "bad-part", None]
+
+        rendered = _render_content(
+            content,
+            variables={},
+            supported_modalities={"vision": True},
+        )
+
+        assert rendered == [{"type": "text", "text": "ok"}]
+
+    def test_custom_part_registration_allows_new_parts(self) -> None:
+        registry = ChatContentRendererRegistry()
+        registry.register_part_renderer("text", chat_prompt_template.render_text_part)
+        registry.register_part_renderer(
+            "image_url",
+            chat_prompt_template.render_image_url_part,
+            modality="vision",
+            placeholder=("<<<image>>>", "<<</image>>>"),
+        )
+
+        custom_part = {"type": "thumbnail", "image_url": {"url": "{{thumb_url}}"}}
+
+        def _render_thumbnail(
+            part: Dict[str, Any], variables: Dict[str, Any], template_type: Any
+        ) -> Dict[str, Any]:
+            rendered = chat_prompt_template.render_image_url_part(
+                part, variables, template_type
+            )
+            assert rendered is not None
+            return {"type": "thumbnail", "image_url": rendered["image_url"]}
+
+        registry.register_part_renderer(
+            "thumbnail",
+            _render_thumbnail,
+            modality="vision",
+        )
+
+        rendered = _render_content(
+            [custom_part],
+            variables={"thumb_url": "https://example.com/thumb.png"},
+            supported_modalities={"vision": True, "video": True},
+            registry=registry,
+        )
+
+        assert rendered[0]["type"] == "thumbnail"
+        assert rendered[0]["image_url"]["url"] == "https://example.com/thumb.png"
+
+        flattened = _render_content(
+            [custom_part],
+            variables={"thumb_url": "https://example.com/thumb.png"},
+            supported_modalities={"vision": False, "video": False},
+            registry=registry,
+        )
+
+        assert isinstance(flattened, str)
+        assert "thumbnail" in flattened
+
+    def test_required_modalities_detects_vision(self) -> None:
+        template = ChatPromptTemplate(
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Describe the image"},
+                        {"type": "image_url", "image_url": {"url": "{{image_url}}"}},
+                    ],
+                }
+            ]
+        )
+
+        assert template.required_modalities() == {"vision"}
+
+    def test_required_modalities_detects_video(self) -> None:
+        template = ChatPromptTemplate(
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Summarize the video"},
+                        {"type": "video_url", "video_url": {"url": "{{video_url}}"}},
+                    ],
+                }
+            ]
+        )
+
+        assert template.required_modalities() == {"video"}

sdks/python/tests/unit/evaluation/models/test_model_capabilities.py

@@ -0,0 +1,31 @@
+from opik.evaluation.models import ModelCapabilities
+
+
+def test_supports_vision_defaults_to_false_without_name() -> None:
+    assert ModelCapabilities.supports_vision(None) is False
+
+
+def test_custom_capability_registration() -> None:
+    ModelCapabilities.register_capability_detector(
+        "custom", lambda model: model.startswith("custom-")
+    )
+
+    assert ModelCapabilities.supports("custom", "custom-model") is True
+    assert ModelCapabilities.supports("custom", "text-model") is False
+
+
+def test_supports_vision_handles_provider_prefix() -> None:
+    assert ModelCapabilities.supports_vision("anthropic/claude-3-opus") is True
+
+
+def test_supports_vision_detects_common_suffixes() -> None:
+    assert ModelCapabilities.supports_vision("provider/new-model-vision") is True
+    assert ModelCapabilities.supports_vision("provider/new-model-vl") is True
+    assert ModelCapabilities.supports_vision("gpt-4.1") is True
+    assert ModelCapabilities.supports_vision("gpt-4.1-mini") is True
+
+
+def test_supports_video_detects_keywords() -> None:
+    assert ModelCapabilities.supports_video("provider/some-video-model") is True
+    assert ModelCapabilities.supports_video("qwen/qwen2.5-vl-32b-instruct") is True
+    assert ModelCapabilities.supports_video("text-only-model") is False