chore: import upstream snapshot with attribution

sdks/python/tests/unit/validation/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for validation helpers."""

sdks/python/tests/unit/validation/test_chat_prompt_messages_validator.py

@@ -0,0 +1,290 @@
+import pytest
+from opik.validation import chat_prompt_messages
+
+
+@pytest.mark.parametrize(
+    argnames="messages, is_valid",
+    argvalues=[
+        # Valid cases
+        ([{"role": "system", "content": "You are helpful"}], True),
+        ([{"role": "user", "content": "Hello"}], True),
+        ([{"role": "assistant", "content": "Hi there"}], True),
+        (
+            [
+                {"role": "system", "content": "You are helpful"},
+                {"role": "user", "content": "Hello"},
+                {"role": "assistant", "content": "Hi"},
+            ],
+            True,
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Hello"},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "https://example.com/image.jpg"},
+                        },
+                    ],
+                }
+            ],
+            True,
+        ),
+        (
+            [
+                {"role": "system", "content": "You are helpful"},
+                {"role": "user", "content": [{"type": "text", "text": "Hello"}]},
+            ],
+            True,
+        ),
+        ([], True),  # Empty list is valid
+        # Invalid cases - not a list
+        (None, False),
+        ("not a list", False),
+        (123, False),
+        ({"role": "user", "content": "hello"}, False),  # Single dict instead of list
+        # Invalid cases - message structure
+        (["not a dict"], False),
+        ([{"content": "hello"}], False),  # Missing role
+        ([{"role": "user"}], False),  # Missing content
+        (
+            [{"role": "user", "content": "hello", "extra": "key"}],
+            False,
+        ),  # Extra key
+        ([{}], False),  # Empty dict
+        # Invalid cases - role values
+        ([{"role": "invalid", "content": "hello"}], False),
+        ([{"role": "System", "content": "hello"}], False),  # Wrong case
+        ([{"role": "", "content": "hello"}], False),  # Empty string
+        ([{"role": None, "content": "hello"}], False),  # None
+        ([{"role": 123, "content": "hello"}], False),  # Wrong type
+        # Invalid cases - content types
+        ([{"role": "user", "content": None}], False),
+        ([{"role": "user", "content": 123}], False),  # Wrong type
+        ([{"role": "user", "content": ["not", "dicts"]}], False),  # List of non-dicts
+        ([{"role": "user", "content": []}], True),  # Empty list is valid
+        # Invalid cases - content parts (when content is array)
+        ([{"role": "user", "content": [{"text": "hello"}]}], False),  # Missing type
+        ([{"role": "user", "content": [{}]}], False),  # Empty dict in array
+        ([{"role": "user", "content": ["not a dict"]}], False),  # Non-dict in array
+        # Multiple validation errors
+        (
+            [
+                {"role": "invalid", "content": "hello"},
+                {"role": "user", "content": None},
+                {"role": "user", "content": [{"text": "hello"}]},
+            ],
+            False,
+        ),
+        # Valid cases - content list with type-specific keys
+        (
+            [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
+            True,
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "https://example.com/image.jpg"},
+                        }
+                    ],
+                }
+            ],
+            True,
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "video_url",
+                            "video_url": {"url": "https://example.com/video.mp4"},
+                        }
+                    ],
+                }
+            ],
+            True,
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "audio_url",
+                            "audio_url": {"url": "https://example.com/audio.mp3"},
+                        }
+                    ],
+                }
+            ],
+            True,
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Hello"},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "https://example.com/image.jpg"},
+                        },
+                        {
+                            "type": "video_url",
+                            "video_url": {"url": "https://example.com/video.mp4"},
+                        },
+                    ],
+                }
+            ],
+            True,
+        ),
+        # Invalid cases - missing required keys for specific types
+        (
+            [{"role": "user", "content": [{"type": "text"}]}],
+            False,  # Missing text key
+        ),
+        (
+            [{"role": "user", "content": [{"type": "image_url"}]}],
+            False,  # Missing image_url key
+        ),
+        (
+            [{"role": "user", "content": [{"type": "video_url"}]}],
+            False,  # Missing video_url key
+        ),
+        (
+            [{"role": "user", "content": [{"type": "audio_url"}]}],
+            False,  # Missing audio_url key
+        ),
+        # Invalid cases - wrong type for URL objects
+        (
+            [{"role": "user", "content": [{"type": "image_url", "image_url": 123}]}],
+            False,  # image_url is not a dict
+        ),
+        (
+            [{"role": "user", "content": [{"type": "image_url", "image_url": None}]}],
+            False,  # image_url is None
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [{"type": "image_url", "image_url": "string"}],
+                }
+            ],
+            False,  # image_url is a string, should be dict
+        ),
+        (
+            [{"role": "user", "content": [{"type": "image_url", "image_url": {}}]}],
+            False,  # image_url dict missing 'url' key
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [{"type": "image_url", "image_url": {"url": 123}}],
+                }
+            ],
+            False,  # image_url.url is not a string
+        ),
+        (
+            [{"role": "user", "content": [{"type": "video_url", "video_url": 123}]}],
+            False,  # video_url is not a dict
+        ),
+        (
+            [{"role": "user", "content": [{"type": "video_url", "video_url": {}}]}],
+            False,  # video_url dict missing 'url' key
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "audio_url", "audio_url": ["not", "a", "dict"]}
+                    ],
+                }
+            ],
+            False,  # audio_url is not a dict
+        ),
+        (
+            [{"role": "user", "content": [{"type": "audio_url", "audio_url": {}}]}],
+            False,  # audio_url dict missing 'url' key
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [{"type": "audio_url", "audio_url": {"url": None}}],
+                }
+            ],
+            False,  # audio_url.url is None
+        ),
+        (
+            [{"role": "user", "content": [{"type": "text", "text": 123}]}],
+            False,  # text is not a string
+        ),
+        (
+            [{"role": "user", "content": [{"type": "text", "text": None}]}],
+            False,  # text is None
+        ),
+        # Invalid cases - wrong key for type
+        (
+            [{"role": "user", "content": [{"type": "image_url", "text": "Hello"}]}],
+            False,  # Has text but missing image_url
+        ),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "image_url": {"url": "https://example.com/image.jpg"},
+                        }
+                    ],
+                }
+            ],
+            False,  # Has image_url but missing text
+        ),
+    ],
+)
+def test_chat_prompt_messages_validator(messages, is_valid):
+    validator = chat_prompt_messages.ChatPromptMessagesValidator(messages)
+    assert validator.validate().ok() is is_valid, f"Failed with {messages}"
+
+
+def test_chat_prompt_messages_validator_error_messages():
+    """Test that error messages include message indices."""
+    validator = chat_prompt_messages.ChatPromptMessagesValidator(
+        [
+            {"role": "invalid", "content": "hello"},
+            {"role": "user", "content": None},
+        ]
+    )
+    result = validator.validate()
+    assert result.failed() is True
+    assert len(result.failure_reasons) > 0
+    # Check that error messages include indices
+    assert any("messages[0]" in reason for reason in result.failure_reasons)
+    assert any("messages[1]" in reason for reason in result.failure_reasons)
+
+
+def test_chat_prompt_messages_validator_raise_validation_error():
+    """Test that raise_if_validation_failed raises ValidationError."""
+    import opik.exceptions
+
+    validator = chat_prompt_messages.ChatPromptMessagesValidator(
+        [{"role": "invalid", "content": "hello"}]
+    )
+    validator.validate()
+
+    with pytest.raises(opik.exceptions.ValidationError) as exc_info:
+        validator.raise_if_validation_failed()
+
+    assert "ChatPrompt.__init__" in str(exc_info.value)
+    assert "messages[0].role" in str(exc_info.value)

sdks/python/tests/unit/validation/test_feedback_score_validator.py

@@ -0,0 +1,118 @@
+import pytest
+from opik.validation import feedback_score
+
+
+@pytest.mark.parametrize(
+    argnames="feedback_score_dict, is_valid",
+    argvalues=[
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "reason": "good reason",
+                "category_name": "sentiment",
+            },
+            True,
+        ),
+        (
+            {
+                "id": 123213232,
+                "name": "toxicity",
+                "value": 0.5,
+                "reason": "good reason",
+                "category_name": "sentiment",
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+            },
+            True,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "unknown-key": "any-value",
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": "0.5",
+            },
+            True,
+        ),
+        ("not-even-a-dict", False),
+        # Test cases for project_name type coercion prevention (StrictStr)
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": "valid-project",
+            },
+            True,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": 123,  # Integer should be rejected for project_name
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": [
+                    "project-list"
+                ],  # List should be rejected for project_name
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": {
+                    "name": "project"
+                },  # Dict should be rejected for project_name
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": True,  # Boolean should be rejected for project_name
+            },
+            False,
+        ),
+        (
+            {
+                "id": "some-id",
+                "name": "toxicity",
+                "value": 0.5,
+                "project_name": 3.14,  # Float should be rejected for project_name
+            },
+            False,
+        ),
+    ],
+)
+def test_feedback_score_validator(feedback_score_dict, is_valid):
+    tested = feedback_score.FeedbackScoreValidator(feedback_score_dict)
+
+    assert tested.validate().ok() is is_valid, f"Failed with {feedback_score_dict}"

sdks/python/tests/unit/validation/test_parameters_validator.py

@@ -0,0 +1,56 @@
+import pytest
+
+from opik.validation import parameters_validator
+from opik import exceptions
+
+
+@pytest.mark.parametrize(
+    "value, allow_empty", [("string", False), ("string", True), (None, True)]
+)
+def test_parameters_type_validator_ok(value, allow_empty):
+    class_name = "TestClass"
+    method_name = "test_method"
+    validator = parameters_validator.create_validator(
+        method_name=method_name, class_name=class_name
+    )
+
+    validator.add_str_parameter(value, name="string_method", allow_empty=allow_empty)
+    result = validator.validate()
+    assert result.ok() is True
+
+
+@pytest.mark.parametrize("value, allow_empty", [(1, False), (1, True), (None, False)])
+def test_parameters_type_validator_fail(value, allow_empty):
+    class_name = "TestClass"
+    method_name = "test_method"
+    validator = parameters_validator.create_validator(
+        method_name=method_name, class_name=class_name
+    )
+    validator.add_str_parameter(value, name="string_method", allow_empty=allow_empty)
+    result = validator.validate()
+    assert result.ok() is False
+    assert len(result.failure_reasons) == 1
+
+
+@pytest.mark.parametrize("value, valid", [(1, True), (1.0, True), ("1", False)])
+def test_parameters_type_validator__add_numeric_parameter(value, valid):
+    class_name = "TestClass"
+    method_name = "test_method"
+    validator = parameters_validator.create_validator(
+        method_name=method_name, class_name=class_name
+    )
+    validator.add_numeric_parameter(value, name="numeric_method")
+    result = validator.validate()
+    assert result.ok() == valid
+
+
+def test_validate_throw_error():
+    with pytest.raises(exceptions.ValidationError):
+        class_name = "TestClass"
+        method_name = "test_method"
+        validator = parameters_validator.create_validator(
+            method_name=method_name, class_name=class_name
+        )
+        validator.add_str_parameter(1, name="string_method")
+        validator.validate()
+        validator.raise_validation_error()

sdks/python/tests/unit/validation/test_validator_helpers.py

@@ -0,0 +1,376 @@
+from typing import Any, List, Optional
+
+from opik.validation import validator_helpers
+from opik.validation import parameter
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        ("String", False, True),
+        ("", False, False),
+        (1, False, False),
+        ((1, 2, 3), False, False),
+        ([2], False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_str(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_str(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        (1, False, True),
+        ("String", False, False),
+        ((1, 2, 3), False, False),
+        ([2], False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_int(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_int(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        (1.0, False, True),
+        (2, False, False),
+        ("String", False, False),
+        ((1, 2, 3), False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_float(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_float(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        (1.0, False, True),
+        (2, False, True),
+        ("String", False, False),
+        ((1, 2, 3), False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_numeric(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_numeric(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        (True, False, True),
+        (False, False, True),
+        ("String", False, False),
+        ((1, 2, 3), False, False),
+        ([2], False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_bool(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_bool(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        ([1.0], False, True),
+        ([2], False, True),
+        (["String"], False, True),
+        ("not_a_list", False, False),
+        (10, False, False),
+        ((1, 2, 3), False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_type_list(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_list(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        ({"key": 1.0}, False, True),
+        ([2], False, False),
+        ("not_a_dict", False, False),
+        (10, False, False),
+        ((1, 2, 3), False, False),
+    ],
+)
+def test_validate_type_dict(value: Any, allow_empty: bool, expected: bool):
+    assert (
+        validator_helpers.validate_type_dict(value=value, allow_empty=allow_empty)
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "value,possible_values,allow_empty,expected",
+    [
+        (None, None, False, False),
+        (None, None, True, True),
+        ("String", None, False, True),
+        (1, None, False, False),
+        ((1, 2, 3), None, False, False),
+        ([2], None, False, False),
+        ({"key": 1.0}, None, False, False),
+        ("1", ["1", "2"], False, True),
+        ("3", ["1", "2"], False, False),
+    ],
+)
+def test_validate_parameter_type_str(
+    value: Any, possible_values: Optional[List], allow_empty: bool, expected: bool
+):
+    param = parameter.create_str_parameter(
+        name="parameter_type_str",
+        value=value,
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,possible_values,allow_empty,expected",
+    [
+        (None, None, False, False),
+        (None, None, True, True),
+        (1, None, False, True),
+        ("String", None, False, False),
+        ((1, 2, 3), None, False, False),
+        ([2], None, False, False),
+        (1, [1, 2, 3], False, True),
+        (4, [1, 2, 3], False, False),
+    ],
+)
+def test_validate_parameter_type_int(
+    value: Any, possible_values: Optional[List], allow_empty: bool, expected: bool
+):
+    param = parameter.create_int_parameter(
+        name="parameter_type_int",
+        value=value,
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,possible_values,allow_empty,expected",
+    [
+        (None, None, False, False),
+        (None, None, True, True),
+        (1.0, None, False, True),
+        (2, None, False, False),
+        ("String", None, False, False),
+        ((1, 2, 3), None, False, False),
+        ({"key": 1.0}, None, False, False),
+        (1.1, [1.1, 2.1, 3.1], False, True),
+        (4.1, [1.1, 2.1, 3.1], False, False),
+    ],
+)
+def test_validate_parameter_type_float(
+    value: Any, possible_values: Optional[List], allow_empty: bool, expected: bool
+):
+    param = parameter.create_float_parameter(
+        name="parameter_type_float",
+        value=value,
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,possible_values,allow_empty,expected",
+    [
+        (None, None, False, False),
+        (None, None, True, True),
+        (1.0, None, False, True),
+        (2, None, False, True),
+        ("String", None, False, False),
+        ((1, 2, 3), None, False, False),
+        ({"key": 1.0}, None, False, False),
+        (1, [1, 2.2, 3], False, True),
+        (2.2, [1.1, 2.2, 3], False, True),
+        (4, [1, 2.2, 3], False, False),
+        (4.2, [1.1, 2.2, 3], False, False),
+    ],
+)
+def test_validate_parameter_type_numeric(
+    value: Any, possible_values: Optional[List], allow_empty: bool, expected: bool
+):
+    param = parameter.create_numeric_parameter(
+        name="parameter_type_numeric",
+        value=value,
+        possible_values=possible_values,
+        allow_empty=allow_empty,
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        (True, False, True),
+        (False, False, True),
+        ("String", False, False),
+        ((1, 2, 3), False, False),
+        ([2], False, False),
+        ({"key": 1.0}, False, False),
+    ],
+)
+def test_validate_parameter_type_bool(value: Any, allow_empty: bool, expected: bool):
+    param = parameter.create_bool_parameter(
+        name="parameter_type_bool", value=value, allow_empty=allow_empty
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        ([1.0], False, True),
+        ([2], False, True),
+        (["String"], False, True),
+        ("not_a_list", False, False),
+        (10, False, False),
+        ((1, 2, 3), False, False),
+        ({"key": 1.0}, False, False),
+        ([], False, False),
+    ],
+)
+def test_validate_parameter_value_list(value: Any, allow_empty: bool, expected: bool):
+    param = parameter.create_list_parameter(
+        name="parameter_value_list", value=value, allow_empty=allow_empty
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+@pytest.mark.parametrize(
+    "value,allow_empty,expected",
+    [
+        (None, False, False),
+        (None, True, True),
+        ({"key": 1.0}, False, True),
+        ([2], False, False),
+        ("not_a_list", False, False),
+        (10, False, False),
+        ((1, 2, 3), False, False),
+        ([], False, False),
+        ({}, False, False),
+    ],
+)
+def test_validate_parameter_value_dict(value: Any, allow_empty: bool, expected: bool):
+    param = parameter.create_dict_parameter(
+        name="parameter_value_dict", value=value, allow_empty=allow_empty
+    )
+    _check_validate_parameter(test_parameter=param, expected=expected)
+
+
+def _check_validate_parameter(test_parameter: parameter.Parameter, expected: bool):
+    valid, msg = validator_helpers.validate_parameter(parameter=test_parameter)
+    assert valid == expected
+
+    if expected:
+        return
+
+    if test_parameter.value is None:
+        value_not_empty = False
+    elif not isinstance(test_parameter.value, (bool, int, float)):
+        value_not_empty = (
+            hasattr(test_parameter.value, "__len__") and len(test_parameter.value) > 0
+        )
+    else:
+        value_not_empty = True
+
+    if test_parameter.possible_values is not None:
+        possible_values_str = [str(v) for v in test_parameter.possible_values]
+
+        if test_parameter.allow_empty:
+            expected_msg = (
+                "parameter %r must be one of [%s] or None but %r was given"
+                % (
+                    test_parameter.name,
+                    ", ".join(possible_values_str),
+                    test_parameter.value,
+                )
+            )
+        else:
+            expected_msg = "parameter %r must be one of [%s] but %r was given" % (
+                test_parameter.name,
+                ", ".join(possible_values_str),
+                test_parameter.value,
+            )
+    elif value_not_empty:
+        param_type = (
+            None
+            if test_parameter.value is None
+            else type(test_parameter.value).__name__
+        )
+
+        if test_parameter.allow_empty:
+            expected_msg = (
+                "parameter %r must be of type(s) %r or None but %r was given"
+                % (
+                    test_parameter.name,
+                    validator_helpers.types_list(test_parameter.types),
+                    param_type,
+                )
+            )
+        else:
+            expected_msg = "parameter %r must be of type(s) %r but %r was given" % (
+                test_parameter.name,
+                validator_helpers.types_list(test_parameter.types),
+                param_type,
+            )
+    else:
+        expected_msg = "parameter %r must have non empty value but %r was given" % (
+            test_parameter.name,
+            test_parameter.value,
+        )
+
+    assert expected_msg == msg