chore: import upstream snapshot with attribution

sdks/python/tests/unit/api_objects/dataset/test_converters.py

@@ -0,0 +1,432 @@
+import pandas as pd
+import pandas.testing
+import json
+import tempfile
+import os
+
+from opik.api_objects.dataset import converters
+from opik.api_objects.dataset.dataset_item import DatasetItem
+from ....testlib import ANY_BUT_NONE
+
+
+def test_from_pandas__all_columns_from_dataframe_represent_all_dataset_item_fields():
+    data_for_dataframe = {
+        "id": ["id-1", "id-2"],
+        "input": [{"input-key-1": "input-1"}, {"input-key-2": "input-2"}],
+        "expected_output": [
+            {"expected-output-key-1": "expected-output-1"},
+            {"expected-output-key-2": "expected-output-2"},
+        ],
+        "metadata": [{"metadata-key-1": "v1"}, {"metadata-key-2": "v2"}],
+        "span_id": ["span-id-1", "span-id-2"],
+        "trace_id": ["trace-id-1", "trace-id-2"],
+        "source": ["some-source-1", "some-source-2"],
+    }
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id="id-1",
+            input={"input-key-1": "input-1"},
+            expected_output={"expected-output-key-1": "expected-output-1"},
+            metadata={"metadata-key-1": "v1"},
+            span_id="span-id-1",
+            trace_id="trace-id-1",
+            source="some-source-1",
+        ),
+        DatasetItem(
+            id="id-2",
+            input={"input-key-2": "input-2"},
+            expected_output={"expected-output-key-2": "expected-output-2"},
+            metadata={"metadata-key-2": "v2"},
+            span_id="span-id-2",
+            trace_id="trace-id-2",
+            source="some-source-2",
+        ),
+    ]
+
+    dataframe = pd.DataFrame(data_for_dataframe)
+
+    actual_items = converters.from_pandas(
+        dataframe=dataframe, keys_mapping={}, ignore_keys=[]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_pandas__only_input_presented_in_dataframe__items_are_constructed_with_default_values_for_missing_fields():
+    data_for_dataframe = {
+        "input": [{"input-key-1": "input-1"}, {"input-key-2": "input-2"}],
+    }
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    dataframe = pd.DataFrame(data_for_dataframe)
+
+    actual_items = converters.from_pandas(
+        dataframe=dataframe, keys_mapping={}, ignore_keys=[]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_pandas__dataframe_column_does_not_have_the_same_name_as_dataset_item_field__keys_mapping_is_used():
+    data_for_dataframe = {
+        "Input column name": [{"input-key-1": "input-1"}, {"input-key-2": "input-2"}],
+    }
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    dataframe = pd.DataFrame(data_for_dataframe)
+
+    actual_items = converters.from_pandas(
+        dataframe=dataframe, keys_mapping={"Input column name": "input"}, ignore_keys=[]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_pandas__dataframe_contains_extra_column_not_needed_for_dataset_item__ignore_keys_is_used():
+    data_for_dataframe = {
+        "input": [{"input-key-1": "input-1"}, {"input-key-2": "input-2"}],
+        "some-extra-column": [1, 2],
+    }
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    dataframe = pd.DataFrame(data_for_dataframe)
+
+    actual_items = converters.from_pandas(
+        dataframe=dataframe, keys_mapping={}, ignore_keys=["some-extra-column"]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_to_pandas__with_keys_mapping__span_id_trace_id_and_source_ignored__happyflow():
+    EXPECTED_DATAFRAME = pd.DataFrame(
+        {
+            "id": ["id-1", "id-2"],
+            "input": [{"input-key-1": "input-1"}, {"input-key-2": "input-2"}],
+            "Customized expected output": [
+                {"expected-output-key-1": "expected-output-1"},
+                {"expected-output-key-2": "expected-output-2"},
+            ],
+            "metadata": [{"metadata-key-1": "v1"}, {"metadata-key-2": "v2"}],
+        }
+    )
+
+    input_items = [
+        DatasetItem(
+            id="id-1",
+            input={"input-key-1": "input-1"},
+            expected_output={"expected-output-key-1": "expected-output-1"},
+            metadata={"metadata-key-1": "v1"},
+            span_id="span-id-1",
+            trace_id="trace-id-1",
+            source="some-source-1",
+        ),
+        DatasetItem(
+            id="id-2",
+            input={"input-key-2": "input-2"},
+            expected_output={"expected-output-key-2": "expected-output-2"},
+            metadata={"metadata-key-2": "v2"},
+            span_id="span-id-2",
+            trace_id="trace-id-2",
+            source="some-source-2",
+        ),
+    ]
+
+    actual_dataframe = converters.to_pandas(
+        input_items, keys_mapping={"expected_output": "Customized expected output"}
+    )
+
+    # check_like ignores columns and rows order
+    pandas.testing.assert_frame_equal(
+        actual_dataframe, EXPECTED_DATAFRAME, check_like=True
+    )
+
+
+def test_from_json__all_columns_from_dataframe_represent_all_dataset_item_fields():
+    input_json = """
+    [
+        {
+            "id": "id-1",
+            "input": {"input-key-1": "input-1"},
+            "expected_output": {"expected-output-key-1": "expected-output-1"},
+            "metadata": {"metadata-key-1": "v1"}
+        },
+        {
+            "id": "id-2",
+            "input": {"input-key-2": "input-2"},
+            "expected_output": {"expected-output-key-2": "expected-output-2"},
+            "metadata": {"metadata-key-2": "v2"}
+        }
+    ]
+"""
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id="id-1",
+            input={"input-key-1": "input-1"},
+            expected_output={"expected-output-key-1": "expected-output-1"},
+            metadata={"metadata-key-1": "v1"},
+        ),
+        DatasetItem(
+            id="id-2",
+            input={"input-key-2": "input-2"},
+            expected_output={"expected-output-key-2": "expected-output-2"},
+            metadata={"metadata-key-2": "v2"},
+        ),
+    ]
+
+    actual_items = converters.from_json(input_json, keys_mapping={}, ignore_keys=[])
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_json__only_input_presented_in_json__items_are_constructed_with_default_values_for_missing_fields():
+    input_json = """
+    [
+        {
+            "input": {"input-key-1": "input-1"}
+        },
+        {
+            "input": {"input-key-2": "input-2"}
+        }
+    ]
+    """
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    actual_items = converters.from_json(input_json, keys_mapping={}, ignore_keys=[])
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_json__json_objects_contain_extra_key_not_needed_for_dataset_item__ignore_keys_is_used():
+    input_json = """
+    [
+        {
+            "input": {"input-key-1": "input-1"},
+            "extra_key": 42
+        },
+        {
+            "input": {"input-key-2": "input-2"},
+            "extra_key": 4242
+        }
+    ]
+    """
+
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    actual_items = converters.from_json(
+        input_json, keys_mapping={}, ignore_keys=["extra_key"]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_from_json__json_objects_dont_have_the_same_name_as_dataset_item_field__keys_mapping_is_used():
+    input_json = """
+    [
+        {
+            "JSON input key": {"input-key-1": "input-1"}
+        },
+        {
+            "JSON input key": {"input-key-2": "input-2"}
+        }
+    ]
+    """
+    EXPECTED_ITEMS = [
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-1": "input-1"},
+            source="sdk",
+        ),
+        DatasetItem(
+            id=ANY_BUT_NONE,
+            input={"input-key-2": "input-2"},
+            source="sdk",
+        ),
+    ]
+
+    actual_items = converters.from_json(
+        input_json, keys_mapping={"JSON input key": "input"}, ignore_keys=[]
+    )
+
+    assert actual_items == EXPECTED_ITEMS
+
+
+def test_to_json__with_keys_mapping__span_id_trace_id_and_source_ignored__happyflow():
+    EXPECTED_JSON = """
+    [
+        {
+            "id": "id-1",
+            "input": {"input-key-1": "input-1"},
+            "Customized expected output": {"expected-output-key-1": "expected-output-1"},
+            "metadata": {"metadata-key-1": "v1"}
+        },
+        {
+            "id": "id-2",
+            "input": {"input-key-2": "input-2"},
+            "Customized expected output": {"expected-output-key-2": "expected-output-2"},
+            "metadata": {"metadata-key-2": "v2"}
+        }
+    ]
+    """
+    input_items = [
+        DatasetItem(
+            id="id-1",
+            input={"input-key-1": "input-1"},
+            expected_output={"expected-output-key-1": "expected-output-1"},
+            metadata={"metadata-key-1": "v1"},
+            span_id="span-id-1",
+            trace_id="trace-id-1",
+            source="some-source-1",
+        ),
+        DatasetItem(
+            id="id-2",
+            input={"input-key-2": "input-2"},
+            expected_output={"expected-output-key-2": "expected-output-2"},
+            metadata={"metadata-key-2": "v2"},
+            span_id="span-id-2",
+            trace_id="trace-id-2",
+            source="some-source-2",
+        ),
+    ]
+
+    actual_json = converters.to_json(
+        input_items, keys_mapping={"expected_output": "Customized expected output"}
+    )
+
+    assert json.loads(actual_json) == json.loads(EXPECTED_JSON)
+
+
+def test_from_jsonl_file__happyflow():
+    jsonl_content = """
+{"input": {"user_question": "What is the capital of France?"}, "expected_output": {"assistant_answer": "The capital of France is Paris."}}
+{"input": {"user_question": "How many planets are in our solar system?"}, "expected_output": {"assistant_answer": "There are 8 planets in our solar system: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune."}}
+"""
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file:
+        temp_file.write(jsonl_content)
+        temp_file_path = temp_file.name
+
+    try:
+        result = converters.from_jsonl_file(
+            temp_file_path, keys_mapping={}, ignore_keys=[]
+        )
+
+        assert result[0].input == {"user_question": "What is the capital of France?"}
+        assert result[0].expected_output == {
+            "assistant_answer": "The capital of France is Paris."
+        }
+
+        assert result[1].input == {
+            "user_question": "How many planets are in our solar system?"
+        }
+        assert result[1].expected_output == {
+            "assistant_answer": "There are 8 planets in our solar system: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune."
+        }
+    finally:
+        os.unlink(temp_file_path)
+
+
+def test_from_jsonl_file__empty_file():
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file:
+        temp_file_path = temp_file.name
+
+    try:
+        result = converters.from_jsonl_file(
+            temp_file_path, keys_mapping={}, ignore_keys=[]
+        )
+        assert isinstance(result, list)
+        assert len(result) == 0
+    finally:
+        os.unlink(temp_file_path)
+
+
+def test_from_jsonl_file__file_with_empty_lines():
+    jsonl_content = """
+{"input": {"user_question": "What is the capital of France?"}, "expected_output": {"assistant_answer": "The capital of France is Paris."}}
+
+{"input": {"user_question": "How many planets are in our solar system?"}, "expected_output": {"assistant_answer": "There are 8 planets in our solar system: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune."}}
+
+"""
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file:
+        temp_file.write(jsonl_content)
+        temp_file_path = temp_file.name
+
+    try:
+        result = converters.from_jsonl_file(
+            temp_file_path, keys_mapping={}, ignore_keys=[]
+        )
+
+        assert len(result) == 2
+
+        assert result[0].input == {"user_question": "What is the capital of France?"}
+        assert result[0].expected_output == {
+            "assistant_answer": "The capital of France is Paris."
+        }
+
+        assert result[1].input == {
+            "user_question": "How many planets are in our solar system?"
+        }
+        assert result[1].expected_output == {
+            "assistant_answer": "There are 8 planets in our solar system: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune."
+        }
+    finally:
+        os.unlink(temp_file_path)

sdks/python/tests/unit/api_objects/dataset/test_dataset_client.py

@@ -0,0 +1,165 @@
+from unittest.mock import Mock
+from opik.api_objects.dataset.dataset import Dataset
+
+
+def test_insert_deduplication__two_dicts_passed_with_the_same_content__only_one_is_inserted():
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+    )
+
+    item_dict = {
+        "input": {"key": "value", "key2": "value2"},
+        "expected_output": {"key": "value", "key2": "value2"},
+        "metadata": {"key": "value", "key2": "value2"},
+    }
+
+    # Insert the identical items
+    dataset.insert([item_dict, item_dict])
+
+    assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 1, (
+        "create_or_update_dataset_items should be called only once"
+    )
+
+    call_args = mock_rest_client.datasets.create_or_update_dataset_items.call_args
+    inserted_items = call_args[1]["items"]
+
+    assert len(inserted_items) == 1, "Only one item should be inserted"
+
+
+def test_insert_deduplication__two_dicts_passed_with_the_different_content__both_are_inserted():
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+    )
+
+    item_dict1 = {
+        "input": {"key": "value1"},
+        "expected_output": {"key": "output1"},
+        "metadata": {"key": "meta1"},
+    }
+    item_dict2 = {
+        "input": {"key": "value2"},
+        "expected_output": {"key": "output2"},
+        "metadata": {"key": "meta2"},
+    }
+
+    # Insert the different items
+    dataset.insert([item_dict1, item_dict2])
+
+    assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 1, (
+        "create_or_update_dataset_items should be called only once"
+    )
+
+    call_args = mock_rest_client.datasets.create_or_update_dataset_items.call_args
+    inserted_items = call_args[1]["items"]
+
+    assert len(inserted_items) == 2, "Two items should be inserted"
+
+
+def test_insert_deduplication__three_dicts_passed__one_unique__two_duplicates__two_different_items_are_inserted():
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+    )
+
+    item_dict1 = {
+        "input": {"key": "value1"},
+        "expected_output": {"key": "output1"},
+        "metadata": {"key": "meta1"},
+    }
+    item_dict2 = {
+        "input": {"key": "value2"},
+        "expected_output": {"key": "output2"},
+        "metadata": {"key": "meta2"},
+    }
+
+    # Insert 3 items: one unique and two duplicates
+    dataset.insert([item_dict1, item_dict2, item_dict1])
+
+    assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 1, (
+        "create_or_update_dataset_items should be called only once"
+    )
+
+    call_args = mock_rest_client.datasets.create_or_update_dataset_items.call_args
+    inserted_rest_items = call_args[1]["items"]
+
+    assert len(inserted_rest_items) == 2, "Two items should be inserted"
+
+
+def test_update__happyflow():
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+    )
+
+    initial_item = {
+        "input": {"key": "initial_value"},
+        "expected_output": {"key": "initial_output"},
+        "metadata": {"key": "initial_metadata"},
+    }
+
+    dataset.insert([initial_item])
+
+    assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 1, (
+        "create_or_update_dataset_items should be called once for insertion"
+    )
+
+    insert_call_args = (
+        mock_rest_client.datasets.create_or_update_dataset_items.call_args
+    )
+    inserted_items = insert_call_args[1]["items"]
+
+    assert len(inserted_items) == 1, "One item should be inserted"
+
+    # Create an updated version of the item
+    updated_item = {
+        "id": inserted_items[0].id,
+        "input": {"key": "updated_value"},
+        "expected_output": {"key": "updated_output"},
+        "metadata": {"key": "updated_metadata"},
+    }
+
+    # Update the item
+    dataset.update([updated_item])
+
+    # Check that create_or_update_dataset_items was called twice in total (once for insertion, once for update)
+    assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 2, (
+        "create_or_update_dataset_items should be called twice in total"
+    )
+
+    # Get the arguments passed to create_or_update_dataset_items for update
+    update_call_args = (
+        mock_rest_client.datasets.create_or_update_dataset_items.call_args
+    )
+    updated_rest_items = update_call_args[1]["items"]
+
+    # Check that one item was updated
+    assert len(updated_rest_items) == 1, "One item should be updated"
+
+    # Verify the content of the updated item
+    assert updated_rest_items[0].data["input"] == {"key": "updated_value"}, (
+        "Input should be updated"
+    )
+    assert updated_rest_items[0].data["expected_output"] == {"key": "updated_output"}, (
+        "Expected output should be updated"
+    )
+    assert updated_rest_items[0].data["metadata"] == {"key": "updated_metadata"}, (
+        "Metadata should be updated"
+    )

sdks/python/tests/unit/api_objects/dataset/test_dataset_items_count.py

@@ -0,0 +1,178 @@
+from unittest.mock import Mock
+from opik.api_objects.dataset.dataset import Dataset
+from opik.rest_api.types.dataset_public import DatasetPublic
+
+
+def test_dataset_items_count__cached_value__returns_cached_count():
+    """Test that dataset_items_count returns cached value when available."""
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=5,
+    )
+
+    count = dataset.dataset_items_count
+
+    assert count == 5
+    mock_rest_client.datasets.get_dataset_by_id.assert_not_called()
+
+
+def test_dataset_items_count__no_cached_value__fetches_from_backend():
+    """Test that dataset_items_count fetches from backend when cache is None."""
+    mock_rest_client = Mock()
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=10)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=None,
+    )
+
+    count = dataset.dataset_items_count
+
+    assert count == 10
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once_with(id=dataset.id)
+
+
+def test_dataset_items_count__fetched_once__cached_for_subsequent_calls():
+    """Test that dataset_items_count is fetched once and then cached."""
+    mock_rest_client = Mock()
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=10)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=None,
+    )
+
+    count1 = dataset.dataset_items_count
+    count2 = dataset.dataset_items_count
+    count3 = dataset.dataset_items_count
+
+    assert count1 == 10
+    assert count2 == 10
+    assert count3 == 10
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once()
+
+
+def test_delete__invalidates_cached_count():
+    """Test that delete() invalidates the cached count."""
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=5,
+    )
+
+    assert dataset.dataset_items_count == 5
+
+    dataset.delete(["item1", "item2"])
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=3)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    count = dataset.dataset_items_count
+
+    assert count == 3
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once()
+
+
+def test_update__invalidates_cached_count():
+    """Test that update() invalidates the cached count."""
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=5,
+    )
+
+    assert dataset.dataset_items_count == 5
+
+    updated_item = {
+        "id": "item1",
+        "input": {"key": "updated_value"},
+        "expected_output": {"key": "updated_output"},
+    }
+    dataset.update([updated_item])
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=5)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    count = dataset.dataset_items_count
+
+    assert count == 5
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once()
+
+
+def test_insert__invalidates_cached_count():
+    """Test that insert() invalidates the cached count."""
+    mock_rest_client = Mock()
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=5,
+    )
+
+    assert dataset.dataset_items_count == 5
+
+    new_items = [
+        {
+            "input": {"key": "value1"},
+            "expected_output": {"key": "output1"},
+        },
+        {
+            "input": {"key": "value2"},
+            "expected_output": {"key": "output2"},
+        },
+    ]
+    dataset.insert(new_items)
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=7)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    count = dataset.dataset_items_count
+
+    assert count == 7
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once()
+
+
+def test_backend_returns_none_count__property_returns_none():
+    """Test that if backend returns None for count, property returns None."""
+    mock_rest_client = Mock()
+
+    mock_dataset_public = DatasetPublic(name="test_dataset", dataset_items_count=None)
+    mock_rest_client.datasets.get_dataset_by_id.return_value = mock_dataset_public
+
+    dataset = Dataset(
+        name="test_dataset",
+        description="Test description",
+        project_name="Test project",
+        rest_client=mock_rest_client,
+        dataset_items_count=None,
+    )
+
+    count = dataset.dataset_items_count
+
+    assert count is None
+    mock_rest_client.datasets.get_dataset_by_id.assert_called_once()

sdks/python/tests/unit/api_objects/dataset/test_rate_limit_retry.py

@@ -0,0 +1,122 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from opik.api_objects.dataset import dataset
+from opik.rest_api.core.api_error import ApiError
+
+
+class TestDatasetRateLimitRetry(unittest.TestCase):
+    """Test rate limit retry behavior for dataset operations using public API."""
+
+    def _create_dataset_with_mock_client(self) -> tuple[dataset.Dataset, Mock]:
+        """Create a Dataset instance with a mocked REST client."""
+        mock_rest_client = Mock()
+        dataset_obj = dataset.Dataset(
+            name="test_dataset",
+            description="test",
+            project_name="Test project",
+            rest_client=mock_rest_client,
+        )
+        return dataset_obj, mock_rest_client
+
+    @patch("opik.api_objects.rest_helpers._sleep")
+    def test_insert__429_with_retry_after_header__retries_with_correct_delay(
+        self, mock_sleep: Mock
+    ) -> None:
+        """Test that 429 errors with RateLimit-Reset header are retried with correct delay."""
+        dataset_obj, mock_rest_client = self._create_dataset_with_mock_client()
+
+        # First call raises 429 with rate limit headers, second call succeeds
+        rate_limit_error = ApiError(
+            status_code=429,
+            headers={
+                "RateLimit-Reset": "5",  # 5 seconds retry after
+            },
+            body="Rate limit exceeded",
+        )
+        mock_rest_client.datasets.create_or_update_dataset_items.side_effect = [
+            rate_limit_error,
+            None,  # Success on second attempt
+        ]
+
+        # Execute using public API
+        dataset_obj.insert([{"input": "test"}])
+
+        # Verify retry behavior
+        assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 2
+        mock_sleep.assert_called_once_with(5.0)
+
+    @patch("opik.api_objects.rest_helpers._sleep")
+    def test_insert__429_without_header__uses_fallback_delay(
+        self, mock_sleep: Mock
+    ) -> None:
+        """Test that 429 errors without headers use fallback 1 second delay."""
+        dataset_obj, mock_rest_client = self._create_dataset_with_mock_client()
+
+        # First two calls raise 429 without headers, third succeeds
+        rate_limit_error = ApiError(
+            status_code=429,
+            headers={},
+            body="Rate limit exceeded",
+        )
+        mock_rest_client.datasets.create_or_update_dataset_items.side_effect = [
+            rate_limit_error,
+            rate_limit_error,
+            None,  # Success on third attempt
+        ]
+
+        # Execute using public API
+        dataset_obj.insert([{"input": "test"}])
+
+        # Verify fallback delay: always 1 second when no header
+        assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 3
+        assert mock_sleep.call_count == 2
+        # Both retries should use 1 second delay
+        assert all(call[0][0] == 1 for call in mock_sleep.call_args_list)
+
+    def test_insert__non_429_error__raises_immediately(self) -> None:
+        """Test that non-429 errors are raised immediately without retry."""
+        dataset_obj, mock_rest_client = self._create_dataset_with_mock_client()
+
+        # Simulate a 500 error
+        error = ApiError(
+            status_code=500,
+            headers={},
+            body="Internal server error",
+        )
+        mock_rest_client.datasets.create_or_update_dataset_items.side_effect = error
+
+        # Execute & Verify
+        with self.assertRaises(ApiError) as context:
+            dataset_obj.insert([{"input": "test"}])
+
+        assert context.exception.status_code == 500
+        # Should only try once for non-429 errors
+        assert mock_rest_client.datasets.create_or_update_dataset_items.call_count == 1
+
+    @patch("opik.api_objects.rest_helpers._sleep")
+    def test_delete__429_with_retry_after_header__retries_with_correct_delay(
+        self, mock_sleep: Mock
+    ) -> None:
+        """Test that delete operation also handles 429 errors correctly."""
+        dataset_obj, mock_rest_client = self._create_dataset_with_mock_client()
+
+        # First call raises 429 with rate limit headers, second call succeeds
+        rate_limit_error = ApiError(
+            status_code=429,
+            headers={
+                "RateLimit-Reset": "3",
+            },
+            body="Rate limit exceeded",
+        )
+        mock_rest_client.datasets.delete_dataset_items.side_effect = [
+            rate_limit_error,
+            None,  # Success on second attempt
+        ]
+
+        # Execute using public API
+        dataset_obj.delete(["item-id-1"])
+
+        # Verify retry behavior
+        assert mock_rest_client.datasets.delete_dataset_items.call_count == 2
+        mock_sleep.assert_called_once_with(3.0)

sdks/python/tests/unit/api_objects/dataset/test_stream_dataset_items.py

@@ -0,0 +1,74 @@
+"""Unit tests for stream_dataset_items() in rest_operations."""
+
+from unittest.mock import Mock, patch
+
+from opik.api_objects.dataset import rest_operations
+from opik.rest_api.types import dataset_item as rest_dataset_item
+
+_SHADOW_WARNING = (
+    "Dataset item data contains keys that shadow DatasetItem fields and will be ignored: %s. "
+    "Rename these keys in your dataset to preserve them."
+)
+
+
+def _make_rest_item(item_id: str, data: dict) -> rest_dataset_item.DatasetItem:
+    return rest_dataset_item.DatasetItem(
+        id=item_id,
+        source="sdk",
+        data=data,
+    )
+
+
+def test_stream_dataset_items__colliding_id_key__uses_real_id_and_warns():
+    real_id = "real-uuid-1234"
+    rest_item = _make_rest_item(real_id, {"id": "COLLISION", "question": "What?"})
+
+    mock_rest_client = Mock()
+
+    with (
+        patch(
+            "opik.api_objects.dataset.rest_operations.rest_stream_parser.read_and_parse_stream",
+            side_effect=[[rest_item], []],
+        ),
+        patch.object(rest_operations.LOGGER, "warning") as mock_warn,
+    ):
+        items = list(
+            rest_operations.stream_dataset_items(
+                rest_client=mock_rest_client,
+                dataset_name="test-dataset",
+                project_name=None,
+            )
+        )
+
+    assert len(items) == 1
+    assert items[0].id == real_id
+
+    mock_warn.assert_called_once_with(_SHADOW_WARNING, ["id"])
+
+
+def test_stream_dataset_items__colliding_id_key__warning_emitted_only_once():
+    """Warning is logged once per stream even when multiple items have the collision."""
+    items_data = [
+        _make_rest_item(f"uuid-{i}", {"id": f"hotpot-{i}", "question": "Q?"})
+        for i in range(3)
+    ]
+
+    mock_rest_client = Mock()
+
+    with (
+        patch(
+            "opik.api_objects.dataset.rest_operations.rest_stream_parser.read_and_parse_stream",
+            side_effect=[items_data, []],
+        ),
+        patch.object(rest_operations.LOGGER, "warning") as mock_warn,
+    ):
+        result = list(
+            rest_operations.stream_dataset_items(
+                rest_client=mock_rest_client,
+                dataset_name="test-dataset",
+                project_name=None,
+            )
+        )
+
+    assert len(result) == 3
+    mock_warn.assert_called_once_with(_SHADOW_WARNING, ["id"])

sdks/python/tests/unit/api_objects/dataset/test_suite/test_converters.py

@@ -0,0 +1,513 @@
+"""Unit tests for test_suite.converters module."""
+
+import json
+import os
+import tempfile
+
+import pandas as pd
+import pandas.testing
+import pytest
+from unittest import mock
+
+from opik.api_objects.dataset import dataset_item
+from opik.api_objects.dataset.test_suite import converters
+
+
+# ---------------------------------------------------------------------------
+# evaluators_to_assertions
+# ---------------------------------------------------------------------------
+
+
+def test_evaluators_to_assertions__multiple_evaluators__concatenates():
+    e1 = mock.MagicMock()
+    e1.assertions = ["A1"]
+    e2 = mock.MagicMock()
+    e2.assertions = ["A2", "A3"]
+
+    assert converters.evaluators_to_assertions([e1, e2]) == ["A1", "A2", "A3"]
+
+
+def test_evaluators_to_assertions__empty_list__returns_empty():
+    assert converters.evaluators_to_assertions([]) == []
+
+
+# ---------------------------------------------------------------------------
+# version_evaluators_to_assertions
+# ---------------------------------------------------------------------------
+
+
+def test_version_evaluators_to_assertions__llm_judge__extracts_assertions():
+    from opik.evaluation.suite_evaluators import LLMJudge
+
+    judge = LLMJudge(assertions=["Response is accurate"], track=False)
+    config = judge.to_config().model_dump(by_alias=True)
+
+    evaluator_item = mock.MagicMock()
+    evaluator_item.type = "llm_judge"
+    evaluator_item.config = config
+
+    assert converters.version_evaluators_to_assertions([evaluator_item]) == [
+        "Response is accurate"
+    ]
+
+
+def test_version_evaluators_to_assertions__none__returns_empty():
+    assert converters.version_evaluators_to_assertions(None) == []
+
+
+def test_version_evaluators_to_assertions__non_llm_judge__skipped():
+    evaluator_item = mock.MagicMock()
+    evaluator_item.type = "custom_scorer"
+
+    assert converters.version_evaluators_to_assertions([evaluator_item]) == []
+
+
+# ---------------------------------------------------------------------------
+# version_policy_to_execution_policy
+# ---------------------------------------------------------------------------
+
+
+def test_version_policy_to_execution_policy__converts():
+    policy = mock.MagicMock()
+    policy.runs_per_item = 5
+    policy.pass_threshold = 3
+
+    assert converters.version_policy_to_execution_policy(policy) == {
+        "runs_per_item": 5,
+        "pass_threshold": 3,
+    }
+
+
+def test_version_policy_to_execution_policy__none__returns_default():
+    assert converters.version_policy_to_execution_policy(None) == {
+        "runs_per_item": 1,
+        "pass_threshold": 1,
+    }
+
+
+# ---------------------------------------------------------------------------
+# dataset_item_to_suite_item_dict (DatasetItem → TestSuiteItem)
+# ---------------------------------------------------------------------------
+
+
+def test_dataset_item_to_suite_item_dict__all_fields():
+    item = dataset_item.DatasetItem(
+        id="item-1",
+        description="Test item",
+        question="What is 2+2?",
+        execution_policy=dataset_item.ExecutionPolicyItem(
+            runs_per_item=3, pass_threshold=2
+        ),
+    )
+
+    assert converters.dataset_item_to_suite_item_dict(item) == {
+        "id": "item-1",
+        "data": {"question": "What is 2+2?"},
+        "assertions": [],
+        "description": "Test item",
+        "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+    }
+
+
+def test_dataset_item_to_suite_item_dict__minimal():
+    item = dataset_item.DatasetItem(id="item-1", question="Hello")
+
+    result = converters.dataset_item_to_suite_item_dict(item)
+
+    assert result == {"id": "item-1", "data": {"question": "Hello"}, "assertions": []}
+    assert "description" not in result
+    assert "execution_policy" not in result
+
+
+def test_dataset_item_to_suite_item_dict__with_evaluators__extracts_assertions():
+    from opik.evaluation.suite_evaluators import LLMJudge
+
+    judge = LLMJudge(assertions=["Is correct"], track=False)
+    config = judge.to_config().model_dump(by_alias=True)
+
+    item = dataset_item.DatasetItem(
+        id="item-2",
+        evaluators=[
+            dataset_item.EvaluatorItem(
+                name="llm_judge", type="llm_judge", config=config
+            ),
+        ],
+        question="Hello",
+    )
+
+    assert converters.dataset_item_to_suite_item_dict(item)["assertions"] == [
+        "Is correct"
+    ]
+
+
+# ---------------------------------------------------------------------------
+# suite_item_dict_to_dataset_item (TestSuiteItem → DatasetItem)
+# ---------------------------------------------------------------------------
+
+
+def test_suite_item_dict_to_dataset_item__all_fields():
+    item = {
+        "id": "item-1",
+        "data": {"question": "Hello", "context": "test"},
+        "assertions": ["Is polite"],
+        "description": "A test case",
+        "execution_policy": {"runs_per_item": 5, "pass_threshold": 3},
+    }
+
+    ds_item = converters.suite_item_dict_to_dataset_item(item)
+
+    assert ds_item.id == "item-1"
+    assert ds_item.get_content() == {"question": "Hello", "context": "test"}
+    assert ds_item.description == "A test case"
+    assert ds_item.execution_policy is not None
+    assert ds_item.execution_policy.runs_per_item == 5
+    assert ds_item.execution_policy.pass_threshold == 3
+    assert ds_item.evaluators is not None
+    assert len(ds_item.evaluators) == 1
+    assert ds_item.evaluators[0].type == "llm_judge"
+
+
+def test_suite_item_dict_to_dataset_item__minimal__generates_id():
+    ds_item = converters.suite_item_dict_to_dataset_item(
+        {"data": {"question": "Hello"}}
+    )
+
+    assert ds_item.id is not None
+    assert len(ds_item.id) > 0
+    assert ds_item.get_content() == {"question": "Hello"}
+    assert ds_item.evaluators is None
+    assert ds_item.execution_policy is None
+
+
+# ---------------------------------------------------------------------------
+# to_json / from_json
+# ---------------------------------------------------------------------------
+
+
+SAMPLE_ITEMS = [
+    {
+        "id": "item-1",
+        "data": {"question": "How do I get a refund?", "context": "Premium user"},
+        "assertions": ["Response is polite"],
+        "description": "Refund scenario",
+        "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+    },
+    {
+        "id": "item-2",
+        "data": {"question": "Is my account hacked?"},
+        "assertions": [],
+    },
+]
+
+
+def test_to_json__happyflow():
+    EXPECTED = [
+        {
+            "id": "item-1",
+            "data": {"question": "How do I get a refund?", "context": "Premium user"},
+            "assertions": ["Response is polite"],
+            "description": "Refund scenario",
+            "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+        },
+        {
+            "id": "item-2",
+            "data": {"question": "Is my account hacked?"},
+            "assertions": [],
+        },
+    ]
+
+    assert json.loads(converters.to_json(SAMPLE_ITEMS)) == EXPECTED
+
+
+def test_to_json__empty_list():
+    assert json.loads(converters.to_json([])) == []
+
+
+def test_from_json__happyflow():
+    json_str = json.dumps(
+        [
+            {"data": {"question": "Hello"}, "assertions": ["Is polite"]},
+            {"data": {"question": "Bye"}},
+        ]
+    )
+
+    EXPECTED = [
+        {"data": {"question": "Hello"}, "assertions": ["Is polite"]},
+        {"data": {"question": "Bye"}},
+    ]
+
+    assert converters.from_json(json_str, {}, []) == EXPECTED
+
+
+def test_from_json__with_keys_mapping():
+    json_str = json.dumps(
+        [
+            {"test_data": {"question": "Hello"}, "checks": ["Is polite"]},
+        ]
+    )
+
+    EXPECTED = [{"data": {"question": "Hello"}, "assertions": ["Is polite"]}]
+
+    assert (
+        converters.from_json(
+            json_str, {"test_data": "data", "checks": "assertions"}, []
+        )
+        == EXPECTED
+    )
+
+
+def test_from_json__with_ignore_keys():
+    json_str = json.dumps(
+        [
+            {"data": {"question": "Hello"}, "internal_note": "skip this"},
+        ]
+    )
+
+    EXPECTED = [{"data": {"question": "Hello"}}]
+
+    assert converters.from_json(json_str, {}, ["internal_note"]) == EXPECTED
+
+
+def test_from_json__non_array__raises_value_error():
+    json_str = json.dumps({"data": {"question": "Hello"}})
+
+    with pytest.raises(ValueError, match="must be an array"):
+        converters.from_json(json_str, {}, [])
+
+
+# ---------------------------------------------------------------------------
+# to_pandas / from_pandas
+# ---------------------------------------------------------------------------
+
+
+def test_to_pandas__happyflow():
+    EXPECTED = pd.DataFrame(
+        [
+            {
+                "id": "item-1",
+                "data": {
+                    "question": "How do I get a refund?",
+                    "context": "Premium user",
+                },
+                "assertions": ["Response is polite"],
+                "description": "Refund scenario",
+                "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+            },
+            {
+                "id": "item-2",
+                "data": {"question": "Is my account hacked?"},
+                "assertions": [],
+            },
+        ]
+    )
+
+    pandas.testing.assert_frame_equal(converters.to_pandas(SAMPLE_ITEMS), EXPECTED)
+
+
+def test_to_pandas__empty_list():
+    assert len(converters.to_pandas([])) == 0
+
+
+def test_from_pandas__happyflow():
+    dataframe = pd.DataFrame(
+        [
+            {"data": {"question": "Hello"}, "assertions": ["Is polite"]},
+            {"data": {"question": "Bye"}},
+        ]
+    )
+
+    EXPECTED = [
+        {"data": {"question": "Hello"}, "assertions": ["Is polite"]},
+        {"data": {"question": "Bye"}},
+    ]
+
+    assert converters.from_pandas(dataframe, {}, []) == EXPECTED
+
+
+def test_from_pandas__with_keys_mapping():
+    dataframe = pd.DataFrame([{"test_data": {"question": "Hello"}}])
+
+    EXPECTED = [{"data": {"question": "Hello"}}]
+
+    assert converters.from_pandas(dataframe, {"test_data": "data"}, []) == EXPECTED
+
+
+def test_from_pandas__nan_values__skipped():
+    dataframe = pd.DataFrame(
+        [
+            {"data": {"question": "Hello"}, "assertions": ["Is polite"]},
+            {"data": {"question": "Bye"}, "assertions": float("nan")},
+        ]
+    )
+
+    result = converters.from_pandas(dataframe, {}, [])
+
+    assert result[0] == {"data": {"question": "Hello"}, "assertions": ["Is polite"]}
+    assert result[1] == {"data": {"question": "Bye"}}
+
+
+# ---------------------------------------------------------------------------
+# from_jsonl_file
+# ---------------------------------------------------------------------------
+
+
+def test_from_jsonl_file__happyflow():
+    jsonl_content = (
+        '{"data": {"question": "What is 2+2?"}, "assertions": ["Is correct"]}\n'
+        '{"data": {"question": "Capital of France?"}}\n'
+    )
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+        f.write(jsonl_content)
+        path = f.name
+
+    try:
+        EXPECTED = [
+            {"data": {"question": "What is 2+2?"}, "assertions": ["Is correct"]},
+            {"data": {"question": "Capital of France?"}},
+        ]
+
+        assert converters.from_jsonl_file(path, {}, []) == EXPECTED
+    finally:
+        os.unlink(path)
+
+
+def test_from_jsonl_file__empty_lines__skipped():
+    jsonl_content = '{"data": {"question": "Q1"}}\n\n{"data": {"question": "Q2"}}\n\n'
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+        f.write(jsonl_content)
+        path = f.name
+
+    try:
+        assert len(converters.from_jsonl_file(path, {}, [])) == 2
+    finally:
+        os.unlink(path)
+
+
+def test_from_jsonl_file__with_keys_mapping():
+    jsonl_content = '{"test_data": {"question": "Hello"}}\n'
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+        f.write(jsonl_content)
+        path = f.name
+
+    try:
+        EXPECTED = [{"data": {"question": "Hello"}}]
+
+        assert converters.from_jsonl_file(path, {"test_data": "data"}, []) == EXPECTED
+    finally:
+        os.unlink(path)
+
+
+# ---------------------------------------------------------------------------
+# Round-trip tests
+# ---------------------------------------------------------------------------
+
+
+def test_adapter_roundtrip__suite_to_dataset_to_suite__preserves_all_fields():
+    original = {
+        "id": "item-1",
+        "data": {"question": "Hello", "context": "Premium"},
+        "assertions": ["Is polite", "Is helpful"],
+        "description": "Test case",
+        "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+    }
+
+    ds_item = converters.suite_item_dict_to_dataset_item(original)
+    recovered = converters.dataset_item_to_suite_item_dict(ds_item)
+
+    assert recovered["id"] == original["id"]
+    assert recovered["data"] == original["data"]
+    assert sorted(recovered["assertions"]) == sorted(original["assertions"])
+    assert recovered["description"] == original["description"]
+    assert recovered["execution_policy"] == original["execution_policy"]
+
+
+def test_adapter_roundtrip__two_cycles__stable():
+    original = {
+        "id": "item-1",
+        "data": {"question": "Hello"},
+        "assertions": ["Is polite"],
+        "execution_policy": {"runs_per_item": 5, "pass_threshold": 3},
+    }
+
+    suite_item = original
+    for _ in range(2):
+        ds_item = converters.suite_item_dict_to_dataset_item(suite_item)
+        suite_item = converters.dataset_item_to_suite_item_dict(ds_item)
+
+    assert suite_item["data"] == original["data"]
+    assert suite_item["assertions"] == original["assertions"]
+    assert suite_item["execution_policy"] == original["execution_policy"]
+
+
+def test_json_roundtrip__export_import_export__stable():
+    json_str_1 = converters.to_json(SAMPLE_ITEMS)
+    imported = converters.from_json(json_str_1, {}, [])
+    json_str_2 = converters.to_json(imported)
+
+    assert json.loads(json_str_1) == json.loads(json_str_2)
+
+
+def test_json_roundtrip__import_export_import__stable():
+    json_str = json.dumps(
+        [
+            {
+                "data": {"question": "Hello"},
+                "assertions": ["Is polite"],
+                "description": "Test",
+                "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+            },
+        ]
+    )
+
+    items_1 = converters.from_json(json_str, {}, [])
+    exported = converters.to_json(items_1)
+    items_2 = converters.from_json(exported, {}, [])
+
+    assert items_1 == items_2
+
+
+def test_pandas_roundtrip__export_import_export__stable():
+    df_1 = converters.to_pandas(SAMPLE_ITEMS)
+    imported = converters.from_pandas(df_1, {}, [])
+    df_2 = converters.to_pandas(imported)
+
+    pandas.testing.assert_frame_equal(df_1, df_2)
+
+
+def test_full_roundtrip__json_through_adapters__two_cycles():
+    """JSON → from_json → adapter → adapter → to_json, repeated twice."""
+    original_json = json.dumps(
+        [
+            {
+                "id": "item-1",
+                "data": {"question": "Refund?", "tier": "premium"},
+                "assertions": ["Is polite", "No hallucination"],
+                "description": "Refund scenario",
+                "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+            },
+        ]
+    )
+
+    current_json = original_json
+    for _ in range(2):
+        suite_items = converters.from_json(current_json, {}, [])
+        ds_items = [converters.suite_item_dict_to_dataset_item(i) for i in suite_items]
+        suite_items = [converters.dataset_item_to_suite_item_dict(i) for i in ds_items]
+        current_json = converters.to_json(suite_items)
+
+    EXPECTED = {
+        "id": "item-1",
+        "data": {"question": "Refund?", "tier": "premium"},
+        "assertions": ["Is polite", "No hallucination"],
+        "description": "Refund scenario",
+        "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
+    }
+
+    final = json.loads(current_json)
+    assert len(final) == 1
+    result = final[0]
+    assert result["id"] == EXPECTED["id"]
+    assert result["data"] == EXPECTED["data"]
+    assert sorted(result["assertions"]) == sorted(EXPECTED["assertions"])
+    assert result["description"] == EXPECTED["description"]
+    assert result["execution_policy"] == EXPECTED["execution_policy"]

sdks/python/tests/unit/api_objects/dataset/test_suite/test_report_file.py

@@ -0,0 +1,317 @@
+"""Unit tests for test suite result file generation."""
+
+import json
+import os
+
+
+from opik.api_objects.dataset.test_suite import (
+    suite_result_constructor,
+    types as suite_types,
+)
+from opik.api_objects.dataset.test_suite.report_processors import file_writer
+from opik.api_objects.dataset import dataset_item
+from opik.evaluation import evaluation_result, test_result, test_case
+from opik.evaluation.metrics import score_result
+
+
+def _make_test_result(
+    dataset_item_id: str,
+    trial_id: int,
+    scores: list[tuple[str, float]],
+    task_output: dict | None = None,
+    dataset_item_content: dict | None = None,
+    execution_policy: dict | None = None,
+    task_execution_time: float | None = None,
+    scoring_time: float | None = None,
+) -> test_result.TestResult:
+    ds_item = None
+    if execution_policy is not None:
+        ds_item = dataset_item.DatasetItem(
+            id=dataset_item_id,
+            execution_policy=dataset_item.ExecutionPolicyItem(
+                runs_per_item=execution_policy.get("runs_per_item"),
+                pass_threshold=execution_policy.get("pass_threshold"),
+            ),
+        )
+
+    return test_result.TestResult(
+        test_case=test_case.TestCase(
+            trace_id=f"trace-{dataset_item_id}-{trial_id}",
+            dataset_item_id=dataset_item_id,
+            task_output=task_output or {"input": "test", "output": "result"},
+            dataset_item_content=dataset_item_content or {"question": "What?"},
+            dataset_item=ds_item,
+        ),
+        score_results=[
+            score_result.ScoreResult(name=name, value=value) for name, value in scores
+        ],
+        trial_id=trial_id,
+        task_execution_time=task_execution_time,
+        scoring_time=scoring_time,
+    )
+
+
+def _make_suite_result(
+    test_results_list: list[test_result.TestResult],
+    suite_name: str | None = None,
+    total_time: float | None = None,
+) -> suite_types.TestSuiteResult:
+    eval_result = evaluation_result.EvaluationResult(
+        experiment_id="exp-123",
+        dataset_id="dataset-456",
+        experiment_name="my-experiment",
+        test_results=test_results_list,
+        experiment_url="http://example.com/experiment/exp-123",
+        trial_count=1,
+    )
+    return suite_result_constructor.build_suite_result(
+        eval_result,
+        suite_name=suite_name,
+        total_time=total_time,
+    )
+
+
+class TestToReportDict:
+    def test_to_report_dict__single_item_with_mixed_scores__returns_correct_structure(
+        self,
+    ):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("Is polite", True), ("Is helpful", False)],
+                task_output={"input": "hi", "output": "hello"},
+                dataset_item_content={"question": "hi"},
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+                task_execution_time=1.234,
+                scoring_time=0.567,
+            )
+        ]
+        suite_result = _make_suite_result(test_results_list, suite_name="My Suite")
+
+        result = suite_result.to_report_dict()
+
+        assert result["suite_passed"] is False
+        assert result["items_passed"] == 0
+        assert result["items_total"] == 1
+        assert result["pass_rate"] == 0.0
+        assert result["experiment_id"] == "exp-123"
+        assert result["experiment_name"] == "my-experiment"
+        assert result["experiment_url"] == "http://example.com/experiment/exp-123"
+        assert result["suite_name"] == "My Suite"
+        assert "generated_at" in result
+
+        assert len(result["items"]) == 1
+        item = result["items"][0]
+        assert item["dataset_item_id"] == "item-1"
+        assert item["passed"] is False
+        assert item["runs_passed"] == 0
+        assert item["execution_policy"] == {"runs_per_item": 1, "pass_threshold": 1}
+
+        assert len(item["runs"]) == 1
+        run = item["runs"][0]
+        assert run["trial_id"] == 0
+        assert run["passed"] is False
+        assert run["input"] == "hi"
+        assert run["output"] == "hello"
+        assert run["trace_id"] == "trace-item-1-0"
+        assert run["task_execution_time_seconds"] == 1.234
+        assert run["scoring_time_seconds"] == 0.567
+
+        assert len(run["assertions"]) == 2
+        assert run["assertions"][0]["name"] == "Is polite"
+        assert run["assertions"][0]["passed"] is True
+        assert run["assertions"][1]["name"] == "Is helpful"
+        assert run["assertions"][1]["passed"] is False
+
+    def test_to_report_dict__all_items_pass__suite_passed_true(self):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+            _make_test_result(
+                dataset_item_id="item-2",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list)
+
+        result = suite_result.to_report_dict()
+
+        assert result["suite_passed"] is True
+        assert result["items_passed"] == 2
+        assert result["pass_rate"] == 1.0
+
+    def test_to_report_dict__with_total_time__includes_rounded_value(self):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list, total_time=12.3456)
+
+        result = suite_result.to_report_dict()
+
+        assert result["total_time_seconds"] == 12.346
+
+    def test_to_report_dict__scoring_failed__marks_assertion_failed_with_reason(self):
+        test_results_list = [
+            test_result.TestResult(
+                test_case=test_case.TestCase(
+                    trace_id="trace-1",
+                    dataset_item_id="item-1",
+                    task_output={"output": "test"},
+                    dataset_item_content={},
+                    dataset_item=dataset_item.DatasetItem(
+                        id="item-1",
+                        execution_policy=dataset_item.ExecutionPolicyItem(
+                            runs_per_item=1,
+                            pass_threshold=1,
+                        ),
+                    ),
+                ),
+                score_results=[
+                    score_result.ScoreResult(
+                        name="A1",
+                        value=0,
+                        scoring_failed=True,
+                        reason="Model error",
+                    ),
+                ],
+                trial_id=0,
+            )
+        ]
+        suite_result = _make_suite_result(test_results_list)
+
+        result = suite_result.to_report_dict()
+
+        assertion = result["items"][0]["runs"][0]["assertions"][0]
+        assert assertion["passed"] is False
+        assert assertion["scoring_failed"] is True
+        assert assertion["reason"] == "Model error"
+
+
+class TestSaveReport:
+    def test_save_report__valid_input__writes_json_file(self, tmp_path):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(
+            test_results_list, suite_name="Test Suite", total_time=5.0
+        )
+        output_path = str(tmp_path / "report.json")
+
+        result_path = file_writer.save_report(suite_result, output_path)
+
+        assert result_path == output_path
+        assert os.path.exists(output_path)
+
+        with open(output_path) as f:
+            data = json.load(f)
+
+        assert data["suite_name"] == "Test Suite"
+        assert data["suite_passed"] is True
+        assert len(data["items"]) == 1
+
+    def test_save_report__no_path__uses_experiment_name(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list)
+
+        result_path = file_writer.save_report(suite_result)
+
+        assert "my-experiment" in os.path.basename(result_path)
+        assert result_path.endswith(".json")
+        assert os.path.exists(result_path)
+
+    def test_save_report__nested_path__creates_parent_directories(self, tmp_path):
+        output_path = str(tmp_path / "nested" / "dir" / "report.json")
+
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list)
+
+        result_path = file_writer.save_report(suite_result, output_path)
+
+        assert os.path.exists(result_path)
+
+
+class TestBuildDefaultReportPath:
+    def test_build_default_report_path__unsafe_characters__replaces_with_underscore(
+        self,
+    ):
+        path = file_writer.build_default_report_path("my/suite:name")
+        assert os.path.basename(path) == "my_suite_name.json"
+
+    def test_build_default_report_path__safe_characters__keeps_unchanged(self):
+        path = file_writer.build_default_report_path("my-suite_v1.0")
+        assert os.path.basename(path) == "my-suite_v1.0.json"
+
+    def test_build_default_report_path__spaces__replaces_with_underscore(self):
+        path = file_writer.build_default_report_path("my suite")
+        assert os.path.basename(path) == "my_suite.json"
+
+
+class TestTestSuiteResultMethods:
+    def test_to_dict__passing_suite__returns_dict_with_suite_passed_true(self):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list)
+
+        result = suite_result.to_dict()
+
+        assert isinstance(result, dict)
+        assert result["suite_passed"] is True
+        assert "items" in result
+
+    def test_to_report_dict__save_to_file__produces_valid_json(self, tmp_path):
+        test_results_list = [
+            _make_test_result(
+                dataset_item_id="item-1",
+                trial_id=0,
+                scores=[("A1", True)],
+                execution_policy={"runs_per_item": 1, "pass_threshold": 1},
+            ),
+        ]
+        suite_result = _make_suite_result(test_results_list)
+        output_path = str(tmp_path / "result.json")
+
+        path = file_writer.save_report(suite_result, output_path)
+
+        assert os.path.exists(path)
+        with open(path) as f:
+            data = json.load(f)
+        assert data["suite_passed"] is True