chore: import upstream snapshot with attribution

sdks/python/tests/e2e/evaluation/test_evaluate_dataset_version.py

@@ -0,0 +1,90 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.evaluation.metrics import score_result
+
+from .. import verifiers
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+def test_evaluate__with_dataset_version__evaluates_version_items_only__happyflow(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that opik.evaluate works with DatasetVersion and only evaluates items from that version."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    # Insert first batch - creates v1 with 2 items
+    dataset.insert(
+        [
+            {"input": {"question": "Q1"}, "expected_output": {"answer": "A1"}},
+            {"input": {"question": "Q2"}, "expected_output": {"answer": "A2"}},
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Insert second batch - creates v2 with 4 items total
+    dataset.insert(
+        [
+            {"input": {"question": "Q3"}, "expected_output": {"answer": "A3"}},
+            {"input": {"question": "Q4"}, "expected_output": {"answer": "A4"}},
+        ]
+    )
+    _wait_for_version(dataset, "v2")
+
+    # Get v1 view - should have only 2 items
+    v1_view = dataset.get_version_view("v1")
+    assert v1_view.items_total == 2
+
+    # Simple task that returns the expected output
+    def task(item: Dict[str, Any]) -> Dict[str, Any]:
+        return item["expected_output"]
+
+    # Simple scoring function
+    def scoring_function(
+        dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+    ) -> score_result.ScoreResult:
+        return score_result.ScoreResult(name="test_score", value=1.0)
+
+    # Evaluate using DatasetVersion (v1)
+    result = opik.evaluate(
+        dataset=v1_view,
+        task=task,
+        scoring_functions=[scoring_function],
+        verbose=0,
+        project_name=PROJECT_NAME,
+    )
+
+    # Should have evaluated only 2 items (from v1), not 4 (from v2/current)
+    assert len(result.test_results) == 2
+
+    # Verify the items evaluated were from v1
+    evaluated_questions = {
+        tr.test_case.dataset_item_content["input"]["question"]
+        for tr in result.test_results
+    }
+    assert evaluated_questions == {"Q1", "Q2"}
+
+    # Verify the experiment is linked to v1's version ID
+    v1_version_info = v1_view.get_version_info()
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=result.experiment_id,
+        experiment_name=result.experiment_name,
+        experiment_metadata=None,
+        traces_amount=2,
+        feedback_scores_amount=1,
+        dataset_version_id=v1_version_info.id,
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_evaluate_filter_string.py

@@ -0,0 +1,141 @@
+from typing import Dict, Any
+
+import opik
+from opik import id_helpers
+from opik.evaluation import metrics
+from opik.evaluation import evaluator as evaluator_module
+
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_evaluate__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "geography"',
+        project_name=PROJECT_NAME,
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items (filtered by geography category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+
+def test_evaluate_optimization_trial__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate_optimization_trial correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    evaluator_module.evaluate_optimization_trial(
+        optimization_id=id_helpers.generate_id(),
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "math"',
+        project_name=PROJECT_NAME,
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1, (
+        f"Expected 1 experiment item (filtered by math category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME

sdks/python/tests/e2e/evaluation/test_evaluate_scoring_functions.py

@@ -0,0 +1,320 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.api_objects.experiment import experiment_item
+from opik.evaluation import metrics
+from opik.evaluation.metrics import score_result
+from opik.message_processing.emulation import models
+from opik.types import FeedbackScoreDict
+
+from .. import verifiers
+from ...testlib import assert_equal, ANY_BUT_NONE, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+DATASET_ITEMS = [
+    {
+        "input": {"question": "What is the of capital of Ukraine?"},
+        "expected_model_output": {"output": "Kyiv"},
+    },
+]
+
+
+def llm_task(item: Dict[str, Any]):
+    if item["input"] == {"question": "What is the of capital of Ukraine?"}:
+        return {"output": "Kyiv"}
+
+    raise AssertionError(
+        f"Task received dataset item with an unexpected input: {item['input']}"
+    )
+
+
+def equals_scoring_function(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]):
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="Correct output value" if value == 1.0 else "Incorrect output value",
+    )
+
+
+def task_span_name_scoring_function(
+    task_span: models.SpanModel, **ignored_kwargs: Any
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_scoring_function",
+        reason="Correct task span name" if score == 1.0 else "Incorrect task span name",
+    )
+
+
+def task_span_name_and_equals_scoring_function(
+    dataset_item: Dict[str, Any],
+    task_outputs: Dict[str, Any],
+    task_span: models.SpanModel,
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+
+    score *= value
+
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_and_equals_scoring_function",
+        reason="Correct task span name and output value"
+        if score == 1.0
+        else "Incorrect task span name and output value",
+    )
+
+
+def test_evaluate__scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that ordinary scoring functions work correctly.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+    _wait_for_version(dataset, "v1")
+
+    # Get the version ID to verify it's passed to the experiment
+    version_info = dataset.get_version_info()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=None,
+        dataset_version_id=version_info.id,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        )
+    ]
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+        actual=experiment_items_contents,
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_scoring_metrics__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and scoring metrics work correctly.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_metrics=[equals_metric],
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=2,
+        prompts=None,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None, name="equals_metric", reason=None, value=1.0
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+            ],
+        )
+    ]
+
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_task_span_scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and task span scoring functions work correctly.
+    # Also, it checks that task span scoring functions can access:
+    # task span, dataset item content (dataset_item), and task output (task_outputs) parameters.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[
+            equals_scoring_function,
+            task_span_name_scoring_function,
+            task_span_name_and_equals_scoring_function,
+        ],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=3,
+        prompts=None,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_and_equals_scoring_function",
+                    reason="Correct task span name and output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_scoring_function",
+                    reason="Correct task span name",
+                    value=1.0,
+                ),
+            ],
+        ),
+    ]
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )

sdks/python/tests/e2e/evaluation/test_evaluate_streaming.py

@@ -0,0 +1,158 @@
+import threading
+from typing import Dict, Any, List
+from unittest import mock
+
+import opik
+from opik.api_objects import rest_stream_parser
+from opik.evaluation import helpers as helpers_module
+from opik.evaluation.metrics import score_result
+from opik.types import FeedbackScoreDict
+
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+# Simple dataset items for testing
+DATASET_ITEMS = [
+    {
+        "input": {"question": f"Question {i}"},
+        "expected_output": {"answer": f"Answer {i}"},
+    }
+    for i in range(20)  # Create 20 items to test streaming behavior
+]
+
+# Batch size for streaming - must be less than len(DATASET_ITEMS) to test multi-batch streaming
+TEST_BATCH_SIZE = 5
+
+
+def simple_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Simple task that echoes the expected output."""
+    return item["expected_output"]
+
+
+def simple_scoring_function(
+    dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+) -> score_result.ScoreResult:
+    """Simple scoring function that always returns 1.0."""
+    return score_result.ScoreResult(
+        name="simple_score",
+        value=1.0,
+        reason="Test score",
+    )
+
+
+def test_streaming_starts_evaluation_before_complete_download(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    Test that streaming evaluation starts processing items before all items are downloaded.
+
+    This test verifies the core streaming behavior by:
+    1. Creating a dataset with multiple items
+    2. Setting a small batch size to force multiple batches
+    3. Patching read_and_parse_stream to track when items are yielded
+    4. Patching the task execution to track when tasks start
+    5. Verifying that the first task starts before the last item is yielded
+    """
+    # Create dataset with multiple items
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    # Track the sequence of events: 'yield' or 'task'
+    events: List[str] = []
+    events_lock = threading.Lock()
+
+    # Store original read_and_parse_stream function
+    original_read_and_parse_stream = rest_stream_parser.read_and_parse_stream
+
+    def tracked_read_and_parse_stream(stream, item_class, nb_samples=None):
+        """Wrapper that tracks when items are yielded from the stream."""
+        items = original_read_and_parse_stream(stream, item_class, nb_samples)
+        tracked_items = []
+        for item in items:
+            with events_lock:
+                events.append("yield")
+            tracked_items.append(item)
+        return tracked_items
+
+    def tracked_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        """Wrapper that tracks when tasks start executing."""
+        with events_lock:
+            events.append("task")
+        return simple_task(item)
+
+    # Patch both read_and_parse_stream to track yields and EVALUATION_STREAM_DATASET_BATCH_SIZE
+    # to ensure we stream in multiple batches
+    with (
+        mock.patch.object(
+            rest_stream_parser,
+            "read_and_parse_stream",
+            side_effect=tracked_read_and_parse_stream,
+        ),
+        mock.patch.object(
+            helpers_module, "EVALUATION_STREAM_DATASET_BATCH_SIZE", TEST_BATCH_SIZE
+        ),
+    ):
+        # Run evaluation with streaming
+        evaluation_result = opik.evaluate(
+            dataset=dataset,
+            task=tracked_task,
+            scoring_functions=[simple_scoring_function],
+            experiment_name=experiment_name,
+            verbose=1,
+            project_name=PROJECT_NAME,
+        )
+
+    # Verify evaluation completed successfully
+    assert evaluation_result.dataset_id == dataset.id
+    assert len(events) == len(DATASET_ITEMS) * 2  # Each item has yield + task
+
+    # Count yields and tasks
+    yield_count = events.count("yield")
+    task_count = events.count("task")
+    assert yield_count == len(DATASET_ITEMS)
+    assert task_count == len(DATASET_ITEMS)
+
+    # Critical assertion: Tasks should be interleaved with yields
+    # This proves streaming is working - we don't wait for all yields before starting tasks
+    # Find the index of the last yield
+    last_yield_index = len(events) - 1 - events[::-1].index("yield")
+    # Find the index of the first task
+    first_task_index = events.index("task")
+
+    assert first_task_index < last_yield_index, (
+        f"Streaming not working! First task appeared at index {first_task_index}, "
+        f"but last yield appeared at index {last_yield_index}. "
+        f"Tasks should start before all items are yielded. "
+        f"Event sequence: {events}"
+    )
+
+    # Verify the experiment was created correctly
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items = retrieved_experiment.get_items()
+    assert len(experiment_items) == len(DATASET_ITEMS)
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+    # Verify scoring output: each item should have a score with name "simple_score" and value 1.0
+    for item in experiment_items:
+        # Check that feedback_scores exists and is not empty
+        assert item.feedback_scores is not None and len(item.feedback_scores) == 1, (
+            f"Experiment item {item.id} should have exactly 1 feedback score, got {len(item.feedback_scores) if item.feedback_scores else 0}"
+        )
+
+        # Verify the score matches expected structure
+        expected_score = FeedbackScoreDict(
+            category_name=None,
+            name="simple_score",
+            reason="Test score",
+            value=1.0,
+        )
+
+        actual_score = item.feedback_scores[0]
+        assert actual_score == expected_score, (
+            f"Experiment item {item.id} has incorrect feedback score. "
+            f"Expected: {expected_score}, Got: {actual_score}"
+        )

sdks/python/tests/e2e/evaluation/test_evaluate_task_span.py

@@ -0,0 +1,487 @@
+from typing import Dict, Any, List, Optional
+
+import opik
+from opik import Prompt, synchronization
+from opik.evaluation import metrics
+from opik.evaluation.metrics import BaseMetric, score_result
+from opik.message_processing.emulation import models
+
+from .. import verifiers
+from ..conftest import random_chars
+
+
+def _wait_for_per_item_feedback_scores(
+    opik_client: opik.Opik,
+    experiment_name: str,
+    expected_count_per_item: int,
+    expected_item_count: int,
+    max_try_seconds: float = 10.0,
+) -> List[Any]:
+    """
+    Poll the backend until every experiment item has ``expected_count_per_item``
+    feedback scores. Returns the materialized list of experiment items.
+
+    The verifier ``verifiers.verify_experiment`` only checks the **experiment-
+    level** ``feedback_scores`` aggregate (one row per unique score name).
+    Per-item feedback scores can lag for an extra moment after the
+    aggregate has converged — especially when task-span scoring writes the
+    second batch of scores via ``client.log_traces_feedback_scores`` after
+    the per-item trace has been emitted. Polling avoids race-condition
+    flakes against the read-back without masking real bugs (a bounded
+    timeout still fails the test if the count never settles).
+    """
+    last_items: List[Any] = []
+
+    def _check() -> bool:
+        nonlocal last_items
+        experiment = opik_client.get_experiment_by_name(experiment_name)
+        last_items = experiment.get_items()
+        if len(last_items) != expected_item_count:
+            return False
+        return all(
+            len(item.feedback_scores) == expected_count_per_item for item in last_items
+        )
+
+    converged = synchronization.until(
+        _check, max_try_seconds=max_try_seconds, allow_errors=True
+    )
+    assert converged, (
+        f"Per-item feedback score counts did not converge to "
+        f"{expected_count_per_item} within {max_try_seconds}s. "
+        f"Last observed: items={len(last_items)}, "
+        f"counts={[len(item.feedback_scores) for item in last_items]}"
+    )
+    return last_items
+
+
+class TaskSpanTestMetric(BaseMetric):
+    def __init__(
+        self,
+        name: str = "task_span_test_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self, task_span: models.SpanModel, **ignored_kwargs: Any
+    ) -> score_result.ScoreResult:
+        score = 1.0 if task_span.name == "task" else 0.0
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason="Correct task span name"
+            if score == 1.0
+            else "Incorrect task span name",
+        )
+
+
+class TaskSpanInputTestMetric(BaseMetric):
+    def __init__(
+        self,
+        name: str = "task_span_input_test_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self, task_span: models.SpanModel, **ignored_kwargs: Any
+    ) -> score_result.ScoreResult:
+        input_data = task_span.input
+        has_question = (
+            isinstance(input_data, dict)
+            and "item" in input_data
+            and "input" in input_data["item"]
+            and isinstance(input_data["item"]["input"], dict)
+            and "question" in input_data["item"]["input"]
+        )
+        score = 1.0 if has_question else 0.0
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason="Task span has question input"
+            if score == 1.0
+            else "Task span missing question input",
+        )
+
+
+def test_evaluate__with_task_span_metrics__single_metric__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_model_output": {"output": "Berlin"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-task-span-prompt-{random_chars()}",
+        prompt=f"test-task-span-prompt-template-{random_chars()}",
+    )
+
+    task_span_metric = TaskSpanTestMetric()
+    equals_metric = metrics.Equals()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric, task_span_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompts=[prompt],
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=2,
+        feedback_scores_amount=2,  # equals_metric + task_span_metric
+        prompts=[prompt],
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=2,
+        expected_item_count=2,
+    )
+
+    for item in experiment_items_contents:
+        score_names = [score["name"] for score in item.feedback_scores]
+        assert "equals_metric" in score_names
+        assert "task_span_test_metric" in score_names
+
+        # Find task span metric score
+        task_span_score = next(
+            score
+            for score in item.feedback_scores
+            if score["name"] == "task_span_test_metric"
+        )
+        assert task_span_score["value"] == 1.0
+        assert "Correct task span name" in task_span_score["reason"]
+
+
+def test_evaluate__with_task_span_metrics__multiple_task_span_metrics__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Spain?"},
+                "expected_model_output": {"output": "Madrid"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Spain?"}:
+            return {"output": "Madrid"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    task_span_metric_1 = TaskSpanTestMetric(name="task_span_metric_1")
+    task_span_metric_2 = TaskSpanInputTestMetric(name="task_span_metric_2")
+    equals_metric = metrics.Equals()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric, task_span_metric_1, task_span_metric_2],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model-v2",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model-v2"},
+        traces_amount=1,
+        feedback_scores_amount=3,  # equals_metric + 2 task_span_metrics
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=3,
+        expected_item_count=1,
+    )
+    item = experiment_items_contents[0]
+    score_names = [score["name"] for score in item.feedback_scores]
+    assert "equals_metric" in score_names
+    assert "task_span_metric_1" in score_names
+    assert "task_span_metric_2" in score_names
+
+    # Verify all task span metrics scored correctly
+    for score in item.feedback_scores:
+        if score["name"] in ["task_span_metric_1", "task_span_metric_2"]:
+            assert score["value"] == 1.0
+
+
+def test_evaluate__with_task_span_metrics__only_task_span_metrics__no_regular_metrics(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Italy?"},
+                "expected_model_output": {"output": "Rome"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Italy?"}:
+            return {"output": "Rome"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    task_span_metric = TaskSpanTestMetric()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[task_span_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "task-span-only-model",
+        },
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "task-span-only-model"},
+        traces_amount=1,
+        feedback_scores_amount=1,  # only task_span_metric
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=1,
+        expected_item_count=1,
+    )
+    item = experiment_items_contents[0]
+    score = item.feedback_scores[0]
+    assert score["name"] == "task_span_test_metric"
+    assert score["value"] == 1.0
+
+
+def test_evaluate__with_task_span_metrics__mixed_with_regular_metrics__multiple_trials(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Japan?"},
+                "expected_model_output": {"output": "Tokyo"},
+            },
+            {
+                "input": {"question": "What is the capital of Canada?"},
+                "expected_model_output": {"output": "Ottawa"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Japan?"}:
+            return {"output": "Tokyo"}
+        if item["input"] == {"question": "What is the capital of Canada?"}:
+            return {"output": "Ottawa"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-mixed-metrics-prompt-{random_chars()}",
+        prompt=f"test-mixed-metrics-prompt-template-{random_chars()}",
+    )
+
+    # Mix of regular and task span metrics
+    equals_metric = metrics.Equals(name="regular_equals")
+    contains_metric = metrics.Contains(name="regular_contains")
+    task_span_metric = TaskSpanTestMetric(name="span_name_check")
+    task_span_input_metric = TaskSpanInputTestMetric(name="span_input_check")
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[
+            equals_metric,
+            task_span_metric,
+            contains_metric,
+            task_span_input_metric,
+        ],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "mixed-metrics-model",
+            "version": "1.0",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        trial_count=5,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "mixed-metrics-model", "version": "1.0"},
+        traces_amount=2 * 5,  # 2 traces per dataset item per trial
+        feedback_scores_amount=4,  # 2 regular + 2 task_span metrics
+        prompts=[prompt],
+    )
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=4,
+        expected_item_count=2 * 5,
+    )
+
+    expected_score_names = {
+        "regular_equals",
+        "regular_contains",
+        "span_name_check",
+        "span_input_check",
+    }
+
+    for item in experiment_items_contents:
+        actual_score_names = {score["name"] for score in item.feedback_scores}
+        assert actual_score_names == expected_score_names
+
+        # Verify all metrics scored correctly (assuming perfect matches)
+        for score in item.feedback_scores:
+            assert score["value"] == 1.0
+
+
+class TaskSpanWithMultipleParametersMetric(BaseMetric):
+    """
+    Metric that verifies multiple parameters are passed correctly:
+    - task_span: the span information
+    - input: from dataset item
+    - output: from task output
+    - **ignored_kwargs: to handle any other parameters
+    """
+
+    def __init__(
+        self,
+        name: str = "task_span_multi_param_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self,
+        task_span: models.SpanModel,
+        input: Dict[str, Any],
+        output: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        # Simply verify all expected parameters are present and store them in metadata
+        return score_result.ScoreResult(
+            value=1.0,
+            name=self.name,
+            reason=f"Received task_span={type(task_span).__name__}, input={type(input).__name__}, output={type(output).__name__}",
+            metadata={
+                "input": input,
+                "output": output,
+                "task_span_name": task_span.name,
+            },
+        )
+
+
+def test_evaluate__with_task_span_metrics__metric_with_multiple_parameters__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    Test that task_span metrics can access task_span, dataset item content (input),
+    and task output (output) parameters. Verifies arguments are passed correctly.
+    """
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert([{"input": {"question": "What is 2+2?"}}])
+
+    def task(item: Dict[str, Any]):
+        return {"output": "4"}
+
+    multi_param_metric = TaskSpanWithMultipleParametersMetric()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[multi_param_metric],
+        experiment_name=experiment_name,
+    )
+
+    # Verify the metric received all expected parameters in local test results
+    assert len(evaluation_result.test_results) == 1
+    test_result = evaluation_result.test_results[0]
+    assert len(test_result.score_results) == 1
+
+    score_result = test_result.score_results[0]
+    assert score_result.name == "task_span_multi_param_metric"
+    assert score_result.value == 1.0
+    assert "task_span=SpanModel" in score_result.reason
+    assert "input=dict" in score_result.reason
+    assert "output=str" in score_result.reason
+
+    # Verify the parameters were stored correctly in metadata
+    assert score_result.metadata is not None
+    assert score_result.metadata["input"] == {"question": "What is 2+2?"}
+    assert score_result.metadata["output"] == "4"
+    assert score_result.metadata["task_span_name"] == "task"

sdks/python/tests/e2e/evaluation/test_experiment_evaluate.py

@@ -0,0 +1,749 @@
+from typing import Dict, Any, List
+
+import opik
+
+from opik import Prompt, synchronization, id_helpers
+from opik.api_objects.dataset import dataset_item
+from opik.evaluation import metrics
+from opik.evaluation import test_result
+from opik.evaluation.metrics import score_result
+from opik.api_objects.experiment import experiment_item
+from .. import verifiers
+from ..conftest import random_chars
+from ...testlib import assert_equal, ANY_BUT_NONE, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_experiment_creation_via_evaluate_function__single_prompt_arg_used__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    experiment_tags = ["capital", "geography", "europe"]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        experiment_tags=experiment_tags,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=[prompt],
+        experiment_tags=experiment_tags,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 0.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+
+def test_experiment_creation_via_evaluate_function__single_prompt_arg_used__filter_dataset_items_by_id(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the of capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the of capital of Poland?"},
+            "expected_model_output": {"output": "Warsaw"},
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    # Keep France, drop Poland; append a fake id so the filter still covers
+    # the "non-existent id is ignored" case.
+    dataset_item_ids = [item["id"] for item in dataset_items]
+    dataset_item_ids.pop(1)
+    dataset_item_ids.append(id_helpers.generate_id())
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        dataset_item_ids=dataset_item_ids,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item (fake id is skipped)
+        feedback_scores_amount=1,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiments = opik_client.get_experiments_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert len(retrieved_experiments) == 1, (
+        f"Expected 1 experiment, but got {len(retrieved_experiments)}. "
+        f"Experiments: {retrieved_experiments}"
+    )
+    retrieved_experiment = retrieved_experiments[0]
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1, (
+        f"Expected 1 experiment item, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+
+def test_experiment_creation_via_evaluate_function__multiple_prompts_arg_used__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt1 = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+    prompt2 = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompts=[prompt1, prompt2],
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=[prompt1, prompt2],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 0.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+    verifiers.verify_experiment_traces_have_opik_prompts(
+        opik_client=opik_client,
+        trace_ids=[item.trace_id for item in experiment_items_contents],
+        prompts=[prompt1, prompt2],
+    )
+
+
+def test_experiment_creation__name_can_be_omitted(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    We can send "None" as experiment_name and the backend will set it for us
+    """
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "reference": "Paris",
+            },
+        ]
+    )
+
+    def task(item: dataset_item.DatasetItem):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=None,
+    )
+
+    experiment_id = evaluation_result.experiment_id
+
+    if not synchronization.until(
+        lambda: (
+            opik_client._rest_client.experiments.get_experiment_by_id(experiment_id)
+            is not None
+        ),
+        allow_errors=True,
+    ):
+        raise AssertionError(f"Failed to get experiment with id {experiment_id}.")
+
+    experiment_content = opik_client._rest_client.experiments.get_experiment_by_id(
+        experiment_id
+    )
+
+    assert experiment_content.name is not None, (
+        f"Expected experiment name to be set by backend, but got None. "
+        f"Experiment ID: {experiment_id}, Experiment content: {experiment_content}"
+    )
+
+
+def test_evaluate_experiment__an_experiment_created_with_evaluate__then_new_scores_are_added_to_existing_experiment_items__amount_of_feedback_scores_increased(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {
+                "output": "Paris",
+                "reference": item["expected_model_output"]["output"],
+            }
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    # Create the experiment first
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        prompt=prompt,
+        project_name=PROJECT_NAME,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={
+            "model_name": "gpt-3.5",
+        },
+        traces_amount=1,
+        feedback_scores_amount=0,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    # Populate the existing experiment with a new feedback score
+    evaluation_result = opik.evaluate_experiment(
+        experiment_name=experiment_name,
+        scoring_metrics=[
+            metrics.Equals(name="metric1"),
+            metrics.Equals(name="metric2"),
+            metrics.Equals(name="metric3"),
+        ],
+        project_name=PROJECT_NAME,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={
+            "model_name": "gpt-3.5",
+        },
+        traces_amount=1,
+        feedback_scores_amount=3,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+
+def test_experiment__get_experiments_by_name(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+        project_name=PROJECT_NAME,
+    )
+
+    experiments_names = [experiment_name, experiment_name, random_chars(10)]
+
+    evaluation_results = []
+    equals_metric = metrics.Equals()
+    for name in experiments_names:
+        evaluation_result = opik.evaluate(
+            dataset=dataset,
+            task=task,
+            scoring_metrics=[equals_metric],
+            experiment_name=name,
+            experiment_config={
+                "model_name": "gpt-3.5",
+            },
+            scoring_key_mapping={
+                "reference": lambda x: x["expected_model_output"]["output"],
+            },
+            prompt=prompt,
+            project_name=PROJECT_NAME,
+        )
+        evaluation_results.append(evaluation_result)
+
+    # make sure experiments saved and available
+    for result in evaluation_results:
+        verifiers.verify_experiment(
+            opik_client=opik_client,
+            id=result.experiment_id,
+            experiment_name=result.experiment_name,
+            experiment_metadata={"model_name": "gpt-3.5"},
+            traces_amount=2,  # one trace per dataset item
+            feedback_scores_amount=1,
+            prompts=[prompt],
+            project_name=PROJECT_NAME,
+        )
+
+    # check getting experiment by name
+    experiments = opik_client.get_experiments_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert len(experiments) == 2, (
+        f"Expected 2 experiments with name '{experiment_name}', but got {len(experiments)}. "
+        f"Experiment IDs: {[e.id for e in experiments]}"
+    )
+    assert all(experiment.project_name == PROJECT_NAME for experiment in experiments)
+
+    single = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert single is not None
+    assert single.id in [e.id for e in experiments]
+
+    experiments = opik_client.get_experiments_by_name(
+        experiments_names[2], project_name=PROJECT_NAME
+    )
+    assert len(experiments) == 1, (
+        f"Expected 1 experiment with name '{experiments_names[2]}', but got {len(experiments)}. "
+        f"Experiment IDs: {[e.id for e in experiments]}"
+    )
+
+
+def test_experiment__get_experiment_items__no_feedback_scores(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]) -> Dict[str, Any]:
+        return {
+            "output": "Paris",
+        }
+
+    opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[],
+        experiment_name=experiment_name,
+    )
+
+    experiment = opik_client.get_experiment_by_name(experiment_name)
+    items = experiment.get_items()
+
+    assert len(items) == 1, (
+        f"Expected 1 experiment item, but got {len(items)}. Items: {items}"
+    )
+    assert items[0].feedback_scores == [], (
+        f"Expected empty feedback scores, but got {items[0].feedback_scores}. "
+        f"Item: {items[0]}"
+    )
+
+
+def test_experiment_creation_via_evaluate_function__with_experiment_scoring_functions__scores_computed_and_logged(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions compute and log experiment-level scores."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Poland?"}:
+            return {"output": "Krakow"}  # Wrong answer
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    def constant_score(
+        test_results: List[test_result.TestResult],
+    ) -> List[score_result.ScoreResult]:
+        """Compute a random number based on the number of test results."""
+        return [
+            score_result.ScoreResult(
+                name="fixed_number",
+                value=0.8,
+                reason="Fixed score of 0.8",
+            )
+        ]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        experiment_scoring_functions=[constant_score],
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are in the result
+    assert len(evaluation_result.experiment_scores) == 1, (
+        f"Expected 1 experiment score in evaluation result, but got {len(evaluation_result.experiment_scores)}. "
+        f"Experiment scores: {evaluation_result.experiment_scores}"
+    )
+    assert evaluation_result.experiment_scores[0].name == "fixed_number", (
+        f"Expected experiment score name 'fixed_number', but got '{evaluation_result.experiment_scores[0].name}'. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].value == 0.8, (
+        f"Expected experiment score value 0.8, but got {evaluation_result.experiment_scores[0].value}. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].reason is not None, (
+        f"Expected experiment score reason to be set, but got None. "
+        f"Score: {evaluation_result.experiment_scores[0]}"
+    )
+    assert "Fixed score of 0.8" in evaluation_result.experiment_scores[0].reason, (
+        f"Expected reason to contain 'Fixed score of 0.8', but got: '{evaluation_result.experiment_scores[0].reason}'"
+    )
+
+    # Verify experiment was created with experiment scores
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        experiment_scores={"fixed_number": 0.8},
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_experiment_scoring_functions.py

@@ -0,0 +1,138 @@
+import statistics
+from typing import Dict, Any, List
+
+import pytest
+
+import opik
+from opik.evaluation import metrics
+from opik.evaluation import test_result
+from opik.evaluation.metrics import score_result
+from .. import verifiers
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_experiment_scoring_functions__standard_deviation__computed_and_logged(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions can compute and log standard deviation of metric scores."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": "Paris",
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_output": "Warsaw",
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        # Return correct answer for France, wrong answer for Poland so
+        # metric scores are [1.0, 0.0] — enough to exercise stdev.
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris", "reference": item["expected_output"]}
+        if item["input"] == {"question": "What is the capital of Poland?"}:
+            return {"output": "Krakow", "reference": item["expected_output"]}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    def compute_std_deviation(
+        test_results: List[test_result.TestResult],
+    ) -> List[score_result.ScoreResult]:
+        """Compute standard deviation of metric scores across all test results."""
+        scores = [x.score_results[0].value for x in test_results]
+        return [
+            score_result.ScoreResult(
+                name="equals_metric_std_dev",
+                value=statistics.stdev(scores) if len(scores) > 1 else 0.0,
+                reason=f"Standard deviation of {len(scores)} metric scores",
+            )
+        ]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        experiment_scoring_functions=[compute_std_deviation],
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are in the result
+    assert len(evaluation_result.experiment_scores) == 1, (
+        f"Expected 1 experiment score in evaluation result, but got {len(evaluation_result.experiment_scores)}. "
+        f"Experiment scores: {evaluation_result.experiment_scores}"
+    )
+    assert evaluation_result.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev', but got '{evaluation_result.experiment_scores[0].name}'. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+
+    expected_stdev = statistics.stdev([1.0, 0.0])
+    assert evaluation_result.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev}, but got {evaluation_result.experiment_scores[0].value}. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].reason is not None, (
+        f"Expected experiment score reason to be set, but got None. "
+        f"Score: {evaluation_result.experiment_scores[0]}"
+    )
+    assert (
+        "Standard deviation of 2 metric scores"
+        in evaluation_result.experiment_scores[0].reason
+    ), (
+        f"Expected reason to contain 'Standard deviation of 2 metric scores', but got: '{evaluation_result.experiment_scores[0].reason}'"
+    )
+
+    # Verify experiment was created
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are logged to backend by retrieving the experiment
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_data = retrieved_experiment.get_experiment_data()
+
+    # Check that experiment scores are present in the experiment data
+    assert experiment_data.experiment_scores is not None, (
+        f"Expected experiment_scores to be set in experiment data, but got None. "
+        f"Experiment ID: {evaluation_result.experiment_id}, "
+        f"Experiment name: {evaluation_result.experiment_name}, "
+        f"Experiment data: {experiment_data}"
+    )
+    assert len(experiment_data.experiment_scores) == 1, (
+        f"Expected 1 experiment score in backend, but got {len(experiment_data.experiment_scores) if experiment_data.experiment_scores else 0}. "
+        f"Experiment scores: {experiment_data.experiment_scores}"
+    )
+    assert experiment_data.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev' in backend, but got '{experiment_data.experiment_scores[0].name}'. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )
+    assert experiment_data.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev} in backend, but got {experiment_data.experiment_scores[0].value}. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )

sdks/python/tests/e2e/evaluation/test_test_suite_agentic.py

@@ -0,0 +1,314 @@
+"""E2E tests for the agentic LLM-judge path on test suites.
+
+The one-shot LLMJudge sees only the dataset item's top-level `input` /
+`output`. The agentic judge — auto-engaged when the local emulator is
+active during `opik.run_tests` — receives a pre-rendered flat overview
+of the trace plus `read` / `scan` / `search` tools for drilling in.
+These tests craft assertions that are decidable ONLY from the
+intermediate span state, so a passing verdict is direct evidence the
+agentic judge ran:
+
+1. A span-structure assertion ("the agent called the `fetch_user_data`
+   helper") — verifiable only by looking at span names in the trace.
+2. A span-error assertion ("no errors occurred without errors") on a
+   trace whose inner span recorded an error caught by the task —
+   verifiable only by looking at span `error_info`.
+3. A buried-keyword assertion — the marker sits past the overview's
+   floor-tier 500-char truncation, so the judge must call at least one
+   of `read` / `scan` / `search` to recover it. The test pins the
+   overview sizer's ladder to the floor entry to guarantee truncation
+   regardless of the judge model's context budget.
+
+**Judge-model choice.** These tests deliberately override the SDK's
+default judge model (`gpt-5-nano`) with a stronger one. `gpt-5-nano`
+is the canonical "doesn't engage the tool loop" failure mode the
+backend doc (`SupportedJudgeProvider.java`) warns about — it tends to
+judge from the inline overview alone, never calling `read`. The SDK
+can't prompt-engineer around that; the design doc §9 explicitly
+classifies model engagement as the operator's call. Here we pick
+`gpt-4o-mini` to match the backend's allow-list second choice — same
+OpenAI dependency story, but actually engages with tools.
+
+All require `OPENAI_API_KEY`; the assertions are deliberately
+unambiguous to keep judge flakiness low even on a competent judge.
+"""
+
+from typing import Any, Dict
+
+import pytest
+
+import opik
+from .. import verifiers
+from ...testlib import environment, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+# Stronger than the SDK default (`gpt-5-nano`) and known to engage the
+# tool loop — backend `SupportedJudgeProvider.java` lists it as the
+# OpenAI allow-list entry. If you change this, re-run all three tests:
+# nano will pass the first two (structural assertions) but reliably
+# fail the third (it never calls `read` on truncated content).
+AGENTIC_JUDGE_MODEL = "gpt-5-mini"  #
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_test_suite_agentic__assertion_about_span_name__passes(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """The agentic judge must inspect the trace's span tree to verify
+    that a specifically-named tool span was actually called. The task
+    output deliberately omits the helper's name, so a one-shot judge
+    has no way to verify the assertion from input/output alone.
+    """
+    span_assertion = "The agent called a step named `fetch_user_data`"
+
+    @opik.track(name="fetch_user_data", project_name=PROJECT_NAME)
+    def fetch_user_data(user_id: str) -> Dict[str, Any]:
+        return {"id": user_id, "name": "alice"}
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        fetch_user_data("u-1")
+        # Output deliberately doesn't mention `fetch_user_data` — the
+        # only evidence the helper was called lives in the span tree.
+        return {"input": item["input"], "output": "ok"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — span-name assertion",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "fetch user u-1"}},
+                "assertions": [span_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # The one assertion must surface in the feedback scores.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={span_assertion},
+        project_name=PROJECT_NAME,
+    )
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+@pytest.mark.xfail(reason="flaky on CI with simple OpenAI models")
+def test_test_suite_agentic__assertion_about_span_error__detects_failure(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """The agentic judge must inspect `error_info` on a nested span to
+    confirm a *specific* exception type and message. The task catches
+    the exception so the trace's top-level output looks healthy.
+
+    Why the assertion names the exception type AND a phrase from the
+    message: the inline overview already carries each span's
+    `has_error` flag, so an assertion of the form "no errors occurred"
+    would be decidable from the overview alone — the test would pass
+    without ever calling `read` and prove nothing about drill-in. By
+    pinning the verdict on the exception's type and message — both of
+    which live only in `error_info` and are NOT in the overview — the
+    correct verdict is only reachable by reading the span. A judge
+    that stops at `has_error: true` lacks the evidence to confirm
+    "type=RuntimeError" or "message contains 'internal failure'" and
+    must drill in.
+    """
+    span_error_assertion = (
+        "An internal step raised an exception of type `RuntimeError` "
+        "whose message contains the phrase 'internal failure'"
+    )
+
+    @opik.track(name="risky_step", project_name=PROJECT_NAME)
+    def risky_step() -> str:
+        # @opik.track captures `error_info` on the inner span when this
+        # raises, even though the caller swallows the exception. The
+        # exception type and message are recorded in `error_info` —
+        # *not* exposed via the overview's `has_error` flag — which is
+        # what forces the judge to call `read`.
+        raise RuntimeError("internal failure")
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            risky_step()
+        except RuntimeError:
+            pass
+        return {"input": item["input"], "output": "completed"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — span-error assertion (type + message)",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "do the thing"}},
+                "assertions": [span_error_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # The assertion is TRUE — the inner step did raise a RuntimeError
+    # with message "internal failure". A judge that stayed at the
+    # overview level would only see `has_error: true` with no type or
+    # message info, so per the "absence of evidence" rule it would
+    # fail the assertion (score=false). A passing verdict means it
+    # drilled into `error_info` to verify both the type and the
+    # message phrase.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={span_error_assertion},
+        project_name=PROJECT_NAME,
+    )
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_test_suite_agentic__assertion_requires_buried_keyword_lookup__passes(
+    opik_client: opik.Opik,
+    dataset_name: str,
+    experiment_name: str,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """The marker is placed past the floor-tier truncation the overview
+    applies to every span's I/O. The overview alone physically cannot
+    include it, so the judge MUST reach for at least one of
+    `read` / `scan` / `search` to find the marker and pass the
+    assertion. We don't pin which tool the model picks — that's a
+    model-quality question. What we prove here is that the agentic loop
+    engages beyond the overview, since a one-shot judge would lack the
+    evidence.
+
+    The marker is a synthetic token (`MARKER-` + opaque tail) chosen so
+    it can't appear anywhere else in the trace or in the model's
+    pretraining. A `score: true` verdict here means the judge actually
+    extracted it from span content.
+
+    NOTE on the ladder monkeypatch: the overview sizer normally picks
+    the largest per-field limit that fits the model's context budget,
+    which on `gpt-4o-mini` (128k window) would render this small trace
+    at the `NO_OVERVIEW_TRUNCATION` tier — the marker would be visible
+    inline and the test premise would silently regress. Forcing the
+    ladder to its single floor entry (`OVERVIEW_IO_FLOOR_CHAR_LIMIT`) keeps
+    the truncation guaranteed without needing to produce a multi-MB
+    trace just to overflow the budget. The sizer's own behavior is
+    covered by `test_overview_sizer.py`.
+    """
+    from opik.evaluation.suite_evaluators.agentic.compression import (
+        span_tree_serializer,
+    )
+
+    monkeypatch.setattr(
+        span_tree_serializer,
+        "OVERVIEW_IO_LIMIT_LADDER",
+        (span_tree_serializer.OVERVIEW_IO_FLOOR_CHAR_LIMIT,),
+    )
+
+    secret_marker = "MARKER-7f3a8c2e9b1d4f60"
+    keyword_assertion = (
+        f"At least one intermediate step processed a payload containing "
+        f"the literal token '{secret_marker}'"
+    )
+
+    @opik.track(name="process_step", project_name=PROJECT_NAME)
+    def process_step(payload: str) -> str:
+        # The output deliberately doesn't echo the marker — it only
+        # references the payload length, so the agent can't shortcut
+        # through `output` alone. The marker lives only in the input,
+        # which is what the overview truncates at the floor tier.
+        return f"step processed {len(payload)} chars"
+
+    @opik.track(name="noop_step", project_name=PROJECT_NAME)
+    def noop_step() -> str:
+        return "noop"
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        # Pad with filler so the marker lands well past the overview's
+        # OVERVIEW_IO_FLOOR_CHAR_LIMIT (500 chars) in the JSON-rendered span
+        # input `{"payload": "<filler><marker>"}`. `padding ` is 8 chars
+        # × 70 = 560 chars; with the ~13-char JSON prefix the marker
+        # starts past offset 573, so it sits beyond the overview cap by
+        # construction (the ladder is pinned to the floor by the
+        # monkeypatch above). Mixed-in noop_step guarantees more than
+        # one span exists, so the judge can't trivially infer "must be
+        # the only span."
+        filler = "padding " * 170
+        process_step(payload=f"{filler}{secret_marker} trailer")
+        noop_step()
+        return {"input": item["input"], "output": "completed"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — buried-keyword requires read/scan/search",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "process the payload"}},
+                "assertions": [keyword_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # If the judge stayed at the overview level, it would have seen the
+    # truncated string (no marker) and could only have guessed at the
+    # assertion. A passing verdict means it drilled into span content
+    # via one of read / scan / search.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={keyword_assertion},
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_threads_evaluate.py

@@ -0,0 +1,220 @@
+import time
+import uuid
+import pytest
+
+import opik
+from opik import synchronization
+from opik.evaluation import metrics
+from opik.evaluation.models import LiteLLMChatModel
+from opik.evaluation.threads import evaluator
+from opik.types import FeedbackScoreDict
+
+from ...testlib import environment
+from .. import verifiers
+
+
+@pytest.fixture
+def real_model_conversation():
+    return [
+        {
+            "role": "user",
+            "content": "I need to book a flight to New York and find a hotel.",
+        },
+        {
+            "role": "assistant",
+            "content": "I can help you with that. For flights to New York, what dates are you looking to travel?",
+        },
+    ]
+
+
+@pytest.fixture
+def active_thread_and_project_name(
+    opik_client, real_model_conversation, temporary_project_name
+):
+    thread_id = str(uuid.uuid4())[-6:]
+
+    # create conversation traces
+    i = 0
+    while i < len(real_model_conversation) - 1:
+        opik_client.trace(
+            name=f"trace-name-{i}:{thread_id}",
+            input={"input": f"{real_model_conversation[i]['content']}"},
+            output={"output": f"{real_model_conversation[i + 1]['content']}"},
+            project_name=temporary_project_name,
+            thread_id=thread_id,
+        )
+        time.sleep(0.1)
+        i += 2
+
+    opik_client.flush()
+
+    return thread_id, temporary_project_name
+
+
+def _one_thread_is_active(project_name: str, opik_client: opik.Opik) -> bool:
+    threads = opik_client.search_threads(
+        project_name=project_name, filter_string='status = "active"'
+    )
+    return len(threads) == 1
+
+
+@pytest.fixture
+def eval_project_name(temporary_project_name: str) -> str:
+    return temporary_project_name
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_evaluate_threads__happy_path(
+    opik_client, active_thread_and_project_name, eval_project_name
+):
+    active_thread, project_name = active_thread_and_project_name
+    # wait for active threads to propagate
+    if not synchronization.until(
+        lambda: _one_thread_is_active(project_name, opik_client), max_try_seconds=30
+    ):
+        raise AssertionError(f"Failed to create threads in project '{project_name}'")
+
+    judge_model = LiteLLMChatModel(reasoning_effort="minimal")
+
+    metrics_ = [
+        metrics.ConversationalCoherenceMetric(model=judge_model),
+    ]
+
+    result = evaluator.evaluate_threads(
+        project_name=project_name,
+        filter_string=f'id = "{active_thread}"',
+        metrics=metrics_,
+        eval_project_name=eval_project_name,
+        trace_input_transform=lambda x: x["input"],
+        trace_output_transform=lambda x: x["output"],
+        verbose=1,
+    )
+
+    assert result is not None
+    assert len(result.results) == 1  # we have only one thread
+
+    thread_result = result.results[0]
+    assert thread_result.thread_id == active_thread
+    assert len(thread_result.scores) == len(metrics_)
+
+    feedback_scores = [
+        FeedbackScoreDict(
+            id=active_thread,
+            name=score.name,
+            value=score.value,
+            reason=score.reason.strip(),
+            category_name=None,
+        )
+        for score in thread_result.scores
+        if not score.scoring_failed
+    ]
+
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=active_thread,
+        project_name=project_name,
+        feedback_scores=feedback_scores,
+    )
+
+
+def test_evaluate_threads__no_truncation_for_long_traces(
+    opik_client, temporary_project_name
+):
+    """E2E test verifying that long trace content is not truncated during evaluation.
+
+    The test creates a trace with output exceeding 15,000 characters with a unique marker
+    at the end, then verifies the marker is present in the transform function, proving
+    that truncation did not occur.
+    """
+    thread_id = str(uuid.uuid4())[-6:]
+
+    # Create a long output that exceeds the truncation threshold (~9935 chars)
+    # with a unique marker at the end that would be lost if truncated
+    marker = "UNIQUE_END_MARKER_XYZ123"
+    long_content = "a" * 15000 + marker
+
+    # Create a trace with very long output
+    opik_client.trace(
+        name=f"long-trace:{thread_id}",
+        input={"input": "test input"},
+        output={"output": long_content},
+        project_name=temporary_project_name,
+        thread_id=thread_id,
+    )
+
+    opik_client.flush()
+
+    # Wait for thread to be created
+    if not synchronization.until(
+        lambda: _one_thread_is_active(temporary_project_name, opik_client),
+        max_try_seconds=30,
+    ):
+        raise AssertionError(
+            f"Failed to create thread in project '{temporary_project_name}'"
+        )
+
+    # Track what the transform receives
+    received_outputs = []
+
+    def input_transform(x):
+        return x.get("input", "")
+
+    def output_transform(x):
+        """Transform that captures the output to verify it's not truncated."""
+        # When truncated, x might be a string (malformed JSON) instead of dict
+        if isinstance(x, str):
+            # This is the bug! Truncation causes malformed JSON string
+            received_outputs.append(f"TRUNCATED_STRING:{x[:100]}...")
+            return "TRUNCATED"
+        output = x.get("output", "")
+        received_outputs.append(output)
+        return output
+
+    # Create a simple metric that just checks the output
+    class ContentVerificationMetric(metrics.base_metric.BaseMetric):
+        def __init__(self):
+            super().__init__(
+                name="content_verification",
+                track=False,
+            )
+
+        def score(self, conversation, **ignored_kwargs):
+            # Just return a dummy score - we're really testing the transform
+            return metrics.score_result.ScoreResult(
+                name=self.name,
+                value=1.0,
+                reason="Content verification",
+            )
+
+    # Run evaluation
+    result = evaluator.evaluate_threads(
+        project_name=temporary_project_name,
+        filter_string=f'id = "{thread_id}"',
+        metrics=[ContentVerificationMetric()],
+        eval_project_name=temporary_project_name,
+        trace_input_transform=input_transform,
+        trace_output_transform=output_transform,
+        verbose=0,
+    )
+
+    assert result is not None
+    assert len(result.results) == 1
+
+    # Verify that the transform received the full content with the marker
+    assert len(received_outputs) > 0, "Transform should have been called"
+    transformed_content = received_outputs[0]
+
+    # This is the critical assertion: if truncation occurred, the marker would be missing
+    assert marker in transformed_content, (
+        f"Content was truncated! Expected marker '{marker}' not found. "
+        f"Content length: {len(transformed_content)}, expected: {len(long_content)}. "
+        f"Last 100 chars: {transformed_content[-100:]}"
+    )
+
+    # Also verify the full length is preserved
+    assert len(transformed_content) == len(long_content), (
+        f"Content length mismatch: got {len(transformed_content)}, "
+        f"expected {len(long_content)}"
+    )