chore: import upstream snapshot with attribution

sdks/python/tests/.gitignore

@@ -0,0 +1,2 @@
+# Local pytest artifacts — raw `--durations=0` dumps from e2e runs
+e2e_*.out

sdks/python/tests/conftest.py

@@ -0,0 +1,373 @@
+import os
+import random
+import string
+import tempfile
+import warnings
+from typing import cast
+
+import numpy as np
+
+from unittest import mock
+import pytest
+
+from opik import context_storage
+from opik.api_objects import connection_resources, opik_client
+from opik.healthcheck import connection_monitor, connection_probe
+from opik.message_processing import streamer_constructors
+from opik.message_processing.replay import replay_manager
+
+from . import testlib
+from .testlib import (
+    backend_emulator_message_processor,
+    environment,
+    noop_file_upload_manager,
+)
+
+
+@pytest.fixture(autouse=True)
+def clear_context_storage():
+    yield
+    context_storage.clear_all()
+
+
+@pytest.fixture(autouse=True)
+def shutdown_cached_client_after_test():
+    yield
+    opik_client.reset_global_client(end_client=False)
+    # Drop any shared connection bundle so the process-wide cache doesn't leak a
+    # streamer/emulator from one test into the next. Critical with fake_backend,
+    # which installs a fresh emulator-backed streamer per test: without this, a
+    # reused bundle keeps the previous test's (closed) streamer and the new
+    # emulator records nothing. Fire-and-forget: no flush, no network.
+    connection_resources.MANAGER.close_all(flush=False)
+
+
+@pytest.fixture(autouse=True)
+def skip_local_configuration_file(tmp_path):
+    # Point config path to a non-existent file so the global config is never read
+    # We need to do this because there are a lot of tests that compare the logged project name
+    # in saved traces/spans using "Default Project" as value. But the global config
+    # can already have the project name saved, so the comparison will fail.
+    os.environ["OPIK_CONFIG_PATH"] = str(tmp_path / "nonexistent_opik.config")
+
+
+@pytest.fixture
+def fake_file_upload_manager():
+    upload_manager_emulator = noop_file_upload_manager.FileUploadManagerEmulator()
+    yield upload_manager_emulator
+
+
+@pytest.fixture
+def fake_replay_manager():
+    probe = mock.Mock(spec=connection_probe.ConnectionProbe)
+    probe.check_connection.return_value = connection_probe.ProbeResult(True, None)
+
+    monitor = connection_monitor.OpikConnectionMonitor(
+        ping_interval=10,
+        check_timeout=1,
+        probe=probe,
+    )
+
+    fallback_replay = replay_manager.ReplayManager(
+        monitor=monitor,
+        batch_size=10,
+        batch_replay_delay=0.5,
+        tick_interval_seconds=5.0,
+    )
+    return fallback_replay
+
+
+@pytest.fixture
+def patch_streamer(
+    fake_file_upload_manager: noop_file_upload_manager.FileUploadManagerEmulator,
+    fake_replay_manager: replay_manager.ReplayManager,
+):
+    streamer = None
+    try:
+        # Pass the upload manager to the emulator so it can access attachments
+        fake_message_processor_ = (
+            backend_emulator_message_processor.BackendEmulatorMessageProcessor(
+                file_upload_manager=fake_file_upload_manager
+            )
+        )
+        streamer = streamer_constructors.construct_streamer(
+            message_processor=fake_message_processor_,
+            n_consumers=1,
+            use_batching=True,
+            file_uploader=fake_file_upload_manager,
+            max_queue_size=None,
+            use_attachment_extraction=False,
+            fallback_replay_manager=fake_replay_manager,
+        )
+
+        yield streamer, fake_message_processor_
+    finally:
+        if streamer is not None:
+            # Tests that rely on fake_backend already call `opik.flush_tracker()`
+            # (or access `.trace_trees` after the work is done) during the test
+            # body, so the teardown has nothing meaningful left to drain.
+            streamer.close(flush=False)
+
+
+@pytest.fixture
+def patch_streamer_without_batching(
+    fake_replay_manager: replay_manager.ReplayManager,
+):
+    streamer = None
+    try:
+        # Create an upload manager first
+        fake_upload_manager = noop_file_upload_manager.FileUploadManagerEmulator()
+        # Pass the upload manager to the emulator so it can access attachments
+        fake_message_processor_ = (
+            backend_emulator_message_processor.BackendEmulatorMessageProcessor(
+                file_upload_manager=fake_upload_manager
+            )
+        )
+        streamer = streamer_constructors.construct_streamer(
+            message_processor=fake_message_processor_,
+            n_consumers=1,
+            use_batching=False,
+            file_uploader=fake_upload_manager,
+            max_queue_size=None,
+            use_attachment_extraction=False,
+            fallback_replay_manager=fake_replay_manager,
+        )
+
+        yield streamer, fake_message_processor_
+    finally:
+        if streamer is not None:
+            streamer.close(flush=False)
+
+
+@pytest.fixture
+def fake_backend(patch_streamer):
+    """
+    Patches the function that creates an instance of Streamer under the hood of Opik.
+    As a result, instead of sending data to the backend, it's being passed to
+    the backend emulator, which uses this data to build span and trace trees.
+
+    The resulting trees can be accessed via `fake_backend.trace_trees` or
+    `fake_backend.span_trees` and then used for comparing with expected span/trace trees.
+
+    The trees are built via special classes stored in testlib.models.
+    """
+    streamer, fake_message_processor_ = patch_streamer
+
+    fake_message_processor_ = cast(
+        backend_emulator_message_processor.BackendEmulatorMessageProcessor,
+        fake_message_processor_,
+    )
+
+    mock_construct_online_streamer = mock.Mock()
+    mock_construct_online_streamer.return_value = streamer
+
+    with mock.patch.object(
+        streamer_constructors,
+        "construct_online_streamer",
+        mock_construct_online_streamer,
+    ):
+        yield fake_message_processor_
+
+
+@pytest.fixture
+def fake_backend_without_batching(patch_streamer_without_batching):
+    """
+    Same as fake_backend but must be used when batching is not supported
+    (e.g., when there are Span/Trace update requests)
+    """
+    streamer, fake_message_processor_ = patch_streamer_without_batching
+
+    fake_message_processor_ = cast(
+        backend_emulator_message_processor.BackendEmulatorMessageProcessor,
+        fake_message_processor_,
+    )
+
+    mock_construct_online_streamer = mock.Mock()
+    mock_construct_online_streamer.return_value = streamer
+
+    with mock.patch.object(
+        streamer_constructors,
+        "construct_online_streamer",
+        mock_construct_online_streamer,
+    ):
+        yield fake_message_processor_
+
+
+@pytest.fixture
+def fake_backend_with_patched_environment(request, patch_streamer):
+    """
+    Allows patching environment variables for the duration of the test.
+    Creates a fake backend as in the `fake_backend` fixture.
+    """
+    with testlib.patch_environ(add_keys=request.param):
+        streamer, fake_message_processor_ = patch_streamer
+
+        fake_message_processor_ = cast(
+            backend_emulator_message_processor.BackendEmulatorMessageProcessor,
+            fake_message_processor_,
+        )
+
+        mock_construct_online_streamer = mock.Mock()
+        mock_construct_online_streamer.return_value = streamer
+
+        with mock.patch.object(
+            streamer_constructors,
+            "construct_online_streamer",
+            mock_construct_online_streamer,
+        ):
+            yield fake_message_processor_
+
+
+def random_chars(n: int = 6) -> str:
+    return "".join(random.choice(string.ascii_letters) for _ in range(n))
+
+
+@pytest.fixture()
+def configure_opik_local_env_vars():
+    with testlib.patch_environ(
+        {
+            "OPIK_URL_OVERRIDE": "http://localhost:5173/api",
+        }
+    ):
+        yield
+
+
+@pytest.fixture()
+def configure_opik_not_configured():
+    with testlib.patch_environ(
+        add_keys={},
+        remove_keys=[
+            "OPIK_URL_OVERRIDE",
+            "OPIK_API_KEY",
+            "OPIK_WORKSPACE",
+            "OPIK_PROJECT_NAME",
+        ],
+    ):
+        yield
+
+
+@pytest.fixture()
+def temp_file_15kb():
+    file_size = 15 * 1024
+    with tempfile.NamedTemporaryFile(delete=True) as f:
+        f.write(np.random.bytes(file_size))
+        f.flush()
+
+        yield f
+
+
+@pytest.fixture()
+def temp_file_15mb():
+    file_size = 15 * 1024 * 1024
+    with tempfile.NamedTemporaryFile(delete=True) as f:
+        f.write(np.random.bytes(file_size))
+        f.flush()
+
+        yield f
+
+
+@pytest.fixture(scope="session")
+def ensure_openai_configured():
+    # don't use assertion here to prevent printing os.environ with all env variables
+    if not environment.has_openai_api_key():
+        raise Exception(
+            "OpenAI not configured! Ensure you have set the OPENAI_API_KEY and OPENAI_ORG_ID environment variables."
+        )
+
+
+@pytest.fixture(scope="session")
+def ensure_google_project_and_location_configured():
+    if not (
+        "GOOGLE_CLOUD_PROJECT" in os.environ and "GOOGLE_CLOUD_LOCATION" in os.environ
+    ):
+        raise Exception(
+            "GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION env vars must be set!"
+        )
+
+
+@pytest.fixture(scope="session")
+def ensure_anthropic_configured():
+    # don't use assertion here to prevent printing os.environ with all env variables
+
+    if "ANTHROPIC_API_KEY" not in os.environ:
+        raise Exception("Anthropic not configured!")
+
+
+@pytest.fixture(scope="session")
+def ensure_mistral_configured():
+    # don't use assertion here to prevent printing os.environ with all env variables
+
+    if "MISTRAL_API_KEY" not in os.environ:
+        raise Exception("Mistral not configured!")
+
+
+@pytest.fixture(scope="session")
+def ensure_vertexai_configured(ensure_google_project_and_location_configured):
+    GOOGLE_APPLICATION_CREDENTIALS_PATH = "gcp_credentials.json"
+
+    if "GITHUB_ACTIONS" not in os.environ:
+        if "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ:
+            raise Exception("GOOGLE_APPLICATION_CREDENTIALS env var must be configured")
+        yield
+        return
+
+    if "GCP_CREDENTIALS_JSON" not in os.environ:
+        raise Exception(
+            "GCP_CREDENTIALS_JSON env var with credentials json content must be set"
+        )
+
+    try:
+        gcp_credentials = os.environ["GCP_CREDENTIALS_JSON"]
+        with open(GOOGLE_APPLICATION_CREDENTIALS_PATH, mode="wt") as output_file:
+            output_file.write(gcp_credentials)
+
+        with testlib.patch_environ(
+            add_keys={
+                "GOOGLE_APPLICATION_CREDENTIALS": GOOGLE_APPLICATION_CREDENTIALS_PATH
+            }
+        ):
+            yield
+    finally:
+        if os.path.exists(GOOGLE_APPLICATION_CREDENTIALS_PATH):
+            os.remove(GOOGLE_APPLICATION_CREDENTIALS_PATH)
+
+
+@pytest.fixture(scope="session")
+def ensure_google_api_configured():
+    GOOGLE_API_KEY = "GOOGLE_API_KEY"
+    if GOOGLE_API_KEY not in os.environ:
+        raise Exception(f"{GOOGLE_API_KEY} env var must be set")
+
+
+@pytest.fixture(scope="session")
+def ensure_aws_bedrock_configured():
+    import boto3
+
+    session = boto3.Session()
+
+    bedrock_client = session.client(service_name="bedrock")
+    try:
+        available_models = bedrock_client.list_foundation_models()["modelSummaries"]
+        if not available_models:
+            raise Exception("AWS Bedrock not configured! No models available")
+    except Exception as e:
+        raise Exception(f"AWS Bedrock not configured! {e}")
+
+
+@pytest.fixture(scope="session")
+def ensure_groq_configured():
+    # don't use assertion here to prevent printing os.environ with all env variables
+    if "GROQ_API_KEY" not in os.environ:
+        raise Exception("Groq is not configured!")
+
+
+warnings.filterwarnings(
+    "ignore",
+    message=".*PydanticDeprecatedSince20.*",
+    category=DeprecationWarning,
+)
+warnings.filterwarnings(
+    "ignore",
+    message=".*0 counts of 2-gram overlaps.*",
+    category=UserWarning,
+)

sdks/python/tests/e2e/cli/_cascade_comparison.py

@@ -0,0 +1,473 @@
+"""Deep-equal helpers for ``opik migrate dataset`` cascade e2e tests.
+
+The cascade copies four kinds of entities -- experiment + experiment items
++ traces + spans -- with FK fields remapped to the destination. Counts
+alone aren't enough; we also need to verify that the content (input,
+output, tags, metadata, feedback scores, assertion results, span tree
+shape) round-trips byte-for-byte modulo the remapped IDs.
+
+This module provides ``compare_cascade(source_state, destination_state, rest_client)``
+that recursively diff-walks both sides and raises ``AssertionError`` with
+a precise message on any mismatch.
+
+What's compared
+---------------
+Experiment level:
+  - name, type, evaluation_method, tags, metadata
+  - prompt_versions must be None on destination (epic decision: strip)
+
+Experiment items (paired via source/dest item ordinal, which corresponds
+to the source/dest dataset_item_id pairing the cascade builds):
+  - assertion_results compared as a set keyed by (value, passed, reason)
+  - feedback_scores compared as a set keyed by (name, value, reason, source)
+  - status NOT compared -- BE computes it from assertion_results
+
+Traces (paired via cascade's trace_id_remap):
+  - name, input, output, metadata, tags, start_time, end_time,
+    thread_id, error_info, ttft, environment
+  - feedback_scores compared as a set keyed by (name, value, reason, source)
+
+Spans (tree-aware):
+  - both sides sorted topologically (parent before child)
+  - parent_span_id remap verified by reconstructing each side's tree and
+    walking in lockstep
+  - per-span: name, type, input, output, metadata, model, provider,
+    tags, usage, start_time, end_time, error_info, ttft,
+    total_estimated_cost, environment
+  - feedback_scores on spans compared as a set
+
+What's NOT compared (intentional)
+---------------------------------
+  - any id field (id, project_id, experiment_id, dataset_id,
+    dataset_version_id, dataset_item_id, trace_id, span_id,
+    parent_span_id, optimization_id) -- they all change during cascade
+  - audit fields (created_at, last_updated_at, created_by, last_updated_by)
+  - BE-computed aggregates on traces/items (trace_count,
+    total_estimated_cost, duration, usage, span_count, llm_span_count,
+    has_tool_spans, providers, span_feedback_scores)
+  - ``project_name`` on experiment metadata (Slice 3 stamps it on the
+    destination as part of recreate_experiment; differs intentionally)
+  - ``prompt_versions`` (stripped on destination per epic decision)
+  - ``optimization_id`` (stripped on destination -- Slice 4's territory)
+
+Trace ``input`` / ``output`` JSON that embeds source-side IDs (e.g.
+``{'item': '<src-dataset-item-id>'}``) round-trips verbatim. The cascade
+deliberately does not recursively remap arbitrary JSON content. Tests
+that seed embedded IDs in trace I/O and care about post-migration
+freshness need their own narrower assertion; this module compares the
+JSON shape verbatim because that IS the cascade's contract.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from opik.rest_api import OpikApi
+
+
+# ---------------------------------------------------------------------------
+# Top-level entrypoint
+# ---------------------------------------------------------------------------
+
+
+def compare_cascade(
+    *,
+    rest_client: OpikApi,
+    source_experiment: Any,
+    destination_experiment: Any,
+    source_item_ids: List[str],
+    destination_item_ids: List[str],
+    source_trace_ids: List[str],
+    destination_trace_ids: List[str],
+    source_items_compare: List[Any],
+    destination_items_compare: List[Any],
+) -> None:
+    """Deep-equal the experiment + items + traces + spans between source and
+    destination, modulo remapped IDs.
+
+    Raises ``AssertionError`` with a focused message on any divergence.
+
+    The trace pairing is positional: ``source_trace_ids[i]`` must correspond
+    to ``destination_trace_ids[i]`` (callers maintain this ordering when
+    they seed + read). Same for items.
+    """
+    _compare_experiment(source_experiment, destination_experiment)
+
+    if len(source_items_compare) != len(destination_items_compare):
+        raise AssertionError(
+            f"item count diverged: source={len(source_items_compare)}, "
+            f"destination={len(destination_items_compare)}"
+        )
+    if len(source_trace_ids) != len(destination_trace_ids):
+        raise AssertionError(
+            f"trace count diverged: source={len(source_trace_ids)}, "
+            f"destination={len(destination_trace_ids)}"
+        )
+
+    # Items are typically returned in BE-imposed order (e.g. by created_at
+    # desc). Pair by dataset_item_id round-trip: source item with source
+    # dataset_item_id S maps to destination item with destination
+    # dataset_item_id D where D = item_id_remap[S]. The callers pass the
+    # already-paired ordered lists, so positional zip works.
+    for src_item, dst_item in zip(source_items_compare, destination_items_compare):
+        _compare_experiment_item(src_item, dst_item)
+
+    # Traces compared in pairs.
+    for src_tid, dst_tid in zip(source_trace_ids, destination_trace_ids):
+        src_trace = rest_client.traces.get_trace_by_id(id=src_tid)
+        dst_trace = rest_client.traces.get_trace_by_id(id=dst_tid)
+        _compare_trace(src_trace, dst_trace)
+
+        # Spans for this trace. ``project_id`` lives on the trace's read
+        # shape and scopes the spans query correctly without needing the
+        # caller to plumb project_name everywhere.
+        src_spans = _fetch_spans_for_trace(
+            rest_client, trace_id=src_tid, project_id=src_trace.project_id
+        )
+        dst_spans = _fetch_spans_for_trace(
+            rest_client, trace_id=dst_tid, project_id=dst_trace.project_id
+        )
+        _compare_span_trees(src_spans, dst_spans)
+
+
+# ---------------------------------------------------------------------------
+# Experiment-level
+# ---------------------------------------------------------------------------
+
+
+def _compare_experiment(src: Any, dst: Any) -> None:
+    if src.name != dst.name:
+        raise AssertionError(
+            f"experiment.name diverged: source={src.name!r}, destination={dst.name!r}"
+        )
+    if src.type != dst.type:
+        raise AssertionError(
+            f"experiment.type diverged: source={src.type!r}, destination={dst.type!r}"
+        )
+    if src.evaluation_method != dst.evaluation_method:
+        raise AssertionError(
+            f"experiment.evaluation_method diverged: source={src.evaluation_method!r}, "
+            f"destination={dst.evaluation_method!r}"
+        )
+    if (src.tags or None) != (dst.tags or None):
+        raise AssertionError(
+            f"experiment.tags diverged: source={src.tags!r}, destination={dst.tags!r}"
+        )
+
+    # Metadata: compare modulo Slice 3's injections.
+    # - ``project_name`` is stamped on the destination by recreate_experiment
+    #   (kept as a forward-import hint); on source it depends on how the
+    #   experiment was created. Strip from both for comparison.
+    # - ``prompt_versions`` is stripped on the destination by design.
+    src_meta = dict(src.metadata or {})
+    dst_meta = dict(dst.metadata or {})
+    src_meta.pop("project_name", None)
+    dst_meta.pop("project_name", None)
+    src_meta.pop("prompt_versions", None)
+    dst_meta.pop("prompt_versions", None)
+    if src_meta != dst_meta:
+        raise AssertionError(
+            f"experiment.metadata diverged (after stripping project_name + "
+            f"prompt_versions): source={src_meta!r}, destination={dst_meta!r}"
+        )
+
+    # Per epic decision, destination must have prompt_versions stripped.
+    if dst.prompt_versions:
+        raise AssertionError(
+            f"experiment.prompt_versions should be stripped on destination "
+            f"(epic decision); got {dst.prompt_versions!r}"
+        )
+
+    # Per epic decision, destination must have optimization_id stripped.
+    if dst.optimization_id:
+        raise AssertionError(
+            f"experiment.optimization_id should be stripped on destination "
+            f"(Slice 4 cascades the optimization entity); "
+            f"got {dst.optimization_id!r}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Experiment item (Compare view)
+# ---------------------------------------------------------------------------
+
+
+def _compare_experiment_item(src: Any, dst: Any) -> None:
+    src_ars = _normalize_assertions(src.assertion_results)
+    dst_ars = _normalize_assertions(dst.assertion_results)
+    if src_ars != dst_ars:
+        raise AssertionError(
+            f"experiment item assertion_results diverged: "
+            f"source={src_ars}, destination={dst_ars}"
+        )
+
+    src_fs = _normalize_feedback_scores(src.feedback_scores)
+    dst_fs = _normalize_feedback_scores(dst.feedback_scores)
+    if src_fs != dst_fs:
+        raise AssertionError(
+            f"experiment item feedback_scores diverged: "
+            f"source={src_fs}, destination={dst_fs}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Trace
+# ---------------------------------------------------------------------------
+
+
+_TRACE_DIRECT_FIELDS: Tuple[str, ...] = (
+    "name",
+    "input",
+    "output",
+    "metadata",
+    "tags",
+    "thread_id",
+    "ttft",
+    "environment",
+)
+
+
+def _compare_trace(src: Any, dst: Any) -> None:
+    for field in _TRACE_DIRECT_FIELDS:
+        s = getattr(src, field, None)
+        d = getattr(dst, field, None)
+        if (s or None) != (d or None):
+            raise AssertionError(
+                f"trace.{field} diverged: source={s!r}, destination={d!r}"
+            )
+
+    # ``error_info`` model_dump for content comparison; the read shape is
+    # ErrorInfoPublic on both sides so dicts should be equal.
+    s_err = _safe_dump(src.error_info)
+    d_err = _safe_dump(dst.error_info)
+    if s_err != d_err:
+        raise AssertionError(
+            f"trace.error_info diverged: source={s_err}, destination={d_err}"
+        )
+
+    # start_time / end_time round-trip as-is; the cascade copies them
+    # verbatim from the source trace. ms precision differences would
+    # surface here.
+    if src.start_time != dst.start_time:
+        raise AssertionError(
+            f"trace.start_time diverged: source={src.start_time}, "
+            f"destination={dst.start_time}"
+        )
+    if (src.end_time or None) != (dst.end_time or None):
+        raise AssertionError(
+            f"trace.end_time diverged: source={src.end_time}, "
+            f"destination={dst.end_time}"
+        )
+
+    # Feedback scores compared as a set keyed by name+value+reason+source.
+    src_fs = _normalize_feedback_scores(src.feedback_scores)
+    dst_fs = _normalize_feedback_scores(dst.feedback_scores)
+    if src_fs != dst_fs:
+        raise AssertionError(
+            f"trace.feedback_scores diverged: source={src_fs}, destination={dst_fs}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Span tree
+# ---------------------------------------------------------------------------
+
+
+_SPAN_DIRECT_FIELDS: Tuple[str, ...] = (
+    "name",
+    "type",
+    "input",
+    "output",
+    "metadata",
+    "model",
+    "provider",
+    "tags",
+    "usage",
+    "total_estimated_cost",
+    "ttft",
+    "environment",
+)
+
+
+def _compare_span_trees(src_spans: List[Any], dst_spans: List[Any]) -> None:
+    """Walk both span trees in parallel, comparing per-node fields and
+    verifying parent_span_id remap (children's new parent must be the
+    remapped new root, etc.).
+
+    Pairs spans across the two sides by tree position: both lists are
+    sorted topologically (parents first) and within a parent's children
+    by (name, start_time). The cascade preserves source order via
+    ``sort_spans_topologically`` so a stable sort makes this
+    deterministic.
+    """
+    if len(src_spans) != len(dst_spans):
+        raise AssertionError(
+            f"span count diverged: source={len(src_spans)}, "
+            f"destination={len(dst_spans)}"
+        )
+
+    src_sorted = _topo_sort_for_compare(src_spans)
+    dst_sorted = _topo_sort_for_compare(dst_spans)
+
+    src_to_dst_span_id: Dict[Optional[str], Optional[str]] = {None: None}
+    for src_span, dst_span in zip(src_sorted, dst_sorted):
+        src_to_dst_span_id[src_span.id] = dst_span.id
+
+        for field in _SPAN_DIRECT_FIELDS:
+            s = getattr(src_span, field, None)
+            d = getattr(dst_span, field, None)
+            if (s or None) != (d or None):
+                raise AssertionError(
+                    f"span.{field} diverged (source span id={src_span.id!r}, "
+                    f"dest span id={dst_span.id!r}): source={s!r}, destination={d!r}"
+                )
+
+        # Timestamps verbatim.
+        if src_span.start_time != dst_span.start_time:
+            raise AssertionError(
+                f"span.start_time diverged (source span id={src_span.id!r}): "
+                f"source={src_span.start_time}, destination={dst_span.start_time}"
+            )
+        if (src_span.end_time or None) != (dst_span.end_time or None):
+            raise AssertionError(
+                f"span.end_time diverged (source span id={src_span.id!r}): "
+                f"source={src_span.end_time}, destination={dst_span.end_time}"
+            )
+
+        # Error info.
+        s_err = _safe_dump(getattr(src_span, "error_info", None))
+        d_err = _safe_dump(getattr(dst_span, "error_info", None))
+        if s_err != d_err:
+            raise AssertionError(
+                f"span.error_info diverged (source span id={src_span.id!r}): "
+                f"source={s_err}, destination={d_err}"
+            )
+
+        # Feedback scores compared as a set.
+        s_fs = _normalize_feedback_scores(getattr(src_span, "feedback_scores", None))
+        d_fs = _normalize_feedback_scores(getattr(dst_span, "feedback_scores", None))
+        if s_fs != d_fs:
+            raise AssertionError(
+                f"span.feedback_scores diverged (source span id={src_span.id!r}): "
+                f"source={s_fs}, destination={d_fs}"
+            )
+
+        # parent_span_id remap correctness: the destination span's
+        # parent_span_id must be the destination id of the source span's
+        # parent (or None for root).
+        expected_dst_parent = src_to_dst_span_id.get(src_span.parent_span_id)
+        if dst_span.parent_span_id != expected_dst_parent:
+            raise AssertionError(
+                f"span.parent_span_id remap incorrect "
+                f"(source span id={src_span.id!r}, source parent={src_span.parent_span_id!r}): "
+                f"expected destination parent={expected_dst_parent!r}, "
+                f"got destination parent={dst_span.parent_span_id!r}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Normalisation helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalize_assertions(items: Optional[List[Any]]) -> List[Tuple[Any, Any, Any]]:
+    """Set-equality-friendly tuples keyed by the AssertionResult identity:
+    (value, passed, reason). Sorted so list-equality also works."""
+    if not items:
+        return []
+    return sorted(
+        ((a.value, a.passed, a.reason) for a in items),
+        key=lambda t: (str(t[0]), bool(t[1]), str(t[2] or "")),
+    )
+
+
+def _normalize_feedback_scores(
+    items: Optional[List[Any]],
+) -> List[Tuple[Any, ...]]:
+    """Set-equality-friendly tuples keyed by (name, value, reason, source).
+    Source vs destination scores might come back in different orders; the
+    sort makes the comparison stable."""
+    if not items:
+        return []
+    return sorted(
+        (
+            (
+                getattr(f, "name", None),
+                getattr(f, "value", None),
+                getattr(f, "category_name", None),
+                getattr(f, "reason", None),
+                getattr(f, "source", None),
+            )
+            for f in items
+        ),
+        key=lambda t: tuple(str(x) for x in t),
+    )
+
+
+def _safe_dump(obj: Any) -> Optional[Dict[str, Any]]:
+    if obj is None:
+        return None
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    if isinstance(obj, dict):
+        return obj
+    return {"_raw": str(obj)}
+
+
+def _topo_sort_for_compare(spans: List[Any]) -> List[Any]:
+    """Topological sort that's also stable on (name, start_time).
+
+    The cascade re-emits spans in source topological order. The BE may
+    return them in a different ordering on read; this helper produces a
+    deterministic order on both sides so paired comparison works.
+    """
+    by_id: Dict[Optional[str], Any] = {s.id: s for s in spans}
+    children: Dict[Optional[str], List[Any]] = {None: []}
+    for s in spans:
+        children.setdefault(s.parent_span_id, []).append(s)
+    # Sort each parent's children deterministically.
+    for parent_id, kids in children.items():
+        kids.sort(key=lambda s: (s.name or "", str(s.start_time)))
+
+    out: List[Any] = []
+
+    def _walk(parent_id: Optional[str]) -> None:
+        for s in children.get(parent_id, []):
+            out.append(s)
+            _walk(s.id)
+
+    _walk(None)
+    # Defensive: catch orphans (spans whose parent isn't in the same tree).
+    if len(out) != len(spans):
+        # Append orphans at the end in deterministic order.
+        seen = {s.id for s in out}
+        orphans = [s for s in spans if s.id not in seen]
+        orphans.sort(key=lambda s: (s.name or "", str(s.start_time)))
+        out.extend(orphans)
+    _ = by_id  # by_id retained for clarity / potential future use
+    return out
+
+
+def _fetch_spans_for_trace(
+    rest_client: OpikApi, *, trace_id: str, project_id: Optional[str]
+) -> List[Any]:
+    """Pull all spans for one trace from the BE.
+
+    Scopes by ``project_id`` (off the trace's read shape), required by
+    the BE.
+    """
+    collected: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.spans.get_spans_by_project(
+            project_id=project_id,
+            trace_id=trace_id,
+            page=page,
+            size=200,
+        )
+        page_content = resp.content or []
+        collected.extend(page_content)
+        if len(page_content) < 200:
+            break
+        page += 1
+    return collected

sdks/python/tests/e2e/cli/conftest.py

@@ -0,0 +1,701 @@
+"""Shared fixtures + helpers for ``opik migrate`` e2e tests.
+
+These tests drive ``opik migrate dataset`` against a real backend
+(localhost during dev, the CI-provisioned Opik in CI). They verify
+per-version fidelity end-to-end: items, item-level fields (data,
+description, tags, evaluators, execution_policy, source), version-level
+fields (suite evaluators, execution_policy, user tags, metadata), and
+display order.
+
+Helpers live here so individual test files stay focused on the scenarios
+they exercise. Wire-type item reads (via ``rest_stream_parser`` directly)
+mirror what ``cli/migrate/datasets/version_replay.py`` does in production
+— the SDK dataclass strips per-item ``tags`` during reconstruction, so
+asserting tag fidelity requires the wire type.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import subprocess
+import os
+import sys
+from typing import Any, Dict, Iterator, List, Optional, Set
+
+import pytest
+
+import opik
+from opik.api_objects import rest_stream_parser
+from opik.rest_api import OpikApi
+from opik.rest_api.core.api_error import ApiError
+from opik.rest_api.types import dataset_item_public, dataset_version_public
+
+from ...conftest import random_chars
+
+
+# ---------------------------------------------------------------------------
+# Project fixtures — ephemeral source + target, deleted on teardown
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def source_project_name(opik_client: opik.Opik) -> Iterator[str]:
+    """Create an ephemeral source project for the migration test.
+
+    Deleted on teardown (best-effort — tolerates already-deleted state).
+    Each test gets its own project so parallel runs don't collide on
+    dataset-name uniqueness (datasets are workspace-scoped, not
+    project-scoped, in Opik's BE — see Slice 1's collision pre-flight).
+    """
+    name = f"e2e-cli-migrate-source-{random_chars()}"
+    opik_client.rest_client.projects.create_project(name=name)
+    yield name
+    _best_effort_delete_project(opik_client.rest_client, name)
+
+
+@pytest.fixture
+def target_project_name(opik_client: opik.Opik) -> Iterator[str]:
+    """Create an ephemeral target project for the migration test."""
+    name = f"e2e-cli-migrate-target-{random_chars()}"
+    opik_client.rest_client.projects.create_project(name=name)
+    yield name
+    _best_effort_delete_project(opik_client.rest_client, name)
+
+
+def _best_effort_delete_project(rest_client: OpikApi, name: str) -> None:
+    try:
+        project_id = rest_client.projects.retrieve_project(name=name).id
+        rest_client.projects.delete_project_by_id(project_id)
+    except ApiError:
+        # Already gone (404) or insufficient permissions — either way
+        # cleanup is non-blocking; leave the project for the next run to
+        # garbage-collect or for a maintenance task to clean up.
+        pass
+
+
+# ---------------------------------------------------------------------------
+# CLI invocation
+# ---------------------------------------------------------------------------
+
+
+def run_migrate_cli(
+    args: List[str],
+    audit_log_path: Optional[str] = None,
+    extra_env: Optional[Dict[str, str]] = None,
+) -> subprocess.CompletedProcess:
+    """Invoke ``opik migrate`` via the installed CLI entrypoint.
+
+    Uses subprocess (not Click's ``CliRunner``) so the test exercises the
+    same code path real users hit — module import, Click group setup,
+    config-chain resolution, exit-code handling, stderr routing. Returns
+    the completed process so the caller can assert on ``returncode``,
+    ``stdout``, ``stderr``.
+
+    ``--audit-log`` is appended when provided. Tests typically write to a
+    tmp_path so the JSON can be re-read and asserted on.
+
+    ``extra_env`` is merged into the child process environment — the resume
+    E2E test uses it to put a test-only ``sitecustomize.py`` seam on
+    ``PYTHONPATH`` that injects a deterministic mid-cascade crash (the child
+    ``os._exit``s, so ``returncode`` is the hard-exit code, not a clean CLI
+    exit) and redirects the checkpoint dir into a tmp path.
+    """
+    cmd = [sys.executable, "-m", "opik.cli", "migrate"] + args
+    if audit_log_path is not None:
+        cmd.extend(["--audit-log", audit_log_path])
+    env = None
+    if extra_env is not None:
+        env = {**os.environ, **extra_env}
+    return subprocess.run(cmd, capture_output=True, text=True, env=env)
+
+
+# ---------------------------------------------------------------------------
+# Multi-version source seeding
+# ---------------------------------------------------------------------------
+
+
+def create_dataset_shell(
+    rest_client: OpikApi,
+    name: str,
+    project_name: str,
+    *,
+    type: Optional[str] = None,
+) -> str:
+    """Create an empty dataset (or test suite) and return its id.
+
+    ``type='evaluation_suite'`` produces a test suite (carries version-
+    level evaluators + execution_policy); omit for a plain dataset.
+    Caller is responsible for seeding versions via ``apply_changes``.
+    """
+    kwargs: Dict[str, Any] = {"name": name, "project_name": project_name}
+    if type is not None:
+        kwargs["type"] = type
+    rest_client.datasets.create_dataset(**kwargs)
+    ds = rest_client.datasets.get_dataset_by_identifier(
+        dataset_name=name, project_name=project_name
+    )
+    return ds.id
+
+
+def apply_changes(
+    rest_client: OpikApi,
+    dataset_id: str,
+    *,
+    base_version_id: Optional[str],
+    added_items: Optional[List[Dict[str, Any]]] = None,
+    edited_items: Optional[List[Dict[str, Any]]] = None,
+    deleted_ids: Optional[List[str]] = None,
+    change_description: Optional[str] = None,
+    suite_evaluators: Optional[List[Dict[str, Any]]] = None,
+    suite_execution_policy: Optional[Dict[str, int]] = None,
+    metadata: Optional[Dict[str, str]] = None,
+    user_tags: Optional[List[str]] = None,
+    override: bool = False,
+) -> str:
+    """Send ``apply_dataset_item_changes`` and return the new version id.
+
+    Thin wrapper over the raw REST endpoint that mirrors the BE schema's
+    field names. Used to seed multi-version source datasets for migration
+    tests. ``override=True`` is required for the first version (when
+    ``base_version_id=None``); see the BE validation in
+    ``DatasetItemService.applyDeltaChanges``.
+    """
+    request: Dict[str, Any] = {}
+    if change_description is not None:
+        request["change_description"] = change_description
+    if base_version_id is not None:
+        request["base_version"] = base_version_id
+    if added_items:
+        request["added_items"] = added_items
+    if edited_items:
+        request["edited_items"] = edited_items
+    if deleted_ids:
+        request["deleted_ids"] = deleted_ids
+    if suite_evaluators is not None:
+        request["evaluators"] = suite_evaluators
+    if suite_execution_policy is not None:
+        request["execution_policy"] = suite_execution_policy
+    if metadata is not None:
+        request["metadata"] = metadata
+    if user_tags is not None:
+        request["tags"] = user_tags
+    new_version = rest_client.datasets.apply_dataset_item_changes(
+        id=dataset_id, request=request, override=override
+    )
+    return new_version.id
+
+
+# ---------------------------------------------------------------------------
+# Verification helpers (read side)
+# ---------------------------------------------------------------------------
+
+
+def chronological_versions(
+    rest_client: OpikApi, dataset_id: str
+) -> List[dataset_version_public.DatasetVersionPublic]:
+    """Return every version of ``dataset_id`` oldest-first.
+
+    The REST endpoint returns newest-first; we paginate to exhaustion and
+    reverse so tests can iterate alongside source-version order for per-
+    version comparisons.
+    """
+    out: List[dataset_version_public.DatasetVersionPublic] = []
+    page = 1
+    while True:
+        resp = rest_client.datasets.list_dataset_versions(
+            id=dataset_id, page=page, size=100
+        )
+        if not resp.content:
+            break
+        out.extend(resp.content)
+        if len(resp.content) < 100:
+            break
+        page += 1
+    out.reverse()
+    return out
+
+
+def stream_items_wire(
+    rest_client: OpikApi,
+    *,
+    dataset_name: str,
+    project_name: Optional[str],
+    version_hash: Optional[str],
+) -> List[dataset_item_public.DatasetItemPublic]:
+    """Read items at ``version_hash`` via the raw REST stream + wire type.
+
+    The SDK helper ``rest_operations.stream_dataset_items`` drops per-item
+    ``tags`` during dataclass reconstruction, so tests that assert tag
+    fidelity must go through the wire type directly. Mirrors the same
+    approach used by ``cli/migrate/datasets/version_replay.py`` in
+    production.
+    """
+    raw_stream = rest_client.datasets.stream_dataset_items(
+        dataset_name=dataset_name,
+        project_name=project_name,
+        dataset_version=version_hash,
+    )
+    return rest_stream_parser.read_and_parse_stream(
+        stream=raw_stream,
+        item_class=dataset_item_public.DatasetItemPublic,
+    )
+
+
+def item_content_hash(item: dataset_item_public.DatasetItemPublic) -> str:
+    """Full-fidelity per-item hash covering every persisted user field.
+
+    Mirrors the production hash in ``cli/migrate/datasets/version_replay._content_hash_for``
+    so source-version vs target-version set-equality checks behave the
+    same way the migration code does internally (i.e. any field change
+    is treated as a content change).
+    """
+    content: Dict[str, Any] = {"data": dict(item.data) if item.data else {}}
+    if item.description is not None:
+        content["description"] = item.description
+    if item.tags is not None:
+        content["tags"] = sorted(item.tags)
+    if item.evaluators is not None:
+        content["evaluators"] = [
+            {"name": e.name, "type": e.type, "config": e.config}
+            for e in item.evaluators
+        ]
+    if item.execution_policy is not None:
+        content["execution_policy"] = {
+            "runs_per_item": item.execution_policy.runs_per_item,
+            "pass_threshold": item.execution_policy.pass_threshold,
+        }
+    if item.source is not None:
+        content["source"] = item.source
+    return hashlib.sha256(
+        json.dumps(content, sort_keys=True, default=str).encode()
+    ).hexdigest()
+
+
+def item_hashes(items: List[dataset_item_public.DatasetItemPublic]) -> Set[str]:
+    return {item_content_hash(it) for it in items}
+
+
+def display_order(
+    items: List[dataset_item_public.DatasetItemPublic], key: str = "q"
+) -> List[Optional[Any]]:
+    """Extract one ``data`` field per item in stream order (newest-first).
+
+    The stream's order *is* the UI's display order, so two versions' lists
+    of (e.g.) ``q`` values match iff the visible order matches.
+    """
+    return [(item.data.get(key) if item.data else None) for item in items]
+
+
+def normalize_evaluators(evals: Optional[List[Any]]) -> List[Dict[str, Any]]:
+    """Compare-friendly form of a suite evaluator list.
+
+    Strips wire-type wrapping and sorts by name so identical
+    configurations hash equal regardless of how the BE happened to
+    serialise them.
+    """
+    if not evals:
+        return []
+    return sorted(
+        ({"name": e.name, "type": e.type, "config": e.config} for e in evals),
+        key=lambda d: d["name"],
+    )
+
+
+def normalize_policy(pol: Any) -> Optional[Dict[str, int]]:
+    """Compare-friendly form of an execution_policy."""
+    if pol is None:
+        return None
+    return {
+        "runs_per_item": pol.runs_per_item,
+        "pass_threshold": pol.pass_threshold,
+    }
+
+
+def strip_be_managed_version_tags(
+    tags: Optional[List[str]],
+) -> List[str]:
+    """Drop the BE-managed ``'latest'`` marker so source/target tag lists compare equal.
+
+    The BE auto-injects ``'latest'`` on the newest version of any dataset
+    on read; the migration code filters it out before forwarding to avoid
+    409 conflicts. Tests strip it on both sides for the same reason.
+    """
+    return sorted(t for t in (tags or []) if t != "latest")
+
+
+# ---------------------------------------------------------------------------
+# Cascade seeding (Slice 3: experiment + traces + spans)
+#
+# Tests seed an experiment + its trace data directly via REST. We do this
+# rather than going through ``opik.evaluate`` because the BE-side wire
+# shapes are what the cascade reads from, and we want full control over
+# trace ids, span tree topology, and feedback score payloads.
+# ---------------------------------------------------------------------------
+
+
+def seed_experiment_with_trace_tree(
+    rest_client: OpikApi,
+    *,
+    experiment_name: str,
+    dataset_name: str,
+    dataset_id: str,
+    dataset_version_id: Optional[str],
+    project_name: str,
+    item_ids: List[str],
+    experiment_config: Optional[Dict[str, Any]] = None,
+    experiment_type: str = "regular",
+    evaluation_method: str = "dataset",
+    experiment_tags: Optional[List[str]] = None,
+    spans_per_trace: int = 2,
+    feedback_scores_per_trace: Optional[List[Dict[str, Any]]] = None,
+    per_item_extras: Optional[List[Dict[str, Any]]] = None,
+    optimization_id: Optional[str] = None,
+    trace_environment: Optional[str] = None,
+    span_environment: Optional[str] = None,
+    thread_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Create a source experiment + one trace per ``item_id`` + a small span
+    tree per trace, then attach everything via ``create_experiment_items``.
+
+    Returns a dict the cascade tests assert against:
+
+      {
+        "experiment_id": str,
+        "trace_ids": [str, ...],          # one per item_id, same order
+        "span_ids_by_trace": {trace_id: [root_span_id, child_span_id, ...]},
+        "feedback_scores_by_trace": {trace_id: [score_dicts...]},
+      }
+
+    ``spans_per_trace`` controls the tree size; ``spans_per_trace >= 2``
+    produces a root + child(ren) layout so the cascade has to remap
+    parent_span_id. We deliberately do NOT use ``opik.evaluate`` here -- we
+    want the wire shape, deterministic ids, and to assert on it without
+    flush/streamer timing concerns.
+
+    ``trace_environment`` / ``span_environment`` stamp the ClickHouse
+    ``environment`` column on every seeded trace / span; ``thread_id``
+    groups the traces into one thread so the cascade's env preservation
+    (OPIK-6695) can be round-trip asserted on traces, spans, and the
+    BE-materialized thread row.
+    """
+    import datetime as dt
+    import opik.id_helpers as id_helpers_module
+    from opik.rest_api.types.experiment_item import ExperimentItem
+    from opik.rest_api.types.feedback_score_batch_item import (
+        FeedbackScoreBatchItem,
+    )
+    from opik.rest_api.types.span_write import SpanWrite
+    from opik.rest_api.types.trace_write import TraceWrite
+
+    if spans_per_trace < 1:
+        raise ValueError("spans_per_trace must be >= 1")
+
+    now = dt.datetime.now(dt.timezone.utc)
+
+    trace_ids: List[str] = []
+    span_ids_by_trace: Dict[str, List[str]] = {}
+    feedback_scores_by_trace: Dict[str, List[Dict[str, Any]]] = {}
+
+    trace_writes: List[TraceWrite] = []
+    span_writes: List[SpanWrite] = []
+    feedback_batch: List[FeedbackScoreBatchItem] = []
+
+    for index, item_id in enumerate(item_ids):
+        trace_id = id_helpers_module.generate_id()
+        trace_ids.append(trace_id)
+        trace_writes.append(
+            TraceWrite(
+                id=trace_id,
+                project_name=project_name,
+                name=f"task-{index}",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"item": item_id},
+                output={"answer": f"output-{index}"},
+                metadata={"item_id": item_id},
+                tags=["e2e-cascade"],
+                thread_id=thread_id,
+                environment=trace_environment,
+            )
+        )
+
+        # Span tree: root + (spans_per_trace - 1) children of the root.
+        # Children all parent on the root so the cascade has to remap
+        # parent_span_id at least once.
+        root_span_id = id_helpers_module.generate_id()
+        span_ids_by_trace[trace_id] = [root_span_id]
+        span_writes.append(
+            SpanWrite(
+                id=root_span_id,
+                project_name=project_name,
+                trace_id=trace_id,
+                parent_span_id=None,
+                name=f"root-{index}",
+                type="general",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"item": item_id},
+                output={"answer": f"output-{index}"},
+                environment=span_environment,
+            )
+        )
+        for child_index in range(spans_per_trace - 1):
+            child_span_id = id_helpers_module.generate_id()
+            span_ids_by_trace[trace_id].append(child_span_id)
+            span_writes.append(
+                SpanWrite(
+                    id=child_span_id,
+                    project_name=project_name,
+                    trace_id=trace_id,
+                    parent_span_id=root_span_id,
+                    name=f"llm-call-{index}-{child_index}",
+                    type="llm",
+                    start_time=now + dt.timedelta(milliseconds=1),
+                    end_time=now + dt.timedelta(milliseconds=9),
+                    input={"prompt": "..."},
+                    output={"completion": f"output-{index}"},
+                    model="gpt-mock",
+                    provider="mock",
+                    usage={"prompt_tokens": 5, "completion_tokens": 10},
+                    environment=span_environment,
+                )
+            )
+
+        # Optional: attach feedback scores to the trace.
+        if feedback_scores_per_trace:
+            scores_for_this_trace: List[Dict[str, Any]] = []
+            for score in feedback_scores_per_trace:
+                feedback_batch.append(
+                    FeedbackScoreBatchItem(
+                        id=trace_id,
+                        project_name=project_name,
+                        name=score["name"],
+                        value=score["value"],
+                        reason=score.get("reason"),
+                        source="sdk",
+                    )
+                )
+                scores_for_this_trace.append(score)
+            feedback_scores_by_trace[trace_id] = scores_for_this_trace
+
+    rest_client.traces.create_traces(traces=trace_writes)
+    rest_client.spans.create_spans(spans=span_writes)
+    if feedback_batch:
+        rest_client.traces.score_batch_of_traces(scores=feedback_batch)
+
+    # Create the experiment, then attach experiment items wiring item_id
+    # to trace_id 1:1.
+    import opik.id_helpers as _id_helpers
+
+    new_experiment_id = _id_helpers.generate_id()
+    create_experiment_kwargs: Dict[str, Any] = {
+        "id": new_experiment_id,
+        "name": experiment_name,
+        "dataset_name": dataset_name,
+        "type": experiment_type,
+        "evaluation_method": evaluation_method,
+        "tags": experiment_tags,
+        "metadata": experiment_config,
+        "dataset_version_id": dataset_version_id,
+        "project_name": project_name,
+    }
+    if optimization_id is not None:
+        create_experiment_kwargs["optimization_id"] = optimization_id
+    rest_client.experiments.create_experiment(**create_experiment_kwargs)
+
+    extras_list = per_item_extras or [{} for _ in item_ids]
+    if len(extras_list) != len(item_ids):
+        raise ValueError("per_item_extras must have the same length as item_ids")
+
+    # ``assertion_results`` are persisted via the dedicated
+    # ``assertion_results.store_assertions_batch(entity_type='TRACE', ...)``
+    # endpoint -- the ``ExperimentItem.assertion_results`` field is dropped
+    # silently on the BE Write view (it's READ-ONLY on the Compare view,
+    # computed from the underlying assertion-results entity table). Same
+    # for the other per-item fidelity fields like input/output -- those
+    # are BE-computed read aggregates.
+    #
+    # The seed builds a separate assertion-batch from each item's extras
+    # before constructing the ExperimentItem write (which only carries the
+    # FK fields). This mirrors how the cascade itself writes assertions.
+    from opik.rest_api.types.assertion_result_batch_item import (
+        AssertionResultBatchItem,
+    )
+
+    assertion_batch: List[AssertionResultBatchItem] = []
+    assertion_results_by_trace: Dict[str, List[Dict[str, Any]]] = {}
+    experiment_items_to_create: List[ExperimentItem] = []
+    for item_id, trace_id, extras in zip(item_ids, trace_ids, extras_list):
+        per_item_assertions = extras.get("assertion_results") or []
+        for ar in per_item_assertions:
+            value = (
+                ar.get("value") if isinstance(ar, dict) else getattr(ar, "value", None)
+            )
+            passed = (
+                ar.get("passed")
+                if isinstance(ar, dict)
+                else getattr(ar, "passed", None)
+            )
+            reason = (
+                ar.get("reason")
+                if isinstance(ar, dict)
+                else getattr(ar, "reason", None)
+            )
+            if value is None or passed is None:
+                continue
+            assertion_batch.append(
+                AssertionResultBatchItem(
+                    entity_id=trace_id,
+                    project_name=project_name,
+                    name=value,
+                    status="passed" if passed else "failed",
+                    reason=reason,
+                    source="sdk",
+                )
+            )
+            assertion_results_by_trace.setdefault(trace_id, []).append(
+                {"value": value, "passed": passed, "reason": reason}
+            )
+
+        # The remaining extras are READ-ONLY on the BE; we don't write
+        # them. Forwarding them on the ExperimentItem create payload would
+        # be silently dropped (BE Write view doesn't include them).
+        experiment_items_to_create.append(
+            ExperimentItem(
+                id=_id_helpers.generate_id(),
+                experiment_id=new_experiment_id,
+                dataset_item_id=item_id,
+                trace_id=trace_id,
+            )
+        )
+
+    rest_client.experiments.create_experiment_items(
+        experiment_items=experiment_items_to_create
+    )
+
+    if assertion_batch:
+        rest_client.assertion_results.store_assertions_batch(
+            entity_type="TRACE",
+            assertion_results=assertion_batch,
+        )
+
+    return {
+        "experiment_id": new_experiment_id,
+        "trace_ids": trace_ids,
+        "span_ids_by_trace": span_ids_by_trace,
+        "feedback_scores_by_trace": feedback_scores_by_trace,
+        "assertion_results_by_trace": assertion_results_by_trace,
+    }
+
+
+def find_destination_experiment(
+    rest_client: OpikApi,
+    *,
+    destination_dataset_id: str,
+    experiment_name: str,
+) -> Any:
+    """Locate the cascaded experiment at the destination by name + dataset.
+
+    Returns the ``ExperimentPublic``. Raises if zero or multiple match;
+    the cascade is supposed to recreate one experiment per source
+    experiment, so neither outcome is silently acceptable.
+    """
+    page = rest_client.experiments.find_experiments(
+        dataset_id=destination_dataset_id,
+        page=1,
+        size=100,
+        name=experiment_name,
+    )
+    matched = [e for e in (page.content or []) if e.name == experiment_name]
+    if len(matched) != 1:
+        raise AssertionError(
+            f"expected exactly one destination experiment named "
+            f"{experiment_name!r} under dataset {destination_dataset_id}, "
+            f"got {len(matched)}"
+        )
+    return matched[0]
+
+
+def destination_experiment_items(
+    rest_client: OpikApi,
+    *,
+    experiment_id: str,
+    dataset_id: str,
+) -> List[Any]:
+    """Materialise the destination experiment's items via the Compare view.
+
+    The cascade's source-side read uses
+    ``datasets.find_dataset_items_with_experiment_items`` because only the
+    Compare view surfaces ``assertion_results`` / ``feedback_scores`` /
+    ``input`` / ``output``. We use the same endpoint for destination
+    verification so tests can assert on those fields directly (the slim
+    ``stream_experiment_items`` Public view drops them).
+
+    Returns a flat list of ``ExperimentItemCompare`` -- one per source
+    experiment item.
+    """
+    experiment_ids_filter = json.dumps([experiment_id])
+    collected: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.datasets.find_dataset_items_with_experiment_items(
+            id=dataset_id,
+            experiment_ids=experiment_ids_filter,
+            page=page,
+            size=100,
+        )
+        content = resp.content or []
+        if not content:
+            break
+        for ds_item in content:
+            for ei in ds_item.experiment_items or []:
+                if ei.experiment_id == experiment_id:
+                    collected.append(ei)
+        if len(content) < 100:
+            break
+        page += 1
+    return collected
+
+
+def destination_spans_for_trace(
+    rest_client: OpikApi, *, trace_id: str, project_name: str
+) -> List[Any]:
+    """Read all destination spans for one destination trace, paginating.
+
+    ``get_spans_by_project`` requires ``project_name`` (or ``project_id``)
+    on the request; without it the BE 400s. We pass the destination
+    project name explicitly.
+    """
+    out: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.spans.get_spans_by_project(
+            project_name=project_name,
+            trace_id=trace_id,
+            page=page,
+            size=100,
+        )
+        if not resp.content:
+            break
+        out.extend(resp.content)
+        if len(resp.content) < 100:
+            break
+        page += 1
+    return out
+
+
+def destination_feedback_scores_for_trace(
+    rest_client: OpikApi, *, trace_id: str
+) -> List[Any]:
+    """Read feedback scores on a destination trace.
+
+    The trace's ``feedback_scores`` field on read is the authoritative
+    source -- the cascade copies them implicitly because trace metadata
+    isn't the only place they live (per-trace ``feedback_scores`` table).
+    Today's cascade does NOT explicitly re-emit feedback scores; this
+    helper lets a test assert that as an explicit known-gap or as
+    "preserved if and only if the cascade adds the copy".
+    """
+    trace = rest_client.traces.get_trace_by_id(id=trace_id)
+    return list(trace.feedback_scores or [])

sdks/python/tests/e2e/cli/test_migrate_dataset_cross_project_cascade_e2e.py

@@ -0,0 +1,318 @@
+"""End-to-end test for ``opik migrate dataset`` with a cross-project experiment.
+
+``opik.evaluate`` co-locates an experiment's traces in one project, but the
+BE permits ``experiment_items`` rows that point at traces living in a
+*different* project from the experiment's own project. The BE populates
+``experiment_items.project_id`` from ``traces.project_id`` at write time
+(``ExperimentItemService.populateProjectIdFromTraces``), so the rows are
+the authoritative source of truth for "which projects do this experiment's
+traces actually live in".
+
+This test pins the cascade's cross-project contract: regardless of how
+many distinct source projects the experiment's traces are spread across,
+the migration funnels every one of them into the single ``--to-project``
+destination.
+
+Shape (three distinct source projects per experiment, plus the target):
+
+  1. Source dataset lives in project A (``source_project_name``)
+  2. ``seed_experiment_with_trace_tree`` seeds the experiment + 2 traces in A
+  3. We mint a trace + span in project B (``cross_project_name_b``) and
+     another in project C (``cross_project_name_c``), then append two
+     ``experiment_items`` rows pointing the same experiment at them. The
+     experiment now has traces spread across A, B, and C — the user only
+     "sees" A (the experiment's own project) but the cascade must reach
+     into B and C as well.
+  4. Run ``opik migrate dataset ... --to-project=<destination>``
+  5. Assert: destination experiment has all 4 items, every destination
+     trace lives under the destination project, and the source trace ids
+     (across all three source projects) are not reused.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+from pathlib import Path
+from typing import Any, Iterator, List
+
+import pytest
+
+import opik
+import opik.id_helpers as id_helpers_module
+from opik.rest_api.core.api_error import ApiError
+from opik.rest_api.types.experiment_item import ExperimentItem
+from opik.rest_api.types.span_write import SpanWrite
+from opik.rest_api.types.trace_write import TraceWrite
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .. import verifiers
+from .conftest import (
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_spans_for_trace,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _make_cross_project(opik_client: opik.Opik, label: str) -> Iterator[str]:
+    name = f"e2e-cli-migrate-cross-{label}-{random_chars()}"
+    rest = opik_client.rest_client
+    rest.projects.create_project(name=name)
+    yield name
+    try:
+        project_id = rest.projects.retrieve_project(name=name).id
+        rest.projects.delete_project_by_id(project_id)
+    except ApiError:
+        pass
+
+
+@pytest.fixture
+def cross_project_name_b(opik_client: opik.Opik) -> Iterator[str]:
+    yield from _make_cross_project(opik_client, "b")
+
+
+@pytest.fixture
+def cross_project_name_c(opik_client: opik.Opik) -> Iterator[str]:
+    yield from _make_cross_project(opik_client, "c")
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-cross-{random_chars()}"
+
+
+def _mint_cross_project_trace(
+    rest: Any,
+    *,
+    project_name: str,
+    label: str,
+) -> str:
+    """Create one trace + one root span in ``project_name`` and return the trace id."""
+    now = dt.datetime.now(dt.timezone.utc)
+    trace_id = id_helpers_module.generate_id()
+    span_id = id_helpers_module.generate_id()
+    rest.traces.create_traces(
+        traces=[
+            TraceWrite(
+                id=trace_id,
+                project_name=project_name,
+                name=f"task-cross-{label}",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"q": f"Q-cross-{label}"},
+                output={"answer": f"from-{label}"},
+                tags=["e2e-cross"],
+            )
+        ]
+    )
+    rest.spans.create_spans(
+        spans=[
+            SpanWrite(
+                id=span_id,
+                project_name=project_name,
+                trace_id=trace_id,
+                parent_span_id=None,
+                name=f"root-{label}",
+                type="general",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"q": f"Q-cross-{label}"},
+                output={"answer": f"from-{label}"},
+            )
+        ]
+    )
+    return trace_id
+
+
+def test_migrate_dataset__cross_project_experiment__all_traces_land_in_target(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    cross_project_name_b: str,
+    cross_project_name_c: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset in project A ──
+    # All items (same-project + both cross-project) are added to a single
+    # v1 so the experiment's items all resolve under the same dataset
+    # version -- mixing experiment_items rows against different dataset
+    # versions causes the Compare endpoint to surface incomplete rows
+    # (missing ``source``). That's a known corner case unrelated to the
+    # cross-project plumbing under test, so we sidestep it by seeding a
+    # single version.
+    from opik import id_helpers
+    from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+    source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+            DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+            DatasetItemWrite(source="manual", data={"q": "Q-cross-b", "a": "A-b"}),
+            DatasetItemWrite(source="manual", data={"q": "Q-cross-c", "a": "A-c"}),
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[0]
+
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+    same_project_item_ids = [by_q["Q1"], by_q["Q2"]]
+    cross_b_item_id = by_q["Q-cross-b"]
+    cross_c_item_id = by_q["Q-cross-c"]
+    assert all(same_project_item_ids) and cross_b_item_id and cross_c_item_id
+
+    # ── Seed the experiment with same-project traces in A ──
+    experiment_name = f"e2e-cross-{random_chars()}"
+    cascade_seed = seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=experiment_name,
+        dataset_name=dataset_name,
+        dataset_id=source_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=same_project_item_ids,
+        spans_per_trace=2,
+    )
+    source_experiment_id = cascade_seed["experiment_id"]
+    same_project_trace_ids = cascade_seed["trace_ids"]
+
+    # ── Mint cross-project traces in two distinct projects (B and C) ──
+    # The experiment now has traces in 3 projects total: A (same-project,
+    # 2 traces), B (1 trace), and C (1 trace). The user only "sees" A as
+    # the experiment's project; the cascade has to discover B and C via
+    # ``streamExperimentItems``' per-item ``project_id`` field and issue a
+    # ``search_traces`` / ``search_spans`` against each.
+    cross_b_trace_id = _mint_cross_project_trace(
+        rest, project_name=cross_project_name_b, label="b"
+    )
+    cross_c_trace_id = _mint_cross_project_trace(
+        rest, project_name=cross_project_name_c, label="c"
+    )
+
+    rest.experiments.create_experiment_items(
+        experiment_items=[
+            ExperimentItem(
+                id=id_helpers_module.generate_id(),
+                experiment_id=source_experiment_id,
+                dataset_item_id=cross_b_item_id,
+                trace_id=cross_b_trace_id,
+            ),
+            ExperimentItem(
+                id=id_helpers_module.generate_id(),
+                experiment_id=source_experiment_id,
+                dataset_item_id=cross_c_item_id,
+                trace_id=cross_c_trace_id,
+            ),
+        ]
+    )
+
+    # ── Run the migration to the destination project ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify destination ─────────────────────────────────────────────
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=experiment_name,
+    )
+
+    # All 4 source experiment items should round-trip (2 same-project +
+    # 1 from project B + 1 from project C).
+    expected_count = len(same_project_trace_ids) + 2
+    dest_items = destination_experiment_items(
+        rest, experiment_id=dest_exp.id, dataset_id=dest_dataset.id
+    )
+    assert len(dest_items) == expected_count, (
+        f"expected {expected_count} destination experiment items "
+        f"(same-project + 2 cross-project), got {len(dest_items)}"
+    )
+
+    # Every destination trace must live in the target project — including
+    # the ones that originally lived in project B and project C. This is
+    # the contract the cross-project plumbing protects: without
+    # ``_discover_trace_projects``, the search_traces call would scope to
+    # the experiment's own project (A) only and both cross-project traces
+    # would be silently dropped, leaving 2 fewer items at the destination.
+    dest_trace_ids: List[str] = []
+    for it in dest_items:
+        assert it.trace_id is not None, "destination experiment item missing trace_id"
+        dest_trace_ids.append(it.trace_id)
+        verifiers.verify_trace(
+            opik_client=opik_client,
+            trace_id=it.trace_id,
+            project_name=target_project_name,
+        )
+
+    # Per-span fidelity per source project: spans live under the same project
+    # as their parent trace and are discovered+read by the cascade in the
+    # same per-project loop as traces. So the cross-project safety we just
+    # verified for traces only holds if the per-project ``search_spans``
+    # loop also fired for B and C; if it didn't, the cross-project traces
+    # would land at the destination but with zero spans attached.
+    #
+    # Trace ``name`` is the stable handle for classifying which source
+    # project a destination trace came from:
+    #   * ``task-N``        -> same-project (A), seeded with root + 1 LLM child
+    #   * ``task-cross-b``  -> cross-project from B, seeded with 1 root span
+    #   * ``task-cross-c``  -> cross-project from C, seeded with 1 root span
+    expected_spans_by_trace_name = {"task-cross-b": 1, "task-cross-c": 1}
+    for i in range(len(same_project_trace_ids)):
+        expected_spans_by_trace_name[f"task-{i}"] = 2  # root + 1 LLM child
+
+    for it in dest_items:
+        dest_trace = rest.traces.get_trace_by_id(id=it.trace_id)
+        assert dest_trace.name in expected_spans_by_trace_name, (
+            f"unexpected destination trace name {dest_trace.name!r} -- the "
+            f"cross-project trace naming contract changed and this test "
+            f"doesn't know which source project to attribute it to"
+        )
+        dest_spans = destination_spans_for_trace(
+            rest, trace_id=it.trace_id, project_name=target_project_name
+        )
+        expected = expected_spans_by_trace_name[dest_trace.name]
+        assert len(dest_spans) == expected, (
+            f"destination trace {dest_trace.name!r} should have {expected} "
+            f"span(s) at the destination, got {len(dest_spans)} -- if 0 for "
+            f"a cross-project trace, the per-project search_spans loop didn't "
+            f"fire for that source project"
+        )
+
+    # Fresh destination ids — migrate is copy-not-move. No source trace id
+    # (from any of the 3 source projects) may be reused at the destination.
+    source_trace_ids = set(same_project_trace_ids) | {
+        cross_b_trace_id,
+        cross_c_trace_id,
+    }
+    overlap = source_trace_ids & set(dest_trace_ids)
+    assert not overlap, (
+        f"destination trace ids must be fresh, but found source ids reused: {overlap}"
+    )

sdks/python/tests/e2e/cli/test_migrate_dataset_e2e.py

@@ -0,0 +1,483 @@
+"""End-to-end tests for ``opik migrate dataset`` against a real Opik backend.
+
+Covers the plain-dataset path: full version replay, plus the experiment +
+trace + span cascade that rides along with the dataset. The test-suite
+path lives in ``test_migrate_test_suite_e2e.py``.
+
+Each test:
+
+1. Seeds a multi-version source dataset directly via the REST API (so we
+   control every per-version delta the migration has to replay) and,
+   where the test calls for it, an experiment + traces + spans attached
+   to one of the source versions
+2. Runs ``opik migrate dataset`` as a subprocess so the actual CLI
+   entrypoint, Click group, and exit-code handling are exercised
+3. Reads back the target via the raw REST stream + wire type (the SDK
+   helper drops per-item tags) and asserts per-version content +
+   display-order fidelity, plus -- where relevant -- destination
+   experiment + trace + span fidelity and FK remapping
+
+Shared helpers live in ``conftest.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+from opik import synchronization
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from ._cascade_comparison import compare_cascade
+from .conftest import (
+    apply_changes,
+    chronological_versions,
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_feedback_scores_for_trace,
+    destination_spans_for_trace,
+    display_order,
+    find_destination_experiment,
+    item_hashes,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+# Per ``sdks/python/AGENTS.md``: every e2e module sources PROJECT_NAME from
+# ``generate_project_name("e2e", __name__)`` so backend project names are
+# isolated per test module + the autouse ``configure_e2e_tests_env`` fixture
+# can patch ``OPIK_PROJECT_NAME`` to match.
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-{random_chars()}"
+
+
+class TestMigrateDatasetVersionReplay:
+    """Default ``opik migrate dataset`` flow: full version-history replay.
+
+    Pin the slice 2 contract: target version count == source version
+    count, per-version content set-equal under hash, display order
+    preserved at every version.
+    """
+
+    def test_three_version_dataset_with_mixed_deltas_round_trips(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        dataset_name: str,
+        tmp_path: Path,
+    ) -> None:
+        rest = opik_client.rest_client
+
+        # Seed: 3 versions exercising every delta kind.
+        # v1: adds Q1, Q2, Q3.
+        # v2: edits Q1 (data change), adds Q4.
+        # v3: edits Q3 (data change), deletes Q2, adds Q5.
+        source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+
+        # v1 of a plain dataset is created via the REST insert path (mirrors
+        # what ``Dataset.insert`` does at higher level). Use create_or_update
+        # so we get exactly one BE version with all three items.
+        from opik import id_helpers
+        from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+        rest.datasets.create_or_update_dataset_items(
+            dataset_id=source_id,
+            items=[
+                DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+                DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+                DatasetItemWrite(source="manual", data={"q": "Q3", "a": "A3"}),
+            ],
+            batch_group_id=id_helpers.generate_id(),
+        )
+        v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[
+            0
+        ]
+        v1_items = stream_items_wire(
+            rest,
+            dataset_name=dataset_name,
+            project_name=source_project_name,
+            version_hash=v1.version_hash,
+        )
+        by_q = {item.data["q"]: item for item in v1_items if item.data}
+
+        # v2: edit Q1's answer + add Q4.
+        v2_id = apply_changes(
+            rest,
+            source_id,
+            base_version_id=v1.id,
+            edited_items=[
+                {"id": by_q["Q1"].id, "data": {"q": "Q1", "a": "A1-EDITED"}},
+            ],
+            added_items=[{"data": {"q": "Q4", "a": "A4"}, "source": "manual"}],
+            change_description="v2 — edit Q1, add Q4",
+        )
+
+        # v3: edit Q3, delete Q2, add Q5.
+        apply_changes(
+            rest,
+            source_id,
+            base_version_id=v2_id,
+            edited_items=[
+                {"id": by_q["Q3"].id, "data": {"q": "Q3", "a": "A3-EDITED"}},
+            ],
+            deleted_ids=[by_q["Q2"].id],
+            added_items=[{"data": {"q": "Q5", "a": "A5"}, "source": "manual"}],
+            change_description="v3 — delete Q2, edit Q3, add Q5",
+        )
+
+        # Snapshot source expectations per version.
+        src_versions = chronological_versions(rest, source_id)
+        assert len(src_versions) == 3
+        expected_hashes = []
+        expected_orders = []
+        for v in src_versions:
+            items = stream_items_wire(
+                rest,
+                dataset_name=dataset_name,
+                project_name=source_project_name,
+                version_hash=v.version_hash,
+            )
+            expected_hashes.append(item_hashes(items))
+            expected_orders.append(display_order(items))
+
+        # ── Seed an experiment on v1 items so the cascade has something to
+        # round-trip. Regular-dataset experiments carry per-trace feedback
+        # scores (test suites carry assertion_results -- covered in
+        # test_migrate_test_suite_e2e.py). Each item gets one trace with a
+        # root + 1 LLM child span and a feedback score on the trace. ──
+        experiment_name = f"e2e-exp-{random_chars()}"
+        v1_item_ids = [by_q["Q1"].id, by_q["Q2"].id, by_q["Q3"].id]
+        cascade_seed = seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=experiment_name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=v1_item_ids,
+            experiment_config={"runner": "e2e-cascade-test"},
+            experiment_tags=["e2e", "cascade"],
+            spans_per_trace=2,  # root + 1 LLM child -> exercises parent_span_id remap
+            feedback_scores_per_trace=[
+                {"name": "correctness", "value": 0.9, "reason": "matches reference"},
+                {"name": "latency_p95", "value": 230.5},
+            ],
+        )
+
+        # Run the migration.
+        audit_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            [
+                "dataset",
+                dataset_name,
+                "--to-project",
+                target_project_name,
+            ],
+            audit_log_path=str(audit_path),
+        )
+        assert result.returncode == 0, result.stdout + result.stderr
+
+        # Verify target: same version count, per-version content set-equal
+        # under hash, display order matches at every version.
+        target = rest.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name, project_name=target_project_name
+        )
+        tgt_versions = chronological_versions(rest, target.id)
+        assert len(tgt_versions) == len(src_versions), (
+            f"target version count {len(tgt_versions)} != source {len(src_versions)} "
+            "— Slice 2 contract requires N=N"
+        )
+
+        for src_v, tgt_v, exp_hashes, exp_order in zip(
+            src_versions, tgt_versions, expected_hashes, expected_orders
+        ):
+            items = stream_items_wire(
+                rest,
+                dataset_name=dataset_name,
+                project_name=target_project_name,
+                version_hash=tgt_v.version_hash,
+            )
+            actual_hashes = item_hashes(items)
+            actual_order = display_order(items)
+            assert actual_hashes == exp_hashes, (
+                f"version {tgt_v.version_name}: target items don't match source. "
+                f"Missing on target: {exp_hashes - actual_hashes}; "
+                f"extra on target: {actual_hashes - exp_hashes}"
+            )
+            assert actual_order == exp_order, (
+                f"version {tgt_v.version_name}: display order diverged "
+                f"(source: {exp_order}, target: {actual_order})"
+            )
+
+        # Audit log records one per-version entry per replayed source version.
+        audit = json.loads(audit_path.read_text())
+        per_version_records = [
+            a for a in audit["actions"] if a["type"] == "replay_dataset_version"
+        ]
+        assert len(per_version_records) == len(src_versions)
+        # Per-version deltas: v1=(3 adds), v2=(1 add, 1 mod), v3=(1 add, 1 mod, 1 del).
+        assert (
+            per_version_records[0]["items_added"],
+            per_version_records[0]["items_modified"],
+            per_version_records[0]["items_deleted"],
+        ) == (3, 0, 0)
+        assert (
+            per_version_records[1]["items_added"],
+            per_version_records[1]["items_modified"],
+            per_version_records[1]["items_deleted"],
+        ) == (1, 1, 0)
+        assert (
+            per_version_records[2]["items_added"],
+            per_version_records[2]["items_modified"],
+            per_version_records[2]["items_deleted"],
+        ) == (1, 1, 1)
+
+        # ── Cascade fidelity ──
+        # The destination project should now have a copy of the source
+        # experiment with: a remapped dataset_version_id, fresh item ids
+        # carrying remapped trace ids, traces+spans landing under the
+        # destination project, feedback scores re-emitted on the destination
+        # traces, and per-item write-side fidelity (input/output) preserved.
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=target.id,
+            experiment_name=experiment_name,
+        )
+        # FKs remapped.
+        assert dest_exp.id != cascade_seed["experiment_id"]
+        assert dest_exp.dataset_id == target.id
+        # The destination experiment must reference one of the target
+        # versions (the cascade picks the remap of v1).
+        target_version_ids = {v.id for v in tgt_versions}
+        assert dest_exp.dataset_version_id in target_version_ids
+
+        # Items: one per source item, with FRESH trace ids (disjoint from
+        # source). Per-item input/output/usage/cost are READ-ONLY on the BE
+        # (computed/aggregated from the underlying trace + span entities);
+        # we assert the trace + span fidelity below instead.
+        dest_items = destination_experiment_items(
+            rest,
+            experiment_id=dest_exp.id,
+            dataset_id=target.id,
+        )
+        assert len(dest_items) == len(v1_item_ids)
+        dest_trace_ids = {it.trace_id for it in dest_items}
+        assert dest_trace_ids.isdisjoint(set(cascade_seed["trace_ids"])), (
+            "destination experiment items should reference new trace ids, "
+            "not the source's"
+        )
+
+        # Each destination trace exists under the target project and has the
+        # same span shape as the source (root + 1 child = 2 spans).
+        for new_trace_id in dest_trace_ids:
+            dest_spans = destination_spans_for_trace(
+                rest,
+                trace_id=new_trace_id,
+                project_name=target_project_name,
+            )
+            assert len(dest_spans) == 2, (
+                f"trace {new_trace_id} should have 2 spans (root + child), "
+                f"got {len(dest_spans)}"
+            )
+            # Topological remap: exactly one root (parent_span_id=None),
+            # the other span points at the root via parent_span_id.
+            roots = [s for s in dest_spans if s.parent_span_id is None]
+            assert len(roots) == 1, f"trace {new_trace_id} should have one root span"
+            children = [s for s in dest_spans if s.parent_span_id is not None]
+            assert all(c.parent_span_id == roots[0].id for c in children), (
+                f"trace {new_trace_id} child spans should remap parent_span_id "
+                "to the new root id"
+            )
+
+            # Trace-level feedback scores re-emitted on the destination trace.
+            dest_scores = destination_feedback_scores_for_trace(
+                rest, trace_id=new_trace_id
+            )
+            score_names = {s.name for s in dest_scores}
+            assert score_names == {"correctness", "latency_p95"}, (
+                f"trace {new_trace_id}: expected feedback score names "
+                f"{{'correctness', 'latency_p95'}}, got {score_names}"
+            )
+
+        # ── Deep-equal source vs. destination ──
+        # Verify field-by-field that experiment + items + traces + spans
+        # round-trip the cascade modulo remapped IDs. Pairing strategy:
+        # both sides sorted by trace ``name`` (assigned by the seed as
+        # "task-0", "task-1", "task-2" and carried verbatim through the
+        # cascade), guaranteeing stable positional correspondence.
+        src_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=source_id,
+            experiment_name=experiment_name,
+        )
+        src_items_compare = destination_experiment_items(
+            rest,
+            experiment_id=cascade_seed["experiment_id"],
+            dataset_id=source_id,
+        )
+        # Sort both sides by trace name for stable pairing. Build a
+        # trace_id -> name map by reading each trace once.
+        src_trace_names = {
+            it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+            for it in src_items_compare
+        }
+        dst_trace_names = {
+            it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+            for it in dest_items
+        }
+        src_items_compare.sort(key=lambda it: src_trace_names[it.trace_id])
+        dest_items_sorted = sorted(
+            dest_items, key=lambda it: dst_trace_names[it.trace_id]
+        )
+        src_trace_ids_sorted = [it.trace_id for it in src_items_compare]
+        dst_trace_ids_sorted = [it.trace_id for it in dest_items_sorted]
+
+        compare_cascade(
+            rest_client=rest,
+            source_experiment=src_exp,
+            destination_experiment=dest_exp,
+            source_item_ids=v1_item_ids,
+            destination_item_ids=[it.dataset_item_id for it in dest_items_sorted],
+            source_trace_ids=src_trace_ids_sorted,
+            destination_trace_ids=dst_trace_ids_sorted,
+            source_items_compare=src_items_compare,
+            destination_items_compare=dest_items_sorted,
+        )
+
+
+class TestMigrateDatasetEnvironmentPreservation:
+    """OPIK-6695: the cascade must preserve the ``environment`` column on
+    traces, spans, and the BE-materialized ``trace_threads`` row.
+
+    Pre-2026-05-07 rows default to ``''`` in ClickHouse and replay
+    trivially; the regression this guards is the post-migration reset of a
+    non-empty ``environment`` to ``''`` because the re-emit payload didn't
+    carry the field.
+    """
+
+    def test_environment_round_trips_on_traces_spans_and_threads(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        dataset_name: str,
+        tmp_path: Path,
+    ) -> None:
+        from opik import id_helpers
+        from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+        rest = opik_client.rest_client
+
+        # Single-version dataset with two items -> two cascaded traces.
+        source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+        rest.datasets.create_or_update_dataset_items(
+            dataset_id=source_id,
+            items=[
+                DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+                DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+            ],
+            batch_group_id=id_helpers.generate_id(),
+        )
+        v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[
+            0
+        ]
+        v1_items = stream_items_wire(
+            rest,
+            dataset_name=dataset_name,
+            project_name=source_project_name,
+            version_hash=v1.version_hash,
+        )
+        item_ids = [it.id for it in v1_items]
+
+        # Seed: traces tagged environment="production" + grouped into one
+        # thread; spans tagged environment="staging". The trace env and
+        # span env differ deliberately so a single shared value couldn't
+        # mask a per-entity bug, and the thread inherits the trace env.
+        experiment_name = f"e2e-env-{random_chars()}"
+        thread_id = f"env-thread-{random_chars()}"
+        seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=experiment_name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=item_ids,
+            spans_per_trace=2,
+            trace_environment="production",
+            span_environment="staging",
+            thread_id=thread_id,
+        )
+
+        audit_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["dataset", dataset_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_path),
+        )
+        assert result.returncode == 0, result.stdout + result.stderr
+
+        target = rest.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name, project_name=target_project_name
+        )
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=target.id,
+            experiment_name=experiment_name,
+        )
+        dest_items = destination_experiment_items(
+            rest,
+            experiment_id=dest_exp.id,
+            dataset_id=target.id,
+        )
+        assert len(dest_items) == len(item_ids)
+
+        # (a) traces and (b) spans keep their source environment verbatim.
+        for dest_item in dest_items:
+            dest_trace = rest.traces.get_trace_by_id(id=dest_item.trace_id)
+            assert dest_trace.environment == "production", (
+                f"trace {dest_item.trace_id} lost environment: "
+                f"got {dest_trace.environment!r}"
+            )
+            dest_spans = destination_spans_for_trace(
+                rest,
+                trace_id=dest_item.trace_id,
+                project_name=target_project_name,
+            )
+            assert dest_spans, f"trace {dest_item.trace_id} has no destination spans"
+            assert all(span.environment == "staging" for span in dest_spans), (
+                "destination spans lost environment: "
+                f"{[span.environment for span in dest_spans]}"
+            )
+
+        # (c) the destination thread row -- materialized by the BE from the
+        # cascaded traces -- inherits the same environment. Polls because
+        # thread materialization is eventually consistent.
+        assert synchronization.until(
+            lambda: bool(
+                opik_client.search_threads(
+                    project_name=target_project_name,
+                    filter_string=f'id = "{thread_id}"',
+                )
+            ),
+            max_try_seconds=30,
+        ), f"destination thread {thread_id!r} never materialized"
+        threads = opik_client.search_threads(
+            project_name=target_project_name,
+            filter_string=f'id = "{thread_id}"',
+        )
+        assert len(threads) == 1
+        assert threads[0].environment == "production", (
+            f"destination thread {thread_id!r} lost environment: "
+            f"got {threads[0].environment!r}"
+        )

sdks/python/tests/e2e/cli/test_migrate_dataset_evaluate_shape_e2e.py

@@ -0,0 +1,467 @@
+"""End-to-end test for ``opik migrate dataset`` using the natural
+``opik.evaluate(...)`` flow as seeding (rather than direct REST writes).
+
+Complements ``test_migrate_dataset_e2e.py`` (precise per-version delta
+coverage via REST seeding) by exercising the realistic shape that real
+users produce:
+
+  1. Create a dataset, ``dataset.insert(...)`` a first batch of items
+  2. Run ``opik.evaluate(...)`` (experiment E1) under the dataset's project
+  3. Insert more items
+  4. Run ``opik.evaluate(...)`` (experiment E2) -- E1 saw 2 items, E2 sees 3
+  5. ``opik migrate dataset ... --to-project=<dest>``
+  6. Verify both experiments + items + traces + spans + feedback scores
+     round-trip with fresh destination ids
+
+In Opik V2 ``opik.evaluate(...)`` always lands its traces in the dataset's
+project (the per-evaluate ``project_name`` override is deprecated). The
+per-trace project_id scoping that ``_copy_traces_and_spans`` does is
+defense-in-depth for BE shapes the SDK doesn't produce today; that
+contract is covered by the unit test
+``test_span_read_uses_trace_project_not_experiment_project``.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any, Callable, Dict, Iterator
+from unittest import mock
+
+import pytest
+
+import opik
+from opik.evaluation.metrics import score_result
+
+from ...testlib import generate_project_name
+from .. import verifiers
+from .conftest import (
+    destination_experiment_items,
+    find_destination_experiment,
+    run_migrate_cli,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+# ``dataset_name`` and ``experiment_name`` fixtures come from
+# ``tests/e2e/conftest.py`` (shared per AGENTS.md). For the second
+# experiment we derive a sibling name off the standard fixture rather
+# than introducing a parallel per-test fixture.
+
+
+@pytest.fixture
+def experiment_name_two(experiment_name: str) -> Iterator[str]:
+    yield f"{experiment_name}-second"
+
+
+def _llm_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Deterministic stand-in for an LLM call.
+
+    Returns ``"Paris"`` regardless of input so the equality scorer
+    produces a deterministic 1.0/0.0 split based on the dataset items'
+    expected outputs.
+    """
+    return {"output": "Paris"}
+
+
+def _equals_score(
+    dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+) -> score_result.ScoreResult:
+    reference = dataset_item.get("expected", {}).get("output")
+    prediction = task_outputs["output"]
+    value = 1.0 if reference == prediction else 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="match" if value == 1.0 else "mismatch",
+    )
+
+
+def _wait_until(
+    predicate: Callable[[], Any],
+    timeout_seconds: int = 30,
+    poll_seconds: float = 1.0,
+) -> Any:
+    """Poll ``predicate()`` until it returns truthy or the timeout elapses.
+
+    The streamer flushes traces / spans / feedback scores asynchronously,
+    so we poll the BE for the artifacts we care about rather than racing
+    a hardcoded ``sleep()``. Returns the predicate's final value so the
+    caller can capture e.g. the experiment items in one go.
+    """
+    deadline = time.time() + timeout_seconds
+    last: Any = None
+    while time.time() < deadline:
+        last = predicate()
+        if last:
+            return last
+        time.sleep(poll_seconds)
+    return last
+
+
+def test_migrate_dataset__evaluate_shape__round_trips(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    experiment_name: str,
+    experiment_name_two: str,
+    tmp_path: Path,
+) -> None:
+    # ── Seed via the natural opik.evaluate(...) flow ──
+    # Dataset and both evaluate() runs land in source_project_name. We
+    # add items between runs so E1 sees 2 items and E2 sees 3 -- this
+    # exercises the "experiments reference different dataset version
+    # snapshots" case during migration.
+    dataset = opik_client.create_dataset(dataset_name, project_name=source_project_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of France?"},
+                "expected": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "Capital of Italy?"},
+                "expected": {"output": "Rome"},
+            },
+        ]
+    )
+
+    # Experiment 1 -- 2 items.
+    eval1 = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name,
+        experiment_config={"phase": "first-eval"},
+    )
+
+    # Add a third item between runs.
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of Spain?"},
+                "expected": {"output": "Madrid"},
+            },
+        ]
+    )
+
+    # Experiment 2 -- 3 items.
+    eval2 = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name_two,
+        experiment_config={"phase": "second-eval"},
+    )
+
+    # Flush the SDK streamer so all traces / spans / feedback scores land
+    # on the BE before the migration reads them.
+    opik.flush_tracker()
+
+    # Belt-and-suspenders: poll until both experiments are visible on the
+    # BE (the streamer's flush returns once the local queue drains, but
+    # the BE's eventual-consistency window can lag a beat).
+    rest = opik_client.rest_client
+
+    def _experiments_ready() -> bool:
+        e1 = rest.experiments.get_experiment_by_id(id=eval1.experiment_id)
+        e2 = rest.experiments.get_experiment_by_id(id=eval2.experiment_id)
+        return e1 is not None and e2 is not None
+
+    assert _wait_until(_experiments_ready), (
+        "experiments not visible on BE after flush; streamer may have failed"
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify the destination via the shared verifier layer ─────────────
+    # ``verifiers.verify_experiment`` / ``verify_trace`` have built-in
+    # ``synchronization.until`` retry loops for the BE's eventual-consistency
+    # window right after the migrate completes -- spurious failures from
+    # not-yet-readable rows are eliminated.
+    #
+    # Two name->id lookups remain: the destination dataset id (cascade keeps
+    # the source name at the destination after the rename) and each
+    # destination experiment id (cascade is name-preserving on experiments,
+    # so we resolve by name). Everything else routes through verifiers.
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_dataset_id = dest_dataset.id
+
+    dest_e1 = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset_id,
+        experiment_name=experiment_name,
+    )
+    dest_e2 = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset_id,
+        experiment_name=experiment_name_two,
+    )
+
+    # Fresh destination ids -- migrate is copy-not-move; new experiments
+    # must have new ids.
+    assert dest_e1.id != eval1.experiment_id, (
+        "destination experiment 1 must have a fresh id, not the source's"
+    )
+    assert dest_e2.id != eval2.experiment_id, (
+        "destination experiment 2 must have a fresh id, not the source's"
+    )
+
+    # Experiment-level shape via the shared verifier (with eventual-
+    # consistency retry baked in). ``recreate_experiment`` injects
+    # ``project_name`` into the destination experiment's metadata
+    # (intentional -- it's recorded so future imports/migrations can
+    # re-derive the project context). The verifier's metadata check is
+    # exact-equality, so include it in the expected shape.
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=dest_e1.id,
+        experiment_name=experiment_name,
+        experiment_metadata={
+            "phase": "first-eval",
+            "project_name": target_project_name,
+        },
+        feedback_scores_amount=1,  # the equals scorer emits one aggregate
+        traces_amount=2,
+        project_name=target_project_name,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=dest_e2.id,
+        experiment_name=experiment_name_two,
+        experiment_metadata={
+            "phase": "second-eval",
+            "project_name": target_project_name,
+        },
+        feedback_scores_amount=1,
+        traces_amount=3,
+        project_name=target_project_name,
+    )
+
+    # Per-item shape: every destination experiment item has a fresh
+    # trace_id + dataset_item_id, and the trace exists under the
+    # destination project (verified via ``verify_trace`` for its built-in
+    # eventual-consistency retry). The trace's feedback_scores value /
+    # reason varies per item (the equals scorer's mismatch produces 0.0
+    # for some items and 1.0 for others), so per-trace feedback scores
+    # aren't asserted by exact value here -- ``verify_experiment(
+    # feedback_scores_amount=1)`` above already pins that the cascade
+    # re-emitted the equals_scoring_function at the experiment-aggregate
+    # level.
+    dest_e1_items = destination_experiment_items(
+        rest, experiment_id=dest_e1.id, dataset_id=dest_dataset_id
+    )
+    dest_e2_items = destination_experiment_items(
+        rest, experiment_id=dest_e2.id, dataset_id=dest_dataset_id
+    )
+    for it in dest_e1_items + dest_e2_items:
+        assert it.trace_id is not None, "destination experiment item missing trace_id"
+        assert it.dataset_item_id is not None, (
+            "destination item missing dataset_item_id"
+        )
+        verifiers.verify_trace(
+            opik_client=opik_client,
+            trace_id=it.trace_id,
+            project_name=target_project_name,
+        )
+
+
+def test_migrate_dataset__cascade_trace_and_span_comments__round_trip(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    experiment_name: str,
+    tmp_path: Path,
+) -> None:
+    """Slice 4 (OPIK-6417): comments on traces + spans survive the cascade.
+
+    Seeds an evaluate() run, POSTs a handful of comments on one of the
+    resulting traces + one of its spans, runs the migrate, and asserts
+    the destination trace + span carry the same comments in the same
+    order on read-back. ``add_trace_comment`` / ``add_span_comment`` are
+    the only write surface (comments are ``READ_ONLY`` on the trace/span
+    Write payload); the cascade re-emits them via those endpoints after
+    the destination trace/span lands.
+    """
+    rest = opik_client.rest_client
+
+    # ── Seed ──
+    dataset = opik_client.create_dataset(dataset_name, project_name=source_project_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of France?"},
+                "expected": {"output": "Paris"},
+            },
+        ]
+    )
+    eval_result = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name,
+        experiment_config={"phase": "comments-cascade"},
+    )
+    opik.flush_tracker()
+
+    def _experiment_ready() -> bool:
+        return (
+            rest.experiments.get_experiment_by_id(id=eval_result.experiment_id)
+            is not None
+        )
+
+    assert _wait_until(_experiment_ready), (
+        "experiment not visible on BE after flush; streamer may have failed"
+    )
+
+    # Pick one source trace + one of its spans to attach comments to. The
+    # cascade reads ``comments`` off the bulk trace/span read, so the
+    # specific id we attach to doesn't matter -- any trace/span in the
+    # experiment is fine.
+    source_traces = _wait_until(
+        lambda: list(
+            opik_client.search_traces(
+                project_name=source_project_name,
+                filter_string=f'experiment_id = "{eval_result.experiment_id}"',
+                max_results=10,
+                truncate=False,
+            )
+        )
+    )
+    assert source_traces, "no source traces visible to seed comments on"
+    target_trace = source_traces[0]
+    source_spans = _wait_until(
+        lambda: list(
+            opik_client.search_spans(
+                project_name=source_project_name,
+                trace_id=target_trace.id,
+                max_results=10,
+                truncate=False,
+            )
+        )
+    )
+    assert source_spans, "no source spans visible to seed comments on"
+    target_span = source_spans[0]
+
+    # POST trace + span comments in a deterministic order so we can assert
+    # round-trip ordering at the destination.
+    trace_comment_texts = [
+        "qa-note: golden path verified",
+        "pm-note: tracked in OPS-1234",
+        "debug-note: see screenshot in #incidents",
+    ]
+    span_comment_texts = [
+        "first span observation",
+        "second span observation",
+    ]
+    for text in trace_comment_texts:
+        rest.traces.add_trace_comment(id_=target_trace.id, text=text)
+    for text in span_comment_texts:
+        rest.spans.add_span_comment(id_=target_span.id, text=text)
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify the destination trace + span round-trip the comments ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_experiment = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=experiment_name,
+    )
+
+    dest_items = _wait_until(
+        lambda: destination_experiment_items(
+            rest,
+            experiment_id=dest_experiment.id,
+            dataset_id=dest_dataset.id,
+        )
+    )
+    assert dest_items, "no destination experiment items after migrate"
+    dest_trace_id = dest_items[0].trace_id
+    assert dest_trace_id is not None
+
+    # Verify the destination trace's comments via the shared verifier
+    # (it polls through ``_retry_until_assertions_pass`` for BE eventual
+    # consistency, so we don't need a separate hand-rolled wait).
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=dest_trace_id,
+        project_name=target_project_name,
+        comments=trace_comment_texts,
+    )
+
+    # Span comments: the cascade mints fresh span ids, so we resolve the
+    # destination span via name match on the source ``target_span``
+    # before delegating to the shared verifier for the content check.
+    dest_spans = _wait_until(
+        lambda: list(
+            opik_client.search_spans(
+                project_name=target_project_name,
+                trace_id=dest_trace_id,
+                max_results=50,
+                truncate=False,
+            )
+        )
+    )
+    assert dest_spans, "no destination spans after migrate"
+    matching = [s for s in dest_spans if s.name == target_span.name]
+    assert matching, (
+        f"destination trace has no span matching source name {target_span.name!r}"
+    )
+    # If multiple destination spans share the source span's name (rare;
+    # the seeded experiment has one root + a few children with distinct
+    # names), pick the one whose comment count matches before delegating
+    # to verify_span -- the verifier's exact-equality assert would fail
+    # against an unrelated namesake otherwise.
+    candidate_ids = [s.id for s in matching]
+    if len(candidate_ids) > 1:
+        chosen = next(
+            (
+                s.id
+                for s in matching
+                if rest.spans.get_span_by_id(id=s.id).comments
+                and len(rest.spans.get_span_by_id(id=s.id).comments)
+                == len(span_comment_texts)
+            ),
+            candidate_ids[0],
+        )
+    else:
+        chosen = candidate_ids[0]
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=chosen,
+        trace_id=dest_trace_id,
+        parent_span_id=mock.ANY,
+        project_name=target_project_name,
+        comments=span_comment_texts,
+    )

sdks/python/tests/e2e/cli/test_migrate_dataset_optimizations_e2e.py

@@ -0,0 +1,269 @@
+"""End-to-end test for ``opik migrate dataset`` Slice 5 — optimization cascade.
+
+Optimization is a real BE entity, not a logical group-by: it has its own
+UUID, its own row in ``OptimizationDAO``, and its own ``dataset_id`` /
+``project_id`` FKs. The Optimization Studio "group" view in the UI is
+computed by the BE at read time from each experiment's ``optimization_id``
+FK plus the optimization's aggregated stats (``numTrials``, etc.); the
+persisted shape is one Optimization row + N Experiment rows where
+``Experiment.optimization_id == Optimization.id``.
+
+Slice 5's contract: when ``opik migrate dataset`` runs, every optimization
+referencing the source dataset is recreated under the destination project
+with a fresh id, and every experiment carrying that ``optimization_id``
+is recreated at the destination with its FK re-pointed at the new
+destination optimization id. The destination's Optimization Studio shows
+the same rows + trial groupings as the source.
+
+Shape pinned by this test:
+
+  1. Source dataset in project A with 3 items
+  2. One Optimization (``opt``) on the source dataset
+  3. Two TRIAL experiments seeded as ``opt``'s trials (each has 1 item)
+  4. One CONTROL ``regular`` experiment with no optimization_id (1 item)
+  5. Run ``opik migrate dataset ... --to-project=<destination>``
+  6. Assert:
+     a. Destination has exactly one optimization tied to the
+        destination dataset, with a fresh id (NOT the source id).
+     b. Both trial experiments are recreated under the destination
+        project with their ``optimization_id`` pointing at the
+        destination optimization (NOT the source optimization id).
+     c. The control experiment has no ``optimization_id`` (untouched).
+     d. The destination optimization's fidelity fields
+        (``name`` / ``objective_name`` / ``status``) match the source.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+import opik.id_helpers as id_helpers_module
+from opik.rest_api import OpikApi
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .conftest import (
+    create_dataset_shell,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-opt-{random_chars()}"
+
+
+def _create_source_optimization(
+    rest: OpikApi,
+    *,
+    dataset_name: str,
+    project_name: str,
+    optimization_name: str,
+) -> str:
+    """Create an Optimization tied to ``dataset_name`` in ``project_name``
+    and return its id. The cascade will discover this via
+    ``find_optimizations(dataset_id=...)``.
+    """
+    optimization_id = id_helpers_module.generate_id()
+    rest.optimizations.create_optimization(
+        id=optimization_id,
+        name=optimization_name,
+        dataset_name=dataset_name,
+        project_name=project_name,
+        objective_name="accuracy",
+        status="completed",
+    )
+    return optimization_id
+
+
+def test_migrate_dataset__optimization_cascade__trials_grouped_at_destination(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset with 3 items (2 for trials, 1 for control) ──
+    from opik import id_helpers
+    from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+    source_dataset_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_dataset_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": "trial-a", "a": "A"}),
+            DatasetItemWrite(source="manual", data={"q": "trial-b", "a": "B"}),
+            DatasetItemWrite(source="manual", data={"q": "control", "a": "C"}),
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(
+        id=source_dataset_id, page=1, size=1
+    ).content[0]
+
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+    trial_a_item_id = by_q["trial-a"]
+    trial_b_item_id = by_q["trial-b"]
+    control_item_id = by_q["control"]
+    assert trial_a_item_id and trial_b_item_id and control_item_id
+
+    # ── Create the source optimization ──
+    optimization_name = f"e2e-opt-{random_chars()}"
+    source_optimization_id = _create_source_optimization(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        optimization_name=optimization_name,
+    )
+
+    # ── Seed two trial experiments pointing at the optimization ──
+    # ``experiment_type="trial"`` is what the Optimization Studio UI
+    # filters on for the trial sub-list under an optimization row.
+    trial_a_name = f"trial-a-{random_chars()}"
+    trial_b_name = f"trial-b-{random_chars()}"
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=trial_a_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[trial_a_item_id],
+        experiment_type="trial",
+        optimization_id=source_optimization_id,
+    )
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=trial_b_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[trial_b_item_id],
+        experiment_type="trial",
+        optimization_id=source_optimization_id,
+    )
+
+    # ── Seed a control regular experiment with no optimization_id ──
+    # Pins that the cascade only touches experiments that actually had
+    # an optimization_id; the control experiment must round-trip with
+    # no optimization_id at the destination.
+    control_name = f"control-{random_chars()}"
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=control_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[control_item_id],
+        experiment_type="regular",
+        optimization_id=None,
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--from-project",
+            source_project_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify destination optimization fidelity ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_optimizations_page = rest.optimizations.find_optimizations(
+        dataset_id=dest_dataset.id, page=1, size=10
+    )
+    dest_optimizations = list(dest_optimizations_page.content or [])
+    assert len(dest_optimizations) == 1, (
+        f"expected exactly one destination optimization tied to dataset "
+        f"{dest_dataset.id}, got {len(dest_optimizations)}"
+    )
+    dest_optimization = dest_optimizations[0]
+
+    # Fresh destination id -- migrate is copy-not-move; source id must
+    # not be reused.
+    assert dest_optimization.id != source_optimization_id, (
+        "destination optimization id must be fresh; source id reused"
+    )
+    # Fidelity fields round-trip verbatim.
+    assert dest_optimization.name == optimization_name
+    assert dest_optimization.objective_name == "accuracy"
+    assert dest_optimization.status == "completed"
+    assert dest_optimization.dataset_id == dest_dataset.id
+
+    # ── Verify trial experiments are re-pointed at the destination optimization ──
+    dest_trial_a = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=trial_a_name,
+    )
+    dest_trial_b = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=trial_b_name,
+    )
+    assert dest_trial_a.optimization_id == dest_optimization.id, (
+        f"trial a should FK to destination optimization {dest_optimization.id}, "
+        f"got {dest_trial_a.optimization_id}"
+    )
+    assert dest_trial_b.optimization_id == dest_optimization.id, (
+        f"trial b should FK to destination optimization {dest_optimization.id}, "
+        f"got {dest_trial_b.optimization_id}"
+    )
+    # Defence-in-depth: neither trial should still point at the source id.
+    for dest_trial, label in ((dest_trial_a, "a"), (dest_trial_b, "b")):
+        assert dest_trial.optimization_id != source_optimization_id, (
+            f"trial {label}'s destination optimization_id must NOT be the "
+            f"source id (cascade failed to remap)"
+        )
+
+    # ── Verify control experiment is untouched (no optimization_id) ──
+    dest_control = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=control_name,
+    )
+    assert not dest_control.optimization_id, (
+        f"control experiment must not have an optimization_id at the "
+        f"destination, got {dest_control.optimization_id!r}"
+    )
+
+    # ── Audit log carries per-optimization records ──
+    import json
+
+    audit = json.loads(audit_path.read_text())
+    migrate_optimization_records = [
+        a for a in audit["actions"] if a["type"] == "migrate_optimization"
+    ]
+    assert len(migrate_optimization_records) == 1
+    record = migrate_optimization_records[0]
+    assert record["status"] == "ok"
+    assert record["source_id"] == source_optimization_id
+    assert record["destination_id"] == dest_optimization.id

sdks/python/tests/e2e/cli/test_migrate_dataset_resume_e2e.py

@@ -0,0 +1,273 @@
+"""End-to-end test for ``opik migrate dataset`` checkpoint/resume (OPIK-7168).
+
+The unit tests prove the resume *logic* against a stand-in client. This test
+proves the thing that actually matters for the feature: a **real process
+death** mid-cascade leaves a checkpoint on disk that a **real re-run** picks
+up, against a **real backend** whose trace-delete cascades spans — ending with
+a duplicate-free destination.
+
+Shape:
+
+  1. Seed a source dataset (one version) and N experiments, each with its own
+     trace + span tree, all referencing that dataset.
+  2. Run ``opik migrate dataset`` with a test-only ``sitecustomize.py`` seam on
+     ``PYTHONPATH`` (nothing test-only lives in the product) that ``os._exit(137)``s
+     partway through one experiment — after its destination traces are written
+     and recorded on the checkpoint but before the experiment row is recreated.
+     This is the uncatchable-kill (OOM-like) interruption the feature exists for;
+     a clean ``except`` never runs, so only the incrementally-flushed checkpoint
+     survives. The seam also redirects the checkpoint dir into tmp_path.
+  3. Assert the run died hard and the checkpoint on disk shows partial progress
+     (some experiments done, one in flight with recorded dest trace ids).
+  4. Re-run the *same* command with no crash env. Assert it exits 0, resumes
+     (skips the already-done experiments, cleans up the interrupted one's
+     partial traces, re-migrates it), and the destination ends with exactly one
+     experiment per source experiment — no duplicates — and the checkpoint file
+     is gone.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Iterator, List
+
+import pytest
+
+import opik
+from opik import id_helpers, synchronization
+from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+from ...conftest import random_chars
+from .conftest import (
+    create_dataset_shell,
+    destination_experiment_items,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+# Number of source experiments and the count of experiments completed before
+# the first run hard-exits. Crashing after 2 completed experiments (mid the 3rd)
+# leaves a non-trivial done-set to skip AND an in-flight experiment to clean up
+# + re-migrate on resume.
+_NUM_EXPERIMENTS = 5
+_CRASH_AFTER_N_EXPERIMENTS = 2
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-resume-{random_chars()}"
+
+
+def _write_test_seam(
+    seam_dir: Path, *, crash_after_n: int, checkpoint_dir: Path
+) -> None:
+    """Write a test-only ``sitecustomize.py`` the migrate subprocess auto-imports.
+
+    Python imports ``sitecustomize`` at interpreter startup, so dropping it on
+    the subprocess's ``PYTHONPATH`` lets the test inject behaviour **without any
+    test-only logic living in the product**. Two patches, both test-owned:
+
+    * Redirect the checkpoint dir to a tmp path (via ``checkpoint_dir``) so the
+      test can find/assert the checkpoint without touching the developer's real
+      ``~/.opik`` and without repointing ``HOME`` (which would break the SDK's
+      backend-config resolution the E2E run relies on).
+    * Wrap ``_copy_traces_and_spans`` so the real one runs first (destination
+      traces written + recorded on the checkpoint), then ``os._exit(137)`` after
+      the Nth experiment — reproducing the exact "traces written, experiment row
+      not yet created" partial state an OOM leaves mid-cascade. ``crash_after_n``
+      < 0 disables the crash (used for the resume run).
+    """
+    seam_dir.mkdir(parents=True, exist_ok=True)
+    (seam_dir / "sitecustomize.py").write_text(
+        f"""
+import os
+from pathlib import Path
+from opik.cli.migrate import checkpoint as _cp
+from opik.cli.migrate.datasets import experiments as _exp
+
+_cp.checkpoint_dir = lambda: Path({str(checkpoint_dir)!r})
+
+_crash_after = {crash_after_n}
+if _crash_after >= 0:
+    _seen = {{"n": 0}}
+    _real_copy = _exp._copy_traces_and_spans
+
+    def _copy_then_maybe_crash(*args, **kwargs):
+        result = _real_copy(*args, **kwargs)
+        _seen["n"] += 1
+        if _seen["n"] > _crash_after:
+            os._exit(137)  # 128 + SIGKILL(9): uncatchable, mimics an OOM kill
+        return result
+
+    _exp._copy_traces_and_spans = _copy_then_maybe_crash
+"""
+    )
+
+
+def test_migrate_dataset__crash_mid_cascade__resumes_without_duplicates(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset (single version, one item per experiment) ──
+    source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": f"Q{i}", "a": f"A{i}"})
+            for i in range(_NUM_EXPERIMENTS)
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[0]
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    item_id_by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+
+    # ── Seed N experiments, each referencing one dataset item ──
+    # Names derive from the (already-random) ``dataset_name`` fixture so they're
+    # unique per run yet reproducible from the fixture, per sdks/python/AGENTS.md.
+    experiment_names: List[str] = []
+    for i in range(_NUM_EXPERIMENTS):
+        name = f"{dataset_name}-exp-{i}"
+        experiment_names.append(name)
+        seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=[item_id_by_q[f"Q{i}"]],
+            spans_per_trace=2,
+        )
+
+    audit_path = tmp_path / "audit.json"
+    migrate_args = ["dataset", dataset_name, "--to-project", target_project_name]
+
+    # Test seam: a sitecustomize.py the migrate subprocess auto-imports. It
+    # redirects the checkpoint dir into tmp_path (hermetic; no real ~/.opik) and
+    # injects the deterministic mid-cascade crash — all test-owned, nothing in
+    # the product. PYTHONPATH must keep the SDK's own src on it (the E2E run
+    # imports opik from source), so the seam dir is prepended.
+    checkpoint_dir = tmp_path / "checkpoints"
+    seam_dir = tmp_path / "seam"
+    existing_pythonpath = os.environ.get("PYTHONPATH", "")
+
+    def _seam_env(crash_after_n: int) -> dict:
+        _write_test_seam(
+            seam_dir, crash_after_n=crash_after_n, checkpoint_dir=checkpoint_dir
+        )
+        return {
+            "PYTHONPATH": os.pathsep.join(
+                [str(seam_dir)] + ([existing_pythonpath] if existing_pythonpath else [])
+            )
+        }
+
+    # ── Run 1: crash deterministically mid-cascade ──
+    crashed = run_migrate_cli(
+        migrate_args,
+        audit_log_path=str(audit_path),
+        extra_env=_seam_env(_CRASH_AFTER_N_EXPERIMENTS),
+    )
+    # os._exit(137) is a hard, uncatchable exit — not a clean CLI exit code.
+    assert crashed.returncode == 137, (
+        f"expected hard-exit 137, got {crashed.returncode}\n"
+        f"stdout={crashed.stdout}\nstderr={crashed.stderr}"
+    )
+
+    # The incrementally-flushed checkpoint must have survived the kill and show
+    # partial progress: the experiments before the crash completed, and the one
+    # at the crash is in flight with recorded destination trace ids (its traces
+    # were written to the backend before the crash).
+    checkpoints = list(checkpoint_dir.glob("opik-migrate-checkpoint-*.json"))
+    assert len(checkpoints) == 1, f"expected one checkpoint file, got {checkpoints}"
+    checkpoint_data = json.loads(checkpoints[0].read_text())
+    assert (
+        len(checkpoint_data["completed_experiment_ids"]) == _CRASH_AFTER_N_EXPERIMENTS
+    )
+    # The dataset phase (create-temp/replay/optimizations) finished before the
+    # cascade started, so resume must skip it and reconstruct the remaps from
+    # the temp destination rather than re-running (and duplicating) it. Under the
+    # OPIK-7162 ordering the source keeps its original name until the run
+    # succeeds, and the destination is still under the temp name at crash time —
+    # the checkpoint records both so resume can re-resolve them.
+    assert checkpoint_data["dataset_phase_done"] is True
+    assert checkpoint_data["source_name"] == dataset_name
+    assert checkpoint_data["temp_dest_name"] == f"{dataset_name}__migrating"
+    in_flight = checkpoint_data["in_flight"]
+    assert in_flight is not None, "interrupted experiment must be recorded in flight"
+    assert in_flight["dest_trace_ids"], (
+        "the in-flight experiment's destination trace ids must be recorded "
+        "before the crash so resume can delete them"
+    )
+    # Crash landed before the experiment row was created.
+    assert in_flight["dest_experiment_id"] is None
+
+    # ── Run 2: resume to completion (same seam, crash disabled) ──
+    resumed = run_migrate_cli(
+        migrate_args, audit_log_path=str(audit_path), extra_env=_seam_env(-1)
+    )
+    assert resumed.returncode == 0, resumed.stdout + resumed.stderr
+    assert "Resuming migration" in resumed.stdout, (
+        "resumed run should announce it is resuming"
+    )
+
+    # ── Verify destination: exactly one experiment per source, no duplicates ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+
+    # Migrated experiments become readable asynchronously, so poll until the
+    # destination shows the full set before asserting — otherwise the read can
+    # race ahead of eventual consistency and flake. ``until`` returns False on
+    # timeout; the count assertion below then fails with the observed number.
+    def _all_experiments_present() -> bool:
+        page = rest.experiments.find_experiments(
+            dataset_id=dest_dataset.id, page=1, size=100
+        )
+        return len(page.content or []) == _NUM_EXPERIMENTS
+
+    synchronization.until(_all_experiments_present, max_try_seconds=30)
+
+    for name in experiment_names:
+        # find_destination_experiment RAISES if zero or >1 match — so this is
+        # the duplicate check: the interrupted experiment must not have been
+        # migrated twice (once by the crashed run, once by the resume).
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=dest_dataset.id,
+            experiment_name=name,
+        )
+        dest_items = destination_experiment_items(
+            rest, experiment_id=dest_exp.id, dataset_id=dest_dataset.id
+        )
+        assert len(dest_items) == 1, (
+            f"experiment {name!r} should have exactly 1 item at the "
+            f"destination, got {len(dest_items)}"
+        )
+
+    # Total destination experiments equals the source count — no orphaned
+    # partial experiment left over from the crashed run.
+    all_dest = rest.experiments.find_experiments(
+        dataset_id=dest_dataset.id, page=1, size=100
+    )
+    assert len(all_dest.content or []) == _NUM_EXPERIMENTS, (
+        f"expected {_NUM_EXPERIMENTS} destination experiments, "
+        f"got {len(all_dest.content or [])}"
+    )
+
+    # Checkpoint deleted on successful completion.
+    assert list(checkpoint_dir.glob("opik-migrate-checkpoint-*.json")) == []

sdks/python/tests/e2e/cli/test_migrate_prompt_e2e.py

@@ -0,0 +1,229 @@
+"""End-to-end test for ``opik migrate prompt`` against a real Opik backend.
+
+Seeds a multi-version source prompt directly via the REST API, runs
+``opik migrate prompt`` as a subprocess (exercising the actual Click
+entrypoint + exit-code handling), then reads back the destination prompt
+and asserts:
+
+  - Destination has the same number of versions as the source
+  - Each destination version carries the source commit hash verbatim
+    (the architectural promise this slice exists to deliver)
+  - Source has been renamed to ``<name>_v1``
+  - Audit log is finalised to ``ok`` with one ``replay_prompt_version``
+    record per source version
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterator, List
+
+import pytest
+
+import opik
+from opik.rest_api import OpikApi
+from opik.rest_api.types.prompt_version_detail import PromptVersionDetail
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .conftest import run_migrate_cli
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def prompt_name() -> Iterator[str]:
+    yield f"e2e-migrate-prompt-{random_chars()}"
+
+
+def _seed_source_prompt_with_versions(
+    rest_client: OpikApi,
+    *,
+    name: str,
+    project_name: str,
+    version_specs: List[dict],
+) -> List[str]:
+    """Create a source prompt container plus N versions.
+
+    First call to ``create_prompt`` carries the v1 template so the BE
+    mints v1 with its own commit hash (we don't control v1's commit at
+    seed time — that's fine for the test, we just need a multi-version
+    history). Subsequent versions are minted via
+    ``create_prompt_version`` so we control their commit + payload.
+
+    Returns the list of source version commits in chronological order.
+    """
+    rest_client.prompts.create_prompt(
+        name=name,
+        project_name=project_name,
+        description="e2e source description",
+        tags=["e2e", "source"],
+        template=version_specs[0]["template"],
+        type=version_specs[0].get("type"),
+        change_description=version_specs[0].get("change_description"),
+    )
+    versions_page = rest_client.prompts.get_prompts(name=name, size=10)
+    source_row = versions_page.content[0]
+    versions = rest_client.prompts.get_prompt_versions(
+        id=source_row.id, page=1, size=100
+    )
+    v1_commit = versions.content[0].commit
+    commits = [v1_commit]
+
+    for spec in version_specs[1:]:
+        version_payload = PromptVersionDetail(
+            template=spec["template"],
+            type=spec.get("type"),
+            change_description=spec.get("change_description"),
+            tags=spec.get("tags"),
+            environments=spec.get("environments"),
+        )
+        created = rest_client.prompts.create_prompt_version(
+            name=name,
+            version=version_payload,
+            project_name=project_name,
+        )
+        commits.append(created.commit)
+
+    return commits
+
+
+class TestMigratePromptE2E:
+    def test_three_version_prompt_round_trips_with_commits_verbatim(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        prompt_name: str,
+        tmp_path: Path,
+    ) -> None:
+        rest = opik_client.rest_client
+
+        source_commits = _seed_source_prompt_with_versions(
+            rest,
+            name=prompt_name,
+            project_name=source_project_name,
+            version_specs=[
+                {"template": "hi {{name}}", "type": "mustache"},
+                {
+                    "template": "hello {{name}}",
+                    "type": "mustache",
+                    "change_description": "be friendlier",
+                },
+                {
+                    "template": "greetings {{name}}",
+                    "type": "mustache",
+                    "tags": ["polished"],
+                },
+            ],
+        )
+
+        audit_log_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["prompt", prompt_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_log_path),
+        )
+        assert result.returncode == 0, (
+            f"migrate prompt failed: stdout={result.stdout!r} stderr={result.stderr!r}"
+        )
+
+        # Source must have been renamed (workspace-unique name freed for
+        # the destination to claim).
+        renamed_page = rest.prompts.get_prompts(name=f"{prompt_name}_v1", size=10)
+        assert renamed_page.content, (
+            f"source prompt was not renamed to {prompt_name}_v1"
+        )
+
+        # Destination claims the original name.
+        dest_page = rest.prompts.get_prompts(name=prompt_name, size=10)
+        assert dest_page.content, "destination prompt not found at original name"
+        dest_prompt = dest_page.content[0]
+
+        # Every source version is replayed onto the destination with the
+        # commit hash preserved verbatim. BE orders newest-first; reverse
+        # to chronological for comparison against source_commits.
+        dest_versions_page = rest.prompts.get_prompt_versions(
+            id=dest_prompt.id, page=1, size=100
+        )
+        dest_commits = [v.commit for v in reversed(dest_versions_page.content or [])]
+        assert dest_commits == source_commits
+
+        # Audit log finalised to "ok" with one replay record per version.
+        audit = json.loads(audit_log_path.read_text())
+        assert audit["status"] == "ok"
+        replay_records = [
+            a for a in audit["actions"] if a["type"] == "replay_prompt_version"
+        ]
+        assert len(replay_records) == len(source_commits)
+        assert all(a["status"] == "ok" for a in replay_records)
+
+    def test_environment_ownership_round_trips(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        prompt_name: str,
+        tmp_path: Path,
+    ) -> None:
+        # A version that owns an environment on the source must own the
+        # same environment on the destination after migration. The BE
+        # accepts ``environments`` inline on create_prompt_version and
+        # auto-registers unknown names in the workspace registry, so the
+        # replay carries the set verbatim without a follow-up PATCH.
+        rest = opik_client.rest_client
+        environment_name = f"prod-{random_chars()}"
+        source_commits = _seed_source_prompt_with_versions(
+            rest,
+            name=prompt_name,
+            project_name=source_project_name,
+            version_specs=[
+                {"template": "hi {{name}}", "type": "mustache"},
+                {"template": "hello {{name}}", "type": "mustache"},
+                {
+                    "template": "greetings {{name}}",
+                    "type": "mustache",
+                    "environments": [environment_name],
+                },
+            ],
+        )
+
+        audit_log_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["prompt", prompt_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_log_path),
+        )
+        assert result.returncode == 0, (
+            f"migrate prompt failed: stdout={result.stdout!r} stderr={result.stderr!r}"
+        )
+
+        dest_page = rest.prompts.get_prompts(name=prompt_name, size=10)
+        assert dest_page.content, "destination prompt not found at original name"
+        dest_prompt = dest_page.content[0]
+
+        dest_versions_page = rest.prompts.get_prompt_versions(
+            id=dest_prompt.id, page=1, size=100
+        )
+        dest_versions = list(reversed(dest_versions_page.content or []))
+        dest_commits = [v.commit for v in dest_versions]
+        assert dest_commits == source_commits
+
+        # Exactly one destination version owns the environment, and it is
+        # the last one (the source owner), proving ownership round-tripped
+        # without leaking onto the env-less versions.
+        owners = [
+            v.commit
+            for v in dest_versions
+            if environment_name in (v.environments or [])
+        ]
+        assert owners == [source_commits[-1]]
+
+        audit = json.loads(audit_log_path.read_text())
+        env_records = [
+            a
+            for a in audit["actions"]
+            if a["type"] == "replay_prompt_version" and a.get("source_environments")
+        ]
+        assert len(env_records) == 1
+        assert env_records[0]["source_environments"] == [environment_name]
+        assert env_records[0]["target_environments"] == [environment_name]

sdks/python/tests/e2e/cli/test_migrate_test_suite_e2e.py

@@ -0,0 +1,488 @@
+"""End-to-end test for ``opik migrate dataset`` against a test suite.
+
+Test suites have an extra dimension over plain datasets: each version
+carries suite-level ``evaluators`` + ``execution_policy`` that the
+migration must replay per-version. Plus the BE-natural test-suite shape
+is "config-only v1 + content versions v2+" (the SDK's
+``create_test_suite(global_assertions=...)`` produces a v1 with zero
+items but the suite config attached).
+
+This test covers the full surface:
+
+- 4 versions: config-only v1 + 3 content versions with adds/edits/deletes
+- Per-version suite-level evaluators that change across versions
+- Per-version suite-level execution_policy that changes across versions
+- Per-version user tags + metadata
+- Per-item evaluators + execution_policy + tags on individual items
+- Clear transitions: items that drop their per-item overrides between versions
+
+Verifies all of the above round-trips at every replayed version on the
+target.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from ._cascade_comparison import compare_cascade
+from .conftest import (
+    apply_changes,
+    chronological_versions,
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_spans_for_trace,
+    display_order,
+    find_destination_experiment,
+    item_hashes,
+    normalize_evaluators,
+    normalize_policy,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    strip_be_managed_version_tags,
+    stream_items_wire,
+)
+
+# Per ``sdks/python/AGENTS.md``: every e2e module sources PROJECT_NAME from
+# ``generate_project_name("e2e", __name__)`` so backend project names are
+# isolated per test module + the autouse ``configure_e2e_tests_env`` fixture
+# can patch ``OPIK_PROJECT_NAME`` to match.
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def suite_name() -> Iterator[str]:
+    yield f"e2e-migrate-suite-{random_chars()}"
+
+
+# Suite-level evaluators per source version. Each entry is what gets
+# attached to that version (changes across versions to exercise replay).
+SUITE_EVALUATORS_PER_VERSION = [
+    # v1: one suite-level assertion
+    [{"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}}],
+    # v2: still v1's evaluator (no suite-level change)
+    [{"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}}],
+    # v3: add a second suite-level assertion + bump policy
+    [
+        {"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}},
+        {"name": "v2-judge", "type": "llm_judge", "config": {"model": "sonnet"}},
+    ],
+    # v4: replace evaluators entirely
+    [{"name": "v3-only-judge", "type": "llm_judge", "config": {"model": "opus"}}],
+]
+
+SUITE_EXEC_POLICY_PER_VERSION = [
+    {"runs_per_item": 1, "pass_threshold": 1},
+    {"runs_per_item": 1, "pass_threshold": 1},
+    {"runs_per_item": 3, "pass_threshold": 2},
+    {"runs_per_item": 5, "pass_threshold": 3},
+]
+
+
+def test_test_suite_full_fidelity_round_trip(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    suite_name: str,
+    tmp_path: Path,
+) -> None:
+    """Migrate a 4-version test suite end-to-end and verify per-version
+    fidelity across items, item-level overrides, suite-level config,
+    metadata, and user tags.
+
+    Exercises every fidelity dimension Slice 2 supports in one test
+    because the BE round-trip is the expensive part; multiple smaller
+    tests would each pay the same setup cost. If any dimension regresses
+    this single test pins it.
+    """
+    rest = opik_client.rest_client
+
+    # ── Seed v1: config-only (no items, just suite-level evaluators+policy) ──
+    suite_id = create_dataset_shell(
+        rest, suite_name, source_project_name, type="evaluation_suite"
+    )
+    apply_changes(
+        rest,
+        suite_id,
+        base_version_id=None,
+        change_description="v1 — initial config-only version",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[0],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[0],
+        override=True,  # required when base_version=None
+    )
+    v1 = chronological_versions(rest, suite_id)[0]
+
+    # ── Seed v2: add 3 items (Q1, Q2 with per-item evaluator, Q3 with
+    # per-item exec policy). Suite config unchanged. ──
+    v2_items = [
+        {"data": {"q": "Q1", "a": "A1"}, "source": "manual"},
+        {
+            "data": {"q": "Q2", "a": "A2"},
+            "source": "manual",
+            "evaluators": [
+                {
+                    "name": "q2-item-judge",
+                    "type": "llm_judge",
+                    "config": {"model": "haiku-item"},
+                }
+            ],
+            "tags": ["smoke"],
+        },
+        {
+            "data": {"q": "Q3", "a": "A3"},
+            "source": "manual",
+            "execution_policy": {"runs_per_item": 7, "pass_threshold": 4},
+        },
+    ]
+    v2_id = apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v1.id,
+        added_items=v2_items,
+        change_description="v2 — initial items with per-item overrides",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[1],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[1],
+    )
+    v2_streamed = stream_items_wire(
+        rest,
+        dataset_name=suite_name,
+        project_name=source_project_name,
+        version_hash=None,  # latest
+    )
+    by_q = {it.data["q"]: it for it in v2_streamed if it.data}
+
+    # ── Seed v3: edit Q1 (add per-item policy + extend tags), edit Q2
+    # (clear tags), add Q4. Bump suite evaluators to v3 config, bump policy. ──
+    v3_id = apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v2_id,
+        edited_items=[
+            {
+                "id": by_q["Q1"].id,
+                "data": {"q": "Q1", "a": "A1"},
+                "execution_policy": {"runs_per_item": 9, "pass_threshold": 5},
+                "tags": ["regression", "v3-touched"],
+            },
+            {
+                "id": by_q["Q2"].id,
+                "data": {"q": "Q2", "a": "A2"},
+                # Clear Q2's tags by passing empty list.
+                "tags": [],
+            },
+        ],
+        added_items=[{"data": {"q": "Q4", "a": "A4"}, "source": "manual"}],
+        change_description="v3 — edit Q1, clear Q2 tags, add Q4",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[2],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[2],
+        metadata={"phase": "experimentation", "owner": "ada"},
+        user_tags=["baseline"],
+    )
+
+    # ── Seed v4: delete Q2, replace Q3's per-item evaluator, add Q5 with a
+    # per-item evaluator. Replace suite config entirely + bump policy. ──
+    apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v3_id,
+        edited_items=[
+            {
+                "id": by_q["Q3"].id,
+                "data": {"q": "Q3", "a": "A3"},
+                "evaluators": [
+                    {
+                        "name": "q3-item-judge-NEW",
+                        "type": "llm_judge",
+                        "config": {"model": "sonnet-item"},
+                    }
+                ],
+            }
+        ],
+        deleted_ids=[by_q["Q2"].id],
+        added_items=[
+            {
+                "data": {"q": "Q5", "a": "A5"},
+                "source": "manual",
+                "evaluators": [
+                    {
+                        "name": "q5-item-judge",
+                        "type": "llm_judge",
+                        "config": {"model": "opus-item"},
+                    }
+                ],
+            }
+        ],
+        change_description="v4 — delete Q2, replace Q3 evaluator, add Q5",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[3],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[3],
+        metadata={"phase": "production", "owner": "ada"},
+        user_tags=["v1.0", "release-candidate"],
+    )
+
+    # ── Snapshot source expectations per version ──
+    src_versions = chronological_versions(rest, suite_id)
+    assert len(src_versions) == 4
+    expected = []
+    for v in src_versions:
+        items = stream_items_wire(
+            rest,
+            dataset_name=suite_name,
+            project_name=source_project_name,
+            version_hash=v.version_hash,
+        )
+        expected.append(
+            {
+                "hashes": item_hashes(items),
+                "order": display_order(items),
+                "suite_evals": normalize_evaluators(v.evaluators),
+                "suite_pol": normalize_policy(v.execution_policy),
+                "metadata": v.metadata or None,
+                "user_tags": strip_be_managed_version_tags(v.tags),
+            }
+        )
+
+    # ── Seed a suite-driven experiment on v2 items so the cascade has
+    # something to round-trip. Test-suite experiments carry per-trace
+    # assertion_results (regular-dataset experiments carry feedback_scores
+    # instead -- covered in test_migrate_dataset_e2e.py). Each item gets
+    # one trace, one span, and per-item assertion_results that the seed
+    # helper writes via store_assertions_batch(entity_type='TRACE', ...).
+    # The cascade re-emits them scoped to the new destination trace ids
+    # via the same endpoint. ──
+    experiment_name = f"e2e-suite-exp-{random_chars()}"
+    v2_item_ids = [by_q["Q1"].id, by_q["Q2"].id, by_q["Q3"].id]
+    cascade_seed = seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=experiment_name,
+        dataset_name=suite_name,
+        dataset_id=suite_id,
+        dataset_version_id=v2_id,
+        project_name=source_project_name,
+        item_ids=v2_item_ids,
+        # ``type="regular"`` + ``evaluation_method="evaluation_suite"`` is
+        # the canonical shape for a non-optimizer test-suite run, mirroring
+        # what ``opik.evaluate(...)`` produces against a test suite. The
+        # BE only updates ``last_created_experiment_at`` on the dataset for
+        # ``type="regular"`` experiments, so ``with_experiments_only=true``
+        # (the UI's project-page filter) correctly surfaces this one.
+        # Optimizer-driven trials use ``type="trial"`` + ``optimization_id``;
+        # those are Slice 4's cascade scope.
+        experiment_type="regular",
+        evaluation_method="evaluation_suite",
+        experiment_config={"runner": "e2e-suite-cascade-test"},
+        experiment_tags=["e2e", "test-suite", "cascade"],
+        spans_per_trace=2,
+        # Per-item runtime assertion results are seeded with the SAME name
+        # as the v2 suite-level evaluator (``v1-judge``). That's what a real
+        # test-suite run produces: for each item, one assertion result per
+        # evaluator that ran against it. Q2 also has a per-item override
+        # evaluator ``q2-item-judge``; we seed a second result there to
+        # mirror what an actual evaluator run would produce.
+        per_item_extras=[
+            {
+                "assertion_results": [
+                    {
+                        "value": "v1-judge",  # matches v2's suite-level evaluator
+                        "passed": i % 2 == 0,
+                        "reason": f"v1-judge ran on Q{i + 1}",
+                    }
+                ]
+                # Q2 has a per-item override evaluator ``q2-item-judge``; a
+                # real run would also produce a result for that. We tack it
+                # on for the second item.
+                + (
+                    [
+                        {
+                            "value": "q2-item-judge",
+                            "passed": True,
+                            "reason": "q2-item-judge ran on Q2",
+                        }
+                    ]
+                    if i == 1
+                    else []
+                ),
+            }
+            for i in range(len(v2_item_ids))
+        ],
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            suite_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify target ──
+    target = rest.datasets.get_dataset_by_identifier(
+        dataset_name=suite_name, project_name=target_project_name
+    )
+    assert target.type == "evaluation_suite", (
+        f"target should be a test suite, got type={target.type!r}"
+    )
+
+    tgt_versions = chronological_versions(rest, target.id)
+    assert len(tgt_versions) == len(src_versions), (
+        f"target version count {len(tgt_versions)} != source {len(src_versions)} "
+        "— Slice 2 contract requires N=N for test suites too"
+    )
+
+    for src_v, tgt_v, exp in zip(src_versions, tgt_versions, expected):
+        items = stream_items_wire(
+            rest,
+            dataset_name=suite_name,
+            project_name=target_project_name,
+            version_hash=tgt_v.version_hash,
+        )
+        # Items: set-equal under content hash + display order matches.
+        actual_hashes = item_hashes(items)
+        assert actual_hashes == exp["hashes"], (
+            f"{tgt_v.version_name}: items diverged from {src_v.version_name}. "
+            f"Missing on target: {exp['hashes'] - actual_hashes}; "
+            f"extra on target: {actual_hashes - exp['hashes']}"
+        )
+        assert display_order(items) == exp["order"], (
+            f"{tgt_v.version_name}: display order diverged"
+        )
+        # Suite-level config matches.
+        assert normalize_evaluators(tgt_v.evaluators) == exp["suite_evals"], (
+            f"{tgt_v.version_name}: suite evaluators diverged"
+        )
+        assert normalize_policy(tgt_v.execution_policy) == exp["suite_pol"], (
+            f"{tgt_v.version_name}: suite execution_policy diverged"
+        )
+        # Version-level metadata + user tags match.
+        assert (tgt_v.metadata or None) == exp["metadata"], (
+            f"{tgt_v.version_name}: metadata diverged"
+        )
+        assert strip_be_managed_version_tags(tgt_v.tags) == exp["user_tags"], (
+            f"{tgt_v.version_name}: user version tags diverged"
+        )
+
+    # ── Cascade fidelity (test-suite-specific) ──
+    # The destination project should now have a copy of the source
+    # suite-driven experiment: type + evaluation_method preserved,
+    # per-item assertion_results + execution_policy round-trip, FKs
+    # remapped, traces+spans land under the destination project.
+    dest_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=target.id,
+        experiment_name=experiment_name,
+    )
+    assert dest_exp.id != cascade_seed["experiment_id"]
+    assert dest_exp.dataset_id == target.id
+    # Type + evaluation_method preserved (test-suite-specific fields).
+    assert dest_exp.type == "regular", (
+        f"destination experiment type should be 'regular', got {dest_exp.type!r}"
+    )
+    assert dest_exp.evaluation_method == "evaluation_suite", (
+        f"destination experiment evaluation_method should be 'evaluation_suite', "
+        f"got {dest_exp.evaluation_method!r}"
+    )
+    target_version_ids = {v.id for v in tgt_versions}
+    assert dest_exp.dataset_version_id in target_version_ids
+
+    # Per-item assertion_results survive on the destination via the
+    # Compare view (the BE persists them through the dedicated
+    # ``assertion_results.store_assertions_batch`` endpoint scoped to the
+    # destination trace ids; the Compare view aggregates them onto each
+    # ExperimentItemCompare for read).
+    dest_items = destination_experiment_items(
+        rest,
+        experiment_id=dest_exp.id,
+        dataset_id=target.id,
+    )
+    assert len(dest_items) == len(v2_item_ids)
+    dest_trace_ids = {it.trace_id for it in dest_items}
+    assert dest_trace_ids.isdisjoint(set(cascade_seed["trace_ids"]))
+
+    # Each destination item has at least one assertion result named
+    # ``v1-judge`` (the v2 suite-level evaluator's name), matching what a
+    # real test-suite run would produce. The Q2 item additionally has a
+    # ``q2-item-judge`` result because of the per-item evaluator override.
+    items_with_q2_override = 0
+    for dest_item in dest_items:
+        ars = dest_item.assertion_results
+        assert ars, "destination experiment item should have assertion_results"
+        names = {a.value for a in ars}
+        assert "v1-judge" in names, (
+            f"destination item should carry a 'v1-judge' assertion result "
+            f"(matches the v2 suite-level evaluator); got names={names}"
+        )
+        if "q2-item-judge" in names:
+            items_with_q2_override += 1
+            # Find the q2-item-judge result and verify its reason text.
+            q2_ar = next(a for a in ars if a.value == "q2-item-judge")
+            assert q2_ar.passed is True
+            assert "Q2" in (q2_ar.reason or "")
+    assert items_with_q2_override == 1, (
+        "exactly one item (Q2) should carry the per-item q2-item-judge "
+        f"assertion result; got {items_with_q2_override}"
+    )
+
+    # Each destination trace exists under the target project with the
+    # span tree shape preserved (root + 1 child, parent_span_id remapped).
+    for new_trace_id in dest_trace_ids:
+        dest_spans = destination_spans_for_trace(
+            rest,
+            trace_id=new_trace_id,
+            project_name=target_project_name,
+        )
+        assert len(dest_spans) == 2
+        roots = [s for s in dest_spans if s.parent_span_id is None]
+        assert len(roots) == 1
+        children = [s for s in dest_spans if s.parent_span_id is not None]
+        assert all(c.parent_span_id == roots[0].id for c in children)
+
+    # ── Deep-equal source vs. destination ──
+    # Pair source/destination items by trace ``name`` (assigned by the
+    # seed as "task-0", "task-1", "task-2" and carried verbatim through
+    # the cascade), then run a field-by-field comparison of experiment +
+    # items (assertion_results + feedback_scores) + traces + spans
+    # modulo remapped IDs.
+    src_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=suite_id,
+        experiment_name=experiment_name,
+    )
+    src_items_compare = destination_experiment_items(
+        rest,
+        experiment_id=cascade_seed["experiment_id"],
+        dataset_id=suite_id,
+    )
+    src_trace_names = {
+        it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+        for it in src_items_compare
+    }
+    dst_trace_names = {
+        it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+        for it in dest_items
+    }
+    src_items_compare.sort(key=lambda it: src_trace_names[it.trace_id])
+    dest_items_sorted = sorted(dest_items, key=lambda it: dst_trace_names[it.trace_id])
+    src_trace_ids_sorted = [it.trace_id for it in src_items_compare]
+    dst_trace_ids_sorted = [it.trace_id for it in dest_items_sorted]
+
+    compare_cascade(
+        rest_client=rest,
+        source_experiment=src_exp,
+        destination_experiment=dest_exp,
+        source_item_ids=v2_item_ids,
+        destination_item_ids=[it.dataset_item_id for it in dest_items_sorted],
+        source_trace_ids=src_trace_ids_sorted,
+        destination_trace_ids=dst_trace_ids_sorted,
+        source_items_compare=src_items_compare,
+        destination_items_compare=dest_items_sorted,
+    )

sdks/python/tests/e2e/compatibility_v1/__init__.py

@@ -0,0 +1 @@
+# This is tests to check compatibility with OPIK 1.x

sdks/python/tests/e2e/compatibility_v1/evaluation/test_evaluate_dataset_version.py

@@ -0,0 +1,87 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.evaluation.metrics import score_result
+
+from tests.e2e import verifiers
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+def test_evaluate__with_dataset_version__evaluates_version_items_only__happyflow(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that opik.evaluate works with DatasetVersion and only evaluates items from that version."""
+    dataset = opik_client.create_dataset(dataset_name)
+
+    # Insert first batch - creates v1 with 2 items
+    dataset.insert(
+        [
+            {"input": {"question": "Q1"}, "expected_output": {"answer": "A1"}},
+            {"input": {"question": "Q2"}, "expected_output": {"answer": "A2"}},
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Insert second batch - creates v2 with 4 items total
+    dataset.insert(
+        [
+            {"input": {"question": "Q3"}, "expected_output": {"answer": "A3"}},
+            {"input": {"question": "Q4"}, "expected_output": {"answer": "A4"}},
+        ]
+    )
+    _wait_for_version(dataset, "v2")
+
+    # Get v1 view - should have only 2 items
+    v1_view = dataset.get_version_view("v1")
+    assert v1_view.items_total == 2
+
+    # Simple task that returns the expected output
+    def task(item: Dict[str, Any]) -> Dict[str, Any]:
+        return item["expected_output"]
+
+    # Simple scoring function
+    def scoring_function(
+        dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+    ) -> score_result.ScoreResult:
+        return score_result.ScoreResult(name="test_score", value=1.0)
+
+    # Evaluate using DatasetVersion (v1)
+    result = opik.evaluate(
+        dataset=v1_view,
+        task=task,
+        scoring_functions=[scoring_function],
+        verbose=0,
+    )
+
+    opik.flush_tracker()
+
+    # Should have evaluated only 2 items (from v1), not 4 (from v2/current)
+    assert len(result.test_results) == 2
+
+    # Verify the items evaluated were from v1
+    evaluated_questions = {
+        tr.test_case.dataset_item_content["input"]["question"]
+        for tr in result.test_results
+    }
+    assert evaluated_questions == {"Q1", "Q2"}
+
+    # Verify the experiment is linked to v1's version ID
+    v1_version_info = v1_view.get_version_info()
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=result.experiment_id,
+        experiment_name=result.experiment_name,
+        experiment_metadata=None,
+        traces_amount=2,
+        feedback_scores_amount=1,
+        dataset_version_id=v1_version_info.id,
+    )

sdks/python/tests/e2e/compatibility_v1/evaluation/test_evaluate_filter_string.py

@@ -0,0 +1,133 @@
+from typing import Dict, Any
+
+import opik
+from opik import id_helpers
+from opik.evaluation import metrics
+from opik.evaluation import evaluator as evaluator_module
+
+
+def test_evaluate__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "geography"',
+    )
+
+    opik.flush_tracker()
+
+    retrieved_experiment = opik_client.get_experiment_by_name(experiment_name)
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items (filtered by geography category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+
+def test_evaluate_optimization_trial__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate_optimization_trial correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    evaluator_module.evaluate_optimization_trial(
+        optimization_id=id_helpers.generate_id(),
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "math"',
+    )
+
+    opik.flush_tracker()
+
+    retrieved_experiment = opik_client.get_experiment_by_name(experiment_name)
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1, (
+        f"Expected 1 experiment item (filtered by math category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )

sdks/python/tests/e2e/compatibility_v1/evaluation/test_evaluate_scoring_functions.py

@@ -0,0 +1,318 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.api_objects.experiment import experiment_item
+from opik.evaluation import metrics
+from opik.evaluation.metrics import score_result
+from opik.message_processing.emulation import models
+from opik.types import FeedbackScoreDict
+
+from tests.e2e import verifiers
+from tests.testlib import assert_equal, ANY_BUT_NONE
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+DATASET_ITEMS = [
+    {
+        "input": {"question": "What is the of capital of Ukraine?"},
+        "expected_model_output": {"output": "Kyiv"},
+    },
+]
+
+
+def llm_task(item: Dict[str, Any]):
+    if item["input"] == {"question": "What is the of capital of Ukraine?"}:
+        return {"output": "Kyiv"}
+
+    raise AssertionError(
+        f"Task received dataset item with an unexpected input: {item['input']}"
+    )
+
+
+def equals_scoring_function(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]):
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="Correct output value" if value == 1.0 else "Incorrect output value",
+    )
+
+
+def task_span_name_scoring_function(
+    task_span: models.SpanModel, **ignored_kwargs: Any
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_scoring_function",
+        reason="Correct task span name" if score == 1.0 else "Incorrect task span name",
+    )
+
+
+def task_span_name_and_equals_scoring_function(
+    dataset_item: Dict[str, Any],
+    task_outputs: Dict[str, Any],
+    task_span: models.SpanModel,
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+
+    score *= value
+
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_and_equals_scoring_function",
+        reason="Correct task span name and output value"
+        if score == 1.0
+        else "Incorrect task span name and output value",
+    )
+
+
+def test_evaluate__scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that ordinary scoring functions work correctly.
+    dataset = opik_client.create_dataset(dataset_name)
+    dataset.insert(DATASET_ITEMS)
+    _wait_for_version(dataset, "v1")
+
+    # Get the version ID to verify it's passed to the experiment
+    version_info = dataset.get_version_info()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+    )
+
+    opik.flush_tracker()
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=None,
+        dataset_version_id=version_info.id,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        )
+    ]
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+        actual=experiment_items_contents,
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_scoring_metrics__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and scoring metrics work correctly.
+    dataset = opik_client.create_dataset(dataset_name)
+    dataset.insert(DATASET_ITEMS)
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_metrics=[equals_metric],
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    opik.flush_tracker()
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=2,
+        prompts=None,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None, name="equals_metric", reason=None, value=1.0
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+            ],
+        )
+    ]
+
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_task_span_scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and task span scoring functions work correctly.
+    # Also, it checks that task span scoring functions can access:
+    # task span, dataset item content (dataset_item), and task output (task_outputs) parameters.
+    dataset = opik_client.create_dataset(dataset_name)
+    dataset.insert(DATASET_ITEMS)
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[
+            equals_scoring_function,
+            task_span_name_scoring_function,
+            task_span_name_and_equals_scoring_function,
+        ],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+    )
+
+    opik.flush_tracker()
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=3,
+        prompts=None,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_and_equals_scoring_function",
+                    reason="Correct task span name and output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_scoring_function",
+                    reason="Correct task span name",
+                    value=1.0,
+                ),
+            ],
+        ),
+    ]
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )

sdks/python/tests/e2e/compatibility_v1/evaluation/test_evaluate_streaming.py

@@ -0,0 +1,154 @@
+import threading
+from typing import Dict, Any, List
+from unittest import mock
+
+import opik
+from opik.api_objects import rest_stream_parser
+from opik.evaluation import helpers as helpers_module
+from opik.evaluation.metrics import score_result
+from opik.types import FeedbackScoreDict
+
+
+# Simple dataset items for testing
+DATASET_ITEMS = [
+    {
+        "input": {"question": f"Question {i}"},
+        "expected_output": {"answer": f"Answer {i}"},
+    }
+    for i in range(20)  # Create 20 items to test streaming behavior
+]
+
+# Batch size for streaming - must be less than len(DATASET_ITEMS) to test multi-batch streaming
+TEST_BATCH_SIZE = 5
+
+
+def simple_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Simple task that echoes the expected output."""
+    return item["expected_output"]
+
+
+def simple_scoring_function(
+    dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+) -> score_result.ScoreResult:
+    """Simple scoring function that always returns 1.0."""
+    return score_result.ScoreResult(
+        name="simple_score",
+        value=1.0,
+        reason="Test score",
+    )
+
+
+def test_streaming_starts_evaluation_before_complete_download(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    Test that streaming evaluation starts processing items before all items are downloaded.
+
+    This test verifies the core streaming behavior by:
+    1. Creating a dataset with multiple items
+    2. Setting a small batch size to force multiple batches
+    3. Patching read_and_parse_stream to track when items are yielded
+    4. Patching the task execution to track when tasks start
+    5. Verifying that the first task starts before the last item is yielded
+    """
+    # Create dataset with multiple items
+    dataset = opik_client.create_dataset(dataset_name)
+    dataset.insert(DATASET_ITEMS)
+
+    # Track the sequence of events: 'yield' or 'task'
+    events: List[str] = []
+    events_lock = threading.Lock()
+
+    # Store original read_and_parse_stream function
+    original_read_and_parse_stream = rest_stream_parser.read_and_parse_stream
+
+    def tracked_read_and_parse_stream(stream, item_class, nb_samples=None):
+        """Wrapper that tracks when items are yielded from the stream."""
+        items = original_read_and_parse_stream(stream, item_class, nb_samples)
+        tracked_items = []
+        for item in items:
+            with events_lock:
+                events.append("yield")
+            tracked_items.append(item)
+        return tracked_items
+
+    def tracked_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        """Wrapper that tracks when tasks start executing."""
+        with events_lock:
+            events.append("task")
+        return simple_task(item)
+
+    # Patch both read_and_parse_stream to track yields and EVALUATION_STREAM_DATASET_BATCH_SIZE
+    # to ensure we stream in multiple batches
+    with (
+        mock.patch.object(
+            rest_stream_parser,
+            "read_and_parse_stream",
+            side_effect=tracked_read_and_parse_stream,
+        ),
+        mock.patch.object(
+            helpers_module, "EVALUATION_STREAM_DATASET_BATCH_SIZE", TEST_BATCH_SIZE
+        ),
+    ):
+        # Run evaluation with streaming
+        evaluation_result = opik.evaluate(
+            dataset=dataset,
+            task=tracked_task,
+            scoring_functions=[simple_scoring_function],
+            experiment_name=experiment_name,
+            verbose=1,
+        )
+
+        opik.flush_tracker()
+
+    # Verify evaluation completed successfully
+    assert evaluation_result.dataset_id == dataset.id
+    assert len(events) == len(DATASET_ITEMS) * 2  # Each item has yield + task
+
+    # Count yields and tasks
+    yield_count = events.count("yield")
+    task_count = events.count("task")
+    assert yield_count == len(DATASET_ITEMS)
+    assert task_count == len(DATASET_ITEMS)
+
+    # Critical assertion: Tasks should be interleaved with yields
+    # This proves streaming is working - we don't wait for all yields before starting tasks
+    # Find the index of the last yield
+    last_yield_index = len(events) - 1 - events[::-1].index("yield")
+    # Find the index of the first task
+    first_task_index = events.index("task")
+
+    assert first_task_index < last_yield_index, (
+        f"Streaming not working! First task appeared at index {first_task_index}, "
+        f"but last yield appeared at index {last_yield_index}. "
+        f"Tasks should start before all items are yielded. "
+        f"Event sequence: {events}"
+    )
+
+    # Verify experiment was created correctly
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items = retrieved_experiment.get_items()
+    assert len(experiment_items) == len(DATASET_ITEMS)
+
+    # Verify scoring output: each item should have a score with name "simple_score" and value 1.0
+    for item in experiment_items:
+        # Check that feedback_scores exists and is not empty
+        assert item.feedback_scores is not None and len(item.feedback_scores) == 1, (
+            f"Experiment item {item.id} should have exactly 1 feedback score, got {len(item.feedback_scores) if item.feedback_scores else 0}"
+        )
+
+        # Verify the score matches expected structure
+        expected_score = FeedbackScoreDict(
+            category_name=None,
+            name="simple_score",
+            reason="Test score",
+            value=1.0,
+        )
+
+        actual_score = item.feedback_scores[0]
+        assert actual_score == expected_score, (
+            f"Experiment item {item.id} has incorrect feedback score. "
+            f"Expected: {expected_score}, Got: {actual_score}"
+        )

sdks/python/tests/e2e/compatibility_v1/evaluation/test_experiment_scoring_functions.py

@@ -0,0 +1,141 @@
+import statistics
+from typing import Dict, Any, List
+
+import pytest
+
+import opik
+from opik.evaluation import metrics
+from opik.evaluation import test_result
+from opik.evaluation.metrics import score_result
+from tests.e2e import verifiers
+
+
+def test_experiment_scoring_functions__standard_deviation__computed_and_logged(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions can compute and log standard deviation of metric scores."""
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": "Paris",
+            },
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_output": "Berlin",
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_output": "Warsaw",
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        # Return correct answers for France and Germany, wrong answer for Poland
+        # This will generate scores: 1.0, 1.0, 0.0
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris", "reference": item["expected_output"]}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin", "reference": item["expected_output"]}
+        if item["input"] == {"question": "What is the capital of Poland?"}:
+            return {"output": "Krakow", "reference": item["expected_output"]}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    def compute_std_deviation(
+        test_results: List[test_result.TestResult],
+    ) -> List[score_result.ScoreResult]:
+        """Compute standard deviation of metric scores across all test results."""
+        scores = [x.score_results[0].value for x in test_results]
+        return [
+            score_result.ScoreResult(
+                name="equals_metric_std_dev",
+                value=statistics.stdev(scores) if len(scores) > 1 else 0.0,
+                reason=f"Standard deviation of {len(scores)} metric scores",
+            )
+        ]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        experiment_scoring_functions=[compute_std_deviation],
+    )
+
+    opik.flush_tracker()
+
+    # Verify experiment scores are in the result
+    assert len(evaluation_result.experiment_scores) == 1, (
+        f"Expected 1 experiment score in evaluation result, but got {len(evaluation_result.experiment_scores)}. "
+        f"Experiment scores: {evaluation_result.experiment_scores}"
+    )
+    assert evaluation_result.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev', but got '{evaluation_result.experiment_scores[0].name}'. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+
+    expected_stdev = statistics.stdev([1.0, 1.0, 0.0])
+    assert evaluation_result.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev}, but got {evaluation_result.experiment_scores[0].value}. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].reason is not None, (
+        f"Expected experiment score reason to be set, but got None. "
+        f"Score: {evaluation_result.experiment_scores[0]}"
+    )
+    assert (
+        "Standard deviation of 3 metric scores"
+        in evaluation_result.experiment_scores[0].reason
+    ), (
+        f"Expected reason to contain 'Standard deviation of 3 metric scores', but got: '{evaluation_result.experiment_scores[0].reason}'"
+    )
+
+    # Verify experiment was created
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=3,  # one trace per dataset item
+        feedback_scores_amount=1,
+    )
+
+    # Verify experiment scores are logged to backend by retrieving the experiment
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_data = retrieved_experiment.get_experiment_data()
+
+    # Check that experiment scores are present in the experiment data
+    assert experiment_data.experiment_scores is not None, (
+        f"Expected experiment_scores to be set in experiment data, but got None. "
+        f"Experiment ID: {evaluation_result.experiment_id}, "
+        f"Experiment name: {evaluation_result.experiment_name}, "
+        f"Experiment data: {experiment_data}"
+    )
+    assert len(experiment_data.experiment_scores) == 1, (
+        f"Expected 1 experiment score in backend, but got {len(experiment_data.experiment_scores) if experiment_data.experiment_scores else 0}. "
+        f"Experiment scores: {experiment_data.experiment_scores}"
+    )
+    assert experiment_data.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev' in backend, but got '{experiment_data.experiment_scores[0].name}'. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )
+    assert experiment_data.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev} in backend, but got {experiment_data.experiment_scores[0].value}. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )

sdks/python/tests/e2e/compatibility_v1/test_chat_prompt.py

@@ -0,0 +1,605 @@
+import uuid
+import opik
+from opik.api_objects.prompt import PromptType, ChatPrompt
+from .. import verifiers
+import opik.exceptions
+import pytest
+
+
+def test_chat_prompt__create__happyflow(opik_client: opik.Opik):
+    """Test creating a chat prompt with multiple messages."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Hello, {{name}}!"},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        metadata={"version": "1.0", "type": "customer_support"},
+    )
+
+    # Verify the prompt was created correctly
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        metadata={"version": "1.0", "type": "customer_support"},
+        type=PromptType.MUSTACHE,
+    )
+
+
+def test_chat_prompt__format__happyflow(opik_client: opik.Opik):
+    """Test formatting a chat prompt with variables."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-format-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a {{role}}."},
+        {"role": "user", "content": "Hello, {{name}}! Tell me about {{topic}}."},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Verify the prompt was created correctly
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Format with variables
+    formatted = chat_prompt.format(
+        variables={"role": "coding assistant", "name": "Alice", "topic": "Python"}
+    )
+
+    assert len(formatted) == 2
+    assert formatted[0]["role"] == "system"
+    assert formatted[0]["content"] == "You are a coding assistant."
+    assert formatted[1]["role"] == "user"
+    assert formatted[1]["content"] == "Hello, Alice! Tell me about Python."
+
+
+def test_chat_prompt__create_new_version__happyflow(opik_client: opik.Opik):
+    """Test creating a new version of a chat prompt."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-versioning-{unique_identifier}"
+    messages_v1 = [
+        {"role": "system", "content": "You are helpful."},
+        {"role": "user", "content": "Hi!"},
+    ]
+
+    # Create initial version
+    chat_prompt_v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages_v1,
+    )
+
+    # Create new version with different messages
+    messages_v2 = [
+        {"role": "system", "content": "You are very helpful."},
+        {"role": "user", "content": "Hello there!"},
+        {"role": "assistant", "content": "How can I assist you?"},
+    ]
+
+    chat_prompt_v2 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages_v2,
+    )
+
+    # Verify both versions
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v1,
+        name=prompt_name,
+        messages=messages_v1,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v2,
+        name=prompt_name,
+        messages=messages_v2,
+    )
+
+    # Verify they share the same prompt ID but have different version IDs
+    assert (
+        chat_prompt_v2.__internal_api__prompt_id__
+        == chat_prompt_v1.__internal_api__prompt_id__
+    )
+    assert (
+        chat_prompt_v2.__internal_api__version_id__
+        != chat_prompt_v1.__internal_api__version_id__
+    )
+    assert chat_prompt_v2.commit != chat_prompt_v1.commit
+
+
+def test_chat_prompt__do_not_create_new_version_with_same_messages(
+    opik_client: opik.Opik,
+):
+    """Test that creating a chat prompt with identical messages doesn't create a new version."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-no-dup-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are helpful."},
+        {"role": "user", "content": "Hello!"},
+    ]
+
+    # Create initial version
+    chat_prompt_v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Try to create with same messages
+    chat_prompt_v2 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Verify both prompts
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v1,
+        name=prompt_name,
+        messages=messages,
+        commit=chat_prompt_v1.commit,
+        prompt_id=chat_prompt_v1.__internal_api__prompt_id__,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v2,
+        name=prompt_name,
+        messages=messages,
+        commit=chat_prompt_v1.commit,
+        prompt_id=chat_prompt_v1.__internal_api__prompt_id__,
+    )
+
+
+def test_chat_prompt__direct_class_instantiation__happyflow(opik_client: opik.Opik):
+    """Test creating a ChatPrompt directly using the class."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-direct-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are an expert."},
+        {"role": "user", "content": "Explain {{concept}}."},
+    ]
+
+    # Create directly via ChatPrompt class
+    chat_prompt = ChatPrompt(
+        name=prompt_name,
+        messages=messages,
+        metadata={"category": "education"},
+    )
+
+    # Verify it was synced to backend
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        metadata={"category": "education"},
+    )
+
+
+def test_chat_prompt__multimodal_content__happyflow(opik_client: opik.Opik):
+    """Test chat prompt with multimodal content (text + images)."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-multimodal-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You analyze images."},
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image of {{subject}}?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/image.jpg"},
+                },
+            ],
+        },
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Verify multimodal content is preserved
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+    )
+
+    assert len(chat_prompt.template[1]["content"]) == 2
+    assert chat_prompt.template[1]["content"][0]["type"] == "text"
+    assert chat_prompt.template[1]["content"][1]["type"] == "image_url"
+
+
+def test_chat_prompt__different_types__mustache_and_jinja2(opik_client: opik.Opik):
+    """Test chat prompts with different template types."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    # Test with Mustache
+    prompt_name_mustache = f"chat-prompt-mustache-{unique_identifier}"
+    messages_mustache = [
+        {"role": "user", "content": "Hello {{name}}!"},
+    ]
+
+    chat_prompt_mustache = opik_client.create_chat_prompt(
+        name=prompt_name_mustache,
+        messages=messages_mustache,
+        type=PromptType.MUSTACHE,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_mustache,
+        name=prompt_name_mustache,
+        messages=messages_mustache,
+        type=PromptType.MUSTACHE,
+    )
+
+    formatted_mustache = chat_prompt_mustache.format(variables={"name": "Bob"})
+    assert formatted_mustache[0]["content"] == "Hello Bob!"
+
+    # Test with Jinja2
+    prompt_name_jinja = f"chat-prompt-jinja-{unique_identifier}"
+    messages_jinja = [
+        {"role": "user", "content": "Hello {{ name }}!"},
+    ]
+
+    chat_prompt_jinja = opik_client.create_chat_prompt(
+        name=prompt_name_jinja,
+        messages=messages_jinja,
+        type=PromptType.JINJA2,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_jinja,
+        name=prompt_name_jinja,
+        messages=messages_jinja,
+        type=PromptType.JINJA2,
+    )
+
+    formatted_jinja = chat_prompt_jinja.format(variables={"name": "Carol"})
+    assert formatted_jinja[0]["content"] == "Hello Carol!"
+
+
+def test_chat_prompt__empty_messages__should_work(opik_client: opik.Opik):
+    """Test chat prompt with empty messages list."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-empty-{unique_identifier}"
+    messages = []
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=[],
+    )
+
+
+def test_chat_prompt__multiple_roles__happyflow(opik_client: opik.Opik):
+    """Test chat prompt with system, user, and assistant messages."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-multirole-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is the capital of {{country}}?"},
+        {"role": "assistant", "content": "I'd be happy to help with that!"},
+        {"role": "user", "content": "Please tell me."},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+    )
+
+    formatted = chat_prompt.format(variables={"country": "France"})
+    assert len(formatted) == 4
+    assert formatted[0]["role"] == "system"
+    assert formatted[1]["role"] == "user"
+    assert formatted[1]["content"] == "What is the capital of France?"
+    assert formatted[2]["role"] == "assistant"
+    assert formatted[3]["role"] == "user"
+
+
+def test_get_chat_prompt__chat_prompt__returns_chat_prompt(opik_client: opik.Opik):
+    """Test that get_chat_prompt() returns a ChatPrompt object for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-get-{unique_id}"
+
+    # Create a chat prompt
+    created_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are helpful"},
+            {"role": "user", "content": "Hello {{name}}"},
+        ],
+    )
+
+    # Retrieve it with get_chat_prompt()
+    retrieved_prompt = opik_client.get_chat_prompt(name=prompt_name)
+
+    assert retrieved_prompt is not None
+    assert isinstance(retrieved_prompt, opik.ChatPrompt)
+    assert retrieved_prompt.name == prompt_name
+    assert retrieved_prompt.commit == created_prompt.commit
+    assert len(retrieved_prompt.template) == 2
+
+
+def test_get_chat_prompt__string_prompt__prompt_structure_mismatch_error(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt() raises an error for text prompts (type mismatch)."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"text-prompt-{unique_id}"
+
+    # Create a text prompt
+    text_prompt = opik_client.create_prompt(
+        name=prompt_name,
+        prompt="Hello {{name}}",
+    )
+
+    # Try to retrieve it with get_chat_prompt() - should raise an error due to type mismatch
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.get_chat_prompt(name=prompt_name)
+
+    # Verify the text prompt remains unchanged
+    retrieved_prompt = opik_client.get_prompt(name=prompt_name)
+    assert retrieved_prompt is not None
+    verifiers.verify_prompt_version(
+        retrieved_prompt,
+        name=prompt_name,
+        template=text_prompt.prompt,
+        commit=text_prompt.commit,
+        prompt_id=text_prompt.__internal_api__prompt_id__,
+        version_id=text_prompt.__internal_api__version_id__,
+    )
+
+
+def test_get_chat_prompt_history__chat_prompt__returns_chat_prompts(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt_history() returns ChatPrompt objects for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-history-{unique_id}"
+
+    # Create multiple versions of a chat prompt
+    v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 1"}],
+    )
+    v2 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 2"}],
+    )
+    v3 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "System message"},
+            {"role": "user", "content": "Version 3"},
+        ],
+    )
+
+    # Retrieve history
+    history = opik_client.get_chat_prompt_history(name=prompt_name)
+
+    assert len(history) == 3
+    assert all(isinstance(p, opik.ChatPrompt) for p in history)
+    assert history[0].name == prompt_name
+
+    # Verify commits are in the history
+    commits = {p.commit for p in history}
+    assert v1.commit in commits
+    assert v2.commit in commits
+    assert v3.commit in commits
+
+
+def test_get_chat_prompt_history__string_prompt__prompt_structure_mismatch_error(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt_history() raises an error for text prompts (type mismatch)."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"text-prompt-history-{unique_id}"
+
+    # Create a text prompt
+    text_prompt = opik_client.create_prompt(name=prompt_name, prompt="Hello")
+
+    # Try to get history with get_chat_prompt_history() - should raise an error due to type mismatch
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.get_chat_prompt_history(name=prompt_name)
+
+    # Verify the text prompt remains unchanged
+    retrieved_prompt = opik_client.get_prompt(name=prompt_name)
+    assert retrieved_prompt is not None
+    verifiers.verify_prompt_version(
+        retrieved_prompt,
+        name=prompt_name,
+        template=text_prompt.prompt,
+        commit=text_prompt.commit,
+        prompt_id=text_prompt.__internal_api__prompt_id__,
+        version_id=text_prompt.__internal_api__version_id__,
+    )
+
+
+def test_search_prompts__filter_chat_prompts_only(opik_client: opik.Opik):
+    """Test that search_prompts() with template_structure filter returns only ChatPrompt instances."""
+    unique_id = str(uuid.uuid4())[-6:]
+
+    # Create text prompts
+    text_prompt_1 = opik_client.create_prompt(
+        name=f"text-search-{unique_id}-1",
+        prompt="Text prompt 1",
+    )
+    text_prompt_2 = opik_client.create_prompt(
+        name=f"text-search-{unique_id}-2",
+        prompt="Text prompt 2",
+    )
+
+    # Create chat prompts
+    chat_prompt_1 = opik_client.create_chat_prompt(
+        name=f"chat-search-{unique_id}-1",
+        messages=[{"role": "user", "content": "Chat 1"}],
+    )
+    chat_prompt_2 = opik_client.create_chat_prompt(
+        name=f"chat-search-{unique_id}-2",
+        messages=[{"role": "user", "content": "Chat 2"}],
+    )
+
+    # Search for only chat prompts using template_structure filter
+    results = opik_client.search_prompts(
+        filter_string=f'name contains "{unique_id}" AND template_structure = "chat"'
+    )
+
+    # Should only return chat prompts, not text prompts
+    assert len(results) == 2
+    assert all(isinstance(p, opik.ChatPrompt) for p in results)
+
+    result_names = {p.name for p in results}
+    assert chat_prompt_1.name in result_names
+    assert chat_prompt_2.name in result_names
+    assert text_prompt_1.name not in result_names
+    assert text_prompt_2.name not in result_names
+
+
+def test_get_chat_prompt__with_commit__chat_prompt(opik_client: opik.Opik):
+    """Test that get_chat_prompt() with commit works for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-commit-{unique_id}"
+
+    # Create multiple versions
+    v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 1"}],
+    )
+    _ = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 2"}],
+    )
+
+    # Retrieve specific version by commit
+    retrieved_v1 = opik_client.get_chat_prompt(name=prompt_name, commit=v1.commit)
+
+    assert retrieved_v1 is not None
+    assert isinstance(retrieved_v1, opik.ChatPrompt)
+    assert retrieved_v1.commit == v1.commit
+    assert retrieved_v1.template[0]["content"] == "Version 1"
+
+
+def test_get_chat_prompt__nonexistent__returns_none(opik_client: opik.Opik):
+    """Test that get_chat_prompt() returns None for non-existent prompts."""
+    result = opik_client.get_chat_prompt(name="nonexistent-chat-prompt-12345")
+    assert result is None
+
+
+def test_chat_prompt__template_structure_immutable__error(opik_client: opik.Opik):
+    """Test that template_structure is immutable for chat prompts."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+    prompt_name = f"test-immutable-chat-structure-{unique_identifier}"
+
+    # Create initial chat prompt
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Hello!"},
+        ],
+    )
+
+    # Verify chat prompt was created
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Hello!"},
+        ],
+    )
+
+    # Attempt to create a text prompt version with the same name should fail
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.create_prompt(
+            name=prompt_name,
+            prompt="This is a text prompt: {{variable}}",
+        )
+
+    # Verify the original chat prompt remains unchanged
+    retrieved_chat_prompt = opik_client.get_chat_prompt(name=prompt_name)
+    assert retrieved_chat_prompt is not None
+    verifiers.verify_chat_prompt_version(
+        retrieved_chat_prompt,
+        name=prompt_name,
+        messages=chat_prompt.template,
+        commit=chat_prompt.commit,
+        prompt_id=chat_prompt.__internal_api__prompt_id__,
+        version_id=chat_prompt.__internal_api__version_id__,
+    )
+
+
+def test_chat_prompt__invalid_messages__raises_validation_error(opik_client: opik.Opik):
+    """Test that invalid messages raise ValidationError."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-invalid-{unique_identifier}"
+
+    with pytest.raises(opik.exceptions.ValidationError) as exc_info:
+        ChatPrompt(
+            name=prompt_name,
+            messages=[{"role": "invalid", "content": "hello"}],
+        )
+
+    assert "ChatPrompt.__init__" in str(exc_info.value)
+    assert "messages[0].role" in str(exc_info.value)
+
+
+def test_chat_prompt__create_with_tags__happyflow(opik_client: opik.Opik):
+    """Test that create_chat_prompt() accepts tags parameter."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-with-tags-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Hello!"},
+    ]
+    tags = ["chat-tag1", "chat-tag2", "production"]
+
+    # Create chat prompt with tags
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        tags=tags,
+    )
+
+    # Verify prompt was created
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+    )
+
+    # Verify tags were set by searching for the prompt
+    filtered_prompts = opik_client.search_prompts(
+        filter_string=f'name = "{prompt_name}" AND tags contains "chat-tag1"',
+    )
+    assert len(filtered_prompts) == 1
+    assert filtered_prompts[0].name == prompt_name
+    assert set(filtered_prompts[0].tags) == set(tags)

sdks/python/tests/e2e/compatibility_v1/test_dataset.py

@@ -0,0 +1,348 @@
+import opik
+import opik.exceptions
+from opik import synchronization
+
+from opik.api_objects.dataset import dataset_item
+from opik.api_objects import helpers
+from .. import verifiers
+import pytest
+
+
+def test_create_and_populate_dataset__happyflow(
+    opik_client: opik.Opik, dataset_name: str
+):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    EXPECTED_DATASET_ITEMS = [
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of France?"},
+            expected_output={"output": "Paris"},
+        ),
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Germany?"},
+            expected_output={"output": "Berlin"},
+        ),
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Poland?"},
+            expected_output={"output": "Warsaw"},
+        ),
+    ]
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=EXPECTED_DATASET_ITEMS,
+    )
+
+
+def test_insert_and_update_item__dataset_size_should_be_the_same__an_item_with_the_same_id_should_have_new_content(
+    opik_client: opik.Opik, dataset_name: str
+):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    ITEM_ID = helpers.generate_id()
+    dataset.insert(
+        [
+            {
+                "id": ITEM_ID,
+                "input": {"question": "What is the of capital of France?"},
+            },
+        ]
+    )
+    dataset.update(
+        [
+            {
+                "id": ITEM_ID,
+                "input": {"question": "What is the of capital of Belarus?"},
+            },
+        ]
+    )
+    EXPECTED_DATASET_ITEMS = [
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Belarus?"},
+        ),
+    ]
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=EXPECTED_DATASET_ITEMS,
+    )
+
+
+def test_deduplication(opik_client: opik.Opik, dataset_name: str):
+    DESCRIPTION = "E2E test dataset"
+
+    item = {
+        "user_input": {"question": "What is the of capital of France?"},
+        "expected_model_output": {"output": "Paris"},
+    }
+
+    # Write the dataset
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+    dataset.insert([item])
+
+    # Read the dataset and insert the same item
+    new_dataset = opik_client.get_dataset(dataset_name)
+    new_dataset.insert([item])
+
+    # Verify the dataset
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=[
+            dataset_item.DatasetItem(**item),
+        ],
+    )
+
+
+def test_dataset_clearing(opik_client: opik.Opik, dataset_name: str):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+        ]
+    )
+    dataset.clear()
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=[],
+    )
+
+
+def test_get_items_with_filter__returns_filtered_items(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_items with filter_string returns correct filtered items."""
+    DESCRIPTION = "E2E test dataset for filtering"
+
+    # Create dataset with items that have different data.category values
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+                "category": "geography",
+            },
+            {
+                "input": {"question": "What is 2 + 2?"},
+                "expected_output": {"output": "4"},
+                "category": "math",
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_output": {"output": "Warsaw"},
+                "category": "geography",
+            },
+        ]
+    )
+
+    verifiers.verify_dataset_filtered_items(
+        opik_client=opik_client,
+        dataset_name=dataset_name,
+        filter_string='data.category = "geography"',
+        expected_count=2,
+        expected_inputs={
+            "What is the capital of France?",
+            "What is the capital of Poland?",
+        },
+    )
+
+
+def test_get_items_with_filter__filter_excludes_all_items__returns_empty_list(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_items with filter that matches no items returns empty list."""
+    DESCRIPTION = "E2E test dataset for empty filter"
+
+    # Create dataset with items
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "category": "geography",
+            },
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "category": "geography",
+            },
+        ]
+    )
+
+    verifiers.verify_dataset_filtered_items(
+        opik_client=opik_client,
+        dataset_name=dataset_name,
+        filter_string='data.category = "nonexistent"',
+        expected_count=0,
+        expected_inputs=set(),
+    )
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+def test_get_version_view__returns_items_from_specific_version(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_version_view returns items from a specific dataset version.
+
+    Also tests that get_current_version_name returns correct version after mutations.
+    """
+    DESCRIPTION = "E2E test dataset for version view"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    # Version should be None before any items are inserted
+    assert dataset.get_current_version_name() is None
+
+    # Insert first batch of items - creates v1
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Insert second batch of items - creates v2
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v2")
+
+    # Get version view for v1 - should only have 1 item
+    v1_view = dataset.get_version_view("v1")
+    v1_items = v1_view.get_items()
+    assert len(v1_items) == 1
+    assert v1_items[0]["input"] == {"question": "What is the capital of France?"}
+    assert v1_view.version_name == "v1"
+    assert v1_view.items_total == 1
+
+    # Get version view for v2 - should have 2 items
+    v2_view = dataset.get_version_view("v2")
+    v2_items = v2_view.get_items()
+    assert len(v2_items) == 2
+    assert v2_view.version_name == "v2"
+    assert v2_view.items_total == 2
+
+    # Current dataset should also have 2 items
+    current_items = dataset.get_items()
+    assert len(current_items) == 2
+
+    # Delete an item - should create v3
+    dataset.delete([current_items[0]["id"]])
+    _wait_for_version(dataset, "v3")
+
+    # Get version view for v3 - should have 1 item
+    v3_view = dataset.get_version_view("v3")
+    v3_items = v3_view.get_items()
+    assert len(v3_items) == 1
+    assert v3_view.version_name == "v3"
+    assert v3_view.items_total == 1
+
+
+def test_get_items__dataset_created_without_project_name__returns_inserted_items(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Regression test for OPIK-5704.
+
+    Datasets created via the Opik 1.0 UI (or REST API without project_name) have
+    project_id=NULL in the DB. When the SDK fetches such a dataset via get_dataset()
+    and then calls get_items(), items must be returned — not an empty list.
+    """
+    # Simulate Opik 1.0 UI: create dataset at workspace level (no project_name)
+    opik_client._rest_client.datasets.create_dataset(name=dataset_name)
+
+    # SDK lookup — this is the path that used to return [] for items
+    dataset = opik_client.get_dataset(dataset_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+            }
+        ]
+    )
+
+    items = dataset.get_items()
+    assert len(items) == 1, (
+        f"Expected 1 item but got {len(items)}. "
+        "Dataset items are not returned for datasets created without project_name (OPIK-5704)."
+    )
+    assert items[0]["input"] == {"question": "What is the capital of France?"}
+    assert items[0]["expected_output"] == {"output": "Paris"}
+
+
+def test_get_version_view__version_not_found__raises_exception(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_version_view raises DatasetVersionNotFound for non-existent version."""
+    DESCRIPTION = "E2E test dataset for version not found"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    # Insert items to create v1
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Try to get a non-existent version
+    with pytest.raises(opik.exceptions.DatasetVersionNotFound):
+        dataset.get_version_view("v999")

sdks/python/tests/e2e/compatibility_v1/test_experiments.py

@@ -0,0 +1,392 @@
+from typing import Dict, Any
+
+import opik
+from opik.api_objects.experiment import experiment_item
+from opik.evaluation.metrics import score_result
+from opik.types import FeedbackScoreDict
+
+from .. import verifiers
+from tests.testlib import assert_equal, ANY_BUT_NONE
+
+
+def llm_task(item: Dict[str, Any]):
+    if item["input"] == {"question": "What is the capital of Ukraine?"}:
+        return {"output": "Kyiv"}
+    if item["input"] == {"question": "What is the capital of France?"}:
+        return {"output": "Paris"}
+    if item["input"] == {"question": "What is the capital of Germany?"}:
+        return {"output": "Berlin"}
+    if item["input"] == {"question": "What is the capital of Poland?"}:
+        return {"output": "Krakow"}
+
+    raise AssertionError(
+        f"Task received dataset item with an unexpected input: {item['input']}"
+    )
+
+
+def equals_scoring_function(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]):
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="Correct output value" if value == 1.0 else "Incorrect output value",
+    )
+
+
+def test__find_experiment_items_for_dataset__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_model_output": {"output": "Berlin"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+        ]
+    )
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    opik.flush_tracker()
+
+    # make sure experiments saved and available
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=3,  # one trace per dataset item
+        feedback_scores_amount=1,
+    )
+
+    # find experiment items for dataset
+    retrieved_experiment = opik_client.get_experiment_by_name(experiment_name)
+    experiments = opik_client.get_experiments_client()
+    experiment_items_contents = experiments.find_experiment_items_for_dataset(
+        dataset_name=dataset_name,
+        experiment_ids=[retrieved_experiment.id],
+    )
+
+    assert len(experiment_items_contents) == 3
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Ukraine?"},
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Poland?"},
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Incorrect output value",
+                    value=0.0,
+                )
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Berlin"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Germany?"},
+            },
+            evaluation_task_output={"output": "Berlin"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        ),
+    ]
+
+    assert_equal(
+        expected=sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.evaluation_task_output),
+        ),
+        actual=sorted(
+            experiment_items_contents, key=lambda item: str(item.evaluation_task_output)
+        ),
+    )
+
+
+def test__find_experiment_items_for_dataset__filtered__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_model_output": {"output": "Berlin"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+        ]
+    )
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    opik.flush_tracker()
+
+    # make sure experiments saved and available
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=3,  # one trace per dataset item
+        feedback_scores_amount=1,
+    )
+
+    # find experiment items for dataset
+    retrieved_experiment = opik_client.get_experiment_by_name(experiment_name)
+    experiments = opik_client.get_experiments_client()
+    experiment_items_contents = experiments.find_experiment_items_for_dataset(
+        dataset_name=dataset_name,
+        experiment_ids=[retrieved_experiment.id],
+        filter_string="feedback_scores.equals_scoring_function = 0.0",
+    )
+
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Poland?"},
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Incorrect output value",
+                    value=0.0,
+                )
+            ],
+        )
+    ]
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+        actual=experiment_items_contents,
+    )
+
+
+def test__experiment_scores__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions are executed and scores are logged."""
+
+    def compute_experiment_scores(test_results):
+        """Aggregate scores across all test results."""
+        # Extract all scoring function values
+        all_scores = []
+        for result in test_results:
+            if result.score_results:
+                all_scores.extend([score.value for score in result.score_results])
+
+        if not all_scores:
+            return []
+
+        # Compute aggregate metrics
+        return [
+            score_result.ScoreResult(
+                name="max_score",
+                value=max(all_scores),
+                reason=f"Maximum score across {len(all_scores)} measurements",
+            ),
+            score_result.ScoreResult(
+                name="min_score",
+                value=min(all_scores),
+                reason=f"Minimum score across {len(all_scores)} measurements",
+            ),
+            score_result.ScoreResult(
+                name="avg_score",
+                value=sum(all_scores) / len(all_scores),
+                reason=f"Average score across {len(all_scores)} measurements",
+            ),
+        ]
+
+    # Create dataset
+    dataset = opik_client.create_dataset(dataset_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_model_output": {"output": "Berlin"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+        ]
+    )
+
+    # Run evaluation with experiment scoring functions
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_scoring_functions=[compute_experiment_scores],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    opik.flush_tracker()
+
+    # Verify experiment was created with experiment scores
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=3,
+        feedback_scores_amount=1,
+    )
+
+    # Verify experiment scores are present in evaluation result
+    assert evaluation_result.experiment_scores is not None, (
+        "Experiment scores should not be None"
+    )
+    assert len(evaluation_result.experiment_scores) == 3, (
+        f"Expected 3 experiment scores, got {len(evaluation_result.experiment_scores)}"
+    )
+
+    score_names = {score.name for score in evaluation_result.experiment_scores}
+    assert score_names == {
+        "max_score",
+        "min_score",
+        "avg_score",
+    }, f"Expected score names {{max_score, min_score, avg_score}}, got {score_names}"
+
+    # Verify experiment scores are retrievable via SDK API
+    retrieved_experiment = opik_client.get_experiment_by_name(experiment_name)
+    rest_client = opik_client._rest_client
+    experiment_content = rest_client.experiments.get_experiment_by_id(
+        retrieved_experiment.id
+    )
+
+    assert experiment_content.experiment_scores is not None, (
+        "Experiment scores should be persisted in backend"
+    )
+    assert len(experiment_content.experiment_scores) == 3, (
+        f"Expected 3 experiment scores in backend, got {len(experiment_content.experiment_scores)}"
+    )
+
+    backend_score_names = {score.name for score in experiment_content.experiment_scores}
+    assert backend_score_names == {"max_score", "min_score", "avg_score"}, (
+        f"Expected backend score names {{max_score, min_score, avg_score}}, got {backend_score_names}"
+    )
+
+    # Verify score values are reasonable
+    max_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "max_score"
+    )
+    min_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "min_score"
+    )
+    avg_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "avg_score"
+    )
+
+    assert 0.0 <= max_score.value <= 1.0, (
+        f"max_score should be in [0,1], got {max_score.value}"
+    )
+    assert 0.0 <= min_score.value <= 1.0, (
+        f"min_score should be in [0,1], got {min_score.value}"
+    )
+    assert 0.0 <= avg_score.value <= 1.0, (
+        f"avg_score should be in [0,1], got {avg_score.value}"
+    )
+    assert min_score.value <= avg_score.value <= max_score.value, (
+        f"Score ordering should be min <= avg <= max, got {min_score.value} <= {avg_score.value} <= {max_score.value}"
+    )

sdks/python/tests/e2e/compatibility_v1/test_optimization.py

@@ -0,0 +1,40 @@
+import opik
+import pytest
+from opik.rest_api import core as rest_api_core
+from .. import verifiers
+
+
+def test_optimization_lifecycle__happyflow(opik_client: opik.Opik, dataset_name: str):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    # Create optimization
+    optimization = opik_client.create_optimization(
+        objective_name="some-objective-name",
+        dataset_name=dataset.name,
+        name="some-optimization-name",
+    )
+
+    verifiers.verify_optimization(
+        opik_client=opik_client,
+        optimization_id=optimization.id,
+        name="some-optimization-name",
+        dataset_name=dataset.name,
+        status="running",
+        objective_name="some-objective-name",
+    )
+
+    # Update optimization name and status
+    optimization.update(name="new-optimization-name", status="completed")
+    verifiers.verify_optimization(
+        opik_client=opik_client,
+        optimization_id=optimization.id,
+        name="new-optimization-name",
+        dataset_name=dataset.name,
+        status="completed",
+        objective_name="some-objective-name",
+    )
+
+    opik_client.delete_optimizations([optimization.id])
+
+    with pytest.raises(rest_api_core.ApiError):
+        opik_client.get_optimization_by_id(optimization.id)

sdks/python/tests/e2e/conftest.py

@@ -0,0 +1,127 @@
+import os
+import tempfile
+import numpy as np
+import pytest
+
+import opik
+import opik.api_objects.opik_client
+from opik.evaluation.suite_evaluators.llm_judge import config as llm_judge_config
+from opik.rest_api import core as rest_api_core
+from .. import testlib
+from ..conftest import random_chars
+from ..testlib import generate_project_name
+
+ATTACHMENT_FILE_SIZE = 2 * 1024 * 1024
+
+
+@pytest.fixture(autouse=True)
+def _fast_llm_judge_reasoning_effort(monkeypatch):
+    """Force LLMJudge's default reasoning_effort to "minimal" for the e2e
+    suite so LLM-bound assertion runs (test_test_suite, etc.) don't burn
+    time on reasoning tokens. Production default stays "low"."""
+    monkeypatch.setattr(llm_judge_config, "DEFAULT_REASONING_EFFORT", "minimal")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def configure_e2e_tests_env(request):
+    """Patch OPIK_PROJECT_NAME for the duration of the test module.
+
+    Reads the test module's ``PROJECT_NAME`` constant (the single source
+    of truth) so the env var the SDK writes to and the constant the tests
+    verify against can never drift. Files that don't declare
+    ``PROJECT_NAME`` get a fresh per-module project name — they never
+    reference it in Python, so no comparison can fail.
+
+    Module-scoped because all tests in a file share one project under
+    ``--dist=loadfile``.
+
+    On env-var visibility: ``Opik(...)`` reads ``OPIK_PROJECT_NAME``
+    once at construction (caches it as ``self._project_name``); see
+    ``opik.api_objects.opik_client.Opik.__init__``. The patch therefore
+    only takes effect because the ``opik_client`` fixture builds a fresh
+    client per test, and the function-scoped autouse
+    ``shutdown_cached_client_after_test`` resets the global cached
+    client. ``@opik.track`` resolves the client lazily via
+    ``get_client_cached()`` at call time (not at decorator-definition
+    time), so there is no import-time capture to worry about."""
+    project_name = getattr(request.module, "PROJECT_NAME", None)
+    if project_name is None:
+        project_name = generate_project_name("e2e", request.module.__name__)
+    with testlib.patch_environ({"OPIK_PROJECT_NAME": project_name}):
+        yield
+
+
+@pytest.fixture()
+def opik_client(shutdown_cached_client_after_test):
+    opik_client_ = opik.api_objects.opik_client.Opik(batching=True)
+
+    yield opik_client_
+
+    # Tests explicitly poll the backend for anything they care about during
+    # the call phase, so teardown doesn't need to wait for the upload/flush
+    # pipeline to drain. Skip `flush=True` to avoid the 5-s polling budget
+    # in `file_upload_manager.flush`.
+    opik_client_.end(flush=False)
+
+
+@pytest.fixture
+def dataset_name(opik_client: opik.Opik):
+    name = f"e2e-tests-dataset-{random_chars()}"
+    yield name
+
+
+@pytest.fixture
+def experiment_name(opik_client: opik.Opik):
+    name = f"e2e-tests-experiment-{random_chars()}"
+    yield name
+
+
+@pytest.fixture
+def prompt_name():
+    """Unique prompt / chat-prompt name for tests that create prompts.
+
+    Function-scoped because most tests create one prompt and assert on its
+    versions; sharing across tests would entangle version histories."""
+    yield f"e2e-tests-prompt-{random_chars()}"
+
+
+@pytest.fixture
+def environment_name(opik_client: opik.Opik):
+    """A unique environment name for the test; the environment is deleted on teardown.
+
+    Environments are workspace-capped (default 20), so leaking them across runs
+    quickly fills the cap. Cleanup is best-effort — ``delete_environment`` is a
+    no-op if the test never created it."""
+    name = f"e2e-tests-environment-{random_chars()}"
+    yield name
+    try:
+        opik_client.delete_environment(name)
+    except rest_api_core.ApiError:
+        pass
+
+
+@pytest.fixture
+def temporary_project_name(opik_client: opik.Opik):
+    """A unique project name for the test; the project is deleted on teardown.
+
+    Tolerant of projects that were never created (e.g. test bailed before
+    creating one) or already deleted — cleanup is best-effort."""
+    name = f"e2e-tests-temporary-project-{random_chars()}"
+    yield name
+    try:
+        project_id = opik_client.rest_client.projects.retrieve_project(name=name).id
+        opik_client.rest_client.projects.delete_project_by_id(project_id)
+    except rest_api_core.ApiError:
+        pass
+
+
+@pytest.fixture
+def attachment_data_file():
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        temp_file.write(np.random.bytes(ATTACHMENT_FILE_SIZE))
+        temp_file.seek(0)
+        yield temp_file
+    finally:
+        temp_file.close()
+        os.unlink(temp_file.name)

sdks/python/tests/e2e/evaluation/test_evaluate_dataset_version.py

@@ -0,0 +1,90 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.evaluation.metrics import score_result
+
+from .. import verifiers
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+def test_evaluate__with_dataset_version__evaluates_version_items_only__happyflow(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that opik.evaluate works with DatasetVersion and only evaluates items from that version."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    # Insert first batch - creates v1 with 2 items
+    dataset.insert(
+        [
+            {"input": {"question": "Q1"}, "expected_output": {"answer": "A1"}},
+            {"input": {"question": "Q2"}, "expected_output": {"answer": "A2"}},
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Insert second batch - creates v2 with 4 items total
+    dataset.insert(
+        [
+            {"input": {"question": "Q3"}, "expected_output": {"answer": "A3"}},
+            {"input": {"question": "Q4"}, "expected_output": {"answer": "A4"}},
+        ]
+    )
+    _wait_for_version(dataset, "v2")
+
+    # Get v1 view - should have only 2 items
+    v1_view = dataset.get_version_view("v1")
+    assert v1_view.items_total == 2
+
+    # Simple task that returns the expected output
+    def task(item: Dict[str, Any]) -> Dict[str, Any]:
+        return item["expected_output"]
+
+    # Simple scoring function
+    def scoring_function(
+        dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+    ) -> score_result.ScoreResult:
+        return score_result.ScoreResult(name="test_score", value=1.0)
+
+    # Evaluate using DatasetVersion (v1)
+    result = opik.evaluate(
+        dataset=v1_view,
+        task=task,
+        scoring_functions=[scoring_function],
+        verbose=0,
+        project_name=PROJECT_NAME,
+    )
+
+    # Should have evaluated only 2 items (from v1), not 4 (from v2/current)
+    assert len(result.test_results) == 2
+
+    # Verify the items evaluated were from v1
+    evaluated_questions = {
+        tr.test_case.dataset_item_content["input"]["question"]
+        for tr in result.test_results
+    }
+    assert evaluated_questions == {"Q1", "Q2"}
+
+    # Verify the experiment is linked to v1's version ID
+    v1_version_info = v1_view.get_version_info()
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=result.experiment_id,
+        experiment_name=result.experiment_name,
+        experiment_metadata=None,
+        traces_amount=2,
+        feedback_scores_amount=1,
+        dataset_version_id=v1_version_info.id,
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_evaluate_filter_string.py

@@ -0,0 +1,141 @@
+from typing import Dict, Any
+
+import opik
+from opik import id_helpers
+from opik.evaluation import metrics
+from opik.evaluation import evaluator as evaluator_module
+
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_evaluate__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "geography"',
+        project_name=PROJECT_NAME,
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items (filtered by geography category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+
+def test_evaluate_optimization_trial__with_filter_string__filters_dataset_items(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that evaluate_optimization_trial correctly filters dataset items using filter_string."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+            "category": "geography",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is 2+2?"},
+            "expected_model_output": {"output": "4"},
+            "category": "math",
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the capital of Germany?"},
+            "expected_model_output": {"output": "Berlin"},
+            "category": "geography",
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+        if item["input"] == {"question": "What is 2+2?"}:
+            return {"output": "4"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    evaluator_module.evaluate_optimization_trial(
+        optimization_id=id_helpers.generate_id(),
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        dataset_filter_string='data.category = "math"',
+        project_name=PROJECT_NAME,
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1, (
+        f"Expected 1 experiment item (filtered by math category), but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME

sdks/python/tests/e2e/evaluation/test_evaluate_scoring_functions.py

@@ -0,0 +1,320 @@
+from typing import Dict, Any
+
+import opik
+from opik import synchronization
+from opik.api_objects.experiment import experiment_item
+from opik.evaluation import metrics
+from opik.evaluation.metrics import score_result
+from opik.message_processing.emulation import models
+from opik.types import FeedbackScoreDict
+
+from .. import verifiers
+from ...testlib import assert_equal, ANY_BUT_NONE, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+DATASET_ITEMS = [
+    {
+        "input": {"question": "What is the of capital of Ukraine?"},
+        "expected_model_output": {"output": "Kyiv"},
+    },
+]
+
+
+def llm_task(item: Dict[str, Any]):
+    if item["input"] == {"question": "What is the of capital of Ukraine?"}:
+        return {"output": "Kyiv"}
+
+    raise AssertionError(
+        f"Task received dataset item with an unexpected input: {item['input']}"
+    )
+
+
+def equals_scoring_function(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]):
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="Correct output value" if value == 1.0 else "Incorrect output value",
+    )
+
+
+def task_span_name_scoring_function(
+    task_span: models.SpanModel, **ignored_kwargs: Any
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_scoring_function",
+        reason="Correct task span name" if score == 1.0 else "Incorrect task span name",
+    )
+
+
+def task_span_name_and_equals_scoring_function(
+    dataset_item: Dict[str, Any],
+    task_outputs: Dict[str, Any],
+    task_span: models.SpanModel,
+) -> score_result.ScoreResult:
+    score = 1.0 if task_span.name == "llm_task" else 0.0
+
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+
+    score *= value
+
+    return score_result.ScoreResult(
+        value=score,
+        name="task_span_name_and_equals_scoring_function",
+        reason="Correct task span name and output value"
+        if score == 1.0
+        else "Incorrect task span name and output value",
+    )
+
+
+def test_evaluate__scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that ordinary scoring functions work correctly.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+    _wait_for_version(dataset, "v1")
+
+    # Get the version ID to verify it's passed to the experiment
+    version_info = dataset.get_version_info()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=None,
+        dataset_version_id=version_info.id,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        )
+    ]
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+        actual=experiment_items_contents,
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_scoring_metrics__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and scoring metrics work correctly.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_metrics=[equals_metric],
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=2,
+        prompts=None,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None, name="equals_metric", reason=None, value=1.0
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+            ],
+        )
+    ]
+
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )
+
+
+def test_evaluate__scoring_functions_mixed_with_task_span_scoring_functions__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    # Tests that mix of ordinary scoring functions and task span scoring functions work correctly.
+    # Also, it checks that task span scoring functions can access:
+    # task span, dataset item content (dataset_item), and task output (task_outputs) parameters.
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[
+            equals_scoring_function,
+            task_span_name_scoring_function,
+            task_span_name_and_equals_scoring_function,
+        ],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item
+        feedback_scores_amount=3,
+        prompts=None,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_and_equals_scoring_function",
+                    reason="Correct task span name and output value",
+                    value=1.0,
+                ),
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="task_span_name_scoring_function",
+                    reason="Correct task span name",
+                    value=1.0,
+                ),
+            ],
+        ),
+    ]
+    # sort feedback scores by name
+    for item in experiment_items_contents:
+        item.feedback_scores = sorted(item.feedback_scores, key=lambda x: x["name"])
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT, actual=experiment_items_contents
+    )

sdks/python/tests/e2e/evaluation/test_evaluate_streaming.py

@@ -0,0 +1,158 @@
+import threading
+from typing import Dict, Any, List
+from unittest import mock
+
+import opik
+from opik.api_objects import rest_stream_parser
+from opik.evaluation import helpers as helpers_module
+from opik.evaluation.metrics import score_result
+from opik.types import FeedbackScoreDict
+
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+# Simple dataset items for testing
+DATASET_ITEMS = [
+    {
+        "input": {"question": f"Question {i}"},
+        "expected_output": {"answer": f"Answer {i}"},
+    }
+    for i in range(20)  # Create 20 items to test streaming behavior
+]
+
+# Batch size for streaming - must be less than len(DATASET_ITEMS) to test multi-batch streaming
+TEST_BATCH_SIZE = 5
+
+
+def simple_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Simple task that echoes the expected output."""
+    return item["expected_output"]
+
+
+def simple_scoring_function(
+    dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+) -> score_result.ScoreResult:
+    """Simple scoring function that always returns 1.0."""
+    return score_result.ScoreResult(
+        name="simple_score",
+        value=1.0,
+        reason="Test score",
+    )
+
+
+def test_streaming_starts_evaluation_before_complete_download(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    Test that streaming evaluation starts processing items before all items are downloaded.
+
+    This test verifies the core streaming behavior by:
+    1. Creating a dataset with multiple items
+    2. Setting a small batch size to force multiple batches
+    3. Patching read_and_parse_stream to track when items are yielded
+    4. Patching the task execution to track when tasks start
+    5. Verifying that the first task starts before the last item is yielded
+    """
+    # Create dataset with multiple items
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(DATASET_ITEMS)
+
+    # Track the sequence of events: 'yield' or 'task'
+    events: List[str] = []
+    events_lock = threading.Lock()
+
+    # Store original read_and_parse_stream function
+    original_read_and_parse_stream = rest_stream_parser.read_and_parse_stream
+
+    def tracked_read_and_parse_stream(stream, item_class, nb_samples=None):
+        """Wrapper that tracks when items are yielded from the stream."""
+        items = original_read_and_parse_stream(stream, item_class, nb_samples)
+        tracked_items = []
+        for item in items:
+            with events_lock:
+                events.append("yield")
+            tracked_items.append(item)
+        return tracked_items
+
+    def tracked_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        """Wrapper that tracks when tasks start executing."""
+        with events_lock:
+            events.append("task")
+        return simple_task(item)
+
+    # Patch both read_and_parse_stream to track yields and EVALUATION_STREAM_DATASET_BATCH_SIZE
+    # to ensure we stream in multiple batches
+    with (
+        mock.patch.object(
+            rest_stream_parser,
+            "read_and_parse_stream",
+            side_effect=tracked_read_and_parse_stream,
+        ),
+        mock.patch.object(
+            helpers_module, "EVALUATION_STREAM_DATASET_BATCH_SIZE", TEST_BATCH_SIZE
+        ),
+    ):
+        # Run evaluation with streaming
+        evaluation_result = opik.evaluate(
+            dataset=dataset,
+            task=tracked_task,
+            scoring_functions=[simple_scoring_function],
+            experiment_name=experiment_name,
+            verbose=1,
+            project_name=PROJECT_NAME,
+        )
+
+    # Verify evaluation completed successfully
+    assert evaluation_result.dataset_id == dataset.id
+    assert len(events) == len(DATASET_ITEMS) * 2  # Each item has yield + task
+
+    # Count yields and tasks
+    yield_count = events.count("yield")
+    task_count = events.count("task")
+    assert yield_count == len(DATASET_ITEMS)
+    assert task_count == len(DATASET_ITEMS)
+
+    # Critical assertion: Tasks should be interleaved with yields
+    # This proves streaming is working - we don't wait for all yields before starting tasks
+    # Find the index of the last yield
+    last_yield_index = len(events) - 1 - events[::-1].index("yield")
+    # Find the index of the first task
+    first_task_index = events.index("task")
+
+    assert first_task_index < last_yield_index, (
+        f"Streaming not working! First task appeared at index {first_task_index}, "
+        f"but last yield appeared at index {last_yield_index}. "
+        f"Tasks should start before all items are yielded. "
+        f"Event sequence: {events}"
+    )
+
+    # Verify the experiment was created correctly
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items = retrieved_experiment.get_items()
+    assert len(experiment_items) == len(DATASET_ITEMS)
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+    # Verify scoring output: each item should have a score with name "simple_score" and value 1.0
+    for item in experiment_items:
+        # Check that feedback_scores exists and is not empty
+        assert item.feedback_scores is not None and len(item.feedback_scores) == 1, (
+            f"Experiment item {item.id} should have exactly 1 feedback score, got {len(item.feedback_scores) if item.feedback_scores else 0}"
+        )
+
+        # Verify the score matches expected structure
+        expected_score = FeedbackScoreDict(
+            category_name=None,
+            name="simple_score",
+            reason="Test score",
+            value=1.0,
+        )
+
+        actual_score = item.feedback_scores[0]
+        assert actual_score == expected_score, (
+            f"Experiment item {item.id} has incorrect feedback score. "
+            f"Expected: {expected_score}, Got: {actual_score}"
+        )

sdks/python/tests/e2e/evaluation/test_evaluate_task_span.py

@@ -0,0 +1,487 @@
+from typing import Dict, Any, List, Optional
+
+import opik
+from opik import Prompt, synchronization
+from opik.evaluation import metrics
+from opik.evaluation.metrics import BaseMetric, score_result
+from opik.message_processing.emulation import models
+
+from .. import verifiers
+from ..conftest import random_chars
+
+
+def _wait_for_per_item_feedback_scores(
+    opik_client: opik.Opik,
+    experiment_name: str,
+    expected_count_per_item: int,
+    expected_item_count: int,
+    max_try_seconds: float = 10.0,
+) -> List[Any]:
+    """
+    Poll the backend until every experiment item has ``expected_count_per_item``
+    feedback scores. Returns the materialized list of experiment items.
+
+    The verifier ``verifiers.verify_experiment`` only checks the **experiment-
+    level** ``feedback_scores`` aggregate (one row per unique score name).
+    Per-item feedback scores can lag for an extra moment after the
+    aggregate has converged — especially when task-span scoring writes the
+    second batch of scores via ``client.log_traces_feedback_scores`` after
+    the per-item trace has been emitted. Polling avoids race-condition
+    flakes against the read-back without masking real bugs (a bounded
+    timeout still fails the test if the count never settles).
+    """
+    last_items: List[Any] = []
+
+    def _check() -> bool:
+        nonlocal last_items
+        experiment = opik_client.get_experiment_by_name(experiment_name)
+        last_items = experiment.get_items()
+        if len(last_items) != expected_item_count:
+            return False
+        return all(
+            len(item.feedback_scores) == expected_count_per_item for item in last_items
+        )
+
+    converged = synchronization.until(
+        _check, max_try_seconds=max_try_seconds, allow_errors=True
+    )
+    assert converged, (
+        f"Per-item feedback score counts did not converge to "
+        f"{expected_count_per_item} within {max_try_seconds}s. "
+        f"Last observed: items={len(last_items)}, "
+        f"counts={[len(item.feedback_scores) for item in last_items]}"
+    )
+    return last_items
+
+
+class TaskSpanTestMetric(BaseMetric):
+    def __init__(
+        self,
+        name: str = "task_span_test_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self, task_span: models.SpanModel, **ignored_kwargs: Any
+    ) -> score_result.ScoreResult:
+        score = 1.0 if task_span.name == "task" else 0.0
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason="Correct task span name"
+            if score == 1.0
+            else "Incorrect task span name",
+        )
+
+
+class TaskSpanInputTestMetric(BaseMetric):
+    def __init__(
+        self,
+        name: str = "task_span_input_test_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self, task_span: models.SpanModel, **ignored_kwargs: Any
+    ) -> score_result.ScoreResult:
+        input_data = task_span.input
+        has_question = (
+            isinstance(input_data, dict)
+            and "item" in input_data
+            and "input" in input_data["item"]
+            and isinstance(input_data["item"]["input"], dict)
+            and "question" in input_data["item"]["input"]
+        )
+        score = 1.0 if has_question else 0.0
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason="Task span has question input"
+            if score == 1.0
+            else "Task span missing question input",
+        )
+
+
+def test_evaluate__with_task_span_metrics__single_metric__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_model_output": {"output": "Berlin"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Germany?"}:
+            return {"output": "Berlin"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-task-span-prompt-{random_chars()}",
+        prompt=f"test-task-span-prompt-template-{random_chars()}",
+    )
+
+    task_span_metric = TaskSpanTestMetric()
+    equals_metric = metrics.Equals()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric, task_span_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompts=[prompt],
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=2,
+        feedback_scores_amount=2,  # equals_metric + task_span_metric
+        prompts=[prompt],
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=2,
+        expected_item_count=2,
+    )
+
+    for item in experiment_items_contents:
+        score_names = [score["name"] for score in item.feedback_scores]
+        assert "equals_metric" in score_names
+        assert "task_span_test_metric" in score_names
+
+        # Find task span metric score
+        task_span_score = next(
+            score
+            for score in item.feedback_scores
+            if score["name"] == "task_span_test_metric"
+        )
+        assert task_span_score["value"] == 1.0
+        assert "Correct task span name" in task_span_score["reason"]
+
+
+def test_evaluate__with_task_span_metrics__multiple_task_span_metrics__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Spain?"},
+                "expected_model_output": {"output": "Madrid"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Spain?"}:
+            return {"output": "Madrid"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    task_span_metric_1 = TaskSpanTestMetric(name="task_span_metric_1")
+    task_span_metric_2 = TaskSpanInputTestMetric(name="task_span_metric_2")
+    equals_metric = metrics.Equals()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric, task_span_metric_1, task_span_metric_2],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model-v2",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model-v2"},
+        traces_amount=1,
+        feedback_scores_amount=3,  # equals_metric + 2 task_span_metrics
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=3,
+        expected_item_count=1,
+    )
+    item = experiment_items_contents[0]
+    score_names = [score["name"] for score in item.feedback_scores]
+    assert "equals_metric" in score_names
+    assert "task_span_metric_1" in score_names
+    assert "task_span_metric_2" in score_names
+
+    # Verify all task span metrics scored correctly
+    for score in item.feedback_scores:
+        if score["name"] in ["task_span_metric_1", "task_span_metric_2"]:
+            assert score["value"] == 1.0
+
+
+def test_evaluate__with_task_span_metrics__only_task_span_metrics__no_regular_metrics(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Italy?"},
+                "expected_model_output": {"output": "Rome"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Italy?"}:
+            return {"output": "Rome"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    task_span_metric = TaskSpanTestMetric()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[task_span_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "task-span-only-model",
+        },
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "task-span-only-model"},
+        traces_amount=1,
+        feedback_scores_amount=1,  # only task_span_metric
+    )
+
+    assert evaluation_result.dataset_id == dataset.id
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=1,
+        expected_item_count=1,
+    )
+    item = experiment_items_contents[0]
+    score = item.feedback_scores[0]
+    assert score["name"] == "task_span_test_metric"
+    assert score["value"] == 1.0
+
+
+def test_evaluate__with_task_span_metrics__mixed_with_regular_metrics__multiple_trials(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Japan?"},
+                "expected_model_output": {"output": "Tokyo"},
+            },
+            {
+                "input": {"question": "What is the capital of Canada?"},
+                "expected_model_output": {"output": "Ottawa"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of Japan?"}:
+            return {"output": "Tokyo"}
+        if item["input"] == {"question": "What is the capital of Canada?"}:
+            return {"output": "Ottawa"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-mixed-metrics-prompt-{random_chars()}",
+        prompt=f"test-mixed-metrics-prompt-template-{random_chars()}",
+    )
+
+    # Mix of regular and task span metrics
+    equals_metric = metrics.Equals(name="regular_equals")
+    contains_metric = metrics.Contains(name="regular_contains")
+    task_span_metric = TaskSpanTestMetric(name="span_name_check")
+    task_span_input_metric = TaskSpanInputTestMetric(name="span_input_check")
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[
+            equals_metric,
+            task_span_metric,
+            contains_metric,
+            task_span_input_metric,
+        ],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "mixed-metrics-model",
+            "version": "1.0",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        trial_count=5,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "mixed-metrics-model", "version": "1.0"},
+        traces_amount=2 * 5,  # 2 traces per dataset item per trial
+        feedback_scores_amount=4,  # 2 regular + 2 task_span metrics
+        prompts=[prompt],
+    )
+
+    experiment_items_contents = _wait_for_per_item_feedback_scores(
+        opik_client,
+        experiment_name,
+        expected_count_per_item=4,
+        expected_item_count=2 * 5,
+    )
+
+    expected_score_names = {
+        "regular_equals",
+        "regular_contains",
+        "span_name_check",
+        "span_input_check",
+    }
+
+    for item in experiment_items_contents:
+        actual_score_names = {score["name"] for score in item.feedback_scores}
+        assert actual_score_names == expected_score_names
+
+        # Verify all metrics scored correctly (assuming perfect matches)
+        for score in item.feedback_scores:
+            assert score["value"] == 1.0
+
+
+class TaskSpanWithMultipleParametersMetric(BaseMetric):
+    """
+    Metric that verifies multiple parameters are passed correctly:
+    - task_span: the span information
+    - input: from dataset item
+    - output: from task output
+    - **ignored_kwargs: to handle any other parameters
+    """
+
+    def __init__(
+        self,
+        name: str = "task_span_multi_param_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self,
+        task_span: models.SpanModel,
+        input: Dict[str, Any],
+        output: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        # Simply verify all expected parameters are present and store them in metadata
+        return score_result.ScoreResult(
+            value=1.0,
+            name=self.name,
+            reason=f"Received task_span={type(task_span).__name__}, input={type(input).__name__}, output={type(output).__name__}",
+            metadata={
+                "input": input,
+                "output": output,
+                "task_span_name": task_span.name,
+            },
+        )
+
+
+def test_evaluate__with_task_span_metrics__metric_with_multiple_parameters__happy_flow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    Test that task_span metrics can access task_span, dataset item content (input),
+    and task output (output) parameters. Verifies arguments are passed correctly.
+    """
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert([{"input": {"question": "What is 2+2?"}}])
+
+    def task(item: Dict[str, Any]):
+        return {"output": "4"}
+
+    multi_param_metric = TaskSpanWithMultipleParametersMetric()
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[multi_param_metric],
+        experiment_name=experiment_name,
+    )
+
+    # Verify the metric received all expected parameters in local test results
+    assert len(evaluation_result.test_results) == 1
+    test_result = evaluation_result.test_results[0]
+    assert len(test_result.score_results) == 1
+
+    score_result = test_result.score_results[0]
+    assert score_result.name == "task_span_multi_param_metric"
+    assert score_result.value == 1.0
+    assert "task_span=SpanModel" in score_result.reason
+    assert "input=dict" in score_result.reason
+    assert "output=str" in score_result.reason
+
+    # Verify the parameters were stored correctly in metadata
+    assert score_result.metadata is not None
+    assert score_result.metadata["input"] == {"question": "What is 2+2?"}
+    assert score_result.metadata["output"] == "4"
+    assert score_result.metadata["task_span_name"] == "task"

sdks/python/tests/e2e/evaluation/test_experiment_evaluate.py

@@ -0,0 +1,749 @@
+from typing import Dict, Any, List
+
+import opik
+
+from opik import Prompt, synchronization, id_helpers
+from opik.api_objects.dataset import dataset_item
+from opik.evaluation import metrics
+from opik.evaluation import test_result
+from opik.evaluation.metrics import score_result
+from opik.api_objects.experiment import experiment_item
+from .. import verifiers
+from ..conftest import random_chars
+from ...testlib import assert_equal, ANY_BUT_NONE, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_experiment_creation_via_evaluate_function__single_prompt_arg_used__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    experiment_tags = ["capital", "geography", "europe"]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        experiment_tags=experiment_tags,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=[prompt],
+        experiment_tags=experiment_tags,
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 0.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+
+def test_experiment_creation_via_evaluate_function__single_prompt_arg_used__filter_dataset_items_by_id(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset_items = [
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the of capital of France?"},
+            "expected_model_output": {"output": "Paris"},
+        },
+        {
+            "id": id_helpers.generate_id(),
+            "input": {"question": "What is the of capital of Poland?"},
+            "expected_model_output": {"output": "Warsaw"},
+        },
+    ]
+
+    dataset.insert(dataset_items)
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    # Keep France, drop Poland; append a fake id so the filter still covers
+    # the "non-existent id is ignored" case.
+    dataset_item_ids = [item["id"] for item in dataset_items]
+    dataset_item_ids.pop(1)
+    dataset_item_ids.append(id_helpers.generate_id())
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompt=prompt,
+        dataset_item_ids=dataset_item_ids,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=1,  # one trace per dataset item (fake id is skipped)
+        feedback_scores_amount=1,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiments = opik_client.get_experiments_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert len(retrieved_experiments) == 1, (
+        f"Expected 1 experiment, but got {len(retrieved_experiments)}. "
+        f"Experiments: {retrieved_experiments}"
+    )
+    retrieved_experiment = retrieved_experiments[0]
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 1, (
+        f"Expected 1 experiment item, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+
+def test_experiment_creation_via_evaluate_function__multiple_prompts_arg_used__happyflow(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt1 = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+    prompt2 = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        prompts=[prompt1, prompt2],
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        prompts=[prompt1, prompt2],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_items_contents = retrieved_experiment.get_items()
+    assert len(experiment_items_contents) == 2, (
+        f"Expected 2 experiment items, but got {len(experiment_items_contents)}. "
+        f"Experiment items: {experiment_items_contents}"
+    )
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Paris"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 1.0,
+                }
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                {
+                    "category_name": None,
+                    "name": "equals_metric",
+                    "reason": None,
+                    "value": 0.0,
+                }
+            ],
+        ),
+    ]
+    assert_equal(
+        sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.dataset_item_data),
+        ),
+        sorted(experiment_items_contents, key=lambda item: str(item.dataset_item_data)),
+    )
+
+    verifiers.verify_experiment_traces_have_opik_prompts(
+        opik_client=opik_client,
+        trace_ids=[item.trace_id for item in experiment_items_contents],
+        prompts=[prompt1, prompt2],
+    )
+
+
+def test_experiment_creation__name_can_be_omitted(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """
+    We can send "None" as experiment_name and the backend will set it for us
+    """
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "reference": "Paris",
+            },
+        ]
+    )
+
+    def task(item: dataset_item.DatasetItem):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=None,
+    )
+
+    experiment_id = evaluation_result.experiment_id
+
+    if not synchronization.until(
+        lambda: (
+            opik_client._rest_client.experiments.get_experiment_by_id(experiment_id)
+            is not None
+        ),
+        allow_errors=True,
+    ):
+        raise AssertionError(f"Failed to get experiment with id {experiment_id}.")
+
+    experiment_content = opik_client._rest_client.experiments.get_experiment_by_id(
+        experiment_id
+    )
+
+    assert experiment_content.name is not None, (
+        f"Expected experiment name to be set by backend, but got None. "
+        f"Experiment ID: {experiment_id}, Experiment content: {experiment_content}"
+    )
+
+
+def test_evaluate_experiment__an_experiment_created_with_evaluate__then_new_scores_are_added_to_existing_experiment_items__amount_of_feedback_scores_increased(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {
+                "output": "Paris",
+                "reference": item["expected_model_output"]["output"],
+            }
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+    )
+
+    # Create the experiment first
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        prompt=prompt,
+        project_name=PROJECT_NAME,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={
+            "model_name": "gpt-3.5",
+        },
+        traces_amount=1,
+        feedback_scores_amount=0,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    # Populate the existing experiment with a new feedback score
+    evaluation_result = opik.evaluate_experiment(
+        experiment_name=experiment_name,
+        scoring_metrics=[
+            metrics.Equals(name="metric1"),
+            metrics.Equals(name="metric2"),
+            metrics.Equals(name="metric3"),
+        ],
+        project_name=PROJECT_NAME,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={
+            "model_name": "gpt-3.5",
+        },
+        traces_amount=1,
+        feedback_scores_amount=3,
+        prompts=[prompt],
+        project_name=PROJECT_NAME,
+    )
+
+    assert evaluation_result.dataset_id == dataset.id, (
+        f"Expected evaluation result dataset_id '{dataset.id}', but got '{evaluation_result.dataset_id}'"
+    )
+
+
+def test_experiment__get_experiments_by_name(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the of capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the of capital of Poland?"}:
+            return {"output": "Krakow"}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    prompt = Prompt(
+        name=f"test-experiment-prompt-{random_chars()}",
+        prompt=f"test-experiment-prompt-template-{random_chars()}",
+        project_name=PROJECT_NAME,
+    )
+
+    experiments_names = [experiment_name, experiment_name, random_chars(10)]
+
+    evaluation_results = []
+    equals_metric = metrics.Equals()
+    for name in experiments_names:
+        evaluation_result = opik.evaluate(
+            dataset=dataset,
+            task=task,
+            scoring_metrics=[equals_metric],
+            experiment_name=name,
+            experiment_config={
+                "model_name": "gpt-3.5",
+            },
+            scoring_key_mapping={
+                "reference": lambda x: x["expected_model_output"]["output"],
+            },
+            prompt=prompt,
+            project_name=PROJECT_NAME,
+        )
+        evaluation_results.append(evaluation_result)
+
+    # make sure experiments saved and available
+    for result in evaluation_results:
+        verifiers.verify_experiment(
+            opik_client=opik_client,
+            id=result.experiment_id,
+            experiment_name=result.experiment_name,
+            experiment_metadata={"model_name": "gpt-3.5"},
+            traces_amount=2,  # one trace per dataset item
+            feedback_scores_amount=1,
+            prompts=[prompt],
+            project_name=PROJECT_NAME,
+        )
+
+    # check getting experiment by name
+    experiments = opik_client.get_experiments_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert len(experiments) == 2, (
+        f"Expected 2 experiments with name '{experiment_name}', but got {len(experiments)}. "
+        f"Experiment IDs: {[e.id for e in experiments]}"
+    )
+    assert all(experiment.project_name == PROJECT_NAME for experiment in experiments)
+
+    single = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    assert single is not None
+    assert single.id in [e.id for e in experiments]
+
+    experiments = opik_client.get_experiments_by_name(
+        experiments_names[2], project_name=PROJECT_NAME
+    )
+    assert len(experiments) == 1, (
+        f"Expected 1 experiment with name '{experiments_names[2]}', but got {len(experiments)}. "
+        f"Experiment IDs: {[e.id for e in experiments]}"
+    )
+
+
+def test_experiment__get_experiment_items__no_feedback_scores(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]) -> Dict[str, Any]:
+        return {
+            "output": "Paris",
+        }
+
+    opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[],
+        experiment_name=experiment_name,
+    )
+
+    experiment = opik_client.get_experiment_by_name(experiment_name)
+    items = experiment.get_items()
+
+    assert len(items) == 1, (
+        f"Expected 1 experiment item, but got {len(items)}. Items: {items}"
+    )
+    assert items[0].feedback_scores == [], (
+        f"Expected empty feedback scores, but got {items[0].feedback_scores}. "
+        f"Item: {items[0]}"
+    )
+
+
+def test_experiment_creation_via_evaluate_function__with_experiment_scoring_functions__scores_computed_and_logged(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions compute and log experiment-level scores."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_model_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris"}
+        if item["input"] == {"question": "What is the capital of Poland?"}:
+            return {"output": "Krakow"}  # Wrong answer
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    def constant_score(
+        test_results: List[test_result.TestResult],
+    ) -> List[score_result.ScoreResult]:
+        """Compute a random number based on the number of test results."""
+        return [
+            score_result.ScoreResult(
+                name="fixed_number",
+                value=0.8,
+                reason="Fixed score of 0.8",
+            )
+        ]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        experiment_scoring_functions=[constant_score],
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are in the result
+    assert len(evaluation_result.experiment_scores) == 1, (
+        f"Expected 1 experiment score in evaluation result, but got {len(evaluation_result.experiment_scores)}. "
+        f"Experiment scores: {evaluation_result.experiment_scores}"
+    )
+    assert evaluation_result.experiment_scores[0].name == "fixed_number", (
+        f"Expected experiment score name 'fixed_number', but got '{evaluation_result.experiment_scores[0].name}'. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].value == 0.8, (
+        f"Expected experiment score value 0.8, but got {evaluation_result.experiment_scores[0].value}. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].reason is not None, (
+        f"Expected experiment score reason to be set, but got None. "
+        f"Score: {evaluation_result.experiment_scores[0]}"
+    )
+    assert "Fixed score of 0.8" in evaluation_result.experiment_scores[0].reason, (
+        f"Expected reason to contain 'Fixed score of 0.8', but got: '{evaluation_result.experiment_scores[0].reason}'"
+    )
+
+    # Verify experiment was created with experiment scores
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        experiment_scores={"fixed_number": 0.8},
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_experiment_scoring_functions.py

@@ -0,0 +1,138 @@
+import statistics
+from typing import Dict, Any, List
+
+import pytest
+
+import opik
+from opik.evaluation import metrics
+from opik.evaluation import test_result
+from opik.evaluation.metrics import score_result
+from .. import verifiers
+from ...testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_experiment_scoring_functions__standard_deviation__computed_and_logged(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions can compute and log standard deviation of metric scores."""
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": "Paris",
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_output": "Warsaw",
+            },
+        ]
+    )
+
+    def task(item: Dict[str, Any]):
+        # Return correct answer for France, wrong answer for Poland so
+        # metric scores are [1.0, 0.0] — enough to exercise stdev.
+        if item["input"] == {"question": "What is the capital of France?"}:
+            return {"output": "Paris", "reference": item["expected_output"]}
+        if item["input"] == {"question": "What is the capital of Poland?"}:
+            return {"output": "Krakow", "reference": item["expected_output"]}
+
+        raise AssertionError(
+            f"Task received dataset item with an unexpected input: {item['input']}"
+        )
+
+    def compute_std_deviation(
+        test_results: List[test_result.TestResult],
+    ) -> List[score_result.ScoreResult]:
+        """Compute standard deviation of metric scores across all test results."""
+        scores = [x.score_results[0].value for x in test_results]
+        return [
+            score_result.ScoreResult(
+                name="equals_metric_std_dev",
+                value=statistics.stdev(scores) if len(scores) > 1 else 0.0,
+                reason=f"Standard deviation of {len(scores)} metric scores",
+            )
+        ]
+
+    equals_metric = metrics.Equals()
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=task,
+        scoring_metrics=[equals_metric],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        experiment_scoring_functions=[compute_std_deviation],
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are in the result
+    assert len(evaluation_result.experiment_scores) == 1, (
+        f"Expected 1 experiment score in evaluation result, but got {len(evaluation_result.experiment_scores)}. "
+        f"Experiment scores: {evaluation_result.experiment_scores}"
+    )
+    assert evaluation_result.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev', but got '{evaluation_result.experiment_scores[0].name}'. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+
+    expected_stdev = statistics.stdev([1.0, 0.0])
+    assert evaluation_result.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev}, but got {evaluation_result.experiment_scores[0].value}. "
+        f"Full score object: {evaluation_result.experiment_scores[0]}"
+    )
+    assert evaluation_result.experiment_scores[0].reason is not None, (
+        f"Expected experiment score reason to be set, but got None. "
+        f"Score: {evaluation_result.experiment_scores[0]}"
+    )
+    assert (
+        "Standard deviation of 2 metric scores"
+        in evaluation_result.experiment_scores[0].reason
+    ), (
+        f"Expected reason to contain 'Standard deviation of 2 metric scores', but got: '{evaluation_result.experiment_scores[0].reason}'"
+    )
+
+    # Verify experiment was created
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are logged to backend by retrieving the experiment
+    retrieved_experiment = opik_client.get_experiment_by_id(
+        evaluation_result.experiment_id
+    )
+    experiment_data = retrieved_experiment.get_experiment_data()
+
+    # Check that experiment scores are present in the experiment data
+    assert experiment_data.experiment_scores is not None, (
+        f"Expected experiment_scores to be set in experiment data, but got None. "
+        f"Experiment ID: {evaluation_result.experiment_id}, "
+        f"Experiment name: {evaluation_result.experiment_name}, "
+        f"Experiment data: {experiment_data}"
+    )
+    assert len(experiment_data.experiment_scores) == 1, (
+        f"Expected 1 experiment score in backend, but got {len(experiment_data.experiment_scores) if experiment_data.experiment_scores else 0}. "
+        f"Experiment scores: {experiment_data.experiment_scores}"
+    )
+    assert experiment_data.experiment_scores[0].name == "equals_metric_std_dev", (
+        f"Expected experiment score name 'equals_metric_std_dev' in backend, but got '{experiment_data.experiment_scores[0].name}'. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )
+    assert experiment_data.experiment_scores[0].value == pytest.approx(
+        expected_stdev, abs=0.0001
+    ), (
+        f"Expected experiment score value {expected_stdev} in backend, but got {experiment_data.experiment_scores[0].value}. "
+        f"Full score object: {experiment_data.experiment_scores[0]}"
+    )

sdks/python/tests/e2e/evaluation/test_test_suite_agentic.py

@@ -0,0 +1,314 @@
+"""E2E tests for the agentic LLM-judge path on test suites.
+
+The one-shot LLMJudge sees only the dataset item's top-level `input` /
+`output`. The agentic judge — auto-engaged when the local emulator is
+active during `opik.run_tests` — receives a pre-rendered flat overview
+of the trace plus `read` / `scan` / `search` tools for drilling in.
+These tests craft assertions that are decidable ONLY from the
+intermediate span state, so a passing verdict is direct evidence the
+agentic judge ran:
+
+1. A span-structure assertion ("the agent called the `fetch_user_data`
+   helper") — verifiable only by looking at span names in the trace.
+2. A span-error assertion ("no errors occurred without errors") on a
+   trace whose inner span recorded an error caught by the task —
+   verifiable only by looking at span `error_info`.
+3. A buried-keyword assertion — the marker sits past the overview's
+   floor-tier 500-char truncation, so the judge must call at least one
+   of `read` / `scan` / `search` to recover it. The test pins the
+   overview sizer's ladder to the floor entry to guarantee truncation
+   regardless of the judge model's context budget.
+
+**Judge-model choice.** These tests deliberately override the SDK's
+default judge model (`gpt-5-nano`) with a stronger one. `gpt-5-nano`
+is the canonical "doesn't engage the tool loop" failure mode the
+backend doc (`SupportedJudgeProvider.java`) warns about — it tends to
+judge from the inline overview alone, never calling `read`. The SDK
+can't prompt-engineer around that; the design doc §9 explicitly
+classifies model engagement as the operator's call. Here we pick
+`gpt-4o-mini` to match the backend's allow-list second choice — same
+OpenAI dependency story, but actually engages with tools.
+
+All require `OPENAI_API_KEY`; the assertions are deliberately
+unambiguous to keep judge flakiness low even on a competent judge.
+"""
+
+from typing import Any, Dict
+
+import pytest
+
+import opik
+from .. import verifiers
+from ...testlib import environment, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+# Stronger than the SDK default (`gpt-5-nano`) and known to engage the
+# tool loop — backend `SupportedJudgeProvider.java` lists it as the
+# OpenAI allow-list entry. If you change this, re-run all three tests:
+# nano will pass the first two (structural assertions) but reliably
+# fail the third (it never calls `read` on truncated content).
+AGENTIC_JUDGE_MODEL = "gpt-5-mini"  #
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_test_suite_agentic__assertion_about_span_name__passes(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """The agentic judge must inspect the trace's span tree to verify
+    that a specifically-named tool span was actually called. The task
+    output deliberately omits the helper's name, so a one-shot judge
+    has no way to verify the assertion from input/output alone.
+    """
+    span_assertion = "The agent called a step named `fetch_user_data`"
+
+    @opik.track(name="fetch_user_data", project_name=PROJECT_NAME)
+    def fetch_user_data(user_id: str) -> Dict[str, Any]:
+        return {"id": user_id, "name": "alice"}
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        fetch_user_data("u-1")
+        # Output deliberately doesn't mention `fetch_user_data` — the
+        # only evidence the helper was called lives in the span tree.
+        return {"input": item["input"], "output": "ok"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — span-name assertion",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "fetch user u-1"}},
+                "assertions": [span_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # The one assertion must surface in the feedback scores.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={span_assertion},
+        project_name=PROJECT_NAME,
+    )
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+@pytest.mark.xfail(reason="flaky on CI with simple OpenAI models")
+def test_test_suite_agentic__assertion_about_span_error__detects_failure(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """The agentic judge must inspect `error_info` on a nested span to
+    confirm a *specific* exception type and message. The task catches
+    the exception so the trace's top-level output looks healthy.
+
+    Why the assertion names the exception type AND a phrase from the
+    message: the inline overview already carries each span's
+    `has_error` flag, so an assertion of the form "no errors occurred"
+    would be decidable from the overview alone — the test would pass
+    without ever calling `read` and prove nothing about drill-in. By
+    pinning the verdict on the exception's type and message — both of
+    which live only in `error_info` and are NOT in the overview — the
+    correct verdict is only reachable by reading the span. A judge
+    that stops at `has_error: true` lacks the evidence to confirm
+    "type=RuntimeError" or "message contains 'internal failure'" and
+    must drill in.
+    """
+    span_error_assertion = (
+        "An internal step raised an exception of type `RuntimeError` "
+        "whose message contains the phrase 'internal failure'"
+    )
+
+    @opik.track(name="risky_step", project_name=PROJECT_NAME)
+    def risky_step() -> str:
+        # @opik.track captures `error_info` on the inner span when this
+        # raises, even though the caller swallows the exception. The
+        # exception type and message are recorded in `error_info` —
+        # *not* exposed via the overview's `has_error` flag — which is
+        # what forces the judge to call `read`.
+        raise RuntimeError("internal failure")
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            risky_step()
+        except RuntimeError:
+            pass
+        return {"input": item["input"], "output": "completed"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — span-error assertion (type + message)",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "do the thing"}},
+                "assertions": [span_error_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # The assertion is TRUE — the inner step did raise a RuntimeError
+    # with message "internal failure". A judge that stayed at the
+    # overview level would only see `has_error: true` with no type or
+    # message info, so per the "absence of evidence" rule it would
+    # fail the assertion (score=false). A passing verdict means it
+    # drilled into `error_info` to verify both the type and the
+    # message phrase.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={span_error_assertion},
+        project_name=PROJECT_NAME,
+    )
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_test_suite_agentic__assertion_requires_buried_keyword_lookup__passes(
+    opik_client: opik.Opik,
+    dataset_name: str,
+    experiment_name: str,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """The marker is placed past the floor-tier truncation the overview
+    applies to every span's I/O. The overview alone physically cannot
+    include it, so the judge MUST reach for at least one of
+    `read` / `scan` / `search` to find the marker and pass the
+    assertion. We don't pin which tool the model picks — that's a
+    model-quality question. What we prove here is that the agentic loop
+    engages beyond the overview, since a one-shot judge would lack the
+    evidence.
+
+    The marker is a synthetic token (`MARKER-` + opaque tail) chosen so
+    it can't appear anywhere else in the trace or in the model's
+    pretraining. A `score: true` verdict here means the judge actually
+    extracted it from span content.
+
+    NOTE on the ladder monkeypatch: the overview sizer normally picks
+    the largest per-field limit that fits the model's context budget,
+    which on `gpt-4o-mini` (128k window) would render this small trace
+    at the `NO_OVERVIEW_TRUNCATION` tier — the marker would be visible
+    inline and the test premise would silently regress. Forcing the
+    ladder to its single floor entry (`OVERVIEW_IO_FLOOR_CHAR_LIMIT`) keeps
+    the truncation guaranteed without needing to produce a multi-MB
+    trace just to overflow the budget. The sizer's own behavior is
+    covered by `test_overview_sizer.py`.
+    """
+    from opik.evaluation.suite_evaluators.agentic.compression import (
+        span_tree_serializer,
+    )
+
+    monkeypatch.setattr(
+        span_tree_serializer,
+        "OVERVIEW_IO_LIMIT_LADDER",
+        (span_tree_serializer.OVERVIEW_IO_FLOOR_CHAR_LIMIT,),
+    )
+
+    secret_marker = "MARKER-7f3a8c2e9b1d4f60"
+    keyword_assertion = (
+        f"At least one intermediate step processed a payload containing "
+        f"the literal token '{secret_marker}'"
+    )
+
+    @opik.track(name="process_step", project_name=PROJECT_NAME)
+    def process_step(payload: str) -> str:
+        # The output deliberately doesn't echo the marker — it only
+        # references the payload length, so the agent can't shortcut
+        # through `output` alone. The marker lives only in the input,
+        # which is what the overview truncates at the floor tier.
+        return f"step processed {len(payload)} chars"
+
+    @opik.track(name="noop_step", project_name=PROJECT_NAME)
+    def noop_step() -> str:
+        return "noop"
+
+    @opik.track(name="task", project_name=PROJECT_NAME)
+    def run_task(item: Dict[str, Any]) -> Dict[str, Any]:
+        # Pad with filler so the marker lands well past the overview's
+        # OVERVIEW_IO_FLOOR_CHAR_LIMIT (500 chars) in the JSON-rendered span
+        # input `{"payload": "<filler><marker>"}`. `padding ` is 8 chars
+        # × 70 = 560 chars; with the ~13-char JSON prefix the marker
+        # starts past offset 573, so it sits beyond the overview cap by
+        # construction (the ladder is pinned to the floor by the
+        # monkeypatch above). Mixed-in noop_step guarantees more than
+        # one span exists, so the judge can't trivially infer "must be
+        # the only span."
+        filler = "padding " * 170
+        process_step(payload=f"{filler}{secret_marker} trailer")
+        noop_step()
+        return {"input": item["input"], "output": "completed"}
+
+    suite = opik_client.create_test_suite(
+        name=dataset_name,
+        description="Agentic judge — buried-keyword requires read/scan/search",
+        project_name=PROJECT_NAME,
+    )
+    suite.insert(
+        [
+            {
+                "data": {"input": {"question": "process the payload"}},
+                "assertions": [keyword_assertion],
+            }
+        ]
+    )
+
+    suite_result = opik.run_tests(
+        test_suite=suite,
+        task=run_task,
+        experiment_name=experiment_name,
+        verbose=0,
+        model=AGENTIC_JUDGE_MODEL,
+        scoring_tool_strategy="auto",
+    )
+
+    # If the judge stayed at the overview level, it would have seen the
+    # truncated string (no marker) and could only have guessed at the
+    # assertion. A passing verdict means it drilled into span content
+    # via one of read / scan / search.
+    verifiers.verify_test_suite_result(
+        opik_client=opik_client,
+        suite_result=suite_result,
+        items_total=1,
+        items_passed=1,
+        experiment_items_count=1,
+        total_feedback_scores=1,
+        expected_score_names={keyword_assertion},
+        project_name=PROJECT_NAME,
+    )

sdks/python/tests/e2e/evaluation/test_threads_evaluate.py

@@ -0,0 +1,220 @@
+import time
+import uuid
+import pytest
+
+import opik
+from opik import synchronization
+from opik.evaluation import metrics
+from opik.evaluation.models import LiteLLMChatModel
+from opik.evaluation.threads import evaluator
+from opik.types import FeedbackScoreDict
+
+from ...testlib import environment
+from .. import verifiers
+
+
+@pytest.fixture
+def real_model_conversation():
+    return [
+        {
+            "role": "user",
+            "content": "I need to book a flight to New York and find a hotel.",
+        },
+        {
+            "role": "assistant",
+            "content": "I can help you with that. For flights to New York, what dates are you looking to travel?",
+        },
+    ]
+
+
+@pytest.fixture
+def active_thread_and_project_name(
+    opik_client, real_model_conversation, temporary_project_name
+):
+    thread_id = str(uuid.uuid4())[-6:]
+
+    # create conversation traces
+    i = 0
+    while i < len(real_model_conversation) - 1:
+        opik_client.trace(
+            name=f"trace-name-{i}:{thread_id}",
+            input={"input": f"{real_model_conversation[i]['content']}"},
+            output={"output": f"{real_model_conversation[i + 1]['content']}"},
+            project_name=temporary_project_name,
+            thread_id=thread_id,
+        )
+        time.sleep(0.1)
+        i += 2
+
+    opik_client.flush()
+
+    return thread_id, temporary_project_name
+
+
+def _one_thread_is_active(project_name: str, opik_client: opik.Opik) -> bool:
+    threads = opik_client.search_threads(
+        project_name=project_name, filter_string='status = "active"'
+    )
+    return len(threads) == 1
+
+
+@pytest.fixture
+def eval_project_name(temporary_project_name: str) -> str:
+    return temporary_project_name
+
+
+@pytest.mark.skipif(
+    not environment.has_openai_api_key(), reason="OPENAI_API_KEY is not set"
+)
+def test_evaluate_threads__happy_path(
+    opik_client, active_thread_and_project_name, eval_project_name
+):
+    active_thread, project_name = active_thread_and_project_name
+    # wait for active threads to propagate
+    if not synchronization.until(
+        lambda: _one_thread_is_active(project_name, opik_client), max_try_seconds=30
+    ):
+        raise AssertionError(f"Failed to create threads in project '{project_name}'")
+
+    judge_model = LiteLLMChatModel(reasoning_effort="minimal")
+
+    metrics_ = [
+        metrics.ConversationalCoherenceMetric(model=judge_model),
+    ]
+
+    result = evaluator.evaluate_threads(
+        project_name=project_name,
+        filter_string=f'id = "{active_thread}"',
+        metrics=metrics_,
+        eval_project_name=eval_project_name,
+        trace_input_transform=lambda x: x["input"],
+        trace_output_transform=lambda x: x["output"],
+        verbose=1,
+    )
+
+    assert result is not None
+    assert len(result.results) == 1  # we have only one thread
+
+    thread_result = result.results[0]
+    assert thread_result.thread_id == active_thread
+    assert len(thread_result.scores) == len(metrics_)
+
+    feedback_scores = [
+        FeedbackScoreDict(
+            id=active_thread,
+            name=score.name,
+            value=score.value,
+            reason=score.reason.strip(),
+            category_name=None,
+        )
+        for score in thread_result.scores
+        if not score.scoring_failed
+    ]
+
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=active_thread,
+        project_name=project_name,
+        feedback_scores=feedback_scores,
+    )
+
+
+def test_evaluate_threads__no_truncation_for_long_traces(
+    opik_client, temporary_project_name
+):
+    """E2E test verifying that long trace content is not truncated during evaluation.
+
+    The test creates a trace with output exceeding 15,000 characters with a unique marker
+    at the end, then verifies the marker is present in the transform function, proving
+    that truncation did not occur.
+    """
+    thread_id = str(uuid.uuid4())[-6:]
+
+    # Create a long output that exceeds the truncation threshold (~9935 chars)
+    # with a unique marker at the end that would be lost if truncated
+    marker = "UNIQUE_END_MARKER_XYZ123"
+    long_content = "a" * 15000 + marker
+
+    # Create a trace with very long output
+    opik_client.trace(
+        name=f"long-trace:{thread_id}",
+        input={"input": "test input"},
+        output={"output": long_content},
+        project_name=temporary_project_name,
+        thread_id=thread_id,
+    )
+
+    opik_client.flush()
+
+    # Wait for thread to be created
+    if not synchronization.until(
+        lambda: _one_thread_is_active(temporary_project_name, opik_client),
+        max_try_seconds=30,
+    ):
+        raise AssertionError(
+            f"Failed to create thread in project '{temporary_project_name}'"
+        )
+
+    # Track what the transform receives
+    received_outputs = []
+
+    def input_transform(x):
+        return x.get("input", "")
+
+    def output_transform(x):
+        """Transform that captures the output to verify it's not truncated."""
+        # When truncated, x might be a string (malformed JSON) instead of dict
+        if isinstance(x, str):
+            # This is the bug! Truncation causes malformed JSON string
+            received_outputs.append(f"TRUNCATED_STRING:{x[:100]}...")
+            return "TRUNCATED"
+        output = x.get("output", "")
+        received_outputs.append(output)
+        return output
+
+    # Create a simple metric that just checks the output
+    class ContentVerificationMetric(metrics.base_metric.BaseMetric):
+        def __init__(self):
+            super().__init__(
+                name="content_verification",
+                track=False,
+            )
+
+        def score(self, conversation, **ignored_kwargs):
+            # Just return a dummy score - we're really testing the transform
+            return metrics.score_result.ScoreResult(
+                name=self.name,
+                value=1.0,
+                reason="Content verification",
+            )
+
+    # Run evaluation
+    result = evaluator.evaluate_threads(
+        project_name=temporary_project_name,
+        filter_string=f'id = "{thread_id}"',
+        metrics=[ContentVerificationMetric()],
+        eval_project_name=temporary_project_name,
+        trace_input_transform=input_transform,
+        trace_output_transform=output_transform,
+        verbose=0,
+    )
+
+    assert result is not None
+    assert len(result.results) == 1
+
+    # Verify that the transform received the full content with the marker
+    assert len(received_outputs) > 0, "Transform should have been called"
+    transformed_content = received_outputs[0]
+
+    # This is the critical assertion: if truncation occurred, the marker would be missing
+    assert marker in transformed_content, (
+        f"Content was truncated! Expected marker '{marker}' not found. "
+        f"Content length: {len(transformed_content)}, expected: {len(long_content)}. "
+        f"Last 100 chars: {transformed_content[-100:]}"
+    )
+
+    # Also verify the full length is preserved
+    assert len(transformed_content) == len(long_content), (
+        f"Content length mismatch: got {len(transformed_content)}, "
+        f"expected {len(long_content)}"
+    )

sdks/python/tests/e2e/runner/conftest.py

@@ -0,0 +1,285 @@
+import base64
+import dataclasses
+import hashlib
+import hmac as hmac_mod
+import os
+import re
+import shutil
+import subprocess
+import sys
+import threading
+import time
+import uuid
+
+import pytest
+
+import opik
+import opik.api_objects.opik_client
+from opik.api_objects import rest_helpers
+from opik.cli.local_runner.pairing import hkdf_sha256
+from opik.rest_api import core as rest_api_core
+
+
+ECHO_APP = os.path.join(os.path.dirname(__file__), "echo_app.py")
+OPIK_CLI = shutil.which("opik") or "opik"
+
+RUNNER_STARTUP_TIMEOUT = 30
+
+_phase_report_key = pytest.StashKey[dict]()
+
+
+@dataclasses.dataclass
+class RunnerInfo:
+    runner_id: str
+    process: subprocess.Popen
+    output_lines: list
+    bridge_key: bytes = b""
+
+
+@dataclasses.dataclass
+class TestProject:
+    id: str
+    name: str
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    rep = outcome.get_result()
+    item.stash.setdefault(_phase_report_key, {})[rep.when] = rep
+
+
+@pytest.fixture()
+def opik_client(configure_e2e_tests_env, shutdown_cached_client_after_test):
+    client = opik.api_objects.opik_client.Opik(batching=True)
+    yield client
+    client.end()
+
+
+@pytest.fixture()
+def api_client(opik_client):
+    return opik_client.rest_client
+
+
+@pytest.fixture()
+def subprocess_env(opik_client):
+    cfg = opik_client.config
+    env = os.environ.copy()
+    env["OPIK_URL_OVERRIDE"] = cfg.url_override
+    if cfg.api_key:
+        env["OPIK_API_KEY"] = cfg.api_key
+    if cfg.workspace:
+        env["OPIK_WORKSPACE"] = cfg.workspace
+    return env
+
+
+@pytest.fixture()
+def project(api_client, temporary_project_name) -> TestProject:
+    """Per-test project, created on setup. The shared ``temporary_project_name``
+    fixture handles name generation and teardown."""
+    try:
+        api_client.projects.create_project(name=temporary_project_name)
+    except rest_api_core.ApiError:
+        pass
+    project_id = rest_helpers.resolve_project_id_by_name(
+        api_client, temporary_project_name
+    )
+    return TestProject(id=project_id, name=temporary_project_name)
+
+
+def _drain_stdout(proc, output_lines):
+    for line in proc.stdout:
+        output_lines.append(line.rstrip())
+
+
+def _parse_pairing_url(output_lines, timeout):
+    """Wait for the CLI to print the pairing URL, then parse the fragment.
+
+    The URL may be split across multiple stdout lines by the terminal.
+    Join all output and search for the full URL.
+    """
+    deadline = time.monotonic() + timeout
+    url_pattern = re.compile(
+        r"(https?://\S+/opik/pair/v1(?:\?[^#\s]*)?#[A-Za-z0-9_\-]+)"
+    )
+
+    while time.monotonic() < deadline:
+        joined = "".join(list(output_lines))
+        match = url_pattern.search(joined)
+        if match:
+            url = match.group(1)
+            fragment = url.split("#", 1)[1]
+            padded = fragment + "=" * ((4 - len(fragment) % 4) % 4)
+            raw = base64.urlsafe_b64decode(padded)
+            if len(raw) < 65:
+                time.sleep(0.05)
+                continue
+            name_len = raw[64]
+            if len(raw) < 65 + name_len:
+                time.sleep(0.05)
+                continue
+            return {
+                "session_id": str(uuid.UUID(bytes=raw[0:16])),
+                "activation_key": raw[16:48],
+                "project_id": str(uuid.UUID(bytes=raw[48:64])),
+                "runner_name": raw[65 : 65 + name_len].decode("utf-8"),
+            }
+        time.sleep(0.05)
+    return None
+
+
+def _activate_session(api_client, activation_key, session_id, runner_name):
+    session_id_bytes = uuid.UUID(session_id).bytes
+    runner_name_hash = hashlib.sha256(runner_name.encode("utf-8")).digest()
+    message = session_id_bytes + runner_name_hash
+    sig = hmac_mod.new(activation_key, message, hashlib.sha256).digest()
+    hmac_b64 = base64.b64encode(sig).decode("ascii")
+
+    api_client.pairing.activate_pairing_session(
+        session_id, runner_name=runner_name, hmac=hmac_b64
+    )
+
+
+def _start_runner(
+    api_client, subprocess_env, project_id, opik_client, request, cli_args
+):
+    proc = subprocess.Popen(
+        cli_args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=subprocess_env,
+    )
+
+    output_lines = []
+    drain_thread = threading.Thread(
+        target=_drain_stdout, args=(proc, output_lines), daemon=True
+    )
+    drain_thread.start()
+
+    # Wait for the CLI to print its pairing URL, then parse the fragment
+    payload = _parse_pairing_url(output_lines, timeout=15)
+    if payload is None:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail(
+            "CLI did not print pairing URL within 15s.\n"
+            "Output:\n" + "\n".join(output_lines)
+        )
+
+    # Activate the CLI's session (simulates browser clicking Connect)
+    _activate_session(
+        api_client,
+        payload["activation_key"],
+        payload["session_id"],
+        payload["runner_name"],
+    )
+
+    # Wait for the CLI to detect connected status
+    deadline = time.monotonic() + RUNNER_STARTUP_TIMEOUT
+    connected = False
+    while time.monotonic() < deadline:
+        for line in list(output_lines):
+            if "Paired" in line:
+                connected = True
+                break
+        if connected:
+            break
+        if proc.poll() is not None:
+            break
+        time.sleep(0.1)
+
+    if not connected:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail(
+            f"Runner did not connect within {RUNNER_STARTUP_TIMEOUT}s.\n"
+            f"Output:\n" + "\n".join(output_lines)
+        )
+
+    # Get runner_id by polling the API
+    runners_page = api_client.runners.list_runners(project_id=project_id, size=10)
+    runner_id = None
+    if runners_page.content:
+        for r in runners_page.content:
+            if r.status == "connected":
+                runner_id = r.id
+                break
+
+    if runner_id is None:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail("No connected runner found after activation")
+
+    bridge_key = hkdf_sha256(
+        ikm=payload["activation_key"],
+        salt=uuid.UUID(payload["session_id"]).bytes,
+        info=b"opik-bridge-v1",
+    )
+
+    info = RunnerInfo(
+        runner_id=runner_id,
+        process=proc,
+        output_lines=output_lines,
+        bridge_key=bridge_key,
+    )
+
+    yield info
+
+    # Runner CLI + echo_app don't need a graceful window in tests; SIGTERM →
+    # short wait → SIGKILL is fine. Drops per-test teardown from ~10s to ~2s.
+    proc.terminate()
+    try:
+        proc.wait(timeout=2)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait()
+    drain_thread.join(timeout=2)
+
+    report = request.node.stash.get(_phase_report_key, {})
+    if report.get("call") and report["call"].failed:
+        print(f"\n--- Runner output (runner_id={runner_id}) ---")
+        print("\n".join(output_lines))
+        print("--- End runner output ---")
+
+
+@pytest.fixture()
+def runner_process(api_client, subprocess_env, project, opik_client, request):
+    """Endpoint runner — for job/agent E2E tests."""
+    yield from _start_runner(
+        api_client,
+        subprocess_env,
+        project.id,
+        opik_client,
+        request,
+        cli_args=[
+            OPIK_CLI,
+            "endpoint",
+            "--project",
+            project.name,
+            "--",
+            sys.executable,
+            ECHO_APP,
+        ],
+    )
+
+
+@pytest.fixture()
+def bridge_runner_process(api_client, subprocess_env, project, opik_client, request):
+    """Connect runner — for bridge command E2E tests."""
+    yield from _start_runner(
+        api_client,
+        subprocess_env,
+        project.id,
+        opik_client,
+        request,
+        cli_args=[
+            OPIK_CLI,
+            "connect",
+            "--project",
+            project.name,
+        ],
+    )

sdks/python/tests/e2e/runner/echo_app.py

@@ -0,0 +1,36 @@
+"""Agent app used by the runner e2e tests.
+
+Contains two entrypoints:
+- echo: basic echo function
+- echo_config: echo with a configurable greeting (used in mask tests)
+"""
+
+import threading
+
+import opik
+
+
+@opik.track(entrypoint=True)
+def echo(message: str) -> str:
+    print(f"echo stdout: {message}")
+    return f"echo: {message}"
+
+
+class EchoConfig(opik.Config):
+    greeting: str
+
+
+@opik.track(entrypoint=True)
+def echo_config(message: str) -> str:
+    client = opik.Opik()
+    cfg = client.get_or_create_config(
+        fallback=EchoConfig(greeting="fallback-greeting"),
+    )
+    return f"{cfg.greeting}: {message}"
+
+
+if __name__ == "__main__":
+    # Analogous to uvicorn.run() in a FastAPI app: @opik.track(entrypoint=True) registers
+    # handlers at import time, and this blocks the main thread so the runner loop stays
+    # alive to process jobs.
+    threading.Event().wait()

sdks/python/tests/e2e/runner/test_bridge_e2e.py

@@ -0,0 +1,195 @@
+"""Happy-path e2e tests for the bridge command system.
+
+Exec tests: echo, nonzero exit, background process.
+File operation flow: write → list → search → edit → read, all via the API.
+"""
+
+import time
+
+from opik.rest_api.errors.conflict_error import ConflictError
+from opik.runner.bridge_auth import sign
+
+from .conftest import RunnerInfo
+
+_BRIDGE_READY_TIMEOUT = 15
+
+
+def _submit_and_wait(api_client, runner_id, cmd_type, args, bridge_key=b""):
+    if bridge_key:
+        args = dict(args)
+        args["_hmac"] = sign(bridge_key, cmd_type, args)
+
+    deadline = time.monotonic() + _BRIDGE_READY_TIMEOUT
+    while True:
+        try:
+            resp = api_client.runners.create_bridge_command(
+                runner_id,
+                type=cmd_type,
+                args=args,
+            )
+            break
+        except ConflictError:
+            if time.monotonic() > deadline:
+                raise
+            time.sleep(1)
+
+    return api_client.runners.get_bridge_command(
+        runner_id,
+        resp.command_id,
+        wait=True,
+        timeout=15,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Auth tests
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_command__unsigned__fails_with_auth_failed(
+    api_client, bridge_runner_process: RunnerInfo
+):
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "echo should-not-run"},
+    )
+    assert cmd.status == "failed"
+    assert cmd.error["code"] == "auth_failed"
+
+
+def test_bridge_command__invalid_hmac__fails_with_auth_failed(
+    api_client, bridge_runner_process: RunnerInfo
+):
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "echo should-not-run", "_hmac": "dGhpcyBpcyBub3QgYSB2YWxpZCBobWFj"},
+    )
+    assert cmd.status == "failed"
+    assert cmd.error["code"] == "auth_failed"
+
+
+# ---------------------------------------------------------------------------
+# Exec tests
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_exec_echo(api_client, bridge_runner_process: RunnerInfo):
+    """Submit a simple echo command and verify the result."""
+    marker = f"bridge-e2e-{int(time.time())}"
+
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": f"echo {marker}"},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert marker in cmd.result["stdout"]
+    assert cmd.result["exit_code"] == 0
+
+
+def test_bridge_exec_nonzero_exit(api_client, bridge_runner_process: RunnerInfo):
+    """Verify non-zero exit codes are returned correctly."""
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "exit 42"},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert cmd.result["exit_code"] == 42
+
+
+def test_bridge_exec_background(api_client, bridge_runner_process: RunnerInfo):
+    """Submit a background command, verify PID is returned."""
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "sleep 30", "background": True},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert "pid" in cmd.result
+    assert cmd.result["status"] == "running"
+
+
+# ---------------------------------------------------------------------------
+# File operation flow: write → list → search → edit → read
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_file_operations(api_client, bridge_runner_process: RunnerInfo):
+    """Write a file, find it with list/search, edit it, and read back."""
+    rid = bridge_runner_process.runner_id
+    bk = bridge_runner_process.bridge_key
+    marker = f"xyzzy_{int(time.time())}"
+    filename = f"bridge_e2e_{int(time.time())}.py"
+    original_content = f"# {marker}\n"
+
+    # 1. Write
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "WriteFile",
+        {"path": filename, "content": original_content},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert cmd.result["created"] is True
+
+    # 2. ListFiles — new file should appear in the root
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "ListFiles",
+        {"pattern": f"{filename}"},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert any(filename in f for f in cmd.result["files"]), (
+        f"{filename} not in {cmd.result['files']}"
+    )
+
+    # 3. EditFile — replace content
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "EditFile",
+        {
+            "path": filename,
+            "edits": [{"old_string": marker, "new_string": f"edited_{marker}"}],
+        },
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert cmd.result["edits_applied"] == 1
+
+    # 4. ReadFile — verify the edit took effect
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "ReadFile",
+        {"path": filename},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert f"edited_{marker}" in cmd.result["content"]
+
+    # 5. Cleanup via Exec
+    _submit_and_wait(
+        api_client,
+        rid,
+        "Exec",
+        {"command": f"rm {filename}"},
+        bridge_key=bk,
+    )

sdks/python/tests/e2e/runner/test_runner_e2e.py

@@ -0,0 +1,191 @@
+"""Happy-path e2e tests for the local runner.
+
+Test 1 (basic): register echo agent, create job, verify trace output.
+Test 2 (mask):  register echo_config agent, create mask, create job with
+                mask_id, verify the mask value appears in the trace output.
+"""
+
+import time
+from typing import Optional
+
+
+import opik
+import opik.rest_api.client as rest_api_client
+from opik.api_objects.agent_config.config import ConfigManager
+from .conftest import RunnerInfo
+
+
+JOB_COMPLETION_TIMEOUT = 30
+TRACE_PROPAGATION_TIMEOUT = 30
+AGENT_REGISTRATION_TIMEOUT = 10
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def submit_job(
+    api: rest_api_client.OpikApi,
+    agent_name: str,
+    message: str,
+    project_id: str,
+    mask_id: Optional[str] = None,
+) -> None:
+    """Create a job for the given agent."""
+    api.runners.create_job(
+        agent_name=agent_name,
+        inputs={"message": message},
+        project_id=project_id,
+        mask_id=mask_id,
+    )
+
+
+def wait_for_completed_job(
+    api: rest_api_client.OpikApi, runner_id: str, match_text: str
+):
+    """Poll list_jobs until a completed job whose inputs contain *match_text* appears."""
+    result = []
+
+    def _find():
+        page = api.runners.list_jobs(runner_id=runner_id, size=20)
+        if page.content:
+            for j in page.content:
+                if j.status == "completed" and j.inputs and match_text in str(j.inputs):
+                    result.clear()
+                    result.append(j)
+                    return True
+        return False
+
+    assert opik.synchronization.until(
+        _find,
+        max_try_seconds=JOB_COMPLETION_TIMEOUT,
+        allow_errors=True,
+    ), f"No completed job with '{match_text}' found within {JOB_COMPLETION_TIMEOUT}s"
+
+    return result[0]
+
+
+def find_trace_by_input(
+    api: rest_api_client.OpikApi, project_name: str, match_text: str
+):
+    """Poll until a trace whose input contains *match_text* appears and has output."""
+    result = []
+
+    def _find():
+        page = api.traces.get_traces_by_project(
+            project_name=project_name,
+            size=20,
+        )
+        if page.content:
+            for t in page.content:
+                if t.input and match_text in str(t.input) and t.output:
+                    result.clear()
+                    result.append(t)
+                    return True
+        return False
+
+    assert opik.synchronization.until(
+        _find,
+        max_try_seconds=TRACE_PROPAGATION_TIMEOUT,
+        allow_errors=True,
+    ), f"No trace with '{match_text}' found within {TRACE_PROPAGATION_TIMEOUT}s"
+
+    return result[0]
+
+
+def wait_for_agent_registration(
+    api: rest_api_client.OpikApi, agent_name: str, project_id: str
+) -> None:
+    """Poll until the agent is registered with any runner in the project."""
+
+    def _is_agent_registered():
+        runners_page = api.runners.list_runners(project_id=project_id, size=50)
+        if runners_page.content:
+            for runner in runners_page.content:
+                if runner.agents:
+                    for agent in runner.agents:
+                        if agent.name == agent_name:
+                            return True
+        return False
+
+    if not opik.synchronization.until(
+        _is_agent_registered,
+        max_try_seconds=AGENT_REGISTRATION_TIMEOUT,
+        allow_errors=True,
+    ):
+        raise AssertionError(
+            f"Agent '{agent_name}' was not registered within {AGENT_REGISTRATION_TIMEOUT}s"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_runner_happy_path(api_client, runner_process: RunnerInfo, project):
+    """Basic: register echo agent, run job, verify job result, trace output, and job logs."""
+    message = f"hello-e2e-{int(time.time())}"
+
+    wait_for_agent_registration(api_client, "echo", project.id)
+
+    submit_job(api_client, "echo", message, project.id)
+
+    job = wait_for_completed_job(api_client, runner_process.runner_id, message)
+    assert job.result is not None, "Completed job should have a result"
+    assert f"echo: {message}" in str(job.result)
+    assert job.trace_id is not None, "Completed job should have a trace_id"
+
+    trace = find_trace_by_input(api_client, project.name, message)
+    assert f"echo: {message}" in str(trace.output)
+
+    logs_result = []
+
+    def _find_logs():
+        logs = api_client.runners.get_job_logs(job.id)
+        if logs:
+            logs_result.clear()
+            logs_result.extend(logs)
+            return True
+        return False
+
+    assert opik.synchronization.until(
+        _find_logs,
+        max_try_seconds=5,
+        allow_errors=True,
+    ), f"Expected job logs for job {job.id}, got none"
+
+    log_text = " ".join(entry.text for entry in logs_result)
+    assert message in log_text, f"Expected '{message}' in job logs, got: {log_text}"
+
+
+def test_runner_with_mask(opik_client, api_client, runner_process: RunnerInfo, project):
+    """Mask: register echo_config agent, create mask, verify mask value in job result and trace."""
+    message = f"mask-e2e-{int(time.time())}"
+    custom_greeting = f"custom-greeting-{int(time.time())}"
+
+    wait_for_agent_registration(api_client, "echo_config", project.id)
+
+    manager = ConfigManager(
+        project_name=project.name,
+        rest_client_=opik_client.rest_client,
+    )
+    manager.create_blueprint(
+        parameters={"greeting": "default-greeting"},
+    )
+    mask_id = manager.create_mask(
+        parameters={"greeting": custom_greeting},
+    )
+
+    submit_job(api_client, "echo_config", message, project.id, mask_id=mask_id)
+
+    job = wait_for_completed_job(api_client, runner_process.runner_id, message)
+    assert job.result is not None, "Completed job should have a result"
+    assert custom_greeting in str(job.result)
+    assert job.trace_id is not None, "Completed job should have a trace_id"
+
+    trace = find_trace_by_input(api_client, project.name, message)
+    assert custom_greeting in str(trace.output), (
+        f"Expected '{custom_greeting}' in trace output, got: {trace.output}"
+    )

sdks/python/tests/e2e/test_agent_config.py

@@ -0,0 +1,276 @@
+import uuid
+from typing import Annotated, Optional
+
+import pytest
+import opik
+from opik import opik_context
+from opik.api_objects.agent_config.cache import get_global_registry
+from opik.api_objects.agent_config.config import ConfigManager
+from opik.api_objects.agent_config.context import agent_config_context
+
+from opik.api_objects.prompt.text.prompt import Prompt
+from opik.api_objects.prompt.chat.chat_prompt import ChatPrompt
+from . import verifiers
+from ..testlib import ANY_DICT, ANY_BUT_NONE
+
+
+@pytest.fixture(autouse=True)
+def clear_caches_after_test():
+    yield
+    get_global_registry().clear()
+
+
+@pytest.fixture
+def project_name(temporary_project_name: str) -> str:
+    """Alias to the shared per-test project fixture so test bodies stay
+    readable (``project_name`` matches the SDK kwarg name)."""
+    return temporary_project_name
+
+
+def test_multi_class_publishes_store_only_sent_values__happyflow(
+    opik_client: opik.Opik,
+    project_name: str,
+):
+    """Each published blueprint stores only the values that were sent — no carry-forward."""
+
+    class ConfigA(opik.Config):
+        temperature: float
+        model: str
+
+    class ConfigB(opik.Config):
+        retries: int
+
+    opik_client.create_config(
+        ConfigA(temperature=0.5, model="gpt-4"), project_name=project_name
+    )
+    opik_client.create_config(ConfigB(retries=3), project_name=project_name)
+
+    manager = ConfigManager(
+        project_name=project_name,
+        rest_client_=opik_client.rest_client,
+    )
+    latest = manager.get_blueprint()
+    assert latest is not None
+    assert sorted(latest.keys()) == ["retries"]
+
+    project_id = opik_client.rest_client.projects.retrieve_project(name=project_name).id
+    history = opik_client.rest_client.agent_configs.get_blueprint_history(
+        project_id=project_id
+    ).content
+    assert len(history) == 2
+
+
+def test_publish_version_and_retrieve__happyflow(
+    opik_client: opik.Opik,
+    project_name: str,
+):
+    """Core lifecycle: publish, dedup, new version, get by latest / version name / env."""
+
+    class MyConfig(opik.Config):
+        temperature: Annotated[float, "Sampling temperature"]
+        model: str
+        hint: Optional[str]
+
+    # Publish v1 with hint=None and verify the version name comes back.
+    v1_name = opik_client.create_config(
+        MyConfig(temperature=0.5, model="gpt-3.5", hint=None), project_name=project_name
+    )
+    assert isinstance(v1_name, str) and v1_name != ""
+
+    # Backend auto-tags the first blueprint as "prod" — verify without any manual deploy_to.
+    get_global_registry().clear()
+
+    @opik.track(project_name=project_name)
+    def fetch_auto_prod():
+        return opik_client.get_or_create_config(
+            fallback=MyConfig(temperature=0.0, model="fallback", hint=None),
+            project_name=project_name,
+            env="prod",
+        )
+
+    auto_prod = fetch_auto_prod()
+    assert auto_prod.temperature == pytest.approx(0.5)
+    assert auto_prod.model == "gpt-3.5"
+
+    # Publishing different values (hint filled in) creates a new version.
+    get_global_registry().clear()
+    v2_name = opik_client.create_config(
+        MyConfig(temperature=0.8, model="gpt-4", hint="use chain-of-thought"),
+        project_name=project_name,
+    )
+    assert v2_name != v1_name
+
+    # version="latest" returns v2; hint is now a real value.
+    get_global_registry().clear()
+
+    @opik.track(project_name=project_name)
+    def fetch_latest():
+        return opik_client.get_or_create_config(
+            fallback=MyConfig(temperature=0.0, model="fallback", hint=None),
+            project_name=project_name,
+            version="latest",
+        )
+
+    latest = fetch_latest()
+    assert latest.temperature == pytest.approx(0.8)
+    assert latest.model == "gpt-4"
+    assert latest.hint == "use chain-of-thought"
+
+    # version= by name returns v1; hint must be None as originally published.
+    get_global_registry().clear()
+
+    @opik.track(project_name=project_name)
+    def fetch_by_name():
+        return opik_client.get_or_create_config(
+            fallback=MyConfig(temperature=0.0, model="fallback", hint=None),
+            project_name=project_name,
+            version=v1_name,
+        )
+
+    by_name = fetch_by_name()
+    assert by_name.temperature == pytest.approx(0.5)
+    assert by_name.hint is None
+
+    # Deploy v1 to prod; env= fetch returns v1 despite v2 being latest.
+    opik_client.set_config_env(project_name=project_name, version=v1_name, env="prod")
+    get_global_registry().clear()
+
+    @opik.track(project_name=project_name)
+    def fetch_by_env():
+        return opik_client.get_or_create_config(
+            fallback=MyConfig(temperature=0.0, model="fallback", hint=None),
+            project_name=project_name,
+            env="prod",
+        )
+
+    by_env = fetch_by_env()
+    assert by_env.temperature == pytest.approx(0.5)
+    assert by_env.hint is None
+
+
+def test_prompt_field_and_trace_metadata__happyflow(
+    opik_client: opik.Opik,
+    project_name: str,
+):
+    """Prompt-typed and ChatPrompt-typed fields survive the roundtrip with the correct
+    class; field access inside a tracked function injects agent_configuration into
+    trace and span metadata."""
+
+    prompt_name = f"e2e-prompt-{uuid.uuid4().hex[:8]}"
+    chat_prompt_name = f"e2e-chat-prompt-{uuid.uuid4().hex[:8]}"
+
+    prompt_v1 = opik_client.create_prompt(
+        name=prompt_name, prompt="Hello v1", project_name=project_name
+    )
+    chat_prompt_v1 = opik_client.create_chat_prompt(
+        name=chat_prompt_name,
+        messages=[{"role": "user", "content": "Hi v1"}],
+        project_name=project_name,
+    )
+
+    class PromptConfig(opik.Config):
+        system_prompt: Prompt
+        chat_template: ChatPrompt
+        temperature: float
+
+    opik_client.create_config(
+        PromptConfig(
+            system_prompt=prompt_v1,
+            chat_template=chat_prompt_v1,
+            temperature=0.3,
+        ),
+        project_name=project_name,
+    )
+
+    get_global_registry().clear()
+
+    id_storage = {}
+
+    @opik.track(project_name=project_name)
+    def run():
+        cfg = opik_client.get_or_create_config(
+            fallback=PromptConfig(
+                system_prompt=prompt_v1,
+                chat_template=chat_prompt_v1,
+                temperature=0.0,
+            ),
+            project_name=project_name,
+            version="latest",
+        )
+        id_storage["trace_id"] = opik_context.get_current_trace_data().id
+        id_storage["span_id"] = opik_context.get_current_span_data().id
+        id_storage["system_prompt"] = cfg.system_prompt
+        id_storage["system_prompt_version_id"] = cfg.system_prompt.version_id
+        id_storage["chat_template"] = cfg.chat_template
+        id_storage["chat_template_version_id"] = cfg.chat_template.version_id
+        _ = cfg.temperature
+        return cfg
+
+    run()
+    opik.flush_tracker()
+
+    # Prompt field roundtrip — must come back as Prompt, not ChatPrompt.
+    assert isinstance(id_storage["system_prompt"], Prompt)
+    assert not isinstance(id_storage["system_prompt"], ChatPrompt)
+    assert id_storage["system_prompt_version_id"] == prompt_v1.version_id
+
+    # ChatPrompt field roundtrip — must come back as ChatPrompt, not plain Prompt.
+    assert isinstance(id_storage["chat_template"], ChatPrompt)
+    assert id_storage["chat_template_version_id"] == chat_prompt_v1.version_id
+
+    expected_meta = {
+        "_blueprint_id": ANY_BUT_NONE,
+        "blueprint_version": ANY_BUT_NONE,
+        "values": ANY_DICT,
+    }
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=id_storage["trace_id"],
+        metadata={"agent_configuration": ANY_DICT.containing(expected_meta)},
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=id_storage["span_id"],
+        trace_id=id_storage["trace_id"],
+        parent_span_id=None,
+        metadata={"agent_configuration": ANY_DICT.containing(expected_meta)},
+    )
+
+
+def test_mask_overrides_config__happyflow(
+    opik_client: opik.Opik,
+    project_name: str,
+):
+    """A mask overrides selected fields while leaving untouched fields intact."""
+
+    class MyConfig(opik.Config):
+        temperature: float
+        model: str
+
+    opik_client.create_config(
+        MyConfig(temperature=0.5, model="gpt-4"), project_name=project_name
+    )
+
+    get_global_registry().clear()
+
+    manager = ConfigManager(
+        project_name=project_name,
+        rest_client_=opik_client.rest_client,
+    )
+    mask_id = manager.create_mask(parameters={"temperature": 0.9})
+
+    get_global_registry().clear()
+
+    with agent_config_context(mask_id):
+
+        @opik.track(project_name=project_name)
+        def fetch_with_mask():
+            return opik_client.get_or_create_config(
+                fallback=MyConfig(temperature=0.0, model="fallback"),
+                project_name=project_name,
+                version="latest",
+            )
+
+        result = fetch_with_mask()
+        assert result.temperature == pytest.approx(0.9)
+        assert result.model == "gpt-4"

sdks/python/tests/e2e/test_annotation_queue.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import uuid
+import pytest
+
+import opik
+from opik import synchronization
+from . import verifiers
+
+
+@pytest.fixture
+def annotation_queue_name():
+    return f"e2e-tests-annotation-queue-{str(uuid.uuid4())[-6:]}"
+
+
+def test_annotation_queue__add_traces__happyflow(
+    opik_client: opik.Opik,
+    temporary_project_name: str,
+    annotation_queue_name: str,
+):
+    """Test creating a traces annotation queue and adding traces to it."""
+    # Create traces and capture the returned Trace objects
+    trace1 = opik_client.trace(
+        name="trace-1",
+        input={"input": "trace-1-input"},
+        output={"output": "trace-1-output"},
+        project_name=temporary_project_name,
+    )
+    trace2 = opik_client.trace(
+        name="trace-2",
+        input={"input": "trace-2-input"},
+        output={"output": "trace-2-output"},
+        project_name=temporary_project_name,
+    )
+
+    opik_client.flush()
+
+    # Wait for traces to be available (which also ensures project is created)
+    def traces_available():
+        traces = opik_client.search_traces(
+            project_name=temporary_project_name,
+            max_results=10,
+        )
+        return len(traces) >= 2
+
+    synchronization.wait_for_done(traces_available, timeout=30)
+
+    # Create traces annotation queue after project exists
+    queue = opik_client.create_traces_annotation_queue(
+        name=annotation_queue_name,
+        project_name=temporary_project_name,
+        description="Test queue for traces",
+        instructions="Please review these traces",
+    )
+
+    verifiers.verify_traces_annotation_queue(
+        opik_client=opik_client,
+        queue_id=queue.id,
+        name=annotation_queue_name,
+        scope="trace",
+        description="Test queue for traces",
+        instructions="Please review these traces",
+    )
+
+    # Add traces directly using the Trace objects returned by opik_client.trace()
+    queue.add_traces([trace1, trace2])
+
+    # Verify queue items count
+    def check_items_count():
+        updated_queue = opik_client.get_traces_annotation_queue(queue.id)
+        return updated_queue.items_count == 2
+
+    synchronization.wait_for_done(check_items_count, timeout=30)
+
+    # Verify we can retrieve the queue
+    retrieved_queue = opik_client.get_traces_annotation_queue(queue.id)
+    assert retrieved_queue.id == queue.id
+    assert retrieved_queue.name == annotation_queue_name
+    assert retrieved_queue.items_count == 2
+
+    # Retrieve the items in the queue
+    def queue_items_available():
+        return len(queue.get_items()) == 2
+
+    synchronization.wait_for_done(queue_items_available, timeout=30)
+
+    items = queue.get_items()
+    assert {item.id for item in items} == {trace1.id, trace2.id}
+
+    # Remove one trace from queue
+    queue.remove_traces([trace1])
+
+    # Verify queue items count after removal
+    def check_items_count_after_removal():
+        updated_queue = opik_client.get_traces_annotation_queue(queue.id)
+        return updated_queue.items_count == 1
+
+    synchronization.wait_for_done(check_items_count_after_removal, timeout=30)
+
+    # Cleanup: delete the queue
+    queue.delete()
+
+    # Verify queue is deleted
+    with pytest.raises(opik.exceptions.OpikException):
+        opik_client.get_traces_annotation_queue(queue.id)
+
+
+def test_annotation_queue__add_threads__happyflow(
+    opik_client: opik.Opik,
+    temporary_project_name: str,
+    annotation_queue_name: str,
+):
+    """Test creating a threads annotation queue and adding threads to it."""
+    # Thread IDs must be valid UUIDs for annotation queue API
+    # Use full UUIDs and store them to filter later
+    thread_id_1 = "thread_1"
+    thread_id_2 = "thread_2"
+    thread_ids = {thread_id_1, thread_id_2}
+
+    # Create traces that belong to threads
+    for i in range(3):
+        opik_client.trace(
+            name=f"trace-thread1-{i}",
+            input={"input": f"input-{i}"},
+            output={"output": f"output-{i}"},
+            project_name=temporary_project_name,
+            thread_id=thread_id_1,
+        )
+
+    for i in range(2):
+        opik_client.trace(
+            name=f"trace-thread2-{i}",
+            input={"input": f"input-{i}"},
+            output={"output": f"output-{i}"},
+            project_name=temporary_project_name,
+            thread_id=thread_id_2,
+        )
+
+    opik_client.flush()
+
+    # Wait for threads to be available (which also ensures project is created)
+    def threads_available():
+        threads = opik_client.search_threads(
+            project_name=temporary_project_name,
+            max_results=100,
+        )
+        found_ids = {t.id for t in threads}
+        return thread_ids.issubset(found_ids)
+
+    synchronization.wait_for_done(threads_available, timeout=30)
+
+    # Create threads annotation queue after project exists
+    queue = opik_client.create_threads_annotation_queue(
+        name=annotation_queue_name,
+        project_name=temporary_project_name,
+        description="Test queue for threads",
+        instructions="Please review these threads",
+    )
+
+    verifiers.verify_threads_annotation_queue(
+        opik_client=opik_client,
+        queue_id=queue.id,
+        name=annotation_queue_name,
+        scope="thread",
+        description="Test queue for threads",
+        instructions="Please review these threads",
+    )
+
+    # Get threads from API and filter by our thread IDs
+    threads = opik_client.search_threads(
+        project_name=temporary_project_name,
+        max_results=10,
+    )
+    assert len(threads) == 2
+
+    # Add threads to queue
+    queue.add_threads(threads)
+
+    # Verify queue items count
+    def check_items_count():
+        updated_queue = opik_client.get_threads_annotation_queue(queue.id)
+        return updated_queue.items_count == 2
+
+    synchronization.wait_for_done(check_items_count, timeout=30)
+
+    # Verify we can retrieve the queue
+    retrieved_queue = opik_client.get_threads_annotation_queue(queue.id)
+    assert retrieved_queue.id == queue.id
+    assert retrieved_queue.name == annotation_queue_name
+    assert retrieved_queue.items_count == 2
+
+    # Retrieve the items in the queue
+    def queue_items_available():
+        return len(queue.get_items()) == 2
+
+    synchronization.wait_for_done(queue_items_available, timeout=30)
+
+    items = queue.get_items()
+    assert {item.id for item in items} == thread_ids
+
+    # Remove one thread from queue
+    queue.remove_threads([threads[0]])
+
+    # Verify queue items count after removal
+    def check_items_count_after_removal():
+        updated_queue = opik_client.get_threads_annotation_queue(queue.id)
+        return updated_queue.items_count == 1
+
+    synchronization.wait_for_done(check_items_count_after_removal, timeout=30)
+
+    # Cleanup: delete the queue
+    queue.delete()
+
+    # Verify queue is deleted
+    with pytest.raises(opik.exceptions.OpikException):
+        opik_client.get_threads_annotation_queue(queue.id)

sdks/python/tests/e2e/test_anonymization.py

@@ -0,0 +1,211 @@
+import pytest
+
+import opik
+import opik.hooks
+from opik import opik_context
+from opik.anonymizer import factory, anonymizer
+
+from . import verifiers
+from ..testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+# Email pattern
+EMAIL_RULE = (
+    r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+    "[EMAIL_REDACTED]",
+)
+
+# Credit card pattern (simplified)
+CC_RULE = (r"\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b", "[CARD_REDACTED]")
+
+# SSN pattern
+SSN_RULE = {"regex": r"\b\d{3}-\d{2}-\d{4}\b", "replace": "[SSN_REDACTED]"}
+
+
+@pytest.mark.parametrize("override_project_name", [True, False])
+def test_tracked_function__regexp_rules_anonymization__happy_flow(
+    opik_client, override_project_name
+):
+    """Test that sensitive fields are masked in input, output and metadata."""
+    project_name = (
+        generate_project_name("e2e", "anonymization", "override")
+        if override_project_name
+        else None
+    )
+
+    # create and register anonymizer
+    opik.hooks.clear_anonymizers()
+    rules_anonymizer = factory.create_anonymizer([EMAIL_RULE, CC_RULE, SSN_RULE])
+    opik.hooks.add_anonymizer(rules_anonymizer)
+
+    # Setup
+    ID_STORAGE = {}
+
+    # Example PII values used in this test:
+    # Email: john.doe@example.com
+    # Credit Card: 1234 5678 9012 3456
+    # SSN: 123-45-6789
+
+    @opik.track(
+        tags=["outer-tag1", "outer-tag2"],
+        metadata={
+            "outer-metadata-key": "outer-metadata-value",
+            "card-number": "1234 5678 9012 3456",
+        },
+        project_name=project_name,
+    )
+    def f_outer(x: str, email: str):
+        ID_STORAGE["f_outer-trace-id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["f_outer-span-id"] = opik_context.get_current_span_data().id
+
+        f_inner("inner-input")
+        return email
+
+    @opik.track(
+        tags=["inner-tag1", "inner-tag2"],
+        metadata={"inner-metadata-key": "inner-metadata-value", "ssn": "123-45-6789"},
+        project_name=project_name,
+    )
+    def f_inner(y):
+        ID_STORAGE["f_inner-span-id"] = opik_context.get_current_span_data().id
+        return "inner-output"
+
+    # Call
+    f_outer("outer-input", email="john.doe@example.com")
+    opik.flush_tracker()
+
+    anonymized_outer_metadata = {
+        "outer-metadata-key": "outer-metadata-value",
+        "card-number": "[CARD_REDACTED]",
+    }
+
+    # Verify trace
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_outer",
+        input={"x": "outer-input", "email": "[EMAIL_REDACTED]"},
+        output={"output": "[EMAIL_REDACTED]"},
+        metadata=anonymized_outer_metadata,
+        tags=["outer-tag1", "outer-tag2"],
+        project_name=project_name or PROJECT_NAME,
+    )
+
+    # Verify the top level span
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=ID_STORAGE["f_outer-span-id"],
+        parent_span_id=None,
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_outer",
+        input={"x": "outer-input", "email": "[EMAIL_REDACTED]"},
+        output={"output": "[EMAIL_REDACTED]"},
+        metadata=anonymized_outer_metadata,
+        tags=["outer-tag1", "outer-tag2"],
+        project_name=project_name or PROJECT_NAME,
+    )
+
+    anonymized_inner_metadata = {
+        "inner-metadata-key": "inner-metadata-value",
+        "ssn": "[SSN_REDACTED]",
+    }
+
+    # Verify nested span
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=ID_STORAGE["f_inner-span-id"],
+        parent_span_id=ID_STORAGE["f_outer-span-id"],
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_inner",
+        input={"y": "inner-input"},
+        output={"output": "inner-output"},
+        metadata=anonymized_inner_metadata,
+        tags=["inner-tag1", "inner-tag2"],
+        project_name=project_name or PROJECT_NAME,
+    )
+
+
+@pytest.mark.parametrize("override_project_name", [True, False])
+def test_tracked_function__rules_anonymization_remove_sensitive_key__happy_flow(
+    opik_client, override_project_name
+):
+    """Test that sensitive keys are removed from metadata and other sensitive fields are masked in input and output."""
+    project_name = (
+        generate_project_name("e2e", "anonymization", "override")
+        if override_project_name
+        else None
+    )
+
+    class ApiKeyAnonymizer(anonymizer.Anonymizer):
+        def anonymize(self, data, **kwargs):
+            field_name = kwargs.get("field_name")
+            object_type = kwargs.get("object_type")
+            if (
+                field_name == "metadata"
+                and object_type in ["span", "trace"]
+                and "api_key" in data
+            ):
+                del data["api_key"]
+            return data
+
+    # create and register anonymizer
+    opik.hooks.clear_anonymizers()
+    api_key_anonymizer = ApiKeyAnonymizer()
+    opik.hooks.add_anonymizer(api_key_anonymizer)
+    rules_anonymizer = factory.create_anonymizer([EMAIL_RULE, CC_RULE, SSN_RULE])
+    opik.hooks.add_anonymizer(rules_anonymizer)
+
+    # Setup
+    ID_STORAGE = {}
+
+    @opik.track(
+        tags=["outer-tag1", "outer-tag2"],
+        metadata={
+            "outer-metadata-key": "outer-metadata-value",
+            "card-number": "1234 5678 9012 3456",
+            "api_key": "secret-api-key",
+        },
+        project_name=project_name,
+    )
+    def f_outer(x: str, email: str):
+        ID_STORAGE["f_outer-trace-id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["f_outer-span-id"] = opik_context.get_current_span_data().id
+
+        return email
+
+    # Call
+    f_outer("outer-input", email="john.doe@example.com")
+    opik.flush_tracker()
+
+    anonymized_metadata = {
+        "outer-metadata-key": "outer-metadata-value",
+        "card-number": "[CARD_REDACTED]",
+    }
+
+    # Verify trace
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_outer",
+        input={"x": "outer-input", "email": "[EMAIL_REDACTED]"},
+        output={"output": "[EMAIL_REDACTED]"},
+        metadata=anonymized_metadata,
+        tags=["outer-tag1", "outer-tag2"],
+        project_name=project_name or PROJECT_NAME,
+    )
+
+    # Verify the top level span
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=ID_STORAGE["f_outer-span-id"],
+        parent_span_id=None,
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_outer",
+        input={"x": "outer-input", "email": "[EMAIL_REDACTED]"},
+        output={"output": "[EMAIL_REDACTED]"},
+        metadata=anonymized_metadata,
+        tags=["outer-tag1", "outer-tag2"],
+        project_name=project_name or PROJECT_NAME,
+    )

sdks/python/tests/e2e/test_attachments_client.py

@@ -0,0 +1,266 @@
+import os
+
+import pytest
+
+import opik
+from opik import Attachment, id_helpers
+from . import verifiers
+from .conftest import ATTACHMENT_FILE_SIZE
+from ..testlib import generate_project_name
+from opik.rest_api import core as rest_api_core
+from opik import synchronization
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_attachments_client__get_attachment_list_for_trace__happyflow(
+    opik_client: opik.Opik, attachment_data_file
+):
+    trace_id = id_helpers.generate_id()
+
+    file_name = os.path.basename(attachment_data_file.name)
+    attachment = Attachment(
+        data=attachment_data_file.name,
+        file_name=file_name,
+        content_type="application/octet-stream",
+    )
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-with-attachment",
+        project_name=PROJECT_NAME,
+        attachments=[attachment],
+    )
+
+    opik_client.flush()
+
+    attachments_client = opik_client.get_attachment_client()
+
+    synchronization.wait_for_done(
+        lambda: len(
+            attachments_client.get_attachment_list(
+                project_name=PROJECT_NAME,
+                entity_id=trace_id,
+                entity_type="trace",
+            )
+        )
+        > 0,
+        timeout=30,
+    )
+
+    attachments_list = attachments_client.get_attachment_list(
+        project_name=PROJECT_NAME,
+        entity_id=trace_id,
+        entity_type="trace",
+    )
+    assert len(attachments_list) == 1
+    assert attachments_list[0].file_name == file_name
+    assert attachments_list[0].mime_type == "application/octet-stream"
+
+
+def test_attachments_client__download_attachment_for_trace__happyflow(
+    opik_client: opik.Opik, attachment_data_file
+):
+    trace_id = id_helpers.generate_id()
+
+    file_name = os.path.basename(attachment_data_file.name)
+    attachment = Attachment(
+        data=attachment_data_file.name,
+        file_name=file_name,
+        content_type="text/plain",
+    )
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-download-attachment",
+        project_name=PROJECT_NAME,
+        attachments=[attachment],
+    )
+
+    opik_client.flush()
+    attachments_client = opik_client.get_attachment_client()
+
+    synchronization.wait_for_done(
+        lambda: len(
+            attachments_client.get_attachment_list(
+                project_name=PROJECT_NAME,
+                entity_id=trace_id,
+                entity_type="trace",
+            )
+        )
+        > 0,
+        timeout=30,
+    )
+
+    attachment_data = attachments_client.download_attachment(
+        project_name=PROJECT_NAME,
+        entity_type="trace",
+        entity_id=trace_id,
+        file_name=file_name,
+        mime_type="text/plain",
+    )
+    downloaded_content = b"".join(attachment_data)
+
+    # Read the original file content to compare
+    attachment_data_file.seek(0)
+    expected_content = attachment_data_file.read()
+    assert downloaded_content == expected_content
+
+
+def test_attachments_client__get_attachment_list_for_span__happyflow(
+    opik_client: opik.Opik, attachment_data_file
+):
+    span_id = id_helpers.generate_id()
+    trace_id = id_helpers.generate_id()
+
+    file_name = os.path.basename(attachment_data_file.name)
+    attachment = Attachment(
+        data=attachment_data_file.name,
+        file_name=file_name,
+        content_type="application/octet-stream",
+    )
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-span",
+        project_name=PROJECT_NAME,
+    )
+    opik_client.span(
+        id=span_id,
+        trace_id=trace_id,
+        name="test-span-with-attachment",
+        attachments=[attachment],
+    )
+
+    opik_client.flush()
+
+    attachments_client = opik_client.get_attachment_client()
+
+    synchronization.wait_for_done(
+        lambda: len(
+            attachments_client.get_attachment_list(
+                project_name=PROJECT_NAME,
+                entity_id=span_id,
+                entity_type="span",
+            )
+        )
+        > 0,
+        timeout=30,
+    )
+
+    attachments_list = attachments_client.get_attachment_list(
+        project_name=PROJECT_NAME,
+        entity_id=span_id,
+        entity_type="span",
+    )
+    assert len(attachments_list) == 1
+    assert attachments_list[0].file_name == file_name
+    assert attachments_list[0].mime_type == "application/octet-stream"
+
+
+def test_attachments_client__invalid_project_name__or_non_existing_entity_id__raises_error(
+    opik_client: opik.Opik,
+):
+    attachments_client = opik_client.get_attachment_client()
+
+    with pytest.raises(rest_api_core.ApiError):
+        attachments_client.get_attachment_list(
+            project_name="non-existent-project",
+            entity_id="some-entity-id",
+            entity_type="trace",
+        )
+
+    with pytest.raises(rest_api_core.ApiError):
+        attachments_client.get_attachment_list(
+            project_name=PROJECT_NAME,
+            entity_id="non-existent-entity-id",
+            entity_type="trace",
+        )
+
+
+def test_attachments_client__upload_attachment_for_trace__happyflow(
+    opik_client: opik.Opik, attachment_data_file
+):
+    """Test uploading an attachment for a trace."""
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-upload",
+        project_name=PROJECT_NAME,
+    )
+
+    opik_client.flush()
+
+    attachments_client = opik_client.get_attachment_client()
+
+    file_name = os.path.basename(attachment_data_file.name)
+    attachments_client.upload_attachment(
+        project_name=PROJECT_NAME,
+        entity_type="trace",
+        entity_id=trace_id,
+        file_path=attachment_data_file.name,
+        file_name=file_name,
+        mime_type="application/octet-stream",
+    )
+
+    verifiers.verify_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        attachments={
+            file_name: Attachment(
+                data=attachment_data_file.name,
+                file_name=file_name,
+                content_type="application/octet-stream",
+            )
+        },
+        data_sizes={file_name: ATTACHMENT_FILE_SIZE},
+    )
+
+
+def test_attachments_client__upload_attachment_for_span__happyflow(
+    opik_client: opik.Opik, attachment_data_file
+):
+    """Test uploading an attachment for a span."""
+    span_id = id_helpers.generate_id()
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-span-upload",
+        project_name=PROJECT_NAME,
+    )
+    opik_client.span(
+        id=span_id,
+        trace_id=trace_id,
+        name="test-span-for-upload",
+    )
+
+    opik_client.flush()
+
+    attachments_client = opik_client.get_attachment_client()
+
+    file_name = os.path.basename(attachment_data_file.name)
+    attachments_client.upload_attachment(
+        project_name=PROJECT_NAME,
+        entity_type="span",
+        entity_id=span_id,
+        file_path=attachment_data_file.name,
+        file_name=file_name,
+        mime_type="application/octet-stream",
+    )
+
+    verifiers.verify_attachments(
+        opik_client=opik_client,
+        entity_type="span",
+        entity_id=span_id,
+        attachments={
+            file_name: Attachment(
+                data=attachment_data_file.name,
+                file_name=file_name,
+                content_type="application/octet-stream",
+            )
+        },
+        data_sizes={file_name: ATTACHMENT_FILE_SIZE},
+    )

sdks/python/tests/e2e/test_attachments_extraction.py

@@ -0,0 +1,558 @@
+"""
+E2E tests for automatic attachment extraction from trace/span input, output, and metadata.
+
+This module tests the automatic extraction and upload of base64-encoded attachments
+embedded in trace and span data when attachment extraction is enabled.
+"""
+
+import base64
+
+import pytest
+
+import opik
+from opik import id_helpers, datetime_helpers, synchronization
+
+from . import verifiers
+from ..testlib import generate_project_name
+from ..unit.api_objects.attachment import constants
+from .. import testlib
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture(autouse=True)
+def disable_tests_if_attachment_extraction_not_enabled(opik_client: opik.Opik):
+    """Disable tests if attachment extraction is not enabled."""
+    if not opik_client.config.is_attachment_extraction_active:
+        pytest.skip("Attachment extraction is not enabled - skipping E2E tests")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def configure_min_size_tests_env():
+    with testlib.patch_environ({"OPIK_MIN_BASE64_EMBEDDED_ATTACHMENT_SIZE": "30"}):
+        yield
+
+
+def _create_base64_url(media_type: str, base64_data: str) -> str:
+    """Create a data URL with base64-encoded content."""
+    return f"data:{media_type};base64,{base64_data}"
+
+
+def test_extraction__trace_with_end_time__extracts_attachments_from_input(
+    opik_client: opik.Opik,
+):
+    """Test that traces with end_time has attachments extracted from the input field."""
+    trace_id = id_helpers.generate_id()
+
+    # Create a trace with end_time and base64-encoded images in input
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-extraction-input",
+        project_name=PROJECT_NAME,
+        input={
+            "image1": _create_base64_url("image/png", constants.PNG_BASE64),
+            "image2": _create_base64_url("image/jpeg", constants.JPEG_BASE64),
+            "text": "regular text field",
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+        len(base64.b64decode(constants.JPEG_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__trace_without_end_time__does_not_extract_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that traces without end_time does NOT have attachments extracted."""
+    trace_id = id_helpers.generate_id()
+
+    # Create a trace WITHOUT calling end() - no end_time set
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-no-extraction",
+        project_name=PROJECT_NAME,
+        input={
+            "image": _create_base64_url("image/png", constants.PNG_BASE64),
+        },
+    )
+    # Note: NOT calling trace.end()
+
+    opik_client.flush()
+
+    # Wait for the trace to arrive at the backend — once queryable, message
+    # processing is done and attachment-extraction would already have fired.
+    assert synchronization.until(
+        lambda: opik_client.get_trace_content(id=trace_id) is not None,
+        allow_errors=True,
+    ), f"Failed to get trace with id {trace_id}."
+
+    # Verify NO attachments were extracted
+    attachments_client = opik_client.get_attachment_client()
+
+    attachment_list = attachments_client.get_attachment_list(
+        project_name=PROJECT_NAME,
+        entity_id=trace_id,
+        entity_type="trace",
+    )
+
+    assert len(attachment_list) == 0, (
+        f"Expected no attachments, but found {len(attachment_list)}"
+    )
+
+
+def test_extraction__trace_with_end_time__extracts_attachments_from_output(
+    opik_client: opik.Opik,
+):
+    """Test that traces with end_time has attachments extracted from the output field."""
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-extraction-output",
+        project_name=PROJECT_NAME,
+        input={"prompt": "generate an image"},
+        output={
+            "result_image": _create_base64_url("image/png", constants.PNG_BASE64),
+            "result_pdf": _create_base64_url("application/pdf", constants.PDF_BASE64),
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+        len(base64.b64decode(constants.PDF_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__trace_with_end_time__extracts_attachments_from_metadata(
+    opik_client: opik.Opik,
+):
+    """Test that traces with end_time has attachments extracted from the metadata field."""
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-extraction-metadata",
+        project_name=PROJECT_NAME,
+        metadata={
+            "screenshot": _create_base64_url("image/png", constants.PNG_BASE64),
+            "version": "1.0",
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [len(base64.b64decode(constants.PNG_BASE64))]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__trace_with_end_time__extracts_from_all_fields(
+    opik_client: opik.Opik,
+):
+    """Test extraction from input, output, and metadata simultaneously."""
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-extraction-all-fields",
+        project_name=PROJECT_NAME,
+        input={
+            "input_img": _create_base64_url("image/png", constants.PNG_BASE64),
+        },
+        output={
+            "output_img": _create_base64_url("image/jpeg", constants.JPEG_BASE64),
+        },
+        metadata={
+            "meta_gif": _create_base64_url("image/gif", constants.GIF89_BASE64),
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+        len(base64.b64decode(constants.JPEG_BASE64)),
+        len(base64.b64decode(constants.GIF89_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__span_with_end_time__extracts_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that spans with end_time has attachments extracted."""
+    trace_id = id_helpers.generate_id()
+    span_id = id_helpers.generate_id()
+
+    # Create trace first
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-span-extraction",
+        project_name=PROJECT_NAME,
+    )
+
+    # Create a span with end_time and attachments
+    trace.span(
+        id=span_id,
+        name="test-span-extraction",
+        input={
+            "image": _create_base64_url("image/png", constants.PNG_BASE64),
+        },
+        output={
+            "result": _create_base64_url("image/webp", constants.WEBP_BASE64),
+        },
+        metadata={
+            "meta_gif": _create_base64_url("image/gif", constants.GIF89_BASE64),
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+        len(base64.b64decode(constants.WEBP_BASE64)),
+        len(base64.b64decode(constants.GIF89_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="span",
+        entity_id=span_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__span_without_end_time__does_not_extract_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that spans without end_time does NOT have attachments extracted."""
+    trace_id = id_helpers.generate_id()
+    span_id = id_helpers.generate_id()
+
+    # Create trace first
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-span-no-extraction",
+        project_name=PROJECT_NAME,
+    )
+
+    # Create a span WITHOUT an end_time set
+    trace.span(
+        id=span_id,
+        name="test-span-no-extraction",
+        input={
+            "image": _create_base64_url("image/png", constants.PNG_BASE64),
+        },
+    )
+
+    opik_client.flush()
+
+    # Wait for the span to arrive at the backend — once queryable, message
+    # processing is done and attachment-extraction would already have fired.
+    assert synchronization.until(
+        lambda: opik_client.get_span_content(id=span_id) is not None,
+        allow_errors=True,
+    ), f"Failed to get span with id {span_id}."
+
+    # Verify NO attachments were extracted
+    attachments_client = opik_client.get_attachment_client()
+
+    attachment_list = attachments_client.get_attachment_list(
+        project_name=PROJECT_NAME,
+        entity_id=span_id,
+        entity_type="span",
+    )
+
+    assert len(attachment_list) == 0, (
+        f"Expected no attachments, but found {len(attachment_list)}"
+    )
+
+
+def test_extraction__trace_update__extracts_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that trace updates have attachments extracted (updates are always processed)."""
+    trace_id = id_helpers.generate_id()
+
+    # Create an initial trace without attachments
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-update-extraction",
+        project_name=PROJECT_NAME,
+        input={"prompt": "initial input"},
+    )
+
+    opik_client.flush()
+
+    # Update the trace with attachment data
+    trace.update(
+        output={
+            "result_image": _create_base64_url("image/png", constants.PNG_BASE64),
+        }
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [len(base64.b64decode(constants.PNG_BASE64))]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__span_update__extracts_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that span updates have attachments extracted (updates are always processed)."""
+    trace_id = id_helpers.generate_id()
+    span_id = id_helpers.generate_id()
+
+    # Create trace
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-span-update",
+        project_name=PROJECT_NAME,
+    )
+
+    # Create an initial span without attachments
+    span = trace.span(
+        id=span_id,
+        name="test-span-update-extraction",
+        input={"data": "initial"},
+    )
+
+    opik_client.flush()
+
+    # Update the span with attachment data
+    span.update(
+        output={
+            "chart": _create_base64_url("image/svg+xml", constants.SVG_BASE64),
+        }
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [len(base64.b64decode(constants.SVG_BASE64))]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="span",
+        entity_id=span_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__various_file_types__all_extracted(
+    opik_client: opik.Opik,
+):
+    """Test extraction of various file types (PNG, JPEG, PDF, GIF, WebP, SVG, JSON)."""
+    trace_id = id_helpers.generate_id()
+
+    opik_client.trace(
+        id=trace_id,
+        name="test-trace-various-types",
+        project_name=PROJECT_NAME,
+        input={
+            "png": _create_base64_url("image/png", constants.PNG_BASE64),
+            "jpeg": _create_base64_url("image/jpeg", constants.JPEG_BASE64),
+            "pdf": _create_base64_url("application/pdf", constants.PDF_BASE64),
+            "gif": _create_base64_url("image/gif", constants.GIF89_BASE64),
+            "webp": _create_base64_url("image/webp", constants.WEBP_BASE64),
+            "svg": _create_base64_url("image/svg+xml", constants.SVG_BASE64),
+            "json": _create_base64_url("application/json", constants.JSON_BASE64),
+        },
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+        len(base64.b64decode(constants.JPEG_BASE64)),
+        len(base64.b64decode(constants.PDF_BASE64)),
+        len(base64.b64decode(constants.GIF89_BASE64)),
+        len(base64.b64decode(constants.WEBP_BASE64)),
+        len(base64.b64decode(constants.SVG_BASE64)),
+        len(base64.b64decode(constants.JSON_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+
+def test_extraction__backend_reinjects_extracted_attachments(
+    opik_client: opik.Opik,
+):
+    """Test that backend reinjects extracted attachments."""
+    trace_id = id_helpers.generate_id()
+    span_id = id_helpers.generate_id()
+
+    # Create a trace with end_time and base64-encoded images in input
+    trace_input = {
+        "image1": _create_base64_url("image/png", constants.PNG_BASE64),
+        "text": "regular text field",
+    }
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-backend_reinjects_extracted_attachments",
+        project_name=PROJECT_NAME,
+        input=trace_input,
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    # Create a span with end_time and attachments
+    span_input = {
+        "image": _create_base64_url("image/png", constants.PNG_BASE64),
+    }
+    trace.span(
+        id=span_id,
+        name="test-span--backend_reinjects_extracted_attachments",
+        input=span_input,
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    #
+    # Verify attachments were extracted and uploaded for trace and span
+    #
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="trace",
+        entity_id=trace_id,
+        expected_sizes=expected_sizes,
+    )
+
+    expected_sizes = [
+        len(base64.b64decode(constants.PNG_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="span",
+        entity_id=span_id,
+        expected_sizes=expected_sizes,
+    )
+
+    #
+    # Verify trace and span returned by backend has extracted attachments injected back into
+    #
+
+    # The SDK consumes the `data:<mime>;base64,` prefix as part of the
+    # extraction match (OPIK-6608), so the placeholder replaces the whole data
+    # URI. On reinjection the backend only knows the raw base64 payload —
+    # round-trip strips the data URI prefix by design.
+    expected_trace_input = {
+        "image1": constants.PNG_BASE64,
+        "text": "regular text field",
+    }
+    expected_span_input = {
+        "image": constants.PNG_BASE64,
+    }
+
+    # Verify trace
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace.id,
+        name="test-trace-backend_reinjects_extracted_attachments",
+        input=expected_trace_input,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify span
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span_id,
+        parent_span_id=None,
+        trace_id=trace_id,
+        name="test-span--backend_reinjects_extracted_attachments",
+        input=expected_span_input,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_extraction__input_as_top_level_list(
+    opik_client: opik.Opik,
+):
+    """Test that top-level lists are extracted and uploaded as separate attachments."""
+    trace_id = id_helpers.generate_id()
+    span_id = id_helpers.generate_id()
+
+    # Create trace first
+    trace = opik_client.trace(
+        id=trace_id,
+        name="test-trace-for-attachment-list-extraction",
+        project_name=PROJECT_NAME,
+    )
+
+    # Create a span with end_time and a top-level list in the input with attachments
+    data = [
+        {"image": constants.JPEG_BASE64},
+        {"pdf": constants.PDF_BASE64},
+    ]
+
+    trace.span(
+        id=span_id,
+        name="test-span-extraction",
+        input=data,
+        end_time=datetime_helpers.local_timestamp(),
+    )
+
+    opik_client.flush()
+
+    # Verify attachments were extracted and uploaded
+    expected_sizes = [
+        len(base64.b64decode(constants.JPEG_BASE64)),
+        len(base64.b64decode(constants.PDF_BASE64)),
+    ]
+    verifiers.verify_auto_extracted_attachments(
+        opik_client=opik_client,
+        entity_type="span",
+        entity_id=span_id,
+        expected_sizes=expected_sizes,
+    )

sdks/python/tests/e2e/test_backend_check.py

@@ -0,0 +1,17 @@
+import opik
+import pytest
+from opik.rest_api.core import api_error
+
+
+def test_auth_check__happyflow(opik_client: opik.Opik):
+    # Assuming opik client is correctly configured for tests, no
+    # exceptions must be raised.
+    assert opik_client.auth_check() is None
+
+
+def test_auth_check__not_existing_workspace__api_error_raised():
+    opik_client = opik.Opik(
+        workspace="workspace-that-does-not-exist-in-any-installation"
+    )
+    with pytest.raises(api_error.ApiError):
+        opik_client.auth_check()

sdks/python/tests/e2e/test_chat_prompt.py

@@ -0,0 +1,666 @@
+import uuid
+import opik
+from opik.api_objects.prompt import PromptType, ChatPrompt
+from . import verifiers
+from ..testlib import generate_project_name
+import opik.exceptions
+import pytest
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_chat_prompt__create__happyflow(opik_client: opik.Opik):
+    """Test creating a chat prompt with multiple messages."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Hello, {{name}}!"},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        metadata={"version": "1.0", "type": "customer_support"},
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify the prompt was created correctly
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        metadata={"version": "1.0", "type": "customer_support"},
+        type=PromptType.MUSTACHE,
+    )
+
+
+def test_chat_prompt__format__happyflow(opik_client: opik.Opik):
+    """Test formatting a chat prompt with variables."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-format-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a {{role}}."},
+        {"role": "user", "content": "Hello, {{name}}! Tell me about {{topic}}."},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify the prompt was created correctly
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    # Format with variables
+    formatted = chat_prompt.format(
+        variables={"role": "coding assistant", "name": "Alice", "topic": "Python"}
+    )
+
+    assert len(formatted) == 2
+    assert formatted[0]["role"] == "system"
+    assert formatted[0]["content"] == "You are a coding assistant."
+    assert formatted[1]["role"] == "user"
+    assert formatted[1]["content"] == "Hello, Alice! Tell me about Python."
+
+
+def test_chat_prompt__create_new_version__happyflow(opik_client: opik.Opik):
+    """Test creating a new version of a chat prompt."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-versioning-{unique_identifier}"
+    messages_v1 = [
+        {"role": "system", "content": "You are helpful."},
+        {"role": "user", "content": "Hi!"},
+    ]
+
+    # Create initial version
+    chat_prompt_v1 = opik_client.create_chat_prompt(
+        name=prompt_name, messages=messages_v1, project_name=PROJECT_NAME
+    )
+
+    # Create new version with different messages
+    messages_v2 = [
+        {"role": "system", "content": "You are very helpful."},
+        {"role": "user", "content": "Hello there!"},
+        {"role": "assistant", "content": "How can I assist you?"},
+    ]
+
+    chat_prompt_v2 = opik_client.create_chat_prompt(
+        name=prompt_name, messages=messages_v2, project_name=PROJECT_NAME
+    )
+
+    # Verify both versions
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v1,
+        name=prompt_name,
+        messages=messages_v1,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v2,
+        name=prompt_name,
+        messages=messages_v2,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify they share the same prompt ID but have different version IDs
+    assert (
+        chat_prompt_v2.__internal_api__prompt_id__
+        == chat_prompt_v1.__internal_api__prompt_id__
+    )
+    assert (
+        chat_prompt_v2.__internal_api__version_id__
+        != chat_prompt_v1.__internal_api__version_id__
+    )
+    assert chat_prompt_v2.commit != chat_prompt_v1.commit
+
+
+def test_chat_prompt__do_not_create_new_version_with_same_messages(
+    opik_client: opik.Opik,
+):
+    """Test that creating a chat prompt with identical messages doesn't create a new version."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-no-dup-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are helpful."},
+        {"role": "user", "content": "Hello!"},
+    ]
+
+    # Create initial version
+    chat_prompt_v1 = opik_client.create_chat_prompt(
+        name=prompt_name, messages=messages, project_name=PROJECT_NAME
+    )
+
+    # Try to create with same messages
+    chat_prompt_v2 = opik_client.create_chat_prompt(
+        name=prompt_name, messages=messages, project_name=PROJECT_NAME
+    )
+
+    # Verify both prompts
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v1,
+        name=prompt_name,
+        messages=messages,
+        commit=chat_prompt_v1.commit,
+        prompt_id=chat_prompt_v1.__internal_api__prompt_id__,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_v2,
+        name=prompt_name,
+        messages=messages,
+        commit=chat_prompt_v1.commit,
+        prompt_id=chat_prompt_v1.__internal_api__prompt_id__,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_chat_prompt__direct_class_instantiation__happyflow(opik_client: opik.Opik):
+    """Test creating a ChatPrompt directly using the class."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-direct-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are an expert."},
+        {"role": "user", "content": "Explain {{concept}}."},
+    ]
+
+    # Create directly via ChatPrompt class
+    chat_prompt = ChatPrompt(
+        name=prompt_name,
+        messages=messages,
+        metadata={"category": "education"},
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify it was synced to backend
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        metadata={"category": "education"},
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_chat_prompt__multimodal_content__happyflow(opik_client: opik.Opik):
+    """Test chat prompt with multimodal content (text + images)."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-multimodal-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You analyze images."},
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image of {{subject}}?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/image.jpg"},
+                },
+            ],
+        },
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify multimodal content is preserved
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    assert len(chat_prompt.template[1]["content"]) == 2
+    assert chat_prompt.template[1]["content"][0]["type"] == "text"
+    assert chat_prompt.template[1]["content"][1]["type"] == "image_url"
+
+
+def test_chat_prompt__different_types__mustache_and_jinja2(opik_client: opik.Opik):
+    """Test chat prompts with different template types."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    # Test with Mustache
+    prompt_name_mustache = f"chat-prompt-mustache-{unique_identifier}"
+    messages_mustache = [
+        {"role": "user", "content": "Hello {{name}}!"},
+    ]
+
+    chat_prompt_mustache = opik_client.create_chat_prompt(
+        name=prompt_name_mustache,
+        messages=messages_mustache,
+        type=PromptType.MUSTACHE,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_mustache,
+        name=prompt_name_mustache,
+        messages=messages_mustache,
+        type=PromptType.MUSTACHE,
+        project_name=PROJECT_NAME,
+    )
+
+    formatted_mustache = chat_prompt_mustache.format(variables={"name": "Bob"})
+    assert formatted_mustache[0]["content"] == "Hello Bob!"
+
+    # Test with Jinja2
+    prompt_name_jinja = f"chat-prompt-jinja-{unique_identifier}"
+    messages_jinja = [
+        {"role": "user", "content": "Hello {{ name }}!"},
+    ]
+
+    chat_prompt_jinja = opik_client.create_chat_prompt(
+        name=prompt_name_jinja,
+        messages=messages_jinja,
+        type=PromptType.JINJA2,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt_jinja,
+        name=prompt_name_jinja,
+        messages=messages_jinja,
+        type=PromptType.JINJA2,
+        project_name=PROJECT_NAME,
+    )
+
+    formatted_jinja = chat_prompt_jinja.format(variables={"name": "Carol"})
+    assert formatted_jinja[0]["content"] == "Hello Carol!"
+
+
+def test_chat_prompt__empty_messages__should_work(opik_client: opik.Opik):
+    """Test chat prompt with empty messages list."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-empty-{unique_identifier}"
+    messages = []
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=[],
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_chat_prompt__multiple_roles__happyflow(opik_client: opik.Opik):
+    """Test chat prompt with system, user, and assistant messages."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-multirole-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What is the capital of {{country}}?"},
+        {"role": "assistant", "content": "I'd be happy to help with that!"},
+        {"role": "user", "content": "Please tell me."},
+    ]
+
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    formatted = chat_prompt.format(variables={"country": "France"})
+    assert len(formatted) == 4
+    assert formatted[0]["role"] == "system"
+    assert formatted[1]["role"] == "user"
+    assert formatted[1]["content"] == "What is the capital of France?"
+    assert formatted[2]["role"] == "assistant"
+    assert formatted[3]["role"] == "user"
+
+
+def test_get_chat_prompt__chat_prompt__returns_chat_prompt(opik_client: opik.Opik):
+    """Test that get_chat_prompt() returns a ChatPrompt object for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-get-{unique_id}"
+
+    # Create a chat prompt
+    created_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are helpful"},
+            {"role": "user", "content": "Hello {{name}}"},
+        ],
+        project_name=PROJECT_NAME,
+    )
+
+    # Retrieve it with get_chat_prompt()
+    retrieved_prompt = opik_client.get_chat_prompt(
+        name=prompt_name,
+        project_name=PROJECT_NAME,
+    )
+
+    assert retrieved_prompt is not None
+    assert isinstance(retrieved_prompt, opik.ChatPrompt)
+    assert retrieved_prompt.name == prompt_name
+    assert retrieved_prompt.commit == created_prompt.commit
+    assert len(retrieved_prompt.template) == 2
+    assert retrieved_prompt.project_name == PROJECT_NAME
+
+
+def test_get_chat_prompt__string_prompt__prompt_structure_mismatch_error(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt() raises an error for text prompts (type mismatch)."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"text-prompt-{unique_id}"
+
+    # Create a text prompt
+    text_prompt = opik_client.create_prompt(
+        name=prompt_name,
+        prompt="Hello {{name}}",
+        project_name=PROJECT_NAME,
+    )
+
+    # Try to retrieve it with get_chat_prompt() - should raise an error due to type mismatch
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.get_chat_prompt(name=prompt_name, project_name=PROJECT_NAME)
+
+    # Verify the text prompt remains unchanged
+    retrieved_prompt = opik_client.get_prompt(
+        name=prompt_name, project_name=PROJECT_NAME
+    )
+    assert retrieved_prompt is not None
+    verifiers.verify_prompt_version(
+        retrieved_prompt,
+        name=prompt_name,
+        template=text_prompt.prompt,
+        commit=text_prompt.commit,
+        prompt_id=text_prompt.__internal_api__prompt_id__,
+        version_id=text_prompt.__internal_api__version_id__,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_get_chat_prompt_history__chat_prompt__returns_chat_prompts(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt_history() returns ChatPrompt objects for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-history-{unique_id}"
+
+    # Create multiple versions of a chat prompt
+    v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 1"}],
+        project_name=PROJECT_NAME,
+    )
+    v2 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 2"}],
+        project_name=PROJECT_NAME,
+    )
+    v3 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "System message"},
+            {"role": "user", "content": "Version 3"},
+        ],
+        project_name=PROJECT_NAME,
+    )
+
+    # Retrieve history
+    history = opik_client.get_chat_prompt_history(
+        name=prompt_name, project_name=PROJECT_NAME
+    )
+
+    assert len(history) == 3
+    assert all(isinstance(p, opik.ChatPrompt) for p in history)
+    assert history[0].name == prompt_name
+
+    # Verify commits are in the history
+    commits = {p.commit for p in history}
+    assert v1.commit in commits
+    assert v2.commit in commits
+    assert v3.commit in commits
+
+
+def test_get_chat_prompt_history__string_prompt__prompt_structure_mismatch_error(
+    opik_client: opik.Opik,
+):
+    """Test that get_chat_prompt_history() raises an error for text prompts (type mismatch)."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"text-prompt-history-{unique_id}"
+
+    # Create a text prompt
+    text_prompt = opik_client.create_prompt(
+        name=prompt_name, prompt="Hello", project_name=PROJECT_NAME
+    )
+
+    # Try to get history with get_chat_prompt_history() - should raise an error due to type mismatch
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.get_chat_prompt_history(name=prompt_name, project_name=PROJECT_NAME)
+
+    # Verify the text prompt remains unchanged
+    retrieved_prompt = opik_client.get_prompt(
+        name=prompt_name, project_name=PROJECT_NAME
+    )
+    assert retrieved_prompt is not None
+    verifiers.verify_prompt_version(
+        retrieved_prompt,
+        name=prompt_name,
+        template=text_prompt.prompt,
+        commit=text_prompt.commit,
+        prompt_id=text_prompt.__internal_api__prompt_id__,
+        version_id=text_prompt.__internal_api__version_id__,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_search_prompts__filter_chat_prompts_only(opik_client: opik.Opik):
+    """Test that search_prompts() with template_structure filter returns only ChatPrompt instances."""
+    unique_id = str(uuid.uuid4())[-6:]
+
+    # Create text prompts
+    text_prompt_1 = opik_client.create_prompt(
+        name=f"text-search-{unique_id}-1",
+        prompt="Text prompt 1",
+        project_name=PROJECT_NAME,
+    )
+    text_prompt_2 = opik_client.create_prompt(
+        name=f"text-search-{unique_id}-2",
+        prompt="Text prompt 2",
+        project_name=PROJECT_NAME,
+    )
+
+    # Create chat prompts
+    chat_prompt_1 = opik_client.create_chat_prompt(
+        name=f"chat-search-{unique_id}-1",
+        messages=[{"role": "user", "content": "Chat 1"}],
+        project_name=PROJECT_NAME,
+    )
+    chat_prompt_2 = opik_client.create_chat_prompt(
+        name=f"chat-search-{unique_id}-2",
+        messages=[{"role": "user", "content": "Chat 2"}],
+        project_name=PROJECT_NAME,
+    )
+
+    # Search for only chat prompts using template_structure filter
+    results = opik_client.search_prompts(
+        filter_string=f'name contains "{unique_id}" AND template_structure = "chat"',
+        project_name=PROJECT_NAME,
+    )
+
+    # Should only return chat prompts, not text prompts
+    assert len(results) == 2
+    assert all(isinstance(p, opik.ChatPrompt) for p in results)
+
+    result_names = {p.name for p in results}
+    assert chat_prompt_1.name in result_names
+    assert chat_prompt_2.name in result_names
+    assert text_prompt_1.name not in result_names
+    assert text_prompt_2.name not in result_names
+
+
+def test_get_chat_prompt__with_commit__chat_prompt(opik_client: opik.Opik):
+    """Test that get_chat_prompt() with commit works for chat prompts."""
+    unique_id = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-commit-{unique_id}"
+
+    # Create multiple versions
+    v1 = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 1"}],
+        project_name=PROJECT_NAME,
+    )
+    _ = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[{"role": "user", "content": "Version 2"}],
+        project_name=PROJECT_NAME,
+    )
+
+    # Retrieve specific version by commit
+    retrieved_v1 = opik_client.get_chat_prompt(
+        name=prompt_name, commit=v1.commit, project_name=PROJECT_NAME
+    )
+
+    assert retrieved_v1 is not None
+    assert isinstance(retrieved_v1, opik.ChatPrompt)
+    assert retrieved_v1.commit == v1.commit
+    assert retrieved_v1.template[0]["content"] == "Version 1"
+    assert retrieved_v1.name == prompt_name
+    assert retrieved_v1.project_name == PROJECT_NAME
+
+
+def test_get_chat_prompt__nonexistent__returns_none(opik_client: opik.Opik):
+    """Test that get_chat_prompt() returns None for non-existent prompts."""
+    result = opik_client.get_chat_prompt(
+        name="nonexistent-chat-prompt-12345", project_name=opik_client.project_name
+    )
+    assert result is None
+
+
+def test_chat_prompt__template_structure_immutable__error(opik_client: opik.Opik):
+    """Test that template_structure is immutable for chat prompts."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+    prompt_name = f"test-immutable-chat-structure-{unique_identifier}"
+
+    # Create initial chat prompt
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Hello!"},
+        ],
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify chat prompt was created
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Hello!"},
+        ],
+        project_name=PROJECT_NAME,
+    )
+
+    # Attempt to create a text prompt version with the same name should fail
+    with pytest.raises(opik.exceptions.PromptTemplateStructureMismatch):
+        opik_client.create_prompt(
+            name=prompt_name,
+            prompt="This is a text prompt: {{variable}}",
+            project_name=PROJECT_NAME,
+        )
+
+    # Verify the original chat prompt remains unchanged
+    retrieved_chat_prompt = opik_client.get_chat_prompt(
+        name=prompt_name, project_name=PROJECT_NAME
+    )
+    assert retrieved_chat_prompt is not None
+    verifiers.verify_chat_prompt_version(
+        retrieved_chat_prompt,
+        name=prompt_name,
+        messages=chat_prompt.template,
+        commit=chat_prompt.commit,
+        prompt_id=chat_prompt.__internal_api__prompt_id__,
+        version_id=chat_prompt.__internal_api__version_id__,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_chat_prompt__invalid_messages__raises_validation_error(opik_client: opik.Opik):
+    """Test that invalid messages raise ValidationError."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+    prompt_name = f"chat-prompt-invalid-{unique_identifier}"
+
+    with pytest.raises(opik.exceptions.ValidationError) as exc_info:
+        ChatPrompt(
+            name=prompt_name,
+            messages=[{"role": "invalid", "content": "hello"}],
+            project_name=PROJECT_NAME,
+        )
+
+    assert "ChatPrompt.__init__" in str(exc_info.value)
+    assert "messages[0].role" in str(exc_info.value)
+
+
+def test_chat_prompt__create_with_tags__happyflow(opik_client: opik.Opik):
+    """Test that create_chat_prompt() accepts tags parameter."""
+    unique_identifier = str(uuid.uuid4())[-6:]
+
+    prompt_name = f"chat-prompt-with-tags-{unique_identifier}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Hello!"},
+    ]
+    tags = ["chat-tag1", "chat-tag2", "production"]
+
+    # Create chat prompt with tags
+    chat_prompt = opik_client.create_chat_prompt(
+        name=prompt_name,
+        messages=messages,
+        tags=tags,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify prompt was created
+    verifiers.verify_chat_prompt_version(
+        chat_prompt,
+        name=prompt_name,
+        messages=messages,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify tags were set by searching for the prompt
+    filtered_prompts = opik_client.search_prompts(
+        filter_string=f'name = "{prompt_name}" AND tags contains "chat-tag1"',
+        project_name=PROJECT_NAME,
+    )
+    assert len(filtered_prompts) == 1
+    assert filtered_prompts[0].name == prompt_name
+    assert set(filtered_prompts[0].tags) == set(tags)
+    assert filtered_prompts[0].project_name == PROJECT_NAME

sdks/python/tests/e2e/test_dashboard.py

@@ -0,0 +1,220 @@
+"""End-to-end contract and connectivity tests for the dashboard wrapper.
+
+These intentionally cover only the happy-path lifecycle against a real backend —
+that the SDK can create, read, mutate, find, and delete a dashboard, and that the
+config blob round-trips through the backend with the schema the frontend expects.
+Exhaustive validation/serialization cases live in the unit tests.
+"""
+
+import opik
+from opik import dashboard
+from opik.rest_api.core.api_error import ApiError
+from ..testlib import generate_project_name
+from . import verifiers
+import pytest
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def created_dashboards(opik_client: opik.Opik):
+    """Track created dashboards and clean them up even if an assertion fails."""
+    ids = []
+    yield ids
+    for dashboard_id in ids:
+        try:
+            opik_client.delete_dashboard(dashboard_id)
+        except ApiError:
+            pass
+
+
+def test_dashboard_lifecycle__happyflow(
+    opik_client: opik.Opik, created_dashboards: list
+):
+    dash = opik_client.create_dashboard(
+        name=f"e2e-dashboard-{PROJECT_NAME}",
+        type=dashboard.DashboardType.MULTI_PROJECT,
+        description="E2E test dashboard",
+        project_name=PROJECT_NAME,
+    )
+    created_dashboards.append(dash.id)
+
+    assert dash.id is not None
+    assert dash.type == "multi_project"
+
+    # Stats card uses the lowercase-dotted metric namespace.
+    dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.PROJECT_STATS_CARD,
+            title="Total traces",
+            config=dashboard.ProjectStatsCardConfig(
+                metric=dashboard.StatsCardMetric.TRACE_COUNT,
+            ),
+        ),
+    )
+    # Time-series uses the ALL-CAPS metricType namespace, plus a breakdown.
+    dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.PROJECT_METRICS,
+            title="Duration by model",
+            config=dashboard.ProjectMetricsConfig(
+                metric_type=dashboard.ProjectMetricType.DURATION,
+                breakdown=dashboard.BreakdownConfig(
+                    field=dashboard.BreakdownField.MODEL
+                ),
+            ),
+        ),
+    )
+
+    # Re-fetch from the backend and assert the persisted config contract.
+    verifiers.verify_dashboard(
+        opik_client,
+        dashboard_id=dash.id,
+        name=f"e2e-dashboard-{PROJECT_NAME}",
+        type="multi_project",
+        version=dashboard.DashboardState().version,
+        section_count=1,
+        expected_widget_configs={
+            "project_stats_card": {"metric": "trace_count"},
+            "project_metrics": {
+                "metricType": "DURATION",
+                "breakdown": {"field": "model"},
+            },
+        },
+    )
+
+    # The dashboard is discoverable via find.
+    found = opik_client.get_dashboards(
+        name=f"e2e-dashboard-{PROJECT_NAME}", max_results=10
+    )
+    assert any(found_dashboard.id == dash.id for found_dashboard in found)
+
+    # Delete removes it.
+    opik_client.delete_dashboard(dash.id)
+    with pytest.raises(ApiError) as exc_info:
+        opik_client.get_dashboard(dash.id)
+    assert exc_info.value.status_code == 404
+
+
+def test_update_and_remove_widget__persisted(
+    opik_client: opik.Opik, created_dashboards: list
+):
+    dash = opik_client.create_dashboard(
+        name=f"e2e-dashboard-mutate-{PROJECT_NAME}",
+        type=dashboard.DashboardType.MULTI_PROJECT,
+    )
+    created_dashboards.append(dash.id)
+
+    widget_id = dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.TEXT_MARKDOWN,
+            title="Notes",
+            config=dashboard.TextMarkdownConfig(content="initial"),
+        ),
+    )
+
+    dash.update_widget(widget_id, config={"content": "updated"})
+    dash.rename(f"e2e-dashboard-renamed-{PROJECT_NAME}")
+
+    verifiers.verify_dashboard(
+        opik_client,
+        dashboard_id=dash.id,
+        name=f"e2e-dashboard-renamed-{PROJECT_NAME}",
+        type="multi_project",
+        expected_widget_configs={"text_markdown": {"content": "updated"}},
+    )
+
+    dash.remove_widget(widget_id)
+    verifiers.verify_dashboard(
+        opik_client,
+        dashboard_id=dash.id,
+        section_count=1,
+        expected_widget_configs={},
+    )
+    refetched = opik_client.get_dashboard(dash.id)
+    assert refetched.sections[0].widgets == []
+
+
+def test_experiments_dashboard_widgets__all_types(
+    opik_client: opik.Opik, created_dashboards: list
+):
+    dash = opik_client.create_dashboard(
+        name=f"e2e-dashboard-experiments-{PROJECT_NAME}",
+        type=dashboard.DashboardType.EXPERIMENTS,
+    )
+    created_dashboards.append(dash.id)
+
+    dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.EXPERIMENTS_FEEDBACK_SCORES,
+            title="Feedback scores",
+            config=dashboard.ExperimentsFeedbackScoresConfig(
+                chart_type=dashboard.ChartType.BAR,
+                max_experiments_count=5,
+            ),
+        ),
+    )
+    dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.EXPERIMENT_LEADERBOARD,
+            title="Leaderboard",
+            config=dashboard.ExperimentLeaderboardConfig(enable_ranking=False),
+        ),
+    )
+
+    verifiers.verify_dashboard(
+        opik_client,
+        dashboard_id=dash.id,
+        type="experiments",
+        section_count=1,
+        expected_widget_configs={
+            "experiments_feedback_scores": {"chartType": "bar"},
+            "experiment_leaderboard": {"enableRanking": False},
+        },
+    )
+
+
+def test_move_widget_position__persisted(
+    opik_client: opik.Opik, created_dashboards: list
+):
+    dash = opik_client.create_dashboard(
+        name=f"e2e-dashboard-move-{PROJECT_NAME}",
+        type=dashboard.DashboardType.MULTI_PROJECT,
+        project_name=PROJECT_NAME,
+    )
+    created_dashboards.append(dash.id)
+
+    # Two stats-card widgets; auto-layout places them at x=0 and x=1 (w=1 each).
+    w1_id = dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.PROJECT_STATS_CARD,
+            title="Widget A",
+            config=dashboard.ProjectStatsCardConfig(
+                metric=dashboard.StatsCardMetric.TRACE_COUNT,
+            ),
+        ),
+    )
+    dash.add_widget(
+        dashboard.DashboardWidget(
+            type=dashboard.WidgetType.PROJECT_STATS_CARD,
+            title="Widget B",
+            config=dashboard.ProjectStatsCardConfig(
+                metric=dashboard.StatsCardMetric.ERROR_COUNT,
+            ),
+        ),
+    )
+
+    # Move w1 to the far right (x=5) via replace_sections.
+    sections = dash.sections
+    for item in sections[0].layout:
+        if item.id == w1_id:
+            item.x = 5
+            item.y = 0
+    dash.replace_sections(sections)
+
+    verifiers.verify_dashboard(
+        opik_client,
+        dashboard_id=dash.id,
+        section_count=1,
+        expected_widget_positions={w1_id: {"x": 5, "y": 0}},
+    )

sdks/python/tests/e2e/test_dataset.py

@@ -0,0 +1,375 @@
+import opik
+import opik.exceptions
+from opik import synchronization
+
+from opik.api_objects.dataset import dataset_item
+from opik.api_objects import helpers
+from . import verifiers
+from ..testlib import generate_project_name
+import pytest
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_create_and_populate_dataset__happyflow(
+    opik_client: opik.Opik, dataset_name: str
+):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+            {
+                "input": {"question": "What is the of capital of Poland?"},
+                "expected_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    EXPECTED_DATASET_ITEMS = [
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of France?"},
+            expected_output={"output": "Paris"},
+        ),
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Germany?"},
+            expected_output={"output": "Berlin"},
+        ),
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Poland?"},
+            expected_output={"output": "Warsaw"},
+        ),
+    ]
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=EXPECTED_DATASET_ITEMS,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_insert_and_update_item__dataset_size_should_be_the_same__an_item_with_the_same_id_should_have_new_content(
+    opik_client: opik.Opik, dataset_name: str
+):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+
+    ITEM_ID = helpers.generate_id()
+    dataset.insert(
+        [
+            {
+                "id": ITEM_ID,
+                "input": {"question": "What is the of capital of France?"},
+            },
+        ]
+    )
+    dataset.update(
+        [
+            {
+                "id": ITEM_ID,
+                "input": {"question": "What is the of capital of Belarus?"},
+            },
+        ]
+    )
+    EXPECTED_DATASET_ITEMS = [
+        dataset_item.DatasetItem(
+            input={"question": "What is the of capital of Belarus?"},
+        ),
+    ]
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=EXPECTED_DATASET_ITEMS,
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_deduplication(opik_client: opik.Opik, dataset_name: str):
+    DESCRIPTION = "E2E test dataset"
+
+    item = {
+        "user_input": {"question": "What is the of capital of France?"},
+        "expected_model_output": {"output": "Paris"},
+    }
+
+    # Write the dataset
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+    dataset.insert([item])
+
+    # Read the dataset and insert the same item
+    new_dataset = opik_client.get_dataset(dataset_name, project_name=PROJECT_NAME)
+    new_dataset.insert([item])
+
+    # Verify the dataset
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=[
+            dataset_item.DatasetItem(**item),
+        ],
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_dataset_clearing(opik_client: opik.Opik, dataset_name: str):
+    DESCRIPTION = "E2E test dataset"
+
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the of capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is the of capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+        ]
+    )
+    dataset.clear()
+
+    verifiers.verify_dataset(
+        opik_client=opik_client,
+        name=dataset_name,
+        description=DESCRIPTION,
+        dataset_items=[],
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_get_items_with_filter__returns_filtered_items(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_items with filter_string returns correct filtered items."""
+    DESCRIPTION = "E2E test dataset for filtering"
+
+    # Create dataset with items that have different data.category values
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+                "category": "geography",
+            },
+            {
+                "input": {"question": "What is 2 + 2?"},
+                "expected_output": {"output": "4"},
+                "category": "math",
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_output": {"output": "Warsaw"},
+                "category": "geography",
+            },
+        ]
+    )
+
+    verifiers.verify_dataset_filtered_items(
+        opik_client=opik_client,
+        dataset_name=dataset_name,
+        filter_string='data.category = "geography"',
+        expected_count=2,
+        expected_inputs={
+            "What is the capital of France?",
+            "What is the capital of Poland?",
+        },
+        project_name=PROJECT_NAME,
+    )
+
+
+def test_get_items_with_filter__filter_excludes_all_items__returns_empty_list(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_items with filter that matches no items returns empty list."""
+    DESCRIPTION = "E2E test dataset for empty filter"
+
+    # Create dataset with items
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "What is 2 + 2?"},
+                "expected_output": {"output": "4"},
+            },
+        ]
+    )
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "category": "geography",
+            },
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "category": "geography",
+            },
+        ]
+    )
+
+    verifiers.verify_dataset_filtered_items(
+        opik_client=opik_client,
+        dataset_name=dataset_name,
+        filter_string='data.category = "nonexistent"',
+        expected_count=0,
+        expected_inputs=set(),
+        project_name=PROJECT_NAME,
+    )
+
+
+def _wait_for_version(dataset, expected_version: str, timeout: float = 10) -> None:
+    """Wait for dataset to have the expected version, fail if not reached."""
+    success = synchronization.until(
+        lambda: dataset.get_current_version_name() == expected_version,
+        max_try_seconds=timeout,
+    )
+    assert success, f"Expected version '{expected_version}' was not created in time"
+
+
+def test_get_version_view__returns_items_from_specific_version(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_version_view returns items from a specific dataset version.
+
+    Also tests that get_current_version_name returns correct version after mutations.
+    """
+    DESCRIPTION = "E2E test dataset for version view"
+
+    dataset = opik_client.create_dataset(
+        dataset_name, description=DESCRIPTION, project_name=PROJECT_NAME
+    )
+
+    # Version should be None before any items are inserted
+    assert dataset.get_current_version_name() is None
+
+    # Insert first batch of items - creates v1
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+                "expected_output": {"output": "Paris"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Insert second batch of items - creates v2
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Germany?"},
+                "expected_output": {"output": "Berlin"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v2")
+
+    # Get version view for v1 - should only have 1 item
+    v1_view = dataset.get_version_view("v1")
+    v1_items = v1_view.get_items()
+    assert len(v1_items) == 1
+    assert v1_items[0]["input"] == {"question": "What is the capital of France?"}
+    assert v1_view.version_name == "v1"
+    assert v1_view.items_total == 1
+    assert v1_view.project_name == PROJECT_NAME
+
+    # Get version view for v2 - should have 2 items
+    v2_view = dataset.get_version_view("v2")
+    v2_items = v2_view.get_items()
+    assert len(v2_items) == 2
+    assert v2_view.version_name == "v2"
+    assert v2_view.items_total == 2
+    assert v2_view.project_name == PROJECT_NAME
+
+    # Current dataset should also have 2 items
+    current_items = dataset.get_items()
+    assert len(current_items) == 2
+
+    # Delete an item - should create v3
+    dataset.delete([current_items[0]["id"]])
+    _wait_for_version(dataset, "v3")
+
+    # Get version view for v3 - should have 1 item
+    v3_view = dataset.get_version_view("v3")
+    v3_items = v3_view.get_items()
+    assert len(v3_items) == 1
+    assert v3_view.version_name == "v3"
+    assert v3_view.items_total == 1
+    assert v3_view.project_name == PROJECT_NAME
+
+
+def test_get_version_view__version_not_found__raises_exception(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that get_version_view raises DatasetVersionNotFound for non-existent version."""
+    DESCRIPTION = "E2E test dataset for version not found"
+
+    dataset = opik_client.create_dataset(dataset_name, description=DESCRIPTION)
+
+    # Insert items to create v1
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of France?"},
+            },
+        ]
+    )
+    _wait_for_version(dataset, "v1")
+
+    # Try to get a non-existent version
+    with pytest.raises(opik.exceptions.DatasetVersionNotFound):
+        dataset.get_version_view("v999")
+
+
+def test_dataset_items_count__returns_correct_count_after_insert(
+    opik_client: opik.Opik, dataset_name: str
+):
+    """Test that dataset_items_count returns the correct count after insert."""
+    dataset = opik_client.create_dataset(dataset_name, description="items_count test")
+
+    dataset.insert(
+        [
+            {"input": {"question": "What is 2+2?"}},
+            {"input": {"question": "What is 3+3?"}},
+            {"input": {"question": "What is 4+4?"}},
+        ]
+    )
+
+    success = synchronization.until(
+        lambda: dataset.dataset_items_count == 3,
+        max_try_seconds=30,
+    )
+    assert success, f"Expected dataset_items_count=3, got {dataset.dataset_items_count}"

sdks/python/tests/e2e/test_environment.py

@@ -0,0 +1,112 @@
+"""End-to-end tests for the SDK ``environment`` feature.
+
+Exercises:
+- Environment CRUD round-trip via the Opik client.
+- Per-call ``environment=`` on ``Opik.trace`` is persisted on the trace, and
+  spans (manual + ``@track``) inherit that value from the parent trace.
+- Backwards compat: existing call sites without environment still log successfully.
+"""
+
+from typing import Dict
+
+import opik
+from opik import id_helpers
+from . import verifiers
+from ..testlib import generate_project_name
+
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def test_environment_crud__round_trip(opik_client: opik.Opik, environment_name: str):
+    created = opik_client.create_environment(
+        name=environment_name,
+        description="created by SDK e2e test",
+        color="#abcdef",
+    )
+    assert created.name == environment_name
+    assert created.description == "created by SDK e2e test"
+    assert created.color == "#abcdef"
+    assert created.id is not None
+
+    listed = opik_client.get_environments()
+    assert any(env.name == environment_name for env in listed)
+
+    updated = opik_client.update_environment(
+        environment_name, description="updated by SDK e2e test"
+    )
+    assert updated.description == "updated by SDK e2e test"
+    assert updated.name == environment_name
+
+    opik_client.delete_environment(environment_name)
+
+    # Calling delete on a missing name is a no-op.
+    opik_client.delete_environment(environment_name)
+
+
+def test_trace__environment_kwarg_is_persisted_on_the_trace(
+    opik_client: opik.Opik, environment_name: str
+):
+    opik_client.create_environment(name=environment_name)
+
+    trace_id = id_helpers.generate_id()
+    opik_client.trace(
+        id=trace_id,
+        name="env-trace",
+        project_name=PROJECT_NAME,
+        environment=environment_name,
+    )
+    opik_client.flush()
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace_id,
+        project_name=PROJECT_NAME,
+        environment=environment_name,
+    )
+
+
+def test_span__inherits_trace_environment_and_ignores_per_call_override(
+    opik_client: opik.Opik, environment_name: str
+):
+    """A span persists its parent trace's environment. Once the trace is open
+    with an environment, an inner ``@track`` cannot override it — the inner
+    span's per-call ``environment=`` is ignored and it inherits the trace's
+    environment."""
+    opik_client.create_environment(name=environment_name)
+
+    captured: Dict[str, str] = {}
+
+    @opik.track(name="inner-fn", environment="this-should-be-ignored")
+    def inner():
+        ctx = opik.opik_context.get_current_span_data()
+        captured["span_id"] = ctx.id
+        captured["trace_id"] = ctx.trace_id
+        captured["parent_span_id"] = ctx.parent_span_id
+
+    @opik.track(name="outer-fn", environment=environment_name)
+    def outer():
+        inner()
+
+    outer()
+    opik.flush_tracker()
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=captured["span_id"],
+        trace_id=captured["trace_id"],
+        parent_span_id=captured["parent_span_id"],
+        environment=environment_name,
+    )
+
+
+def test_trace_without_environment__still_works_for_backwards_compat(
+    opik_client: opik.Opik,
+):
+    trace_id = id_helpers.generate_id()
+    opik_client.trace(id=trace_id, name="no-env-trace", project_name=PROJECT_NAME)
+    opik_client.flush()
+
+    verifiers.verify_trace(
+        opik_client=opik_client, trace_id=trace_id, project_name=PROJECT_NAME
+    )

sdks/python/tests/e2e/test_experiments.py

@@ -0,0 +1,379 @@
+from typing import Dict, Any
+
+import opik
+from opik.api_objects.experiment import experiment_item
+from opik.evaluation.metrics import score_result
+from opik.types import FeedbackScoreDict
+
+from . import verifiers
+from ..testlib import assert_equal, ANY_BUT_NONE, generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def llm_task(item: Dict[str, Any]):
+    if item["input"] == {"question": "What is the capital of Ukraine?"}:
+        return {"output": "Kyiv"}
+    if item["input"] == {"question": "What is the capital of France?"}:
+        return {"output": "Paris"}
+    if item["input"] == {"question": "What is the capital of Germany?"}:
+        return {"output": "Berlin"}
+    if item["input"] == {"question": "What is the capital of Poland?"}:
+        return {"output": "Krakow"}
+
+    raise AssertionError(
+        f"Task received dataset item with an unexpected input: {item['input']}"
+    )
+
+
+def equals_scoring_function(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]):
+    reference = dataset_item["expected_model_output"]["output"]
+    prediction = task_outputs["output"]
+    if reference == prediction:
+        value = 1.0
+    else:
+        value = 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="Correct output value" if value == 1.0 else "Incorrect output value",
+    )
+
+
+def test__find_experiment_items_for_dataset__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    opik.flush_tracker()
+
+    # make sure experiments saved and available
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        project_name=PROJECT_NAME,
+    )
+
+    # find experiment items for dataset
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiments = opik_client.get_experiments_client()
+    experiment_items_contents = experiments.find_experiment_items_for_dataset(
+        dataset_name=dataset_name,
+        experiment_ids=[retrieved_experiment.id],
+        project_name=opik_client.project_name,
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME
+    assert len(experiment_items_contents) == 2
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Poland?"},
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Incorrect output value",
+                    value=0.0,
+                )
+            ],
+        ),
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Kyiv"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Ukraine?"},
+            },
+            evaluation_task_output={"output": "Kyiv"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Correct output value",
+                    value=1.0,
+                )
+            ],
+        ),
+    ]
+
+    assert_equal(
+        expected=sorted(
+            EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+            key=lambda item: str(item.evaluation_task_output),
+        ),
+        actual=sorted(
+            experiment_items_contents, key=lambda item: str(item.evaluation_task_output)
+        ),
+    )
+
+
+def test__find_experiment_items_for_dataset__filtered__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "gpt-3.5",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    opik.flush_tracker()
+
+    # make sure experiments saved and available
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "gpt-3.5"},
+        traces_amount=2,  # one trace per dataset item
+        feedback_scores_amount=1,
+        project_name=PROJECT_NAME,
+    )
+
+    # find experiment items for dataset
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    experiments = opik_client.get_experiments_client()
+    experiment_items_contents = experiments.find_experiment_items_for_dataset(
+        dataset_name=dataset_name,
+        experiment_ids=[retrieved_experiment.id],
+        filter_string="feedback_scores.equals_scoring_function = 0.0",
+        project_name=PROJECT_NAME,
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+    assert len(experiment_items_contents) == 1
+
+    EXPECTED_EXPERIMENT_ITEMS_CONTENT = [
+        experiment_item.ExperimentItemContent(
+            id=ANY_BUT_NONE,
+            dataset_item_id=ANY_BUT_NONE,
+            trace_id=ANY_BUT_NONE,
+            dataset_item_data={
+                "expected_model_output": {"output": "Warsaw"},
+                "id": ANY_BUT_NONE,
+                "input": {"question": "What is the capital of Poland?"},
+            },
+            evaluation_task_output={"output": "Krakow"},
+            feedback_scores=[
+                FeedbackScoreDict(
+                    category_name=None,
+                    name="equals_scoring_function",
+                    reason="Incorrect output value",
+                    value=0.0,
+                )
+            ],
+        )
+    ]
+
+    assert_equal(
+        expected=EXPECTED_EXPERIMENT_ITEMS_CONTENT,
+        actual=experiment_items_contents,
+    )
+
+
+def test__experiment_scores__happy_path(
+    opik_client: opik.Opik, dataset_name: str, experiment_name: str
+):
+    """Test that experiment scoring functions are executed and scores are logged."""
+
+    def compute_experiment_scores(test_results):
+        """Aggregate scores across all test results."""
+        # Extract all scoring function values
+        all_scores = []
+        for result in test_results:
+            if result.score_results:
+                all_scores.extend([score.value for score in result.score_results])
+
+        if not all_scores:
+            return []
+
+        # Compute aggregate metrics
+        return [
+            score_result.ScoreResult(
+                name="max_score",
+                value=max(all_scores),
+                reason=f"Maximum score across {len(all_scores)} measurements",
+            ),
+            score_result.ScoreResult(
+                name="min_score",
+                value=min(all_scores),
+                reason=f"Minimum score across {len(all_scores)} measurements",
+            ),
+            score_result.ScoreResult(
+                name="avg_score",
+                value=sum(all_scores) / len(all_scores),
+                reason=f"Average score across {len(all_scores)} measurements",
+            ),
+        ]
+
+    # Create dataset
+    dataset = opik_client.create_dataset(dataset_name, project_name=PROJECT_NAME)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "What is the capital of Ukraine?"},
+                "expected_model_output": {"output": "Kyiv"},
+            },
+            {
+                "input": {"question": "What is the capital of Poland?"},
+                "expected_model_output": {"output": "Warsaw"},
+            },
+        ]
+    )
+
+    # Run evaluation with experiment scoring functions
+    evaluation_result = opik.evaluate(
+        dataset=dataset,
+        task=llm_task,
+        scoring_functions=[equals_scoring_function],
+        experiment_scoring_functions=[compute_experiment_scores],
+        experiment_name=experiment_name,
+        experiment_config={
+            "model_name": "test-model",
+        },
+        scoring_key_mapping={
+            "reference": lambda x: x["expected_model_output"]["output"],
+        },
+        project_name=PROJECT_NAME,
+    )
+
+    opik.flush_tracker()
+
+    # Verify experiment was created with experiment scores
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=evaluation_result.experiment_id,
+        experiment_name=evaluation_result.experiment_name,
+        experiment_metadata={"model_name": "test-model"},
+        traces_amount=2,
+        feedback_scores_amount=1,
+        project_name=PROJECT_NAME,
+    )
+
+    # Verify experiment scores are present in evaluation result
+    assert evaluation_result.experiment_scores is not None, (
+        "Experiment scores should not be None"
+    )
+    assert len(evaluation_result.experiment_scores) == 3, (
+        f"Expected 3 experiment scores, got {len(evaluation_result.experiment_scores)}"
+    )
+
+    score_names = {score.name for score in evaluation_result.experiment_scores}
+    assert score_names == {
+        "max_score",
+        "min_score",
+        "avg_score",
+    }, f"Expected score names {{max_score, min_score, avg_score}}, got {score_names}"
+
+    # Verify experiment scores are retrievable via SDK API
+    retrieved_experiment = opik_client.get_experiment_by_name(
+        experiment_name, project_name=PROJECT_NAME
+    )
+    rest_client = opik_client._rest_client
+    experiment_content = rest_client.experiments.get_experiment_by_id(
+        retrieved_experiment.id
+    )
+    assert retrieved_experiment.project_name == PROJECT_NAME
+
+    assert experiment_content.experiment_scores is not None, (
+        "Experiment scores should be persisted in backend"
+    )
+    assert len(experiment_content.experiment_scores) == 3, (
+        f"Expected 3 experiment scores in backend, got {len(experiment_content.experiment_scores)}"
+    )
+
+    backend_score_names = {score.name for score in experiment_content.experiment_scores}
+    assert backend_score_names == {"max_score", "min_score", "avg_score"}, (
+        f"Expected backend score names {{max_score, min_score, avg_score}}, got {backend_score_names}"
+    )
+
+    # Verify score values are reasonable
+    max_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "max_score"
+    )
+    min_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "min_score"
+    )
+    avg_score = next(
+        s for s in evaluation_result.experiment_scores if s.name == "avg_score"
+    )
+
+    assert 0.0 <= max_score.value <= 1.0, (
+        f"max_score should be in [0,1], got {max_score.value}"
+    )
+    assert 0.0 <= min_score.value <= 1.0, (
+        f"min_score should be in [0,1], got {min_score.value}"
+    )
+    assert 0.0 <= avg_score.value <= 1.0, (
+        f"avg_score should be in [0,1], got {avg_score.value}"
+    )
+    assert min_score.value <= avg_score.value <= max_score.value, (
+        f"Score ordering should be min <= avg <= max, got {min_score.value} <= {avg_score.value} <= {max_score.value}"
+    )

sdks/python/tests/e2e/test_feedback_scores.py

@@ -0,0 +1,432 @@
+from typing import List
+import opik
+from opik import opik_context
+from opik.types import BatchFeedbackScoreDict, FeedbackScoreDict
+import pytest
+from . import verifiers
+from ..testlib import generate_project_name
+
+_PROJECT_MANUAL = generate_project_name("manually-specified-project")
+
+
+def test_feedbacks_are_logged_via_trace_and_span__happyflow(opik_client: opik.Opik):
+    trace = opik_client.trace(
+        name="trace-name",
+    )
+    trace.log_feedback_score(
+        "trace-metric-1", value=0.5, category_name="category-1", reason="some-reason-1"
+    )
+    trace.log_feedback_score(
+        "trace-metric-2", value=0.95, category_name="category-2", reason="some-reason-2"
+    )
+
+    span = trace.span(
+        name="span-name",
+    )
+    span.log_feedback_score(
+        "span-metric-1", value=0.75, category_name="category-3", reason="some-reason-3"
+    )
+    span.log_feedback_score(
+        "span-metric-2", value=0.25, category_name="category-4", reason="some-reason-4"
+    )
+
+    opik_client.flush()
+
+    EXPECTED_TRACE_FEEDBACK_SCORES: List[FeedbackScoreDict] = [
+        {
+            "id": trace.id,
+            "name": "trace-metric-1",
+            "value": 0.5,
+            "category_name": "category-1",
+            "reason": "some-reason-1",
+        },
+        {
+            "id": trace.id,
+            "name": "trace-metric-2",
+            "value": 0.95,
+            "category_name": "category-2",
+            "reason": "some-reason-2",
+        },
+    ]
+
+    EXPECTED_SPAN_FEEDBACK_SCORES: List[FeedbackScoreDict] = [
+        {
+            "id": span.id,
+            "name": "span-metric-1",
+            "value": 0.75,
+            "category_name": "category-3",
+            "reason": "some-reason-3",
+        },
+        {
+            "id": span.id,
+            "name": "span-metric-2",
+            "value": 0.25,
+            "category_name": "category-4",
+            "reason": "some-reason-4",
+        },
+    ]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace.id,
+        name="trace-name",
+        feedback_scores=EXPECTED_TRACE_FEEDBACK_SCORES,
+        source="sdk",
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span.id,
+        trace_id=span.trace_id,
+        parent_span_id=None,
+        name="span-name",
+        feedback_scores=EXPECTED_SPAN_FEEDBACK_SCORES,
+        source="sdk",
+    )
+
+
+def test_feedbacks_are_logged_via_trace_and_span__and_deleted(opik_client: opik.Opik):
+    trace = opik_client.trace(
+        name="trace-name",
+    )
+    trace.log_feedback_score(
+        "trace-metric-1", value=0.5, category_name="category-1", reason="some-reason-1"
+    )
+    trace.log_feedback_score(
+        "trace-metric-2", value=0.95, category_name="category-2", reason="some-reason-2"
+    )
+
+    span = trace.span(
+        name="span-name",
+    )
+    span.log_feedback_score(
+        "span-metric-1", value=0.75, category_name="category-3", reason="some-reason-3"
+    )
+    span.log_feedback_score(
+        "span-metric-2", value=0.25, category_name="category-4", reason="some-reason-4"
+    )
+
+    opik_client.flush()
+
+    opik_client.delete_trace_feedback_score(trace.id, "trace-metric-1")
+    opik_client.delete_span_feedback_score(span.id, "span-metric-1")
+    opik_client.delete_span_feedback_score(span.id, "span-metric-2")
+
+    opik_client.flush()
+
+    EXPECTED_TRACE_FEEDBACK_SCORES: List[FeedbackScoreDict] = [
+        {
+            "id": trace.id,
+            "name": "trace-metric-2",
+            "value": 0.95,
+            "category_name": "category-2",
+            "reason": "some-reason-2",
+        },
+    ]
+
+    EXPECTED_SPAN_FEEDBACK_SCORES = None
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace.id,
+        name="trace-name",
+        feedback_scores=EXPECTED_TRACE_FEEDBACK_SCORES,
+        source="sdk",
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span.id,
+        trace_id=span.trace_id,
+        parent_span_id=None,
+        name="span-name",
+        feedback_scores=EXPECTED_SPAN_FEEDBACK_SCORES,
+        source="sdk",
+    )
+
+
+def test_feedbacks_are_logged_via_client__happyflow(opik_client: opik.Opik):
+    trace = opik_client.trace(name="trace-name-1")
+    span = trace.span(name="span-name-1")
+
+    EXPECTED_TRACE_FEEDBACK_SCORES: List[BatchFeedbackScoreDict] = [
+        {
+            "id": trace.id,
+            "name": "trace-metric-1",
+            "value": 0.5,
+            "category_name": "category-1",
+            "reason": "some-reason-1",
+        },
+        {
+            "id": trace.id,
+            "name": "trace-metric-2",
+            "value": 0.95,
+            "category_name": "category-2",
+            "reason": "some-reason-2",
+        },
+    ]
+
+    EXPECTED_SPAN_FEEDBACK_SCORES: List[BatchFeedbackScoreDict] = [
+        {
+            "id": span.id,
+            "name": "span-metric-1",
+            "value": 0.75,
+            "category_name": "category-3",
+            "reason": "some-reason-3",
+        },
+        {
+            "id": span.id,
+            "name": "span-metric-2",
+            "value": 0.25,
+            "category_name": "category-4",
+            "reason": "some-reason-4",
+        },
+    ]
+
+    opik_client.log_spans_feedback_scores(scores=EXPECTED_SPAN_FEEDBACK_SCORES)
+    opik_client.log_traces_feedback_scores(scores=EXPECTED_TRACE_FEEDBACK_SCORES)
+
+    opik_client.flush()
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace.id,
+        name="trace-name-1",
+        feedback_scores=EXPECTED_TRACE_FEEDBACK_SCORES,
+        project_name=opik_client.project_name,
+        source="sdk",
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span.id,
+        trace_id=span.trace_id,
+        parent_span_id=None,
+        name="span-name-1",
+        feedback_scores=EXPECTED_SPAN_FEEDBACK_SCORES,
+        project_name=opik_client.project_name,
+        source="sdk",
+    )
+
+
+def test_feedback_scores_added_via_update_current_span_and_trace__project_specified_manually__feedback_scores_correctly_attached_to_span_and_trace(
+    opik_client: opik.Opik,
+):
+    ID_STORAGE = {}
+
+    @opik.track
+    def f_inner():
+        ID_STORAGE["f_inner-span-id"] = opik_context.get_current_span_data().id
+        opik_context.update_current_span(
+            feedback_scores=[
+                {
+                    "name": "inner-span-feedback-score",
+                    "value": 0.25,
+                    "category_name": "span-score-category",
+                    "reason": "span-score-reason",
+                }
+            ]
+        )
+
+    @opik.track(project_name=_PROJECT_MANUAL)
+    def f_outer():
+        ID_STORAGE["f_outer-trace-id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["f_outer-span-id"] = opik_context.get_current_span_data().id
+        f_inner()
+        opik_context.update_current_trace(
+            feedback_scores=[
+                {
+                    "name": "trace-feedback-score",
+                    "value": 0.75,
+                    "category_name": "trace-score-category",
+                    "reason": "trace-score-reason",
+                },
+                {
+                    "name": "trace-feedback-score-2",
+                    "value": 0.5,
+                    "category_name": "trace-score-category-2",
+                    "reason": "trace-score-reason-2",
+                },
+            ]
+        )
+
+    f_outer()
+    opik.flush_tracker()
+
+    EXPECTED_INNER_SPAN_FEEDBACK_SCORES: List[FeedbackScoreDict] = [
+        {
+            "id": ID_STORAGE["f_inner-span-id"],
+            "name": "inner-span-feedback-score",
+            "value": 0.25,
+            "category_name": "span-score-category",
+            "reason": "span-score-reason",
+        },
+    ]
+
+    EXPECTED_TRACE_FEEDBACK_SCORES: List[FeedbackScoreDict] = [
+        {
+            "id": ID_STORAGE["f_outer-trace-id"],
+            "name": "trace-feedback-score",
+            "value": 0.75,
+            "category_name": "trace-score-category",
+            "reason": "trace-score-reason",
+        },
+        {
+            "id": ID_STORAGE["f_outer-trace-id"],
+            "name": "trace-feedback-score-2",
+            "value": 0.5,
+            "category_name": "trace-score-category-2",
+            "reason": "trace-score-reason-2",
+        },
+    ]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        name="f_outer",
+        feedback_scores=EXPECTED_TRACE_FEEDBACK_SCORES,
+        source="sdk",
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=ID_STORAGE["f_inner-span-id"],
+        trace_id=ID_STORAGE["f_outer-trace-id"],
+        parent_span_id=ID_STORAGE["f_outer-span-id"],
+        name="f_inner",
+        feedback_scores=EXPECTED_INNER_SPAN_FEEDBACK_SCORES,
+        source="sdk",
+    )
+
+
+@pytest.mark.skip(
+    reason="Backend bug (OPIK-3630): If incorrect project_name is specified, the project gets created and the score is not attached to the correct thread."
+)
+def test_log_threads_feedback_scores__project_name_fallback_logic(
+    opik_client: opik.Opik,
+):
+    # Setup projects
+    project_1 = opik_client.project_name + "-1"
+    project_2 = opik_client.project_name + "-2"
+    project_default = opik_client.project_name
+
+    # Create threads in different projects
+    thread_id_p1 = "thread-p1"
+    thread_id_p2 = "thread-p2"
+    thread_id_default = "thread-default"
+
+    # Create traces with thread_ids to create threads
+    opik_client.trace(name="trace-p1", project_name=project_1, thread_id=thread_id_p1)
+    opik_client.trace(name="trace-p2", project_name=project_2, thread_id=thread_id_p2)
+    opik_client.trace(name="trace-default", thread_id=thread_id_default)
+
+    opik_client.flush()
+
+    # Close threads before logging scores - otherwise backend will return 409 error
+    opik_client.rest_client.traces.close_trace_thread(
+        project_name=project_1, thread_id=thread_id_p1
+    )
+    opik_client.rest_client.traces.close_trace_thread(
+        project_name=project_2, thread_id=thread_id_p2
+    )
+    opik_client.rest_client.traces.close_trace_thread(
+        project_name=project_default, thread_id=thread_id_default
+    )
+
+    # Wait for threads to be closed
+    from opik import synchronization
+
+    def check_threads_closed() -> bool:
+        threads_p1 = opik_client.search_threads(
+            project_name=project_1, filter_string=f'id = "{thread_id_p1}"'
+        )
+        threads_p2 = opik_client.search_threads(
+            project_name=project_2, filter_string=f'id = "{thread_id_p2}"'
+        )
+        threads_default = opik_client.search_threads(
+            project_name=project_default, filter_string=f'id = "{thread_id_default}"'
+        )
+        return (
+            len(threads_p1) > 0
+            and threads_p1[0].status == "closed"
+            and len(threads_p2) > 0
+            and threads_p2[0].status == "closed"
+            and len(threads_default) > 0
+            and threads_default[0].status == "closed"
+        )
+
+    synchronization.wait_for_done(lambda: check_threads_closed(), timeout=30)
+
+    # Define scores with different project_name combinations
+    scores: List[BatchFeedbackScoreDict] = [
+        # 1. Per-score project_name (highest priority) - should go to project_1
+        {
+            "id": thread_id_p1,
+            "name": "metric-p1",
+            "value": 1.0,
+            "project_name": project_1,
+            "reason": "reason-p1",
+        },
+        # 2. Function parameter fallback - should go to project_2
+        {
+            "id": thread_id_p2,
+            "name": "metric-p2",
+            "value": 0.5,
+            "reason": "reason-p2",
+        },
+        # 3. If no project_name is specified, the project gets created and
+        # the score is not attached to the correct thread.
+        {
+            "id": thread_id_default,
+            "name": "metric-default",
+            "value": 0.0,
+            "reason": "reason-default",
+        },
+    ]
+
+    # Log scores with project_2 as parameter fallback
+    opik_client.log_threads_feedback_scores(scores=scores, project_name=project_2)
+
+    opik_client.flush()
+
+    # Verifications
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=thread_id_p1,
+        project_name=project_1,
+        feedback_scores=[
+            {
+                "id": thread_id_p1,
+                "name": "metric-p1",
+                "value": 1.0,
+                "reason": "reason-p1",
+                "category_name": None,
+            },
+        ],
+    )
+
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=thread_id_p2,
+        project_name=project_2,
+        feedback_scores=[
+            {
+                "id": thread_id_p2,
+                "name": "metric-p2",
+                "value": 0.5,
+                "reason": "reason-p2",
+                "category_name": None,
+            },
+        ],
+    )
+
+    verifiers.verify_thread(  # TODO: This fails (OPIK-3630)
+        opik_client=opik_client,
+        thread_id=thread_id_default,
+        project_name=project_default,
+        feedback_scores=[
+            {
+                "id": thread_id_default,
+                "name": "metric-default",
+                "value": 0.0,
+                "reason": "reason-default",
+                "category_name": None,
+            },
+        ],
+    )

sdks/python/tests/e2e/test_guardrails.py

@@ -0,0 +1,156 @@
+import opik
+from opik import opik_context
+from . import verifiers
+from opik.guardrails import Guardrail, PII, Topic
+from opik import exceptions
+from unittest import mock
+import pytest
+
+from ..testlib import generate_project_name
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def find_guardrail_span(opik_client, trace_id, project_name):
+    # We cannot easily access the created span id from the test so we need to query it dynamically
+    spans = opik_client.search_spans(
+        trace_id=trace_id,
+        project_name=project_name,
+        filter_string='name = "Guardrail"',
+    )
+
+    assert len(spans) == 1, "Expected to find 1 span"
+    span = spans[0]
+
+    return span.id
+
+
+@pytest.mark.parametrize("override_project_name", [True, False])
+def test_passing_guardrails__happyflow(opik_client, override_project_name):
+    project_name = (
+        generate_project_name("e2e", "guardrails", "override")
+        if override_project_name
+        else None
+    )
+
+    # Setup
+    ID_STORAGE = {}
+
+    guard = Guardrail(
+        guards=[
+            Topic(restricted_topics=["finance"], threshold=0.8),
+            PII(blocked_entities=["CREDIT_CARD", "PERSON"], threshold=0.4),
+        ]
+    )
+
+    generation = "Hi how are you doing?"
+
+    @opik.track(project_name=project_name)
+    def test_function():
+        ID_STORAGE["trace-id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["parent-span-id"] = opik_context.get_current_span_data().id
+
+        result = guard.validate(generation)
+
+        return result
+
+    _ = test_function()
+
+    opik_client.flush()
+
+    # Check trace guardrails validations
+    EXPECTED_TRACE_GUARDRAILS_VALIDATIONS = [
+        {
+            "span_id": mock.ANY,
+            "checks": [
+                {"name": "PII", "result": "passed"},
+                {"name": "TOPIC", "result": "passed"},
+            ],
+        },
+    ]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=ID_STORAGE["trace-id"],
+        guardrails_validations=EXPECTED_TRACE_GUARDRAILS_VALIDATIONS,
+        project_name=project_name or PROJECT_NAME,
+    )
+
+    span_id = find_guardrail_span(opik_client, ID_STORAGE["trace-id"], project_name)
+
+    EXPECTED_OUTPUT = {
+        "guardrail_result": "passed",
+        "validations": [mock.ANY, mock.ANY],
+        "validation_passed": mock.ANY,
+    }
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span_id,
+        parent_span_id=ID_STORAGE["parent-span-id"],
+        trace_id=ID_STORAGE["trace-id"],
+        name="Guardrail",
+        input={"generation": generation},
+        output=EXPECTED_OUTPUT,
+        project_name=project_name or PROJECT_NAME,
+    )
+
+
+def test_failing_guardrails__happyflow(opik_client: opik.Opik):
+    # Setup
+    ID_STORAGE = {}
+
+    guard = Guardrail(
+        guards=[
+            Topic(restricted_topics=["finance"], threshold=0.8),
+            PII(blocked_entities=["CREDIT_CARD", "PERSON"], threshold=0.4),
+        ]
+    )
+
+    generation = "First name: Samantha Last name: Martinez"
+
+    @opik.track
+    def test_function():
+        ID_STORAGE["trace-id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["parent-span-id"] = opik_context.get_current_span_data().id
+
+        with pytest.raises(exceptions.GuardrailValidationFailed):
+            guard.validate(generation)
+
+    test_function()
+
+    opik_client.flush()
+
+    EXPECTED_TRACE_GUARDRAILS_VALIDATIONS = [
+        {
+            "span_id": mock.ANY,
+            "checks": [
+                {"name": "PII", "result": "failed"},
+                {"name": "TOPIC", "result": "passed"},
+            ],
+        },
+    ]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=ID_STORAGE["trace-id"],
+        guardrails_validations=EXPECTED_TRACE_GUARDRAILS_VALIDATIONS,
+    )
+
+    span_id = find_guardrail_span(opik_client, ID_STORAGE["trace-id"], PROJECT_NAME)
+
+    EXPECTED_OUTPUT = {
+        "guardrail_result": "failed",
+        "validations": [mock.ANY, mock.ANY],
+        "validation_passed": mock.ANY,
+    }
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=span_id,
+        parent_span_id=ID_STORAGE["parent-span-id"],
+        trace_id=ID_STORAGE["trace-id"],
+        name="Guardrail",
+        input={"generation": generation},
+        output=EXPECTED_OUTPUT,
+    )

sdks/python/tests/e2e/test_local_recording.py

@@ -0,0 +1,181 @@
+import pytest
+
+import opik
+from opik import exceptions
+from opik.api_objects import opik_client
+from opik.message_processing.emulation.models import TraceModel, SpanModel
+from opik.message_processing.processors import message_processors_chain
+
+from .. import testlib
+from ..testlib import assert_equal, ANY_BUT_NONE
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _local_recording_default_project():
+    """Override the e2e default-project env patch for this module.
+
+    `record_traces_locally()` captures traces in-process and tests assert
+    against the SDK's hardcoded "Default Project" fallback. The autouse
+    `configure_e2e_tests_env` fixture in conftest.py would otherwise set
+    OPIK_PROJECT_NAME to the per-module backend project."""
+    with testlib.patch_environ({}, remove_keys=["OPIK_PROJECT_NAME"]):
+        yield
+
+
+@opik.track
+def _sample_tracked_function(x: str) -> str:
+    return f"out:{x}"
+
+
+def test_records_spans_and_traces__happy_path():
+    with opik.record_traces_locally() as storage:
+        _sample_tracked_function("a")
+
+        span_trees = storage.span_trees
+        trace_trees = storage.trace_trees
+
+    assert isinstance(span_trees, list)
+    assert isinstance(trace_trees, list)
+    assert len(span_trees) == 1
+    assert len(trace_trees) == 1
+
+    EXPECTED_TRACE_TREE = TraceModel(
+        id=ANY_BUT_NONE,
+        start_time=ANY_BUT_NONE,
+        name="_sample_tracked_function",
+        project_name="Default Project",
+        input={"x": "a"},
+        output={"output": "out:a"},
+        end_time=ANY_BUT_NONE,
+        spans=[
+            SpanModel(
+                id=ANY_BUT_NONE,
+                start_time=ANY_BUT_NONE,
+                name="_sample_tracked_function",
+                input={"x": "a"},
+                output={"output": "out:a"},
+                type="general",
+                end_time=ANY_BUT_NONE,
+                project_name="Default Project",
+                last_updated_at=ANY_BUT_NONE,
+                source="sdk",
+            )
+        ],
+        last_updated_at=ANY_BUT_NONE,
+        source="sdk",
+    )
+
+    assert_equal(expected=EXPECTED_TRACE_TREE, actual=trace_trees[0])
+
+
+def test_prevents_nested_usage():
+    with opik.record_traces_locally():
+        with pytest.raises(exceptions.LocalRecordingAlreadyActive):
+            with opik.record_traces_locally():
+                pass
+
+
+def test_cleanup_and_reuse_after_exit__should_save_new_data():
+    client = opik_client.get_global_client()
+
+    # First run: record and ensure the local processor becomes active
+    with opik.record_traces_locally() as storage:
+        _ = _sample_tracked_function("first run")
+        assert len(storage.span_trees) == 1
+        assert len(storage.trace_trees) == 1
+
+        trace_trees = storage.trace_trees
+
+        local = message_processors_chain.get_local_emulator_message_processor(
+            chain=client.__internal_api__message_processor__
+        )
+        assert local is not None and local.is_active()
+
+    EXPECTED_TRACE_TREE = TraceModel(
+        id=ANY_BUT_NONE,
+        start_time=ANY_BUT_NONE,
+        name="_sample_tracked_function",
+        project_name="Default Project",
+        input={"x": "first run"},
+        output={"output": "out:first run"},
+        end_time=ANY_BUT_NONE,
+        spans=[
+            SpanModel(
+                id=ANY_BUT_NONE,
+                start_time=ANY_BUT_NONE,
+                name="_sample_tracked_function",
+                input={"x": "first run"},
+                output={"output": "out:first run"},
+                type="general",
+                end_time=ANY_BUT_NONE,
+                project_name="Default Project",
+                last_updated_at=ANY_BUT_NONE,
+                source="sdk",
+            )
+        ],
+        last_updated_at=ANY_BUT_NONE,
+        source="sdk",
+    )
+
+    assert_equal(expected=EXPECTED_TRACE_TREE, actual=trace_trees[0])
+
+    # After context exit: local processor should be inactive and reset on the next activation
+    local = message_processors_chain.get_local_emulator_message_processor(
+        chain=client.__internal_api__message_processor__
+    )
+    assert local is not None and not local.is_active()
+
+    # The second run should work independently
+    with opik.record_traces_locally() as storage:
+        _ = _sample_tracked_function("second run")
+
+        assert len(storage.span_trees) == 1
+        assert len(storage.trace_trees) == 1
+
+        trace_trees = storage.trace_trees
+
+    EXPECTED_TRACE_TREE = TraceModel(
+        id=ANY_BUT_NONE,
+        start_time=ANY_BUT_NONE,
+        name="_sample_tracked_function",
+        project_name="Default Project",
+        input={"x": "second run"},
+        output={"output": "out:second run"},
+        end_time=ANY_BUT_NONE,
+        spans=[
+            SpanModel(
+                id=ANY_BUT_NONE,
+                start_time=ANY_BUT_NONE,
+                name="_sample_tracked_function",
+                input={"x": "second run"},
+                output={"output": "out:second run"},
+                type="general",
+                end_time=ANY_BUT_NONE,
+                project_name="Default Project",
+                last_updated_at=ANY_BUT_NONE,
+                source="sdk",
+            )
+        ],
+        last_updated_at=ANY_BUT_NONE,
+        source="sdk",
+    )
+
+    assert_equal(expected=EXPECTED_TRACE_TREE, actual=trace_trees[0])
+
+
+def test_handle_flushes_before_access(monkeypatch):
+    flushed = {"called": False}
+
+    original_flush = opik.Opik.flush
+
+    def _flush(self, timeout=None):  # type: ignore[no-redef]
+        flushed["called"] = True
+        return original_flush(self, timeout)
+
+    monkeypatch.setattr(opik.Opik, "flush", _flush, raising=True)
+
+    with opik.record_traces_locally() as storage:
+        _ = _sample_tracked_function("c")
+
+        _ = storage.span_trees
+        assert flushed["called"] is True

sdks/python/tests/e2e/test_optimization.py

@@ -0,0 +1,52 @@
+import opik
+import pytest
+from opik.rest_api import core as rest_api_core
+from . import verifiers
+
+
+def test_optimization_lifecycle__happyflow(opik_client: opik.Opik, dataset_name: str):
+    dataset = opik_client.create_dataset(dataset_name)
+
+    project_name = f"test_optimization_{dataset_name}"
+    optimization_name = f"some-optimization-name-{dataset_name}"
+
+    # Create optimization
+    optimization = opik_client.create_optimization(
+        objective_name="some-objective-name",
+        dataset_name=dataset.name,
+        name=optimization_name,
+        project_name=project_name,
+    )
+
+    assert optimization.project_name == project_name
+
+    verifiers.verify_optimization(
+        opik_client=opik_client,
+        optimization_id=optimization.id,
+        name=optimization_name,
+        dataset_name=dataset.name,
+        status="running",
+        objective_name="some-objective-name",
+        project_name=project_name,
+    )
+
+    # Update optimization name and status
+    optimization.update(name="new-optimization-name", status="completed")
+    verifiers.verify_optimization(
+        opik_client=opik_client,
+        optimization_id=optimization.id,
+        name="new-optimization-name",
+        dataset_name=dataset.name,
+        status="completed",
+        objective_name="some-objective-name",
+        project_name=project_name,
+    )
+
+    # Check project_name propagation
+    optimization = opik_client.get_optimization_by_id(optimization.id)
+    assert optimization.project_name == project_name
+
+    opik_client.delete_optimizations([optimization.id])
+
+    with pytest.raises(rest_api_core.ApiError):
+        opik_client.get_optimization_by_id(optimization.id)

sdks/python/tests/e2e/test_threads.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from typing import Tuple
+import uuid
+
+import pytest
+
+import opik
+from opik import synchronization
+from opik.types import BatchFeedbackScoreDict
+from . import verifiers
+
+
+@pytest.fixture
+def active_threads_and_project(
+    opik_client: opik.Opik, temporary_project_name: str
+) -> Tuple[list[str], str]:
+    thread_ids = [str(uuid.uuid4())[-6:], str(uuid.uuid4())[-6:]]
+
+    for thread_id in thread_ids:
+        for i in range(5):
+            opik_client.trace(
+                name=f"trace-name-{i}:{thread_id}",
+                input={"input": f"trace-input-{i}:{thread_id}"},
+                output={"output": f"trace-output-{i}:{thread_id}"},
+                project_name=temporary_project_name,
+                thread_id=thread_id,
+            )
+
+    opik_client.flush()
+
+    return thread_ids, temporary_project_name
+
+
+def test_threads_client__search_threads__happy_path(
+    opik_client: opik.Opik, active_threads_and_project: Tuple[list[str], str]
+):
+    active_threads, temporary_project_name = active_threads_and_project
+
+    # Search for the first thread by ID filter
+    filter_string = f'id = "{active_threads[0]}"'
+    threads = opik_client.search_threads(
+        project_name=temporary_project_name, filter_string=filter_string
+    )
+    assert len(threads) == 1
+    assert threads[0].id == active_threads[0]
+
+    # Search for all threads with active status
+    filter_string = 'status = "active"'
+    threads = opik_client.search_threads(
+        project_name=temporary_project_name, filter_string=filter_string
+    )
+    assert len(threads) == 2
+    for thread in threads:
+        assert thread.id in active_threads
+
+
+def test_threads_client__log_threads_feedback_scores__happy_path(
+    opik_client: opik.Opik, active_threads_and_project: Tuple[list[str], str]
+):
+    active_threads, temporary_project_name = active_threads_and_project
+
+    # log feedback scores
+    scores = [
+        BatchFeedbackScoreDict(
+            id=active_threads[0],
+            name="thread-metric-1",
+            value=0.75,
+            category_name="category-3",
+            reason="some-reason-3",
+        ),
+        BatchFeedbackScoreDict(
+            id=active_threads[1],
+            name="thread-metric-2",
+            value=0.25,
+            category_name="category-4",
+            reason="some-reason-4",
+        ),
+    ]
+    opik_client.log_threads_feedback_scores(
+        scores=scores, project_name=temporary_project_name
+    )
+
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=active_threads[0],
+        project_name=temporary_project_name,
+        feedback_scores=[scores[0]],
+    )
+
+    verifiers.verify_thread(
+        opik_client=opik_client,
+        thread_id=active_threads[1],
+        project_name=temporary_project_name,
+        feedback_scores=[scores[1]],
+    )
+
+
+def test_threads_client__close_thread__happy_path(
+    opik_client: opik.Opik, temporary_project_name
+):
+    threads_client = opik_client.get_threads_client()
+
+    thread_id = str(uuid.uuid4())[-6:]
+    opik_client.trace(
+        name="some-trace-name",
+        thread_id=thread_id,
+        project_name=temporary_project_name,
+    )
+
+    opik_client.flush()
+
+    threads = opik_client.search_threads(
+        project_name=temporary_project_name,
+        filter_string='status = "active"',
+    )
+    assert len(threads) == 1
+    assert threads[0].id == thread_id
+
+    threads_client.close_thread(
+        thread_id=thread_id,
+        project_name=temporary_project_name,
+    )
+
+    threads = opik_client.search_threads(
+        project_name=temporary_project_name,
+        filter_string='status = "active"',
+    )
+    assert len(threads) == 0
+
+
+def test_threads_client__search_threads__filter_by_feedback_score(
+    opik_client: opik.Opik, temporary_project_name
+):
+    # Create a unique metric name to avoid conflicts with other tests
+    unique_metric = f"test_metric_{str(uuid.uuid4()).replace('-', '_')[-8:]}"
+
+    # Create thread with feedback score
+    thread_with_score = str(uuid.uuid4())[-6:]
+    opik_client.trace(
+        name="trace-with-score",
+        thread_id=thread_with_score,
+        project_name=temporary_project_name,
+    )
+
+    # Create thread without feedback score
+    thread_without_score = str(uuid.uuid4())[-6:]
+    opik_client.trace(
+        name="trace-without-score",
+        thread_id=thread_without_score,
+        project_name=temporary_project_name,
+    )
+
+    opik_client.flush()
+
+    # Log feedback score to one thread
+    opik_client.log_threads_feedback_scores(
+        scores=[
+            BatchFeedbackScoreDict(
+                id=thread_with_score,
+                name=unique_metric,
+                value=0.85,
+                category_name="test-category",
+                reason="test-reason",
+            )
+        ],
+        project_name=temporary_project_name,
+    )
+
+    # Wait for feedback scores to propagate
+    def check_feedback_score_logged() -> bool:
+        threads = opik_client.search_threads(
+            project_name=temporary_project_name,
+            filter_string=f"feedback_scores.{unique_metric} is_not_empty",
+        )
+        return len(threads) == 1
+
+    synchronization.wait_for_done(lambda: check_feedback_score_logged(), timeout=30)
+
+    # Test filtering with is_not_empty
+    threads_not_empty = opik_client.search_threads(
+        project_name=temporary_project_name,
+        filter_string=f"feedback_scores.{unique_metric} is_not_empty",
+    )
+    thread_ids_not_empty = {thread.id for thread in threads_not_empty}
+    assert thread_with_score in thread_ids_not_empty, (
+        "Thread with score should be found with is_not_empty filter"
+    )
+    assert thread_without_score not in thread_ids_not_empty, (
+        "Thread without score should not be found with is_not_empty filter"
+    )
+
+    # Test filtering with = operator
+    threads_with_value = opik_client.search_threads(
+        project_name=temporary_project_name,
+        filter_string=f"feedback_scores.{unique_metric} = 0.85",
+    )
+    thread_ids_with_value = {thread.id for thread in threads_with_value}
+    assert thread_with_score in thread_ids_with_value, (
+        "Thread with score value 0.85 should be found"
+    )
+    assert thread_without_score not in thread_ids_with_value, (
+        "Thread without score should not be found"
+    )
+
+    # Verify is_not_empty and = return the same thread
+    assert thread_ids_not_empty == thread_ids_with_value, (
+        "is_not_empty and = filters should return the same threads for this test case"
+    )

sdks/python/tests/e2e_library_integration/adk/requirements.txt

@@ -0,0 +1,3 @@
+google-adk
+deprecated
+python-multipart; python_version < "3.10"

sdks/python/tests/e2e_library_integration/adk/sample_agent/__init__.py

@@ -0,0 +1 @@
+from . import agent  # noqa: F401

sdks/python/tests/e2e_library_integration/adk/sample_agent/agent.py

@@ -0,0 +1,105 @@
+import datetime
+import os
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from google.adk.agents import Agent
+from google.adk.agents.callback_context import CallbackContext
+
+from opik.integrations.adk import OpikTracer
+
+# ADK loads this file as a top-level module (importlib.import_module("sample_agent")),
+# so relative imports above this package are unavailable — keep the model id inline.
+MODEL = "gemini-2.5-flash"
+
+
+def get_weather(city: str) -> dict:
+    """Retrieves the current weather report for a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the weather report.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+    if city.lower() == "new york":
+        return {
+            "status": "success",
+            "report": (
+                "The weather in New York is sunny with a temperature of 25 degrees"
+                " Celsius (77 degrees Fahrenheit)."
+            ),
+        }
+    else:
+        return {
+            "status": "error",
+            "error_message": f"Weather information for '{city}' is not available.",
+        }
+
+
+def get_current_time(city: str) -> dict:
+    """Returns the current time in a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the current time.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+
+    if city.lower() == "new york":
+        tz_identifier = "America/New_York"
+    else:
+        return {
+            "status": "error",
+            "error_message": (f"Sorry, I don't have timezone information for {city}."),
+        }
+
+    tz = ZoneInfo(tz_identifier)
+    now = datetime.datetime.now(tz)
+    report = f"The current time in {city} is {now.strftime('%Y-%m-%d %H:%M:%S %Z%z')}"
+    return {"status": "success", "report": report}
+
+
+opik_tracer = OpikTracer()
+print(
+    "Project name:",
+    os.environ.get("OPIK_PROJECT_NAME", "no project name set in env vars!"),
+)
+
+
+def after_agent_callback(
+    callback_context: CallbackContext, *args: Any, **kwargs: Any
+) -> None:
+    opik_tracer.after_agent_callback(callback_context, *args, **kwargs)
+    opik_tracer.flush()
+
+
+root_agent = Agent(
+    name="weather_time_agent",
+    model=MODEL,
+    description="Agent to answer questions about the time and weather in a city.",
+    # Kept inline because ADK's AgentLoader imports each sample_agent as
+    # a top-level module, so relative imports from the rest of the test
+    # tree don't resolve at /run time. Keep in sync with
+    # tests/library_integration/adk/agent_instructions.TOOL_USE_WEATHER_OR_TIME.
+    instruction=(
+        "You MUST invoke one of the provided function tools before replying — "
+        "never fabricate a response, never describe a fake tool call in plain "
+        "text, never paste invented JSON. If the user asks about the weather "
+        "in a city, your next action MUST be a function call to "
+        "`get_weather(city=...)`. If the user asks about the current time in "
+        "a city, your next action MUST be a function call to "
+        "`get_current_time(city=...)`. After the tool returns, write a short "
+        "natural-language reply to the user that reports what the tool said. "
+        "Always produce this reply even if the tool's output is already "
+        "self-contained."
+    ),
+    tools=[get_weather, get_current_time],
+    before_agent_callback=opik_tracer.before_agent_callback,
+    after_agent_callback=after_agent_callback,
+    before_model_callback=opik_tracer.before_model_callback,
+    after_model_callback=opik_tracer.after_model_callback,
+    before_tool_callback=opik_tracer.before_tool_callback,
+    after_tool_callback=opik_tracer.after_tool_callback,
+)

sdks/python/tests/e2e_library_integration/adk/sample_agent_anthropic/__init__.py

@@ -0,0 +1 @@
+from . import agent  # noqa: F401

sdks/python/tests/e2e_library_integration/adk/sample_agent_anthropic/agent.py

@@ -0,0 +1,106 @@
+import datetime
+import os
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from google.adk.agents import LlmAgent
+from google.adk.agents.callback_context import CallbackContext
+from google.adk.models.lite_llm import LiteLlm
+
+from opik.integrations.adk import OpikTracer
+
+# ADK loads this file as a top-level module, so relative imports above this
+# package are unavailable — keep the model id inline.
+MODEL = "anthropic/claude-sonnet-4-6"
+
+
+def get_weather(city: str) -> dict:
+    """Retrieves the current weather report for a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the weather report.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+    if city.lower() == "new york":
+        return {
+            "status": "success",
+            "report": (
+                "The weather in New York is sunny with a temperature of 25 degrees"
+                " Celsius (77 degrees Fahrenheit)."
+            ),
+        }
+    else:
+        return {
+            "status": "error",
+            "error_message": f"Weather information for '{city}' is not available.",
+        }
+
+
+def get_current_time(city: str) -> dict:
+    """Returns the current time in a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the current time.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+
+    if city.lower() == "new york":
+        tz_identifier = "America/New_York"
+    else:
+        return {
+            "status": "error",
+            "error_message": (f"Sorry, I don't have timezone information for {city}."),
+        }
+
+    tz = ZoneInfo(tz_identifier)
+    now = datetime.datetime.now(tz)
+    report = f"The current time in {city} is {now.strftime('%Y-%m-%d %H:%M:%S %Z%z')}"
+    return {"status": "success", "report": report}
+
+
+opik_tracer = OpikTracer()
+print(
+    "Project name:",
+    os.environ.get("OPIK_PROJECT_NAME", "no project name set in env vars!"),
+)
+
+
+def after_agent_callback(
+    callback_context: CallbackContext, *args: Any, **kwargs: Any
+) -> None:
+    opik_tracer.after_agent_callback(callback_context, *args, **kwargs)
+    opik_tracer.flush()
+
+
+root_agent = LlmAgent(
+    name="weather_time_agent",
+    model=LiteLlm(model=MODEL),
+    description="Agent to answer questions about the time and weather in a city.",
+    # Kept inline because ADK's AgentLoader imports each sample_agent as
+    # a top-level module, so relative imports from the rest of the test
+    # tree don't resolve at /run time. Keep in sync with
+    # tests/library_integration/adk/agent_instructions.TOOL_USE_WEATHER_OR_TIME.
+    instruction=(
+        "You MUST invoke one of the provided function tools before replying — "
+        "never fabricate a response, never describe a fake tool call in plain "
+        "text, never paste invented JSON. If the user asks about the weather "
+        "in a city, your next action MUST be a function call to "
+        "`get_weather(city=...)`. If the user asks about the current time in "
+        "a city, your next action MUST be a function call to "
+        "`get_current_time(city=...)`. After the tool returns, write a short "
+        "natural-language reply to the user that reports what the tool said. "
+        "Always produce this reply even if the tool's output is already "
+        "self-contained."
+    ),
+    tools=[get_weather, get_current_time],
+    before_agent_callback=opik_tracer.before_agent_callback,
+    after_agent_callback=after_agent_callback,
+    before_model_callback=opik_tracer.before_model_callback,
+    after_model_callback=opik_tracer.after_model_callback,
+    before_tool_callback=opik_tracer.before_tool_callback,
+    after_tool_callback=opik_tracer.after_tool_callback,
+)

sdks/python/tests/e2e_library_integration/adk/sample_agent_openai/__init__.py

@@ -0,0 +1 @@
+from . import agent  # noqa: F401

sdks/python/tests/e2e_library_integration/adk/sample_agent_openai/agent.py

@@ -0,0 +1,106 @@
+import datetime
+import os
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from google.adk.agents import LlmAgent
+from google.adk.agents.callback_context import CallbackContext
+from google.adk.models.lite_llm import LiteLlm
+
+from opik.integrations.adk import OpikTracer
+
+# ADK loads this file as a top-level module, so relative imports above this
+# package are unavailable — keep the model id inline.
+MODEL = "openai/gpt-5-nano"
+
+
+def get_weather(city: str) -> dict:
+    """Retrieves the current weather report for a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the weather report.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+    if city.lower() == "new york":
+        return {
+            "status": "success",
+            "report": (
+                "The weather in New York is sunny with a temperature of 25 degrees"
+                " Celsius (77 degrees Fahrenheit)."
+            ),
+        }
+    else:
+        return {
+            "status": "error",
+            "error_message": f"Weather information for '{city}' is not available.",
+        }
+
+
+def get_current_time(city: str) -> dict:
+    """Returns the current time in a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the current time.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+
+    if city.lower() == "new york":
+        tz_identifier = "America/New_York"
+    else:
+        return {
+            "status": "error",
+            "error_message": (f"Sorry, I don't have timezone information for {city}."),
+        }
+
+    tz = ZoneInfo(tz_identifier)
+    now = datetime.datetime.now(tz)
+    report = f"The current time in {city} is {now.strftime('%Y-%m-%d %H:%M:%S %Z%z')}"
+    return {"status": "success", "report": report}
+
+
+opik_tracer = OpikTracer()
+print(
+    "Project name:",
+    os.environ.get("OPIK_PROJECT_NAME", "no project name set in env vars!"),
+)
+
+
+def after_agent_callback(
+    callback_context: CallbackContext, *args: Any, **kwargs: Any
+) -> None:
+    opik_tracer.after_agent_callback(callback_context, *args, **kwargs)
+    opik_tracer.flush()
+
+
+root_agent = LlmAgent(
+    name="weather_time_agent",
+    model=LiteLlm(model=MODEL, reasoning_effort="minimal"),
+    description="Agent to answer questions about the time and weather in a city.",
+    # Kept inline because ADK's AgentLoader imports each sample_agent as
+    # a top-level module, so relative imports from the rest of the test
+    # tree don't resolve at /run time. Keep in sync with
+    # tests/library_integration/adk/agent_instructions.TOOL_USE_WEATHER_OR_TIME.
+    instruction=(
+        "You MUST invoke one of the provided function tools before replying — "
+        "never fabricate a response, never describe a fake tool call in plain "
+        "text, never paste invented JSON. If the user asks about the weather "
+        "in a city, your next action MUST be a function call to "
+        "`get_weather(city=...)`. If the user asks about the current time in "
+        "a city, your next action MUST be a function call to "
+        "`get_current_time(city=...)`. After the tool returns, write a short "
+        "natural-language reply to the user that reports what the tool said. "
+        "Always produce this reply even if the tool's output is already "
+        "self-contained."
+    ),
+    tools=[get_weather, get_current_time],
+    before_agent_callback=opik_tracer.before_agent_callback,
+    after_agent_callback=after_agent_callback,
+    before_model_callback=opik_tracer.before_model_callback,
+    after_model_callback=opik_tracer.after_model_callback,
+    before_tool_callback=opik_tracer.before_tool_callback,
+    after_tool_callback=opik_tracer.after_tool_callback,
+)

sdks/python/tests/e2e_library_integration/adk/sample_agent_sse/agent.py

@@ -0,0 +1,106 @@
+import datetime
+import os
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from google.adk.agents import Agent
+from google.adk.agents.callback_context import CallbackContext
+
+from opik.integrations.adk import OpikTracer
+
+# ADK loads this file as a top-level module via importlib.import_module
+# ("sample_agent_sse.agent"), so relative imports above this package are
+# unavailable — keep the model id inline.
+MODEL = "gemini-2.5-flash"
+
+
+def get_weather(city: str) -> dict:
+    """Retrieves the current weather report for a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the weather report.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+    if city.lower() == "new york":
+        return {
+            "status": "success",
+            "report": (
+                "The weather in New York is sunny with a temperature of 25 degrees"
+                " Celsius (77 degrees Fahrenheit)."
+            ),
+        }
+    else:
+        return {
+            "status": "error",
+            "error_message": f"Weather information for '{city}' is not available.",
+        }
+
+
+def get_current_time(city: str) -> dict:
+    """Returns the current time in a specified city.
+
+    Args:
+        city (str): The name of the city for which to retrieve the current time.
+
+    Returns:
+        dict: status and result or error msg.
+    """
+
+    if city.lower() == "new york":
+        tz_identifier = "America/New_York"
+    else:
+        return {
+            "status": "error",
+            "error_message": (f"Sorry, I don't have timezone information for {city}."),
+        }
+
+    tz = ZoneInfo(tz_identifier)
+    now = datetime.datetime.now(tz)
+    report = f"The current time in {city} is {now.strftime('%Y-%m-%d %H:%M:%S %Z%z')}"
+    return {"status": "success", "report": report}
+
+
+opik_tracer = OpikTracer()
+print(
+    "Project name:",
+    os.environ.get("OPIK_PROJECT_NAME", "no project name set in env vars!"),
+)
+
+
+def after_agent_callback(
+    callback_context: CallbackContext, *args: Any, **kwargs: Any
+) -> None:
+    opik_tracer.after_agent_callback(callback_context, *args, **kwargs)
+    opik_tracer.flush()
+
+
+root_agent = Agent(
+    name="weather_time_agent",
+    model=MODEL,
+    description="Agent to answer questions about the time and weather in a city.",
+    # Kept inline because ADK's AgentLoader imports each sample_agent as
+    # a top-level module, so relative imports from the rest of the test
+    # tree don't resolve at /run time. Keep in sync with
+    # tests/library_integration/adk/agent_instructions.TOOL_USE_WEATHER_OR_TIME.
+    instruction=(
+        "You MUST invoke one of the provided function tools before replying — "
+        "never fabricate a response, never describe a fake tool call in plain "
+        "text, never paste invented JSON. If the user asks about the weather "
+        "in a city, your next action MUST be a function call to "
+        "`get_weather(city=...)`. If the user asks about the current time in "
+        "a city, your next action MUST be a function call to "
+        "`get_current_time(city=...)`. After the tool returns, write a short "
+        "natural-language reply to the user that reports what the tool said. "
+        "Always produce this reply even if the tool's output is already "
+        "self-contained."
+    ),
+    tools=[get_weather, get_current_time],
+    before_agent_callback=opik_tracer.before_agent_callback,
+    after_agent_callback=after_agent_callback,
+    before_model_callback=opik_tracer.before_model_callback,
+    after_model_callback=opik_tracer.after_model_callback,
+    before_tool_callback=opik_tracer.before_tool_callback,
+    after_tool_callback=opik_tracer.after_tool_callback,
+)

sdks/python/tests/e2e_library_integration/adk/test_opik_tracer.py

@@ -0,0 +1,306 @@
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+import certifi
+import pytest
+import requests
+
+from opik import synchronization
+from opik.integrations.adk import helpers as adk_helpers
+from opik.llm_usage.openai_chat_completions_usage import OpenAICompletionsUsage
+from ... import llm_constants, testlib
+
+# needed for OpenAI agents tests
+os.environ["SSL_CERT_FILE"] = certifi.where()
+
+ADK_SERVER_PORT = 21345
+ADK_USER = "user_113"
+ADK_SESSION = "session_113"
+
+EXPECTED_USAGE_KEYS_GOOGLE = [
+    "completion_tokens",
+    "prompt_tokens",
+    "total_tokens",
+    "original_usage.candidates_token_count",
+    "original_usage.prompt_token_count",
+    "original_usage.total_token_count",
+]
+
+EXPECTED_USAGE_KEYS_GOOGLE_REASONING = [
+    "completion_tokens",
+    "prompt_tokens",
+    "total_tokens",
+    "original_usage.candidates_token_count",
+    "original_usage.prompt_token_count",
+    "original_usage.total_token_count",
+    "original_usage.thoughts_token_count",
+]
+
+
+def _create_user_session(
+    base_url: str,
+    user_id: str,
+    session_id: str,
+    agent_name: str,
+) -> bool:
+    try:
+        url = f"{base_url}/apps/{agent_name}/users/{user_id}/sessions/{session_id}"
+        response = requests.post(url)
+        # 409 means the session already exists - is OK with us
+        if response.status_code == 200 or response.status_code == 409:
+            print(response.json())
+            return True
+    except requests.exceptions.ConnectionError:
+        return False
+    return False
+
+
+@pytest.fixture()
+def start_api_server(request):
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    os.environ["OPIK_FILE_LOGGING_LEVEL"] = "DEBUG"
+
+    agent_name = getattr(request, "param", None)
+    if agent_name is None:
+        agent_name = "sample_agent"  # default
+
+    # Find the adk command in the current environment
+    adk_path = shutil.which("adk")
+    if adk_path is None:
+        # Fallback: construct path from sys.executable
+        venv_bin = os.path.dirname(sys.executable)
+        adk_path = os.path.join(venv_bin, "adk")
+
+    if not os.path.exists(adk_path):
+        raise RuntimeError(f"ADK command not found. Tried: {adk_path}")
+
+    with subprocess.Popen(
+        [adk_path, "api_server", "--port", str(ADK_SERVER_PORT)],
+        cwd=cwd,
+    ) as proc:
+        base_url = f"http://localhost:{ADK_SERVER_PORT}"
+        time.sleep(1)
+
+        # wait until the server is ready and session created
+        if not synchronization.until(
+            lambda: _create_user_session(
+                base_url=base_url,
+                user_id=ADK_USER,
+                session_id=ADK_SESSION,
+                agent_name=agent_name,
+            ),
+            allow_errors=True,
+            sleep=5,
+            max_try_seconds=60,
+        ):
+            proc.kill()
+            raise Exception("Server did not start in time")
+
+        yield base_url
+
+        proc.terminate()
+        proc.wait()
+
+
+@pytest.mark.parametrize("start_api_server", ["sample_agent"], indirect=True)
+def test_opik_tracer_with_sample_agent(
+    opik_client_unique_project_name, start_api_server
+) -> None:
+    base_url = start_api_server
+
+    # send the request to the ADK API server
+    json_data = {
+        "app_name": "sample_agent",
+        "user_id": ADK_USER,
+        "session_id": ADK_SESSION,
+        "new_message": {
+            "role": "user",
+            "parts": [{"text": "Hey, whats the weather in New York today?"}],
+        },
+    }
+    result = requests.post(
+        f"{base_url}/run",
+        json=json_data,
+    )
+    assert result.status_code == 200, (
+        f"ADK /run returned {result.status_code}. Response: {result.text!r}"
+    )
+
+    traces = opik_client_unique_project_name.search_traces(
+        filter_string='input contains "Hey, whats the weather in New York today?"',
+        wait_for_at_least=1,
+        wait_for_timeout=30,
+    )
+    assert len(traces) == 1
+
+    trace = traces[0]
+    assert trace.span_count == 3  # two LLM calls and one function call
+    assert trace.usage is not None
+    assert "adk_invocation_id" in trace.metadata.keys()
+    assert trace.metadata["created_from"] == "google-adk"
+    testlib.assert_dict_has_keys(trace.usage, EXPECTED_USAGE_KEYS_GOOGLE)
+
+    spans = opik_client_unique_project_name.search_spans(wait_for_at_least=3)
+    assert len(spans) == 3
+    assert spans[0].provider == adk_helpers.get_adk_provider()
+    assert spans[2].provider == adk_helpers.get_adk_provider()
+    testlib.assert_dict_has_keys(spans[0].usage, EXPECTED_USAGE_KEYS_GOOGLE)
+    testlib.assert_dict_has_keys(spans[2].usage, EXPECTED_USAGE_KEYS_GOOGLE)
+
+
+@pytest.mark.parametrize("start_api_server", ["sample_agent_sse"], indirect=True)
+def test_opik_tracer_with_sample_agent_sse(
+    opik_client_unique_project_name, start_api_server
+) -> None:
+    """Run the test against the SSE endpoint with streaming enabled using the gemini-2.5-flash model."""
+    base_url = start_api_server
+
+    # send the request to the ADK API server
+    json_data = {
+        "app_name": "sample_agent_sse",
+        "user_id": ADK_USER,
+        "session_id": ADK_SESSION,
+        "new_message": {
+            "role": "user",
+            "parts": [{"text": "Hey, whats the weather in New York today?"}],
+        },
+        "streaming": True,
+    }
+
+    result = requests.post(
+        f"{base_url}/run_sse",
+        json=json_data,
+    )
+    # print("Response: ", result.text)
+    assert result.status_code == 200
+
+    traces = opik_client_unique_project_name.search_traces(
+        filter_string='input contains "Hey, whats the weather in New York today?"',
+        wait_for_at_least=1,
+        wait_for_timeout=30,
+    )
+    assert len(traces) == 1
+
+    trace = traces[0]
+    assert trace.span_count == 3  # two LLM calls and one function call
+    assert trace.usage is not None
+    assert "adk_invocation_id" in trace.metadata.keys()
+    assert trace.metadata["created_from"] == "google-adk"
+    testlib.assert_dict_keys_in_list(trace.usage, EXPECTED_USAGE_KEYS_GOOGLE_REASONING)
+
+    spans = opik_client_unique_project_name.search_spans()
+    assert len(spans) == 3
+    assert spans[0].provider == adk_helpers.get_adk_provider()
+    assert spans[2].provider == adk_helpers.get_adk_provider()
+    testlib.assert_dict_keys_in_list(
+        spans[0].usage, EXPECTED_USAGE_KEYS_GOOGLE_REASONING
+    )
+    testlib.assert_dict_keys_in_list(
+        spans[2].usage, EXPECTED_USAGE_KEYS_GOOGLE_REASONING
+    )
+
+
+@pytest.mark.skip(
+    reason="Skipping due to flakiness with OpenAI API calls. Re-enable once https://github.com/google/adk-python/pull/4303 is merged."
+)
+@pytest.mark.parametrize("start_api_server", ["sample_agent_openai"], indirect=True)
+def test_opik_tracer_with_sample_agent__openai(
+    opik_client_unique_project_name, start_api_server
+) -> None:
+    base_url = start_api_server
+
+    # send the request to the ADK API server
+    json_data = {
+        "app_name": "sample_agent_openai",
+        "user_id": ADK_USER,
+        "session_id": ADK_SESSION,
+        "new_message": {
+            "role": "user",
+            "parts": [{"text": "Hey, whats the weather in New York today?"}],
+        },
+    }
+    result = requests.post(
+        f"{base_url}/run",
+        json=json_data,
+    )
+    print("Response: ", result.text)
+    assert result.status_code == 200
+
+    traces = opik_client_unique_project_name.search_traces(
+        filter_string='input contains "Hey, whats the weather in New York today?"',
+        wait_for_at_least=1,
+        wait_for_timeout=30,
+    )
+    assert len(traces) == 1
+
+    trace = traces[0]
+    assert trace.span_count >= 3  # two LLM calls and one function call + duplicates
+    assert trace.usage is not None
+    assert "adk_invocation_id" in trace.metadata.keys()
+    assert trace.metadata["created_from"] == "google-adk"
+    OpenAICompletionsUsage.from_original_usage_dict(trace.usage)
+
+    spans = opik_client_unique_project_name.search_spans()
+
+    assert len(spans) >= 3  # sometimes it duplicates calls to the function
+    for span in spans:
+        if span.type == "llm":
+            assert span.provider == "openai"
+            assert span.model.startswith(llm_constants.OPENAI_GPT_NANO)
+            OpenAICompletionsUsage.from_original_usage_dict(span.usage)
+        elif span.type == "tool":
+            assert span.name == "get_weather"
+
+
+@pytest.mark.parametrize("start_api_server", ["sample_agent_anthropic"], indirect=True)
+def test_opik_tracer_with_sample_agent__anthropic(
+    opik_client_unique_project_name, start_api_server
+) -> None:
+    base_url = start_api_server
+
+    # send the request to the ADK API server
+    json_data = {
+        "app_name": "sample_agent_anthropic",
+        "user_id": ADK_USER,
+        "session_id": ADK_SESSION,
+        "new_message": {
+            "role": "user",
+            "parts": [{"text": "Hey, whats the weather in New York today?"}],
+        },
+    }
+    result = requests.post(
+        f"{base_url}/run",
+        json=json_data,
+    )
+    print("Response: ", result.text)
+    assert result.status_code == 200
+
+    traces = opik_client_unique_project_name.search_traces(
+        filter_string='input contains "Hey, whats the weather in New York today?"',
+        wait_for_at_least=1,
+        wait_for_timeout=30,
+    )
+    assert len(traces) == 1
+
+    trace = traces[0]
+    assert trace.span_count == 3  # two LLM calls and one function call
+    assert trace.usage is not None
+    assert "adk_invocation_id" in trace.metadata.keys()
+    assert trace.metadata["created_from"] == "google-adk"
+    OpenAICompletionsUsage.from_original_usage_dict(trace.usage)
+
+    spans = opik_client_unique_project_name.search_spans()
+
+    assert len(spans) == 3
+    assert spans[0].type == "llm"
+    assert spans[0].provider == "anthropic"
+    assert spans[0].model.startswith("claude-sonnet-4")
+    OpenAICompletionsUsage.from_original_usage_dict(spans[0].usage)
+
+    assert spans[2].type == "llm"
+    assert spans[2].provider == "anthropic"
+    assert spans[2].model.startswith("claude-sonnet-4")
+    OpenAICompletionsUsage.from_original_usage_dict(spans[2].usage)

sdks/python/tests/e2e_library_integration/conftest.py

@@ -0,0 +1,50 @@
+from typing import Final
+
+import pytest
+
+import opik
+import opik.api_objects.opik_client
+from .. import testlib
+from ..conftest import random_chars
+
+
+OPIK_E2E_LIB_INTEGRATION_TESTS_PROJECT_NAME: Final[str] = (
+    "e2e-libraries-integration-tests"
+)
+
+
+@pytest.fixture()
+def configure_e2e_tests_env():
+    with testlib.patch_environ(
+        {"OPIK_PROJECT_NAME": OPIK_E2E_LIB_INTEGRATION_TESTS_PROJECT_NAME}
+    ):
+        yield
+
+
+@pytest.fixture()
+def opik_client(
+    configure_e2e_tests_env_unique_project_name, shutdown_cached_client_after_test
+):
+    opik_client_ = opik.api_objects.opik_client.Opik(batching=True)
+
+    yield opik_client_
+
+    opik_client_.end()
+
+
+@pytest.fixture()
+def configure_e2e_tests_env_unique_project_name():
+    project_name = f"{OPIK_E2E_LIB_INTEGRATION_TESTS_PROJECT_NAME}-{random_chars()}"
+    with testlib.patch_environ({"OPIK_PROJECT_NAME": project_name}):
+        yield project_name
+
+
+@pytest.fixture()
+def opik_client_unique_project_name(
+    configure_e2e_tests_env_unique_project_name, shutdown_cached_client_after_test
+):
+    opik_client_ = opik.api_objects.opik_client.Opik(batching=True)
+
+    yield opik_client_
+
+    opik_client_.end()

sdks/python/tests/e2e_library_integration/harbor/constants.py

@@ -0,0 +1,11 @@
+from ... import llm_constants
+
+PROJECT_NAME = "e2e-harbor-integration-tests"
+
+AGENT_NAME = "terminus-2"
+MODEL_NAME = llm_constants.OPENAI_GPT_NANO
+TIMEOUT_SEC = 20
+
+DATASET_NAME = "terminal-bench"
+DATASET_VERSION = "2.0"
+TASK_NAMES = ["fix-git", "overfull-hbox"]

sdks/python/tests/e2e_library_integration/harbor/requirements.txt

@@ -0,0 +1 @@
+harbor>=0.1.0

sdks/python/tests/e2e_library_integration/harbor/test_harbor_e2e.py

@@ -0,0 +1,279 @@
+"""
+E2E integration tests for Harbor with Opik tracking. This file is for running tests locally only and
+is not used in CI.
+
+These tests require:
+- Harbor installed: pip install harbor
+- Docker running (for Harbor environments)
+- OPENAI_API_KEY environment variable set
+"""
+
+import json
+import os
+import yaml
+import tempfile
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from click.testing import CliRunner
+
+from opik.cli.main import cli
+from opik import Opik, synchronization
+from opik.integrations.harbor import track_harbor, reset_harbor_tracking
+from . import constants
+
+from harbor.job import Job
+from harbor.models.job.config import (
+    AgentConfig,
+    JobConfig,
+    EnvironmentConfig,
+    OrchestratorConfig,
+    RegistryDatasetConfig,
+)
+from harbor.models.registry import RemoteRegistryInfo
+
+pytest.importorskip("harbor")
+
+pytestmark = pytest.mark.usefixtures("ensure_openai_configured")
+
+
+def get_job_id_from_harbor_job_dir(jobs_dir: Path, job_name: str) -> str:
+    """Extract job_id from Harbor's trial config.json file.
+
+    Args:
+        jobs_dir: The jobs directory where Harbor stores job results.
+        job_name: The name of the job.
+
+    Returns:
+        The job_id UUID string.
+    """
+    job_dir = jobs_dir / job_name
+    trial_dirs = [d for d in job_dir.iterdir() if d.is_dir()]
+    assert len(trial_dirs) >= 1, "Trial directory should exist"
+    trial_config_path = trial_dirs[0] / "config.json"
+    with open(trial_config_path) as f:
+        trial_config = json.load(f)
+    return trial_config["job_id"]
+
+
+def assert_harbor_experiment_created(
+    opik_client: Opik, jobs_dir: Path, job_name: str
+) -> None:
+    """Verify that a Harbor experiment was created with the correct name and has reward scores.
+
+    The experiment name is based on job_id: "harbor-job-{job_id[:8]}"
+
+    Args:
+        opik_client: The Opik client instance.
+        jobs_dir: The jobs directory where Harbor stores job results.
+        job_name: The name of the job.
+    """
+    job_id = get_job_id_from_harbor_job_dir(jobs_dir, job_name)
+    expected_experiment_name = f"harbor-job-{job_id[:8]}"
+
+    experiments = opik_client.get_experiments_by_name(expected_experiment_name)
+    assert len(experiments) >= 1, (
+        f"Experiment '{expected_experiment_name}' should be created"
+    )
+
+    # Verify the experiment has exactly two items with non-null reward feedback scores
+    experiment = experiments[0]
+    items = experiment.get_items()
+    assert len(items) == 2, (
+        f"Experiment should have exactly two items, got {len(items)}"
+    )
+
+    has_reward_score = any(
+        score.get("name") == "reward" and score.get("value") is not None
+        for item in items
+        for score in item.feedback_scores
+    )
+    assert has_reward_score, (
+        "Experiment should have at least one item with a non-null reward feedback score"
+    )
+
+
+@pytest.fixture(autouse=True)
+def reset_tracking_state():
+    reset_harbor_tracking()
+    yield
+    reset_harbor_tracking()
+
+
+@pytest.fixture
+def temp_jobs_dir():
+    with tempfile.TemporaryDirectory(prefix="harbor_test_") as tmpdir:
+        yield Path(tmpdir)
+
+
+class TestHarborSDKIntegration:
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.getenv("OPENAI_API_KEY"),
+        reason="OPENAI_API_KEY not set",
+    )
+    async def test_track_harbor_creates_traces_and_experiment(
+        self,
+        opik_client: Opik,
+        configure_e2e_tests_env_unique_project_name: str,
+        temp_jobs_dir: Path,
+    ):
+        """Test that track_harbor automatically creates traces, dataset, and experiment."""
+        agent = AgentConfig(
+            name=constants.AGENT_NAME,
+            model_name=constants.MODEL_NAME,
+            override_timeout_sec=constants.TIMEOUT_SEC,
+        )
+
+        dataset = RegistryDatasetConfig(
+            registry=RemoteRegistryInfo(),
+            name=constants.DATASET_NAME,
+            version=constants.DATASET_VERSION,
+            task_names=constants.TASK_NAMES,
+        )
+
+        timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        job_name = f"opik-test-{timestamp}"
+
+        job = Job(
+            JobConfig(
+                job_name=job_name,
+                jobs_dir=temp_jobs_dir,
+                orchestrator=OrchestratorConfig(n_concurrent_trials=1),
+                environment=EnvironmentConfig(delete=True),
+                agents=[agent],
+                datasets=[dataset],
+            )
+        )
+
+        tracked_job = track_harbor(job)
+
+        await tracked_job.run()
+
+        if not synchronization.until(
+            function=lambda: opik_client.search_traces(
+                project_name=configure_e2e_tests_env_unique_project_name
+            )[0].output
+            is not None,
+            allow_errors=True,
+            max_try_seconds=60,
+        ):
+            raise AssertionError("Failed to get traces")
+
+        traces = opik_client.search_traces(
+            project_name=configure_e2e_tests_env_unique_project_name, truncate=False
+        )
+        assert len(traces) == 2, f"Expected 2 traces (one per task), got {len(traces)}"
+
+        for trace in traces:
+            assert trace.metadata.get("created_from") == "harbor"
+            assert "harbor" in (trace.tags or [])
+            assert constants.AGENT_NAME in trace.name, (
+                f"Trace name '{trace.name}' should contain agent name '{constants.AGENT_NAME}'"
+            )
+
+            spans = opik_client.search_spans(trace_id=trace.id, truncate=False)
+            assert len(spans) >= 1
+
+        opik_dataset = opik_client.get_dataset(constants.DATASET_NAME)
+        assert opik_dataset is not None, "Dataset should be created automatically"
+
+        assert_harbor_experiment_created(opik_client, temp_jobs_dir, job_name)
+
+
+class TestHarborCLIIntegration:
+    @pytest.mark.skipif(
+        not os.getenv("OPENAI_API_KEY"),
+        reason="OPENAI_API_KEY not set",
+    )
+    def test_opik_harbor_cli_creates_traces_and_experiment(
+        self,
+        opik_client: Opik,
+        configure_e2e_tests_env_unique_project_name: str,
+        temp_jobs_dir: Path,
+    ):
+        """Test that `opik harbor run` automatically creates traces, dataset, and experiment."""
+
+        timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        job_name = f"opik-cli-test-{timestamp}"
+
+        # Create config file with proper timeout settings
+        # (--ak override_timeout_sec=X doesn't work due to Harbor CLI bug)
+        config = {
+            "job_name": job_name,
+            "jobs_dir": str(temp_jobs_dir),
+            "n_attempts": 1,
+            "timeout_multiplier": 1.0,
+            "orchestrator": {
+                "type": "local",
+                "n_concurrent_trials": 1,
+            },
+            "environment": {
+                "type": "docker",
+                "delete": True,
+            },
+            "agents": [
+                {
+                    "name": constants.AGENT_NAME,
+                    "model_name": constants.MODEL_NAME,
+                    "override_timeout_sec": constants.TIMEOUT_SEC,
+                }
+            ],
+            "datasets": [
+                {
+                    "registry": {"type": "remote"},
+                    "name": constants.DATASET_NAME,
+                    "version": constants.DATASET_VERSION,
+                    "task_names": constants.TASK_NAMES,
+                }
+            ],
+        }
+
+        config_path = temp_jobs_dir / "config.yaml"
+        with open(config_path, "w") as f:
+            yaml.dump(config, f)
+
+        runner = CliRunner()
+
+        result = runner.invoke(
+            cli,
+            [
+                "harbor",
+                "run",
+                "-c",
+                str(config_path),
+            ],
+            catch_exceptions=False,
+        )
+
+        # Print output for debugging visibility
+        print(f"\n=== Harbor CLI Output ===\n{result.output}")
+
+        assert result.exit_code == 0, f"CLI failed: {result.output}"
+
+        if not synchronization.until(
+            function=lambda: opik_client.search_traces(
+                project_name=configure_e2e_tests_env_unique_project_name
+            )[0].output
+            is not None,
+            allow_errors=True,
+            max_try_seconds=60,
+        ):
+            raise AssertionError("Failed to get traces after CLI run")
+
+        traces = opik_client.search_traces(
+            project_name=configure_e2e_tests_env_unique_project_name, truncate=False
+        )
+        assert len(traces) == 2, f"Expected 2 traces (one per task), got {len(traces)}"
+        for trace in traces:
+            assert "harbor" in (trace.tags or [])
+            assert constants.AGENT_NAME in trace.name, (
+                f"Trace name '{trace.name}' should contain agent name '{constants.AGENT_NAME}'"
+            )
+
+        opik_dataset = opik_client.get_dataset(constants.DATASET_NAME)
+        assert opik_dataset is not None, "Dataset should be created automatically"
+
+        assert_harbor_experiment_created(opik_client, temp_jobs_dir, job_name)

sdks/python/tests/e2e_library_integration/litellm/constants.py

@@ -0,0 +1,3 @@
+from ... import llm_constants
+
+MODEL_NAME = llm_constants.OPENAI_GPT_NANO

sdks/python/tests/e2e_library_integration/litellm/requirements.txt

@@ -0,0 +1 @@
+litellm<1.75.0

sdks/python/tests/e2e_library_integration/litellm/test_opik_logging.py

@@ -0,0 +1,85 @@
+import litellm
+
+from opik import Opik, synchronization
+from ...e2e import verifiers
+from ...testlib import ANY_DICT, ANY_STRING
+
+
+from . import constants
+
+
+def test_litellm_opik_logging__happyflow(
+    ensure_openai_configured,
+    opik_client: Opik,
+    configure_e2e_tests_env_unique_project_name: str,
+):
+    litellm.callbacks = ["opik"]
+
+    def streaming_function(input):
+        messages = [{"role": "user", "content": input}]
+        # Note: no reasoning_effort here — the litellm pinned in the e2e env
+        # doesn't recognize it for gpt-5-nano and raises UnsupportedParamsError.
+        response = litellm.completion(
+            model=constants.MODEL_NAME,
+            messages=messages,
+            metadata={
+                "opik": {
+                    "tags": ["streaming-test"],
+                },
+            },
+        )
+        return response
+
+    _response = streaming_function("Why is tracking and evaluation of LLMs important?")
+
+    if not synchronization.until(
+        function=lambda: (len(opik_client.search_traces()) > 0),
+        allow_errors=True,
+        max_try_seconds=30,
+    ):
+        raise AssertionError(
+            f"Failed to get traces from project '{configure_e2e_tests_env_unique_project_name}'"
+        )
+
+    traces = opik_client.search_traces(truncate=False)
+    spans = opik_client.search_spans(truncate=False)
+
+    assert len(traces) == 1
+    assert len(spans) == 1
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=traces[0].id,
+        name="chat.completion",
+        metadata=ANY_DICT.containing({"created_from": "litellm"}),
+        input=[
+            {
+                "content": "Why is tracking and evaluation of LLMs important?",
+                "role": "user",
+            }
+        ],
+        output=ANY_DICT,
+        tags=["openai", "streaming-test"],
+        project_name=configure_e2e_tests_env_unique_project_name,
+        error_info=None,
+    )
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        trace_id=traces[0].id,
+        span_id=spans[0].id,
+        parent_span_id=None,
+        name=ANY_STRING.starting_with(constants.MODEL_NAME),
+        metadata=ANY_DICT.containing({"created_from": "litellm"}),
+        input=[
+            {
+                "content": "Why is tracking and evaluation of LLMs important?",
+                "role": "user",
+            }
+        ],
+        output=ANY_DICT,
+        tags=["openai", "streaming-test"],
+        project_name=configure_e2e_tests_env_unique_project_name,
+        error_info=None,
+        type="llm",
+    )

sdks/python/tests/e2e_library_integration/opentelemetry/requirements.txt

@@ -0,0 +1,3 @@
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp

sdks/python/tests/e2e_library_integration/opentelemetry/test_otel_distributed_trace.py

@@ -0,0 +1,233 @@
+"""
+E2E tests for the opik.integrations.otel distributed tracing module.
+
+These tests verify the full roundtrip through the Opik backend:
+  Service A (Opik SDK) creates a trace/span and extracts distributed headers →
+  Service B receives headers, creates OTel spans with opik attributes,
+  and sends them to the Opik backend via OTLP →
+  Verify the OTel spans appear in the backend linked to the parent trace.
+"""
+
+import urllib.parse
+
+
+import opik
+from opik import opik_context, config as opik_config, synchronization
+from opik.integrations.otel import OpikSpanProcessor, distributed_trace
+
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+from ...e2e import verifiers
+
+
+def _build_otlp_exporter(config: opik_config.OpikConfig) -> OTLPSpanExporter:
+    """Build an OTLP exporter pointing at the Opik backend."""
+    parsed = urllib.parse.urlparse(config.url_override)
+    port = parsed.port
+    # different port for local dev vs test CI environment
+    otel_path = (
+        "/api/v1/private/otel/v1/traces"
+        if port == 5173
+        else "/v1/private/otel/v1/traces"
+    )
+    endpoint = urllib.parse.urljoin(config.url_override, otel_path)
+    headers = {"Authorization": config.api_key or ""}
+    if config.workspace:
+        headers["Comet-Workspace"] = config.workspace
+    headers["projectName"] = config.project_name
+    return OTLPSpanExporter(endpoint=endpoint, headers=headers)
+
+
+def _create_otel_tracer(config: opik_config.OpikConfig):
+    """Create an OTel TracerProvider and tracer configured for the Opik OTLP endpoint.
+
+    Registers ``OpikSpanProcessor`` so descendants of an ``attach_to_parent``
+    boundary span automatically inherit the Opik trace context.
+    """
+    resource = Resource.create({"service.name": "e2e-otel-test"})
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(OpikSpanProcessor())
+    exporter = _build_otlp_exporter(config)
+    provider.add_span_processor(BatchSpanProcessor(exporter))
+    return provider.get_tracer("e2e-otel-test-tracer"), provider
+
+
+def test_otel_distributed_trace_roundtrip__happyflow(opik_client: opik.Opik):
+    """
+    Full E2E roundtrip through the Opik backend:
+    1. Service A: create Opik trace and span, extract distributed headers
+    2. Service B: create OTel span with opik attributes via attach_to_parent,
+       send to Opik backend OTLP endpoint
+    3. Verify: the OTel span appears in the backend under the correct trace
+       with the correct parent-child linkage
+    """
+    ID_STORAGE = {}
+
+    # --- Service A: create parent trace/span and extract headers ---
+    @opik.track(name="service-a-parent")
+    def service_a_handler(request):
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["parent_span_id"] = opik_context.get_current_span_data().id
+        ID_STORAGE["headers"] = opik_context.get_distributed_trace_headers()
+        return "service-a-response"
+
+    service_a_handler("service-a-request")
+    opik.flush_tracker()
+
+    # --- Service B: create OTel span with opik distributed trace attributes ---
+    config = opik_config.OpikConfig()
+    tracer, provider = _create_otel_tracer(config)
+
+    distributed_headers = ID_STORAGE["headers"]
+    with tracer.start_as_current_span("service-b-otel-span") as otel_span:
+        distributed_trace.attach_to_parent(otel_span, distributed_headers)
+        otel_span.set_attribute("input", "service-b-input")
+        otel_span.set_attribute("output", "service-b-output")
+
+        # add child span
+        with tracer.start_as_current_span("service-b-child-span") as child_span:
+            child_span.set_attribute("input", "service-b-child-input")
+            child_span.set_attribute("output", "service-b-child-output")
+
+    provider.force_flush()
+    provider.shutdown()
+
+    # --- Verify: parent trace and span in Opik backend ---
+    trace_id = ID_STORAGE["trace_id"]
+    parent_span_id = ID_STORAGE["parent_span_id"]
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace_id,
+        name="service-a-parent",
+        input={"request": "service-a-request"},
+        output={"output": "service-a-response"},
+    )
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=parent_span_id,
+        trace_id=trace_id,
+        parent_span_id=None,
+        name="service-a-parent",
+        input={"request": "service-a-request"},
+        output={"output": "service-a-response"},
+    )
+
+    if not synchronization.until(
+        lambda: len(
+            opik_client.search_spans(
+                project_name=opik_client.config.project_name, trace_id=trace_id
+            )
+        )
+        == 3,
+        max_try_seconds=10,
+    ):
+        raise AssertionError(
+            "Expected 3 spans in Opik backend: Service A parent, Service B attached, "
+            "and Service B child (chained via OpikSpanProcessor) within timeout"
+        )
+
+    spans = opik_client.search_spans(
+        project_name=opik_client.config.project_name,
+        trace_id=trace_id,
+    )
+
+    # check that the attached span is present and has the correct parent-child linkage
+    attached_span = next(
+        iter([span for span in spans if span.name == "service-b-otel-span"]), None
+    )
+    assert attached_span is not None, "service-b-otel-span not found in Opik backend"
+    assert attached_span.parent_span_id == parent_span_id
+    assert attached_span.trace_id == trace_id
+
+    attached_span_child = next(
+        iter([span for span in spans if span.name == "service-b-child-span"]), None
+    )
+    assert attached_span_child is not None, (
+        "service-b-child-span not found in Opik backend"
+    )
+    assert attached_span_child.parent_span_id == attached_span.id
+
+
+def test_opik_track_with_otel_spans__single_merged_trace__happyflow(
+    opik_client: opik.Opik,
+):
+    """
+    Opik-native tracing composed with OTel through the backend (OPIK-6770).
+
+    No distributed headers, no attach_to_parent: an @opik.track entrypoint
+    opens the trace, and OTel spans started inside it are linked by
+    OpikSpanProcessor's in-process fallback. Guarantees that opik-native
+    logging and OTel ingestion land in a single trace, with the OTel spans
+    nested under the native entrypoint span — the main flow this fix enables.
+    """
+    config = opik_config.OpikConfig()
+    tracer, provider = _create_otel_tracer(config)
+    ID_STORAGE = {}
+
+    @opik.track(name="native-entrypoint")
+    def handler(request):
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["entrypoint_span_id"] = opik_context.get_current_span_data().id
+        with tracer.start_as_current_span("otel-parent"):
+            with tracer.start_as_current_span("otel-child"):
+                pass
+        return "response"
+
+    handler("request")
+    opik.flush_tracker()
+    provider.force_flush()
+    provider.shutdown()
+
+    trace_id = ID_STORAGE["trace_id"]
+    entrypoint_span_id = ID_STORAGE["entrypoint_span_id"]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace_id,
+        name="native-entrypoint",
+        input={"request": "request"},
+        output={"output": "response"},
+    )
+
+    if not synchronization.until(
+        lambda: len(
+            opik_client.search_spans(
+                project_name=opik_client.config.project_name, trace_id=trace_id
+            )
+        )
+        == 3,
+        max_try_seconds=15,
+    ):
+        raise AssertionError(
+            "Expected 3 spans (native entrypoint + 2 OTel spans) in a single trace "
+            "within timeout"
+        )
+
+    spans = opik_client.search_spans(
+        project_name=opik_client.config.project_name,
+        trace_id=trace_id,
+    )
+
+    # single merged trace — every span shares the native trace id
+    assert all(span.trace_id == trace_id for span in spans)
+
+    roots = [span for span in spans if span.parent_span_id is None]
+    assert len(roots) == 1
+    assert roots[0].id == entrypoint_span_id
+    assert roots[0].name == "native-entrypoint"
+
+    otel_parent = next(
+        (span for span in spans if span.parent_span_id == entrypoint_span_id), None
+    )
+    assert otel_parent is not None, "OTel parent span not nested under entrypoint"
+    assert otel_parent.name == "otel-parent"
+
+    otel_child = next(
+        (span for span in spans if span.parent_span_id == otel_parent.id), None
+    )
+    assert otel_child is not None, "OTel child span not chained under OTel parent"
+    assert otel_child.name == "otel-child"

sdks/python/tests/e2e_library_integration/pydantic_ai/requirements.txt

@@ -0,0 +1,4 @@
+pydantic-ai
+logfire
+opentelemetry-sdk
+opentelemetry-exporter-otlp

sdks/python/tests/e2e_library_integration/pydantic_ai/test_pydantic_ai_logfire.py

@@ -0,0 +1,479 @@
+"""
+E2E library integration test for OPIK-6770.
+
+Reproduces the ticket scenario: a PydanticAI agent instrumented via logfire
+emits its own OTel spans, and the whole call is wrapped in ``@opik.track``.
+
+Before the ``OpikSpanProcessor`` in-process fallback this produced *two*
+separate traces — one SDK trace for the tracked entrypoint and one OTel trace
+for the ``agent run`` / model spans, since ``@opik.track`` creates an Opik span
+in Opik's context storage but no OTel span for logfire to inherit from.
+
+With the fallback, registering ``OpikSpanProcessor`` on logfire's
+``TracerProvider`` links the agent's root OTel span to the active tracked span,
+producing a single trace with the model spans nested under the entrypoint.
+
+A ``TestModel`` is used so the test is hermetic (no LLM provider key needed).
+"""
+
+import asyncio
+import json
+import urllib.parse
+
+import logfire
+from pydantic_ai import Agent
+from pydantic_ai.models.test import TestModel
+
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+import opik
+from opik import opik_context, config as opik_config, synchronization
+from opik.integrations.otel import OpikSpanProcessor
+
+from ...e2e import verifiers
+
+
+# PydanticAI instrumentation v<=2 names the agent root span "agent run" and the
+# tool span "running tool". v3+ follows the OpenTelemetry GenAI semantic
+# conventions: "invoke_agent {agent_name}" and "execute_tool {tool_name}". The
+# deps are unpinned, so accept either naming so the test survives version drift.
+def _is_agent_run_span(name: str) -> bool:
+    return name == "agent run" or name.startswith("invoke_agent")
+
+
+def _is_tool_span(name: str) -> bool:
+    return name == "running tool" or name.startswith("execute_tool")
+
+
+# Tool result attribute key: "tool_response" (v<=2) or the GenAI semconv
+# "gen_ai.tool.call.result" (v3+).
+_TOOL_RESULT_KEYS = ("tool_response", "gen_ai.tool.call.result")
+
+
+def _build_otlp_exporter(config: opik_config.OpikConfig) -> OTLPSpanExporter:
+    parsed = urllib.parse.urlparse(config.url_override)
+    # different port for local dev vs test CI environment
+    otel_path = (
+        "/api/v1/private/otel/v1/traces"
+        if parsed.port == 5173
+        else "/v1/private/otel/v1/traces"
+    )
+    endpoint = urllib.parse.urljoin(config.url_override, otel_path)
+    headers = {"Authorization": config.api_key or ""}
+    if config.workspace:
+        headers["Comet-Workspace"] = config.workspace
+    headers["projectName"] = config.project_name
+    return OTLPSpanExporter(endpoint=endpoint, headers=headers)
+
+
+def test_pydantic_ai_logfire_with_opik_track__single_merged_trace(
+    opik_client: opik.Opik,
+):
+    config = opik_config.OpikConfig()
+    exporter = _build_otlp_exporter(config)
+
+    logfire.configure(
+        send_to_logfire=False,
+        console=False,
+        additional_span_processors=[
+            BatchSpanProcessor(exporter),
+            OpikSpanProcessor(),
+        ],
+    )
+    logfire.instrument_pydantic_ai()
+
+    agent = Agent(TestModel(custom_output_text="4"))
+    ID_STORAGE = {}
+
+    @opik.track(name="run_entrypoint")
+    async def run(question: str) -> str:
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["entrypoint_span_id"] = opik_context.get_current_span_data().id
+        result = await agent.run(question)
+        return result.output
+
+    asyncio.run(run("What is 2+2?"))
+    opik.flush_tracker()
+    logfire.force_flush()
+
+    trace_id = ID_STORAGE["trace_id"]
+    entrypoint_span_id = ID_STORAGE["entrypoint_span_id"]
+
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=trace_id,
+        name="run_entrypoint",
+        input={"question": "What is 2+2?"},
+        output={"output": "4"},
+    )
+
+    # entrypoint + at least the "agent run" and a model span all under one trace
+    if not synchronization.until(
+        lambda: (
+            len(
+                opik_client.search_spans(
+                    project_name=opik_client.config.project_name, trace_id=trace_id
+                )
+            )
+            >= 3
+        ),
+        max_try_seconds=15,
+    ):
+        raise AssertionError(
+            "Expected the PydanticAI/logfire OTel spans to be ingested under the "
+            "@opik.track trace within timeout"
+        )
+
+    spans = opik_client.search_spans(
+        project_name=opik_client.config.project_name,
+        trace_id=trace_id,
+    )
+
+    # every span belongs to the single tracked trace — no orphan OTel trace
+    assert all(span.trace_id == trace_id for span in spans)
+
+    # exactly one root, and it is the tracked entrypoint span
+    roots = [span for span in spans if span.parent_span_id is None]
+    assert len(roots) == 1
+    assert roots[0].id == entrypoint_span_id
+    assert roots[0].name == "run_entrypoint"
+
+    # the agent's root OTel span attached directly under the entrypoint span
+    otel_root = next(
+        (span for span in spans if span.parent_span_id == entrypoint_span_id), None
+    )
+    assert otel_root is not None, (
+        "PydanticAI agent-run span was not nested under the tracked entrypoint"
+    )
+    # assert the name matches the agent-run span so a future hierarchy change
+    # can't pass on parentage alone
+    assert _is_agent_run_span(otel_root.name)
+
+    # and the model/LLM span chained below the OTel root (descendant chaining)
+    span_ids = {span.id for span in spans}
+    assert any(
+        span.parent_span_id in span_ids and span.id != otel_root.id
+        for span in spans
+        if span.parent_span_id == otel_root.id
+    ), "Expected a model span nested under the PydanticAI 'agent run' span"
+
+
+def test_pydantic_ai_logfire_with_opik_track__two_agent_calls__correct_nesting(
+    opik_client: opik.Opik,
+):
+    """Two sequential agent calls inside one @opik.track entrypoint must produce
+    two sibling 'agent run' subtrees under the entrypoint — each with its own
+    model span and no cross-linking between the calls — all in a single trace."""
+    config = opik_config.OpikConfig()
+    exporter = _build_otlp_exporter(config)
+
+    logfire.configure(
+        send_to_logfire=False,
+        console=False,
+        additional_span_processors=[
+            BatchSpanProcessor(exporter),
+            OpikSpanProcessor(),
+        ],
+    )
+    logfire.instrument_pydantic_ai()
+
+    agent = Agent(TestModel(custom_output_text="ok"))
+    ID_STORAGE = {}
+
+    @opik.track(name="run_entrypoint")
+    def run() -> str:
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["entrypoint_span_id"] = opik_context.get_current_span_data().id
+        agent.run_sync("first question")
+        agent.run_sync("second question")
+        return "done"
+
+    run()
+    opik.flush_tracker()
+    logfire.force_flush()
+
+    trace_id = ID_STORAGE["trace_id"]
+    entrypoint_span_id = ID_STORAGE["entrypoint_span_id"]
+
+    # entrypoint + 2x (agent run + model span) = 5 spans in a single trace
+    if not synchronization.until(
+        lambda: (
+            len(
+                opik_client.search_spans(
+                    project_name=opik_client.config.project_name, trace_id=trace_id
+                )
+            )
+            == 5
+        ),
+        max_try_seconds=15,
+    ):
+        raise AssertionError(
+            "Expected 5 spans (entrypoint + two agent-run subtrees) in a single "
+            "trace within timeout"
+        )
+
+    spans = opik_client.search_spans(
+        project_name=opik_client.config.project_name,
+        trace_id=trace_id,
+    )
+
+    # the tracked entrypoint is the single root of the trace
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=entrypoint_span_id,
+        trace_id=trace_id,
+        parent_span_id=None,
+        name="run_entrypoint",
+    )
+
+    # both agent-run subtrees are siblings directly under the entrypoint
+    agent_runs = [span for span in spans if _is_agent_run_span(span.name)]
+    assert len(agent_runs) == 2
+    for agent_run in agent_runs:
+        verifiers.verify_span(
+            opik_client=opik_client,
+            span_id=agent_run.id,
+            trace_id=trace_id,
+            parent_span_id=entrypoint_span_id,
+        )
+
+    # each 'agent run' owns exactly one model span parented to that run alone —
+    # with the 5-span total above, this rules out any cross-linking between calls
+    for agent_run in agent_runs:
+        children = [span for span in spans if span.parent_span_id == agent_run.id]
+        assert len(children) == 1, "each agent run should own exactly one model span"
+        verifiers.verify_span(
+            opik_client=opik_client,
+            span_id=children[0].id,
+            trace_id=trace_id,
+            parent_span_id=agent_run.id,
+        )
+
+
+def test_pydantic_ai_logfire__nested_track_and_otel__cross_origin_nesting(
+    opik_client: opik.Opik,
+):
+    """Interleave native @opik.track spans with logfire/PydanticAI OTel spans at
+    different depths. Each OTel 'agent run' must attach to the innermost active
+    native span — one under the nested `inner_step`, the other under the
+    top-level `outer_entrypoint` — so parent-child links are preserved across
+    origins (native <-> OTel) within a single trace."""
+    config = opik_config.OpikConfig()
+    exporter = _build_otlp_exporter(config)
+
+    logfire.configure(
+        send_to_logfire=False,
+        console=False,
+        additional_span_processors=[
+            BatchSpanProcessor(exporter),
+            OpikSpanProcessor(),
+        ],
+    )
+    logfire.instrument_pydantic_ai()
+
+    agent = Agent(TestModel(custom_output_text="ok"))
+    ID_STORAGE = {}
+
+    @opik.track(name="inner_step")
+    def inner_step() -> None:
+        ID_STORAGE["inner_span_id"] = opik_context.get_current_span_data().id
+        agent.run_sync("inner question")
+
+    @opik.track(name="outer_entrypoint")
+    def outer() -> str:
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        ID_STORAGE["outer_span_id"] = opik_context.get_current_span_data().id
+        inner_step()
+        agent.run_sync("outer question")
+        return "done"
+
+    outer()
+    opik.flush_tracker()
+    logfire.force_flush()
+
+    trace_id = ID_STORAGE["trace_id"]
+    outer_span_id = ID_STORAGE["outer_span_id"]
+    inner_span_id = ID_STORAGE["inner_span_id"]
+
+    # outer + inner (native) + 2x (agent run + model) = 6 spans in one trace
+    if not synchronization.until(
+        lambda: (
+            len(
+                opik_client.search_spans(
+                    project_name=opik_client.config.project_name, trace_id=trace_id
+                )
+            )
+            == 6
+        ),
+        max_try_seconds=15,
+    ):
+        raise AssertionError(
+            "Expected 6 spans (nested native steps + two agent-run subtrees) in a "
+            "single trace within timeout"
+        )
+
+    spans = opik_client.search_spans(
+        project_name=opik_client.config.project_name,
+        trace_id=trace_id,
+    )
+
+    # native spans: outer is the root, inner is its child
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=outer_span_id,
+        trace_id=trace_id,
+        parent_span_id=None,
+        name="outer_entrypoint",
+    )
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=inner_span_id,
+        trace_id=trace_id,
+        parent_span_id=outer_span_id,
+        name="inner_step",
+    )
+
+    # the two OTel agent-run spans attach to the innermost native span that was
+    # active when each was started: one under inner_step, one under outer.
+    agent_runs = [span for span in spans if _is_agent_run_span(span.name)]
+    assert len(agent_runs) == 2
+    assert {run.parent_span_id for run in agent_runs} == {inner_span_id, outer_span_id}
+    for agent_run in agent_runs:
+        verifiers.verify_span(
+            opik_client=opik_client,
+            span_id=agent_run.id,
+            trace_id=trace_id,
+            parent_span_id=agent_run.parent_span_id,
+        )
+        children = [span for span in spans if span.parent_span_id == agent_run.id]
+        assert len(children) == 1, "each agent run should own exactly one model span"
+        verifiers.verify_span(
+            opik_client=opik_client,
+            span_id=children[0].id,
+            trace_id=trace_id,
+            parent_span_id=agent_run.id,
+        )
+
+
+def test_pydantic_ai_logfire__tool_span_logs_its_output(
+    opik_client: opik.Opik,
+):
+    """A PydanticAI tool's return value must land on the tool span's OUTPUT.
+
+    The logfire instrumentation emits the result under a tool-result attribute
+    (``tool_response`` in v<=2, ``gen_ai.tool.call.result`` in v3+); without an
+    explicit mapping it falls into the default INPUT bucket, so the tool span
+    renders no output in the UI.
+    """
+    config = opik_config.OpikConfig()
+    exporter = _build_otlp_exporter(config)
+
+    logfire.configure(
+        send_to_logfire=False,
+        console=False,
+        additional_span_processors=[
+            BatchSpanProcessor(exporter),
+            OpikSpanProcessor(),
+        ],
+    )
+    logfire.instrument_pydantic_ai()
+
+    agent = Agent(TestModel())
+
+    @agent.tool_plain
+    def get_weather(city: str) -> str:
+        return f"Sunny in {city}"
+
+    ID_STORAGE = {}
+
+    @opik.track(name="run_entrypoint")
+    def run() -> str:
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        return str(agent.run_sync("What is the weather?").output)
+
+    run()
+    opik.flush_tracker()
+    logfire.force_flush()
+
+    trace_id = ID_STORAGE["trace_id"]
+
+    def _tool_span():
+        spans = opik_client.search_spans(
+            project_name=opik_client.config.project_name, trace_id=trace_id
+        )
+        tool_span = next((span for span in spans if _is_tool_span(span.name)), None)
+        return tool_span if tool_span is not None and tool_span.output else None
+
+    if not synchronization.until(lambda: _tool_span() is not None, max_try_seconds=15):
+        raise AssertionError(
+            "Expected the PydanticAI tool span to be ingested with a non-empty "
+            "output within timeout"
+        )
+
+    tool_span = _tool_span()
+    # the tool's return value is mapped to OUTPUT, not INPUT
+    assert any(key in tool_span.output for key in _TOOL_RESULT_KEYS)
+    assert "Sunny in" in json.dumps(tool_span.output)
+    assert all(key not in (tool_span.input or {}) for key in _TOOL_RESULT_KEYS)
+
+
+def test_pydantic_ai_logfire__tool_error_is_logged_as_error_info(
+    opik_client: opik.Opik,
+):
+    """When a PydanticAI tool raises, the OTel ``exception`` event must surface as
+    the Opik span's ``error_info`` instead of being buried in raw event metadata.
+    """
+    config = opik_config.OpikConfig()
+    exporter = _build_otlp_exporter(config)
+
+    logfire.configure(
+        send_to_logfire=False,
+        console=False,
+        additional_span_processors=[
+            BatchSpanProcessor(exporter),
+            OpikSpanProcessor(),
+        ],
+    )
+    logfire.instrument_pydantic_ai()
+
+    agent = Agent(TestModel())
+
+    @agent.tool_plain
+    def failing_tool() -> str:
+        raise RuntimeError("boom: upstream 502")
+
+    ID_STORAGE = {}
+
+    @opik.track(name="run_entrypoint")
+    def run() -> None:
+        ID_STORAGE["trace_id"] = opik_context.get_current_trace_data().id
+        try:
+            agent.run_sync("call the tool")
+        except Exception:
+            # the tool error propagates out of the agent run; we only care that
+            # it was recorded on the span, not about handling it here
+            pass
+
+    run()
+    opik.flush_tracker()
+    logfire.force_flush()
+
+    trace_id = ID_STORAGE["trace_id"]
+
+    def _error_spans():
+        spans = opik_client.search_spans(
+            project_name=opik_client.config.project_name, trace_id=trace_id
+        )
+        return [span for span in spans if span.error_info is not None]
+
+    if not synchronization.until(lambda: len(_error_spans()) > 0, max_try_seconds=15):
+        raise AssertionError(
+            "Expected at least one span with error_info populated from the tool "
+            "exception within timeout"
+        )
+
+    error_info = _error_spans()[0].error_info
+    assert error_info.exception_type == "RuntimeError"
+    assert "boom: upstream 502" in (error_info.message or "")
+    assert error_info.traceback

sdks/python/tests/e2e_smoke/dry_run_import.py

@@ -0,0 +1,22 @@
+import opik
+
+project_name = "e2e-smoke-test-project"
+
+
+@opik.track(
+    tags=["inner-tag1", "inner-tag2"],
+    metadata={"inner-metadata-key": "inner-metadata-value"},
+    project_name=project_name,
+)
+def hello(x: str) -> str:
+    result = f"Hello {x}!"
+    print(result)
+    return result
+
+
+def main():
+    hello("World")
+
+
+if __name__ == "__main__":
+    main()

sdks/python/tests/e2e_smoke/smoke_tests_runner.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# This is a script to execute smoke tests with naked OPIK installation
+#
+echo "░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░"
+echo ""
+echo "Running smoke tests..."
+echo ""
+
+# Find all Python files in current directory
+python_files=$(find . -maxdepth 1 -name "*.py" ! -name "__init__.py")
+
+# Check if any Python files were found
+if [ -z "$python_files" ]; then
+    echo "No Python files found in current directory: $(pwd)"
+    exit 1
+fi
+
+# Track failures to report aggregate result at the end
+failed=0
+
+# Execute each Python file
+for file in $python_files; do
+    echo "═════════════════════════════════════════════════════════════════"
+    echo ""
+    echo ">>> Executing: $file"
+    python3 "$file"
+
+    # Check execution status
+    if [ $? -eq 0 ]; then
+        echo ">>> ✔ Test passed: $file"
+    else
+        echo ">>> ❌ Test failed: $file"
+        failed=1
+    fi
+    echo ""
+done
+
+echo "═════════════════════════════════════════════════════════════════"
+echo "▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓"
+echo ""
+
+# Final summary
+if [ $failed -eq 0 ]; then
+    echo ">>> ✅ All smoke tests completed successfully!"
+else
+    echo ">>> ❌ Some tests failed!"
+    exit 1
+fi
+
+echo ""
+echo "░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░"

sdks/python/tests/integration/simulation/test_simulation_integration.py

@@ -0,0 +1,202 @@
+"""Integration tests for simulation functionality."""
+
+from unittest.mock import Mock, patch
+from opik.simulation import SimulatedUser, run_simulation
+
+
+class TestSimulationIntegration:
+    """Integration tests for simulation functionality."""
+
+    def test_simulation_with_class_based_app(self):
+        """Test simulation with a class-based app that manages state."""
+
+        class WeatherAgent:
+            def __init__(self):
+                self.histories = {}
+
+            def __call__(self, message, *, thread_id, **kwargs):
+                # Initialize history for this thread
+                if thread_id not in self.histories:
+                    self.histories[thread_id] = []
+
+                # Add user message to history
+                self.histories[thread_id].append(message)
+
+                # Generate response based on full history
+                response_content = f"Response to turn {len(self.histories[thread_id])}"
+                assistant_message = {"role": "assistant", "content": response_content}
+
+                # Add to history
+                self.histories[thread_id].append(assistant_message)
+
+                return assistant_message
+
+        agent = WeatherAgent()
+        user_simulator = SimulatedUser(
+            persona="You are curious about weather",
+            fixed_responses=["What's the weather?", "Tell me more", "Thanks!"],
+        )
+
+        result = run_simulation(app=agent, user_simulator=user_simulator, max_turns=3)
+
+        # Verify conversation structure
+        history = result["conversation_history"]
+        assert len(history) == 6  # 3 turns * 2 messages
+
+        # Verify agent maintained state
+        assert len(agent.histories[result["thread_id"]]) == 6
+
+        # Verify responses reference turn numbers
+        assert "Response to turn 1" in history[1]["content"]
+        assert "Response to turn 3" in history[3]["content"]
+        assert "Response to turn 5" in history[5]["content"]
+
+    def test_simulation_with_multiple_threads(self):
+        """Test that different thread_ids maintain separate state."""
+
+        class StatefulAgent:
+            def __init__(self):
+                self.histories = {}
+
+            def __call__(self, message, *, thread_id, **kwargs):
+                if thread_id not in self.histories:
+                    self.histories[thread_id] = []
+
+                self.histories[thread_id].append(message)
+                response = {
+                    "role": "assistant",
+                    "content": f"Thread {thread_id} turn {len(self.histories[thread_id])}",
+                }
+                self.histories[thread_id].append(response)
+                return response
+
+        agent = StatefulAgent()
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        # Run first simulation
+        result1 = run_simulation(
+            app=agent, user_simulator=user_simulator, thread_id="thread-1", max_turns=2
+        )
+
+        # Run second simulation with different thread
+        result2 = run_simulation(
+            app=agent, user_simulator=user_simulator, thread_id="thread-2", max_turns=2
+        )
+
+        # Verify separate state
+        assert result1["thread_id"] == "thread-1"
+        assert result2["thread_id"] == "thread-2"
+        assert len(agent.histories["thread-1"]) == 4
+        assert len(agent.histories["thread-2"]) == 4
+
+        # Verify responses reference correct threads
+        assert "Thread thread-1 turn 1" in result1["conversation_history"][1]["content"]
+        assert "Thread thread-2 turn 1" in result2["conversation_history"][1]["content"]
+
+    @patch("opik.simulation.simulated_user.OpikBaseModel")
+    def test_simulation_with_llm_user(self, mock_base_model):
+        """Test simulation with LLM-based user simulator."""
+        # Mock LLM responses
+        mock_llm_instance = Mock()
+        mock_llm_instance.generate.side_effect = [
+            "What's the weather like?",
+            "That's interesting, tell me more",
+            "Thank you for the information",
+        ]
+        mock_base_model.return_value = mock_llm_instance
+
+        def mock_app(message, *, thread_id, **kwargs):
+            return {"role": "assistant", "content": f"Assistant: {message['content']}"}
+
+        user_simulator = SimulatedUser(
+            persona="You are curious about weather", model="gpt-4o-mini"
+        )
+        user_simulator._llm = mock_llm_instance
+
+        result = run_simulation(
+            app=mock_app, user_simulator=user_simulator, max_turns=3
+        )
+
+        # Verify LLM was called for each user message
+        assert mock_llm_instance.generate.call_count == 3
+
+        # Verify conversation structure
+        history = result["conversation_history"]
+        assert "What's the weather like?" in history[0]["content"]
+        assert "Assistant: What's the weather like?" in history[1]["content"]
+        assert "That's interesting, tell me more" in history[2]["content"]
+
+    def test_simulation_error_recovery(self):
+        """Test simulation continues after app errors."""
+        call_count = 0
+
+        def error_prone_app(message, *, thread_id, **kwargs):
+            nonlocal call_count
+            call_count += 1
+            if call_count == 2:  # Fail on second call
+                raise Exception("Temporary error")
+            return {"role": "assistant", "content": f"Success {call_count}"}
+
+        user_simulator = SimulatedUser(
+            persona="Test user", fixed_responses=["Message 1", "Message 2", "Message 3"]
+        )
+
+        result = run_simulation(
+            app=error_prone_app, user_simulator=user_simulator, max_turns=3
+        )
+
+        history = result["conversation_history"]
+
+        # First turn should succeed
+        assert "Success 1" in history[1]["content"]
+
+        # Second turn should have error message
+        assert "Error processing message: Temporary error" in history[3]["content"]
+
+        # Third turn should succeed again
+        assert "Success 3" in history[5]["content"]
+
+    def test_simulation_with_complex_app_kwargs(self):
+        """Test simulation with complex app configuration."""
+
+        class ConfigurableAgent:
+            def __init__(self):
+                self.config = {}
+                self.histories = {}
+
+            def __call__(
+                self, message, *, thread_id, model=None, temperature=None, **kwargs
+            ):
+                # Store configuration
+                self.config[thread_id] = {
+                    "model": model,
+                    "temperature": temperature,
+                    "other_kwargs": kwargs,
+                }
+
+                # Manage history
+                if thread_id not in self.histories:
+                    self.histories[thread_id] = []
+
+                self.histories[thread_id].append(message)
+                response = {"role": "assistant", "content": "Configured response"}
+                self.histories[thread_id].append(response)
+                return response
+
+        agent = ConfigurableAgent()
+        user_simulator = SimulatedUser(persona="Test user", fixed_responses=["Message"])
+
+        result = run_simulation(
+            app=agent,
+            user_simulator=user_simulator,
+            model="gpt-4",
+            temperature=0.7,
+            custom_param="test",
+            max_turns=1,
+        )
+
+        # Verify configuration was stored
+        config = agent.config[result["thread_id"]]
+        assert config["model"] == "gpt-4"
+        assert config["temperature"] == 0.7
+        assert config["other_kwargs"]["custom_param"] == "test"