chore: import upstream snapshot with attribution

sdks/python/tests/e2e/cli/_cascade_comparison.py

@@ -0,0 +1,473 @@
+"""Deep-equal helpers for ``opik migrate dataset`` cascade e2e tests.
+
+The cascade copies four kinds of entities -- experiment + experiment items
++ traces + spans -- with FK fields remapped to the destination. Counts
+alone aren't enough; we also need to verify that the content (input,
+output, tags, metadata, feedback scores, assertion results, span tree
+shape) round-trips byte-for-byte modulo the remapped IDs.
+
+This module provides ``compare_cascade(source_state, destination_state, rest_client)``
+that recursively diff-walks both sides and raises ``AssertionError`` with
+a precise message on any mismatch.
+
+What's compared
+---------------
+Experiment level:
+  - name, type, evaluation_method, tags, metadata
+  - prompt_versions must be None on destination (epic decision: strip)
+
+Experiment items (paired via source/dest item ordinal, which corresponds
+to the source/dest dataset_item_id pairing the cascade builds):
+  - assertion_results compared as a set keyed by (value, passed, reason)
+  - feedback_scores compared as a set keyed by (name, value, reason, source)
+  - status NOT compared -- BE computes it from assertion_results
+
+Traces (paired via cascade's trace_id_remap):
+  - name, input, output, metadata, tags, start_time, end_time,
+    thread_id, error_info, ttft, environment
+  - feedback_scores compared as a set keyed by (name, value, reason, source)
+
+Spans (tree-aware):
+  - both sides sorted topologically (parent before child)
+  - parent_span_id remap verified by reconstructing each side's tree and
+    walking in lockstep
+  - per-span: name, type, input, output, metadata, model, provider,
+    tags, usage, start_time, end_time, error_info, ttft,
+    total_estimated_cost, environment
+  - feedback_scores on spans compared as a set
+
+What's NOT compared (intentional)
+---------------------------------
+  - any id field (id, project_id, experiment_id, dataset_id,
+    dataset_version_id, dataset_item_id, trace_id, span_id,
+    parent_span_id, optimization_id) -- they all change during cascade
+  - audit fields (created_at, last_updated_at, created_by, last_updated_by)
+  - BE-computed aggregates on traces/items (trace_count,
+    total_estimated_cost, duration, usage, span_count, llm_span_count,
+    has_tool_spans, providers, span_feedback_scores)
+  - ``project_name`` on experiment metadata (Slice 3 stamps it on the
+    destination as part of recreate_experiment; differs intentionally)
+  - ``prompt_versions`` (stripped on destination per epic decision)
+  - ``optimization_id`` (stripped on destination -- Slice 4's territory)
+
+Trace ``input`` / ``output`` JSON that embeds source-side IDs (e.g.
+``{'item': '<src-dataset-item-id>'}``) round-trips verbatim. The cascade
+deliberately does not recursively remap arbitrary JSON content. Tests
+that seed embedded IDs in trace I/O and care about post-migration
+freshness need their own narrower assertion; this module compares the
+JSON shape verbatim because that IS the cascade's contract.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from opik.rest_api import OpikApi
+
+
+# ---------------------------------------------------------------------------
+# Top-level entrypoint
+# ---------------------------------------------------------------------------
+
+
+def compare_cascade(
+    *,
+    rest_client: OpikApi,
+    source_experiment: Any,
+    destination_experiment: Any,
+    source_item_ids: List[str],
+    destination_item_ids: List[str],
+    source_trace_ids: List[str],
+    destination_trace_ids: List[str],
+    source_items_compare: List[Any],
+    destination_items_compare: List[Any],
+) -> None:
+    """Deep-equal the experiment + items + traces + spans between source and
+    destination, modulo remapped IDs.
+
+    Raises ``AssertionError`` with a focused message on any divergence.
+
+    The trace pairing is positional: ``source_trace_ids[i]`` must correspond
+    to ``destination_trace_ids[i]`` (callers maintain this ordering when
+    they seed + read). Same for items.
+    """
+    _compare_experiment(source_experiment, destination_experiment)
+
+    if len(source_items_compare) != len(destination_items_compare):
+        raise AssertionError(
+            f"item count diverged: source={len(source_items_compare)}, "
+            f"destination={len(destination_items_compare)}"
+        )
+    if len(source_trace_ids) != len(destination_trace_ids):
+        raise AssertionError(
+            f"trace count diverged: source={len(source_trace_ids)}, "
+            f"destination={len(destination_trace_ids)}"
+        )
+
+    # Items are typically returned in BE-imposed order (e.g. by created_at
+    # desc). Pair by dataset_item_id round-trip: source item with source
+    # dataset_item_id S maps to destination item with destination
+    # dataset_item_id D where D = item_id_remap[S]. The callers pass the
+    # already-paired ordered lists, so positional zip works.
+    for src_item, dst_item in zip(source_items_compare, destination_items_compare):
+        _compare_experiment_item(src_item, dst_item)
+
+    # Traces compared in pairs.
+    for src_tid, dst_tid in zip(source_trace_ids, destination_trace_ids):
+        src_trace = rest_client.traces.get_trace_by_id(id=src_tid)
+        dst_trace = rest_client.traces.get_trace_by_id(id=dst_tid)
+        _compare_trace(src_trace, dst_trace)
+
+        # Spans for this trace. ``project_id`` lives on the trace's read
+        # shape and scopes the spans query correctly without needing the
+        # caller to plumb project_name everywhere.
+        src_spans = _fetch_spans_for_trace(
+            rest_client, trace_id=src_tid, project_id=src_trace.project_id
+        )
+        dst_spans = _fetch_spans_for_trace(
+            rest_client, trace_id=dst_tid, project_id=dst_trace.project_id
+        )
+        _compare_span_trees(src_spans, dst_spans)
+
+
+# ---------------------------------------------------------------------------
+# Experiment-level
+# ---------------------------------------------------------------------------
+
+
+def _compare_experiment(src: Any, dst: Any) -> None:
+    if src.name != dst.name:
+        raise AssertionError(
+            f"experiment.name diverged: source={src.name!r}, destination={dst.name!r}"
+        )
+    if src.type != dst.type:
+        raise AssertionError(
+            f"experiment.type diverged: source={src.type!r}, destination={dst.type!r}"
+        )
+    if src.evaluation_method != dst.evaluation_method:
+        raise AssertionError(
+            f"experiment.evaluation_method diverged: source={src.evaluation_method!r}, "
+            f"destination={dst.evaluation_method!r}"
+        )
+    if (src.tags or None) != (dst.tags or None):
+        raise AssertionError(
+            f"experiment.tags diverged: source={src.tags!r}, destination={dst.tags!r}"
+        )
+
+    # Metadata: compare modulo Slice 3's injections.
+    # - ``project_name`` is stamped on the destination by recreate_experiment
+    #   (kept as a forward-import hint); on source it depends on how the
+    #   experiment was created. Strip from both for comparison.
+    # - ``prompt_versions`` is stripped on the destination by design.
+    src_meta = dict(src.metadata or {})
+    dst_meta = dict(dst.metadata or {})
+    src_meta.pop("project_name", None)
+    dst_meta.pop("project_name", None)
+    src_meta.pop("prompt_versions", None)
+    dst_meta.pop("prompt_versions", None)
+    if src_meta != dst_meta:
+        raise AssertionError(
+            f"experiment.metadata diverged (after stripping project_name + "
+            f"prompt_versions): source={src_meta!r}, destination={dst_meta!r}"
+        )
+
+    # Per epic decision, destination must have prompt_versions stripped.
+    if dst.prompt_versions:
+        raise AssertionError(
+            f"experiment.prompt_versions should be stripped on destination "
+            f"(epic decision); got {dst.prompt_versions!r}"
+        )
+
+    # Per epic decision, destination must have optimization_id stripped.
+    if dst.optimization_id:
+        raise AssertionError(
+            f"experiment.optimization_id should be stripped on destination "
+            f"(Slice 4 cascades the optimization entity); "
+            f"got {dst.optimization_id!r}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Experiment item (Compare view)
+# ---------------------------------------------------------------------------
+
+
+def _compare_experiment_item(src: Any, dst: Any) -> None:
+    src_ars = _normalize_assertions(src.assertion_results)
+    dst_ars = _normalize_assertions(dst.assertion_results)
+    if src_ars != dst_ars:
+        raise AssertionError(
+            f"experiment item assertion_results diverged: "
+            f"source={src_ars}, destination={dst_ars}"
+        )
+
+    src_fs = _normalize_feedback_scores(src.feedback_scores)
+    dst_fs = _normalize_feedback_scores(dst.feedback_scores)
+    if src_fs != dst_fs:
+        raise AssertionError(
+            f"experiment item feedback_scores diverged: "
+            f"source={src_fs}, destination={dst_fs}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Trace
+# ---------------------------------------------------------------------------
+
+
+_TRACE_DIRECT_FIELDS: Tuple[str, ...] = (
+    "name",
+    "input",
+    "output",
+    "metadata",
+    "tags",
+    "thread_id",
+    "ttft",
+    "environment",
+)
+
+
+def _compare_trace(src: Any, dst: Any) -> None:
+    for field in _TRACE_DIRECT_FIELDS:
+        s = getattr(src, field, None)
+        d = getattr(dst, field, None)
+        if (s or None) != (d or None):
+            raise AssertionError(
+                f"trace.{field} diverged: source={s!r}, destination={d!r}"
+            )
+
+    # ``error_info`` model_dump for content comparison; the read shape is
+    # ErrorInfoPublic on both sides so dicts should be equal.
+    s_err = _safe_dump(src.error_info)
+    d_err = _safe_dump(dst.error_info)
+    if s_err != d_err:
+        raise AssertionError(
+            f"trace.error_info diverged: source={s_err}, destination={d_err}"
+        )
+
+    # start_time / end_time round-trip as-is; the cascade copies them
+    # verbatim from the source trace. ms precision differences would
+    # surface here.
+    if src.start_time != dst.start_time:
+        raise AssertionError(
+            f"trace.start_time diverged: source={src.start_time}, "
+            f"destination={dst.start_time}"
+        )
+    if (src.end_time or None) != (dst.end_time or None):
+        raise AssertionError(
+            f"trace.end_time diverged: source={src.end_time}, "
+            f"destination={dst.end_time}"
+        )
+
+    # Feedback scores compared as a set keyed by name+value+reason+source.
+    src_fs = _normalize_feedback_scores(src.feedback_scores)
+    dst_fs = _normalize_feedback_scores(dst.feedback_scores)
+    if src_fs != dst_fs:
+        raise AssertionError(
+            f"trace.feedback_scores diverged: source={src_fs}, destination={dst_fs}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Span tree
+# ---------------------------------------------------------------------------
+
+
+_SPAN_DIRECT_FIELDS: Tuple[str, ...] = (
+    "name",
+    "type",
+    "input",
+    "output",
+    "metadata",
+    "model",
+    "provider",
+    "tags",
+    "usage",
+    "total_estimated_cost",
+    "ttft",
+    "environment",
+)
+
+
+def _compare_span_trees(src_spans: List[Any], dst_spans: List[Any]) -> None:
+    """Walk both span trees in parallel, comparing per-node fields and
+    verifying parent_span_id remap (children's new parent must be the
+    remapped new root, etc.).
+
+    Pairs spans across the two sides by tree position: both lists are
+    sorted topologically (parents first) and within a parent's children
+    by (name, start_time). The cascade preserves source order via
+    ``sort_spans_topologically`` so a stable sort makes this
+    deterministic.
+    """
+    if len(src_spans) != len(dst_spans):
+        raise AssertionError(
+            f"span count diverged: source={len(src_spans)}, "
+            f"destination={len(dst_spans)}"
+        )
+
+    src_sorted = _topo_sort_for_compare(src_spans)
+    dst_sorted = _topo_sort_for_compare(dst_spans)
+
+    src_to_dst_span_id: Dict[Optional[str], Optional[str]] = {None: None}
+    for src_span, dst_span in zip(src_sorted, dst_sorted):
+        src_to_dst_span_id[src_span.id] = dst_span.id
+
+        for field in _SPAN_DIRECT_FIELDS:
+            s = getattr(src_span, field, None)
+            d = getattr(dst_span, field, None)
+            if (s or None) != (d or None):
+                raise AssertionError(
+                    f"span.{field} diverged (source span id={src_span.id!r}, "
+                    f"dest span id={dst_span.id!r}): source={s!r}, destination={d!r}"
+                )
+
+        # Timestamps verbatim.
+        if src_span.start_time != dst_span.start_time:
+            raise AssertionError(
+                f"span.start_time diverged (source span id={src_span.id!r}): "
+                f"source={src_span.start_time}, destination={dst_span.start_time}"
+            )
+        if (src_span.end_time or None) != (dst_span.end_time or None):
+            raise AssertionError(
+                f"span.end_time diverged (source span id={src_span.id!r}): "
+                f"source={src_span.end_time}, destination={dst_span.end_time}"
+            )
+
+        # Error info.
+        s_err = _safe_dump(getattr(src_span, "error_info", None))
+        d_err = _safe_dump(getattr(dst_span, "error_info", None))
+        if s_err != d_err:
+            raise AssertionError(
+                f"span.error_info diverged (source span id={src_span.id!r}): "
+                f"source={s_err}, destination={d_err}"
+            )
+
+        # Feedback scores compared as a set.
+        s_fs = _normalize_feedback_scores(getattr(src_span, "feedback_scores", None))
+        d_fs = _normalize_feedback_scores(getattr(dst_span, "feedback_scores", None))
+        if s_fs != d_fs:
+            raise AssertionError(
+                f"span.feedback_scores diverged (source span id={src_span.id!r}): "
+                f"source={s_fs}, destination={d_fs}"
+            )
+
+        # parent_span_id remap correctness: the destination span's
+        # parent_span_id must be the destination id of the source span's
+        # parent (or None for root).
+        expected_dst_parent = src_to_dst_span_id.get(src_span.parent_span_id)
+        if dst_span.parent_span_id != expected_dst_parent:
+            raise AssertionError(
+                f"span.parent_span_id remap incorrect "
+                f"(source span id={src_span.id!r}, source parent={src_span.parent_span_id!r}): "
+                f"expected destination parent={expected_dst_parent!r}, "
+                f"got destination parent={dst_span.parent_span_id!r}"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Normalisation helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalize_assertions(items: Optional[List[Any]]) -> List[Tuple[Any, Any, Any]]:
+    """Set-equality-friendly tuples keyed by the AssertionResult identity:
+    (value, passed, reason). Sorted so list-equality also works."""
+    if not items:
+        return []
+    return sorted(
+        ((a.value, a.passed, a.reason) for a in items),
+        key=lambda t: (str(t[0]), bool(t[1]), str(t[2] or "")),
+    )
+
+
+def _normalize_feedback_scores(
+    items: Optional[List[Any]],
+) -> List[Tuple[Any, ...]]:
+    """Set-equality-friendly tuples keyed by (name, value, reason, source).
+    Source vs destination scores might come back in different orders; the
+    sort makes the comparison stable."""
+    if not items:
+        return []
+    return sorted(
+        (
+            (
+                getattr(f, "name", None),
+                getattr(f, "value", None),
+                getattr(f, "category_name", None),
+                getattr(f, "reason", None),
+                getattr(f, "source", None),
+            )
+            for f in items
+        ),
+        key=lambda t: tuple(str(x) for x in t),
+    )
+
+
+def _safe_dump(obj: Any) -> Optional[Dict[str, Any]]:
+    if obj is None:
+        return None
+    if hasattr(obj, "model_dump"):
+        return obj.model_dump()
+    if isinstance(obj, dict):
+        return obj
+    return {"_raw": str(obj)}
+
+
+def _topo_sort_for_compare(spans: List[Any]) -> List[Any]:
+    """Topological sort that's also stable on (name, start_time).
+
+    The cascade re-emits spans in source topological order. The BE may
+    return them in a different ordering on read; this helper produces a
+    deterministic order on both sides so paired comparison works.
+    """
+    by_id: Dict[Optional[str], Any] = {s.id: s for s in spans}
+    children: Dict[Optional[str], List[Any]] = {None: []}
+    for s in spans:
+        children.setdefault(s.parent_span_id, []).append(s)
+    # Sort each parent's children deterministically.
+    for parent_id, kids in children.items():
+        kids.sort(key=lambda s: (s.name or "", str(s.start_time)))
+
+    out: List[Any] = []
+
+    def _walk(parent_id: Optional[str]) -> None:
+        for s in children.get(parent_id, []):
+            out.append(s)
+            _walk(s.id)
+
+    _walk(None)
+    # Defensive: catch orphans (spans whose parent isn't in the same tree).
+    if len(out) != len(spans):
+        # Append orphans at the end in deterministic order.
+        seen = {s.id for s in out}
+        orphans = [s for s in spans if s.id not in seen]
+        orphans.sort(key=lambda s: (s.name or "", str(s.start_time)))
+        out.extend(orphans)
+    _ = by_id  # by_id retained for clarity / potential future use
+    return out
+
+
+def _fetch_spans_for_trace(
+    rest_client: OpikApi, *, trace_id: str, project_id: Optional[str]
+) -> List[Any]:
+    """Pull all spans for one trace from the BE.
+
+    Scopes by ``project_id`` (off the trace's read shape), required by
+    the BE.
+    """
+    collected: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.spans.get_spans_by_project(
+            project_id=project_id,
+            trace_id=trace_id,
+            page=page,
+            size=200,
+        )
+        page_content = resp.content or []
+        collected.extend(page_content)
+        if len(page_content) < 200:
+            break
+        page += 1
+    return collected

sdks/python/tests/e2e/cli/conftest.py

@@ -0,0 +1,701 @@
+"""Shared fixtures + helpers for ``opik migrate`` e2e tests.
+
+These tests drive ``opik migrate dataset`` against a real backend
+(localhost during dev, the CI-provisioned Opik in CI). They verify
+per-version fidelity end-to-end: items, item-level fields (data,
+description, tags, evaluators, execution_policy, source), version-level
+fields (suite evaluators, execution_policy, user tags, metadata), and
+display order.
+
+Helpers live here so individual test files stay focused on the scenarios
+they exercise. Wire-type item reads (via ``rest_stream_parser`` directly)
+mirror what ``cli/migrate/datasets/version_replay.py`` does in production
+— the SDK dataclass strips per-item ``tags`` during reconstruction, so
+asserting tag fidelity requires the wire type.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import subprocess
+import os
+import sys
+from typing import Any, Dict, Iterator, List, Optional, Set
+
+import pytest
+
+import opik
+from opik.api_objects import rest_stream_parser
+from opik.rest_api import OpikApi
+from opik.rest_api.core.api_error import ApiError
+from opik.rest_api.types import dataset_item_public, dataset_version_public
+
+from ...conftest import random_chars
+
+
+# ---------------------------------------------------------------------------
+# Project fixtures — ephemeral source + target, deleted on teardown
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def source_project_name(opik_client: opik.Opik) -> Iterator[str]:
+    """Create an ephemeral source project for the migration test.
+
+    Deleted on teardown (best-effort — tolerates already-deleted state).
+    Each test gets its own project so parallel runs don't collide on
+    dataset-name uniqueness (datasets are workspace-scoped, not
+    project-scoped, in Opik's BE — see Slice 1's collision pre-flight).
+    """
+    name = f"e2e-cli-migrate-source-{random_chars()}"
+    opik_client.rest_client.projects.create_project(name=name)
+    yield name
+    _best_effort_delete_project(opik_client.rest_client, name)
+
+
+@pytest.fixture
+def target_project_name(opik_client: opik.Opik) -> Iterator[str]:
+    """Create an ephemeral target project for the migration test."""
+    name = f"e2e-cli-migrate-target-{random_chars()}"
+    opik_client.rest_client.projects.create_project(name=name)
+    yield name
+    _best_effort_delete_project(opik_client.rest_client, name)
+
+
+def _best_effort_delete_project(rest_client: OpikApi, name: str) -> None:
+    try:
+        project_id = rest_client.projects.retrieve_project(name=name).id
+        rest_client.projects.delete_project_by_id(project_id)
+    except ApiError:
+        # Already gone (404) or insufficient permissions — either way
+        # cleanup is non-blocking; leave the project for the next run to
+        # garbage-collect or for a maintenance task to clean up.
+        pass
+
+
+# ---------------------------------------------------------------------------
+# CLI invocation
+# ---------------------------------------------------------------------------
+
+
+def run_migrate_cli(
+    args: List[str],
+    audit_log_path: Optional[str] = None,
+    extra_env: Optional[Dict[str, str]] = None,
+) -> subprocess.CompletedProcess:
+    """Invoke ``opik migrate`` via the installed CLI entrypoint.
+
+    Uses subprocess (not Click's ``CliRunner``) so the test exercises the
+    same code path real users hit — module import, Click group setup,
+    config-chain resolution, exit-code handling, stderr routing. Returns
+    the completed process so the caller can assert on ``returncode``,
+    ``stdout``, ``stderr``.
+
+    ``--audit-log`` is appended when provided. Tests typically write to a
+    tmp_path so the JSON can be re-read and asserted on.
+
+    ``extra_env`` is merged into the child process environment — the resume
+    E2E test uses it to put a test-only ``sitecustomize.py`` seam on
+    ``PYTHONPATH`` that injects a deterministic mid-cascade crash (the child
+    ``os._exit``s, so ``returncode`` is the hard-exit code, not a clean CLI
+    exit) and redirects the checkpoint dir into a tmp path.
+    """
+    cmd = [sys.executable, "-m", "opik.cli", "migrate"] + args
+    if audit_log_path is not None:
+        cmd.extend(["--audit-log", audit_log_path])
+    env = None
+    if extra_env is not None:
+        env = {**os.environ, **extra_env}
+    return subprocess.run(cmd, capture_output=True, text=True, env=env)
+
+
+# ---------------------------------------------------------------------------
+# Multi-version source seeding
+# ---------------------------------------------------------------------------
+
+
+def create_dataset_shell(
+    rest_client: OpikApi,
+    name: str,
+    project_name: str,
+    *,
+    type: Optional[str] = None,
+) -> str:
+    """Create an empty dataset (or test suite) and return its id.
+
+    ``type='evaluation_suite'`` produces a test suite (carries version-
+    level evaluators + execution_policy); omit for a plain dataset.
+    Caller is responsible for seeding versions via ``apply_changes``.
+    """
+    kwargs: Dict[str, Any] = {"name": name, "project_name": project_name}
+    if type is not None:
+        kwargs["type"] = type
+    rest_client.datasets.create_dataset(**kwargs)
+    ds = rest_client.datasets.get_dataset_by_identifier(
+        dataset_name=name, project_name=project_name
+    )
+    return ds.id
+
+
+def apply_changes(
+    rest_client: OpikApi,
+    dataset_id: str,
+    *,
+    base_version_id: Optional[str],
+    added_items: Optional[List[Dict[str, Any]]] = None,
+    edited_items: Optional[List[Dict[str, Any]]] = None,
+    deleted_ids: Optional[List[str]] = None,
+    change_description: Optional[str] = None,
+    suite_evaluators: Optional[List[Dict[str, Any]]] = None,
+    suite_execution_policy: Optional[Dict[str, int]] = None,
+    metadata: Optional[Dict[str, str]] = None,
+    user_tags: Optional[List[str]] = None,
+    override: bool = False,
+) -> str:
+    """Send ``apply_dataset_item_changes`` and return the new version id.
+
+    Thin wrapper over the raw REST endpoint that mirrors the BE schema's
+    field names. Used to seed multi-version source datasets for migration
+    tests. ``override=True`` is required for the first version (when
+    ``base_version_id=None``); see the BE validation in
+    ``DatasetItemService.applyDeltaChanges``.
+    """
+    request: Dict[str, Any] = {}
+    if change_description is not None:
+        request["change_description"] = change_description
+    if base_version_id is not None:
+        request["base_version"] = base_version_id
+    if added_items:
+        request["added_items"] = added_items
+    if edited_items:
+        request["edited_items"] = edited_items
+    if deleted_ids:
+        request["deleted_ids"] = deleted_ids
+    if suite_evaluators is not None:
+        request["evaluators"] = suite_evaluators
+    if suite_execution_policy is not None:
+        request["execution_policy"] = suite_execution_policy
+    if metadata is not None:
+        request["metadata"] = metadata
+    if user_tags is not None:
+        request["tags"] = user_tags
+    new_version = rest_client.datasets.apply_dataset_item_changes(
+        id=dataset_id, request=request, override=override
+    )
+    return new_version.id
+
+
+# ---------------------------------------------------------------------------
+# Verification helpers (read side)
+# ---------------------------------------------------------------------------
+
+
+def chronological_versions(
+    rest_client: OpikApi, dataset_id: str
+) -> List[dataset_version_public.DatasetVersionPublic]:
+    """Return every version of ``dataset_id`` oldest-first.
+
+    The REST endpoint returns newest-first; we paginate to exhaustion and
+    reverse so tests can iterate alongside source-version order for per-
+    version comparisons.
+    """
+    out: List[dataset_version_public.DatasetVersionPublic] = []
+    page = 1
+    while True:
+        resp = rest_client.datasets.list_dataset_versions(
+            id=dataset_id, page=page, size=100
+        )
+        if not resp.content:
+            break
+        out.extend(resp.content)
+        if len(resp.content) < 100:
+            break
+        page += 1
+    out.reverse()
+    return out
+
+
+def stream_items_wire(
+    rest_client: OpikApi,
+    *,
+    dataset_name: str,
+    project_name: Optional[str],
+    version_hash: Optional[str],
+) -> List[dataset_item_public.DatasetItemPublic]:
+    """Read items at ``version_hash`` via the raw REST stream + wire type.
+
+    The SDK helper ``rest_operations.stream_dataset_items`` drops per-item
+    ``tags`` during dataclass reconstruction, so tests that assert tag
+    fidelity must go through the wire type directly. Mirrors the same
+    approach used by ``cli/migrate/datasets/version_replay.py`` in
+    production.
+    """
+    raw_stream = rest_client.datasets.stream_dataset_items(
+        dataset_name=dataset_name,
+        project_name=project_name,
+        dataset_version=version_hash,
+    )
+    return rest_stream_parser.read_and_parse_stream(
+        stream=raw_stream,
+        item_class=dataset_item_public.DatasetItemPublic,
+    )
+
+
+def item_content_hash(item: dataset_item_public.DatasetItemPublic) -> str:
+    """Full-fidelity per-item hash covering every persisted user field.
+
+    Mirrors the production hash in ``cli/migrate/datasets/version_replay._content_hash_for``
+    so source-version vs target-version set-equality checks behave the
+    same way the migration code does internally (i.e. any field change
+    is treated as a content change).
+    """
+    content: Dict[str, Any] = {"data": dict(item.data) if item.data else {}}
+    if item.description is not None:
+        content["description"] = item.description
+    if item.tags is not None:
+        content["tags"] = sorted(item.tags)
+    if item.evaluators is not None:
+        content["evaluators"] = [
+            {"name": e.name, "type": e.type, "config": e.config}
+            for e in item.evaluators
+        ]
+    if item.execution_policy is not None:
+        content["execution_policy"] = {
+            "runs_per_item": item.execution_policy.runs_per_item,
+            "pass_threshold": item.execution_policy.pass_threshold,
+        }
+    if item.source is not None:
+        content["source"] = item.source
+    return hashlib.sha256(
+        json.dumps(content, sort_keys=True, default=str).encode()
+    ).hexdigest()
+
+
+def item_hashes(items: List[dataset_item_public.DatasetItemPublic]) -> Set[str]:
+    return {item_content_hash(it) for it in items}
+
+
+def display_order(
+    items: List[dataset_item_public.DatasetItemPublic], key: str = "q"
+) -> List[Optional[Any]]:
+    """Extract one ``data`` field per item in stream order (newest-first).
+
+    The stream's order *is* the UI's display order, so two versions' lists
+    of (e.g.) ``q`` values match iff the visible order matches.
+    """
+    return [(item.data.get(key) if item.data else None) for item in items]
+
+
+def normalize_evaluators(evals: Optional[List[Any]]) -> List[Dict[str, Any]]:
+    """Compare-friendly form of a suite evaluator list.
+
+    Strips wire-type wrapping and sorts by name so identical
+    configurations hash equal regardless of how the BE happened to
+    serialise them.
+    """
+    if not evals:
+        return []
+    return sorted(
+        ({"name": e.name, "type": e.type, "config": e.config} for e in evals),
+        key=lambda d: d["name"],
+    )
+
+
+def normalize_policy(pol: Any) -> Optional[Dict[str, int]]:
+    """Compare-friendly form of an execution_policy."""
+    if pol is None:
+        return None
+    return {
+        "runs_per_item": pol.runs_per_item,
+        "pass_threshold": pol.pass_threshold,
+    }
+
+
+def strip_be_managed_version_tags(
+    tags: Optional[List[str]],
+) -> List[str]:
+    """Drop the BE-managed ``'latest'`` marker so source/target tag lists compare equal.
+
+    The BE auto-injects ``'latest'`` on the newest version of any dataset
+    on read; the migration code filters it out before forwarding to avoid
+    409 conflicts. Tests strip it on both sides for the same reason.
+    """
+    return sorted(t for t in (tags or []) if t != "latest")
+
+
+# ---------------------------------------------------------------------------
+# Cascade seeding (Slice 3: experiment + traces + spans)
+#
+# Tests seed an experiment + its trace data directly via REST. We do this
+# rather than going through ``opik.evaluate`` because the BE-side wire
+# shapes are what the cascade reads from, and we want full control over
+# trace ids, span tree topology, and feedback score payloads.
+# ---------------------------------------------------------------------------
+
+
+def seed_experiment_with_trace_tree(
+    rest_client: OpikApi,
+    *,
+    experiment_name: str,
+    dataset_name: str,
+    dataset_id: str,
+    dataset_version_id: Optional[str],
+    project_name: str,
+    item_ids: List[str],
+    experiment_config: Optional[Dict[str, Any]] = None,
+    experiment_type: str = "regular",
+    evaluation_method: str = "dataset",
+    experiment_tags: Optional[List[str]] = None,
+    spans_per_trace: int = 2,
+    feedback_scores_per_trace: Optional[List[Dict[str, Any]]] = None,
+    per_item_extras: Optional[List[Dict[str, Any]]] = None,
+    optimization_id: Optional[str] = None,
+    trace_environment: Optional[str] = None,
+    span_environment: Optional[str] = None,
+    thread_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Create a source experiment + one trace per ``item_id`` + a small span
+    tree per trace, then attach everything via ``create_experiment_items``.
+
+    Returns a dict the cascade tests assert against:
+
+      {
+        "experiment_id": str,
+        "trace_ids": [str, ...],          # one per item_id, same order
+        "span_ids_by_trace": {trace_id: [root_span_id, child_span_id, ...]},
+        "feedback_scores_by_trace": {trace_id: [score_dicts...]},
+      }
+
+    ``spans_per_trace`` controls the tree size; ``spans_per_trace >= 2``
+    produces a root + child(ren) layout so the cascade has to remap
+    parent_span_id. We deliberately do NOT use ``opik.evaluate`` here -- we
+    want the wire shape, deterministic ids, and to assert on it without
+    flush/streamer timing concerns.
+
+    ``trace_environment`` / ``span_environment`` stamp the ClickHouse
+    ``environment`` column on every seeded trace / span; ``thread_id``
+    groups the traces into one thread so the cascade's env preservation
+    (OPIK-6695) can be round-trip asserted on traces, spans, and the
+    BE-materialized thread row.
+    """
+    import datetime as dt
+    import opik.id_helpers as id_helpers_module
+    from opik.rest_api.types.experiment_item import ExperimentItem
+    from opik.rest_api.types.feedback_score_batch_item import (
+        FeedbackScoreBatchItem,
+    )
+    from opik.rest_api.types.span_write import SpanWrite
+    from opik.rest_api.types.trace_write import TraceWrite
+
+    if spans_per_trace < 1:
+        raise ValueError("spans_per_trace must be >= 1")
+
+    now = dt.datetime.now(dt.timezone.utc)
+
+    trace_ids: List[str] = []
+    span_ids_by_trace: Dict[str, List[str]] = {}
+    feedback_scores_by_trace: Dict[str, List[Dict[str, Any]]] = {}
+
+    trace_writes: List[TraceWrite] = []
+    span_writes: List[SpanWrite] = []
+    feedback_batch: List[FeedbackScoreBatchItem] = []
+
+    for index, item_id in enumerate(item_ids):
+        trace_id = id_helpers_module.generate_id()
+        trace_ids.append(trace_id)
+        trace_writes.append(
+            TraceWrite(
+                id=trace_id,
+                project_name=project_name,
+                name=f"task-{index}",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"item": item_id},
+                output={"answer": f"output-{index}"},
+                metadata={"item_id": item_id},
+                tags=["e2e-cascade"],
+                thread_id=thread_id,
+                environment=trace_environment,
+            )
+        )
+
+        # Span tree: root + (spans_per_trace - 1) children of the root.
+        # Children all parent on the root so the cascade has to remap
+        # parent_span_id at least once.
+        root_span_id = id_helpers_module.generate_id()
+        span_ids_by_trace[trace_id] = [root_span_id]
+        span_writes.append(
+            SpanWrite(
+                id=root_span_id,
+                project_name=project_name,
+                trace_id=trace_id,
+                parent_span_id=None,
+                name=f"root-{index}",
+                type="general",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"item": item_id},
+                output={"answer": f"output-{index}"},
+                environment=span_environment,
+            )
+        )
+        for child_index in range(spans_per_trace - 1):
+            child_span_id = id_helpers_module.generate_id()
+            span_ids_by_trace[trace_id].append(child_span_id)
+            span_writes.append(
+                SpanWrite(
+                    id=child_span_id,
+                    project_name=project_name,
+                    trace_id=trace_id,
+                    parent_span_id=root_span_id,
+                    name=f"llm-call-{index}-{child_index}",
+                    type="llm",
+                    start_time=now + dt.timedelta(milliseconds=1),
+                    end_time=now + dt.timedelta(milliseconds=9),
+                    input={"prompt": "..."},
+                    output={"completion": f"output-{index}"},
+                    model="gpt-mock",
+                    provider="mock",
+                    usage={"prompt_tokens": 5, "completion_tokens": 10},
+                    environment=span_environment,
+                )
+            )
+
+        # Optional: attach feedback scores to the trace.
+        if feedback_scores_per_trace:
+            scores_for_this_trace: List[Dict[str, Any]] = []
+            for score in feedback_scores_per_trace:
+                feedback_batch.append(
+                    FeedbackScoreBatchItem(
+                        id=trace_id,
+                        project_name=project_name,
+                        name=score["name"],
+                        value=score["value"],
+                        reason=score.get("reason"),
+                        source="sdk",
+                    )
+                )
+                scores_for_this_trace.append(score)
+            feedback_scores_by_trace[trace_id] = scores_for_this_trace
+
+    rest_client.traces.create_traces(traces=trace_writes)
+    rest_client.spans.create_spans(spans=span_writes)
+    if feedback_batch:
+        rest_client.traces.score_batch_of_traces(scores=feedback_batch)
+
+    # Create the experiment, then attach experiment items wiring item_id
+    # to trace_id 1:1.
+    import opik.id_helpers as _id_helpers
+
+    new_experiment_id = _id_helpers.generate_id()
+    create_experiment_kwargs: Dict[str, Any] = {
+        "id": new_experiment_id,
+        "name": experiment_name,
+        "dataset_name": dataset_name,
+        "type": experiment_type,
+        "evaluation_method": evaluation_method,
+        "tags": experiment_tags,
+        "metadata": experiment_config,
+        "dataset_version_id": dataset_version_id,
+        "project_name": project_name,
+    }
+    if optimization_id is not None:
+        create_experiment_kwargs["optimization_id"] = optimization_id
+    rest_client.experiments.create_experiment(**create_experiment_kwargs)
+
+    extras_list = per_item_extras or [{} for _ in item_ids]
+    if len(extras_list) != len(item_ids):
+        raise ValueError("per_item_extras must have the same length as item_ids")
+
+    # ``assertion_results`` are persisted via the dedicated
+    # ``assertion_results.store_assertions_batch(entity_type='TRACE', ...)``
+    # endpoint -- the ``ExperimentItem.assertion_results`` field is dropped
+    # silently on the BE Write view (it's READ-ONLY on the Compare view,
+    # computed from the underlying assertion-results entity table). Same
+    # for the other per-item fidelity fields like input/output -- those
+    # are BE-computed read aggregates.
+    #
+    # The seed builds a separate assertion-batch from each item's extras
+    # before constructing the ExperimentItem write (which only carries the
+    # FK fields). This mirrors how the cascade itself writes assertions.
+    from opik.rest_api.types.assertion_result_batch_item import (
+        AssertionResultBatchItem,
+    )
+
+    assertion_batch: List[AssertionResultBatchItem] = []
+    assertion_results_by_trace: Dict[str, List[Dict[str, Any]]] = {}
+    experiment_items_to_create: List[ExperimentItem] = []
+    for item_id, trace_id, extras in zip(item_ids, trace_ids, extras_list):
+        per_item_assertions = extras.get("assertion_results") or []
+        for ar in per_item_assertions:
+            value = (
+                ar.get("value") if isinstance(ar, dict) else getattr(ar, "value", None)
+            )
+            passed = (
+                ar.get("passed")
+                if isinstance(ar, dict)
+                else getattr(ar, "passed", None)
+            )
+            reason = (
+                ar.get("reason")
+                if isinstance(ar, dict)
+                else getattr(ar, "reason", None)
+            )
+            if value is None or passed is None:
+                continue
+            assertion_batch.append(
+                AssertionResultBatchItem(
+                    entity_id=trace_id,
+                    project_name=project_name,
+                    name=value,
+                    status="passed" if passed else "failed",
+                    reason=reason,
+                    source="sdk",
+                )
+            )
+            assertion_results_by_trace.setdefault(trace_id, []).append(
+                {"value": value, "passed": passed, "reason": reason}
+            )
+
+        # The remaining extras are READ-ONLY on the BE; we don't write
+        # them. Forwarding them on the ExperimentItem create payload would
+        # be silently dropped (BE Write view doesn't include them).
+        experiment_items_to_create.append(
+            ExperimentItem(
+                id=_id_helpers.generate_id(),
+                experiment_id=new_experiment_id,
+                dataset_item_id=item_id,
+                trace_id=trace_id,
+            )
+        )
+
+    rest_client.experiments.create_experiment_items(
+        experiment_items=experiment_items_to_create
+    )
+
+    if assertion_batch:
+        rest_client.assertion_results.store_assertions_batch(
+            entity_type="TRACE",
+            assertion_results=assertion_batch,
+        )
+
+    return {
+        "experiment_id": new_experiment_id,
+        "trace_ids": trace_ids,
+        "span_ids_by_trace": span_ids_by_trace,
+        "feedback_scores_by_trace": feedback_scores_by_trace,
+        "assertion_results_by_trace": assertion_results_by_trace,
+    }
+
+
+def find_destination_experiment(
+    rest_client: OpikApi,
+    *,
+    destination_dataset_id: str,
+    experiment_name: str,
+) -> Any:
+    """Locate the cascaded experiment at the destination by name + dataset.
+
+    Returns the ``ExperimentPublic``. Raises if zero or multiple match;
+    the cascade is supposed to recreate one experiment per source
+    experiment, so neither outcome is silently acceptable.
+    """
+    page = rest_client.experiments.find_experiments(
+        dataset_id=destination_dataset_id,
+        page=1,
+        size=100,
+        name=experiment_name,
+    )
+    matched = [e for e in (page.content or []) if e.name == experiment_name]
+    if len(matched) != 1:
+        raise AssertionError(
+            f"expected exactly one destination experiment named "
+            f"{experiment_name!r} under dataset {destination_dataset_id}, "
+            f"got {len(matched)}"
+        )
+    return matched[0]
+
+
+def destination_experiment_items(
+    rest_client: OpikApi,
+    *,
+    experiment_id: str,
+    dataset_id: str,
+) -> List[Any]:
+    """Materialise the destination experiment's items via the Compare view.
+
+    The cascade's source-side read uses
+    ``datasets.find_dataset_items_with_experiment_items`` because only the
+    Compare view surfaces ``assertion_results`` / ``feedback_scores`` /
+    ``input`` / ``output``. We use the same endpoint for destination
+    verification so tests can assert on those fields directly (the slim
+    ``stream_experiment_items`` Public view drops them).
+
+    Returns a flat list of ``ExperimentItemCompare`` -- one per source
+    experiment item.
+    """
+    experiment_ids_filter = json.dumps([experiment_id])
+    collected: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.datasets.find_dataset_items_with_experiment_items(
+            id=dataset_id,
+            experiment_ids=experiment_ids_filter,
+            page=page,
+            size=100,
+        )
+        content = resp.content or []
+        if not content:
+            break
+        for ds_item in content:
+            for ei in ds_item.experiment_items or []:
+                if ei.experiment_id == experiment_id:
+                    collected.append(ei)
+        if len(content) < 100:
+            break
+        page += 1
+    return collected
+
+
+def destination_spans_for_trace(
+    rest_client: OpikApi, *, trace_id: str, project_name: str
+) -> List[Any]:
+    """Read all destination spans for one destination trace, paginating.
+
+    ``get_spans_by_project`` requires ``project_name`` (or ``project_id``)
+    on the request; without it the BE 400s. We pass the destination
+    project name explicitly.
+    """
+    out: List[Any] = []
+    page = 1
+    while True:
+        resp = rest_client.spans.get_spans_by_project(
+            project_name=project_name,
+            trace_id=trace_id,
+            page=page,
+            size=100,
+        )
+        if not resp.content:
+            break
+        out.extend(resp.content)
+        if len(resp.content) < 100:
+            break
+        page += 1
+    return out
+
+
+def destination_feedback_scores_for_trace(
+    rest_client: OpikApi, *, trace_id: str
+) -> List[Any]:
+    """Read feedback scores on a destination trace.
+
+    The trace's ``feedback_scores`` field on read is the authoritative
+    source -- the cascade copies them implicitly because trace metadata
+    isn't the only place they live (per-trace ``feedback_scores`` table).
+    Today's cascade does NOT explicitly re-emit feedback scores; this
+    helper lets a test assert that as an explicit known-gap or as
+    "preserved if and only if the cascade adds the copy".
+    """
+    trace = rest_client.traces.get_trace_by_id(id=trace_id)
+    return list(trace.feedback_scores or [])

sdks/python/tests/e2e/cli/test_migrate_dataset_cross_project_cascade_e2e.py

@@ -0,0 +1,318 @@
+"""End-to-end test for ``opik migrate dataset`` with a cross-project experiment.
+
+``opik.evaluate`` co-locates an experiment's traces in one project, but the
+BE permits ``experiment_items`` rows that point at traces living in a
+*different* project from the experiment's own project. The BE populates
+``experiment_items.project_id`` from ``traces.project_id`` at write time
+(``ExperimentItemService.populateProjectIdFromTraces``), so the rows are
+the authoritative source of truth for "which projects do this experiment's
+traces actually live in".
+
+This test pins the cascade's cross-project contract: regardless of how
+many distinct source projects the experiment's traces are spread across,
+the migration funnels every one of them into the single ``--to-project``
+destination.
+
+Shape (three distinct source projects per experiment, plus the target):
+
+  1. Source dataset lives in project A (``source_project_name``)
+  2. ``seed_experiment_with_trace_tree`` seeds the experiment + 2 traces in A
+  3. We mint a trace + span in project B (``cross_project_name_b``) and
+     another in project C (``cross_project_name_c``), then append two
+     ``experiment_items`` rows pointing the same experiment at them. The
+     experiment now has traces spread across A, B, and C — the user only
+     "sees" A (the experiment's own project) but the cascade must reach
+     into B and C as well.
+  4. Run ``opik migrate dataset ... --to-project=<destination>``
+  5. Assert: destination experiment has all 4 items, every destination
+     trace lives under the destination project, and the source trace ids
+     (across all three source projects) are not reused.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+from pathlib import Path
+from typing import Any, Iterator, List
+
+import pytest
+
+import opik
+import opik.id_helpers as id_helpers_module
+from opik.rest_api.core.api_error import ApiError
+from opik.rest_api.types.experiment_item import ExperimentItem
+from opik.rest_api.types.span_write import SpanWrite
+from opik.rest_api.types.trace_write import TraceWrite
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .. import verifiers
+from .conftest import (
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_spans_for_trace,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+def _make_cross_project(opik_client: opik.Opik, label: str) -> Iterator[str]:
+    name = f"e2e-cli-migrate-cross-{label}-{random_chars()}"
+    rest = opik_client.rest_client
+    rest.projects.create_project(name=name)
+    yield name
+    try:
+        project_id = rest.projects.retrieve_project(name=name).id
+        rest.projects.delete_project_by_id(project_id)
+    except ApiError:
+        pass
+
+
+@pytest.fixture
+def cross_project_name_b(opik_client: opik.Opik) -> Iterator[str]:
+    yield from _make_cross_project(opik_client, "b")
+
+
+@pytest.fixture
+def cross_project_name_c(opik_client: opik.Opik) -> Iterator[str]:
+    yield from _make_cross_project(opik_client, "c")
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-cross-{random_chars()}"
+
+
+def _mint_cross_project_trace(
+    rest: Any,
+    *,
+    project_name: str,
+    label: str,
+) -> str:
+    """Create one trace + one root span in ``project_name`` and return the trace id."""
+    now = dt.datetime.now(dt.timezone.utc)
+    trace_id = id_helpers_module.generate_id()
+    span_id = id_helpers_module.generate_id()
+    rest.traces.create_traces(
+        traces=[
+            TraceWrite(
+                id=trace_id,
+                project_name=project_name,
+                name=f"task-cross-{label}",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"q": f"Q-cross-{label}"},
+                output={"answer": f"from-{label}"},
+                tags=["e2e-cross"],
+            )
+        ]
+    )
+    rest.spans.create_spans(
+        spans=[
+            SpanWrite(
+                id=span_id,
+                project_name=project_name,
+                trace_id=trace_id,
+                parent_span_id=None,
+                name=f"root-{label}",
+                type="general",
+                start_time=now,
+                end_time=now + dt.timedelta(milliseconds=10),
+                input={"q": f"Q-cross-{label}"},
+                output={"answer": f"from-{label}"},
+            )
+        ]
+    )
+    return trace_id
+
+
+def test_migrate_dataset__cross_project_experiment__all_traces_land_in_target(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    cross_project_name_b: str,
+    cross_project_name_c: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset in project A ──
+    # All items (same-project + both cross-project) are added to a single
+    # v1 so the experiment's items all resolve under the same dataset
+    # version -- mixing experiment_items rows against different dataset
+    # versions causes the Compare endpoint to surface incomplete rows
+    # (missing ``source``). That's a known corner case unrelated to the
+    # cross-project plumbing under test, so we sidestep it by seeding a
+    # single version.
+    from opik import id_helpers
+    from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+    source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+            DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+            DatasetItemWrite(source="manual", data={"q": "Q-cross-b", "a": "A-b"}),
+            DatasetItemWrite(source="manual", data={"q": "Q-cross-c", "a": "A-c"}),
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[0]
+
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+    same_project_item_ids = [by_q["Q1"], by_q["Q2"]]
+    cross_b_item_id = by_q["Q-cross-b"]
+    cross_c_item_id = by_q["Q-cross-c"]
+    assert all(same_project_item_ids) and cross_b_item_id and cross_c_item_id
+
+    # ── Seed the experiment with same-project traces in A ──
+    experiment_name = f"e2e-cross-{random_chars()}"
+    cascade_seed = seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=experiment_name,
+        dataset_name=dataset_name,
+        dataset_id=source_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=same_project_item_ids,
+        spans_per_trace=2,
+    )
+    source_experiment_id = cascade_seed["experiment_id"]
+    same_project_trace_ids = cascade_seed["trace_ids"]
+
+    # ── Mint cross-project traces in two distinct projects (B and C) ──
+    # The experiment now has traces in 3 projects total: A (same-project,
+    # 2 traces), B (1 trace), and C (1 trace). The user only "sees" A as
+    # the experiment's project; the cascade has to discover B and C via
+    # ``streamExperimentItems``' per-item ``project_id`` field and issue a
+    # ``search_traces`` / ``search_spans`` against each.
+    cross_b_trace_id = _mint_cross_project_trace(
+        rest, project_name=cross_project_name_b, label="b"
+    )
+    cross_c_trace_id = _mint_cross_project_trace(
+        rest, project_name=cross_project_name_c, label="c"
+    )
+
+    rest.experiments.create_experiment_items(
+        experiment_items=[
+            ExperimentItem(
+                id=id_helpers_module.generate_id(),
+                experiment_id=source_experiment_id,
+                dataset_item_id=cross_b_item_id,
+                trace_id=cross_b_trace_id,
+            ),
+            ExperimentItem(
+                id=id_helpers_module.generate_id(),
+                experiment_id=source_experiment_id,
+                dataset_item_id=cross_c_item_id,
+                trace_id=cross_c_trace_id,
+            ),
+        ]
+    )
+
+    # ── Run the migration to the destination project ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify destination ─────────────────────────────────────────────
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=experiment_name,
+    )
+
+    # All 4 source experiment items should round-trip (2 same-project +
+    # 1 from project B + 1 from project C).
+    expected_count = len(same_project_trace_ids) + 2
+    dest_items = destination_experiment_items(
+        rest, experiment_id=dest_exp.id, dataset_id=dest_dataset.id
+    )
+    assert len(dest_items) == expected_count, (
+        f"expected {expected_count} destination experiment items "
+        f"(same-project + 2 cross-project), got {len(dest_items)}"
+    )
+
+    # Every destination trace must live in the target project — including
+    # the ones that originally lived in project B and project C. This is
+    # the contract the cross-project plumbing protects: without
+    # ``_discover_trace_projects``, the search_traces call would scope to
+    # the experiment's own project (A) only and both cross-project traces
+    # would be silently dropped, leaving 2 fewer items at the destination.
+    dest_trace_ids: List[str] = []
+    for it in dest_items:
+        assert it.trace_id is not None, "destination experiment item missing trace_id"
+        dest_trace_ids.append(it.trace_id)
+        verifiers.verify_trace(
+            opik_client=opik_client,
+            trace_id=it.trace_id,
+            project_name=target_project_name,
+        )
+
+    # Per-span fidelity per source project: spans live under the same project
+    # as their parent trace and are discovered+read by the cascade in the
+    # same per-project loop as traces. So the cross-project safety we just
+    # verified for traces only holds if the per-project ``search_spans``
+    # loop also fired for B and C; if it didn't, the cross-project traces
+    # would land at the destination but with zero spans attached.
+    #
+    # Trace ``name`` is the stable handle for classifying which source
+    # project a destination trace came from:
+    #   * ``task-N``        -> same-project (A), seeded with root + 1 LLM child
+    #   * ``task-cross-b``  -> cross-project from B, seeded with 1 root span
+    #   * ``task-cross-c``  -> cross-project from C, seeded with 1 root span
+    expected_spans_by_trace_name = {"task-cross-b": 1, "task-cross-c": 1}
+    for i in range(len(same_project_trace_ids)):
+        expected_spans_by_trace_name[f"task-{i}"] = 2  # root + 1 LLM child
+
+    for it in dest_items:
+        dest_trace = rest.traces.get_trace_by_id(id=it.trace_id)
+        assert dest_trace.name in expected_spans_by_trace_name, (
+            f"unexpected destination trace name {dest_trace.name!r} -- the "
+            f"cross-project trace naming contract changed and this test "
+            f"doesn't know which source project to attribute it to"
+        )
+        dest_spans = destination_spans_for_trace(
+            rest, trace_id=it.trace_id, project_name=target_project_name
+        )
+        expected = expected_spans_by_trace_name[dest_trace.name]
+        assert len(dest_spans) == expected, (
+            f"destination trace {dest_trace.name!r} should have {expected} "
+            f"span(s) at the destination, got {len(dest_spans)} -- if 0 for "
+            f"a cross-project trace, the per-project search_spans loop didn't "
+            f"fire for that source project"
+        )
+
+    # Fresh destination ids — migrate is copy-not-move. No source trace id
+    # (from any of the 3 source projects) may be reused at the destination.
+    source_trace_ids = set(same_project_trace_ids) | {
+        cross_b_trace_id,
+        cross_c_trace_id,
+    }
+    overlap = source_trace_ids & set(dest_trace_ids)
+    assert not overlap, (
+        f"destination trace ids must be fresh, but found source ids reused: {overlap}"
+    )

sdks/python/tests/e2e/cli/test_migrate_dataset_e2e.py

@@ -0,0 +1,483 @@
+"""End-to-end tests for ``opik migrate dataset`` against a real Opik backend.
+
+Covers the plain-dataset path: full version replay, plus the experiment +
+trace + span cascade that rides along with the dataset. The test-suite
+path lives in ``test_migrate_test_suite_e2e.py``.
+
+Each test:
+
+1. Seeds a multi-version source dataset directly via the REST API (so we
+   control every per-version delta the migration has to replay) and,
+   where the test calls for it, an experiment + traces + spans attached
+   to one of the source versions
+2. Runs ``opik migrate dataset`` as a subprocess so the actual CLI
+   entrypoint, Click group, and exit-code handling are exercised
+3. Reads back the target via the raw REST stream + wire type (the SDK
+   helper drops per-item tags) and asserts per-version content +
+   display-order fidelity, plus -- where relevant -- destination
+   experiment + trace + span fidelity and FK remapping
+
+Shared helpers live in ``conftest.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+from opik import synchronization
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from ._cascade_comparison import compare_cascade
+from .conftest import (
+    apply_changes,
+    chronological_versions,
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_feedback_scores_for_trace,
+    destination_spans_for_trace,
+    display_order,
+    find_destination_experiment,
+    item_hashes,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+# Per ``sdks/python/AGENTS.md``: every e2e module sources PROJECT_NAME from
+# ``generate_project_name("e2e", __name__)`` so backend project names are
+# isolated per test module + the autouse ``configure_e2e_tests_env`` fixture
+# can patch ``OPIK_PROJECT_NAME`` to match.
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-{random_chars()}"
+
+
+class TestMigrateDatasetVersionReplay:
+    """Default ``opik migrate dataset`` flow: full version-history replay.
+
+    Pin the slice 2 contract: target version count == source version
+    count, per-version content set-equal under hash, display order
+    preserved at every version.
+    """
+
+    def test_three_version_dataset_with_mixed_deltas_round_trips(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        dataset_name: str,
+        tmp_path: Path,
+    ) -> None:
+        rest = opik_client.rest_client
+
+        # Seed: 3 versions exercising every delta kind.
+        # v1: adds Q1, Q2, Q3.
+        # v2: edits Q1 (data change), adds Q4.
+        # v3: edits Q3 (data change), deletes Q2, adds Q5.
+        source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+
+        # v1 of a plain dataset is created via the REST insert path (mirrors
+        # what ``Dataset.insert`` does at higher level). Use create_or_update
+        # so we get exactly one BE version with all three items.
+        from opik import id_helpers
+        from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+        rest.datasets.create_or_update_dataset_items(
+            dataset_id=source_id,
+            items=[
+                DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+                DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+                DatasetItemWrite(source="manual", data={"q": "Q3", "a": "A3"}),
+            ],
+            batch_group_id=id_helpers.generate_id(),
+        )
+        v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[
+            0
+        ]
+        v1_items = stream_items_wire(
+            rest,
+            dataset_name=dataset_name,
+            project_name=source_project_name,
+            version_hash=v1.version_hash,
+        )
+        by_q = {item.data["q"]: item for item in v1_items if item.data}
+
+        # v2: edit Q1's answer + add Q4.
+        v2_id = apply_changes(
+            rest,
+            source_id,
+            base_version_id=v1.id,
+            edited_items=[
+                {"id": by_q["Q1"].id, "data": {"q": "Q1", "a": "A1-EDITED"}},
+            ],
+            added_items=[{"data": {"q": "Q4", "a": "A4"}, "source": "manual"}],
+            change_description="v2 — edit Q1, add Q4",
+        )
+
+        # v3: edit Q3, delete Q2, add Q5.
+        apply_changes(
+            rest,
+            source_id,
+            base_version_id=v2_id,
+            edited_items=[
+                {"id": by_q["Q3"].id, "data": {"q": "Q3", "a": "A3-EDITED"}},
+            ],
+            deleted_ids=[by_q["Q2"].id],
+            added_items=[{"data": {"q": "Q5", "a": "A5"}, "source": "manual"}],
+            change_description="v3 — delete Q2, edit Q3, add Q5",
+        )
+
+        # Snapshot source expectations per version.
+        src_versions = chronological_versions(rest, source_id)
+        assert len(src_versions) == 3
+        expected_hashes = []
+        expected_orders = []
+        for v in src_versions:
+            items = stream_items_wire(
+                rest,
+                dataset_name=dataset_name,
+                project_name=source_project_name,
+                version_hash=v.version_hash,
+            )
+            expected_hashes.append(item_hashes(items))
+            expected_orders.append(display_order(items))
+
+        # ── Seed an experiment on v1 items so the cascade has something to
+        # round-trip. Regular-dataset experiments carry per-trace feedback
+        # scores (test suites carry assertion_results -- covered in
+        # test_migrate_test_suite_e2e.py). Each item gets one trace with a
+        # root + 1 LLM child span and a feedback score on the trace. ──
+        experiment_name = f"e2e-exp-{random_chars()}"
+        v1_item_ids = [by_q["Q1"].id, by_q["Q2"].id, by_q["Q3"].id]
+        cascade_seed = seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=experiment_name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=v1_item_ids,
+            experiment_config={"runner": "e2e-cascade-test"},
+            experiment_tags=["e2e", "cascade"],
+            spans_per_trace=2,  # root + 1 LLM child -> exercises parent_span_id remap
+            feedback_scores_per_trace=[
+                {"name": "correctness", "value": 0.9, "reason": "matches reference"},
+                {"name": "latency_p95", "value": 230.5},
+            ],
+        )
+
+        # Run the migration.
+        audit_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            [
+                "dataset",
+                dataset_name,
+                "--to-project",
+                target_project_name,
+            ],
+            audit_log_path=str(audit_path),
+        )
+        assert result.returncode == 0, result.stdout + result.stderr
+
+        # Verify target: same version count, per-version content set-equal
+        # under hash, display order matches at every version.
+        target = rest.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name, project_name=target_project_name
+        )
+        tgt_versions = chronological_versions(rest, target.id)
+        assert len(tgt_versions) == len(src_versions), (
+            f"target version count {len(tgt_versions)} != source {len(src_versions)} "
+            "— Slice 2 contract requires N=N"
+        )
+
+        for src_v, tgt_v, exp_hashes, exp_order in zip(
+            src_versions, tgt_versions, expected_hashes, expected_orders
+        ):
+            items = stream_items_wire(
+                rest,
+                dataset_name=dataset_name,
+                project_name=target_project_name,
+                version_hash=tgt_v.version_hash,
+            )
+            actual_hashes = item_hashes(items)
+            actual_order = display_order(items)
+            assert actual_hashes == exp_hashes, (
+                f"version {tgt_v.version_name}: target items don't match source. "
+                f"Missing on target: {exp_hashes - actual_hashes}; "
+                f"extra on target: {actual_hashes - exp_hashes}"
+            )
+            assert actual_order == exp_order, (
+                f"version {tgt_v.version_name}: display order diverged "
+                f"(source: {exp_order}, target: {actual_order})"
+            )
+
+        # Audit log records one per-version entry per replayed source version.
+        audit = json.loads(audit_path.read_text())
+        per_version_records = [
+            a for a in audit["actions"] if a["type"] == "replay_dataset_version"
+        ]
+        assert len(per_version_records) == len(src_versions)
+        # Per-version deltas: v1=(3 adds), v2=(1 add, 1 mod), v3=(1 add, 1 mod, 1 del).
+        assert (
+            per_version_records[0]["items_added"],
+            per_version_records[0]["items_modified"],
+            per_version_records[0]["items_deleted"],
+        ) == (3, 0, 0)
+        assert (
+            per_version_records[1]["items_added"],
+            per_version_records[1]["items_modified"],
+            per_version_records[1]["items_deleted"],
+        ) == (1, 1, 0)
+        assert (
+            per_version_records[2]["items_added"],
+            per_version_records[2]["items_modified"],
+            per_version_records[2]["items_deleted"],
+        ) == (1, 1, 1)
+
+        # ── Cascade fidelity ──
+        # The destination project should now have a copy of the source
+        # experiment with: a remapped dataset_version_id, fresh item ids
+        # carrying remapped trace ids, traces+spans landing under the
+        # destination project, feedback scores re-emitted on the destination
+        # traces, and per-item write-side fidelity (input/output) preserved.
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=target.id,
+            experiment_name=experiment_name,
+        )
+        # FKs remapped.
+        assert dest_exp.id != cascade_seed["experiment_id"]
+        assert dest_exp.dataset_id == target.id
+        # The destination experiment must reference one of the target
+        # versions (the cascade picks the remap of v1).
+        target_version_ids = {v.id for v in tgt_versions}
+        assert dest_exp.dataset_version_id in target_version_ids
+
+        # Items: one per source item, with FRESH trace ids (disjoint from
+        # source). Per-item input/output/usage/cost are READ-ONLY on the BE
+        # (computed/aggregated from the underlying trace + span entities);
+        # we assert the trace + span fidelity below instead.
+        dest_items = destination_experiment_items(
+            rest,
+            experiment_id=dest_exp.id,
+            dataset_id=target.id,
+        )
+        assert len(dest_items) == len(v1_item_ids)
+        dest_trace_ids = {it.trace_id for it in dest_items}
+        assert dest_trace_ids.isdisjoint(set(cascade_seed["trace_ids"])), (
+            "destination experiment items should reference new trace ids, "
+            "not the source's"
+        )
+
+        # Each destination trace exists under the target project and has the
+        # same span shape as the source (root + 1 child = 2 spans).
+        for new_trace_id in dest_trace_ids:
+            dest_spans = destination_spans_for_trace(
+                rest,
+                trace_id=new_trace_id,
+                project_name=target_project_name,
+            )
+            assert len(dest_spans) == 2, (
+                f"trace {new_trace_id} should have 2 spans (root + child), "
+                f"got {len(dest_spans)}"
+            )
+            # Topological remap: exactly one root (parent_span_id=None),
+            # the other span points at the root via parent_span_id.
+            roots = [s for s in dest_spans if s.parent_span_id is None]
+            assert len(roots) == 1, f"trace {new_trace_id} should have one root span"
+            children = [s for s in dest_spans if s.parent_span_id is not None]
+            assert all(c.parent_span_id == roots[0].id for c in children), (
+                f"trace {new_trace_id} child spans should remap parent_span_id "
+                "to the new root id"
+            )
+
+            # Trace-level feedback scores re-emitted on the destination trace.
+            dest_scores = destination_feedback_scores_for_trace(
+                rest, trace_id=new_trace_id
+            )
+            score_names = {s.name for s in dest_scores}
+            assert score_names == {"correctness", "latency_p95"}, (
+                f"trace {new_trace_id}: expected feedback score names "
+                f"{{'correctness', 'latency_p95'}}, got {score_names}"
+            )
+
+        # ── Deep-equal source vs. destination ──
+        # Verify field-by-field that experiment + items + traces + spans
+        # round-trip the cascade modulo remapped IDs. Pairing strategy:
+        # both sides sorted by trace ``name`` (assigned by the seed as
+        # "task-0", "task-1", "task-2" and carried verbatim through the
+        # cascade), guaranteeing stable positional correspondence.
+        src_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=source_id,
+            experiment_name=experiment_name,
+        )
+        src_items_compare = destination_experiment_items(
+            rest,
+            experiment_id=cascade_seed["experiment_id"],
+            dataset_id=source_id,
+        )
+        # Sort both sides by trace name for stable pairing. Build a
+        # trace_id -> name map by reading each trace once.
+        src_trace_names = {
+            it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+            for it in src_items_compare
+        }
+        dst_trace_names = {
+            it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+            for it in dest_items
+        }
+        src_items_compare.sort(key=lambda it: src_trace_names[it.trace_id])
+        dest_items_sorted = sorted(
+            dest_items, key=lambda it: dst_trace_names[it.trace_id]
+        )
+        src_trace_ids_sorted = [it.trace_id for it in src_items_compare]
+        dst_trace_ids_sorted = [it.trace_id for it in dest_items_sorted]
+
+        compare_cascade(
+            rest_client=rest,
+            source_experiment=src_exp,
+            destination_experiment=dest_exp,
+            source_item_ids=v1_item_ids,
+            destination_item_ids=[it.dataset_item_id for it in dest_items_sorted],
+            source_trace_ids=src_trace_ids_sorted,
+            destination_trace_ids=dst_trace_ids_sorted,
+            source_items_compare=src_items_compare,
+            destination_items_compare=dest_items_sorted,
+        )
+
+
+class TestMigrateDatasetEnvironmentPreservation:
+    """OPIK-6695: the cascade must preserve the ``environment`` column on
+    traces, spans, and the BE-materialized ``trace_threads`` row.
+
+    Pre-2026-05-07 rows default to ``''`` in ClickHouse and replay
+    trivially; the regression this guards is the post-migration reset of a
+    non-empty ``environment`` to ``''`` because the re-emit payload didn't
+    carry the field.
+    """
+
+    def test_environment_round_trips_on_traces_spans_and_threads(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        dataset_name: str,
+        tmp_path: Path,
+    ) -> None:
+        from opik import id_helpers
+        from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+        rest = opik_client.rest_client
+
+        # Single-version dataset with two items -> two cascaded traces.
+        source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+        rest.datasets.create_or_update_dataset_items(
+            dataset_id=source_id,
+            items=[
+                DatasetItemWrite(source="manual", data={"q": "Q1", "a": "A1"}),
+                DatasetItemWrite(source="manual", data={"q": "Q2", "a": "A2"}),
+            ],
+            batch_group_id=id_helpers.generate_id(),
+        )
+        v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[
+            0
+        ]
+        v1_items = stream_items_wire(
+            rest,
+            dataset_name=dataset_name,
+            project_name=source_project_name,
+            version_hash=v1.version_hash,
+        )
+        item_ids = [it.id for it in v1_items]
+
+        # Seed: traces tagged environment="production" + grouped into one
+        # thread; spans tagged environment="staging". The trace env and
+        # span env differ deliberately so a single shared value couldn't
+        # mask a per-entity bug, and the thread inherits the trace env.
+        experiment_name = f"e2e-env-{random_chars()}"
+        thread_id = f"env-thread-{random_chars()}"
+        seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=experiment_name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=item_ids,
+            spans_per_trace=2,
+            trace_environment="production",
+            span_environment="staging",
+            thread_id=thread_id,
+        )
+
+        audit_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["dataset", dataset_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_path),
+        )
+        assert result.returncode == 0, result.stdout + result.stderr
+
+        target = rest.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name, project_name=target_project_name
+        )
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=target.id,
+            experiment_name=experiment_name,
+        )
+        dest_items = destination_experiment_items(
+            rest,
+            experiment_id=dest_exp.id,
+            dataset_id=target.id,
+        )
+        assert len(dest_items) == len(item_ids)
+
+        # (a) traces and (b) spans keep their source environment verbatim.
+        for dest_item in dest_items:
+            dest_trace = rest.traces.get_trace_by_id(id=dest_item.trace_id)
+            assert dest_trace.environment == "production", (
+                f"trace {dest_item.trace_id} lost environment: "
+                f"got {dest_trace.environment!r}"
+            )
+            dest_spans = destination_spans_for_trace(
+                rest,
+                trace_id=dest_item.trace_id,
+                project_name=target_project_name,
+            )
+            assert dest_spans, f"trace {dest_item.trace_id} has no destination spans"
+            assert all(span.environment == "staging" for span in dest_spans), (
+                "destination spans lost environment: "
+                f"{[span.environment for span in dest_spans]}"
+            )
+
+        # (c) the destination thread row -- materialized by the BE from the
+        # cascaded traces -- inherits the same environment. Polls because
+        # thread materialization is eventually consistent.
+        assert synchronization.until(
+            lambda: bool(
+                opik_client.search_threads(
+                    project_name=target_project_name,
+                    filter_string=f'id = "{thread_id}"',
+                )
+            ),
+            max_try_seconds=30,
+        ), f"destination thread {thread_id!r} never materialized"
+        threads = opik_client.search_threads(
+            project_name=target_project_name,
+            filter_string=f'id = "{thread_id}"',
+        )
+        assert len(threads) == 1
+        assert threads[0].environment == "production", (
+            f"destination thread {thread_id!r} lost environment: "
+            f"got {threads[0].environment!r}"
+        )

sdks/python/tests/e2e/cli/test_migrate_dataset_evaluate_shape_e2e.py

@@ -0,0 +1,467 @@
+"""End-to-end test for ``opik migrate dataset`` using the natural
+``opik.evaluate(...)`` flow as seeding (rather than direct REST writes).
+
+Complements ``test_migrate_dataset_e2e.py`` (precise per-version delta
+coverage via REST seeding) by exercising the realistic shape that real
+users produce:
+
+  1. Create a dataset, ``dataset.insert(...)`` a first batch of items
+  2. Run ``opik.evaluate(...)`` (experiment E1) under the dataset's project
+  3. Insert more items
+  4. Run ``opik.evaluate(...)`` (experiment E2) -- E1 saw 2 items, E2 sees 3
+  5. ``opik migrate dataset ... --to-project=<dest>``
+  6. Verify both experiments + items + traces + spans + feedback scores
+     round-trip with fresh destination ids
+
+In Opik V2 ``opik.evaluate(...)`` always lands its traces in the dataset's
+project (the per-evaluate ``project_name`` override is deprecated). The
+per-trace project_id scoping that ``_copy_traces_and_spans`` does is
+defense-in-depth for BE shapes the SDK doesn't produce today; that
+contract is covered by the unit test
+``test_span_read_uses_trace_project_not_experiment_project``.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any, Callable, Dict, Iterator
+from unittest import mock
+
+import pytest
+
+import opik
+from opik.evaluation.metrics import score_result
+
+from ...testlib import generate_project_name
+from .. import verifiers
+from .conftest import (
+    destination_experiment_items,
+    find_destination_experiment,
+    run_migrate_cli,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+# ``dataset_name`` and ``experiment_name`` fixtures come from
+# ``tests/e2e/conftest.py`` (shared per AGENTS.md). For the second
+# experiment we derive a sibling name off the standard fixture rather
+# than introducing a parallel per-test fixture.
+
+
+@pytest.fixture
+def experiment_name_two(experiment_name: str) -> Iterator[str]:
+    yield f"{experiment_name}-second"
+
+
+def _llm_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """Deterministic stand-in for an LLM call.
+
+    Returns ``"Paris"`` regardless of input so the equality scorer
+    produces a deterministic 1.0/0.0 split based on the dataset items'
+    expected outputs.
+    """
+    return {"output": "Paris"}
+
+
+def _equals_score(
+    dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]
+) -> score_result.ScoreResult:
+    reference = dataset_item.get("expected", {}).get("output")
+    prediction = task_outputs["output"]
+    value = 1.0 if reference == prediction else 0.0
+    return score_result.ScoreResult(
+        name="equals_scoring_function",
+        value=value,
+        reason="match" if value == 1.0 else "mismatch",
+    )
+
+
+def _wait_until(
+    predicate: Callable[[], Any],
+    timeout_seconds: int = 30,
+    poll_seconds: float = 1.0,
+) -> Any:
+    """Poll ``predicate()`` until it returns truthy or the timeout elapses.
+
+    The streamer flushes traces / spans / feedback scores asynchronously,
+    so we poll the BE for the artifacts we care about rather than racing
+    a hardcoded ``sleep()``. Returns the predicate's final value so the
+    caller can capture e.g. the experiment items in one go.
+    """
+    deadline = time.time() + timeout_seconds
+    last: Any = None
+    while time.time() < deadline:
+        last = predicate()
+        if last:
+            return last
+        time.sleep(poll_seconds)
+    return last
+
+
+def test_migrate_dataset__evaluate_shape__round_trips(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    experiment_name: str,
+    experiment_name_two: str,
+    tmp_path: Path,
+) -> None:
+    # ── Seed via the natural opik.evaluate(...) flow ──
+    # Dataset and both evaluate() runs land in source_project_name. We
+    # add items between runs so E1 sees 2 items and E2 sees 3 -- this
+    # exercises the "experiments reference different dataset version
+    # snapshots" case during migration.
+    dataset = opik_client.create_dataset(dataset_name, project_name=source_project_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of France?"},
+                "expected": {"output": "Paris"},
+            },
+            {
+                "input": {"question": "Capital of Italy?"},
+                "expected": {"output": "Rome"},
+            },
+        ]
+    )
+
+    # Experiment 1 -- 2 items.
+    eval1 = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name,
+        experiment_config={"phase": "first-eval"},
+    )
+
+    # Add a third item between runs.
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of Spain?"},
+                "expected": {"output": "Madrid"},
+            },
+        ]
+    )
+
+    # Experiment 2 -- 3 items.
+    eval2 = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name_two,
+        experiment_config={"phase": "second-eval"},
+    )
+
+    # Flush the SDK streamer so all traces / spans / feedback scores land
+    # on the BE before the migration reads them.
+    opik.flush_tracker()
+
+    # Belt-and-suspenders: poll until both experiments are visible on the
+    # BE (the streamer's flush returns once the local queue drains, but
+    # the BE's eventual-consistency window can lag a beat).
+    rest = opik_client.rest_client
+
+    def _experiments_ready() -> bool:
+        e1 = rest.experiments.get_experiment_by_id(id=eval1.experiment_id)
+        e2 = rest.experiments.get_experiment_by_id(id=eval2.experiment_id)
+        return e1 is not None and e2 is not None
+
+    assert _wait_until(_experiments_ready), (
+        "experiments not visible on BE after flush; streamer may have failed"
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify the destination via the shared verifier layer ─────────────
+    # ``verifiers.verify_experiment`` / ``verify_trace`` have built-in
+    # ``synchronization.until`` retry loops for the BE's eventual-consistency
+    # window right after the migrate completes -- spurious failures from
+    # not-yet-readable rows are eliminated.
+    #
+    # Two name->id lookups remain: the destination dataset id (cascade keeps
+    # the source name at the destination after the rename) and each
+    # destination experiment id (cascade is name-preserving on experiments,
+    # so we resolve by name). Everything else routes through verifiers.
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_dataset_id = dest_dataset.id
+
+    dest_e1 = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset_id,
+        experiment_name=experiment_name,
+    )
+    dest_e2 = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset_id,
+        experiment_name=experiment_name_two,
+    )
+
+    # Fresh destination ids -- migrate is copy-not-move; new experiments
+    # must have new ids.
+    assert dest_e1.id != eval1.experiment_id, (
+        "destination experiment 1 must have a fresh id, not the source's"
+    )
+    assert dest_e2.id != eval2.experiment_id, (
+        "destination experiment 2 must have a fresh id, not the source's"
+    )
+
+    # Experiment-level shape via the shared verifier (with eventual-
+    # consistency retry baked in). ``recreate_experiment`` injects
+    # ``project_name`` into the destination experiment's metadata
+    # (intentional -- it's recorded so future imports/migrations can
+    # re-derive the project context). The verifier's metadata check is
+    # exact-equality, so include it in the expected shape.
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=dest_e1.id,
+        experiment_name=experiment_name,
+        experiment_metadata={
+            "phase": "first-eval",
+            "project_name": target_project_name,
+        },
+        feedback_scores_amount=1,  # the equals scorer emits one aggregate
+        traces_amount=2,
+        project_name=target_project_name,
+    )
+    verifiers.verify_experiment(
+        opik_client=opik_client,
+        id=dest_e2.id,
+        experiment_name=experiment_name_two,
+        experiment_metadata={
+            "phase": "second-eval",
+            "project_name": target_project_name,
+        },
+        feedback_scores_amount=1,
+        traces_amount=3,
+        project_name=target_project_name,
+    )
+
+    # Per-item shape: every destination experiment item has a fresh
+    # trace_id + dataset_item_id, and the trace exists under the
+    # destination project (verified via ``verify_trace`` for its built-in
+    # eventual-consistency retry). The trace's feedback_scores value /
+    # reason varies per item (the equals scorer's mismatch produces 0.0
+    # for some items and 1.0 for others), so per-trace feedback scores
+    # aren't asserted by exact value here -- ``verify_experiment(
+    # feedback_scores_amount=1)`` above already pins that the cascade
+    # re-emitted the equals_scoring_function at the experiment-aggregate
+    # level.
+    dest_e1_items = destination_experiment_items(
+        rest, experiment_id=dest_e1.id, dataset_id=dest_dataset_id
+    )
+    dest_e2_items = destination_experiment_items(
+        rest, experiment_id=dest_e2.id, dataset_id=dest_dataset_id
+    )
+    for it in dest_e1_items + dest_e2_items:
+        assert it.trace_id is not None, "destination experiment item missing trace_id"
+        assert it.dataset_item_id is not None, (
+            "destination item missing dataset_item_id"
+        )
+        verifiers.verify_trace(
+            opik_client=opik_client,
+            trace_id=it.trace_id,
+            project_name=target_project_name,
+        )
+
+
+def test_migrate_dataset__cascade_trace_and_span_comments__round_trip(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    experiment_name: str,
+    tmp_path: Path,
+) -> None:
+    """Slice 4 (OPIK-6417): comments on traces + spans survive the cascade.
+
+    Seeds an evaluate() run, POSTs a handful of comments on one of the
+    resulting traces + one of its spans, runs the migrate, and asserts
+    the destination trace + span carry the same comments in the same
+    order on read-back. ``add_trace_comment`` / ``add_span_comment`` are
+    the only write surface (comments are ``READ_ONLY`` on the trace/span
+    Write payload); the cascade re-emits them via those endpoints after
+    the destination trace/span lands.
+    """
+    rest = opik_client.rest_client
+
+    # ── Seed ──
+    dataset = opik_client.create_dataset(dataset_name, project_name=source_project_name)
+    dataset.insert(
+        [
+            {
+                "input": {"question": "Capital of France?"},
+                "expected": {"output": "Paris"},
+            },
+        ]
+    )
+    eval_result = opik.evaluate(
+        dataset=dataset,
+        task=_llm_task,
+        scoring_functions=[_equals_score],
+        experiment_name=experiment_name,
+        experiment_config={"phase": "comments-cascade"},
+    )
+    opik.flush_tracker()
+
+    def _experiment_ready() -> bool:
+        return (
+            rest.experiments.get_experiment_by_id(id=eval_result.experiment_id)
+            is not None
+        )
+
+    assert _wait_until(_experiment_ready), (
+        "experiment not visible on BE after flush; streamer may have failed"
+    )
+
+    # Pick one source trace + one of its spans to attach comments to. The
+    # cascade reads ``comments`` off the bulk trace/span read, so the
+    # specific id we attach to doesn't matter -- any trace/span in the
+    # experiment is fine.
+    source_traces = _wait_until(
+        lambda: list(
+            opik_client.search_traces(
+                project_name=source_project_name,
+                filter_string=f'experiment_id = "{eval_result.experiment_id}"',
+                max_results=10,
+                truncate=False,
+            )
+        )
+    )
+    assert source_traces, "no source traces visible to seed comments on"
+    target_trace = source_traces[0]
+    source_spans = _wait_until(
+        lambda: list(
+            opik_client.search_spans(
+                project_name=source_project_name,
+                trace_id=target_trace.id,
+                max_results=10,
+                truncate=False,
+            )
+        )
+    )
+    assert source_spans, "no source spans visible to seed comments on"
+    target_span = source_spans[0]
+
+    # POST trace + span comments in a deterministic order so we can assert
+    # round-trip ordering at the destination.
+    trace_comment_texts = [
+        "qa-note: golden path verified",
+        "pm-note: tracked in OPS-1234",
+        "debug-note: see screenshot in #incidents",
+    ]
+    span_comment_texts = [
+        "first span observation",
+        "second span observation",
+    ]
+    for text in trace_comment_texts:
+        rest.traces.add_trace_comment(id_=target_trace.id, text=text)
+    for text in span_comment_texts:
+        rest.spans.add_span_comment(id_=target_span.id, text=text)
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify the destination trace + span round-trip the comments ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_experiment = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=experiment_name,
+    )
+
+    dest_items = _wait_until(
+        lambda: destination_experiment_items(
+            rest,
+            experiment_id=dest_experiment.id,
+            dataset_id=dest_dataset.id,
+        )
+    )
+    assert dest_items, "no destination experiment items after migrate"
+    dest_trace_id = dest_items[0].trace_id
+    assert dest_trace_id is not None
+
+    # Verify the destination trace's comments via the shared verifier
+    # (it polls through ``_retry_until_assertions_pass`` for BE eventual
+    # consistency, so we don't need a separate hand-rolled wait).
+    verifiers.verify_trace(
+        opik_client=opik_client,
+        trace_id=dest_trace_id,
+        project_name=target_project_name,
+        comments=trace_comment_texts,
+    )
+
+    # Span comments: the cascade mints fresh span ids, so we resolve the
+    # destination span via name match on the source ``target_span``
+    # before delegating to the shared verifier for the content check.
+    dest_spans = _wait_until(
+        lambda: list(
+            opik_client.search_spans(
+                project_name=target_project_name,
+                trace_id=dest_trace_id,
+                max_results=50,
+                truncate=False,
+            )
+        )
+    )
+    assert dest_spans, "no destination spans after migrate"
+    matching = [s for s in dest_spans if s.name == target_span.name]
+    assert matching, (
+        f"destination trace has no span matching source name {target_span.name!r}"
+    )
+    # If multiple destination spans share the source span's name (rare;
+    # the seeded experiment has one root + a few children with distinct
+    # names), pick the one whose comment count matches before delegating
+    # to verify_span -- the verifier's exact-equality assert would fail
+    # against an unrelated namesake otherwise.
+    candidate_ids = [s.id for s in matching]
+    if len(candidate_ids) > 1:
+        chosen = next(
+            (
+                s.id
+                for s in matching
+                if rest.spans.get_span_by_id(id=s.id).comments
+                and len(rest.spans.get_span_by_id(id=s.id).comments)
+                == len(span_comment_texts)
+            ),
+            candidate_ids[0],
+        )
+    else:
+        chosen = candidate_ids[0]
+
+    verifiers.verify_span(
+        opik_client=opik_client,
+        span_id=chosen,
+        trace_id=dest_trace_id,
+        parent_span_id=mock.ANY,
+        project_name=target_project_name,
+        comments=span_comment_texts,
+    )

sdks/python/tests/e2e/cli/test_migrate_dataset_optimizations_e2e.py

@@ -0,0 +1,269 @@
+"""End-to-end test for ``opik migrate dataset`` Slice 5 — optimization cascade.
+
+Optimization is a real BE entity, not a logical group-by: it has its own
+UUID, its own row in ``OptimizationDAO``, and its own ``dataset_id`` /
+``project_id`` FKs. The Optimization Studio "group" view in the UI is
+computed by the BE at read time from each experiment's ``optimization_id``
+FK plus the optimization's aggregated stats (``numTrials``, etc.); the
+persisted shape is one Optimization row + N Experiment rows where
+``Experiment.optimization_id == Optimization.id``.
+
+Slice 5's contract: when ``opik migrate dataset`` runs, every optimization
+referencing the source dataset is recreated under the destination project
+with a fresh id, and every experiment carrying that ``optimization_id``
+is recreated at the destination with its FK re-pointed at the new
+destination optimization id. The destination's Optimization Studio shows
+the same rows + trial groupings as the source.
+
+Shape pinned by this test:
+
+  1. Source dataset in project A with 3 items
+  2. One Optimization (``opt``) on the source dataset
+  3. Two TRIAL experiments seeded as ``opt``'s trials (each has 1 item)
+  4. One CONTROL ``regular`` experiment with no optimization_id (1 item)
+  5. Run ``opik migrate dataset ... --to-project=<destination>``
+  6. Assert:
+     a. Destination has exactly one optimization tied to the
+        destination dataset, with a fresh id (NOT the source id).
+     b. Both trial experiments are recreated under the destination
+        project with their ``optimization_id`` pointing at the
+        destination optimization (NOT the source optimization id).
+     c. The control experiment has no ``optimization_id`` (untouched).
+     d. The destination optimization's fidelity fields
+        (``name`` / ``objective_name`` / ``status``) match the source.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+import opik.id_helpers as id_helpers_module
+from opik.rest_api import OpikApi
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .conftest import (
+    create_dataset_shell,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-opt-{random_chars()}"
+
+
+def _create_source_optimization(
+    rest: OpikApi,
+    *,
+    dataset_name: str,
+    project_name: str,
+    optimization_name: str,
+) -> str:
+    """Create an Optimization tied to ``dataset_name`` in ``project_name``
+    and return its id. The cascade will discover this via
+    ``find_optimizations(dataset_id=...)``.
+    """
+    optimization_id = id_helpers_module.generate_id()
+    rest.optimizations.create_optimization(
+        id=optimization_id,
+        name=optimization_name,
+        dataset_name=dataset_name,
+        project_name=project_name,
+        objective_name="accuracy",
+        status="completed",
+    )
+    return optimization_id
+
+
+def test_migrate_dataset__optimization_cascade__trials_grouped_at_destination(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset with 3 items (2 for trials, 1 for control) ──
+    from opik import id_helpers
+    from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+    source_dataset_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_dataset_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": "trial-a", "a": "A"}),
+            DatasetItemWrite(source="manual", data={"q": "trial-b", "a": "B"}),
+            DatasetItemWrite(source="manual", data={"q": "control", "a": "C"}),
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(
+        id=source_dataset_id, page=1, size=1
+    ).content[0]
+
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+    trial_a_item_id = by_q["trial-a"]
+    trial_b_item_id = by_q["trial-b"]
+    control_item_id = by_q["control"]
+    assert trial_a_item_id and trial_b_item_id and control_item_id
+
+    # ── Create the source optimization ──
+    optimization_name = f"e2e-opt-{random_chars()}"
+    source_optimization_id = _create_source_optimization(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        optimization_name=optimization_name,
+    )
+
+    # ── Seed two trial experiments pointing at the optimization ──
+    # ``experiment_type="trial"`` is what the Optimization Studio UI
+    # filters on for the trial sub-list under an optimization row.
+    trial_a_name = f"trial-a-{random_chars()}"
+    trial_b_name = f"trial-b-{random_chars()}"
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=trial_a_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[trial_a_item_id],
+        experiment_type="trial",
+        optimization_id=source_optimization_id,
+    )
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=trial_b_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[trial_b_item_id],
+        experiment_type="trial",
+        optimization_id=source_optimization_id,
+    )
+
+    # ── Seed a control regular experiment with no optimization_id ──
+    # Pins that the cascade only touches experiments that actually had
+    # an optimization_id; the control experiment must round-trip with
+    # no optimization_id at the destination.
+    control_name = f"control-{random_chars()}"
+    seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=control_name,
+        dataset_name=dataset_name,
+        dataset_id=source_dataset_id,
+        dataset_version_id=v1.id,
+        project_name=source_project_name,
+        item_ids=[control_item_id],
+        experiment_type="regular",
+        optimization_id=None,
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            dataset_name,
+            "--from-project",
+            source_project_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify destination optimization fidelity ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+    dest_optimizations_page = rest.optimizations.find_optimizations(
+        dataset_id=dest_dataset.id, page=1, size=10
+    )
+    dest_optimizations = list(dest_optimizations_page.content or [])
+    assert len(dest_optimizations) == 1, (
+        f"expected exactly one destination optimization tied to dataset "
+        f"{dest_dataset.id}, got {len(dest_optimizations)}"
+    )
+    dest_optimization = dest_optimizations[0]
+
+    # Fresh destination id -- migrate is copy-not-move; source id must
+    # not be reused.
+    assert dest_optimization.id != source_optimization_id, (
+        "destination optimization id must be fresh; source id reused"
+    )
+    # Fidelity fields round-trip verbatim.
+    assert dest_optimization.name == optimization_name
+    assert dest_optimization.objective_name == "accuracy"
+    assert dest_optimization.status == "completed"
+    assert dest_optimization.dataset_id == dest_dataset.id
+
+    # ── Verify trial experiments are re-pointed at the destination optimization ──
+    dest_trial_a = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=trial_a_name,
+    )
+    dest_trial_b = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=trial_b_name,
+    )
+    assert dest_trial_a.optimization_id == dest_optimization.id, (
+        f"trial a should FK to destination optimization {dest_optimization.id}, "
+        f"got {dest_trial_a.optimization_id}"
+    )
+    assert dest_trial_b.optimization_id == dest_optimization.id, (
+        f"trial b should FK to destination optimization {dest_optimization.id}, "
+        f"got {dest_trial_b.optimization_id}"
+    )
+    # Defence-in-depth: neither trial should still point at the source id.
+    for dest_trial, label in ((dest_trial_a, "a"), (dest_trial_b, "b")):
+        assert dest_trial.optimization_id != source_optimization_id, (
+            f"trial {label}'s destination optimization_id must NOT be the "
+            f"source id (cascade failed to remap)"
+        )
+
+    # ── Verify control experiment is untouched (no optimization_id) ──
+    dest_control = find_destination_experiment(
+        rest,
+        destination_dataset_id=dest_dataset.id,
+        experiment_name=control_name,
+    )
+    assert not dest_control.optimization_id, (
+        f"control experiment must not have an optimization_id at the "
+        f"destination, got {dest_control.optimization_id!r}"
+    )
+
+    # ── Audit log carries per-optimization records ──
+    import json
+
+    audit = json.loads(audit_path.read_text())
+    migrate_optimization_records = [
+        a for a in audit["actions"] if a["type"] == "migrate_optimization"
+    ]
+    assert len(migrate_optimization_records) == 1
+    record = migrate_optimization_records[0]
+    assert record["status"] == "ok"
+    assert record["source_id"] == source_optimization_id
+    assert record["destination_id"] == dest_optimization.id

sdks/python/tests/e2e/cli/test_migrate_dataset_resume_e2e.py

@@ -0,0 +1,273 @@
+"""End-to-end test for ``opik migrate dataset`` checkpoint/resume (OPIK-7168).
+
+The unit tests prove the resume *logic* against a stand-in client. This test
+proves the thing that actually matters for the feature: a **real process
+death** mid-cascade leaves a checkpoint on disk that a **real re-run** picks
+up, against a **real backend** whose trace-delete cascades spans — ending with
+a duplicate-free destination.
+
+Shape:
+
+  1. Seed a source dataset (one version) and N experiments, each with its own
+     trace + span tree, all referencing that dataset.
+  2. Run ``opik migrate dataset`` with a test-only ``sitecustomize.py`` seam on
+     ``PYTHONPATH`` (nothing test-only lives in the product) that ``os._exit(137)``s
+     partway through one experiment — after its destination traces are written
+     and recorded on the checkpoint but before the experiment row is recreated.
+     This is the uncatchable-kill (OOM-like) interruption the feature exists for;
+     a clean ``except`` never runs, so only the incrementally-flushed checkpoint
+     survives. The seam also redirects the checkpoint dir into tmp_path.
+  3. Assert the run died hard and the checkpoint on disk shows partial progress
+     (some experiments done, one in flight with recorded dest trace ids).
+  4. Re-run the *same* command with no crash env. Assert it exits 0, resumes
+     (skips the already-done experiments, cleans up the interrupted one's
+     partial traces, re-migrates it), and the destination ends with exactly one
+     experiment per source experiment — no duplicates — and the checkpoint file
+     is gone.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Iterator, List
+
+import pytest
+
+import opik
+from opik import id_helpers, synchronization
+from opik.rest_api.types.dataset_item_write import DatasetItemWrite
+
+from ...conftest import random_chars
+from .conftest import (
+    create_dataset_shell,
+    destination_experiment_items,
+    find_destination_experiment,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    stream_items_wire,
+)
+
+# Number of source experiments and the count of experiments completed before
+# the first run hard-exits. Crashing after 2 completed experiments (mid the 3rd)
+# leaves a non-trivial done-set to skip AND an in-flight experiment to clean up
+# + re-migrate on resume.
+_NUM_EXPERIMENTS = 5
+_CRASH_AFTER_N_EXPERIMENTS = 2
+
+
+@pytest.fixture
+def dataset_name() -> Iterator[str]:
+    yield f"e2e-migrate-resume-{random_chars()}"
+
+
+def _write_test_seam(
+    seam_dir: Path, *, crash_after_n: int, checkpoint_dir: Path
+) -> None:
+    """Write a test-only ``sitecustomize.py`` the migrate subprocess auto-imports.
+
+    Python imports ``sitecustomize`` at interpreter startup, so dropping it on
+    the subprocess's ``PYTHONPATH`` lets the test inject behaviour **without any
+    test-only logic living in the product**. Two patches, both test-owned:
+
+    * Redirect the checkpoint dir to a tmp path (via ``checkpoint_dir``) so the
+      test can find/assert the checkpoint without touching the developer's real
+      ``~/.opik`` and without repointing ``HOME`` (which would break the SDK's
+      backend-config resolution the E2E run relies on).
+    * Wrap ``_copy_traces_and_spans`` so the real one runs first (destination
+      traces written + recorded on the checkpoint), then ``os._exit(137)`` after
+      the Nth experiment — reproducing the exact "traces written, experiment row
+      not yet created" partial state an OOM leaves mid-cascade. ``crash_after_n``
+      < 0 disables the crash (used for the resume run).
+    """
+    seam_dir.mkdir(parents=True, exist_ok=True)
+    (seam_dir / "sitecustomize.py").write_text(
+        f"""
+import os
+from pathlib import Path
+from opik.cli.migrate import checkpoint as _cp
+from opik.cli.migrate.datasets import experiments as _exp
+
+_cp.checkpoint_dir = lambda: Path({str(checkpoint_dir)!r})
+
+_crash_after = {crash_after_n}
+if _crash_after >= 0:
+    _seen = {{"n": 0}}
+    _real_copy = _exp._copy_traces_and_spans
+
+    def _copy_then_maybe_crash(*args, **kwargs):
+        result = _real_copy(*args, **kwargs)
+        _seen["n"] += 1
+        if _seen["n"] > _crash_after:
+            os._exit(137)  # 128 + SIGKILL(9): uncatchable, mimics an OOM kill
+        return result
+
+    _exp._copy_traces_and_spans = _copy_then_maybe_crash
+"""
+    )
+
+
+def test_migrate_dataset__crash_mid_cascade__resumes_without_duplicates(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    dataset_name: str,
+    tmp_path: Path,
+) -> None:
+    rest = opik_client.rest_client
+
+    # ── Seed the source dataset (single version, one item per experiment) ──
+    source_id = create_dataset_shell(rest, dataset_name, source_project_name)
+    rest.datasets.create_or_update_dataset_items(
+        dataset_id=source_id,
+        items=[
+            DatasetItemWrite(source="manual", data={"q": f"Q{i}", "a": f"A{i}"})
+            for i in range(_NUM_EXPERIMENTS)
+        ],
+        batch_group_id=id_helpers.generate_id(),
+    )
+    v1 = rest.datasets.list_dataset_versions(id=source_id, page=1, size=1).content[0]
+    v1_items = stream_items_wire(
+        rest,
+        dataset_name=dataset_name,
+        project_name=source_project_name,
+        version_hash=v1.version_hash,
+    )
+    item_id_by_q = {(it.data or {}).get("q"): it.id for it in v1_items}
+
+    # ── Seed N experiments, each referencing one dataset item ──
+    # Names derive from the (already-random) ``dataset_name`` fixture so they're
+    # unique per run yet reproducible from the fixture, per sdks/python/AGENTS.md.
+    experiment_names: List[str] = []
+    for i in range(_NUM_EXPERIMENTS):
+        name = f"{dataset_name}-exp-{i}"
+        experiment_names.append(name)
+        seed_experiment_with_trace_tree(
+            rest,
+            experiment_name=name,
+            dataset_name=dataset_name,
+            dataset_id=source_id,
+            dataset_version_id=v1.id,
+            project_name=source_project_name,
+            item_ids=[item_id_by_q[f"Q{i}"]],
+            spans_per_trace=2,
+        )
+
+    audit_path = tmp_path / "audit.json"
+    migrate_args = ["dataset", dataset_name, "--to-project", target_project_name]
+
+    # Test seam: a sitecustomize.py the migrate subprocess auto-imports. It
+    # redirects the checkpoint dir into tmp_path (hermetic; no real ~/.opik) and
+    # injects the deterministic mid-cascade crash — all test-owned, nothing in
+    # the product. PYTHONPATH must keep the SDK's own src on it (the E2E run
+    # imports opik from source), so the seam dir is prepended.
+    checkpoint_dir = tmp_path / "checkpoints"
+    seam_dir = tmp_path / "seam"
+    existing_pythonpath = os.environ.get("PYTHONPATH", "")
+
+    def _seam_env(crash_after_n: int) -> dict:
+        _write_test_seam(
+            seam_dir, crash_after_n=crash_after_n, checkpoint_dir=checkpoint_dir
+        )
+        return {
+            "PYTHONPATH": os.pathsep.join(
+                [str(seam_dir)] + ([existing_pythonpath] if existing_pythonpath else [])
+            )
+        }
+
+    # ── Run 1: crash deterministically mid-cascade ──
+    crashed = run_migrate_cli(
+        migrate_args,
+        audit_log_path=str(audit_path),
+        extra_env=_seam_env(_CRASH_AFTER_N_EXPERIMENTS),
+    )
+    # os._exit(137) is a hard, uncatchable exit — not a clean CLI exit code.
+    assert crashed.returncode == 137, (
+        f"expected hard-exit 137, got {crashed.returncode}\n"
+        f"stdout={crashed.stdout}\nstderr={crashed.stderr}"
+    )
+
+    # The incrementally-flushed checkpoint must have survived the kill and show
+    # partial progress: the experiments before the crash completed, and the one
+    # at the crash is in flight with recorded destination trace ids (its traces
+    # were written to the backend before the crash).
+    checkpoints = list(checkpoint_dir.glob("opik-migrate-checkpoint-*.json"))
+    assert len(checkpoints) == 1, f"expected one checkpoint file, got {checkpoints}"
+    checkpoint_data = json.loads(checkpoints[0].read_text())
+    assert (
+        len(checkpoint_data["completed_experiment_ids"]) == _CRASH_AFTER_N_EXPERIMENTS
+    )
+    # The dataset phase (create-temp/replay/optimizations) finished before the
+    # cascade started, so resume must skip it and reconstruct the remaps from
+    # the temp destination rather than re-running (and duplicating) it. Under the
+    # OPIK-7162 ordering the source keeps its original name until the run
+    # succeeds, and the destination is still under the temp name at crash time —
+    # the checkpoint records both so resume can re-resolve them.
+    assert checkpoint_data["dataset_phase_done"] is True
+    assert checkpoint_data["source_name"] == dataset_name
+    assert checkpoint_data["temp_dest_name"] == f"{dataset_name}__migrating"
+    in_flight = checkpoint_data["in_flight"]
+    assert in_flight is not None, "interrupted experiment must be recorded in flight"
+    assert in_flight["dest_trace_ids"], (
+        "the in-flight experiment's destination trace ids must be recorded "
+        "before the crash so resume can delete them"
+    )
+    # Crash landed before the experiment row was created.
+    assert in_flight["dest_experiment_id"] is None
+
+    # ── Run 2: resume to completion (same seam, crash disabled) ──
+    resumed = run_migrate_cli(
+        migrate_args, audit_log_path=str(audit_path), extra_env=_seam_env(-1)
+    )
+    assert resumed.returncode == 0, resumed.stdout + resumed.stderr
+    assert "Resuming migration" in resumed.stdout, (
+        "resumed run should announce it is resuming"
+    )
+
+    # ── Verify destination: exactly one experiment per source, no duplicates ──
+    dest_dataset = rest.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name, project_name=target_project_name
+    )
+
+    # Migrated experiments become readable asynchronously, so poll until the
+    # destination shows the full set before asserting — otherwise the read can
+    # race ahead of eventual consistency and flake. ``until`` returns False on
+    # timeout; the count assertion below then fails with the observed number.
+    def _all_experiments_present() -> bool:
+        page = rest.experiments.find_experiments(
+            dataset_id=dest_dataset.id, page=1, size=100
+        )
+        return len(page.content or []) == _NUM_EXPERIMENTS
+
+    synchronization.until(_all_experiments_present, max_try_seconds=30)
+
+    for name in experiment_names:
+        # find_destination_experiment RAISES if zero or >1 match — so this is
+        # the duplicate check: the interrupted experiment must not have been
+        # migrated twice (once by the crashed run, once by the resume).
+        dest_exp = find_destination_experiment(
+            rest,
+            destination_dataset_id=dest_dataset.id,
+            experiment_name=name,
+        )
+        dest_items = destination_experiment_items(
+            rest, experiment_id=dest_exp.id, dataset_id=dest_dataset.id
+        )
+        assert len(dest_items) == 1, (
+            f"experiment {name!r} should have exactly 1 item at the "
+            f"destination, got {len(dest_items)}"
+        )
+
+    # Total destination experiments equals the source count — no orphaned
+    # partial experiment left over from the crashed run.
+    all_dest = rest.experiments.find_experiments(
+        dataset_id=dest_dataset.id, page=1, size=100
+    )
+    assert len(all_dest.content or []) == _NUM_EXPERIMENTS, (
+        f"expected {_NUM_EXPERIMENTS} destination experiments, "
+        f"got {len(all_dest.content or [])}"
+    )
+
+    # Checkpoint deleted on successful completion.
+    assert list(checkpoint_dir.glob("opik-migrate-checkpoint-*.json")) == []

sdks/python/tests/e2e/cli/test_migrate_prompt_e2e.py

@@ -0,0 +1,229 @@
+"""End-to-end test for ``opik migrate prompt`` against a real Opik backend.
+
+Seeds a multi-version source prompt directly via the REST API, runs
+``opik migrate prompt`` as a subprocess (exercising the actual Click
+entrypoint + exit-code handling), then reads back the destination prompt
+and asserts:
+
+  - Destination has the same number of versions as the source
+  - Each destination version carries the source commit hash verbatim
+    (the architectural promise this slice exists to deliver)
+  - Source has been renamed to ``<name>_v1``
+  - Audit log is finalised to ``ok`` with one ``replay_prompt_version``
+    record per source version
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Iterator, List
+
+import pytest
+
+import opik
+from opik.rest_api import OpikApi
+from opik.rest_api.types.prompt_version_detail import PromptVersionDetail
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from .conftest import run_migrate_cli
+
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def prompt_name() -> Iterator[str]:
+    yield f"e2e-migrate-prompt-{random_chars()}"
+
+
+def _seed_source_prompt_with_versions(
+    rest_client: OpikApi,
+    *,
+    name: str,
+    project_name: str,
+    version_specs: List[dict],
+) -> List[str]:
+    """Create a source prompt container plus N versions.
+
+    First call to ``create_prompt`` carries the v1 template so the BE
+    mints v1 with its own commit hash (we don't control v1's commit at
+    seed time — that's fine for the test, we just need a multi-version
+    history). Subsequent versions are minted via
+    ``create_prompt_version`` so we control their commit + payload.
+
+    Returns the list of source version commits in chronological order.
+    """
+    rest_client.prompts.create_prompt(
+        name=name,
+        project_name=project_name,
+        description="e2e source description",
+        tags=["e2e", "source"],
+        template=version_specs[0]["template"],
+        type=version_specs[0].get("type"),
+        change_description=version_specs[0].get("change_description"),
+    )
+    versions_page = rest_client.prompts.get_prompts(name=name, size=10)
+    source_row = versions_page.content[0]
+    versions = rest_client.prompts.get_prompt_versions(
+        id=source_row.id, page=1, size=100
+    )
+    v1_commit = versions.content[0].commit
+    commits = [v1_commit]
+
+    for spec in version_specs[1:]:
+        version_payload = PromptVersionDetail(
+            template=spec["template"],
+            type=spec.get("type"),
+            change_description=spec.get("change_description"),
+            tags=spec.get("tags"),
+            environments=spec.get("environments"),
+        )
+        created = rest_client.prompts.create_prompt_version(
+            name=name,
+            version=version_payload,
+            project_name=project_name,
+        )
+        commits.append(created.commit)
+
+    return commits
+
+
+class TestMigratePromptE2E:
+    def test_three_version_prompt_round_trips_with_commits_verbatim(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        prompt_name: str,
+        tmp_path: Path,
+    ) -> None:
+        rest = opik_client.rest_client
+
+        source_commits = _seed_source_prompt_with_versions(
+            rest,
+            name=prompt_name,
+            project_name=source_project_name,
+            version_specs=[
+                {"template": "hi {{name}}", "type": "mustache"},
+                {
+                    "template": "hello {{name}}",
+                    "type": "mustache",
+                    "change_description": "be friendlier",
+                },
+                {
+                    "template": "greetings {{name}}",
+                    "type": "mustache",
+                    "tags": ["polished"],
+                },
+            ],
+        )
+
+        audit_log_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["prompt", prompt_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_log_path),
+        )
+        assert result.returncode == 0, (
+            f"migrate prompt failed: stdout={result.stdout!r} stderr={result.stderr!r}"
+        )
+
+        # Source must have been renamed (workspace-unique name freed for
+        # the destination to claim).
+        renamed_page = rest.prompts.get_prompts(name=f"{prompt_name}_v1", size=10)
+        assert renamed_page.content, (
+            f"source prompt was not renamed to {prompt_name}_v1"
+        )
+
+        # Destination claims the original name.
+        dest_page = rest.prompts.get_prompts(name=prompt_name, size=10)
+        assert dest_page.content, "destination prompt not found at original name"
+        dest_prompt = dest_page.content[0]
+
+        # Every source version is replayed onto the destination with the
+        # commit hash preserved verbatim. BE orders newest-first; reverse
+        # to chronological for comparison against source_commits.
+        dest_versions_page = rest.prompts.get_prompt_versions(
+            id=dest_prompt.id, page=1, size=100
+        )
+        dest_commits = [v.commit for v in reversed(dest_versions_page.content or [])]
+        assert dest_commits == source_commits
+
+        # Audit log finalised to "ok" with one replay record per version.
+        audit = json.loads(audit_log_path.read_text())
+        assert audit["status"] == "ok"
+        replay_records = [
+            a for a in audit["actions"] if a["type"] == "replay_prompt_version"
+        ]
+        assert len(replay_records) == len(source_commits)
+        assert all(a["status"] == "ok" for a in replay_records)
+
+    def test_environment_ownership_round_trips(
+        self,
+        opik_client: opik.Opik,
+        source_project_name: str,
+        target_project_name: str,
+        prompt_name: str,
+        tmp_path: Path,
+    ) -> None:
+        # A version that owns an environment on the source must own the
+        # same environment on the destination after migration. The BE
+        # accepts ``environments`` inline on create_prompt_version and
+        # auto-registers unknown names in the workspace registry, so the
+        # replay carries the set verbatim without a follow-up PATCH.
+        rest = opik_client.rest_client
+        environment_name = f"prod-{random_chars()}"
+        source_commits = _seed_source_prompt_with_versions(
+            rest,
+            name=prompt_name,
+            project_name=source_project_name,
+            version_specs=[
+                {"template": "hi {{name}}", "type": "mustache"},
+                {"template": "hello {{name}}", "type": "mustache"},
+                {
+                    "template": "greetings {{name}}",
+                    "type": "mustache",
+                    "environments": [environment_name],
+                },
+            ],
+        )
+
+        audit_log_path = tmp_path / "audit.json"
+        result = run_migrate_cli(
+            ["prompt", prompt_name, "--to-project", target_project_name],
+            audit_log_path=str(audit_log_path),
+        )
+        assert result.returncode == 0, (
+            f"migrate prompt failed: stdout={result.stdout!r} stderr={result.stderr!r}"
+        )
+
+        dest_page = rest.prompts.get_prompts(name=prompt_name, size=10)
+        assert dest_page.content, "destination prompt not found at original name"
+        dest_prompt = dest_page.content[0]
+
+        dest_versions_page = rest.prompts.get_prompt_versions(
+            id=dest_prompt.id, page=1, size=100
+        )
+        dest_versions = list(reversed(dest_versions_page.content or []))
+        dest_commits = [v.commit for v in dest_versions]
+        assert dest_commits == source_commits
+
+        # Exactly one destination version owns the environment, and it is
+        # the last one (the source owner), proving ownership round-tripped
+        # without leaking onto the env-less versions.
+        owners = [
+            v.commit
+            for v in dest_versions
+            if environment_name in (v.environments or [])
+        ]
+        assert owners == [source_commits[-1]]
+
+        audit = json.loads(audit_log_path.read_text())
+        env_records = [
+            a
+            for a in audit["actions"]
+            if a["type"] == "replay_prompt_version" and a.get("source_environments")
+        ]
+        assert len(env_records) == 1
+        assert env_records[0]["source_environments"] == [environment_name]
+        assert env_records[0]["target_environments"] == [environment_name]

sdks/python/tests/e2e/cli/test_migrate_test_suite_e2e.py

@@ -0,0 +1,488 @@
+"""End-to-end test for ``opik migrate dataset`` against a test suite.
+
+Test suites have an extra dimension over plain datasets: each version
+carries suite-level ``evaluators`` + ``execution_policy`` that the
+migration must replay per-version. Plus the BE-natural test-suite shape
+is "config-only v1 + content versions v2+" (the SDK's
+``create_test_suite(global_assertions=...)`` produces a v1 with zero
+items but the suite config attached).
+
+This test covers the full surface:
+
+- 4 versions: config-only v1 + 3 content versions with adds/edits/deletes
+- Per-version suite-level evaluators that change across versions
+- Per-version suite-level execution_policy that changes across versions
+- Per-version user tags + metadata
+- Per-item evaluators + execution_policy + tags on individual items
+- Clear transitions: items that drop their per-item overrides between versions
+
+Verifies all of the above round-trips at every replayed version on the
+target.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+import opik
+
+from ...conftest import random_chars
+from ...testlib import generate_project_name
+from ._cascade_comparison import compare_cascade
+from .conftest import (
+    apply_changes,
+    chronological_versions,
+    create_dataset_shell,
+    destination_experiment_items,
+    destination_spans_for_trace,
+    display_order,
+    find_destination_experiment,
+    item_hashes,
+    normalize_evaluators,
+    normalize_policy,
+    run_migrate_cli,
+    seed_experiment_with_trace_tree,
+    strip_be_managed_version_tags,
+    stream_items_wire,
+)
+
+# Per ``sdks/python/AGENTS.md``: every e2e module sources PROJECT_NAME from
+# ``generate_project_name("e2e", __name__)`` so backend project names are
+# isolated per test module + the autouse ``configure_e2e_tests_env`` fixture
+# can patch ``OPIK_PROJECT_NAME`` to match.
+PROJECT_NAME = generate_project_name("e2e", __name__)
+
+
+@pytest.fixture
+def suite_name() -> Iterator[str]:
+    yield f"e2e-migrate-suite-{random_chars()}"
+
+
+# Suite-level evaluators per source version. Each entry is what gets
+# attached to that version (changes across versions to exercise replay).
+SUITE_EVALUATORS_PER_VERSION = [
+    # v1: one suite-level assertion
+    [{"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}}],
+    # v2: still v1's evaluator (no suite-level change)
+    [{"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}}],
+    # v3: add a second suite-level assertion + bump policy
+    [
+        {"name": "v1-judge", "type": "llm_judge", "config": {"model": "haiku"}},
+        {"name": "v2-judge", "type": "llm_judge", "config": {"model": "sonnet"}},
+    ],
+    # v4: replace evaluators entirely
+    [{"name": "v3-only-judge", "type": "llm_judge", "config": {"model": "opus"}}],
+]
+
+SUITE_EXEC_POLICY_PER_VERSION = [
+    {"runs_per_item": 1, "pass_threshold": 1},
+    {"runs_per_item": 1, "pass_threshold": 1},
+    {"runs_per_item": 3, "pass_threshold": 2},
+    {"runs_per_item": 5, "pass_threshold": 3},
+]
+
+
+def test_test_suite_full_fidelity_round_trip(
+    opik_client: opik.Opik,
+    source_project_name: str,
+    target_project_name: str,
+    suite_name: str,
+    tmp_path: Path,
+) -> None:
+    """Migrate a 4-version test suite end-to-end and verify per-version
+    fidelity across items, item-level overrides, suite-level config,
+    metadata, and user tags.
+
+    Exercises every fidelity dimension Slice 2 supports in one test
+    because the BE round-trip is the expensive part; multiple smaller
+    tests would each pay the same setup cost. If any dimension regresses
+    this single test pins it.
+    """
+    rest = opik_client.rest_client
+
+    # ── Seed v1: config-only (no items, just suite-level evaluators+policy) ──
+    suite_id = create_dataset_shell(
+        rest, suite_name, source_project_name, type="evaluation_suite"
+    )
+    apply_changes(
+        rest,
+        suite_id,
+        base_version_id=None,
+        change_description="v1 — initial config-only version",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[0],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[0],
+        override=True,  # required when base_version=None
+    )
+    v1 = chronological_versions(rest, suite_id)[0]
+
+    # ── Seed v2: add 3 items (Q1, Q2 with per-item evaluator, Q3 with
+    # per-item exec policy). Suite config unchanged. ──
+    v2_items = [
+        {"data": {"q": "Q1", "a": "A1"}, "source": "manual"},
+        {
+            "data": {"q": "Q2", "a": "A2"},
+            "source": "manual",
+            "evaluators": [
+                {
+                    "name": "q2-item-judge",
+                    "type": "llm_judge",
+                    "config": {"model": "haiku-item"},
+                }
+            ],
+            "tags": ["smoke"],
+        },
+        {
+            "data": {"q": "Q3", "a": "A3"},
+            "source": "manual",
+            "execution_policy": {"runs_per_item": 7, "pass_threshold": 4},
+        },
+    ]
+    v2_id = apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v1.id,
+        added_items=v2_items,
+        change_description="v2 — initial items with per-item overrides",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[1],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[1],
+    )
+    v2_streamed = stream_items_wire(
+        rest,
+        dataset_name=suite_name,
+        project_name=source_project_name,
+        version_hash=None,  # latest
+    )
+    by_q = {it.data["q"]: it for it in v2_streamed if it.data}
+
+    # ── Seed v3: edit Q1 (add per-item policy + extend tags), edit Q2
+    # (clear tags), add Q4. Bump suite evaluators to v3 config, bump policy. ──
+    v3_id = apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v2_id,
+        edited_items=[
+            {
+                "id": by_q["Q1"].id,
+                "data": {"q": "Q1", "a": "A1"},
+                "execution_policy": {"runs_per_item": 9, "pass_threshold": 5},
+                "tags": ["regression", "v3-touched"],
+            },
+            {
+                "id": by_q["Q2"].id,
+                "data": {"q": "Q2", "a": "A2"},
+                # Clear Q2's tags by passing empty list.
+                "tags": [],
+            },
+        ],
+        added_items=[{"data": {"q": "Q4", "a": "A4"}, "source": "manual"}],
+        change_description="v3 — edit Q1, clear Q2 tags, add Q4",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[2],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[2],
+        metadata={"phase": "experimentation", "owner": "ada"},
+        user_tags=["baseline"],
+    )
+
+    # ── Seed v4: delete Q2, replace Q3's per-item evaluator, add Q5 with a
+    # per-item evaluator. Replace suite config entirely + bump policy. ──
+    apply_changes(
+        rest,
+        suite_id,
+        base_version_id=v3_id,
+        edited_items=[
+            {
+                "id": by_q["Q3"].id,
+                "data": {"q": "Q3", "a": "A3"},
+                "evaluators": [
+                    {
+                        "name": "q3-item-judge-NEW",
+                        "type": "llm_judge",
+                        "config": {"model": "sonnet-item"},
+                    }
+                ],
+            }
+        ],
+        deleted_ids=[by_q["Q2"].id],
+        added_items=[
+            {
+                "data": {"q": "Q5", "a": "A5"},
+                "source": "manual",
+                "evaluators": [
+                    {
+                        "name": "q5-item-judge",
+                        "type": "llm_judge",
+                        "config": {"model": "opus-item"},
+                    }
+                ],
+            }
+        ],
+        change_description="v4 — delete Q2, replace Q3 evaluator, add Q5",
+        suite_evaluators=SUITE_EVALUATORS_PER_VERSION[3],
+        suite_execution_policy=SUITE_EXEC_POLICY_PER_VERSION[3],
+        metadata={"phase": "production", "owner": "ada"},
+        user_tags=["v1.0", "release-candidate"],
+    )
+
+    # ── Snapshot source expectations per version ──
+    src_versions = chronological_versions(rest, suite_id)
+    assert len(src_versions) == 4
+    expected = []
+    for v in src_versions:
+        items = stream_items_wire(
+            rest,
+            dataset_name=suite_name,
+            project_name=source_project_name,
+            version_hash=v.version_hash,
+        )
+        expected.append(
+            {
+                "hashes": item_hashes(items),
+                "order": display_order(items),
+                "suite_evals": normalize_evaluators(v.evaluators),
+                "suite_pol": normalize_policy(v.execution_policy),
+                "metadata": v.metadata or None,
+                "user_tags": strip_be_managed_version_tags(v.tags),
+            }
+        )
+
+    # ── Seed a suite-driven experiment on v2 items so the cascade has
+    # something to round-trip. Test-suite experiments carry per-trace
+    # assertion_results (regular-dataset experiments carry feedback_scores
+    # instead -- covered in test_migrate_dataset_e2e.py). Each item gets
+    # one trace, one span, and per-item assertion_results that the seed
+    # helper writes via store_assertions_batch(entity_type='TRACE', ...).
+    # The cascade re-emits them scoped to the new destination trace ids
+    # via the same endpoint. ──
+    experiment_name = f"e2e-suite-exp-{random_chars()}"
+    v2_item_ids = [by_q["Q1"].id, by_q["Q2"].id, by_q["Q3"].id]
+    cascade_seed = seed_experiment_with_trace_tree(
+        rest,
+        experiment_name=experiment_name,
+        dataset_name=suite_name,
+        dataset_id=suite_id,
+        dataset_version_id=v2_id,
+        project_name=source_project_name,
+        item_ids=v2_item_ids,
+        # ``type="regular"`` + ``evaluation_method="evaluation_suite"`` is
+        # the canonical shape for a non-optimizer test-suite run, mirroring
+        # what ``opik.evaluate(...)`` produces against a test suite. The
+        # BE only updates ``last_created_experiment_at`` on the dataset for
+        # ``type="regular"`` experiments, so ``with_experiments_only=true``
+        # (the UI's project-page filter) correctly surfaces this one.
+        # Optimizer-driven trials use ``type="trial"`` + ``optimization_id``;
+        # those are Slice 4's cascade scope.
+        experiment_type="regular",
+        evaluation_method="evaluation_suite",
+        experiment_config={"runner": "e2e-suite-cascade-test"},
+        experiment_tags=["e2e", "test-suite", "cascade"],
+        spans_per_trace=2,
+        # Per-item runtime assertion results are seeded with the SAME name
+        # as the v2 suite-level evaluator (``v1-judge``). That's what a real
+        # test-suite run produces: for each item, one assertion result per
+        # evaluator that ran against it. Q2 also has a per-item override
+        # evaluator ``q2-item-judge``; we seed a second result there to
+        # mirror what an actual evaluator run would produce.
+        per_item_extras=[
+            {
+                "assertion_results": [
+                    {
+                        "value": "v1-judge",  # matches v2's suite-level evaluator
+                        "passed": i % 2 == 0,
+                        "reason": f"v1-judge ran on Q{i + 1}",
+                    }
+                ]
+                # Q2 has a per-item override evaluator ``q2-item-judge``; a
+                # real run would also produce a result for that. We tack it
+                # on for the second item.
+                + (
+                    [
+                        {
+                            "value": "q2-item-judge",
+                            "passed": True,
+                            "reason": "q2-item-judge ran on Q2",
+                        }
+                    ]
+                    if i == 1
+                    else []
+                ),
+            }
+            for i in range(len(v2_item_ids))
+        ],
+    )
+
+    # ── Run the migration ──
+    audit_path = tmp_path / "audit.json"
+    result = run_migrate_cli(
+        [
+            "dataset",
+            suite_name,
+            "--to-project",
+            target_project_name,
+        ],
+        audit_log_path=str(audit_path),
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+    # ── Verify target ──
+    target = rest.datasets.get_dataset_by_identifier(
+        dataset_name=suite_name, project_name=target_project_name
+    )
+    assert target.type == "evaluation_suite", (
+        f"target should be a test suite, got type={target.type!r}"
+    )
+
+    tgt_versions = chronological_versions(rest, target.id)
+    assert len(tgt_versions) == len(src_versions), (
+        f"target version count {len(tgt_versions)} != source {len(src_versions)} "
+        "— Slice 2 contract requires N=N for test suites too"
+    )
+
+    for src_v, tgt_v, exp in zip(src_versions, tgt_versions, expected):
+        items = stream_items_wire(
+            rest,
+            dataset_name=suite_name,
+            project_name=target_project_name,
+            version_hash=tgt_v.version_hash,
+        )
+        # Items: set-equal under content hash + display order matches.
+        actual_hashes = item_hashes(items)
+        assert actual_hashes == exp["hashes"], (
+            f"{tgt_v.version_name}: items diverged from {src_v.version_name}. "
+            f"Missing on target: {exp['hashes'] - actual_hashes}; "
+            f"extra on target: {actual_hashes - exp['hashes']}"
+        )
+        assert display_order(items) == exp["order"], (
+            f"{tgt_v.version_name}: display order diverged"
+        )
+        # Suite-level config matches.
+        assert normalize_evaluators(tgt_v.evaluators) == exp["suite_evals"], (
+            f"{tgt_v.version_name}: suite evaluators diverged"
+        )
+        assert normalize_policy(tgt_v.execution_policy) == exp["suite_pol"], (
+            f"{tgt_v.version_name}: suite execution_policy diverged"
+        )
+        # Version-level metadata + user tags match.
+        assert (tgt_v.metadata or None) == exp["metadata"], (
+            f"{tgt_v.version_name}: metadata diverged"
+        )
+        assert strip_be_managed_version_tags(tgt_v.tags) == exp["user_tags"], (
+            f"{tgt_v.version_name}: user version tags diverged"
+        )
+
+    # ── Cascade fidelity (test-suite-specific) ──
+    # The destination project should now have a copy of the source
+    # suite-driven experiment: type + evaluation_method preserved,
+    # per-item assertion_results + execution_policy round-trip, FKs
+    # remapped, traces+spans land under the destination project.
+    dest_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=target.id,
+        experiment_name=experiment_name,
+    )
+    assert dest_exp.id != cascade_seed["experiment_id"]
+    assert dest_exp.dataset_id == target.id
+    # Type + evaluation_method preserved (test-suite-specific fields).
+    assert dest_exp.type == "regular", (
+        f"destination experiment type should be 'regular', got {dest_exp.type!r}"
+    )
+    assert dest_exp.evaluation_method == "evaluation_suite", (
+        f"destination experiment evaluation_method should be 'evaluation_suite', "
+        f"got {dest_exp.evaluation_method!r}"
+    )
+    target_version_ids = {v.id for v in tgt_versions}
+    assert dest_exp.dataset_version_id in target_version_ids
+
+    # Per-item assertion_results survive on the destination via the
+    # Compare view (the BE persists them through the dedicated
+    # ``assertion_results.store_assertions_batch`` endpoint scoped to the
+    # destination trace ids; the Compare view aggregates them onto each
+    # ExperimentItemCompare for read).
+    dest_items = destination_experiment_items(
+        rest,
+        experiment_id=dest_exp.id,
+        dataset_id=target.id,
+    )
+    assert len(dest_items) == len(v2_item_ids)
+    dest_trace_ids = {it.trace_id for it in dest_items}
+    assert dest_trace_ids.isdisjoint(set(cascade_seed["trace_ids"]))
+
+    # Each destination item has at least one assertion result named
+    # ``v1-judge`` (the v2 suite-level evaluator's name), matching what a
+    # real test-suite run would produce. The Q2 item additionally has a
+    # ``q2-item-judge`` result because of the per-item evaluator override.
+    items_with_q2_override = 0
+    for dest_item in dest_items:
+        ars = dest_item.assertion_results
+        assert ars, "destination experiment item should have assertion_results"
+        names = {a.value for a in ars}
+        assert "v1-judge" in names, (
+            f"destination item should carry a 'v1-judge' assertion result "
+            f"(matches the v2 suite-level evaluator); got names={names}"
+        )
+        if "q2-item-judge" in names:
+            items_with_q2_override += 1
+            # Find the q2-item-judge result and verify its reason text.
+            q2_ar = next(a for a in ars if a.value == "q2-item-judge")
+            assert q2_ar.passed is True
+            assert "Q2" in (q2_ar.reason or "")
+    assert items_with_q2_override == 1, (
+        "exactly one item (Q2) should carry the per-item q2-item-judge "
+        f"assertion result; got {items_with_q2_override}"
+    )
+
+    # Each destination trace exists under the target project with the
+    # span tree shape preserved (root + 1 child, parent_span_id remapped).
+    for new_trace_id in dest_trace_ids:
+        dest_spans = destination_spans_for_trace(
+            rest,
+            trace_id=new_trace_id,
+            project_name=target_project_name,
+        )
+        assert len(dest_spans) == 2
+        roots = [s for s in dest_spans if s.parent_span_id is None]
+        assert len(roots) == 1
+        children = [s for s in dest_spans if s.parent_span_id is not None]
+        assert all(c.parent_span_id == roots[0].id for c in children)
+
+    # ── Deep-equal source vs. destination ──
+    # Pair source/destination items by trace ``name`` (assigned by the
+    # seed as "task-0", "task-1", "task-2" and carried verbatim through
+    # the cascade), then run a field-by-field comparison of experiment +
+    # items (assertion_results + feedback_scores) + traces + spans
+    # modulo remapped IDs.
+    src_exp = find_destination_experiment(
+        rest,
+        destination_dataset_id=suite_id,
+        experiment_name=experiment_name,
+    )
+    src_items_compare = destination_experiment_items(
+        rest,
+        experiment_id=cascade_seed["experiment_id"],
+        dataset_id=suite_id,
+    )
+    src_trace_names = {
+        it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+        for it in src_items_compare
+    }
+    dst_trace_names = {
+        it.trace_id: rest.traces.get_trace_by_id(id=it.trace_id).name
+        for it in dest_items
+    }
+    src_items_compare.sort(key=lambda it: src_trace_names[it.trace_id])
+    dest_items_sorted = sorted(dest_items, key=lambda it: dst_trace_names[it.trace_id])
+    src_trace_ids_sorted = [it.trace_id for it in src_items_compare]
+    dst_trace_ids_sorted = [it.trace_id for it in dest_items_sorted]
+
+    compare_cascade(
+        rest_client=rest,
+        source_experiment=src_exp,
+        destination_experiment=dest_exp,
+        source_item_ids=v2_item_ids,
+        destination_item_ids=[it.dataset_item_id for it in dest_items_sorted],
+        source_trace_ids=src_trace_ids_sorted,
+        destination_trace_ids=dst_trace_ids_sorted,
+        source_items_compare=src_items_compare,
+        destination_items_compare=dest_items_sorted,
+    )