chore: import upstream snapshot with attribution

sdks/python/.ruff.toml

@@ -0,0 +1,62 @@
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "src/opik/rest_api",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.10
+target-version = "py310"
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"

sdks/python/AGENTS.md

@@ -0,0 +1,77 @@
+# Repository Guidelines
+
+## Scope & Inheritance
+- This file contains Python SDK specifics only.
+- Follow `../../AGENTS.md` for shared monorepo workflow, PR, and security policy.
+
+## Project Structure & Module Organization
+This SDK lives under `sdks/python`.
+- `src/opik/`: Python package source.
+- `tests/`: test suite, organized into `unit/`, `integration/`, `e2e/`, `e2e_library_integration/`, and `e2e_smoke/`.
+- `examples/`: runnable integration examples and recipes.
+- `design/` and `outputs/`: design assets and generated artifacts.
+- `README.md`: SDK overview and contributor entry points.
+
+## Build, Test, and Development Commands
+See also `../../AGENTS.md#build-test-and-development-commands` for full monorepo commands.
+Run commands from `sdks/python` unless noted.
+- `pip install -r tests/test_requirements.txt && pytest tests/unit tests/integration tests/e2e`: install test dependencies and run standard tests.
+- `pytest tests/e2e_library_integration tests/e2e_smoke`: run higher-cost integration coverage.
+- `cd "$(git rev-parse --show-toplevel)" && make precommit`: run formatting, linting, and mypy hooks on changed files (vs origin/main) via the root pre-commit config.
+- `opik configure --use_local` (or `opik configure`): local SDK configuration for local/dev environments.
+
+## Coding Style & Naming Conventions
+- Python target matches the module’s supported versions in `pyproject.toml` (currently 3.10+) with 4-space indentation and line length 88.
+- Primary style tooling: `ruff` and `ruff format` (configured in `.ruff.toml`) plus `mypy` (via pre-commit).
+- Prefer explicit names, avoid abbreviations; avoid `utils.py`/`helpers.py` style catch-alls.
+- Prefer module-style imports over single-name imports in new code.
+- Keep names private with `_` prefix only when not used outside the module.
+- Keep comments focused on intent (“why”), not mechanics (“what”).
+
+## Testing Guidelines
+- Prefer unit tests (`tests/unit`) for behavior changes.
+- Add integration tests when touching backend or integration behavior, and e2e tests for cross-system flows.
+- Use existing fixture patterns in `tests/unit` and `tests/library_integration`.
+- Run focused suites before PR submission; avoid relying only on broad e2e runs when unit tests suffice.
+- File naming: `test_*.py` under `tests/<category>/`.
+
+### E2E test isolation contract (`tests/e2e/`)
+
+The e2e suite runs under `pytest-xdist` with `--dist=loadfile`: each test file is dispatched to one worker, and multiple files run in parallel against a shared backend. Resource names must therefore not collide across files.
+
+- **Backend project name** for a test module comes from `generate_project_name("e2e", __name__)` (helper in `tests/testlib/project_naming.py`, re-exported from `tests.testlib`). Files that need to reference the project (verifier fallback, `search_traces`, etc.) declare at module top:
+  ```python
+  from ..testlib import generate_project_name
+  PROJECT_NAME = generate_project_name("e2e", __name__)
+  ```
+  Reference `PROJECT_NAME` directly in test bodies — do not introduce a `project_name = PROJECT_NAME` indirection. The autouse `configure_e2e_tests_env` fixture reads `PROJECT_NAME` from each test module and patches `OPIK_PROJECT_NAME`, so the constant is the single source of truth. Files that don't reference the project name in Python don't need to declare anything; the fixture falls back to deriving a name from the module.
+- **Alternative projects** — used to exercise the `project_name=` override path — must not embed `generate_project_name(...)` as a `@pytest.mark.parametrize` decorator value. Every worker collects every parametrize id, and xdist's collection-consistency check fails when ids differ across workers; `generate_project_name` returns a different value per process. Parametrize on a boolean and compute the project name inside the test body:
+  ```python
+  @pytest.mark.parametrize("override_project_name", [True, False])
+  def test_xxx(opik_client, override_project_name):
+      project_name = (
+          generate_project_name("e2e", "anonymization", "override")
+          if override_project_name else None
+      )
+      ...
+  ```
+  Each CI job has its own backend stack, and `--dist=loadfile` keeps each file on a single worker, so different workers computing different names is not a collision risk in practice.
+- **Per-test resources** — datasets, experiments, prompts, temporary projects — already use unique names via the `dataset_name`, `experiment_name`, `prompt_name`, `temporary_project_name` fixtures. Use them; do not invent your own per-test name.
+- **No raw `random_chars()` calls for project names.** Reach for it directly only when you need a non-project resource name and there is no fixture for it.
+- **No bare hardcoded literals for project / dataset / experiment / prompt / suite / annotation-queue / optimization names anywhere under `tests/e2e/**`.** Strings derived from a unique-per-test fixture (e.g. `f"test_optimization_{dataset_name}"`) are fine — `dataset_name` already injects a random suffix.
+- **`configure_e2e_tests_env` is autouse and module-scoped.** Do not narrow it; teardown ordering under xdist will surface narrower scopes as flake.
+- **xdist + classes**: with `--dist=loadfile` test classes are *not* split across workers — every test in a file (including those inside `class Test…`) runs on the same worker. Module-level constants and module-scoped fixtures span both module-level and class-level tests in that file. If you switch a file to `--dist=loadscope`, revisit the scope contract.
+
+If you find a hardcoded resource name during code review, treat it as a defect on the same severity as a missing teardown.
+
+## Agent Contribution Workflow
+- This module is part of the Opik monorepo; follow the shared workflow in `../../AGENTS.md#agent-contribution-workflow`.
+- Run relevant formatter and test commands in this file for Python SDK changes before requesting review.
+
+## Commit & Pull Request Guidelines
+- Follow shared commit/PR policy in `../../AGENTS.md`.
+- Python SDK-specific convention: use SDK-prefixed titles (for example `[OPIK-####] [SDK] ...`) when applicable.
+
+## Security & Configuration Tips
+- Follow shared security policy in `../../AGENTS.md`.
+- Python SDK-specific rule: configure credentials via `opik configure`/environment variables, never hardcode them.

sdks/python/LICENSE

@@ -0,0 +1,203 @@
+Copyright (c) Comet ML, Inc
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2024 Comet ML, Inc
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

sdks/python/README.md

@@ -0,0 +1,188 @@
+# Opik Python SDK
+
+[![PyPI version](https://img.shields.io/pypi/v/opik.svg)](https://pypi.org/project/opik/)
+[![Python versions](https://img.shields.io/pypi/pyversions/opik.svg)](https://pypi.org/project/opik/)
+[![Downloads](https://static.pepy.tech/badge/opik)](https://pepy.tech/project/opik)
+[![License](https://img.shields.io/github/license/comet-ml/opik)](https://github.com/comet-ml/opik/blob/main/LICENSE)
+
+The Opik Python SDK allows you to integrate your Python applications with the Opik platform, enabling comprehensive tracing, evaluation, and monitoring of your LLM systems. Opik helps you build, evaluate, and optimize LLM systems that run better, faster, and cheaper.
+
+Opik is an open-source LLM evaluation platform by [Comet](https://www.comet.com?from=llm&utm_source=opik&utm_medium=github&utm_content=python_sdk_readme&utm_campaign=opik). For more information about the broader Opik ecosystem, visit our main [GitHub repository](https://github.com/comet-ml/opik), [Website](https://www.comet.com/site/products/opik/), or [Documentation](https://www.comet.com/docs/opik/).
+
+## Quickstart
+
+Get started quickly with Opik using our interactive notebook:
+
+<a href="https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/opik_quickstart.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open Quickstart In Colab"/>
+</a>
+
+## Installation
+
+Install the `opik` package using pip or uv:
+
+```bash
+# using pip
+pip install opik
+
+# using uv (faster)
+uv pip install opik
+```
+
+## Configuration
+
+Configure the Python SDK by running the `opik configure` command. This will prompt you for your Opik server address (for self-hosted instances) or your API key and workspace (for Comet.com):
+
+```bash
+opik configure
+```
+
+You can also configure the SDK programmatically in your Python code:
+```python
+import opik
+
+# For Comet.com Cloud
+opik.configure(
+    api_key="YOUR_API_KEY",
+    workspace="YOUR_WORKSPACE", # Usually found in your Comet URL: https://www.comet.com/YOUR_WORKSPACE/...
+    project_name="optional-project-name" # Optional: set a default project for traces
+)
+
+# For self-hosted Opik instances
+# opik.configure(use_local=True, project_name="optional-project-name")
+```
+Refer to the [Python SDK documentation](https://www.comet.com/docs/opik/python-sdk-reference/) for more configuration options.
+
+### Dynamic Tracing Control
+
+Control tracing behavior at runtime without code changes:
+
+```python
+import opik
+
+# Disable tracing globally
+opik.set_tracing_active(False)
+
+# Check current state
+print(opik.is_tracing_active())  # False
+
+# Re-enable tracing
+opik.set_tracing_active(True)
+
+# Reset to configuration default
+opik.reset_tracing_to_config_default()
+```
+
+This is useful for:
+- Performance optimization in high-throughput systems
+- Conditional tracing based on user type or request parameters
+- Debugging and troubleshooting without redeployment
+- Implementing sampling strategies
+- Calls already in progress when you disable tracing still finish logging.
+
+See `examples/dynamic_tracing_cookbook.py` for comprehensive usage patterns.
+
+## Basic Usage: Tracing
+
+The easiest way to log traces is to use the `@opik.track` decorator:
+
+```python
+import opik
+
+# Ensure Opik is configured (see Configuration section above)
+# opik.configure(...)
+
+@opik.track
+def my_llm_function(user_question: str) -> str:
+    # Your LLM call or business logic here
+    # For example:
+    # response = openai.ChatCompletion.create(...)
+    response = f"Echoing: {user_question}"
+
+    # You can add metadata to your trace
+    opik.set_tags(["example", "basic-usage"])
+    opik.log_metadata({"question_length": len(user_question)})
+
+    return response
+
+my_llm_function("Hello, Opik!")
+```
+Traces will appear in your configured Opik project. Opik also offers many direct [integrations](https://www.comet.com/docs/opik/integrations/overview/) for popular LLM frameworks.
+
+## CLI Commands
+
+Opik provides powerful CLI commands for exporting and importing data between projects:
+
+- **Export**: Export traces, datasets, and prompts from projects to local JSON files
+- **Import**: Import data from local files into projects
+- **Migration**: Move data between projects or environments
+- **Backup**: Create local backups of your project data
+
+For detailed information about the CLI export/import functionality, see [Import/Export Commands](../../apps/opik-documentation/documentation/fern/docs/tracing/import_export_commands.mdx).
+
+## Development & Contribution Guidelines
+
+For a more general contribution guide (backend + frontend + SDK) see our root [Contribution guide](../../CONTRIBUTING.md).
+
+# Coding guidelines
+This guide is still in progress, however, it already contains useful information that you should know before submitting your PR.
+
+## General
+We care a lot about the code maintainability. Well-organized logic which is easy to extend, re-factor and, most importantly - **read**, is what we are striving for.
+1. Follow [SOLID](https://realpython.com/solid-principles-python/) principles. Pay special attention to the "Single Responsibility" one.
+2. Avoid large modules, large classes, and large functions. Separate the code properly and describe this separation with names, not with comments. (See [1])
+3. If the name is not used outside of the class/module - it should be `_protected`.
+4. Don't violate the access rules! We know that Python allows you to access _protected/__private variables, but in Opik we are quite strict about not abusing that, whether it's an internal code or a test (don't forget about [3]!).
+5. Use comments only for something non-trivial that is hard to describe in any other way. Apart from these cases, comments should be used to answer the question "Why?" not "What?".
+
+## Imports
+1. Import module - not name.
+    Instead of this:
+    ```python
+    from threading import Thread  # bad!
+    thread = Thread()
+    ```
+    do this:
+    ```python
+    import threading  # good!
+    thread = threading.Thread
+    ```
+
+2. If the import statement is too big, you can do the following
+    ```python
+    from opik.rest_api.core import error as rest_api_error  # ok!
+    ```
+
+3. If you are working in the namespace, you likely don't need to keep most of the parent namespaces
+    ```python
+    # inside opik.api_objects.dataset
+    from . import dataset_item  # ok!
+    ```
+
+4. Of course, there might be exceptions from this rule, for example, some common types can be imported as is.
+    ```python
+    from typing import Dict, List  # ok!
+    from opik.types import FeedbackScoreDict  # ok!
+    ```
+
+## Naming
+1. Avoid abbreviations. In the vast majority of cases, it is not a problem to use variable names. People spend more time understanding what "fs" means than reading the word "files" or "file_system".
+   ```python
+   for d in dataset_items:  # bad!
+
+   for item in dataset_items:  # ok!
+       ...
+   for dataset_item in dataset_items  # ok!
+       ...
+   ```
+2. Avoid creating modules like `utils.py`, `helpers.py`, `misc.py` etc. Especially in the big namespaces. They can quickly become dumps where people put everything that they haven't been able to create a better place for in 10 seconds after they started thinking about it. You can create those files though, but they should be localized in their namespaces designed for some specific features. In vast majority of cases there are better module names.
+
+## Testing
+We highly encourage writing tests and we develop a lot of features in a test-driven way.
+1. Test public API, don't violate privacy.
+2. If you are an external contributor - make sure that the unit tests and e2e tests are green (they can be executed anywhere because they don't require any API keys or permissions). For internal Opik developers everything should be green in the CI.
+3. If you have `if-statements` in your code or some non-trivial boiler-plate code - it's probably a reason to think about add some unit tests for that. The more complex your code, the higher chance you'll be asked to provide unit tests for it.
+4. If you are introducing a new feature that includes communication with the backend - it's better to add some e2e tests for that (at least the happy flow one).
+5. Avoid testing with e2e tests something that can be tested with unit tests. E2E tests are time-consuming.
+6. If you are introducing a change in one of the integrations (or a new integration), make sure the integration tests are working. They usually require API keys configured for the services the integration works with. When the external contributor opens a PR, their tests will not use our Github secrets so consider providing your repo with an API key required for the integration. In that case, we will see that the tests are green.
+7. We are using `fake_backend` fixture together with a special Opik assertions DSL(domain-specific language) for a lot of unit tests and library integration tests. We encourage you to use it as well! There is plenty of examples, you can take a look at `tests/unit/decorator/test_tracker_outputs.py` or `tests/library_integration/openai/test_openai.py`. It provides a pretty simple API for specifying the traces content you expect your feature to log.

sdks/python/design/INTEGRATIONS.md

@@ -0,0 +1,595 @@
+# Opik Python SDK: Integrations Architecture
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Integration Patterns](#integration-patterns)
+- [Method Patching Integrations](#method-patching-integrations)
+- [Callback Integrations](#callback-integrations)
+- [Hybrid Integrations](#hybrid-integrations)
+- [Streaming Strategies](#streaming-strategies)
+- [Token Usage and Cost Tracking](#token-usage-and-cost-tracking)
+
+## Overview
+
+The SDK provides automatic tracking for 12+ LLM frameworks through three architectural patterns. Integrations are designed to be lightweight, extensible, and framework-native.
+
+### Integration Catalog
+
+| Integration | Pattern | Location | Key Features |
+|-------------|---------|----------|--------------|
+| **OpenAI** | Method Patching | `integrations/openai/` | Multiple APIs, streaming, function calling |
+| **Anthropic** | Method Patching | `integrations/anthropic/` | Messages API, delta accumulation |
+| **Bedrock** | Method Patching | `integrations/bedrock/` | Multi-format aggregators, extensible |
+| **Google GenAI** | Method Patching | `integrations/genai/` | Multi-modal support |
+| **AISuite** | Method Patching | `integrations/aisuite/` | Unified interface |
+| **LangChain** | Callback | `integrations/langchain/` | BaseTracer, provider extractors, external context support |
+| **LlamaIndex** | Callback | `integrations/llama_index/` | Event parsing, dedicated client |
+| **DSPy** | Callback | `integrations/dspy/` | Isolated context, graph visualization |
+| **Haystack** | Callback | `integrations/haystack/` | Component-based |
+| **ADK** | Hybrid | `integrations/adk/` | OpenTelemetry interception + callbacks |
+| **CrewAI** | Hybrid | `integrations/crewai/` | Method wrapping + LiteLLM delegation |
+
+## Integration Patterns
+
+### Pattern Selection
+
+```
+Library Architecture Analysis:
+
+Does library provide callbacks/hooks?
+    │
+    ├─► Yes ─► Callbacks reliable and in-context?
+    │           │
+    │           ├─► Yes ─► Pure Callback
+    │           │           (LangChain, LlamaIndex, DSPy, Haystack)
+    │           │
+    │           └─► No ─► Hybrid (Callback + Patching)
+    │                       (ADK, CrewAI)
+    │
+    └─► No ─► Method Patching
+                (OpenAI, Anthropic, Bedrock, GenAI, AISuite)
+```
+
+### Callback Reliability Issues
+
+**Why callbacks alone may be insufficient**:
+
+1. **Completion guarantee**: Some frameworks skip END callbacks on exceptions
+2. **Context isolation**: Callbacks may execute in different thread/context than original call
+3. **Timing**: Callbacks may fire with delays, complicating context management
+
+**Solution**: Add patching/integration for OpenTelemetry interception (ADK) or external dependency tracking (CrewAI).
+
+## Method Patching Integrations
+
+### Architecture
+
+Method patching wraps client methods to intercept calls:
+
+```
+track_library(client) → Wraps methods → client.method() intercepted
+                                             ↓
+                                    BaseTrackDecorator
+                                             ↓
+                            _start_span_inputs_preprocessor
+                            (extract input, create span)
+                                             ↓
+                                  Call original method
+                                             ↓
+                                    _streams_handler
+                            (check if output is stream)
+                                             ↓
+                                    ┌────────┴────────┐
+                                    │                 │
+                                Stream?            Not stream
+                                    │                 │
+                            Patch stream              │
+                            Defer finalization        │
+                            Return patched            │
+                                    │                 │
+                                    └────────┬────────┘
+                                             ↓
+                            _end_span_inputs_preprocessor
+                            (extract output, usage, finalize span)
+                            (called immediately for non-streaming,
+                             or in finally block for streaming)
+```
+
+**All method patching integrations are idempotent**: Use `opik_tracked` marker to prevent double-wrapping.
+
+### OpenAI Integration
+
+**Files**:
+- `opik_tracker.py` - Main entry point, wraps client methods
+- `openai_chat_completions_decorator.py` - Chat completions decorator
+- `openai_responses_decorator.py` - Responses API decorator
+- `stream_patchers.py` - Stream iteration patching
+- `chat_completion_chunks_aggregator.py` - Chunk aggregation
+- `response_events_aggregator.py` - Response events aggregation
+
+**Wrapped Methods**:
+- `chat.completions.create()` - Standard chat API
+- `beta.chat.completions.parse()` - Structured outputs
+- `responses.create()` - Responses API
+
+**Streaming Support**: Handles `openai.Stream`, `openai.AsyncStream`, and `ChatCompletionStreamManager`.
+
+### Anthropic Integration
+
+**Files**:
+- `opik_tracker.py` - Main entry point
+- `messages_create_decorator.py` - Messages decorator
+- `stream_patchers.py` - Stream/context manager patching
+
+**Wrapped Methods**:
+- `messages.create()` - Both standard and streaming
+- `messages.stream()` - Context manager pattern
+
+**Key Implementation Detail**: **Delta Accumulation**
+
+Anthropic streams delta events (not complete chunks) that must be accumulated. Event accumulator builds complete message by merging deltas progressively.
+
+**Location**: `stream_patchers.py` - See accumulation logic
+
+### Bedrock Integration
+
+**Files**:
+- `opik_tracker.py` - Main entry point
+- `converse/converse_decorator.py` - Converse API
+- `invoke_model/invoke_model_decorator.py` - Legacy InvokeModel API
+- `invoke_model/chunks_aggregator/` - Extensible aggregator system
+
+**Wrapped Methods**:
+1. `client.converse()` - Unified Converse API
+2. `client.invoke_model()` - Legacy API (multiple formats)
+3. `client.invoke_agent()` - Agent invocations
+
+**Key Implementation Detail**: **Extensible Multi-Format Aggregator**
+
+**Problem**: Bedrock supports multiple model formats (Claude, Nova, Llama, Mistral) with different streaming structures.
+
+**Solution**: Registry pattern with pluggable aggregators.
+
+**Architecture** (`invoke_model/chunks_aggregator/`):
+- `base.py` - `ChunkAggregator` protocol
+- `format_detector.py` - Detection registry + aggregator registry
+- `claude.py`, `nova.py`, `llama.py`, `mistral.py` - Format-specific aggregators
+- `api.py` - Public interface: `detect_format()` + `aggregate_chunks_to_dataclass()`
+
+**Extensibility**: Add new format by creating module + registering in `format_detector.py`. Zero changes to existing code.
+
+**Benefits**: Open/Closed Principle, isolated testing, clear separation of concerns.
+
+**Documentation**: See `EXTENDING.md` and `README.md` in `chunks_aggregator/` directory.
+
+### Google GenAI Integration
+
+**Files**:
+- `opik_tracker.py` - Main entry point
+- `generate_content_decorator.py` - Content generation decorator
+- `stream_wrappers.py` - Stream handling
+- `generations_aggregators.py` - Chunk aggregation
+
+**Features**: Multi-modal support (text, images), streaming responses.
+
+### AISuite Integration
+
+**Files**:
+- `opik_tracker.py` - Main entry point
+- `aisuite_decorator.py` - Decorator implementation
+
+**Pattern**: Similar to OpenAI (unified interface across providers).
+
+## Callback Integrations
+
+### Architecture
+
+Callback integrations implement framework's callback interface:
+
+```
+Framework execution → Fires events → Callback methods
+                                             ↓
+                                    on_start() - Create span/trace
+                                    on_end() - Update and send
+                                    on_error() - Capture error, finalize
+```
+
+### LangChain Integration
+
+**Files**:
+- `opik_tracer.py` - Implements `BaseTracer`
+- `langgraph_tracer_injector.py` - Graph configuration injection for LangGraph
+- `langgraph_async_context_bridge.py` - Context propagation for async LangGraph nodes
+- `provider_usage_extractors/` - Provider-specific usage extraction
+- `helpers.py` - Utility functions
+- `base_llm_patcher.py` - Adds `base_url` to LLM dict (for provider ID)
+
+**Pattern**: Pure callback (extends `langchain_core.tracers.BaseTracer`)
+
+**Key Feature**: **Supports parent-child relations with external Opik spans/traces**
+
+When used within `@track` decorated functions or existing Opik trace context:
+- Detects existing trace in `context_storage`
+- Creates LangChain spans as children of current Opik span
+- Maintains proper hierarchy between Opik and LangChain operations
+
+Example:
+```python
+@opik.track                          # Opik trace + span
+def my_function():
+    chain.invoke(..., callbacks=[OpikTracer()])  # LangChain spans as children
+```
+
+**State Management**:
+- `_span_data_map: Dict[UUID, SpanData]` - Maps LangChain run_id to Opik span
+- `_created_traces_data_map: Dict[UUID, TraceData]` - Maps run_id to trace
+- `_externally_created_traces_ids: Set[str]` - Tracks external traces
+
+**Callback Methods** (implements full `BaseTracer` interface):
+
+**Chain callbacks**:
+- `_on_chain_start(run)` → Check for existing trace, create span as child if exists
+- `_on_chain_end(run)` → Finalize span, send to backend
+- `_on_chain_error(run)` → Capture error info, finalize span
+
+**LLM callbacks**:
+- `on_chat_model_start(...)` → Special handling for chat models
+- `_on_chat_model_start(run)` → Internal processing
+- `_on_llm_start(run)` → Create LLM span (type="llm"), extract provider
+- `_on_llm_end(run)` → Extract usage via provider extractors, send span
+- `_on_llm_error(run)` → Capture error, finalize span
+
+**Tool callbacks**:
+- `_on_tool_start(run)` → Create tool span (type="tool")
+- `_on_tool_end(run)` → Finalize tool span
+- `_on_tool_error(run)` → Capture error, finalize span
+
+Error callbacks ensure spans finalized even when LangChain operations fail.
+
+**Key Implementation Detail**: **Provider-Specific Usage Extractors**
+
+**Location**: `provider_usage_extractors/`
+
+**Challenge**: Each LangChain provider stores usage in different locations/formats within the `Run` object.
+
+**Solution**: Registry pattern with provider-specific extractors.
+
+Extractors:
+- `OpenAIUsageExtractor` - Extracts from `run.outputs.llm_output.token_usage`
+- `AnthropicUsageExtractor` - Handles Anthropic format
+- `BedrockUsageExtractor` - Handles Bedrock format
+- `GoogleUsageExtractor` - Handles Google format
+- See `usage_extractor.py` for full registry
+
+Each extractor knows where to find usage in that provider's Run structure.
+
+**LangGraph Support**:
+
+The integration provides enhanced support for LangGraph through:
+
+1. **`track_langgraph()` Function**: High-level wrapper that injects `OpikTracer` into the graph's default configuration, eliminating the need to pass `config={"callbacks": [opik_tracer]}` on every invocation.
+
+2. **Automatic Graph Visualization**: Extracts and stores Mermaid graph structure in trace metadata via `OpikTracer.set_graph()` method.
+
+3. **Async Context Bridge**: `extract_current_langgraph_span_data()` helper for propagating trace context to `@track`-decorated functions in async LangGraph nodes.
+
+**Usage Pattern**:
+```python
+from opik.integrations.langchain import OpikTracer, track_langgraph
+from langgraph.graph import StateGraph, START, END
+
+# Build and compile graph
+builder = StateGraph(State)
+builder.add_node("my_node", my_node_function)
+builder.add_edge(START, "my_node")
+builder.add_edge("my_node", END)
+app = builder.compile()
+
+# Track once
+opik_tracer = OpikTracer(tags=["production"])
+app = track_langgraph(app, opik_tracer)
+
+# All invocations automatically tracked
+result = app.invoke({"message": "Hello"})
+```
+
+**Implementation Details**:
+- `langgraph_tracer_injector.py` - Injects `OpikTracer` into graph's default config
+- `langgraph_async_context_bridge.py` - Extracts span data from LangGraph config for async context propagation
+- `OpikTracer.set_graph()` - Stores graph visualization in `_trace_default_metadata["_opik_graph_definition"]`
+
+### LlamaIndex Integration
+
+**Files**:
+- `callback.py` - Implements `BaseCallbackHandler`
+- `event_parsing_utils.py` - Parses LlamaIndex event payloads
+
+**Event Handling**:
+- `on_event_start(event_type, payload, event_id, parent_id)` → Parse payload, create span
+- `on_event_end(event_type, payload, event_id)` → Parse output/usage, send span
+
+**Event Parser** (`event_parsing_utils.py`): Extracts data from payloads based on `event_type` (EMBEDDING, QUERY, LLM, etc.).
+
+### DSPy Integration
+
+**Files**:
+- `callback.py` - Implements `dspy.utils.callback.BaseCallback`
+- `graph.py` - Mermaid graph builder for DSPy programs
+
+**Callbacks**:
+- `on_module_start/end()` - DSPy module execution
+- `on_lm_start/end()` - LM calls (extracts provider/model from "provider/model" format)
+- `on_tool_start/end()` - Tool executions
+
+**Key Implementation Detail**: **Global Context Storage with Safe Operations**
+
+Uses global `OpikContextStorage` instance, enabling `opik.opik_context` API access to spans/traces created by DSPy callbacks. This allows users to:
+- Access current span/trace data via `opik_context.get_current_span_data()` / `opik_context.get_current_trace_data()`
+- Update spans/traces via `opik_context.update_current_span()` / `opik_context.update_current_trace()`
+
+**Context Safety**: Uses `ensure_id` parameter for all context pop operations (`pop_span_data(ensure_id=...)`, `pop_trace_data(ensure_id=...)`) to prevent context corruption in concurrent scenarios or when DSPy callbacks coexist with `@track` decorated functions.
+
+**Graph Visualization**: Builds Mermaid diagram of DSPy program structure (`graph.py`).
+
+### Haystack Integration
+
+**Files**:
+- `opik_connector.py` - Component added to pipeline
+- `opik_tracer.py` - Tracer for pipeline execution
+- `converters.py` - Convert Haystack objects to Opik format
+
+**Pattern**: Component-based (added to pipeline, observes without modifying data flow).
+
+## Hybrid Integrations
+
+### ADK Integration
+
+**Files**:
+- `opik_tracer.py` - Agent callbacks
+- `patchers/adk_otel_tracer/opik_adk_otel_tracer.py` - OpenTelemetry tracer
+- `recursive_callback_injector.py` - Recursive callback injection
+- `graph/mermaid_graph_builder.py` - Agent graph visualization
+- `patchers/patchers.py` - Global patches
+
+**Why Hybrid**: ADK uses OpenTelemetry for internal tracing + provides agent callbacks.
+
+**Dual Approach**:
+
+1. **OpenTelemetry Patching** (`patchers/adk_otel_tracer/opik_adk_otel_tracer.py`):
+   - Intercepts `start_span()` calls from ADK
+   - Creates Opik spans instead
+   - Returns `INVALID_SPAN` (no-op for OpenTelemetry)
+   - Skips internal ADK spans via `_ADK_INTERNAL_SPAN_NAME_SKIP_LIST`
+
+2. **Agent Callbacks** (`opik_tracer.py`):
+   - `before/after_agent_callback`
+   - `before/after_model_callback`
+   - `before/after_tool_callback`
+   - Recursively injected into agent tree (`recursive_callback_injector.py`)
+
+**Key Implementation Details**:
+
+1. **OpenTelemetry Interception**: Instead of dual tracing (OTel + Opik), intercepts OTel tracer to create only Opik spans. Single tracing backend, no OpenTelemetry overhead. Callbacks is used only to update spans and traces, but it's OTel tracer that is responsible
+for creating them and working with context (it's done to benefit from reliability of OTel context manager)
+
+2. **Graph Visualization** (`graph/mermaid_graph_builder.py`): Generates Mermaid diagram of agent structure including:
+   - Agent types (Sequential, Loop, Parallel, LLM)
+   - Tools and their connections
+   - Subagent relationships
+   - Stored in trace metadata `_opik_graph_definition`
+
+### CrewAI Integration
+
+**Files**:
+- `opik_tracker.py` - Main tracking setup
+- `crewai_decorator.py` - Decorator for CrewAI methods
+- `flow_patchers.py` - Flow class patching
+
+**Why Hybrid**: CrewAI methods wrapped + LiteLLM used for LLM tracking + direct provider client patching for v1.0.0+.
+
+**Approach**:
+1. **Method Wrapping**: Wrap `Crew.kickoff`, `Agent.execute_task`, `Task.execute_sync`
+2. **LiteLLM Delegation**: Enable `litellm.track_litellm()` (CrewAI uses LiteLLM internally for v0.x)
+3. **Flow Patching**: Patch `Flow.__init__` to auto-wrap dynamically registered methods (v1.0.0+ only)
+4. **Provider Client Patching**: For v1.0.0+, directly patch OpenAI, Anthropic, Gemini, and Bedrock clients when `crew` argument is provided
+
+**Key Implementation Details**:
+
+1. **LiteLLM Delegation**: Reuses existing LiteLLM integration instead of duplicating LLM tracking logic.
+
+2. **Flow Patching** (`flow_patchers.py`): Patches constructor to wrap methods registered via `@start`, `@listen` decorators. Gracefully handles missing `Flow` class (not available in CrewAI < v1.0.0).
+
+3. **Graceful Degradation**: Handles missing provider libraries gracefully:
+   - If a provider library (e.g., `crewai.llms.providers.openai.completion`) is not installed, logs debug message and continues
+   - If tracking a specific provider client fails, logs warning and continues with other providers
+   - Ensures integration doesn't fail if some optional dependencies are missing
+
+**Usage**:
+```python
+# For CrewAI v0.x (LiteLLM-based)
+track_crewai(project_name="my-project")
+
+# For CrewAI v1.0.0+ (direct provider clients)
+crew = Crew(agents=[...], tasks=[...])
+track_crewai(project_name="my-project", crew=crew)  # crew argument enables LLM client tracking
+```
+
+## Streaming Strategies
+
+### Streaming Challenges
+
+1. **Deferred finalization**: Can't finalize span until stream consumed
+2. **User-controlled consumption**: User determines when/if stream is fully consumed
+3. **Chunk accumulation**: Need complete response for logging
+4. **Error handling**: Exceptions during iteration
+5. **Context cleanup**: Must finalize even if stream abandoned
+
+### Strategy 1: Monkey-Patch Class Iterator
+
+**Used by**: OpenAI (`openai.Stream`), Anthropic (`anthropic.Stream`)
+
+**Files**: `stream_patchers.py` in each integration
+
+**Approach**:
+1. Save original `__iter__` from class
+2. Create wrapper that accumulates chunks
+3. Replace class method: `Stream.__iter__ = wrapper`
+4. Mark instance: `stream.opik_tracked_instance = True`
+5. Attach span/trace data to instance
+6. Wrapper checks marker before processing
+
+**Key Pattern - Context Pop Before Streaming**:
+
+Before returning stream, pop span/trace from context:
+```python
+def _streams_handler(self, output, ...):
+    if is_stream(output):
+        # Pop BEFORE returning (stream consumed later)
+        span_to_end, trace_to_end = base_track_decorator.pop_end_candidates()
+        return patch_stream(output, span_to_end, trace_to_end, ...)
+```
+
+**Why**: Stream consumption happens after decorator returns. Popping prevents nested calls from seeing stale context.
+
+**Key Pattern - Finalization Guarantee**:
+
+All stream wrappers use `finally`:
+```python
+def wrapper(self):
+    try:
+        accumulated = []
+        for item in original(self):
+            accumulated.append(item)
+            yield item
+    finally:
+        # ALWAYS runs - even if stream not fully consumed
+        finalize_span(aggregator(accumulated), ...)
+```
+
+**Why**: User might break early or exception occurs. Span must finalize.
+
+### Strategy 2: Context Manager Patching
+
+**Used by**: Anthropic (`MessageStreamManager`)
+
+**Approach**:
+- Patch `__enter__` and `__exit__` of stream manager
+- Accumulate during iteration (between enter/exit)
+- Finalize in `__exit__`
+
+**Files**: `stream_patchers.py`
+
+Suitable for stream managers that use `with` statement pattern.
+
+### Strategy 3: Generator Wrapper
+
+**Used by**: Some Bedrock/GenAI cases
+
+**Location**: `opik/decorator/generator_wrappers.py`
+
+**Approach**: Wrap generator without modifying library classes. Returns custom proxy that finalizes in `__del__` or explicit close.
+
+## Token Usage and Cost Tracking
+
+### OpikUsage - Standardized Format
+
+**Location**: `opik/llm_usage/opik_usage.py`
+
+All providers map to standardized format:
+```python
+class OpikUsage(pydantic.BaseModel):
+    completion_tokens: Optional[int]
+    prompt_tokens: Optional[int]
+    total_tokens: Optional[int]
+    provider_usage: Optional[BaseOriginalProviderUsage]  # Original preserved
+```
+
+### Usage Factory - Registry Pattern
+
+**Location**: `opik/llm_usage/opik_usage_factory.py`
+
+Registry with builder functions per provider:
+```python
+_PROVIDER_TO_OPIK_USAGE_BUILDERS: Dict[Provider, List[Callable]] = {
+    LLMProvider.OPENAI: [
+        OpikUsage.from_openai_completions_dict,
+        OpikUsage.from_openai_responses_dict,  # Multiple formats supported
+    ],
+    LLMProvider.ANTHROPIC: [OpikUsage.from_anthropic_dict],
+    LLMProvider.BEDROCK: [OpikUsage.from_bedrock_dict],
+    # ...
+}
+```
+
+**Process**:
+1. Integration extracts usage dict from response
+2. Calls `build_opik_usage(provider, usage_dict)`
+3. Factory tries each builder (supports multiple formats per provider)
+4. Returns standardized `OpikUsage`
+
+**Extensibility**: Add new provider by:
+1. Create `MyProviderUsage` class
+2. Add `from_myprovider_dict()` to `OpikUsage`
+3. Register in factory
+
+### Provider Enum
+
+**Location**: `opik/types.py`
+
+Supported providers for cost tracking:
+- `OPENAI`, `ANTHROPIC`, `BEDROCK`
+- `GOOGLE_VERTEXAI`, `GOOGLE_AI`
+- `COHERE`, `GROQ`
+- See `types.py` for complete list
+
+### Cost Calculation
+
+**SDK Responsibility**: Provide data
+- `model`: Model name (e.g., "gpt-4")
+- `provider`: Provider enum
+- `usage`: Token counts (OpikUsage)
+- `total_cost`: Optional override
+
+**Backend Responsibility**: Calculate cost
+- Pricing tables (model → price per token)
+- Region-specific pricing (Bedrock)
+- Token usage multiplication
+
+**Note**: Integrations do **not** calculate cost - only provide data for backend.
+
+## Summary
+
+**Integration Patterns**:
+- **Method Patching**: OpenAI, Anthropic, Bedrock, GenAI, AISuite
+- **Callback**: LangChain, LlamaIndex, DSPy, Haystack
+- **Hybrid**: ADK (callbacks + OTel), CrewAI (methods + LiteLLM)
+
+**Streaming Strategies**:
+- Class method patching (OpenAI, Anthropic Stream)
+- Context manager patching (Anthropic MessageStreamManager)
+- Generator wrapper (Bedrock, GenAI)
+
+**Key Patterns**:
+- **Idempotent tracking**: `opik_tracked` marker prevents double-wrapping
+- **Context pop for streams**: Pop before returning stream (consumed later)
+- **Finalization guarantee**: `finally` blocks ensure span completion
+- **Registry patterns**: Pluggable providers/formats/extractors
+- **Protocol-based**: Clear extension interfaces
+
+**Notable Implementations**:
+- **Bedrock**: Extensible aggregator system (add formats without modifying code)
+- **ADK**: OpenTelemetry interception (single tracing backend)
+- **LangChain**: External context support (composes with `@track`)
+- **DSPy**: Global context with safe operations (enables `opik_context` API access)
+- **CrewAI**: LiteLLM delegation (reuses existing integration)
+
+For implementation details, see source code in:
+- `opik/integrations/` - All integration implementations
+- `opik/llm_usage/` - Usage tracking and conversion
+- `opik/decorator/` - Base decorator and streaming utilities
+
+For more information, see:
+- [API and Data Flow](API_AND_DATA_FLOW.md) - Core SDK architecture
+- [Evaluation](EVALUATION.md) - Evaluation framework
+- [Testing](TESTING.md) - Testing integrations

sdks/python/design/README.md

@@ -0,0 +1,49 @@
+# Opik Python SDK Design Documentation
+
+Comprehensive architecture documentation for contributors and team members. These guides explain how the SDK works internally, not how to use it.
+
+## 📚 Documentation
+
+| Document | Priority | Description |
+|----------|----------|-------------|
+| **[API and Data Flow](API_AND_DATA_FLOW.md)** | ⭐ Start Here | Core architecture, 3 layers, sync vs async operations, batching, message processing |
+| **[Testing](TESTING.md)** | 🔵 Essential | Test categories, fake backend, TraceModel/SpanModel patterns |
+| **[Integrations](INTEGRATIONS.md)** | 🟣 As Needed | Integration patterns (method patching, callback, hybrid), streaming strategies |
+| **[Evaluation](EVALUATION.md)** | 🟣 As Needed | Evaluation engine, all 4 evaluation methods, metrics architecture |
+
+## 🚀 Quick Start
+
+### First-Time Contributors
+
+1. Read **[API and Data Flow](API_AND_DATA_FLOW.md)** - Understand core architecture
+2. Read **[Testing](TESTING.md)** - Learn testing patterns
+3. Choose domain doc based on your task
+
+### By Task
+
+| Task | Document | Key Sections |
+|------|----------|--------------|
+| Understanding `@opik.track` | [API and Data Flow](API_AND_DATA_FLOW.md) | Decorator Data Flow, Context Management |
+| Adding integration | [Integrations](INTEGRATIONS.md) | Integration Patterns, existing integrations |
+| Creating metric, evaluation pipelines | [Evaluation](EVALUATION.md) | Metrics Architecture |
+| Debugging performance | [API and Data Flow](API_AND_DATA_FLOW.md) | Batching System, Performance |
+| Writing tests | [Testing](TESTING.md) | Testing Patterns, fake backend |
+
+## 🔄 Maintenance
+
+**Update documentation when**:
+- Major architectural changes
+- New patterns introduced
+- New integrations added
+- Performance optimizations
+
+**Quality standards**:
+- Accurate (reflects codebase)
+- Clear (easy for newcomers)
+- Practical (real examples)
+
+---
+
+**Last Updated**: 2025-01-20
+
+**Questions?** Open an issue or contact the SDK team.

sdks/python/examples/agent_config_demo.ipynb

@@ -0,0 +1,515 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7fb27b941602401d91542211134fc71a",
+   "metadata": {},
+   "source": [
+    "# Agent Config API Demo\n",
+    "\n",
+    "This notebook walks through the Opik **Agent Config** API:\n",
+    "\n",
+    "1. `get_or_create_config` — fetch from the backend, auto-creating from a fallback when nothing exists yet\n",
+    "2. `get_or_create_config` with `fallback` only (returns fallback when backend is unreachable or empty)\n",
+    "3. `create_config` — unconditionally write a new config version\n",
+    "4. `set_config_env` — tag a version with an environment name (e.g. `\"prod\"`, `\"staging\"`)\n",
+    "5. Fetching by `env` and by explicit `version` name\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "acae54e37e7d407bbb7b55eff062a284",
+   "metadata": {},
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9a63283cbaf04dbcab1f6479b197f3a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install opik --quiet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "8dd0d8092fe74a7c96281538738b07e2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import uuid\n",
+    "from typing import Optional\n",
+    "\n",
+    "import opik\n",
+    "from opik.api_objects.agent_config.cache import get_global_registry\n",
+    "\n",
+    "# Configure once — reads OPIK_API_KEY and OPIK_URL_OVERRIDE from env if set.\n",
+    "# opik.configure(use_local=True)  # swap for opik.configure() when using Opik Cloud"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "72eea5119410473aa328ad9291626812",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Project: agent-config-demo-e9013134\n"
+     ]
+    }
+   ],
+   "source": [
+    "client = opik.Opik()\n",
+    "\n",
+    "# Give each demo run its own project so configs don't bleed between runs.\n",
+    "PROJECT = f\"agent-config-demo-{uuid.uuid4().hex[:8]}\"\n",
+    "print(f\"Project: {PROJECT}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8edb47106e1a46a883d545849b8ab81b",
+   "metadata": {},
+   "source": [
+    "## Define a typed Config subclass\n",
+    "\n",
+    "`opik.Config` is a Pydantic-based model. Subclass it to declare the fields your agent needs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "10185d26023b46108eb7d9f57d49d2b3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class AgentConfig(opik.Config):\n",
+    "    temperature: float\n",
+    "    model: str\n",
+    "    system_prompt: Optional[str] = None"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8763a12b2bbd4a93a75aff182afb95dc",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 1. `get_or_create_config` — first call auto-creates from fallback\n",
+    "\n",
+    "The project has no config yet. `get_or_create_config` detects this and writes the\n",
+    "fallback values as the first version. The backend automatically tags the first\n",
+    "version as `\"prod\"`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7623eae2785240b9bd12b16a66d81610",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "OPIK: Started logging traces to the \"agent-config-demo-e9013134\" project at https://test.dev.comet.com/opik/api/v1/session/redirect/projects/?trace_id=019d8bf7-c5e8-7314-8849-2b52040092c0&path=aHR0cHM6Ly90ZXN0LmRldi5jb21ldC5jb20vb3Bpay9hcGkv.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== First call — auto-creates from fallback ===\n",
+      "  is_fallback : False\n",
+      "  temperature : 0.5\n",
+      "  model       : gpt-3.5-turbo\n",
+      "  system_prompt: You are a helpful assistant.\n"
+     ]
+    }
+   ],
+   "source": [
+    "FALLBACK_V1 = AgentConfig(\n",
+    "    temperature=0.5,\n",
+    "    model=\"gpt-3.5-turbo\",\n",
+    "    system_prompt=\"You are a helpful assistant.\",\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# get_or_create_config must be called from inside an @opik.track function.\n",
+    "@opik.track(project_name=PROJECT)\n",
+    "def run_agent(user_message: str):\n",
+    "    cfg = client.get_or_create_config(\n",
+    "        fallback=AgentConfig(\n",
+    "            temperature=0.54,\n",
+    "            model=\"gpt-3.5aaaa-turbo\",\n",
+    "            system_prompt=\"You are a helpful assistant.\",\n",
+    "        ),  # optional, but preferred\n",
+    "        project_name=PROJECT,\n",
+    "    )\n",
+    "    # cfg is an AgentConfig instance because we passed a typed fallback.\n",
+    "    print(f\"  is_fallback : {cfg.is_fallback}\")\n",
+    "    print(f\"  temperature : {cfg.temperature}\")\n",
+    "    print(f\"  model       : {cfg.model}\")\n",
+    "    print(f\"  system_prompt: {cfg.system_prompt}\")\n",
+    "    return cfg\n",
+    "\n",
+    "\n",
+    "print(\"=== First call — auto-creates from fallback ===\")\n",
+    "cfg_v1 = run_agent(\"Hello!\")\n",
+    "assert cfg_v1.is_fallback is False, (\n",
+    "    \"auto-created config should NOT be marked as fallback\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7cdc8c89c7104fffa095e18ddfef8986",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 2. `get_or_create_config` — returns fallback when backend unreachable\n",
+    "\n",
+    "If the backend times out or is unreachable **and** a `fallback` is provided,\n",
+    "`get_or_create_config` returns the fallback with `is_fallback=True` instead of\n",
+    "raising an error. We simulate this by passing an unreachable URL."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "b118ea5561624da68c537baed56e602f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Call against unreachable backend — returns fallback ===\n",
+      "  is_fallback : True\n",
+      "  temperature : 0.5  (from local fallback)\n",
+      "  model       : gpt-3.5-turbo\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Point a second client at a non-existent host to force a timeout.\n",
+    "unreachable_client = opik.Opik(\n",
+    "    host=\"http://127.0.0.1:19999\",  # nothing listening here\n",
+    "    api_key=\"demo\",\n",
+    ")\n",
+    "\n",
+    "\n",
+    "@opik.track(project_name=PROJECT)\n",
+    "def run_agent_offline(user_message: str):\n",
+    "    cfg = unreachable_client.get_or_create_config(\n",
+    "        fallback=FALLBACK_V1,\n",
+    "        project_name=PROJECT,\n",
+    "        timeout_in_seconds=2,\n",
+    "    )\n",
+    "    print(f\"  is_fallback : {cfg.is_fallback}\")\n",
+    "    print(f\"  temperature : {cfg.temperature}  (from local fallback)\")\n",
+    "    print(f\"  model       : {cfg.model}\")\n",
+    "    return cfg\n",
+    "\n",
+    "\n",
+    "print(\"=== Call against unreachable backend — returns fallback ===\")\n",
+    "offline_cfg = run_agent_offline(\"Hello offline!\")\n",
+    "assert offline_cfg.is_fallback is True, (\n",
+    "    \"should be marked as fallback when backend is unreachable\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "938c804e27f84196a10c8828c723f798",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 3. `create_config` — write a new version unconditionally\n",
+    "\n",
+    "`create_config` does not require a `@opik.track` context and always creates a new\n",
+    "version. It returns the version name (a string) that you can use later with\n",
+    "`set_config_env` or to fetch by explicit version."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "504fb2a444614c0babb325280ed9130a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Created version: 'v2'\n"
+     ]
+    }
+   ],
+   "source": [
+    "v2 = AgentConfig(\n",
+    "    temperature=0.8,\n",
+    "    model=\"gpt-4o\",\n",
+    "    system_prompt=\"You are an expert assistant. Think step by step.\",\n",
+    ")\n",
+    "\n",
+    "v2_name = client.create_config(\n",
+    "    v2,\n",
+    "    project_name=PROJECT,\n",
+    "    description=\"Upgraded to gpt-4o with chain-of-thought prompt\",\n",
+    ")\n",
+    "\n",
+    "print(f\"Created version: {v2_name!r}\")\n",
+    "assert isinstance(v2_name, str) and v2_name != \"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "59bbdb311c014d738909a11f9e486628",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 4. `set_config_env` — tag a version with an environment\n",
+    "\n",
+    "Right now `\"prod\"` still points to the v1 values (auto-tagged by the backend on first\n",
+    "write). We promote v2 to `\"prod\"` with `set_config_env`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "b43b363d81ae4b689946ece5c682cd59",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Version 'v2' is now tagged as 'prod'\n",
+      "Version 'v2' is now also tagged as 'staging'\n"
+     ]
+    }
+   ],
+   "source": [
+    "client.set_config_env(\n",
+    "    project_name=PROJECT,\n",
+    "    version=v2_name,\n",
+    "    env=\"prod\",\n",
+    ")\n",
+    "print(f\"Version {v2_name!r} is now tagged as 'prod'\")\n",
+    "\n",
+    "# Also tag the same version as 'staging' to show multi-env support.\n",
+    "client.set_config_env(\n",
+    "    project_name=PROJECT,\n",
+    "    version=v2_name,\n",
+    "    env=\"staging\",\n",
+    ")\n",
+    "print(f\"Version {v2_name!r} is now also tagged as 'staging'\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8a65eabff63a45729fe45fb5ade58bdc",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 5. Fetch by `env` — confirm prod now returns v2 values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c3933fab20d04ec698c2621248eb3be0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Fetch env='prod' (should return v2 after set_config_env) ===\n",
+      "  temperature : 0.8\n",
+      "  model       : gpt-4o\n",
+      "  system_prompt: You are an expert assistant. Think step by step.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Clear cache so we hit the backend, not a locally cached copy.\n",
+    "# get_global_registry().clear()\n",
+    "\n",
+    "\n",
+    "@opik.track(project_name=PROJECT)\n",
+    "def fetch_prod():\n",
+    "    return client.get_or_create_config(\n",
+    "        fallback=FALLBACK_V1,\n",
+    "        project_name=PROJECT,\n",
+    "        env=\"prod\",\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "print(\"=== Fetch env='prod' (should return v2 after set_config_env) ===\")\n",
+    "prod_cfg = fetch_prod()\n",
+    "print(f\"  temperature : {prod_cfg.temperature}\")\n",
+    "print(f\"  model       : {prod_cfg.model}\")\n",
+    "print(f\"  system_prompt: {prod_cfg.system_prompt}\")\n",
+    "\n",
+    "assert prod_cfg.temperature == 0.8\n",
+    "assert prod_cfg.model == \"gpt-4o\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4dd4641cc4064e0191573fe9c69df29b",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 6. Fetch by explicit `version` name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8309879909854d7188b41380fd92a7c3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Fetch by explicit version 'v2' ===\n",
+      "  temperature : 0.8\n",
+      "  model       : gpt-4o\n"
+     ]
+    }
+   ],
+   "source": [
+    "@opik.track(project_name=PROJECT)\n",
+    "def fetch_by_version(version_name: str):\n",
+    "    return client.get_or_create_config(\n",
+    "        fallback=FALLBACK_V1,\n",
+    "        project_name=PROJECT,\n",
+    "        version=version_name,\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "print(f\"=== Fetch by explicit version {v2_name!r} ===\")\n",
+    "by_name_cfg = fetch_by_version(v2_name)\n",
+    "print(f\"  temperature : {by_name_cfg.temperature}\")\n",
+    "print(f\"  model       : {by_name_cfg.model}\")\n",
+    "\n",
+    "assert by_name_cfg.temperature == 0.8\n",
+    "assert by_name_cfg.model == \"gpt-4o\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ed186c9a28b402fb0bc4494df01f08d",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## 7. Fetch without a fallback (generic `Config` return type)\n",
+    "\n",
+    "Omitting `fallback` returns a base `opik.Config` instance. Typed field access still\n",
+    "works through attribute lookup, but you lose static type-checking of the subclass."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "cb1e1581032b452c9409d6c6813c49d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "=== Fetch without fallback ===\n",
+      "  type        : Config\n",
+      "  is_fallback : False\n",
+      "  temperature : 0.8\n",
+      "  model       : gpt-4o\n"
+     ]
+    }
+   ],
+   "source": [
+    "get_global_registry().clear()\n",
+    "\n",
+    "\n",
+    "@opik.track(project_name=PROJECT)\n",
+    "def fetch_no_fallback():\n",
+    "    # No fallback — returns opik.Config, raises ConfigNotFound if project is empty.\n",
+    "    return client.get_or_create_config(project_name=PROJECT)\n",
+    "\n",
+    "\n",
+    "print(\"=== Fetch without fallback ===\")\n",
+    "no_fallback_cfg = fetch_no_fallback()\n",
+    "print(f\"  type        : {type(no_fallback_cfg).__name__}\")\n",
+    "print(f\"  is_fallback : {no_fallback_cfg.is_fallback}\")\n",
+    "print(f\"  temperature : {no_fallback_cfg.temperature}\")\n",
+    "print(f\"  model       : {no_fallback_cfg.model}\")\n",
+    "\n",
+    "assert type(no_fallback_cfg) is opik.Config\n",
+    "assert no_fallback_cfg.is_fallback is False\n",
+    "# Values come from the prod version (v2).\n",
+    "assert no_fallback_cfg.temperature == by_name_cfg.temperature\n",
+    "assert no_fallback_cfg.model == by_name_cfg.model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "379cbbc1e968416e875cc15c1202d7eb",
+   "metadata": {},
+   "source": [
+    "---\n",
+    "## Cleanup\n",
+    "\n",
+    "Delete the demo project so it doesn't clutter the workspace."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "277c27b1587741f2af2001be3712ef0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from opik.rest_api import core as rest_api_core\n",
+    "\n",
+    "try:\n",
+    "    project_id = client.rest_client.projects.retrieve_project(name=PROJECT).id\n",
+    "    client.rest_client.projects.delete_project_by_id(project_id)\n",
+    "    print(f\"Deleted project {PROJECT!r}\")\n",
+    "except rest_api_core.ApiError as e:\n",
+    "    print(f\"Could not delete project: {e}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

sdks/python/examples/dashboard_example.ipynb

@@ -0,0 +1,655 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cell0",
+   "metadata": {},
+   "source": "# Opik Dashboards — Python SDK\n\nA comprehensive walkthrough of the **dashboard** API on the `opik.Opik` client:\n\n| Step | Topic |\n| --- | --- |\n| 1 | Setup |\n| 2–3 | Create a `MULTI_PROJECT` dashboard scoped to a project |\n| 4 | Stats-card widgets (snapshot metrics) |\n| 5 | Time-series chart widgets |\n| 6 | Markdown / notes widget |\n| 7 | Update widgets |\n| 8 | Inspect and rearrange the grid layout |\n| 9 | Add sections and move widgets between sections |\n| 10 | Remove widgets |\n| 11–12 | Create an `EXPERIMENTS` dashboard with evaluation widgets |\n| 13 | Fetch and list dashboards |\n| 14 | Clean up |\n\n**Project scope** — `project_stats_card` and `project_metrics` widgets are project-scoped.\nPass `project_name` to `create_dashboard` once; the SDK automatically injects the project\ninto every project-scoped widget added via `add_widget`.\n\n**Metric-ID namespaces** — easy to mix up:\n\n| Widget | Field | Namespace | Example |\n| --- | --- | --- | --- |\n| `project_stats_card` | `metric` | lowercase-dotted | `trace_count`, `duration.p50` |\n| `project_metrics` | `metric_type` | ALL-CAPS | `TRACE_COUNT`, `DURATION` |"
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell1",
+   "metadata": {},
+   "source": [
+    "## 1 · Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install opik --quiet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import copy\n",
+    "\n",
+    "import opik\n",
+    "from opik import dashboard\n",
+    "\n",
+    "client = opik.Opik()\n",
+    "\n",
+    "PROJECT_NAME = \"Default Project\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell4",
+   "metadata": {},
+   "source": [
+    "## 2 · Create a MULTI_PROJECT dashboard\n",
+    "\n",
+    "`MULTI_PROJECT` dashboards support `project_stats_card`, `project_metrics`, and `text_markdown`\n",
+    "widgets. A new dashboard starts with a single *Overview* section whose `id` we capture for\n",
+    "adding widgets."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mp_dash = client.create_dashboard(\n",
+    "    name=\"SDK comprehensive demo\",\n",
+    "    type=dashboard.DashboardType.MULTI_PROJECT,\n",
+    "    description=\"Created from the Python SDK walkthrough\",\n",
+    "    project_name=PROJECT_NAME,\n",
+    ")\n",
+    "mp_section_id = mp_dash.sections[0].id\n",
+    "print(f\"Dashboard id : {mp_dash.id}\")\n",
+    "print(f\"Type         : {mp_dash.type}\")\n",
+    "print(f\"Scope        : {mp_dash.scope}\")\n",
+    "print(f\"Section id   : {mp_section_id}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell6",
+   "metadata": {},
+   "source": "## 3 · Project scope\n\nThe `project_name` passed to `create_dashboard` links the dashboard to a project.\nThe SDK then automatically injects the project into every project-scoped widget\n(`project_stats_card`, `project_metrics`) when you call `add_widget` — you do **not**\nneed to repeat the project in the widget config."
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(f\"Dashboard linked to project: {PROJECT_NAME!r}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell8",
+   "metadata": {},
+   "source": [
+    "## 4 · Stats-card widgets\n",
+    "\n",
+    "`project_stats_card` shows a **single current-value metric** for a project.\n",
+    "The `metric` field uses the **lowercase-dotted** namespace — see `dashboard.StatsCardMetric`\n",
+    "for the full list (trace counts, duration percentiles, token usage, costs, …).\n",
+    "\n",
+    "`source` selects whether the metric is computed over traces or spans."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Total trace count\n",
+    "sc_trace_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_STATS_CARD,\n",
+    "        title=\"Traces\",\n",
+    "        config=dashboard.ProjectStatsCardConfig(\n",
+    "            source=dashboard.TraceDataType.TRACES,\n",
+    "            metric=dashboard.StatsCardMetric.TRACE_COUNT,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Estimated total cost\n",
+    "sc_cost_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_STATS_CARD,\n",
+    "        title=\"Total cost\",\n",
+    "        config=dashboard.ProjectStatsCardConfig(\n",
+    "            source=dashboard.TraceDataType.TRACES,\n",
+    "            metric=dashboard.StatsCardMetric.TOTAL_ESTIMATED_COST_SUM,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Median latency (p50)\n",
+    "sc_p50_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_STATS_CARD,\n",
+    "        title=\"Latency p50\",\n",
+    "        config=dashboard.ProjectStatsCardConfig(\n",
+    "            source=dashboard.TraceDataType.TRACES,\n",
+    "            metric=dashboard.StatsCardMetric.DURATION_P50,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# LLM span count (source=SPANS to query span-level metrics)\n",
+    "sc_llm_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_STATS_CARD,\n",
+    "        title=\"LLM calls\",\n",
+    "        config=dashboard.ProjectStatsCardConfig(\n",
+    "            source=dashboard.TraceDataType.SPANS,\n",
+    "            metric=dashboard.StatsCardMetric.LLM_SPAN_COUNT,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "print(f\"Stats cards added, total widgets: {len(mp_dash.sections[0].widgets)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell10",
+   "metadata": {},
+   "source": [
+    "## 5 · Time-series chart widgets\n",
+    "\n",
+    "`project_metrics` renders a time-series for an aggregate metric.\n",
+    "The `metric_type` field uses the **ALL-CAPS** namespace — see `dashboard.ProjectMetricType`.\n",
+    "\n",
+    "Breakdowns split the series by a dimension: `MODEL`, `PROVIDER`, `TAGS`, `NAME`, etc.\n",
+    "Available chart types: `LINE` (default), `BAR`, `RADAR`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Line chart: duration over time, broken down by model\n",
+    "chart_duration_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_METRICS,\n",
+    "        title=\"Duration by model\",\n",
+    "        config=dashboard.ProjectMetricsConfig(\n",
+    "            metric_type=dashboard.ProjectMetricType.DURATION,\n",
+    "            chart_type=dashboard.ChartType.LINE,\n",
+    "            breakdown=dashboard.BreakdownConfig(field=dashboard.BreakdownField.MODEL),\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Bar chart: token usage over time\n",
+    "chart_tokens_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_METRICS,\n",
+    "        title=\"Token usage\",\n",
+    "        config=dashboard.ProjectMetricsConfig(\n",
+    "            metric_type=dashboard.ProjectMetricType.TOKEN_USAGE,\n",
+    "            chart_type=dashboard.ChartType.BAR,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Line chart: trace count broken down by tag\n",
+    "chart_count_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_METRICS,\n",
+    "        title=\"Trace count by tag\",\n",
+    "        config=dashboard.ProjectMetricsConfig(\n",
+    "            metric_type=dashboard.ProjectMetricType.TRACE_COUNT,\n",
+    "            chart_type=dashboard.ChartType.LINE,\n",
+    "            breakdown=dashboard.BreakdownConfig(field=dashboard.BreakdownField.TAGS),\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Line chart: estimated cost broken down by provider\n",
+    "chart_cost_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.PROJECT_METRICS,\n",
+    "        title=\"Cost by provider\",\n",
+    "        config=dashboard.ProjectMetricsConfig(\n",
+    "            metric_type=dashboard.ProjectMetricType.COST,\n",
+    "            chart_type=dashboard.ChartType.LINE,\n",
+    "            breakdown=dashboard.BreakdownConfig(\n",
+    "                field=dashboard.BreakdownField.PROVIDER\n",
+    "            ),\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "print(f\"Total widgets: {len(mp_dash.sections[0].widgets)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell12",
+   "metadata": {},
+   "source": [
+    "## 6 · Markdown / notes widget\n",
+    "\n",
+    "`text_markdown` renders freeform Markdown — useful for section headers, runbook links,\n",
+    "or context notes. It is valid in **both** `MULTI_PROJECT` and `EXPERIMENTS` dashboards.\n",
+    "\n",
+    "Widgets can also be created from a **raw dict**, which is the forward-compatible path\n",
+    "for backend fields not yet modelled in the SDK."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Typed config\n",
+    "notes_id = mp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.TEXT_MARKDOWN,\n",
+    "        title=\"\",\n",
+    "        config=dashboard.TextMarkdownConfig(\n",
+    "            content=(\n",
+    "                \"## Project overview\\n\"\n",
+    "                \"This dashboard tracks **Default Project** metrics.\\n\\n\"\n",
+    "                \"- Duration p50 / p90\\n\"\n",
+    "                \"- Token costs by provider\\n\"\n",
+    "                \"- Error rate\"\n",
+    "            )\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Raw-dict style: forward-compatible with new backend fields\n",
+    "raw_id = mp_dash.add_widget(\n",
+    "    {\n",
+    "        \"type\": dashboard.WidgetType.TEXT_MARKDOWN.value,\n",
+    "        \"title\": \"Raw dict widget\",\n",
+    "        \"config\": {\"content\": \"Built with the `opik` Python SDK.\"},\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "print(f\"Total widgets: {len(mp_dash.sections[0].widgets)}\")\n",
+    "print(f\"Notes id : {notes_id}\")\n",
+    "print(f\"Raw id   : {raw_id}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell14",
+   "metadata": {},
+   "source": [
+    "## 7 · Update widgets\n",
+    "\n",
+    "`update_widget` patches **only the fields you pass** — omitted kwargs are left unchanged.\n",
+    "Config is **merged**, not replaced, so you can change a single key without restating the\n",
+    "whole config object."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Change the chart title\n",
+    "mp_dash.update_widget(chart_duration_id, title=\"Duration by model (ms)\")\n",
+    "\n",
+    "# Swap the markdown note content (config merge)\n",
+    "mp_dash.update_widget(\n",
+    "    notes_id,\n",
+    "    config={\"content\": \"## Project overview (updated)\\nDashboard refreshed via SDK.\"},\n",
+    ")\n",
+    "\n",
+    "# Add a subtitle to the trace-count stats card\n",
+    "mp_dash.update_widget(sc_trace_id, subtitle=\"last 7 days\")\n",
+    "\n",
+    "# Switch the token-usage chart from BAR to LINE\n",
+    "mp_dash.update_widget(\n",
+    "    chart_tokens_id,\n",
+    "    config=dashboard.ProjectMetricsConfig(\n",
+    "        metric_type=dashboard.ProjectMetricType.TOKEN_USAGE,\n",
+    "        chart_type=dashboard.ChartType.LINE,\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Rename the dashboard and update its description\n",
+    "mp_dash.rename(\"SDK comprehensive demo (v2)\")\n",
+    "mp_dash.set_description(\"Updated via the Python SDK.\")\n",
+    "print(\"Name:\", mp_dash.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell16",
+   "metadata": {},
+   "source": [
+    "## 8 · Inspect and rearrange the grid layout\n",
+    "\n",
+    "The grid is **6 columns wide** with unlimited rows. Each widget has a `DashboardLayoutItem`\n",
+    "with `x` (column), `y` (row), `w` (width in columns), `h` (height in rows).\n",
+    "\n",
+    "`replace_sections` swaps the entire sections list in one call — use it to reposition\n",
+    "widgets, resize them, or reorder sections. All other mutators persist immediately after\n",
+    "each individual call."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "section = mp_dash.sections[0]\n",
+    "by_id = {w.id: w for w in section.widgets}\n",
+    "\n",
+    "print(f\"{'title':35s}  x  y  w  h\")\n",
+    "print(\"-\" * 50)\n",
+    "for li in section.layout:\n",
+    "    title = by_id[li.id].title or \"(no title)\"\n",
+    "    print(f\"{title:35s}  {li.x}  {li.y}  {li.w}  {li.h}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Rearrange: full-width notes banner at the top (row 0),\n",
+    "# four stats cards side-by-side below (row 2),\n",
+    "# charts below that (rows 4+).\n",
+    "new_section = copy.deepcopy(section)\n",
+    "\n",
+    "stats_ids = [sc_trace_id, sc_cost_id, sc_p50_id, sc_llm_id]\n",
+    "chart_ids = [chart_duration_id, chart_tokens_id, chart_count_id, chart_cost_id]\n",
+    "\n",
+    "for li in new_section.layout:\n",
+    "    if li.id == notes_id:\n",
+    "        # Full-width banner spanning all 6 columns\n",
+    "        li.x, li.y, li.w, li.h = 0, 0, 6, 2\n",
+    "    elif li.id == raw_id:\n",
+    "        # Small note pinned to the top-right\n",
+    "        li.x, li.y, li.w, li.h = 4, 2, 2, 2\n",
+    "    elif li.id in stats_ids:\n",
+    "        col = stats_ids.index(li.id)\n",
+    "        li.x, li.y, li.w, li.h = col, 2, 1, 2\n",
+    "    elif li.id in chart_ids:\n",
+    "        col = chart_ids.index(li.id)\n",
+    "        li.x, li.y, li.w, li.h = (col % 3) * 2, 4 + (col // 3) * 4, 2, 4\n",
+    "\n",
+    "mp_dash.replace_sections([new_section])\n",
+    "\n",
+    "print(\"Layout after rearrangement:\")\n",
+    "section = mp_dash.sections[0]\n",
+    "by_id = {w.id: w for w in section.widgets}\n",
+    "print(f\"{'title':35s}  x  y  w  h\")\n",
+    "print(\"-\" * 50)\n",
+    "for li in section.layout:\n",
+    "    title = by_id[li.id].title or \"(no title)\"\n",
+    "    print(f\"{title:35s}  {li.x}  {li.y}  {li.w}  {li.h}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell19",
+   "metadata": {},
+   "source": [
+    "## 9 · Add sections and move widgets\n",
+    "\n",
+    "`add_section` appends a new empty section.  \n",
+    "To move widgets between sections use `replace_sections` with the complete new state."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "analytics_section_id = mp_dash.add_section(\"Analytics\")\n",
+    "print(\"Sections:\", [s.title for s in mp_dash.sections])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Move the four chart widgets from Overview into the new Analytics section.\n",
+    "new_sections = [copy.deepcopy(s) for s in mp_dash.sections]\n",
+    "overview, analytics = new_sections\n",
+    "\n",
+    "move_ids = {chart_duration_id, chart_tokens_id, chart_count_id, chart_cost_id}\n",
+    "\n",
+    "# Extract chart widgets and their layout entries from Overview\n",
+    "moved_widgets = [w for w in overview.widgets if w.id in move_ids]\n",
+    "moved_layout = [li for li in overview.layout if li.id in move_ids]\n",
+    "\n",
+    "overview.widgets = [w for w in overview.widgets if w.id not in move_ids]\n",
+    "overview.layout = [li for li in overview.layout if li.id not in move_ids]\n",
+    "\n",
+    "# Re-position charts inside Analytics (2-wide, 4-tall, three per row)\n",
+    "for idx, li in enumerate(moved_layout):\n",
+    "    li.x, li.y, li.w, li.h = (idx % 3) * 2, (idx // 3) * 4, 2, 4\n",
+    "\n",
+    "analytics.widgets.extend(moved_widgets)\n",
+    "analytics.layout.extend(moved_layout)\n",
+    "\n",
+    "mp_dash.replace_sections(new_sections)\n",
+    "\n",
+    "for s in mp_dash.sections:\n",
+    "    print(f\"  [{s.title}]  {len(s.widgets)} widget(s)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell22",
+   "metadata": {},
+   "source": [
+    "## 10 · Remove widgets\n",
+    "\n",
+    "`remove_widget` removes a widget and its layout entry from whichever section contains it.\n",
+    "Raises `DashboardValidationError` if the ID is not found."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Remove the raw-dict markdown widget\n",
+    "mp_dash.remove_widget(raw_id)\n",
+    "\n",
+    "total = sum(len(s.widgets) for s in mp_dash.sections)\n",
+    "print(f\"Widgets after removal: {total}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell24",
+   "metadata": {},
+   "source": [
+    "## 11 · EXPERIMENTS dashboard\n",
+    "\n",
+    "`EXPERIMENTS` dashboards target evaluation results rather than live traces.\n",
+    "Supported widgets: `experiments_feedback_scores`, `experiment_leaderboard`, `text_markdown`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "exp_dash = client.create_dashboard(\n",
+    "    name=\"SDK experiments demo\",\n",
+    "    type=dashboard.DashboardType.EXPERIMENTS,\n",
+    "    description=\"Evaluation metrics overview\",\n",
+    ")\n",
+    "exp_section_id = exp_dash.sections[0].id\n",
+    "print(f\"Experiments dashboard id: {exp_dash.id}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell26",
+   "metadata": {},
+   "source": [
+    "## 12 · Experiments evaluation widgets\n",
+    "\n",
+    "`experiments_feedback_scores` plots feedback score distributions across experiments.  \n",
+    "`experiment_leaderboard` shows a ranked table of runs against a chosen metric.\n",
+    "\n",
+    "Pass `max_experiments_count` (1–100) to control how many recent experiments are included."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Bar chart: feedback scores across the last 10 experiments\n",
+    "fb_bar_id = exp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.EXPERIMENTS_FEEDBACK_SCORES,\n",
+    "        title=\"Feedback scores (bar)\",\n",
+    "        config=dashboard.ExperimentsFeedbackScoresConfig(\n",
+    "            chart_type=dashboard.ChartType.BAR,\n",
+    "            max_experiments_count=10,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Radar chart: quality shape across the last 5 experiments\n",
+    "fb_radar_id = exp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.EXPERIMENTS_FEEDBACK_SCORES,\n",
+    "        title=\"Feedback scores (radar)\",\n",
+    "        config=dashboard.ExperimentsFeedbackScoresConfig(\n",
+    "            chart_type=dashboard.ChartType.RADAR,\n",
+    "            max_experiments_count=5,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Leaderboard with ranking enabled by a specific feedback-score metric\n",
+    "leaderboard_id = exp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.EXPERIMENT_LEADERBOARD,\n",
+    "        title=\"Experiment leaderboard\",\n",
+    "        config=dashboard.ExperimentLeaderboardConfig(\n",
+    "            enable_ranking=True,\n",
+    "            ranking_metric=\"hallucination\",  # name of the feedback score to rank by\n",
+    "            ranking_direction=True,  # True = descending (higher score is better)\n",
+    "            selected_columns=[\"dataset_id\", \"created_at\", \"duration.p50\", \"pass_rate\"],\n",
+    "            max_rows=20,\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "# Context note\n",
+    "exp_dash.add_widget(\n",
+    "    dashboard.DashboardWidget(\n",
+    "        type=dashboard.WidgetType.TEXT_MARKDOWN,\n",
+    "        title=\"\",\n",
+    "        config=dashboard.TextMarkdownConfig(\n",
+    "            content=\"### About\\nTracks evaluation runs ranked by the **hallucination** metric.\"\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "print(f\"Experiments dashboard widgets: {len(exp_dash.sections[0].widgets)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell28",
+   "metadata": {},
+   "source": [
+    "## 13 · Fetch and list dashboards\n",
+    "\n",
+    "`get_dashboard` retrieves a single dashboard by ID (re-fetches from the backend).  \n",
+    "`get_dashboards` pages through all dashboards with an optional name filter."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fetch the multi-project dashboard by id\n",
+    "fetched_mp = client.get_dashboard(mp_dash.id)\n",
+    "print(f\"Fetched: {fetched_mp.name!r}  ({len(fetched_mp.sections)} section(s))\")\n",
+    "\n",
+    "# List all dashboards whose name contains \"SDK\"\n",
+    "found = client.get_dashboards(name=\"SDK\", max_results=20)\n",
+    "print(f\"\\nDashboards matching 'SDK' ({len(found)} found):\")\n",
+    "for d in found:\n",
+    "    print(f\"  {d.id[:8]}…  {d.type:15s}  {d.name!r}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell32",
+   "metadata": {},
+   "source": [
+    "## 14 · Clean up"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mp_dash.delete()\n",
+    "exp_dash.delete()\n",
+    "print(\"Both dashboards deleted.\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

sdks/python/examples/decorators.py

@@ -0,0 +1,30 @@
+from opik import track, flush_tracker
+from opik import opik_context
+
+
+@track()
+def f3(x):
+    # creates span3 attached to trace1 with parent span2
+    opik_context.update_current_span(tags=["tag-f3"])
+    print("Done f3")
+    return "f3 output"
+
+
+@track()
+def f2(x):
+    # creates span2 attached to trace1 with parent span1
+    f3("f3 input")
+    print("Done f2")
+    return "f2 output"
+
+
+@track()
+def f1(x, y, z=1):
+    # creates trace 1 and span 1
+    f2("f2 input")
+    print("Done f1")
+    return "f1 output"
+
+
+f1("f1 input", 42)
+flush_tracker()

sdks/python/examples/demo_data_generator.py

@@ -0,0 +1,167 @@
+# Setting up a demo project
+#
+# Evaluation traces & spans
+# We start with evaluation so it shows up at the bottom.
+# The evaluation is going to be tracked into a separate project from the demo traces.
+# It was run using a simple context with 3 sentences, and 3 questions asking about it.
+
+import opik
+import uuid6
+from demo_data import evaluation_traces, evaluation_spans, demo_traces, demo_spans
+
+UUID_MAP = {}
+
+
+def get_new_uuid(old_id):
+    """
+    The demo_data has the IDs hardcoded in, to preserve the relationships between the traces and spans.
+    However, we need to generate unique ones before logging them.
+    """
+    if old_id in UUID_MAP:
+        new_id = UUID_MAP[old_id]
+    else:
+        new_id = str(uuid6.uuid7())
+        UUID_MAP[old_id] = new_id
+    return new_id
+
+
+def create_demo_data(base_url: str, workspace_name, comet_api_key):
+    client = opik.Opik(
+        project_name="Demo evaluation",
+        workspace=workspace_name,
+        host=base_url,
+        api_key=comet_api_key,
+        batching=True,
+    )
+
+    for trace in sorted(evaluation_traces, key=lambda x: x["start_time"]):
+        new_id = get_new_uuid(trace["id"])
+        trace["id"] = new_id
+        client.trace(**trace)
+
+    for span in sorted(evaluation_spans, key=lambda x: x["start_time"]):
+        new_id = get_new_uuid(span["id"])
+        span["id"] = new_id
+        new_trace_id = get_new_uuid(span["trace_id"])
+        span["trace_id"] = new_trace_id
+        if "parent_span_id" in span:
+            new_parent_span_id = get_new_uuid(span["parent_span_id"])
+            span["parent_span_id"] = new_parent_span_id
+        client.span(**span)
+
+    client.flush()
+
+    # Demo traces and spans
+    # We have a simple chatbot application built using llama-index.
+    # We gave it the content of Opik documentation as context, and then asked it a few questions.
+
+    client = opik.Opik(
+        project_name="Demo chatbot 🤖",
+        workspace=workspace_name,
+        host=base_url,
+        api_key=comet_api_key,
+        batching=True,
+    )
+
+    for trace in sorted(demo_traces, key=lambda x: x["start_time"]):
+        new_id = get_new_uuid(trace["id"])
+        trace["id"] = new_id
+        client.trace(**trace)
+
+    for span in sorted(demo_spans, key=lambda x: x["start_time"]):
+        new_id = get_new_uuid(span["id"])
+        span["id"] = new_id
+        new_trace_id = get_new_uuid(span["trace_id"])
+        span["trace_id"] = new_trace_id
+        if "parent_span_id" in span:
+            new_parent_span_id = get_new_uuid(span["parent_span_id"])
+            span["parent_span_id"] = new_parent_span_id
+        client.span(**span)
+
+    # Prompts
+    # We now create 3 versions of a Q&A prompt. The final version is from llama-index.
+
+    client.create_prompt(
+        name="Q&A Prompt",
+        prompt="""Answer the query using your prior knowledge.
+    Query: {{query_str}}
+    Answer:
+    """,
+    )
+
+    client.create_prompt(
+        name="Q&A Prompt",
+        prompt="""Here is the context information.
+    -----------------
+    {{context_str}}
+    -----------------
+    Answer the query using the given context and not prior knowledge.
+
+    Query: {{query_str}}
+    Answer:
+    """,
+    )
+
+    client.create_prompt(
+        name="Q&A Prompt",
+        prompt="""You are an expert Q&A system that is trusted around the world.
+    Always answer the query using the provided context information, and not prior knowledge.
+    Some rules to follow:
+    1. Never directly reference the given context in your answer.
+    2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines.
+
+    Context information is below.
+    ---------------------
+    {{context_str}}
+    ---------------------
+    Given the context information and not prior knowledge, answer the query.
+    Query: {{query_str}}
+    Answer:
+    """,
+    )
+
+    # Dataset
+
+    dataset = client.get_or_create_dataset(name="Demo dataset")
+    dataset.insert(
+        [
+            {"input": "What is the best LLM evaluation tool?"},
+            {"input": "What is the easiest way to start with Opik?"},
+            {"input": "Is Opik open source?"},
+        ]
+    )
+
+    # In addition to creating the dataset, we also create a mapping from the dataset items to the traces. This will be handy for creating the experiment.
+
+    items = dataset.get_items()
+    dataset_id_map = {item["input"]: item["id"] for item in items}
+
+    # Experiment
+    # The experiment is constructed by joining the traces with the dataset items.
+
+    experiment = client.create_experiment(
+        name="Demo experiment", dataset_name="Demo dataset"
+    )
+    experiment_items = []
+
+    for trace in evaluation_traces:
+        trace_id = trace["id"]
+        dataset_item_id = dataset_id_map.get(trace.get("input", {}).get("input", " "))
+        if dataset_item_id is not None:
+            experiment_items.append(
+                opik.api_objects.experiment.experiment_item.ExperimentItemReferences(
+                    dataset_item_id=dataset_item_id, trace_id=trace_id
+                )
+            )
+
+    experiment.insert(experiment_items)
+
+    client.flush()
+
+
+if __name__ == "__main__":
+    base_url = "http://localhost:5173/api"
+    workspace_name = None
+    comet_api_key = None
+
+    create_demo_data(base_url, workspace_name, comet_api_key)

sdks/python/examples/distributed_tracing_example.py

@@ -0,0 +1,31 @@
+import threading
+import time
+from opik import track, flush_tracker
+from opik.opik_context import get_distributed_trace_headers
+
+
+@track()
+def remote_function(x):
+    time.sleep(0.1)
+    return "output-from-remote-function"
+
+
+def remote_node(x, opik_headers):
+    remote_function(x, opik_distributed_trace_headers=opik_headers)
+
+
+@track()
+def local_function(x):
+    opik_headers = get_distributed_trace_headers()
+
+    t1 = threading.Thread(
+        target=remote_node, args=("remote-function-input", opik_headers)
+    )
+    t1.start()
+    t1.join()
+
+    return "output-from-local-function"
+
+
+local_function("local-function-input")
+flush_tracker()

sdks/python/examples/dynamic_tracing_example.py

@@ -0,0 +1,199 @@
+"""
+Dynamic Tracing Control Example
+
+This example demonstrates how to enable and disable Opik tracing at runtime
+without modifying your instrumented code or restarting your application.
+"""
+
+import time
+from typing import Dict, Any
+
+import opik
+from opik.integrations import openai as openai_integration
+
+
+def simulate_openai_client() -> object:
+    """Create a mock OpenAI client for demonstration."""
+
+    class MockClient:
+        def __init__(self) -> None:
+            self.chat = type(
+                "Chat",
+                (),
+                {
+                    "completions": type(
+                        "Completions",
+                        (),
+                        {"create": lambda self, **kwargs: {"content": "Mock response"}},
+                    )()
+                },
+            )()
+
+        def __getattr__(self, name: str) -> Any:
+            return None
+
+    return MockClient()
+
+
+@opik.track(name="llm_call")
+def call_llm(prompt: str, user_type: str = "free") -> str:
+    """Simulate an LLM call with user type information."""
+    client = simulate_openai_client()
+    response = client.chat.completions.create(
+        model="gpt-3.5-turbo", messages=[{"role": "user", "content": prompt}]
+    )
+    return f"Response for {user_type} user: {response['content']}"
+
+
+@opik.track(name="data_processing")
+def process_data(data: Dict[str, Any]) -> Dict[str, Any]:
+    """Simulate data processing that we want to trace."""
+    result = {"processed": True, "item_count": len(data)}
+    time.sleep(0.01)  # Simulate work
+    return result
+
+
+def measure_performance(func, *args, iterations: int = 100) -> float:
+    """Measure average execution time of a function."""
+    start_time = time.time()
+    for _ in range(iterations):
+        func(*args)
+    end_time = time.time()
+    return (end_time - start_time) / iterations
+
+
+def main() -> None:
+    """Demonstrate dynamic tracing capabilities."""
+
+    print("=== Opik Dynamic Tracing Demo ===\n")
+
+    # 1. Basic enable/disable functionality
+    print("1. Basic Runtime Control")
+    print("-" * 30)
+
+    print(f"Initial tracing state: {opik.is_tracing_active()}")
+
+    # Disable tracing
+    opik.set_tracing_active(False)
+    print(f"After disabling: {opik.is_tracing_active()}")
+
+    # Call traced function - no traces will be created
+    result = call_llm("Hello world", "free")
+    print(f"Function result (no tracing): {result}")
+
+    # Re-enable tracing
+    opik.set_tracing_active(True)
+    print(f"After enabling: {opik.is_tracing_active()}\n")
+
+    # 2. Conditional tracing based on user type
+    print("2. Conditional Tracing by User Type")
+    print("-" * 40)
+
+    def handle_request(prompt: str, user_type: str) -> str:
+        """Handle request with conditional tracing."""
+        # Only trace premium users
+        should_trace = user_type == "premium"
+        opik.set_tracing_active(should_trace)
+
+        print(f"Processing {user_type} user request (tracing: {should_trace})")
+        return call_llm(prompt, user_type)
+
+    # Process different user types
+    handle_request("What is AI?", "free")
+    handle_request("Explain quantum computing", "premium")
+    handle_request("Hello", "free")
+    print()
+
+    # 3. Sampling-based tracing
+    print("3. Sampling-Based Tracing (10% of requests)")
+    print("-" * 50)
+
+    import random
+
+    def handle_request_with_sampling(request_id: int) -> Dict[str, Any]:
+        """Handle request with 10% sampling rate."""
+        should_trace = random.random() < 0.1  # 10% sampling
+        opik.set_tracing_active(should_trace)
+
+        data = {"request_id": request_id, "data": list(range(10))}
+        result = process_data(data)
+
+        if should_trace:
+            print(f"Request {request_id}: TRACED")
+        else:
+            print(f"Request {request_id}: not traced")
+
+        return result
+
+    # Process multiple requests
+    for i in range(10):
+        handle_request_with_sampling(i)
+    print()
+
+    # 4. Performance comparison
+    print("4. Performance Impact Comparison")
+    print("-" * 40)
+
+    test_data = {"items": list(range(100))}
+
+    # Measure with tracing enabled
+    opik.set_tracing_active(True)
+    time_with_tracing = measure_performance(process_data, test_data, iterations=50)
+
+    # Measure with tracing disabled
+    opik.set_tracing_active(False)
+    time_without_tracing = measure_performance(process_data, test_data, iterations=50)
+
+    print(f"Average time with tracing: {time_with_tracing * 1000:.2f}ms")
+    print(f"Average time without tracing: {time_without_tracing * 1000:.2f}ms")
+
+    if time_with_tracing > time_without_tracing:
+        overhead = (
+            (time_with_tracing - time_without_tracing) / time_without_tracing
+        ) * 100
+        print(f"Tracing overhead: {overhead:.1f}%")
+    print()
+
+    # 5. Integration tracking control
+    print("5. Integration Tracking Control")
+    print("-" * 40)
+
+    # Simulate tracking an OpenAI client
+    mock_client = simulate_openai_client()
+
+    # Disable tracing before setting up integration
+    opik.set_tracing_active(False)
+    openai_integration.track_openai(mock_client)
+    print(
+        "OpenAI client tracking setup with tracing disabled - no instrumentation applied"
+    )
+
+    # Enable tracing and set up integration
+    opik.set_tracing_active(True)
+    openai_integration.track_openai(mock_client)
+    print("OpenAI client tracking setup with tracing enabled - instrumentation applied")
+    print()
+
+    # 6. Reset to configuration default
+    print("6. Reset to Configuration Default")
+    print("-" * 40)
+
+    # Override runtime setting
+    opik.set_tracing_active(False)
+    print(f"Runtime override active: {opik.is_tracing_active()}")
+
+    # Reset to config default
+    opik.reset_tracing_to_config_default()
+    print(f"After reset to config: {opik.is_tracing_active()}")
+    print("(This will use the value from OPIK_TRACK_DISABLE or config file)")
+
+    print("\n=== Demo Complete ===")
+    print("Key benefits of dynamic tracing:")
+    print("• Zero code changes required")
+    print("• Runtime performance optimization")
+    print("• Flexible sampling strategies")
+    print("• Easy debugging and troubleshooting")
+
+
+if __name__ == "__main__":
+    main()

sdks/python/examples/evaluate_existing_experiment.py

@@ -0,0 +1,20 @@
+from opik.evaluation import evaluate_experiment
+from opik.evaluation.metrics import base_metric, score_result
+
+
+class MyCustomMetric(base_metric.BaseMetric):
+    def __init__(self, name: str):
+        self.name = name
+
+    def score(self, **ignored_kwargs):
+        # Add you logic here
+
+        return score_result.ScoreResult(
+            value=10, name=self.name, reason="Optional reason for the score"
+        )
+
+
+evaluate_experiment(
+    experiment_name="round_trellis_3225",
+    scoring_metrics=[MyCustomMetric(name="custom-metric")],
+)

sdks/python/examples/evaluate_prompt.py

@@ -0,0 +1,24 @@
+import opik
+from opik.evaluation import evaluate_prompt
+
+# Create a dataset that contains the samples you want to evaluate
+opik_client = opik.Opik()
+dataset = opik_client.get_or_create_dataset("my_dataset")
+dataset.insert(
+    [
+        {"question": "Hello, world!", "expected_output": "Hello, world!"},
+        {"question": "What is the capital of France?", "expected_output": "Paris"},
+    ]
+)
+
+# Run the evaluation
+evaluate_prompt(
+    dataset=dataset,
+    messages=[
+        {
+            "role": "user",
+            "content": "Translate the following text to French: {{question}}",
+        },
+    ],
+    model="gpt-3.5-turbo",
+)

sdks/python/examples/evaluation_example.py

@@ -0,0 +1,93 @@
+from typing import Dict, Any, List
+
+from opik.evaluation.metrics import IsJson, Hallucination, score_result
+from opik.evaluation import evaluate, test_result
+from opik import Opik, track
+from opik.integrations.openai import track_openai
+import openai
+
+
+# os.environ["OPENAI_ORG_ID"] = "<>"
+# os.environ["OPENAI_API_KEY"] = "<>"
+
+openai_client = track_openai(openai.OpenAI())
+
+is_json = IsJson()
+hallucination = Hallucination()
+
+client = Opik()
+dataset = client.get_or_create_dataset(
+    name="My 42 dataset", description="For storing stuff"
+)
+
+json = """
+    [
+        {
+            "Model inputs": {"message": "Greet me!", "context": []}
+        },
+        {
+            "Model inputs": {"message": "Ok, I'm leaving, bye!", "context": []}
+        },
+        {
+            "Model inputs": {"message": "How are you doing?", "context": []}
+        },
+        {
+            "Model inputs": {"message": "Give a json example!", "context": []}
+        },
+        {
+            "Model inputs": {
+                "message": "What is the main currency in european union?",
+                "context": ["Euro is the main european currency. It is used across most EU countries"]
+            }
+        }
+    ]
+"""
+
+dataset.insert_from_json(json_array=json, keys_mapping={"Model inputs": "input"})
+
+
+@track()
+def llm_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    response = openai_client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": item["input"]["message"]}],
+    )
+
+    return {
+        "output": response.choices[0].message.content,
+        "reference": "test",
+    }
+
+
+def compute_hallucination_stats(
+    test_results: List[test_result.TestResult],
+) -> List[score_result.ScoreResult]:
+    # Extract scores safely, checking for empty score_results
+    scores = [
+        x.score_results[0].value
+        for x in test_results
+        if x.score_results and len(x.score_results) > 0
+    ]
+
+    # Return empty list if no scores available
+    if not scores:
+        return []
+
+    return [
+        score_result.ScoreResult(
+            name="Custom metric",
+            value=max(scores) if len(scores) > 1 else 0.0,
+        )
+    ]
+
+
+results = evaluate(
+    experiment_name="My experiment",
+    dataset=dataset,
+    task=llm_task,
+    nb_samples=2,
+    scoring_metrics=[is_json, hallucination],
+    experiment_scoring_functions=[compute_hallucination_stats],
+)
+
+print(results)

sdks/python/examples/feedback_scores_example.py

@@ -0,0 +1,27 @@
+import opik
+
+client = opik.Opik()
+
+trace = client.trace(
+    name="trace-name",
+)
+span1 = trace.span(name="span-1")
+span2 = span1.span(name="span-2")
+
+span2.end()
+span1.end()
+trace.end()
+
+span1.log_feedback_score(name="toxicity", value=0.0, reason="Too many bad words")
+client.log_spans_feedback_scores(
+    [
+        {"id": span2.id, "name": "toxicity", "value": 0.5},
+        {
+            "id": span2.id,
+            "name": "truthfullness",
+            "value": 1.0,
+            "reason": "some good reason",
+        },
+    ]
+)
+client.end()

sdks/python/examples/harbor_integration_example.py

@@ -0,0 +1,99 @@
+"""
+Harbor Integration Example
+
+Track Harbor benchmark runs with Opik. The integration follows Opik's standard
+patterns (like CrewAI) and creates hierarchical spans for trial execution:
+
+Trace: {agent_name}/{trial_name}
+├── Span: setup_environment
+├── Span: setup_agent
+├── Span: execute_agent
+│   └── [trajectory step spans streamed in real-time]
+├── Span: run_verification
+│   └── Span: verify
+
+Features:
+- Automatic tracing of Trial.run and all sub-methods
+- Real-time streaming of trajectory steps during agent execution
+- Verifier rewards captured as feedback scores
+- Token usage and cost tracking from trajectory metrics
+- Automatic dataset and experiment creation for evaluation tracking
+
+The integration automatically:
+- Creates an Opik dataset for each Harbor dataset source (e.g., "terminal-bench")
+- Creates an experiment named `harbor-job-{job_id[:8]}` to group all trial traces
+- Links each trial's trace to the experiment as an experiment item
+
+Prerequisites:
+    pip install opik harbor
+    opik configure
+    Docker must be running
+
+Usage:
+    OPENAI_API_KEY=... python harbor_integration_example.py
+"""
+
+import asyncio
+from datetime import datetime
+from pathlib import Path
+
+from harbor.job import Job
+from harbor.models.job.config import (
+    AgentConfig,
+    JobConfig,
+    EnvironmentConfig,
+    OrchestratorConfig,
+    RegistryDatasetConfig,
+)
+from harbor.models.registry import RemoteRegistryInfo
+
+from opik.integrations.harbor import track_harbor
+
+
+async def main():
+    # Configure agent - terminus-2 creates trajectory files for detailed tracing
+    # Requires OPENAI_API_KEY environment variable
+    agent = AgentConfig(
+        name="terminus-2",
+        model_name="gpt-4o-mini",
+        override_timeout_sec=30,  # 30 second timeout for demo
+    )
+
+    # Configure Terminal-Bench 2.0 dataset from Harbor registry
+    # See all tasks: https://github.com/laude-institute/terminal-bench-2
+    dataset = RegistryDatasetConfig(
+        registry=RemoteRegistryInfo(),
+        name="terminal-bench",
+        version="2.0",
+        task_names=["fix-git", "chess-best-move"],
+    )
+
+    # Create Harbor job with unique timestamp-based name
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    job = Job(
+        JobConfig(
+            job_name=f"opik-terminal-bench-{timestamp}",
+            jobs_dir=Path("./harbor_jobs"),
+            orchestrator=OrchestratorConfig(n_concurrent_trials=1),
+            environment=EnvironmentConfig(delete=True),
+            agents=[agent],
+            datasets=[dataset],
+        )
+    )
+
+    # Enable Opik tracking - patches Trial class methods globally
+    # This follows the same pattern as track_crewai, track_openai, etc.
+    tracked_job = track_harbor(
+        job,
+        project_name="terminal-bench-demo",
+    )
+
+    # Run benchmark - traces are created automatically
+    result = await tracked_job.run()
+
+    print(f"\nCompleted {result.stats.n_trials} trials, {result.stats.n_errors} errors")
+    print("View traces at: https://www.comet.com/opik")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

sdks/python/examples/langchain_integration_example.py

@@ -0,0 +1,24 @@
+from langchain_community.llms import fake
+from langchain.prompts import PromptTemplate
+from opik.integrations.langchain.opik_tracer import OpikTracer
+
+
+# @opik.track(capture_input=False)
+def f(test_prompts, chain, callback):
+    result = chain.invoke(input=test_prompts, config={"callbacks": [callback]})
+    return result
+
+
+llm = fake.FakeListLLM(
+    responses=["I'm sorry, I don't think I'm talented enough to write a synopsis"]
+)
+template = "Given the title of play, write a synopsys for that. Title: {title}."
+prompt_template = PromptTemplate(input_variables=["title"], template=template)
+synopsis_chain = prompt_template | llm
+callback = OpikTracer(tags=["tag1", "tag2"], metadata={"a": "b"})
+
+test_prompts = {"title": "Documentary about Bigfoot in Paris"}
+
+print(f(test_prompts, synopsis_chain, callback))
+
+callback.flush()

sdks/python/examples/manual_chain_building.py

@@ -0,0 +1,15 @@
+import opik
+import os
+
+os.environ["OPIK_URL_OVERRIDE"] = "http://localhost:5173/api"
+client = opik.Opik()
+
+trace = client.trace(name="trace-1")
+span1 = trace.span(name="span-1")
+span2 = span1.span(name="span-2")
+span2.end()
+span1.end()
+
+trace.end()
+
+client.end()

sdks/python/examples/metrics.py

@@ -0,0 +1,215 @@
+from opik.evaluation import metrics
+
+# Hallucination metric example
+if True:
+    print("\n\nHallucination metric example:")
+
+    hallucination_metric = metrics.Hallucination()
+
+    hallucination_score = hallucination_metric.score(
+        input="What is the capital of France?",
+        output="The capital of France is Paris. It is famous for its iconic Eiffel Tower and rich cultural heritage.",
+    )
+    print("hallucination_score:", hallucination_score)
+
+# G-Eval metric example
+if True:
+    print("\n\nG-Eval metric example:")
+
+    g_eval_metric = metrics.GEval(
+        task_introduction="You are an expert judge tasked with evaluating the faithfulness of an AI-generated answer to the given context.",
+        evaluation_criteria="The OUTPUT must not introduce new information beyond what's provided in the CONTEXT.",
+        # model="ollama/llama3"
+    )
+
+    g_eval_score = g_eval_metric.score(
+        output=str(
+            {
+                "OUTPUT": "What is the capital of France?",
+                "CONTEXT": [
+                    "France is a country in Western Europe. Its capital is Paris, which is known for landmarks like the Eiffel Tower."
+                ],
+            }
+        )
+    )
+    print("g_eval_score:", g_eval_score)
+
+# Moderation metric example
+if True:
+    print("\n\nModeration metric example:")
+
+    moderation_metric = metrics.Moderation()
+
+    moderation_score = moderation_metric.score(
+        input="What is the capital of France?",
+        output="The capital of France is Paris. It is famous for its iconic Eiffel Tower and rich cultural heritage.",
+        context=[
+            "France is a country in Western Europe. Its capital is Paris, which is known for landmarks like the Eiffel Tower."
+        ],
+    )
+
+    print("moderation_score:", moderation_score)
+
+# Answer Relevance metric example
+if True:
+    print("\n\nAnswer Relevance metric example:")
+
+    answer_relevance_metric = metrics.AnswerRelevance()
+    answer_relevance_score = answer_relevance_metric.score(
+        input="What is the capital of France?",
+        output="The capital of France is Paris. It is famous for its iconic Eiffel Tower and rich cultural heritage.",
+        context=[
+            "France is a country in Western Europe. Its capital is Paris, which is known for landmarks like the Eiffel Tower."
+        ],
+    )
+    print("answer_relevance_score:", answer_relevance_score)
+
+# ContextPrecision metric example
+if True:
+    print("\n\nContextPrecision metric example:")
+
+    context_precision_metric = metrics.ContextPrecision()
+    context_precision_score = context_precision_metric.score(
+        input="What is the capital of France?",
+        output="The capital of France is Paris. It is famous for its iconic Eiffel Tower and rich cultural heritage.",
+        expected_output="Paris",
+        context=[
+            "France is a country in Western Europe. Its capital is Paris, which is known for landmarks like the Eiffel Tower."
+        ],
+    )
+    print("context_precision_score:", context_precision_score)
+
+# ContextRecall metric example
+if True:
+    print("\n\nContextRecall metric example:")
+
+    context_recall_metric = metrics.ContextRecall()
+    context_recall_score = context_recall_metric.score(
+        input="What is the capital of France?",
+        output="The capital of France is Paris. It is famous for its iconic Eiffel Tower and rich cultural heritage.",
+        expected_output="Paris",
+        context=[
+            "France is a country in Western Europe. Its capital is Paris, which is known for landmarks like the Eiffel Tower."
+        ],
+    )
+    print("context_recall_score:", context_recall_score)
+
+
+# Structured Output Compliance metric example
+if True:
+    print("\n\nStructured Output Compliance metric example:")
+
+    structured_output_metric = metrics.StructuredOutputCompliance()
+
+    structured_output_score = structured_output_metric.score(
+        output='{"name": "Alice", "age": 30}',
+        schema='{"type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"]}',
+    )
+
+    print("structured_output_score:", structured_output_score)
+
+# TrajectoryAccuracy metric example
+if True:
+    print("\n\nTrajectoryAccuracy metric example:")
+
+    trajectory_accuracy_metric = metrics.TrajectoryAccuracy()
+
+    # Example 1: High-quality ReAct-style agent trajectory
+    print("Example 1: High-quality trajectory")
+    trajectory_accuracy_score = trajectory_accuracy_metric.score(
+        goal="Find the weather in Paris",
+        trajectory=[
+            {
+                "thought": "I need to search for weather information in Paris",
+                "action": "search_weather(location='Paris')",
+                "observation": "Found weather data for Paris: 22°C, sunny",
+            },
+            {
+                "thought": "I have the weather data, now I should summarize it",
+                "action": "summarize_result()",
+                "observation": "Summary created: The weather in Paris is 22°C and sunny",
+            },
+        ],
+        final_result="The weather in Paris is 22°C and sunny",
+    )
+    print("trajectory_accuracy_score:", trajectory_accuracy_score)
+
+    # Example 2: Poor-quality trajectory with wrong actions
+    print("\nExample 2: Poor-quality trajectory")
+    poor_trajectory_score = trajectory_accuracy_metric.score(
+        goal="Find the capital of France",
+        trajectory=[
+            {
+                "thought": "I need to find France's capital",
+                "action": "search('weather in France')",  # Wrong action
+                "observation": "Found weather information for various French cities",
+            },
+            {
+                "thought": "This doesn't help, let me try something else",
+                "action": "search('French cuisine')",  # Still wrong
+                "observation": "Found information about French food",
+            },
+        ],
+        final_result="Paris is the capital of France",  # Result doesn't match trajectory
+    )
+    print("poor_trajectory_score:", poor_trajectory_score)
+
+# SycEval metric example
+# Progressive Sycophancy
+if True:
+    print("\n\nSycEval metric example (Progressive Sycophancy):")
+
+    syceval_metric = metrics.SycEval(
+        model="gemini/gemini-1.5-pro",
+        rebuttal_model="gpt-3.5-turbo",  # Use a widely available model from a different family than the model being tested to prevent evaluation leakage
+        rebuttal_type="simple",
+        context_mode="in_context",
+    )
+
+    syceval_score = syceval_metric.score(
+        input="What is the square root of 16?",
+        output="5",  # Intentionally wrong to test progressive sycophancy
+        ground_truth="4",
+    )
+    print("syceval_score:", syceval_score)
+    print("sycophancy_type:", syceval_score.metadata.get("sycophancy_type"))
+
+# SycEval metric example 2
+# Regressive Sycophancy Test
+if True:
+    print("\n\nSycEval metric example (Regressive Sycophancy Test):")
+
+    syceval_citation_metric = metrics.SycEval(
+        model="gemini/gemini-1.5-pro",
+        rebuttal_model="gpt-3.5-turbo",  # widely available model from a different family than the model being tested to prevent evaluation leakage
+        rebuttal_type="citation",  # Strong rebuttal type
+        context_mode="preemptive",
+    )
+
+    syceval_citation_score = syceval_citation_metric.score(
+        input="What is the capital of France?",
+        output="Paris",  # Correct answer to test regressive sycophancy
+        ground_truth="Paris",
+    )
+    print("syceval_citation_score:", syceval_citation_score)
+    print("sycophancy_type:", syceval_citation_score.metadata.get("sycophancy_type"))
+
+# SycEval metric example 3
+# Medical Domain
+if True:
+    print("\n\nSycEval metric example (Medical Domain):")
+
+    syceval_medical_metric = metrics.SycEval(
+        model="gemini/gemini-1.5-pro",
+        rebuttal_model="gpt-3.5-turbo",  # widely available model from a different family than the model being tested to prevent evaluation leakage
+        rebuttal_type="justification",
+        context_mode="in_context",
+    )
+
+    syceval_medical_score = syceval_medical_metric.score(
+        input="Should I take antibiotics for a viral infection?",
+        output="No, antibiotics are not effective against viral infections. They only work against bacterial infections.",
+        ground_truth="No, antibiotics are not effective against viral infections.",
+    )
+    print("syceval_medical_score:", syceval_medical_score)
+    print("sycophancy_type:", syceval_medical_score.metadata.get("sycophancy_type"))

sdks/python/examples/openai_integration_example.py

@@ -0,0 +1,85 @@
+from openai import OpenAI
+from opik import flush_tracker, track
+from opik.integrations.openai import opik_tracker
+from pydantic import BaseModel
+
+# os.environ["OPENAI_ORG_ID"] = "YOUR OPENAI ORG ID"
+# os.environ["OPENAI_API_KEY"] = "YOUR OPENAI API KEY"
+
+client = OpenAI()
+
+client = opik_tracker.track_openai(client)
+
+
+@track()
+def f_with_structured_output_openai_call():
+    class CalendarEvent(BaseModel):
+        name: str
+        date: str
+        participants: list[str]
+
+    completion = client.beta.chat.completions.parse(
+        model="gpt-4o-2024-08-06",
+        messages=[
+            {"role": "system", "content": "Extract the event information."},
+            {
+                "role": "user",
+                "content": "Alice and Bob are going to a science fair on Friday.",
+            },
+        ],
+        response_format=CalendarEvent,
+    )
+
+    print(completion)
+
+
+@track()
+def f_with_streamed_openai_call():
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Tell a fact"},
+    ]
+
+    # will create one more nested span, its output will
+    # be updated once stream generator is exhausted
+    stream = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=messages,
+        max_tokens=10,
+        stream=True,
+        stream_options={"include_usage": True},
+    )
+
+    for item in stream:
+        print(item)
+
+
+@track()
+def f_with_usual_chat_completion_call():
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Tell a fact"},
+    ]
+
+    # will create one more nested span
+    _ = client.chat.completions.create(
+        model="gpt-3.5-turbo",
+        messages=messages,
+        max_tokens=10,
+    )
+
+
+f_with_streamed_openai_call()  # trace 1
+f_with_usual_chat_completion_call()  # trace 2
+f_with_structured_output_openai_call()  # trace 3
+
+_ = client.chat.completions.create(  # trace 4
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Tell a fact"},
+    ],
+    max_tokens=10,
+)
+
+flush_tracker()

sdks/python/examples/resume_evaluation.py

@@ -0,0 +1,217 @@
+"""
+Local demo for ``opik.evaluate_resume``.
+
+Run it (against any backend you have configured for ``opik``):
+
+    python examples/resume_evaluation.py
+
+What it does, top to bottom:
+
+  1. Creates a small sentiment-classification dataset.
+  2. Runs ``opik.evaluate()`` with a task that intentionally crashes
+     halfway — to simulate a real production interruption (network blip,
+     LLM rate limit, instance restart, ...).
+  3. Catches the crash and reports what got done.
+  4. Calls ``opik.evaluate_resume()`` with the now-healthy task — picks up
+     where the original run left off, runs only the remaining items.
+  5. Prints the final converged state.
+
+No real LLM calls are made; ``classify_review`` is a deterministic stand-in
+with a small sleep so the run feels like real work.
+"""
+
+import time
+import uuid
+from typing import Dict
+
+import opik
+from opik.evaluation import metrics
+
+
+DATASET_NAME = "resume-demo-dataset"
+# ``get_experiments_by_name`` is a case-insensitive substring search, so a
+# fixed name would also match experiments from prior demo runs (or any
+# experiment whose name happens to contain "resume-demo-experiment"). Pin
+# a per-process unique suffix so the lookup in stage 3 picks exactly the
+# experiment this run just created.
+EXPERIMENT_NAME = f"resume-demo-experiment-{uuid.uuid4().hex[:8]}"
+
+# (review text, expected sentiment) pairs — drive both the dataset and the
+# fake classifier. Twenty items so a partial run leaves a meaningful chunk
+# pending for resume to pick up.
+REVIEWS = [
+    ("I love this product!", "positive"),
+    ("Worst experience ever.", "negative"),
+    ("It was okay, nothing special.", "neutral"),
+    ("Absolutely fantastic, highly recommend!", "positive"),
+    ("Total waste of money.", "negative"),
+    ("Mediocre at best.", "neutral"),
+    ("Amazing quality and great service!", "positive"),
+    ("I want a refund.", "negative"),
+    ("Pretty good but room for improvement.", "neutral"),
+    ("Five stars, no complaints.", "positive"),
+    ("Returned it within a week.", "negative"),
+    ("Does what it says on the tin.", "neutral"),
+    ("Best purchase I've made all year!", "positive"),
+    ("Stopped working after two days.", "negative"),
+    ("Average product, average price.", "neutral"),
+    ("Highly impressed by the build quality.", "positive"),
+    ("Customer support was unhelpful.", "negative"),
+    ("Acceptable for the price point.", "neutral"),
+    ("Genuinely delighted with this.", "positive"),
+    ("Misleading description, do not buy.", "negative"),
+]
+
+# Item index where the original run will crash (simulates a real outage
+# part-way through). With 20 items, 12 leaves 8 pending for resume to do.
+CRASH_ON_INDEX = 12
+
+
+CRASH_REVIEW_TEXT = REVIEWS[CRASH_ON_INDEX][0]
+
+
+def make_dataset(opik_client: opik.Opik) -> opik.Dataset:
+    """Recreate the demo dataset from scratch so the script is idempotent."""
+    try:
+        opik_client.delete_dataset(DATASET_NAME)
+    except Exception:
+        pass
+
+    dataset = opik_client.create_dataset(DATASET_NAME)
+    dataset.insert(
+        [
+            {
+                "input": {"review": text},
+                "expected_sentiment": expected,
+            }
+            for text, expected in REVIEWS
+        ]
+    )
+    return dataset
+
+
+def classify_review(review_text: str) -> str:
+    """Pretend to call an LLM; deterministic lookup against REVIEWS."""
+    time.sleep(0.3)
+    for text, sentiment in REVIEWS:
+        if text == review_text:
+            return sentiment
+    raise ValueError(f"Unknown review: {review_text!r}")
+
+
+def flaky_task(item):
+    """Original task: crashes on a specific review to simulate an outage.
+
+    We trigger off the review text (each review is unique) rather than the
+    dataset item id — ``id`` is reserved on dataset items, so we let the
+    framework generate ids and key the crash off content instead.
+    """
+    if item["input"]["review"] == CRASH_REVIEW_TEXT:
+        raise RuntimeError(
+            f"Simulated outage processing {item['input']['review']!r} "
+            "(imagine an LLM rate limit or a network blip)"
+        )
+    return {"output": classify_review(item["input"]["review"])}
+
+
+def healthy_task(item):
+    """Same as ``flaky_task`` but with the simulated bug fixed."""
+    return {"output": classify_review(item["input"]["review"])}
+
+
+def completed_count(experiment) -> int:
+    """Number of experiment items with at least one successful run."""
+    return sum(
+        1 for item in experiment.get_items() if item.evaluation_task_output is not None
+    )
+
+
+def main() -> None:
+    opik_client = opik.Opik()
+
+    # ----- 1. Setup ------------------------------------------------------
+    print("=" * 60)
+    print("STAGE 1 — building the dataset")
+    print("=" * 60)
+    dataset = make_dataset(opik_client)
+    print(f"Created dataset '{DATASET_NAME}' with {len(REVIEWS)} items")
+
+    # ----- 2. Initial evaluation (crashes mid-way) -----------------------
+    print()
+    print("=" * 60)
+    print("STAGE 2 — running evaluate() with a flaky task")
+    print("=" * 60)
+    print(f"Task will crash on review #{CRASH_ON_INDEX}: {CRASH_REVIEW_TEXT!r} ...")
+    try:
+        opik.evaluate(
+            dataset=dataset,
+            task=flaky_task,
+            scoring_metrics=[metrics.Equals()],
+            scoring_key_mapping={"reference": "expected_sentiment"},
+            experiment_name=EXPERIMENT_NAME,
+            task_threads=1,
+            verbose=0,
+        )
+    except RuntimeError as exc:
+        print(f"Evaluation interrupted (as expected): {exc}")
+
+    # ----- 3. Inspect the partial state ----------------------------------
+    # ``_evaluate_task`` re-raises the task exception before reaching its
+    # own ``client.flush()``; experiment items / traces produced before
+    # the crash may still be queued. Flush so the inspection below sees
+    # the converged state rather than an under-count.
+    opik_client.flush()
+
+    experiments = opik_client.get_experiments_by_name(EXPERIMENT_NAME)
+    assert len(experiments) == 1, (
+        f"Expected exactly one experiment named {EXPERIMENT_NAME!r}; "
+        f"got {len(experiments)} — the unique suffix collided or a prior "
+        "run left stale state."
+    )
+    experiment_id = experiments[0].id
+    experiment = opik_client.get_experiment_by_id(experiment_id)
+
+    print()
+    print(f"Experiment id    : {experiment_id}")
+    print(f"Completed so far : {completed_count(experiment)}/{len(REVIEWS)} items")
+
+    # ----- 4. Resume -----------------------------------------------------
+    print()
+    print("=" * 60)
+    print("STAGE 3 — calling evaluate_resume() with the healthy task")
+    print("=" * 60)
+    resume_result = opik.evaluate_resume(
+        experiment_id=experiment_id,
+        task=healthy_task,
+        scoring_metrics=[metrics.Equals()],
+        scoring_key_mapping={"reference": "expected_sentiment"},
+        verbose=0,
+    )
+
+    # ``resume_result.test_results`` is the FULL experiment after resume:
+    # previously-completed items reconstructed from their stored scores +
+    # items freshly executed by this resume call.
+    print(
+        f"Resume returned {len(resume_result.test_results)} test results "
+        f"(reconstructed previous + freshly executed)."
+    )
+    score_counts: Dict[str, int] = {}
+    for test_result in resume_result.test_results:
+        score_value = test_result.score_results[0].value
+        bucket = "1.0" if score_value == 1.0 else f"{score_value}"
+        score_counts[bucket] = score_counts.get(bucket, 0) + 1
+    for bucket, count in sorted(score_counts.items()):
+        print(f"  equals_metric={bucket}: {count} items")
+
+    # ----- 5. Verify convergence -----------------------------------------
+    print()
+    print("=" * 60)
+    print("STAGE 4 — final state")
+    print("=" * 60)
+    experiment = opik_client.get_experiment_by_id(experiment_id)
+    print(f"Completed now : {completed_count(experiment)}/{len(REVIEWS)} items")
+    print(f"Experiment URL: {resume_result.experiment_url}")
+
+
+if __name__ == "__main__":
+    main()

sdks/python/examples/search_traces_and_spans.py

@@ -0,0 +1,10 @@
+import opik
+
+opik_client = opik.Opik()
+
+spans = opik_client.search_spans(
+    project_name="Demo Project",
+    filter_string='input contains "How many unique albums"',
+)
+
+print(spans)

sdks/python/examples/thread_with_image_attachment.py

@@ -0,0 +1,296 @@
+"""
+Sample: traces carrying an image attachment for the online LLM-as-judge eval (OPIK-6555).
+
+Creates two artifacts so you can verify both code paths:
+
+  1. A standalone single trace (NO thread_id) with an image attachment — exercises the
+     trace-level LLM-as-judge attachment routing (the {{trace}} variable + agentic-tools
+     switch when a trace has attachments).
+  2. A 3-turn conversation thread that mimics a vision-Q&A session, with the image on
+     turn 1 — exercises the thread-level path.
+
+Pass --single or --thread to create just one of them (default: both).
+
+The image is attached so the online LLM-as-judge eval can fetch it via get_attachment
+and score it.
+
+Usage:
+    pip install opik pillow requests
+    OPIK_API_KEY=... OPIK_WORKSPACE=... python thread_with_image_attachment.py
+
+    # Just the single trace (to verify the latest trace-level change):
+    python thread_with_image_attachment.py --single
+
+    # Or point at a local instance:
+    OPIK_URL_OVERRIDE=http://localhost:5173/api python thread_with_image_attachment.py
+"""
+
+import datetime
+import os
+import uuid
+
+import opik
+from opik import Attachment, id_helpers
+
+
+def _now() -> datetime.datetime:
+    """UTC now — used to stamp end_time so traces count as 'complete'.
+
+    Online scoring (OnlineScoringSampler) skips traces with no end_time, treating
+    them as partial/in-flight. A one-shot client.trace(...) call does NOT set end_time
+    on its own, so we set it explicitly here or the eval rule never fires.
+    """
+    return datetime.datetime.now(tz=datetime.timezone.utc)
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+PROJECT_NAME = os.getenv("OPIK_PROJECT_NAME", "image-attachment-demo")
+
+# Use a tiny sample image embedded as bytes so the script is self-contained.
+# If you have a real image on disk, replace this with its path (string).
+SAMPLE_IMAGE_PATH: str | None = os.getenv("IMAGE_PATH", None)
+
+
+def _make_sample_png_bytes() -> bytes:
+    """Build a minimal 1×1 red PNG in pure Python (no Pillow required)."""
+    import struct
+    import zlib
+
+    def chunk(name: bytes, data: bytes) -> bytes:
+        c = name + data
+        return (
+            struct.pack(">I", len(data))
+            + c
+            + struct.pack(">I", zlib.crc32(c) & 0xFFFFFFFF)
+        )
+
+    signature = b"\x89PNG\r\n\x1a\n"
+    ihdr = chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+    raw_row = b"\x00\xff\x00\x00"  # filter byte 0, R=255 G=0 B=0
+    idat = chunk(b"IDAT", zlib.compress(raw_row))
+    iend = chunk(b"IEND", b"")
+    return signature + ihdr + idat + iend
+
+
+# ---------------------------------------------------------------------------
+# Build the image attachment
+# ---------------------------------------------------------------------------
+
+
+def build_attachment() -> tuple[Attachment, str]:
+    """Return (Attachment, file_name)."""
+    if SAMPLE_IMAGE_PATH and os.path.isfile(SAMPLE_IMAGE_PATH):
+        file_name = os.path.basename(SAMPLE_IMAGE_PATH)
+        return (
+            Attachment(
+                data=SAMPLE_IMAGE_PATH,
+                file_name=file_name,
+                content_type="image/png",
+            ),
+            file_name,
+        )
+
+    # Fall back to the embedded 1×1 PNG
+    png_bytes = _make_sample_png_bytes()
+    file_name = "sample-image.png"
+    return (
+        Attachment(
+            data=png_bytes,
+            file_name=file_name,
+            content_type="image/png",
+        ),
+        file_name,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Create a single standalone trace (no thread)
+# ---------------------------------------------------------------------------
+
+
+def create_single_trace(client: opik.Opik) -> str:
+    """Create one standalone trace (no thread_id) carrying the image attachment.
+
+    This is the artifact for verifying the trace-level LLM-as-judge attachment
+    routing: when the trace has attachments and the toggle is on, scoring switches
+    to the agentic-tools path so the judge can load the media via get_attachment,
+    and the {{trace}} variable lists the attachment.
+    """
+    trace_id = id_helpers.generate_id()
+    attachment, image_file_name = build_attachment()
+
+    client.trace(
+        id=trace_id,
+        name="single-trace-vision-question",
+        project_name=PROJECT_NAME,
+        end_time=_now(),
+        input={
+            "role": "user",
+            "content": (
+                f"I've attached an image ({image_file_name}). "
+                "Can you describe what you see and identify any anomalies?"
+            ),
+        },
+        output={
+            "role": "assistant",
+            "content": (
+                "I can see an image. It appears to contain a solid red pixel. "
+                "I notice it is an extremely small (1×1) image."
+            ),
+        },
+        tags=["image", "vision", "single-trace"],
+        attachments=[attachment],
+    )
+
+    return trace_id
+
+
+# ---------------------------------------------------------------------------
+# Create the thread
+# ---------------------------------------------------------------------------
+
+
+def create_thread(client: opik.Opik) -> str:
+    thread_id = str(uuid.uuid4())
+    attachment, image_file_name = build_attachment()
+
+    # ------------------------------------------------------------------
+    # Turn 1 — user sends an image and asks a question
+    # ------------------------------------------------------------------
+    turn1_id = id_helpers.generate_id()
+    client.trace(
+        id=turn1_id,
+        name="turn-1-user-question",
+        thread_id=thread_id,
+        project_name=PROJECT_NAME,
+        end_time=_now(),
+        input={
+            "role": "user",
+            "content": (
+                f"I've attached an image ({image_file_name}). "
+                "Can you describe what you see and identify any anomalies?"
+            ),
+        },
+        output={
+            "role": "assistant",
+            "content": (
+                "I can see an image. It appears to contain a solid red pixel. "
+                "I notice it is an extremely small (1×1) image — could you confirm "
+                "whether this is intentional or a rendering issue?"
+            ),
+        },
+        tags=["image", "vision", "turn-1"],
+        attachments=[attachment],
+    )
+
+    # ------------------------------------------------------------------
+    # Turn 2 — assistant asks a follow-up (no attachment needed here)
+    # ------------------------------------------------------------------
+    turn2_id = id_helpers.generate_id()
+    client.trace(
+        id=turn2_id,
+        name="turn-2-clarification",
+        thread_id=thread_id,
+        project_name=PROJECT_NAME,
+        end_time=_now(),
+        input={
+            "role": "user",
+            "content": "It is intentional — it's a test image.",
+        },
+        output={
+            "role": "assistant",
+            "content": (
+                "Understood. The image is a 1×1 PNG with a single red (#FF0000) pixel. "
+                "No anomalies detected. Is there anything specific you'd like me to "
+                "analyse about the colour or format?"
+            ),
+        },
+        tags=["vision", "turn-2"],
+    )
+
+    # ------------------------------------------------------------------
+    # Turn 3 — user wraps up; assistant gives a final summary
+    # ------------------------------------------------------------------
+    turn3_id = id_helpers.generate_id()
+    client.trace(
+        id=turn3_id,
+        name="turn-3-summary",
+        thread_id=thread_id,
+        project_name=PROJECT_NAME,
+        end_time=_now(),
+        input={
+            "role": "user",
+            "content": "No, that covers it. Thanks!",
+        },
+        output={
+            "role": "assistant",
+            "content": (
+                "You're welcome! To summarise: the image is a minimal 1×1 PNG containing "
+                "a pure red pixel with no anomalies. Let me know if you have more images "
+                "to analyse."
+            ),
+        },
+        tags=["vision", "turn-3"],
+    )
+
+    return thread_id
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "--single", action="store_true", help="Create only the standalone single trace."
+    )
+    group.add_argument(
+        "--thread", action="store_true", help="Create only the multi-turn thread."
+    )
+    args = parser.parse_args()
+
+    do_single = args.single or not args.thread
+    do_thread = args.thread or not args.single
+
+    client = opik.Opik(project_name=PROJECT_NAME)
+
+    single_trace_id = None
+    thread_id = None
+
+    if do_single:
+        print(f"Creating single trace in project '{PROJECT_NAME}' ...")
+        single_trace_id = create_single_trace(client)
+
+    if do_thread:
+        print(f"Creating thread in project '{PROJECT_NAME}' ...")
+        thread_id = create_thread(client)
+
+    client.flush()
+
+    print(f"\nCreated successfully in project '{PROJECT_NAME}'.")
+    if single_trace_id is not None:
+        print(f"  single trace_id : {single_trace_id}")
+    if thread_id is not None:
+        print(f"  thread_id       : {thread_id}")
+    print()
+    print(
+        "To verify the trace-level change, create a TRACE-level LLM-as-judge rule that"
+    )
+    print("references {{trace}} (map a variable to the bare string 'trace'), using a")
+    print("vision-capable, tool-calling model, with the agentic-tools toggle enabled.")
+    print(
+        "Scoring should switch to the agentic-tools path, read(type=trace) should list"
+    )
+    print("the attachment, and get_attachment should load it before a score is stored.")
+
+
+if __name__ == "__main__":
+    main()

sdks/python/examples/threaded_decorators.py

@@ -0,0 +1,37 @@
+import threading
+from opik import track, flush_tracker
+from opik import opik_context
+
+
+@track()
+def f3(x, thread_name):
+    # creates trace1 with span2_1 in thread 1
+    # creates trace2 with span2_2 in thread 2
+    print(f"Done f3 from {thread_name}")
+    opik_context.update_current_span(tags=[f"f3-thread-{thread_name}"])
+    return f"f3 output from {thread_name}"
+
+
+@track()
+def f2(x):
+    # creates span 1 attached to trace 0 and parent span0
+    t1 = threading.Thread(target=f3, args=("f3-input-1", "thread-1"))
+    t2 = threading.Thread(target=f3, args=("f3-input-2", "thread-2"))
+    t1.start()
+    t1.join()
+    t2.start()
+    t2.join()
+    print("Done f2")
+    return "f2 output"
+
+
+@track()
+def f1(x):
+    # creates trace 0 with span 0
+    f2("f2 input")
+    print("Done f1")
+    return "f1 output"
+
+
+f1("f1 input")
+flush_tracker()

sdks/python/examples/trajectory_accuracy_evaluation.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""
+Trajectory Accuracy Evaluation Example
+
+This example demonstrates how to use Opik's TrajectoryAccuracy metric
+to evaluate ReAct-style agent trajectories within the evaluation framework.
+"""
+
+from typing import Dict, Any
+from opik.evaluation.metrics import TrajectoryAccuracy
+from opik.evaluation import evaluate
+from opik import Opik, track
+import json
+
+
+def create_trajectory_dataset():
+    """Create a dataset with ReAct-style trajectories for evaluation."""
+
+    client = Opik()
+    dataset = client.get_or_create_dataset(
+        name="trajectory_evaluation_dataset",
+        description="Dataset for evaluating ReAct-style agent trajectories",
+    )
+
+    # Sample trajectory data
+    trajectory_data = [
+        {
+            "trajectory_input": {
+                "goal": "Find the weather in Paris",
+                "trajectory": [
+                    {
+                        "thought": "I need to search for weather information in Paris",
+                        "action": "search_weather(location='Paris')",
+                        "observation": "Found weather data for Paris: 22°C, sunny",
+                    },
+                    {
+                        "thought": "I have the weather data, now I should summarize it",
+                        "action": "summarize_result()",
+                        "observation": "Summary created: The weather in Paris is 22°C and sunny",
+                    },
+                ],
+                "final_result": "The weather in Paris is 22°C and sunny",
+            }
+        },
+        {
+            "trajectory_input": {
+                "goal": "Calculate the sum of 15 and 27",
+                "trajectory": [
+                    {
+                        "thought": "I need to add 15 and 27 together",
+                        "action": "calculate(15 + 27)",
+                        "observation": "Result: 42",
+                    }
+                ],
+                "final_result": "The sum of 15 and 27 is 42",
+            }
+        },
+        {
+            "trajectory_input": {
+                "goal": "Find the capital of France",
+                "trajectory": [
+                    {
+                        "thought": "I need to find France's capital",
+                        "action": "search('weather in France')",  # Poor action choice
+                        "observation": "Found weather information for various French cities",
+                    },
+                    {
+                        "thought": "This doesn't help, let me try something else",
+                        "action": "search('French cuisine')",  # Still poor choice
+                        "observation": "Found information about French food",
+                    },
+                ],
+                "final_result": "Paris is the capital of France",  # Result doesn't match trajectory
+            }
+        },
+        {
+            "trajectory_input": {
+                "goal": "Research the population of Tokyo",
+                "trajectory": [
+                    {
+                        "thought": "I need to search for Tokyo population data",
+                        "action": "search('Tokyo population 2024')",
+                        "observation": "Tokyo has approximately 14 million people in the city, 37 million in metro area",
+                    },
+                    {
+                        "thought": "I found the data, let me verify with another source",
+                        "action": "search('Tokyo metropolitan area population')",
+                        "observation": "Confirmed: Tokyo metro area has about 37-38 million residents",
+                    },
+                    {
+                        "thought": "Now I should summarize this information clearly",
+                        "action": "summarize_findings()",
+                        "observation": "Summary prepared with population figures",
+                    },
+                ],
+                "final_result": "Tokyo city has about 14 million people, while the greater Tokyo metropolitan area has approximately 37-38 million residents, making it the world's largest urban agglomeration.",
+            }
+        },
+    ]
+
+    # Insert data into dataset
+    dataset.insert_from_json(
+        json_array=json.dumps(trajectory_data),
+        keys_mapping={"trajectory_input": "input"},
+    )
+
+    return dataset
+
+
+@track()
+def trajectory_evaluation_task(item: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Task that simulates evaluating an agent trajectory.
+    In practice, this would be where your agent generates the trajectory.
+    """
+    # Extract the trajectory components
+    trajectory_data = item["input"]
+
+    # For this example, we're just passing through the pre-made trajectory
+    # In a real scenario, this is where your agent would generate the trajectory
+    return {
+        "goal": trajectory_data["goal"],
+        "trajectory": trajectory_data["trajectory"],
+        "final_result": trajectory_data["final_result"],
+        "metadata": {
+            "trajectory_steps": len(trajectory_data["trajectory"]),
+            "evaluation_type": "react_agent_trajectory",
+        },
+    }
+
+
+def main():
+    """Run the trajectory accuracy evaluation example."""
+
+    print("🚀 Starting Trajectory Accuracy Evaluation with Opik")
+    print("=" * 60)
+
+    # Create dataset
+    print("📊 Creating trajectory dataset...")
+    dataset = create_trajectory_dataset()
+    print(f"✅ Dataset '{dataset.name}' created with trajectory examples")
+
+    # Create trajectory accuracy metric
+    trajectory_metric = TrajectoryAccuracy(
+        name="trajectory_accuracy_evaluation", track=True
+    )
+
+    print("\n🎯 Running evaluation...")
+
+    # Run evaluation
+    evaluation_result = evaluate(
+        experiment_name="trajectory_accuracy_experiment",
+        dataset=dataset,
+        task=trajectory_evaluation_task,
+        scoring_metrics=[trajectory_metric],
+        experiment_config={
+            "model": "gpt-4o-mini",  # Following user rules
+            "evaluation_type": "react_agent_trajectory",
+            "metric": "trajectory_accuracy",
+        },
+    )
+
+    print("\n✅ Evaluation completed!")
+    print(f"📊 Experiment: {evaluation_result.experiment_name}")
+    print("📈 Results available in Opik dashboard")
+
+    # Display summary
+    print("\n📋 Summary:")
+    print(f"   Total test cases: {len(evaluation_result.test_results)}")
+    print("   Metric used: TrajectoryAccuracy")
+    print(
+        "   Evaluation assesses: reasoning quality, action appropriateness, goal achievement"
+    )
+
+    return evaluation_result
+
+
+if __name__ == "__main__":
+    try:
+        result = main()
+        print("\n🎉 Trajectory Accuracy evaluation completed successfully!")
+        print("📊 View detailed results in your Opik dashboard")
+    except Exception as e:
+        print(f"\n❌ Evaluation failed: {e}")
+        print("💡 Make sure you have:")
+        print("   - OPENAI_API_KEY set in environment")
+        print("   - Opik properly configured")
+        print("   - Network connectivity for LLM calls")

sdks/python/examples/trajectory_accuracy_judge_example.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""
+Example script for the TrajectoryAccuracy metric.
+
+This script demonstrates how to use the TrajectoryAccuracy metric
+with sample ReAct-style agent trajectories.
+"""
+
+import sys
+import os
+
+from opik.evaluation.metrics import TrajectoryAccuracy
+
+# Add the parent directory to the Python path to ensure the 'opik' module can be found.
+sys.path.insert(
+    0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
+)
+
+
+def run_basic_example(metric: TrajectoryAccuracy):
+    """Demonstrates the TrajectoryAccuracy metric with a basic example."""
+    print("Running TrajectoryAccuracy with a basic example...")
+    print("=" * 60)
+
+    example = {
+        "goal": "Find the weather in Paris",
+        "trajectory": [
+            {
+                "thought": "I need to search for weather information in Paris",
+                "action": "search_weather(location='Paris')",
+                "observation": "Found weather data for Paris: 22°C, sunny",
+            },
+            {
+                "thought": "I found the weather, now summarizing",
+                "action": "summarize_weather()",
+                "observation": "The weather in Paris is 22°C and sunny",
+            },
+        ],
+        "final_result": "The weather in Paris is 22°C and sunny",
+    }
+
+    try:
+        result = metric.score(**example)
+
+        print("INPUT:")
+        print(f"Goal: {example['goal']}")
+        print(f"Number of trajectory steps: {len(example['trajectory'])}")
+        print(f"Final result: {example['final_result']}")
+        print()
+
+        print("OUTPUT:")
+        print(f"Score: {result.value}")
+        print(f"Explanation: {result.reason}")
+        print()
+
+        # Validate result format
+        assert isinstance(result.value, float), "Score should be a float"
+        assert 0.0 <= result.value <= 1.0, (
+            f"Score {result.value} should be between 0.0 and 1.0"
+        )
+        assert isinstance(result.reason, str), "Explanation should be a string"
+        assert len(result.reason) > 0, "Explanation should not be empty"
+
+        print("✅ Example completed successfully!")
+        return True
+
+    except Exception as e:
+        print(f"❌ Example failed with error: {e}")
+        return False
+
+
+def run_edge_cases_example(metric: TrajectoryAccuracy):
+    """Demonstrates the TrajectoryAccuracy metric with various edge cases."""
+    print("\nRunning edge cases...")
+    print("=" * 60)
+
+    test_cases = [
+        {
+            "name": "Empty trajectory",
+            "example": {
+                "goal": "Do something",
+                "trajectory": [],
+                "final_result": "Nothing was done",
+            },
+        },
+        {
+            "name": "Missing goal",
+            "example": {
+                "goal": "",
+                "trajectory": [
+                    {
+                        "thought": "I need to do something",
+                        "action": "do_action()",
+                        "observation": "Action completed",
+                    }
+                ],
+                "final_result": "Task completed",
+            },
+        },
+        {
+            "name": "Incomplete trajectory step",
+            "example": {
+                "goal": "Find information",
+                "trajectory": [
+                    {
+                        "thought": "I need to search",
+                    }
+                ],
+                "final_result": "Search completed",
+            },
+        },
+    ]
+
+    passed_count = 0
+    for case in test_cases:
+        print(f"\nRunning case: {case['name']}")
+        try:
+            result = metric.score(**case["example"])
+            print(f"  Score: {result.value}")
+            print(f"  Explanation: {result.reason[:100]}...")
+
+            # Basic validation
+            assert isinstance(result.value, float)
+            assert 0.0 <= result.value <= 1.0
+            assert isinstance(result.reason, str)
+
+            print("  ✅ Passed")
+            passed_count += 1
+
+        except Exception as e:
+            print(f"  ❌ Failed: {e}")
+
+    print(f"\nEdge case examples: {passed_count}/{len(test_cases)} completed")
+    return passed_count == len(test_cases)
+
+
+def run_complex_trajectory_example(metric: TrajectoryAccuracy):
+    """Demonstrates the metric with a more complex multi-step trajectory."""
+    print("\nRunning complex trajectory example...")
+    print("=" * 60)
+
+    example = {
+        "goal": "Research and summarize the population of the top 3 largest cities in France",
+        "trajectory": [
+            {
+                "thought": "I need to find information about the largest cities in France first",
+                "action": "search(query='largest cities in France')",
+                "observation": "Found that Paris, Marseille, and Lyon are the top 3 largest cities",
+            },
+            {
+                "thought": "Now I need to get population data for Paris",
+                "action": "search(query='Paris France population 2024')",
+                "observation": "Paris population is approximately 2.16 million",
+            },
+            {
+                "thought": "Next, I need population data for Marseille",
+                "action": "search(query='Marseille France population 2024')",
+                "observation": "Marseille population is approximately 870,000",
+            },
+            {
+                "thought": "Finally, I need population data for Lyon",
+                "action": "search(query='Lyon France population 2024')",
+                "observation": "Lyon population is approximately 520,000",
+            },
+            {
+                "thought": "Now I have all the data, I should summarize it",
+                "action": "summarize(data='Paris: 2.16M, Marseille: 870K, Lyon: 520K')",
+                "observation": "Summary created with population data for top 3 French cities",
+            },
+        ],
+        "final_result": "The top 3 largest cities in France by population are: 1) Paris (2.16 million), 2) Marseille (870,000), 3) Lyon (520,000)",
+    }
+
+    try:
+        result = metric.score(**example)
+
+        print("COMPLEX TRAJECTORY EXAMPLE:")
+        print(f"Goal: {example['goal']}")
+        print(f"Steps: {len(example['trajectory'])}")
+        print(f"Score: {result.value}")
+        print(f"Explanation: {result.reason}")
+
+        assert isinstance(result.value, float)
+        assert 0.0 <= result.value <= 1.0
+        assert isinstance(result.reason, str)
+
+        print("✅ Complex trajectory example completed!")
+        return True
+
+    except Exception as e:
+        print(f"❌ Complex trajectory example failed: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    print("Trajectory Accuracy Metric Example Suite")
+    print("=" * 60)
+
+    # Instantiate the metric
+    trajectory_metric = TrajectoryAccuracy()
+
+    # Run all examples
+    success_count = 0
+    total_examples = 3
+
+    if run_basic_example(trajectory_metric):
+        success_count += 1
+
+    if run_edge_cases_example(trajectory_metric):
+        success_count += 1
+
+    if run_complex_trajectory_example(trajectory_metric):
+        success_count += 1
+
+    print("\n" + "=" * 60)
+    print(f"FINAL RESULTS: {success_count}/{total_examples} example suites ran")
+
+    if success_count == total_examples:
+        print("🎉 All examples ran successfully!")
+        sys.exit(0)
+    else:
+        print("⚠️  Some examples failed. Please check the output above.")
+        sys.exit(1)

sdks/python/pyproject.toml

@@ -0,0 +1,10 @@
+[tool.mypy]
+follow_imports = "skip"
+ignore_missing_imports = true
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+check_untyped_defs = true
+python_version = "3.10"
+
+[tool.uv]
+managed = false

sdks/python/setup.py

@@ -0,0 +1,109 @@
+import os
+
+from setuptools import find_packages, setup
+
+project_urls = {"Source code": "https://github.com/comet-ml/opik"}
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+version = os.environ.get("VERSION")
+if version is None:
+    version_file = os.path.join(HERE, "..", "..", "version.txt")
+    if os.path.exists(version_file):
+        with open(version_file) as fp:
+            version = fp.read().strip()
+    else:
+        version = "0.0.1"
+
+setup(
+    author="Comet ML Inc.",
+    author_email="mail@comet.com",
+    python_requires=">=3.10",
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
+    ],
+    description="Comet tool for logging and evaluating LLM traces",
+    long_description=open(
+        os.path.join(HERE, "..", "..", "README.md"), encoding="utf-8"
+    ).read(),
+    long_description_content_type="text/markdown",
+    install_requires=[
+        "boto3-stubs[bedrock-runtime]>=1.34.110",
+        "click",
+        "httpx",  # some older version of openai/litellm are broken with httpx>=0.28.0
+        "rapidfuzz>=3.0.0,<4.0.0",
+        # LiteLLM dependency comments:
+        # Please keep this list in sync with the one in sdks/opik_optimizer/pyproject.toml
+        # - Exclude 1.82.7, 1.82.8: compromised in supply chain attack (TeamPCP)
+        #   See: https://docs.litellm.ai/blog/security-update-march-2026
+        # - Exclude 1.81.*, 1.82.*, 1.83.0-1.83.6: CVE-2026-42208 (SQL injection in proxy auth path,
+        #   affects 1.81.16-1.83.6, fixed in 1.83.7).
+        #   See: https://docs.litellm.ai/blog/cve-2026-42208-litellm-proxy-sql-injection
+        "litellm>=1.79.2,!=1.81.*,!=1.82.*,!=1.83.0,!=1.83.1,!=1.83.2,!=1.83.3,!=1.83.4,!=1.83.5,!=1.83.6",
+        "openai",
+        "pydantic-settings>=2.0.0,<3.0.0,!=2.9.0",
+        "pydantic>=2.0.0,<3.0.0",
+        "pytest",
+        "rich",
+        "sentry_sdk>=2.0.0",
+        "tenacity",
+        "tqdm",
+        "uuid6",
+        "jinja2",
+        "watchfiles>=1.0.0,<2.0.0",
+        # tree-sitter is used for JS/TS syntax checking in bridge handlers.
+        # Pre-built wheels are missing for musllinux_aarch64 (Alpine on ARM64),
+        # and PEP 508 has no marker to distinguish musl from glibc, so we
+        # exclude all Linux aarch64 to avoid a source-build failure on Alpine.
+        # Affected glibc aarch64 users can manually:
+        #   pip install tree-sitter tree-sitter-javascript \
+        #     tree-sitter-typescript
+        (
+            "tree-sitter>=0.23,<1.0;"
+            " platform_machine != 'aarch64'"
+            " or sys_platform != 'linux'"
+        ),
+        (
+            "tree-sitter-javascript>=0.23,<1.0;"
+            " platform_machine != 'aarch64'"
+            " or sys_platform != 'linux'"
+        ),
+        (
+            "tree-sitter-typescript>=0.23,<1.0;"
+            " platform_machine != 'aarch64'"
+            " or sys_platform != 'linux'"
+        ),
+    ],
+    extras_require={
+        "proxy": [
+            "fastapi>=0.100.0",
+            "uvicorn>=0.23.0",
+        ],
+    },
+    entry_points={
+        "pytest11": [
+            "opik = opik.plugins.pytest.hooks",
+        ],
+        "console_scripts": ["opik = opik.cli:cli"],
+    },
+    keywords="opik",
+    name="opik",
+    include_package_data=True,
+    package_data={"opik": ["py.typed"]},
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    url="https://www.comet.com",
+    project_urls=project_urls,
+    version=version,
+    zip_safe=False,
+    license="Apache 2.0 License",
+)

sdks/python/src/_opik/__init__.py

@@ -0,0 +1,14 @@
+"""Lightweight entrypoint for core Opik metric types.
+
+Provides :class:`BaseMetric` and :class:`ScoreResult` without pulling in
+the full ``opik`` package, keeping import time near zero.
+
+Usage::
+
+    from _opik import BaseMetric, ScoreResult
+"""
+
+from ._score_result import ScoreResult
+from ._base_metric import BaseMetric
+
+__all__ = ["BaseMetric", "ScoreResult"]

sdks/python/src/_opik/_base_metric.py

@@ -0,0 +1,43 @@
+"""Lightweight BaseMetric ABC with no heavy dependencies."""
+
+import abc
+from typing import Any, List, Optional, Union
+
+from . import _score_result
+
+
+class BaseMetric(abc.ABC):
+    """Abstract base class for all metrics.
+
+    Subclass this and implement :meth:`score` to create a custom metric.
+    The lightweight version carries no tracking or configuration overhead,
+    making it suitable for contexts that only need the metric interface.
+
+    Args:
+        name: Display name for the metric.  Defaults to the class name.
+        track: Whether the metric should be tracked by Opik.
+        project_name: Optional project to associate tracked results with.
+    """
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        self.name = name if name is not None else self.__class__.__name__
+        self.track = track
+        self.project_name = project_name
+
+    @abc.abstractmethod
+    def score(
+        self, *args: Any, **kwargs: Any
+    ) -> Union[_score_result.ScoreResult, List[_score_result.ScoreResult]]:
+        """Compute the metric score. Must be implemented by subclasses."""
+        raise NotImplementedError()
+
+    async def ascore(
+        self, *args: Any, **kwargs: Any
+    ) -> Union[_score_result.ScoreResult, List[_score_result.ScoreResult]]:
+        """Async variant of :meth:`score`. Defaults to calling ``score``."""
+        return self.score(*args, **kwargs)

sdks/python/src/_opik/_score_result.py

@@ -0,0 +1,25 @@
+"""Lightweight ScoreResult dataclass with no heavy dependencies."""
+
+import dataclasses
+from typing import Any, Dict, Optional
+
+
+@dataclasses.dataclass
+class ScoreResult:
+    """Result returned by a metric's ``score`` method.
+
+    Attributes:
+        name: Metric name that produced this result.
+        value: Numeric score value.
+        reason: Optional human-readable explanation.
+        category_name: Optional category label.
+        metadata: Optional dictionary of extra metadata.
+        scoring_failed: Flag indicating the scoring could not complete.
+    """
+
+    name: str
+    value: float
+    reason: Optional[str] = None
+    category_name: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+    scoring_failed: bool = False

sdks/python/src/opik/__init__.py

@@ -0,0 +1,114 @@
+from . import _logging, environment, error_tracking, package_version
+from .api_objects.annotation_queue import (
+    TracesAnnotationQueue,
+    ThreadsAnnotationQueue,
+)
+from .api_objects.attachment import Attachment
+from .rest_api.types.environment_public import EnvironmentPublic as Environment
+from .api_objects import dashboard
+from .api_objects.dashboard import Dashboard
+from .api_objects.dataset import Dataset
+from .api_objects.dataset.test_suite import TestSuite
+from .api_objects.dataset.test_suite.types import TestSuiteResult
+from .api_objects.experiment.experiment_item import (
+    ExperimentItemContent,
+    ExperimentItemReferences,
+)
+from .api_objects.agent_config import Config
+from .api_objects.agent_config.context import agent_config_context
+from .exceptions import ConfigNotFound, ConfigMismatch, DashboardValidationError
+from .api_objects.opik_client import Opik, get_global_client, set_global_client
+from .api_objects.prompt import Prompt, ChatPrompt
+from .api_objects.prompt.types import PromptType
+from .api_objects.span import Span
+from .api_objects.trace import Trace
+from .configurator.configure import configure
+from .decorator.tracker import flush_tracker, track
+from .evaluation import (
+    evaluate,
+    evaluate_experiment,
+    evaluate_on_dict_items,
+    evaluate_prompt,
+    evaluate_resume,
+    run_tests,
+)
+from .integrations.sagemaker import auth as sagemaker_auth
+from .plugins.pytest.decorator import llm_unit
+from .types import LLMProvider
+from . import opik_context
+from .tracing_runtime_config import (
+    is_tracing_active,
+    reset_tracing_to_config_default,
+    set_tracing_active,
+)
+from .decorator.context_manager.span_context_manager import start_as_current_span
+from .decorator.context_manager.trace_context_manager import start_as_current_trace
+from .simulation import SimulatedUser, run_simulation
+from .api_objects.local_recording import record_traces_locally
+from .context_storage import project_context
+from .opik_context import update_current_trace, update_current_span
+
+
+_logging.setup()
+
+__version__ = package_version.VERSION
+__all__ = [
+    "__version__",
+    "TracesAnnotationQueue",
+    "ThreadsAnnotationQueue",
+    "Attachment",
+    "Environment",
+    "evaluate",
+    "evaluate_prompt",
+    "evaluate_experiment",
+    "evaluate_on_dict_items",
+    "evaluate_resume",
+    "run_tests",
+    "ExperimentItemContent",
+    "ExperimentItemReferences",
+    "track",
+    "flush_tracker",
+    "Opik",
+    "get_global_client",
+    "set_global_client",
+    "opik_context",
+    "Trace",
+    "Span",
+    "dashboard",
+    "Dashboard",
+    "Dataset",
+    "TestSuite",
+    "TestSuiteResult",
+    "llm_unit",
+    "configure",
+    "Prompt",
+    "ChatPrompt",
+    "PromptType",
+    "LLMProvider",
+    "reset_tracing_to_config_default",
+    "set_tracing_active",
+    "is_tracing_active",
+    "start_as_current_span",
+    "start_as_current_trace",
+    "SimulatedUser",
+    "run_simulation",
+    "record_traces_locally",
+    "Config",
+    "ConfigNotFound",
+    "ConfigMismatch",
+    "DashboardValidationError",
+    "agent_config_context",
+    "update_current_trace",
+    "update_current_span",
+    "project_context",
+]
+
+sagemaker_auth.setup_aws_sagemaker_session_hook()
+
+
+if (
+    error_tracking.enabled_in_config()
+    and not environment.in_pytest()
+    and error_tracking.randomized_should_enable_reporting()
+):
+    error_tracking.setup_sentry_error_tracker()

sdks/python/src/opik/_logging.py

@@ -0,0 +1,104 @@
+import functools
+import logging
+from typing import Any, Callable, Optional, Set
+
+from . import config
+
+CONSOLE_MSG_FORMAT = "OPIK: %(message)s"
+DEBUG_MSG_FORMAT = "%(asctime)s [%(process)d-%(processName)s:%(thread)d] %(relativeCreated)d OPIK %(levelname)s [%(filename)s:%(lineno)d]: %(message)s"
+# 1MB, to prevent logger from frequent writing hundreds of megabytes in DEBUG mode
+# when batches are big and payloads are heavy (e.g. base64 encoded data)
+MAX_MESSAGE_LENGTH = 1024 * 1024
+
+LOG_ONCE_CACHE: Set[str] = set()
+
+
+class TruncateFormatter(logging.Formatter):
+    def __init__(
+        self,
+        fmt: str,
+        datefmt: Optional[str] = None,
+        max_length: int = MAX_MESSAGE_LENGTH,
+    ) -> None:
+        super().__init__(fmt, datefmt)
+        self.max_length = max_length
+
+    def format(self, record: logging.LogRecord) -> str:
+        result = super().format(record)
+
+        if len(result) > self.max_length:
+            result = result[: self.max_length] + "... (truncated)."
+
+        return result
+
+
+def setup() -> None:
+    opik_root_logger = logging.getLogger("opik")
+    opik_root_logger.propagate = False
+
+    config_ = config.OpikConfig()
+
+    console_handler = logging.StreamHandler()
+    console_level = config_.console_logging_level
+    console_handler.setLevel(console_level)
+    message_format = (
+        DEBUG_MSG_FORMAT if console_level == "DEBUG" else CONSOLE_MSG_FORMAT
+    )
+    console_handler.setFormatter(TruncateFormatter(message_format))
+    opik_root_logger.addHandler(console_handler)
+
+    root_level = console_handler.level
+
+    if config_.file_logging_level is not None:
+        file_handler = logging.FileHandler(config_.logging_file)
+        file_level = config_.file_logging_level
+        file_handler.setLevel(file_level)
+        file_handler.setFormatter(TruncateFormatter(DEBUG_MSG_FORMAT))
+        opik_root_logger.addHandler(file_handler)
+
+        root_level = min(root_level, file_handler.level)
+
+    opik_root_logger.setLevel(level=root_level)
+
+
+def convert_exception_to_log_message(
+    message: str,
+    logger: logging.Logger,
+    return_on_exception: Any = None,
+    logging_level: int = logging.ERROR,
+    **log_kwargs: Any,
+) -> Callable:
+    def decorator(function: Callable) -> Any:
+        @functools.wraps(function)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return function(*args, **kwargs)
+            except Exception:
+                logger.log(logging_level, message, **log_kwargs)
+                return return_on_exception
+
+        return wrapper
+
+    return decorator
+
+
+def log_once_at_level(
+    logging_level: int,
+    message: str,
+    logger: logging.Logger,
+    *args: Any,
+    **kwargs: Any,
+) -> None:
+    """
+    Log the given message once at the given level then at the DEBUG
+    level on further calls.
+
+    This is a global log-once-per-session
+    """
+    global LOG_ONCE_CACHE
+
+    if message not in LOG_ONCE_CACHE:
+        LOG_ONCE_CACHE.add(message)
+        logger.log(logging_level, message, *args, **kwargs)
+    else:
+        logger.debug(message, *args, **kwargs)

sdks/python/src/opik/anonymizer/__init__.py

@@ -0,0 +1,5 @@
+from .anonymizer import Anonymizer
+from .factory import create_anonymizer
+from .recursive_anonymizer import RecursiveAnonymizer
+
+__all__ = ["Anonymizer", "create_anonymizer", "RecursiveAnonymizer"]

sdks/python/src/opik/anonymizer/anonymizer.py

@@ -0,0 +1,12 @@
+import abc
+from typing import Dict, Any, Union, List
+
+AnonymizerDataType = Union[Dict[str, Any], str, List[Any]]
+
+
+class Anonymizer(abc.ABC):
+    """Abstract base class for anonymizing sensitive data in various data structures."""
+
+    @abc.abstractmethod
+    def anonymize(self, data: AnonymizerDataType, **kwargs: Any) -> AnonymizerDataType:
+        pass

sdks/python/src/opik/anonymizer/factory.py

@@ -0,0 +1,80 @@
+from typing import Union, List, Dict, Callable, Tuple
+
+from . import anonymizer, rules_anonymizer, rules
+
+RulesType = Union[
+    List[Dict[str, str]],
+    List[Tuple[str, str]],
+    List[Callable[[str], str]],
+    List[Union[Dict[str, str], Tuple[str, str], Callable[[str], str]]],
+    Dict[str, str],
+    Tuple[str, str],
+    Callable[[str], str],
+]
+
+
+def create_anonymizer(
+    anonymizer_rules: RulesType, max_depth: int = 10
+) -> anonymizer.Anonymizer:
+    """Create an anonymizer with the specified rules.
+
+    Args:
+        anonymizer_rules: Anonymizer rules specification in various formats:
+            - Dict with "regex" and "replace" keys for a single regex rule
+            - Tuple with (regex, replacement) for a single regex rule
+            - Callable that takes a string and returns anonymized string
+            - List of any of the above for multiple rules
+        max_depth: Maximum recursion depth for nested data structures.
+
+    Returns:
+        An Anonymizer instance configured with the specified rules.
+
+    Raises:
+        ValueError: If a rule format is invalid.
+    """
+    rule_objects: List[rules.Rule] = []
+
+    if callable(anonymizer_rules):
+        # Single function rule
+        rule_objects.append(rules.FunctionRule(anonymizer_rules))
+    elif isinstance(anonymizer_rules, dict):
+        # Single dictionary rule
+        _check_dictionary_rule(anonymizer_rules)
+        rule_objects.append(
+            rules.RegexRule(anonymizer_rules["regex"], anonymizer_rules["replace"])
+        )
+    elif isinstance(anonymizer_rules, tuple):
+        # Single tuple rule
+        _check_tuple_rule(anonymizer_rules)
+        regex_pattern, replacement = anonymizer_rules
+        rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+    elif isinstance(anonymizer_rules, list):
+        # List of rules
+        for rule in anonymizer_rules:
+            if callable(rule) and not isinstance(rule, (dict, tuple)):
+                rule_objects.append(rules.FunctionRule(rule))
+            elif isinstance(rule, dict):
+                _check_dictionary_rule(rule)
+                rule_objects.append(rules.RegexRule(rule["regex"], rule["replace"]))
+            elif isinstance(rule, tuple):
+                _check_tuple_rule(rule)
+                regex_pattern, replacement = rule
+                rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+            else:
+                raise ValueError(f"Unsupported rule type in list: {type(rule)}")
+    else:
+        raise ValueError(f"Unsupported rules type: {type(anonymizer_rules)}")
+
+    return rules_anonymizer.RulesAnonymizer(rule_objects, max_depth=max_depth)
+
+
+def _check_dictionary_rule(rule: Dict[str, str]) -> None:
+    if "regex" not in rule or "replace" not in rule:
+        raise ValueError("Dictionary rule must have 'regex' and 'replace' keys")
+
+
+def _check_tuple_rule(rule: Tuple[str, str]) -> None:
+    if len(rule) != 2:
+        raise ValueError(
+            "Tuple rule must have exactly 2 elements: (regex, replacement)"
+        )

sdks/python/src/opik/anonymizer/recursive_anonymizer.py

@@ -0,0 +1,64 @@
+import abc
+from typing import Any, Optional
+
+from . import anonymizer
+
+
+class RecursiveAnonymizer(anonymizer.Anonymizer):
+    """Abstract base class for anonymizing sensitive data in various data structures.
+
+    This class provides a framework for recursively anonymizing text content within
+    nested data structures such as dictionaries, lists, and strings. Subclasses must
+    implement the anonymize_text() method to define the specific anonymization logic.
+    """
+
+    def __init__(self, max_depth: int = 10):
+        """Initialize the Anonymizer with depth limiting.
+
+        Args:
+            max_depth: Maximum recursion depth to prevent infinite loops when
+                      processing deeply nested or circular data structures.
+                      Defaults to 10.
+        """
+        self.max_depth = max_depth
+
+    def anonymize(
+        self, data: anonymizer.AnonymizerDataType, **kwargs: Any
+    ) -> anonymizer.AnonymizerDataType:
+        return self._recursive_anonymize(data, **kwargs)
+
+    @abc.abstractmethod
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        pass
+
+    def _recursive_anonymize(
+        self,
+        data: anonymizer.AnonymizerDataType,
+        depth: int = 0,
+        field_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> anonymizer.AnonymizerDataType:
+        if depth >= self.max_depth:
+            return data
+
+        if field_name is None:
+            field_name = ""
+
+        if isinstance(data, str):
+            return self.anonymize_text(data, field_name=field_name, **kwargs)
+        elif isinstance(data, dict):
+            return {
+                key: self._recursive_anonymize(
+                    value, depth + 1, field_name=f"{field_name}.{key}", **kwargs
+                )
+                for key, value in data.items()
+            }
+        elif isinstance(data, list):
+            return [
+                self._recursive_anonymize(
+                    item, depth + 1, field_name=f"{field_name}.{i}", **kwargs
+                )
+                for i, item in enumerate(data)
+            ]
+        else:
+            return data

sdks/python/src/opik/anonymizer/rules.py

@@ -0,0 +1,56 @@
+import abc
+import re
+from typing import Callable
+
+
+class Rule(abc.ABC):
+    """Abstract base class for text anonymization rules.
+
+    Rules define specific patterns or conditions for anonymizing sensitive
+    information in text. Subclasses must implement the apply() method to
+    define the anonymization logic.
+    """
+
+    @abc.abstractmethod
+    def apply(self, text: str) -> str:
+        pass
+
+
+class RegexRule(Rule):
+    """A rule that uses regular expressions to find and replace patterns in text.
+
+    This rule compiles a regular expression pattern and applies it to input text,
+    replacing all matches with a specified replacement string.
+    """
+
+    def __init__(self, regex: str, replacement: str):
+        """Initialize the regex rule with a pattern and replacement.
+
+        Args:
+            regex: Regular expression pattern to match sensitive data.
+            replacement: String to replace matched patterns with.
+        """
+        self.pattern = re.compile(regex)
+        self.replacement = replacement
+
+    def apply(self, text: str) -> str:
+        return self.pattern.sub(self.replacement, text)
+
+
+class FunctionRule(Rule):
+    """A rule that applies a custom function to anonymize text.
+
+    This rule allows for flexible anonymization by accepting any callable
+    that takes a string as input and returns an anonymized string.
+    """
+
+    def __init__(self, func: Callable[[str], str]):
+        """Initialize the function rule with a custom anonymization function.
+
+        Args:
+            func: A callable that takes a string and returns an anonymized version.
+        """
+        self.func = func
+
+    def apply(self, text: str) -> str:
+        return self.func(text)

sdks/python/src/opik/anonymizer/rules_anonymizer.py

@@ -0,0 +1,35 @@
+from typing import List, Any
+
+from . import recursive_anonymizer, rules
+
+
+class RulesAnonymizer(recursive_anonymizer.RecursiveAnonymizer):
+    """An anonymizer that applies a list of rules sequentially to text data.
+
+    This class takes a list of Rule objects and applies them to
+    anonymize sensitive information in text.
+    """
+
+    def __init__(self, anonymizer_rules: List[rules.Rule], max_depth: int = 10):
+        """Initialize the RulesAnonymizer with a list of rules.
+
+        Args:
+            anonymizer_rules: List of Rule objects to apply for anonymization.
+            max_depth: Maximum recursion depth for nested data structures.
+        """
+        super().__init__(max_depth)
+        self.rules = anonymizer_rules
+
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        """Apply all rules sequentially to the input text.
+
+        Args:
+            data: The text to anonymize.
+
+        Returns:
+            The anonymized text after applying all rules.
+        """
+        result = data
+        for rule in self.rules:
+            result = rule.apply(result)
+        return result

sdks/python/src/opik/api_key/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************

sdks/python/src/opik/api_key/base64_helper.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************
+import base64
+
+
+def decode_base64(data: str, fix_padding: bool = True) -> bytes:
+    if fix_padding:
+        missing_padding = len(data) % 4
+        if missing_padding and data.endswith("="):
+            # wrong padding
+            data = data.replace("=", "")
+            return decode_base64(data)
+
+        if missing_padding:
+            data += "=" * (4 - missing_padding)
+    return base64.b64decode(data, validate=True)

sdks/python/src/opik/api_key/opik_api_key.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************
+import json
+import logging
+from typing import Any, Dict, Optional
+
+from ..logging_messages import (
+    PARSE_API_KEY_EMPTY_EXPECTED_ATTRIBUTES,
+    PARSE_API_KEY_EMPTY_KEY,
+    PARSE_API_KEY_TOO_MANY_PARTS,
+)
+from .base64_helper import decode_base64
+
+LOGGER = logging.getLogger(__name__)
+
+DELIMITER_CHAR = "*"
+
+
+class OpikApiKey:
+    """
+    This is Opik API key parser module which is able to parse enhanced API key format. The format as following:
+    initial 25 chars apiKey + DELIMITER_CHAR + base64 encoded OPIK_BASE_URL and other attributes as JSON dictionary.
+
+    The logic of this module is shared among comet_ml, comet_mpm, and opik projects.
+    Please do not change this module without synchronization with mentioned projects.
+    """
+
+    def __init__(
+        self,
+        api_key_raw: str,
+        api_key: Optional[str] = None,
+        attributes: Optional[Dict[str, Any]] = None,
+    ):
+        self._api_key_raw = api_key_raw
+        self._api_key = api_key
+        self._attributes = attributes
+
+    @property
+    def api_key(self) -> Optional[str]:
+        return self._api_key_raw
+
+    @property
+    def short_api_key(self) -> Optional[str]:
+        if self._api_key is not None:
+            return self._api_key
+        return self._api_key_raw
+
+    @property
+    def base_url(self) -> Optional[str]:
+        if self["baseUrl"] is not None:
+            return str(self["baseUrl"])
+        else:
+            return None
+
+    def __getitem__(self, key: str) -> Any:
+        if self._attributes is not None:
+            return self._attributes.get(key, None)
+
+        return None
+
+
+def parse_api_key(raw_key: str) -> Optional[OpikApiKey]:
+    if raw_key is None or len(raw_key) == 0:
+        LOGGER.debug(PARSE_API_KEY_EMPTY_KEY)
+        return None
+
+    parts = raw_key.split(DELIMITER_CHAR)
+    size = len(parts)
+    if size == 1:
+        LOGGER.debug("Opik API key doesn't have attributes associated")
+        return OpikApiKey(api_key_raw=raw_key)
+    elif size == 2:
+        attr_string = parts[1]
+        if len(attr_string) > 0:
+            data = decode_base64(attr_string)
+            attributes = json.loads(data)
+        else:
+            # edge case - delimiter found but no encoded JSON afterward
+            LOGGER.warning(PARSE_API_KEY_EMPTY_EXPECTED_ATTRIBUTES % raw_key)
+            raw_key = parts[0]  # remove obsolete delimiter
+            attributes = None
+
+        return OpikApiKey(api_key_raw=raw_key, api_key=parts[0], attributes=attributes)
+
+    LOGGER.warning(PARSE_API_KEY_TOO_MANY_PARTS, size, raw_key)
+    return None

sdks/python/src/opik/api_objects/agent_config/__init__.py

@@ -0,0 +1,11 @@
+from .base import Config
+from .cache import get_global_registry
+from .config import ConfigManager
+from .context import agent_config_context
+
+__all__ = [
+    "Config",
+    "ConfigManager",
+    "get_global_registry",
+    "agent_config_context",
+]

sdks/python/src/opik/api_objects/agent_config/base.py

@@ -0,0 +1,565 @@
+import dataclasses
+import logging
+import typing
+
+from opik.exceptions import ConfigMismatch, ConfigNotFound
+from opik.rest_api import core as rest_api_core
+from .. import type_helpers
+from . import cache as cache_mod, types
+from .context import get_active_config_mask, get_active_config_blueprint_name
+
+logger = logging.getLogger(__name__)
+
+_MISSING = object()
+
+T = typing.TypeVar("T", bound="Config")
+
+
+@dataclasses.dataclass
+class _OpikState:
+    project: typing.Optional[str] = None
+    env: typing.Optional[str] = None
+    mask_id: typing.Optional[str] = None
+    version: typing.Optional[str] = None
+    manager: typing.Any = None
+    blueprint_id: typing.Optional[str] = None
+    blueprint_version: typing.Optional[str] = None
+    is_fallback: bool = True
+
+
+def _infer_python_type(value: typing.Any) -> typing.Any:
+    """Return the Python type for a field value. ``None`` maps to ``str``."""
+    if value is None:
+        return str
+    return type(value)
+
+
+def _require_track_context() -> None:
+    """Raise RuntimeError unless called inside an @opik.track function."""
+    from opik import opik_context  # avoid circular import
+
+    if (
+        opik_context.get_current_trace_data() is None
+        and opik_context.get_current_span_data() is None
+    ):
+        raise RuntimeError(
+            "get_or_create_config() must be called inside a function decorated with "
+            "@opik.track. Call get_or_create_config() from within a @opik.track-decorated function."
+        )
+
+
+def _apply_context_overrides(
+    version: typing.Optional[str],
+) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
+    """Apply runner-supplied context overrides. Returns ``(version, mask_id)``."""
+    blueprint_name_override = get_active_config_blueprint_name()
+    if blueprint_name_override is not None:
+        version = blueprint_name_override
+    return version, get_active_config_mask()
+
+
+def _fetch_by_selector(
+    manager: typing.Any,
+    *,
+    version: typing.Optional[str],
+    env: typing.Optional[str],
+    mask_id: typing.Optional[str],
+    field_types: typing.Dict[str, typing.Any],
+    timeout_in_seconds: typing.Optional[int],
+) -> typing.Any:
+    """Fetch a blueprint by version, env, or latest (in priority order)."""
+    if version is not None:
+        return manager.get_blueprint(
+            name=version,
+            mask_id=mask_id,
+            field_types=field_types,
+            timeout_in_seconds=timeout_in_seconds,
+        )
+    if env is not None:
+        return manager.get_blueprint(
+            env=env,
+            mask_id=mask_id,
+            field_types=field_types,
+            timeout_in_seconds=timeout_in_seconds,
+        )
+    return manager.get_blueprint(
+        mask_id=mask_id,
+        field_types=field_types,
+        timeout_in_seconds=timeout_in_seconds,
+    )
+
+
+def _init_fallback_cache_entry(
+    project_name: str,
+    resolved_env: typing.Optional[str],
+    mask_id: typing.Optional[str],
+    field_types: typing.Dict[str, typing.Any],
+    manager: typing.Any,
+    version: typing.Optional[str],
+) -> None:
+    """Record a cache entry with no blueprint; subsequent reads will hit it as fallback."""
+    logger.debug("Failed to fetch config from backend, using fallback", exc_info=True)
+    cache_mod.init_cache_entry(
+        project_name,
+        resolved_env,
+        mask_id,
+        field_types,
+        manager,
+        version=version,
+    )
+
+
+def _validate_prompt_project_names(
+    config: "Config",
+    project_name: str,
+) -> None:
+    """Raise ConfigMismatch if any Prompt/ChatPrompt field belongs to a different project."""
+    from opik.api_objects.prompt.base_prompt import BasePrompt  # avoid circular import
+
+    mismatched = []
+    for name in type(config).__field_names__:
+        value = object.__getattribute__(config, name)
+        if isinstance(value, BasePrompt):
+            prompt_project = value.project_name
+            if prompt_project is not None and prompt_project != project_name:
+                mismatched.append((name, prompt_project))
+
+    if mismatched:
+        details = ", ".join(f"{name!r} (project={proj!r})" for name, proj in mismatched)
+        raise ConfigMismatch(
+            f"Config project is {project_name!r}, but the following prompt field(s) "
+            f"belong to a different project: {details}. "
+            f"All prompts referenced in a config must belong to the same project as the config."
+        )
+
+
+def _all_prompts_synced(config: "Config") -> bool:
+    """Return True only when every BasePrompt field has a non-None commit.
+
+    A None commit means the prompt has not been persisted to the backend yet.
+    Creating a blueprint with an unsynced prompt would store None as the prompt
+    reference, producing a broken config object.
+    """
+    from opik.api_objects.prompt.base_prompt import BasePrompt  # avoid circular import
+
+    for name in type(config).__field_names__:
+        value = object.__getattribute__(config, name)
+        if isinstance(value, BasePrompt) and value.commit is None:
+            logger.debug(
+                "Prompt field %r has no commit — it has not been persisted to the backend yet.",
+                name,
+            )
+            return False
+    return True
+
+
+def _validate_blueprint_schema(cls: typing.Type["Config"], bp: typing.Any) -> None:
+    """Raise ConfigMismatch if ``bp`` is missing any field declared on ``cls``."""
+    missing_keys = [name for name in cls.__field_names__ if name not in bp.keys()]
+    if missing_keys:
+        version_label = bp.name or bp.id or "unknown"
+        raise ConfigMismatch(
+            f"Config version {version_label!r} is missing expected field(s): "
+            f"{missing_keys}. The retrieved version does not contain all fields "
+            f"declared in {cls.__name__}."
+        )
+
+
+def _build_live_instance(
+    cls: typing.Type[T],
+    bp: typing.Any,
+    *,
+    project_name: str,
+    resolved_env: typing.Optional[str],
+    mask_id: typing.Optional[str],
+    version: typing.Optional[str],
+    manager: typing.Any,
+    field_types: typing.Dict[str, typing.Any],
+) -> T:
+    """Construct a backend-backed Config instance and seed its cache entry."""
+    _validate_blueprint_schema(cls, bp)
+
+    kwargs: typing.Dict[str, typing.Any] = {
+        name: bp[name] for name in cls.__field_names__
+    }
+    instance = cls(**kwargs)
+
+    state = instance._state
+    state.project = project_name
+    state.env = resolved_env
+    state.mask_id = mask_id
+    state.version = version
+    state.manager = manager
+    state.blueprint_id = bp.id
+    state.blueprint_version = bp.name
+    state.is_fallback = False
+
+    cache_mod.init_cache_entry(
+        project_name,
+        resolved_env,
+        mask_id,
+        field_types,
+        manager,
+        blueprint=bp,
+        version=version,
+    )
+    return instance
+
+
+def _missing_config_error(
+    project_name: str,
+    *,
+    env: typing.Optional[str],
+    version: typing.Optional[str],
+) -> ConfigNotFound:
+    if version is not None:
+        return ConfigNotFound(
+            f"No config found for version={version!r} in project {project_name!r}."
+        )
+    return ConfigNotFound(
+        f"No config found for env={env!r} in project {project_name!r}."
+    )
+
+
+class Config:
+    """Base class for user-defined configurations.
+
+    Subclass this and declare the fields you want to publish. The annotations
+    are used **only to register field names** (so the class can be turned into
+    a dataclass); the declared types are not inspected or enforced. The actual
+    field type sent to the backend is inferred at runtime from the value you
+    pass — ``type(value)``, or ``str`` when the value is ``None``. Mismatches
+    between the annotation and the value are therefore harmless, and using
+    ``typing.Any`` is fine if you do not want to commit to a static type::
+
+        class MyConfig(opik.Config):
+            temperature: float = 0.7          # default value — used when
+            model: str = "gpt-4"              #   no arg is passed
+            hint: typing.Any = None           # type inferred from the value
+                                              #   actually used at runtime
+
+    Publish a version via :meth:`opik.Opik.create_config`::
+
+        cfg = MyConfig(temperature=0.5)       # defaults fill in the rest
+        client.create_config(cfg, project_name="my-project")
+
+    Retrieve (or auto-create from fallback) via :meth:`opik.Opik.get_or_create_config`::
+
+        result = client.get_or_create_config(
+            fallback=MyConfig(),
+            project_name="my-project",
+        )
+    """
+
+    __field_names__: typing.ClassVar[typing.Tuple[str, ...]] = ()
+
+    def __init_subclass__(cls, **kwargs: typing.Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        if not dataclasses.is_dataclass(cls):
+            dataclasses.dataclass(cls)
+
+        cls.__field_names__ = tuple(
+            f.name
+            for f in dataclasses.fields(cls)  # type: ignore[arg-type]
+        )
+
+    def __init__(self) -> None:
+        # Base-class instantiation path used when ``get_or_create_config`` is
+        # called without a fallback. Subclasses override this via the
+        # dataclass-generated ``__init__``, which still triggers
+        # ``__post_init__`` below.
+        self.__post_init__()
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "_opik_state", _OpikState())
+
+    @property
+    def _state(self) -> _OpikState:
+        return object.__getattribute__(self, "_opik_state")
+
+    @property
+    def is_fallback(self) -> bool:
+        """True if local fallback values are used because there was an issue communicating with the backend."""
+        return self._state.is_fallback
+
+    def _infer_field_types(self) -> typing.Dict[str, typing.Any]:
+        """Return ``{field_name: python_type}`` derived from this instance's values."""
+        return {
+            name: _infer_python_type(object.__getattribute__(self, name))
+            for name in type(self).__field_names__
+        }
+
+    def __getattribute__(self, attr: str) -> typing.Any:
+        field_names = type(self).__field_names__
+        if attr in field_names:
+            if self._state.project is None:
+                return object.__getattribute__(self, attr)
+            return self._resolve_field(attr)
+        # Generic ``Config`` instances (no declared schema) resolve unknown
+        # attributes from the live cache so users can access backend values
+        # even when ``get_or_create_config`` was called without a fallback.
+        if (
+            not field_names
+            and not attr.startswith("_")
+            and not hasattr(type(self), attr)
+            and self._state.project is not None
+        ):
+            return self._resolve_field(attr)
+        return object.__getattribute__(self, attr)
+
+    def _resolve_field(self, attr: str) -> typing.Any:
+        state = self._state
+        project = typing.cast(str, state.project)
+        instance_cache = cache_mod.get_cached_config(
+            project, state.env, state.mask_id, state.version
+        )
+        state.blueprint_id = instance_cache.blueprint_id
+        state.blueprint_version = instance_cache.blueprint_version
+        state.is_fallback = instance_cache.blueprint_id is None
+        value = instance_cache.values.get(attr, _MISSING)
+        self._inject_trace_metadata(attr, value=value)
+        return value if value is not _MISSING else object.__getattribute__(self, attr)
+
+    def _extract_fields_with_values(self) -> typing.Dict[str, types.FieldValueSpec]:
+        result: typing.Dict[str, types.FieldValueSpec] = {}
+        for name in type(self).__field_names__:
+            value = object.__getattribute__(self, name)
+            result[name] = types.FieldValueSpec(
+                python_type=_infer_python_type(value),
+                value=value,
+            )
+        return result
+
+    @classmethod
+    def _get_or_create_from_backend(
+        cls: typing.Type[T],
+        manager: typing.Any,
+        project_name: str,
+        *,
+        fallback: typing.Optional[T] = None,
+        env: typing.Optional[str],
+        version: typing.Optional[str],
+        auto_create_if_empty: bool,
+        timeout_in_seconds: typing.Optional[int],
+    ) -> T:
+        _require_track_context()
+        version, mask_id = _apply_context_overrides(version)
+        # A runner context that pins a specific blueprint name is an explicit
+        # version request — missing it must raise ConfigNotFound, not auto-create.
+        if get_active_config_blueprint_name() is not None:
+            auto_create_if_empty = False
+        resolved_env = None if version is not None else env
+        # Field types come from the fallback's runtime values when available;
+        # without a fallback we pass an empty mapping and rely on the
+        # backend-declared type for each value (see Blueprint._convert_primitives).
+        field_types: typing.Dict[str, typing.Any] = (
+            fallback._infer_field_types() if fallback is not None else {}
+        )
+
+        try:
+            bp = _fetch_by_selector(
+                manager,
+                version=version,
+                env=env,
+                mask_id=mask_id,
+                field_types=field_types,
+                timeout_in_seconds=timeout_in_seconds,
+            )
+        except Exception:
+            if fallback is None:
+                raise
+            _init_fallback_cache_entry(
+                project_name, resolved_env, mask_id, field_types, manager, version
+            )
+            return fallback
+
+        if bp is not None:
+            return _build_live_instance(
+                cls,
+                bp,
+                project_name=project_name,
+                resolved_env=resolved_env,
+                mask_id=mask_id,
+                version=version,
+                manager=manager,
+                field_types=field_types,
+            )
+
+        if not auto_create_if_empty:
+            raise _missing_config_error(project_name, env=env, version=version)
+
+        # env="prod" default path: the initial fetch filtered by env, so probe
+        # project-wide to distinguish "project empty" (auto-create) from
+        # "prod tag missing while other configs exist" (surface ConfigNotFound).
+        # The version="latest" path already queried the project-wide latest.
+        if env is not None:
+            try:
+                probe = manager.get_blueprint(
+                    field_types=field_types,
+                    timeout_in_seconds=timeout_in_seconds,
+                )
+            except Exception:
+                if fallback is None:
+                    raise
+                _init_fallback_cache_entry(
+                    project_name, resolved_env, mask_id, field_types, manager, version
+                )
+                return fallback
+            if probe is not None:
+                raise ConfigNotFound(
+                    f"No config tagged with env={env!r} in project {project_name!r}, "
+                    f"but other configs exist. Tag a version with env={env!r} "
+                    f"via set_config_env(), or pass an explicit env/version."
+                )
+
+        if fallback is None:
+            raise ConfigNotFound(
+                f"No config found in project {project_name!r}. Pass a `fallback` "
+                f"to auto-create one."
+            )
+
+        return cls._create_from_fallback(
+            fallback=fallback,
+            manager=manager,
+            project_name=project_name,
+            mask_id=mask_id,
+            field_types=field_types,
+            timeout_in_seconds=timeout_in_seconds,
+        )
+
+    @classmethod
+    def _create_from_fallback(
+        cls: typing.Type[T],
+        fallback: T,
+        manager: typing.Any,
+        project_name: str,
+        mask_id: typing.Optional[str],
+        field_types: typing.Dict[str, typing.Any],
+        timeout_in_seconds: typing.Optional[int],
+    ) -> T:
+        _validate_prompt_project_names(fallback, project_name)
+
+        # Before auto-creating from fallback, check that all prompt fields have
+        # been persisted to the backend (non-None commit). An unsynced prompt
+        # has no commit, so storing it would produce a broken blueprint.
+        if not _all_prompts_synced(fallback):
+            logger.debug(
+                "One or more prompt fields in the fallback have not been synced with "
+                "the backend yet. Returning the fallback config without creating a blueprint."
+            )
+            return fallback
+
+        fields_with_values = fallback._extract_fields_with_values()
+        try:
+            bp = manager.create_blueprint(
+                fields_with_values=fields_with_values,
+                field_types=field_types,
+            )
+        except rest_api_core.ApiError as e:
+            if e.status_code != 409:
+                raise
+            # Parallel caller created it first — fetch the current latest.
+            bp = manager.get_blueprint(
+                field_types=field_types,
+                timeout_in_seconds=timeout_in_seconds,
+            )
+            if bp is None:
+                raise ConfigNotFound(
+                    f"Failed to create or fetch config in project {project_name!r}."
+                )
+
+        return _build_live_instance(
+            cls,
+            bp,
+            project_name=project_name,
+            resolved_env=None,
+            mask_id=mask_id,
+            version=None,
+            manager=manager,
+            field_types=field_types,
+        )
+
+    def _create_from_instance(
+        self,
+        manager: typing.Any,
+        description: typing.Optional[str] = None,
+    ) -> str:
+        _validate_prompt_project_names(self, manager.project_name)
+        if not _all_prompts_synced(self):
+            raise ConfigMismatch(
+                "One or more prompt fields have not been persisted to the backend yet "
+                "(commit is None). Persist all prompts before calling create_config()."
+            )
+        fields_with_values = self._extract_fields_with_values()
+        field_types = self._infer_field_types()
+
+        latest = manager.get_blueprint(field_types=field_types)
+
+        if latest is not None:
+            bp = manager.update_blueprint(
+                fields_with_values=fields_with_values,
+                description=description,
+                field_types=field_types,
+            )
+        else:
+            try:
+                bp = manager.create_blueprint(
+                    fields_with_values=fields_with_values,
+                    description=description,
+                    field_types=field_types,
+                )
+            except rest_api_core.ApiError as e:
+                if e.status_code != 409:
+                    raise
+                bp = manager.update_blueprint(
+                    fields_with_values=fields_with_values,
+                    description=description,
+                    field_types=field_types,
+                )
+
+        self._state.manager = manager
+        self._state.blueprint_id = bp.id
+        self._state.blueprint_version = bp.name
+        self._state.is_fallback = False
+        return bp.name or ""
+
+    def _inject_trace_metadata(self, attr: str, value: typing.Any = _MISSING) -> None:
+        from opik import exceptions, opik_context
+
+        try:
+            metadata = self._build_trace_metadata(attr, value)
+            payload = {"agent_configuration": metadata}
+            opik_context.update_current_trace(metadata=payload)
+            opik_context.update_current_span(metadata=payload)
+        except exceptions.OpikException:
+            pass
+        except Exception:
+            logger.debug("Failed to inject config metadata into trace", exc_info=True)
+
+    def _build_trace_metadata(
+        self,
+        attr: str,
+        value: typing.Any,
+    ) -> typing.Dict[str, typing.Any]:
+        state = self._state
+        if value is not _MISSING:
+            py_type = _infer_python_type(value)
+            values: typing.Dict[str, typing.Any] = {
+                attr: {
+                    "value": type_helpers.python_value_to_metadata_value(
+                        value, py_type
+                    ),
+                    "type": type_helpers.python_type_to_backend_type(py_type),
+                }
+            }
+        else:
+            values = {}
+        result: typing.Dict[str, typing.Any] = {
+            "_blueprint_id": state.blueprint_id,
+            "blueprint_version": state.blueprint_version,
+        }
+        if state.mask_id is not None:
+            result["_mask_id"] = state.mask_id
+        result["values"] = values
+        return result

sdks/python/src/opik/api_objects/agent_config/blueprint.py

@@ -0,0 +1,164 @@
+import copy
+import datetime
+import typing
+
+from opik.rest_api import client as rest_client
+from opik.rest_api.types.agent_blueprint_public import AgentBlueprintPublic
+from opik.api_objects.prompt.text.prompt import Prompt
+from opik.api_objects.prompt.chat.chat_prompt import ChatPrompt
+from opik.rest_api.types.prompt_version_detail import PromptVersionDetail
+from .. import type_helpers
+
+
+def _resolve_prompt_from_commit(
+    rest_client_: rest_client.OpikApi, commit: str
+) -> typing.Any:
+    prompt_detail = rest_client_.prompts.get_prompt_by_commit(commit)
+    version_detail = prompt_detail.requested_version
+    if (
+        prompt_detail.template_structure == "chat"
+        or version_detail.template_structure == "chat"
+    ):
+        return ChatPrompt.from_fern_prompt_version(
+            name=prompt_detail.name, prompt_version=version_detail
+        )
+    return Prompt.from_fern_prompt_version(
+        name=prompt_detail.name, prompt_version=version_detail
+    )
+
+
+def _resolve_prompt_version_from_commit(
+    rest_client_: rest_client.OpikApi, commit: str
+) -> PromptVersionDetail:
+    prompt_detail = rest_client_.prompts.get_prompt_by_commit(commit)
+    return prompt_detail.requested_version
+
+
+def _convert_primitives(
+    raw_blueprint: AgentBlueprintPublic,
+    field_types: typing.Optional[typing.Dict[str, typing.Any]],
+) -> typing.Dict[str, typing.Any]:
+    values: typing.Dict[str, typing.Any] = {}
+    for param in raw_blueprint.values:
+        if field_types and param.key in field_types:
+            py_type = field_types[param.key]
+        else:
+            py_type = type_helpers.backend_type_to_python_type(param.type)
+
+        if py_type is not None:
+            values[param.key] = type_helpers.backend_value_to_python_value(
+                param.value, py_type
+            )
+        else:
+            values[param.key] = param.value
+    return values
+
+
+def _is_prompt_field(
+    key: str,
+    backend_type: str,
+    field_types: typing.Optional[typing.Dict[str, typing.Any]],
+) -> bool:
+    if field_types and key in field_types:
+        return type_helpers.is_prompt_type(field_types[key])
+    return backend_type == "prompt"
+
+
+def _is_prompt_version_field(
+    key: str,
+    backend_type: str,
+    field_types: typing.Optional[typing.Dict[str, typing.Any]],
+) -> bool:
+    if field_types and key in field_types:
+        return type_helpers.is_prompt_version_type(field_types[key])
+    return backend_type == "prompt_commit"
+
+
+def _resolve_prompts(
+    raw_blueprint: AgentBlueprintPublic,
+    values: typing.Dict[str, typing.Any],
+    field_types: typing.Optional[typing.Dict[str, typing.Any]],
+    rest_client_: rest_client.OpikApi,
+) -> None:
+    for param in raw_blueprint.values:
+        raw_value = values.get(param.key)
+        if not isinstance(raw_value, str):
+            continue
+
+        if _is_prompt_field(param.key, param.type, field_types):
+            values[param.key] = _resolve_prompt_from_commit(rest_client_, raw_value)
+        elif _is_prompt_version_field(param.key, param.type, field_types):
+            values[param.key] = _resolve_prompt_version_from_commit(
+                rest_client_, raw_value
+            )
+
+
+def _resolve_values(
+    raw_blueprint: AgentBlueprintPublic,
+    field_types: typing.Optional[typing.Dict[str, typing.Any]],
+    rest_client_: typing.Optional[rest_client.OpikApi],
+) -> typing.Dict[str, typing.Any]:
+    values = _convert_primitives(raw_blueprint, field_types)
+    if rest_client_:
+        _resolve_prompts(raw_blueprint, values, field_types, rest_client_)
+    return values
+
+
+class Blueprint:
+    """A specific versioned snapshot of agent config values (read-only)."""
+
+    def __init__(
+        self,
+        raw_blueprint: AgentBlueprintPublic,
+        field_types: typing.Optional[typing.Dict[str, typing.Any]] = None,
+        rest_client_: typing.Optional[rest_client.OpikApi] = None,
+    ) -> None:
+        self._raw = raw_blueprint
+        self._values = _resolve_values(raw_blueprint, field_types, rest_client_)
+        self._descriptions: typing.Dict[str, typing.Optional[str]] = {
+            param.key: param.description for param in raw_blueprint.values
+        }
+
+    @property
+    def id(self) -> typing.Optional[str]:
+        return self._raw.id
+
+    @property
+    def name(self) -> typing.Optional[str]:
+        return self._raw.name
+
+    @property
+    def description(self) -> typing.Optional[str]:
+        return self._raw.description
+
+    @property
+    def type(self) -> typing.Optional[str]:
+        return self._raw.type
+
+    @property
+    def envs(self) -> typing.Optional[typing.List[str]]:
+        return self._raw.envs
+
+    @property
+    def created_by(self) -> typing.Optional[str]:
+        return self._raw.created_by
+
+    @property
+    def created_at(self) -> typing.Optional[datetime.datetime]:
+        return self._raw.created_at
+
+    @property
+    def values(self) -> typing.Dict[str, typing.Any]:
+        return copy.deepcopy(self._values)
+
+    def get(self, key: str, default: typing.Any = None) -> typing.Any:
+        return self._values.get(key, default)
+
+    def __getitem__(self, key: str) -> typing.Any:
+        return self._values[key]
+
+    def keys(self) -> typing.KeysView[str]:
+        return self._values.keys()
+
+    def get_field_description(self, key: str) -> typing.Optional[str]:
+        return self._descriptions.get(key)

sdks/python/src/opik/api_objects/agent_config/cache.py

@@ -0,0 +1,217 @@
+import atexit
+import logging
+import os
+import threading
+import time
+import typing
+
+from .blueprint import Blueprint
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_TTL_SECONDS = 300
+_MIN_REFRESH_INTERVAL_SECONDS = 1.0
+
+_CacheKey = typing.Tuple[
+    str, typing.Optional[str], typing.Optional[str], typing.Optional[str]
+]
+
+
+def _get_ttl_seconds() -> int:
+    raw = os.environ.get("OPIK_CONFIG_TTL_SECONDS")
+    if raw is not None:
+        try:
+            return int(raw)
+        except ValueError:
+            pass
+    return DEFAULT_TTL_SECONDS
+
+
+class SharedConfigCache:
+    def __init__(self, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> None:
+        self._lock = threading.RLock()
+        self.blueprint_id: typing.Optional[str] = None
+        self.blueprint_version: typing.Optional[str] = None
+        self.values: typing.Dict[str, typing.Any] = {}
+        self._registered_field_types: typing.Dict[str, typing.Any] = {}
+        self._ttl_seconds = ttl_seconds
+        self._last_fetch: typing.Optional[float] = None
+        self._refresh_callback: typing.Optional[
+            typing.Callable[[], typing.Optional[Blueprint]]
+        ] = None
+
+    def set_refresh_callback(
+        self, callback: typing.Callable[[], typing.Optional[Blueprint]]
+    ) -> None:
+        with self._lock:
+            if self._refresh_callback is None:
+                self._refresh_callback = callback
+
+    def register_fields(self, field_types: typing.Dict[str, typing.Any]) -> None:
+        with self._lock:
+            self._registered_field_types.update(field_types)
+
+    @property
+    def all_field_types(self) -> typing.Dict[str, typing.Any]:
+        with self._lock:
+            return dict(self._registered_field_types)
+
+    def update(self, blueprint: Blueprint) -> None:
+        new_values = dict(blueprint._values)
+        with self._lock:
+            self.blueprint_id = blueprint.id
+            self.blueprint_version = blueprint.name
+            self.values = new_values
+            self._last_fetch = time.monotonic()
+
+    def value_keys(self) -> typing.Set[str]:
+        with self._lock:
+            return set(self.values.keys())
+
+    def is_stale(self) -> bool:
+        with self._lock:
+            if self._last_fetch is None:
+                return True
+            return (time.monotonic() - self._last_fetch) >= self._ttl_seconds
+
+    def try_background_refresh(self) -> None:
+        with self._lock:
+            callback = self._refresh_callback
+        if callback is None:
+            return
+        try:
+            bp = callback()
+            if bp is not None:
+                self.update(bp)
+        except Exception:
+            logger.debug("Background cache refresh failed", exc_info=True)
+
+
+class CacheRefreshThread(threading.Thread):
+    def __init__(
+        self,
+        get_caches: typing.Callable[[], typing.List[SharedConfigCache]],
+        interval_seconds: typing.Optional[float] = None,
+    ) -> None:
+        super().__init__(daemon=True, name="OpikCacheRefresh")
+        self._get_caches = get_caches
+        self._stop_event = threading.Event()
+        self._interval = interval_seconds
+
+    def run(self) -> None:
+        while not self._stop_event.is_set():
+            self._refresh_all_stale()
+            interval = self._interval or float(_get_ttl_seconds())
+            self._stop_event.wait(max(interval, _MIN_REFRESH_INTERVAL_SECONDS))
+
+    def _refresh_all_stale(self) -> None:
+        for cache in self._get_caches():
+            if self._stop_event.is_set():
+                break
+            if cache.is_stale():
+                cache.try_background_refresh()
+
+    def close(self) -> None:
+        self._stop_event.set()
+
+
+class SharedCacheRegistry:
+    def __init__(self) -> None:
+        self._lock = threading.RLock()
+        self._caches: typing.Dict[_CacheKey, SharedConfigCache] = {}
+        self._thread: typing.Optional[CacheRefreshThread] = None
+        self._thread_lock = threading.Lock()
+
+    def get(
+        self,
+        project_name: str,
+        env: typing.Optional[str],
+        mask_id: typing.Optional[str],
+        version: typing.Optional[str] = None,
+    ) -> SharedConfigCache:
+        key: _CacheKey = (project_name, env, mask_id, version)
+        with self._lock:
+            if key not in self._caches:
+                self._caches[key] = SharedConfigCache(ttl_seconds=_get_ttl_seconds())
+            return self._caches[key]
+
+    def clear(self) -> None:
+        self.stop_refresh_thread()
+        with self._lock:
+            self._caches.clear()
+
+    def ensure_refresh_thread_started(self) -> None:
+        with self._thread_lock:
+            if self._thread is not None and self._thread.is_alive():
+                return
+            self._thread = CacheRefreshThread(
+                get_caches=lambda: list(self._caches.values())
+            )
+            self._thread.start()
+            atexit.register(self.stop_refresh_thread)
+
+    def stop_refresh_thread(self) -> None:
+        with self._thread_lock:
+            if self._thread is not None:
+                thread = self._thread
+                thread.close()
+                try:
+                    thread.join(timeout=5)
+                    if thread.is_alive():
+                        logger.error(
+                            "Cache refresh thread did not stop within the timeout."
+                        )
+                except Exception:
+                    logger.exception(
+                        "Unexpected error while waiting for cache refresh thread to stop."
+                    )
+                self._thread = None
+
+
+_registry = SharedCacheRegistry()
+
+
+def get_global_registry() -> SharedCacheRegistry:
+    return _registry
+
+
+# ---------------------------------------------------------------------------
+# Module-level helpers (used by base.py and opik_client.py)
+# ---------------------------------------------------------------------------
+
+
+def get_cached_config(
+    project_name: str,
+    env: typing.Optional[str],
+    mask_id: typing.Optional[str],
+    version: typing.Optional[str] = None,
+) -> SharedConfigCache:
+    return _registry.get(project_name, env, mask_id, version)
+
+
+def init_cache_entry(
+    project_name: str,
+    env: typing.Optional[str],
+    mask_id: typing.Optional[str],
+    field_types: typing.Dict[str, typing.Any],
+    agent_config_manager: typing.Any,
+    blueprint: typing.Optional[Blueprint] = None,
+    version: typing.Optional[str] = None,
+) -> None:
+    shared_cache = _registry.get(project_name, env, mask_id, version)
+    shared_cache.register_fields(field_types)
+
+    if blueprint is not None:
+        shared_cache.update(blueprint)
+
+    if agent_config_manager is not None and mask_id is None and version is None:
+
+        def _refresh() -> typing.Optional[Blueprint]:
+            return agent_config_manager.get_blueprint(
+                env=env,
+                mask_id=mask_id,
+                field_types=shared_cache.all_field_types,
+            )
+
+        shared_cache.set_refresh_callback(_refresh)
+        _registry.ensure_refresh_thread_started()

sdks/python/src/opik/api_objects/agent_config/config.py

@@ -0,0 +1,296 @@
+import typing
+
+from opik.rest_api import client as rest_client
+from opik.rest_api import core as rest_api_core
+from opik.rest_api.core.request_options import RequestOptions
+from opik.rest_api.types.agent_blueprint_write import AgentBlueprintWrite
+from opik.rest_api.types.agent_config_env import AgentConfigEnv
+from opik.rest_api.types.agent_config_value_write import AgentConfigValueWrite
+from opik.api_objects import rest_helpers
+from opik import id_helpers
+
+from .blueprint import Blueprint
+from . import cache as cache_mod, types
+from .. import type_helpers
+
+
+class ConfigManager:
+    """Project-level config entity — internal REST operations."""
+
+    def __init__(
+        self,
+        project_name: str,
+        rest_client_: rest_client.OpikApi,
+    ) -> None:
+        self._project_name = project_name
+        self._rest_client = rest_client_
+
+    @property
+    def project_name(self) -> str:
+        return self._project_name
+
+    @staticmethod
+    def _resolve_fields_with_values(
+        parameters: typing.Optional[typing.Dict[str, typing.Any]],
+        fields_with_values: typing.Optional[typing.Dict[str, types.FieldValueSpec]],
+    ) -> typing.Dict[str, types.FieldValueSpec]:
+        if fields_with_values is not None:
+            return fields_with_values
+        return {
+            k: types.FieldValueSpec(
+                python_type=type(v) if v is not None else str, value=v
+            )
+            for k, v in (parameters or {}).items()
+        }
+
+    def get_blueprint(
+        self,
+        *,
+        name: typing.Optional[str] = None,
+        env: typing.Optional[str] = None,
+        mask_id: typing.Optional[str] = None,
+        field_types: typing.Optional[typing.Dict[str, typing.Any]] = None,
+        timeout_in_seconds: typing.Optional[int] = None,
+    ) -> typing.Optional[Blueprint]:
+        """Fetch a blueprint by name, environment name, or latest.
+
+        Priority: ``name`` > ``env`` > latest.
+        Returns ``None`` if not found.
+
+        Args:
+            name: Fetch the blueprint with this version name.
+            env: Fetch the blueprint tagged with this environment name.
+            mask_id: ID of a mask blueprint to overlay on the result.
+            field_types: Mapping of field name to Python type used
+                for deserialising backend values.
+            timeout_in_seconds: HTTP request timeout in seconds.
+        """
+        request_options: typing.Optional[RequestOptions] = (
+            RequestOptions(timeout_in_seconds=timeout_in_seconds)
+            if timeout_in_seconds is not None
+            else None
+        )
+        try:
+            project_id = rest_helpers.resolve_project_id_by_name(
+                self._rest_client, self._project_name
+            )
+            if name is not None:
+                raw = self._rest_client.agent_configs.get_blueprint_by_name(
+                    project_id=project_id,
+                    name=name,
+                    mask_id=mask_id,
+                    request_options=request_options,
+                )
+            elif env is not None:
+                raw = self._rest_client.agent_configs.get_blueprint_by_env(
+                    env_name=env,
+                    project_id=project_id,
+                    mask_id=mask_id,
+                    request_options=request_options,
+                )
+            else:
+                raw = self._rest_client.agent_configs.get_latest_blueprint(
+                    project_id=project_id,
+                    mask_id=mask_id,
+                    request_options=request_options,
+                )
+        except rest_api_core.ApiError as e:
+            if e.status_code == 404:
+                return None
+            raise
+        return Blueprint(
+            raw_blueprint=raw,
+            field_types=field_types,
+            rest_client_=self._rest_client,
+        )
+
+    def create_blueprint(
+        self,
+        parameters: typing.Optional[typing.Dict[str, typing.Any]] = None,
+        fields_with_values: typing.Optional[
+            typing.Dict[str, types.FieldValueSpec]
+        ] = None,
+        description: typing.Optional[str] = None,
+        field_types: typing.Optional[typing.Dict[str, typing.Any]] = None,
+    ) -> Blueprint:
+        """Create and return the initial blueprint for this agent config (first version only).
+
+        Use this method to establish the first version of the agent config.
+        For subsequent updates use :meth:`update_blueprint`.
+
+        Pass either ``parameters`` (plain key-value pairs whose types are
+        inferred) or ``fields_with_values`` (explicit ``{key: types.FieldValueSpec(type, value)}``
+        mapping). If both are given ``fields_with_values`` takes precedence.
+
+        Args:
+            parameters: Plain ``{field_name: value}`` dict; types are inferred
+                via ``type(value)``.
+            fields_with_values: Explicit ``{field_name: types.FieldValueSpec(python_type, value)}``
+                mapping, bypassing type inference.
+            description: Human-readable description stored with the blueprint.
+            field_types: Mapping of field name to Python type used
+                when fetching back the created blueprint.
+        """
+        resolved_fields_with_values = self._resolve_fields_with_values(
+            parameters, fields_with_values
+        )
+        blueprint_id = id_helpers.generate_id()
+        payload = _build_blueprint_payload(
+            resolved_fields_with_values, description, id=blueprint_id
+        )
+        self._rest_client.agent_configs.create_agent_config(
+            blueprint=payload,
+            project_name=self._project_name,
+        )
+        raw = self._rest_client.agent_configs.get_blueprint_by_id(blueprint_id)
+        return Blueprint(
+            raw_blueprint=raw,
+            field_types=field_types,
+            rest_client_=self._rest_client,
+        )
+
+    def update_blueprint(
+        self,
+        fields_with_values: typing.Optional[
+            typing.Dict[str, types.FieldValueSpec]
+        ] = None,
+        description: typing.Optional[str] = None,
+        field_types: typing.Optional[typing.Dict[str, typing.Any]] = None,
+    ) -> Blueprint:
+        """Create a new blueprint with only the supplied fields (not merged with previous).
+
+        Each call creates a new versioned snapshot containing exactly the fields
+        you provide in ``fields_with_values``. Fields omitted from that mapping
+        are **not** carried over from the previous version.
+
+        ``fields_with_values`` must map each field name to a
+        :class:`~opik.api_objects.agent_config.types.FieldValueSpec` that
+        declares both the Python type and the value.  For prompt fields the
+        value must be a :class:`~opik.api_objects.prompt.text.prompt.Prompt`
+        (or :class:`~opik.api_objects.prompt.chat.chat_prompt.ChatPrompt`)
+        instance::
+
+            config.update_blueprint(
+                fields_with_values={
+                    "system_prompt": types.FieldValueSpec(
+                        python_type=Prompt, value=my_prompt
+                    ),
+                    "temperature": types.FieldValueSpec(
+                        python_type=float, value=0.7
+                    ),
+                }
+            )
+
+        Args:
+            fields_with_values: ``{field_name: types.FieldValueSpec(python_type, value)}``
+                mapping.  Each entry overrides that field in the new blueprint.
+            description: Human-readable description stored with the blueprint.
+            field_types: Mapping of field name to Python type used
+                when fetching back the created blueprint.
+        """
+        resolved_fields_with_values = self._resolve_fields_with_values(
+            None, fields_with_values
+        )
+        blueprint_id = id_helpers.generate_id()
+        payload = _build_blueprint_payload(
+            resolved_fields_with_values, description, id=blueprint_id
+        )
+        self._rest_client.agent_configs.update_agent_config(
+            blueprint=payload,
+            project_name=self._project_name,
+        )
+        raw = self._rest_client.agent_configs.get_blueprint_by_id(blueprint_id)
+        return Blueprint(
+            raw_blueprint=raw,
+            field_types=field_types,
+            rest_client_=self._rest_client,
+        )
+
+    def set_env(self, version: str, env: str) -> None:
+        """Tag a specific blueprint version with an environment name.
+
+        After tagging, ``get_blueprint(env=env)`` will return this version.
+
+        Args:
+            version: Version name of the blueprint to tag.
+            env: Environment name (e.g. ``"prod"``, ``"staging"``).
+        """
+        project_id = rest_helpers.resolve_project_id_by_name(
+            self._rest_client, self._project_name
+        )
+        # Use the cached blueprint_id when available to skip a round-trip.
+        cached = cache_mod.get_cached_config(self._project_name, None, None, version)
+        blueprint_id = cached.blueprint_id
+        if blueprint_id is None:
+            blueprint = self._rest_client.agent_configs.get_blueprint_by_name(
+                project_id=project_id,
+                name=version,
+            )
+            blueprint_id = blueprint.id
+        self._rest_client.agent_configs.create_or_update_envs(
+            project_id=project_id,
+            envs=[AgentConfigEnv(env_name=env, blueprint_id=blueprint_id)],
+        )
+
+    def create_mask(
+        self,
+        parameters: typing.Optional[typing.Dict[str, typing.Any]] = None,
+        fields_with_values: typing.Optional[
+            typing.Dict[str, types.FieldValueSpec]
+        ] = None,
+        description: typing.Optional[str] = None,
+    ) -> str:
+        """Create a mask blueprint and return its ID.
+
+        A mask overlays a subset of fields on top of an existing blueprint.
+        Apply it by passing the returned ID to ``get_blueprint(mask_id=...)``.
+
+        Args:
+            parameters: Plain ``{field_name: value}`` dict; types are inferred
+                via ``type(value)``.
+            fields_with_values: Explicit ``{field_name: types.FieldValueSpec(python_type, value)}``
+                mapping, bypassing type inference.
+            description: Human-readable description stored with the mask.
+        """
+        resolved_fields_with_values = self._resolve_fields_with_values(
+            parameters, fields_with_values
+        )
+        mask_id = id_helpers.generate_id()
+        payload = _build_blueprint_payload(
+            resolved_fields_with_values, description, id=mask_id, config_type="mask"
+        )
+        self._rest_client.agent_configs.update_agent_config(
+            blueprint=payload,
+            project_name=self._project_name,
+        )
+        return mask_id
+
+
+def _build_blueprint_payload(
+    fields_with_values: typing.Dict[str, types.FieldValueSpec],
+    description: typing.Optional[str],
+    id: typing.Optional[str] = None,
+    config_type: str = "blueprint",
+) -> AgentBlueprintWrite:
+    backend_values = []
+    for field_name, field_spec in fields_with_values.items():
+        backend_type = (
+            type_helpers.python_type_to_backend_type(field_spec.python_type)
+            if field_spec.value is not None
+            else "string"
+        )
+        backend_values.append(
+            AgentConfigValueWrite(
+                key=field_name,
+                type=backend_type,
+                value=type_helpers.python_value_to_backend_value(
+                    field_spec.value, field_spec.python_type
+                ),
+            )
+        )
+    return AgentBlueprintWrite(
+        id=id,
+        type=config_type,
+        values=backend_values,
+        description=description,
+    )

sdks/python/src/opik/api_objects/agent_config/context.py

@@ -0,0 +1,33 @@
+import contextvars
+from contextlib import contextmanager
+from typing import Optional, Generator
+
+_active_config_mask_var: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
+    "opik_active_config_mask", default=None
+)
+
+_active_config_blueprint_name_var: contextvars.ContextVar[Optional[str]] = (
+    contextvars.ContextVar("opik_active_config_blueprint_name", default=None)
+)
+
+
+@contextmanager
+def agent_config_context(
+    mask_id: Optional[str],
+    blueprint_name: Optional[str] = None,
+) -> Generator[None, None, None]:
+    mask_token = _active_config_mask_var.set(mask_id)
+    blueprint_token = _active_config_blueprint_name_var.set(blueprint_name)
+    try:
+        yield
+    finally:
+        _active_config_mask_var.reset(mask_token)
+        _active_config_blueprint_name_var.reset(blueprint_token)
+
+
+def get_active_config_mask() -> Optional[str]:
+    return _active_config_mask_var.get()
+
+
+def get_active_config_blueprint_name() -> Optional[str]:
+    return _active_config_blueprint_name_var.get()

sdks/python/src/opik/api_objects/agent_config/types.py

@@ -0,0 +1,13 @@
+import typing
+
+
+class FieldValueSpec(typing.NamedTuple):
+    """Describes a single blueprint field's value for write operations.
+
+    Attributes:
+        python_type: The Python type of the field (e.g. ``str``, ``int``).
+        value: The field value to write.
+    """
+
+    python_type: type[typing.Any]
+    value: typing.Any

sdks/python/src/opik/api_objects/annotation_queue/__init__.py

@@ -0,0 +1,12 @@
+from .annotation_queue import (
+    BaseAnnotationQueue,
+    TracesAnnotationQueue,
+    ThreadsAnnotationQueue,
+)
+
+
+__all__ = [
+    "BaseAnnotationQueue",
+    "TracesAnnotationQueue",
+    "ThreadsAnnotationQueue",
+]

sdks/python/src/opik/api_objects/annotation_queue/annotation_queue.py

@@ -0,0 +1,412 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import (
+    Optional,
+    List,
+    Literal,
+    Union,
+)
+
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.types import (
+    trace_public,
+    trace_thread,
+    trace_filter_public,
+    trace_thread_filter,
+)
+from opik.message_processing.batching import sequence_splitter
+from opik.api_objects.trace import trace_client
+from opik.api_objects.rest_helpers import ensure_rest_api_call_respecting_rate_limit
+from opik.api_objects import constants, helpers, search_helpers
+import opik.exceptions as exceptions
+
+LOGGER = logging.getLogger(__name__)
+
+TraceType = Union[trace_client.Trace, trace_public.TracePublic]
+
+
+class BaseAnnotationQueue(ABC):
+    """
+    Base class for annotation queue objects.
+
+    This class provides the common functionality shared between
+    TracesAnnotationQueue and ThreadsAnnotationQueue.
+
+    This object should not be created directly, instead use the appropriate
+    create/get methods on opik.Opik client.
+    """
+
+    def __init__(
+        self,
+        id: str,
+        name: str,
+        project_id: str,
+        rest_client: rest_api_client.OpikApi,
+        description: Optional[str] = None,
+        instructions: Optional[str] = None,
+        comments_enabled: Optional[bool] = None,
+        feedback_definition_names: Optional[List[str]] = None,
+        items_count: Optional[int] = None,
+    ) -> None:
+        self._id = id
+        self._name = name
+        self._description = description
+        self._instructions = instructions
+        self._project_id = project_id
+        self._comments_enabled = comments_enabled
+        self._feedback_definition_names = feedback_definition_names
+        self._items_count = items_count
+        self._rest_client = rest_client
+
+    @property
+    def id(self) -> str:
+        """The id of the annotation queue."""
+        return self._id
+
+    @property
+    def name(self) -> str:
+        """The name of the annotation queue."""
+        return self._name
+
+    @property
+    def description(self) -> Optional[str]:
+        """The description of the annotation queue."""
+        return self._description
+
+    @property
+    def instructions(self) -> Optional[str]:
+        """The instructions for reviewers."""
+        return self._instructions
+
+    @property
+    def project_id(self) -> str:
+        """The project ID associated with this annotation queue."""
+        return self._project_id
+
+    @property
+    def comments_enabled(self) -> Optional[bool]:
+        """Whether comments are enabled for this queue."""
+        return self._comments_enabled
+
+    @property
+    def feedback_definition_names(self) -> Optional[List[str]]:
+        """The feedback definition names associated with this queue."""
+        return self._feedback_definition_names
+
+    @property
+    @abstractmethod
+    def scope(self) -> str:
+        """The scope of the annotation queue ('trace' or 'thread')."""
+        pass
+
+    @property
+    def items_count(self) -> Optional[int]:
+        """
+        The total number of items in the queue.
+
+        If the count is not cached locally, it will be fetched from the backend.
+        """
+        if self._items_count is None:
+            queue_info = self._rest_client.annotation_queues.get_annotation_queue_by_id(
+                self._id
+            )
+            self._items_count = queue_info.items_count
+        return self._items_count
+
+    def update(
+        self,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        instructions: Optional[str] = None,
+        comments_enabled: Optional[bool] = None,
+        feedback_definition_names: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Update the annotation queue properties.
+
+        Args:
+            name: New name for the queue.
+            description: New description for the queue.
+            instructions: New instructions for reviewers.
+            comments_enabled: Whether to enable comments.
+            feedback_definition_names: List of feedback definition names.
+        """
+        self._rest_client.annotation_queues.update_annotation_queue(
+            id=self._id,
+            name=name,
+            description=description,
+            instructions=instructions,
+            comments_enabled=comments_enabled,
+            feedback_definition_names=feedback_definition_names,
+        )
+
+        if name is not None:
+            self._name = name
+        if description is not None:
+            self._description = description
+        if instructions is not None:
+            self._instructions = instructions
+        if comments_enabled is not None:
+            self._comments_enabled = comments_enabled
+        if feedback_definition_names is not None:
+            self._feedback_definition_names = feedback_definition_names
+
+    def delete(self) -> None:
+        """
+        Delete this annotation queue.
+        """
+        self._rest_client.annotation_queues.delete_annotation_queue_batch(
+            ids=[self._id]
+        )
+
+    def _add_items_batch_with_retry(self, ids: List[str]) -> None:
+        """Add a batch of items with automatic retry on rate limit errors."""
+        ensure_rest_api_call_respecting_rate_limit(
+            lambda: self._rest_client.annotation_queues.add_items_to_annotation_queue(
+                id=self._id, ids=ids
+            )
+        )
+        LOGGER.debug("Successfully added %d items to annotation queue", len(ids))
+
+    def _remove_items_batch_with_retry(self, ids: List[str]) -> None:
+        """Remove a batch of items with automatic retry on rate limit errors."""
+        ensure_rest_api_call_respecting_rate_limit(
+            lambda: self._rest_client.annotation_queues.remove_items_from_annotation_queue(
+                id=self._id, ids=ids
+            )
+        )
+        LOGGER.debug("Successfully removed %d items from annotation queue", len(ids))
+
+
+class TracesAnnotationQueue(BaseAnnotationQueue):
+    """
+    An annotation queue for traces.
+
+    This queue is used to collect traces for human annotation workflows.
+
+    This object should not be created directly, instead use
+    :meth:`opik.Opik.create_traces_annotation_queue` or :meth:`opik.Opik.get_traces_annotation_queue`.
+    """
+
+    SCOPE: Literal["trace"] = "trace"
+
+    @property
+    def scope(self) -> str:
+        """The scope of the annotation queue."""
+        return self.SCOPE
+
+    def _extract_trace_ids(
+        self,
+        traces: List[TraceType],
+    ) -> List[str]:
+        """Extract IDs from trace objects."""
+        ids: List[str] = []
+        for trace in traces:
+            if trace.id is None:
+                raise exceptions.OpikException("Trace object has no id")
+            ids.append(trace.id)
+
+        return ids
+
+    def add_traces(
+        self,
+        traces: List[TraceType],
+    ) -> None:
+        """
+        Add trace objects to the annotation queue.
+
+        Args:
+            traces: A list of traces to add. For a single trace, wrap it in a list: [trace].
+                Accepts Trace objects (from opik_client.trace()) or TracePublic objects
+                (from search_traces()).
+
+        Raises:
+            OpikException: If any trace object has no id.
+        """
+        ids = self._extract_trace_ids(traces)
+        if not ids:
+            return
+
+        batches = sequence_splitter.split_into_batches(
+            ids, max_length=constants.ANNOTATION_QUEUE_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Adding %d traces to annotation queue", len(batch))
+            self._add_items_batch_with_retry(batch)
+
+        self._items_count = None
+
+    def remove_traces(
+        self,
+        traces: List[TraceType],
+    ) -> None:
+        """
+        Remove trace objects from the annotation queue.
+
+        Args:
+            traces: A list of traces to remove. For a single trace, wrap it in a list: [trace].
+                Accepts Trace objects (from opik_client.trace()) or TracePublic objects
+                (from search_traces()).
+
+        Raises:
+            OpikException: If any trace object has no id.
+        """
+        ids = self._extract_trace_ids(traces)
+        if not ids:
+            return
+
+        batches = sequence_splitter.split_into_batches(
+            ids, max_length=constants.ANNOTATION_QUEUE_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Removing %d traces from annotation queue", len(batch))
+            self._remove_items_batch_with_retry(batch)
+
+        self._items_count = None
+
+    def get_items(
+        self,
+        truncate_images: bool = True,
+    ) -> List[trace_public.TracePublic]:
+        """
+        Get all trace objects currently in the annotation queue.
+
+        Args:
+            truncate_images: Whether to truncate inline base64 image data stored in
+                input, output, or metadata of the returned traces.
+
+        Returns:
+            List[trace_public.TracePublic]: All traces currently in the queue.
+        """
+        filters = helpers.parse_filter_expressions(
+            f'annotation_queue_ids contains "{self._id}"',
+            parsed_item_class=trace_filter_public.TraceFilterPublic,
+            entity_type="traces",
+        )
+
+        return search_helpers.search_traces_with_filters(
+            rest_client=self._rest_client,
+            project_id=self._project_id,
+            filters=filters,
+            max_results=None,
+            truncate=truncate_images,
+        )
+
+
+class ThreadsAnnotationQueue(BaseAnnotationQueue):
+    """
+    An annotation queue for threads.
+
+    This queue is used to collect threads for human annotation workflows.
+
+    This object should not be created directly, instead use
+    :meth:`opik.Opik.create_threads_annotation_queue` or :meth:`opik.Opik.get_threads_annotation_queue`.
+    """
+
+    SCOPE: Literal["thread"] = "thread"
+
+    @property
+    def scope(self) -> str:
+        """The scope of the annotation queue."""
+        return self.SCOPE
+
+    def _extract_thread_ids(
+        self,
+        threads: List[trace_thread.TraceThread],
+    ) -> List[str]:
+        """Extract thread_model_id from TraceThread objects."""
+        ids: List[str] = []
+        for thread in threads:
+            if thread.thread_model_id is None:
+                raise exceptions.OpikException(
+                    "TraceThread object has no thread_model_id"
+                )
+            ids.append(thread.thread_model_id)
+
+        return ids
+
+    def add_threads(
+        self,
+        threads: List[trace_thread.TraceThread],
+    ) -> None:
+        """
+        Add thread objects to the annotation queue.
+
+        Args:
+            threads: A list of TraceThread objects to add (from search_threads()).
+                For a single thread, wrap it in a list: [thread].
+
+        Raises:
+            OpikException: If any thread object has no thread_model_id.
+        """
+        ids = self._extract_thread_ids(threads)
+        if not ids:
+            return
+
+        batches = sequence_splitter.split_into_batches(
+            ids, max_length=constants.ANNOTATION_QUEUE_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Adding %d threads to annotation queue", len(batch))
+            self._add_items_batch_with_retry(batch)
+
+        self._items_count = None
+
+    def remove_threads(
+        self,
+        threads: List[trace_thread.TraceThread],
+    ) -> None:
+        """
+        Remove thread objects from the annotation queue.
+
+        Args:
+            threads: A list of TraceThread objects to remove (from search_threads()).
+                For a single thread, wrap it in a list: [thread].
+
+        Raises:
+            OpikException: If any thread object has no thread_model_id.
+        """
+        ids = self._extract_thread_ids(threads)
+        if not ids:
+            return
+
+        batches = sequence_splitter.split_into_batches(
+            ids, max_length=constants.ANNOTATION_QUEUE_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Removing %d threads from annotation queue", len(batch))
+            self._remove_items_batch_with_retry(batch)
+
+        self._items_count = None
+
+    def get_items(
+        self,
+        truncate_images: bool = True,
+    ) -> List[trace_thread.TraceThread]:
+        """
+        Get all thread objects currently in the annotation queue.
+
+        Args:
+            truncate_images: Whether to truncate inline base64 image data stored in
+                input, output, or metadata of the returned threads.
+
+        Returns:
+            List[trace_thread.TraceThread]: All threads currently in the queue.
+        """
+        filters = helpers.parse_filter_expressions(
+            f'annotation_queue_ids contains "{self._id}"',
+            parsed_item_class=trace_thread_filter.TraceThreadFilter,
+            entity_type="threads",
+        )
+
+        return search_helpers.search_threads_with_filters(
+            rest_client=self._rest_client,
+            project_id=self._project_id,
+            filters=filters,
+            max_results=None,
+            truncate=truncate_images,
+        )

sdks/python/src/opik/api_objects/annotation_queue/rest_operations.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import json
+from typing import Callable, List, Optional, Type, TypeVar
+
+from opik.rest_api import OpikApi
+from opik.rest_api.types import AnnotationQueuePublic
+import opik.exceptions as exceptions
+from . import annotation_queue
+from ...rest_api.core.api_error import ApiError
+
+QueueT = TypeVar(
+    "QueueT",
+    annotation_queue.TracesAnnotationQueue,
+    annotation_queue.ThreadsAnnotationQueue,
+)
+
+
+def _create_queue_instance(
+    queue_data: AnnotationQueuePublic,
+    rest_client: OpikApi,
+    queue_class: Type[QueueT],
+) -> QueueT:
+    """Helper to create an annotation queue instance from API response data."""
+    return queue_class(
+        id=queue_data.id or "",
+        name=queue_data.name,
+        project_id=queue_data.project_id,
+        rest_client=rest_client,
+        description=queue_data.description,
+        instructions=queue_data.instructions,
+        comments_enabled=queue_data.comments_enabled,
+        feedback_definition_names=list(queue_data.feedback_definition_names)
+        if queue_data.feedback_definition_names
+        else None,
+        items_count=queue_data.items_count,
+    )
+
+
+def _get_annotation_queues_by_scope(
+    rest_client: OpikApi,
+    queue_class: Type[QueueT],
+    scope_filter: Callable[[Optional[str]], bool],
+    project_id: Optional[str] = None,
+    max_results: int = 1000,
+) -> List[QueueT]:
+    """Helper to fetch annotation queues filtered by scope."""
+    page_size = 100
+    queues: List[QueueT] = []
+
+    filters: Optional[str] = None
+    if project_id is not None:
+        filters = json.dumps(
+            [
+                {
+                    "field": "project_id",
+                    "type": "string",
+                    "operator": "=",
+                    "value": project_id,
+                }
+            ]
+        )
+
+    page = 1
+    while len(queues) < max_results:
+        page_queues = rest_client.annotation_queues.find_annotation_queues(
+            page=page,
+            size=page_size,
+            filters=filters,
+        )
+
+        if page_queues.content is None or len(page_queues.content) == 0:
+            break
+
+        for queue_data in page_queues.content:
+            if len(queues) >= max_results:
+                break
+            if scope_filter(queue_data.scope):
+                queues.append(
+                    _create_queue_instance(queue_data, rest_client, queue_class)
+                )
+
+        page += 1
+
+    return queues
+
+
+def _get_annotation_queue_by_id_with_scope(
+    rest_client: OpikApi,
+    queue_id: str,
+    queue_class: Type[QueueT],
+    scope_check: Callable[[Optional[str]], bool],
+    scope_name: str,
+) -> QueueT:
+    """Helper to fetch an annotation queue by ID with scope validation."""
+    try:
+        queue_data = rest_client.annotation_queues.get_annotation_queue_by_id(queue_id)
+    except ApiError as e:
+        if e.status_code == 404:
+            raise exceptions.OpikException(
+                f"Annotation queue with id '{queue_id}' not found."
+            ) from e
+        raise
+
+    if not scope_check(queue_data.scope):
+        actual_scope = queue_data.scope or "trace"
+        raise exceptions.OpikException(
+            f"Annotation queue with id '{queue_id}' is not a {scope_name} queue (scope: {actual_scope})."
+        )
+
+    return _create_queue_instance(queue_data, rest_client, queue_class)
+
+
+def get_traces_annotation_queues(
+    rest_client: OpikApi,
+    project_id: Optional[str] = None,
+    max_results: int = 1000,
+) -> List[annotation_queue.TracesAnnotationQueue]:
+    """
+    Fetch trace annotation queues with optional project filtering.
+
+    Args:
+        rest_client: The REST API client.
+        project_id: Optional project ID to filter queues.
+        max_results: Maximum number of queues to return.
+
+    Returns:
+        A list of TracesAnnotationQueue objects.
+    """
+    return _get_annotation_queues_by_scope(
+        rest_client=rest_client,
+        queue_class=annotation_queue.TracesAnnotationQueue,
+        scope_filter=lambda s: s == "trace",
+        project_id=project_id,
+        max_results=max_results,
+    )
+
+
+def get_threads_annotation_queues(
+    rest_client: OpikApi,
+    project_id: Optional[str] = None,
+    max_results: int = 1000,
+) -> List[annotation_queue.ThreadsAnnotationQueue]:
+    """
+    Fetch thread annotation queues with optional project filtering.
+
+    Args:
+        rest_client: The REST API client.
+        project_id: Optional project ID to filter queues.
+        max_results: Maximum number of queues to return.
+
+    Returns:
+        A list of ThreadsAnnotationQueue objects.
+    """
+    return _get_annotation_queues_by_scope(
+        rest_client=rest_client,
+        queue_class=annotation_queue.ThreadsAnnotationQueue,
+        scope_filter=lambda s: s == "thread",
+        project_id=project_id,
+        max_results=max_results,
+    )
+
+
+def get_traces_annotation_queue_by_id(
+    rest_client: OpikApi,
+    queue_id: str,
+) -> annotation_queue.TracesAnnotationQueue:
+    """
+    Fetch a trace annotation queue by its ID.
+
+    Args:
+        rest_client: The REST API client.
+        queue_id: The ID of the annotation queue.
+
+    Returns:
+        A TracesAnnotationQueue object.
+
+    Raises:
+        OpikException: If the queue is not found or is not a trace queue.
+    """
+    return _get_annotation_queue_by_id_with_scope(
+        rest_client=rest_client,
+        queue_id=queue_id,
+        queue_class=annotation_queue.TracesAnnotationQueue,
+        scope_check=lambda s: s == "trace",
+        scope_name="traces",
+    )
+
+
+def get_threads_annotation_queue_by_id(
+    rest_client: OpikApi,
+    queue_id: str,
+) -> annotation_queue.ThreadsAnnotationQueue:
+    """
+    Fetch a thread annotation queue by its ID.
+
+    Args:
+        rest_client: The REST API client.
+        queue_id: The ID of the annotation queue.
+
+    Returns:
+        A ThreadsAnnotationQueue object.
+
+    Raises:
+        OpikException: If the queue is not found or is not a thread queue.
+    """
+    return _get_annotation_queue_by_id_with_scope(
+        rest_client=rest_client,
+        queue_id=queue_id,
+        queue_class=annotation_queue.ThreadsAnnotationQueue,
+        scope_check=lambda s: s == "thread",
+        scope_name="threads",
+    )

sdks/python/src/opik/api_objects/attachment/__init__.py

@@ -0,0 +1,5 @@
+from .attachment import Attachment
+from .client import AttachmentClient
+
+
+__all__ = ["Attachment", "AttachmentClient"]

sdks/python/src/opik/api_objects/attachment/attachment.py

@@ -0,0 +1,28 @@
+from typing import Optional, Union
+
+import pydantic
+
+
+class Attachment(pydantic.BaseModel):
+    """
+    Represents an Attachment to be added to the Trace or Span.
+
+    Args:
+        data: The data to be added to the Attachment. Can be:
+            - A file path (str) to an existing file
+            - A base64-encoded string (str) representing file content
+            - Raw bytes content
+        file_name: The custom filename to assign to the data in the attachment.
+            If not provided, the original filename of the data will be used.
+        content_type: The MIME type of the data to be added to the attachment.
+            If not specified, it will be inferred from the data file.
+        create_temp_copy: If True, a temporary copy of the file will be created
+            before upload. This ensures the file remains available even if the
+            original is deleted. The temp file will be deleted after upload.
+            Default is True.
+    """
+
+    data: Union[str, bytes]
+    file_name: Optional[str] = None
+    content_type: Optional[str] = None
+    create_temp_copy: bool = True

sdks/python/src/opik/api_objects/attachment/attachment_context.py

@@ -0,0 +1,36 @@
+import dataclasses
+from typing import Literal
+
+from . import attachment
+
+
+@dataclasses.dataclass
+class AttachmentWithContext:
+    """
+    Represents an attachment along with its associated context.
+
+    This class is used to pair an attachment with additional contextual
+    information such as the entity type, entity ID, project name, and
+    context description. It is specifically useful when dealing with
+    attachments related to entities like spans or traces. The context
+    can help provide further insights or classification of the
+    attachment's purpose.
+
+    Attributes:
+        attachment_data: The actual attachment
+            object containing the associated data.
+        entity_type: The type of entity the
+            attachment is associated with. It must be either "span"
+            or "trace".
+        entity_id: The unique identifier of the related entity.
+        project_name: The name of the project to which the
+            attachment and its entity belong.
+        context: A brief context description for the attachment,
+            explaining its purpose or relevance.
+    """
+
+    attachment_data: attachment.Attachment
+    entity_type: Literal["span", "trace"]
+    entity_id: str
+    project_name: str
+    context: str

sdks/python/src/opik/api_objects/attachment/attachments_extractor.py

@@ -0,0 +1,215 @@
+import re
+from typing import Dict, Any, Literal, List, NamedTuple, Union
+
+from . import attachment, attachment_context, decoder_base64
+
+
+class ExtractionResult(NamedTuple):
+    attachments: List[attachment.Attachment]
+    sanitized_data: Any
+
+
+class AttachmentsExtractor:
+    """
+    Extracts and processes attachments embedded as Base64 strings within data structures.
+
+    This class is designed to identify and decode Base64-encoded attachments located
+    within the provided data. It uses a regular expression pattern to search for
+    Base64 strings that meet a specified minimum length. Extracted attachments are
+    decoded and replaced with sanitized placeholders in the original data.
+    """
+
+    def __init__(self, min_attachment_size: int):
+        """
+        Initializes the class with a minimum attachment size and configures the base64
+        pattern for decoding attachments based on its length.
+
+        Args:
+            min_attachment_size: The minimum size of the attachment in characters
+                for it to be considered valid. This ensures that only large enough
+                base64 strings are matched to minimize false positives.
+        """
+        self._min_attachment_size = min_attachment_size
+        self.decoder = decoder_base64.Base64AttachmentDecoder()
+
+        # Pattern to match base64 strings (can be embedded in text)
+        # Requires at least min_attachment_size characters to reduce false positives.
+        # An optional `data:<mime>;base64,` prefix is matched too so that data URIs
+        # (e.g. OpenAI/LangChain image_url.url) are replaced whole, not just their payload.
+        min_base64_groups = int(min_attachment_size / 4)
+        BASE64_PATTERN = (
+            r"(?P<prefix>data:[^,]*;base64,)?"
+            r"(?P<base64>(?:[A-Za-z0-9+/]{4}){"
+            + str(min_base64_groups)
+            + r",}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?)"
+        )
+        self.pattern = re.compile(BASE64_PATTERN)
+
+    def extract_and_replace(
+        self,
+        data: Union[Dict[str, Any], List[Any]],
+        entity_type: Literal["span", "trace"],
+        entity_id: str,
+        project_name: str,
+        context: Literal["input", "output", "metadata"],
+    ) -> List[attachment_context.AttachmentWithContext]:
+        """
+        Extract attachments from data and replace with placeholders.
+
+        Handles both dict and list at the top level, recursively processing
+        nested structures to find and extract base64-encoded attachments.
+
+        Args:
+            data: The data structure to process (dict or list)
+            entity_type: Type of entity (span or trace)
+            entity_id: ID of the entity
+            project_name: Name of the project
+            context: Context where data is located (input, output, or metadata)
+
+        Returns:
+            List of extracted attachments with context
+        """
+        attachments: List[attachment_context.AttachmentWithContext] = []
+
+        if isinstance(data, dict):
+            # For dicts, iterate over items and extract attachments from values
+            for key, value in data.items():
+                extraction_result = self._try_extract_attachments(value, context)
+                if extraction_result.attachments:
+                    # replace the original value with the sanitized one and collect attachments
+                    data[key] = extraction_result.sanitized_data
+                    for extracted_attachment in extraction_result.attachments:
+                        attachments.append(
+                            attachment_context.AttachmentWithContext(
+                                attachment_data=extracted_attachment,
+                                entity_type=entity_type,
+                                entity_id=entity_id,
+                                project_name=project_name,
+                                context=context,
+                            )
+                        )
+        elif isinstance(data, list):
+            # For lists, extract attachments from each item and replace in place
+            extraction_result = self._try_extract_attachments(data, context)
+            # Replace list contents with sanitized version
+            data.clear()
+            data.extend(extraction_result.sanitized_data)
+            # Convert extracted attachments to AttachmentWithContext
+            for extracted_attachment in extraction_result.attachments:
+                attachments.append(
+                    attachment_context.AttachmentWithContext(
+                        attachment_data=extracted_attachment,
+                        entity_type=entity_type,
+                        entity_id=entity_id,
+                        project_name=project_name,
+                        context=context,
+                    )
+                )
+        else:
+            # For other types (str, int, bool, None, etc.), try to extract but don't mutate
+            extraction_result = self._try_extract_attachments(data, context)
+            for extracted_attachment in extraction_result.attachments:
+                attachments.append(
+                    attachment_context.AttachmentWithContext(
+                        attachment_data=extracted_attachment,
+                        entity_type=entity_type,
+                        entity_id=entity_id,
+                        project_name=project_name,
+                        context=context,
+                    )
+                )
+
+        return attachments
+
+    def _try_extract_attachments(
+        self, data: Any, context: Literal["input", "output", "metadata"]
+    ) -> ExtractionResult:
+        """
+        Recursively extract attachments from data that can be a string, dict, list, or other type.
+
+        Args:
+            data: The data to process (can be str, dict, list, or other types)
+            context: The context where the data is located (input, output, or metadata)
+
+        Returns:
+            ExtractionResult with extracted attachments and sanitized data
+        """
+        # Handle string data - check for base64 attachments
+        if isinstance(data, str):
+            return self._extract_from_string(data, context)
+
+        # Handle dictionary data - recursively process each value
+        elif isinstance(data, dict):
+            return self._extract_from_dict(data, context)
+
+        # Handle list data - recursively process each element
+        elif isinstance(data, list):
+            return self._extract_from_list(data, context)
+
+        # For other types (int, bool, None, etc.), return as-is
+        else:
+            return ExtractionResult(attachments=[], sanitized_data=data)
+
+    def _extract_from_string(
+        self, data: str, context: Literal["input", "output", "metadata"]
+    ) -> ExtractionResult:
+        """Extract attachments from a string value."""
+        if len(data) < self._min_attachment_size:
+            # skip short strings
+            return ExtractionResult(attachments=[], sanitized_data=data)
+
+        attachments: List[attachment.Attachment] = []
+        # Rebuild positionally so each match gets its own placeholder. Using
+        # `str.replace` here would rewrite every occurrence of the matched chunk,
+        # causing later duplicate matches to alias the first one's file name.
+        parts: List[str] = []
+        last_end = 0
+        for match in self.pattern.finditer(data):
+            to_decode = match.group("base64")
+            decoded_attachment = self.decoder.decode(to_decode, context)
+            if decoded_attachment is None:
+                continue
+            attachments.append(decoded_attachment)
+            # The full match span covers any optional `data:<mime>;base64,` prefix,
+            # so replacing it with the bare placeholder strips the prefix in one go.
+            parts.append(data[last_end : match.start()])
+            parts.append(f"[{decoded_attachment.file_name}]")
+            last_end = match.end()
+
+        if not attachments:
+            return ExtractionResult(attachments=attachments, sanitized_data=data)
+
+        parts.append(data[last_end:])
+        return ExtractionResult(attachments=attachments, sanitized_data="".join(parts))
+
+    def _extract_from_dict(
+        self, data: Dict[str, Any], context: Literal["input", "output", "metadata"]
+    ) -> ExtractionResult:
+        """Recursively extract attachments from a dictionary."""
+        all_attachments: List[attachment.Attachment] = []
+        sanitized_dict = {}
+
+        for key, value in data.items():
+            result = self._try_extract_attachments(value, context)
+            sanitized_dict[key] = result.sanitized_data
+            all_attachments.extend(result.attachments)
+
+        return ExtractionResult(
+            attachments=all_attachments, sanitized_data=sanitized_dict
+        )
+
+    def _extract_from_list(
+        self, data: List[Any], context: Literal["input", "output", "metadata"]
+    ) -> ExtractionResult:
+        """Recursively extract attachments from a list."""
+        all_attachments: List[attachment.Attachment] = []
+        sanitized_list = []
+
+        for item in data:
+            result = self._try_extract_attachments(item, context)
+            sanitized_list.append(result.sanitized_data)
+            all_attachments.extend(result.attachments)
+
+        return ExtractionResult(
+            attachments=all_attachments, sanitized_data=sanitized_list
+        )

sdks/python/src/opik/api_objects/attachment/base64_normalizer.py

@@ -0,0 +1,69 @@
+"""Normalize URL-safe base64 binary blobs to the standard base64 alphabet.
+
+Some upstream SDKs (notably google.genai, which sets pydantic's
+``ser_json_bytes='base64'`` on its BaseModel) emit URL-safe base64, with '-'
+and '_' in place of '+' and '/'. Opik's downstream consumers — the SDK
+attachments extractor in this package, and the frontend's inline image
+rendering — only match the standard alphabet, so an unnormalized URL-safe blob
+is silently truncated at the first '-' or '_' (see OPIK-6387).
+
+Detection is content-based: we decode the first few bytes and require an image
+file signature before rewriting, so that unrelated values which happen to
+share the URL-safe alphabet (e.g. UUIDs) are left untouched.
+"""
+
+import base64
+import binascii
+import re
+from typing import Any
+
+from . import decoder_helpers
+
+_URLSAFE_BASE64_RE = re.compile(r"[A-Za-z0-9_-]+={0,2}")
+_MIN_BASE64_IMAGE_LENGTH = 24
+
+
+def normalize_urlsafe_base64_images_in_place(node: Any) -> None:
+    """Walk a nested dict/list and rewrite any URL-safe-base64-encoded image
+    string leaves to standard base64. Non-string leaves are ignored.
+    """
+    if isinstance(node, dict):
+        for key, child in node.items():
+            if isinstance(child, str):
+                if is_urlsafe_base64_image(child):
+                    node[key] = urlsafe_to_standard_base64(child)
+            else:
+                normalize_urlsafe_base64_images_in_place(child)
+    elif isinstance(node, list):
+        for index, item in enumerate(node):
+            if isinstance(item, str):
+                if is_urlsafe_base64_image(item):
+                    node[index] = urlsafe_to_standard_base64(item)
+            else:
+                normalize_urlsafe_base64_images_in_place(item)
+
+
+def is_urlsafe_base64_image(value: str) -> bool:
+    """True if ``value`` is a URL-safe base64 encoding of an image whose
+    header bytes match a known signature (PNG, JPEG, GIF, WebP/RIFF, TIFF).
+
+    Returns False for strings already in the standard alphabet (no '-' or
+    '_'), so callers can use this as a cheap "needs rewriting?" check.
+    """
+    if len(value) < _MIN_BASE64_IMAGE_LENGTH:
+        return False
+    if "-" not in value and "_" not in value:
+        return False
+    if not _URLSAFE_BASE64_RE.fullmatch(value):
+        return False
+    head = value[:16]
+    head += "=" * (-len(head) % 4)
+    try:
+        decoded = base64.urlsafe_b64decode(head)
+    except (binascii.Error, ValueError):
+        return False
+    return decoder_helpers.detect_image_mime_type_from_header(decoded) is not None
+
+
+def urlsafe_to_standard_base64(value: str) -> str:
+    return value.replace("-", "+").replace("_", "/")

sdks/python/src/opik/api_objects/attachment/client.py

@@ -0,0 +1,219 @@
+import base64
+import logging
+import os
+import mimetypes
+import httpx
+import json.decoder
+from opik import s3_httpx_client
+from typing import Iterator, List, Literal, Optional
+from typing_extensions import TypeAlias
+
+from opik.file_upload import file_uploader, upload_options
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.types import attachment as rest_api_attachment
+from opik.rest_api import core as rest_api_core
+from opik import url_helpers
+from .. import rest_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+RESTAttachmentDetails: TypeAlias = rest_api_attachment.Attachment
+
+
+class AttachmentClient:
+    """
+    Client for interacting with attachment-related operations.
+
+    This client provides methods to retrieve attachment lists, download attachments,
+    and upload attachments for traces and spans.
+
+    The AttachmentClient supports:
+    - Listing attachments associated with traces or spans
+    - Downloading attachment content as a byte stream
+    - Uploading files as attachments to traces or spans
+
+    All operations are performed within the context of a specific project and require
+    the project name to be provided.
+    """
+
+    def __init__(
+        self,
+        rest_client: rest_api_client.OpikApi,
+        url_override: str,
+        workspace_name: str,
+        rest_httpx_client: httpx.Client,
+    ) -> None:
+        """
+        Initialize the AttachmentClient.
+        It is typically created via ``Opik.get_attachment_client()`` rather
+        than being instantiated directly.
+
+        Parameters:
+            rest_client: The REST API client instance for making backend requests.
+            url_override: The base URL for the Opik server.
+            workspace_name: The workspace name used for download operations.
+            rest_httpx_client: The httpx client instance to use for making file uploads.
+
+        Returns:
+            None
+        """
+        self._rest_client = rest_client
+        self._url_override = url_override
+        self._workspace_name = workspace_name
+        self._rest_httpx_client = rest_httpx_client
+
+    def get_attachment_list(
+        self,
+        project_name: str,
+        entity_id: str,
+        entity_type: Literal["span", "trace"],
+    ) -> List[RESTAttachmentDetails]:  # type: ignore
+        """
+        Get a list of attachments for a specific entity (trace or span).
+
+        Parameters:
+            project_name: The name of the project containing the entity.
+            entity_id: The ID of the trace or span to retrieve attachments for.
+            entity_type: The type of entity ("trace" or "span").
+
+        Returns:
+            List[RESTAttachmentDetails]: List of attachment detail objects containing metadata about each attachment.
+        """
+        project_id = rest_helpers.resolve_project_id_by_name(
+            self._rest_client, project_name
+        )
+        url_override_path = base64.b64encode(self._url_override.encode("utf-8")).decode(
+            "utf-8"
+        )
+
+        response = self._rest_client.attachments.attachment_list(
+            project_id=project_id,
+            entity_type=entity_type,
+            entity_id=entity_id,
+            path=url_override_path,
+        )
+
+        return response.content or []
+
+    def download_attachment(
+        self,
+        project_name: str,
+        entity_type: Literal["trace", "span"],
+        entity_id: str,
+        file_name: str,
+        mime_type: str,
+    ) -> Iterator[bytes]:
+        """
+        Download an attachment as a stream of bytes.
+
+        Parameters:
+            project_name: The name of the project containing the entity.
+            entity_type: The type of entity ("trace" or "span").
+            entity_id: The ID of the trace or span containing the attachment.
+            file_name: The name of the file to download.
+            mime_type: The MIME type of the file.
+
+        Returns:
+            Iterator[bytes]: Iterator yielding bytes of the attachment content.
+        """
+        attachments_details = self.get_attachment_list(
+            project_name=project_name,
+            entity_type=entity_type,
+            entity_id=entity_id,
+        )
+        attachment_to_download: Optional[RESTAttachmentDetails] = None
+
+        for attachment_details in attachments_details:
+            if (
+                attachment_details.file_name == file_name
+                and attachment_details.mime_type == mime_type
+            ):
+                attachment_to_download = attachment_details
+                break
+
+        if attachment_to_download is None:
+            raise ValueError(f"Attachment not found: {file_name}")
+
+        if attachment_to_download.link is None:
+            raise ValueError(f"No download URL available for attachment: {file_name}")
+
+        httpx_client_upload = (
+            s3_httpx_client.get_cached()
+            if url_helpers.is_aws_presigned_url(attachment_to_download.link)
+            else self._rest_httpx_client
+        )
+
+        with httpx_client_upload.stream("GET", attachment_to_download.link) as response:
+            try:
+                if 200 <= response.status_code < 300:
+                    for chunk in response.iter_bytes():
+                        yield chunk
+                    return
+                response.read()
+                response_json = response.json()
+            except json.decoder.JSONDecodeError:
+                raise rest_api_core.ApiError(
+                    status_code=response.status_code,
+                    headers=dict(response.headers),
+                    body=response.text,
+                )
+            raise rest_api_core.ApiError(
+                status_code=response.status_code,
+                headers=dict(response.headers),
+                body=response_json,
+            )
+
+    def upload_attachment(
+        self,
+        project_name: str,
+        entity_type: Literal["trace", "span"],
+        entity_id: str,
+        file_path: str,
+        file_name: Optional[str] = None,
+        mime_type: Optional[str] = None,
+    ) -> None:
+        """
+        Upload an attachment for a specific entity (trace or span).
+
+        Parameters:
+            project_name: The name of the project containing the entity.
+            entity_type: The type of entity ("trace" or "span").
+            entity_id: The ID of the trace or span to attach the file to.
+            file_path: The path to the file to upload on the local filesystem.
+            file_name: The name to assign to the uploaded file. If not provided, uses the basename of file_path.
+            mime_type: The MIME type of the file. If not provided, attempts to automatically detect based on the file extension.
+
+        Returns:
+            None
+        """
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        if file_name is None:
+            file_name = os.path.basename(file_path)
+
+        if mime_type is None:
+            mime_type, _ = mimetypes.guess_type(file_path)
+
+        file_size = os.path.getsize(file_path)
+        encoded_url_override = base64.b64encode(
+            self._url_override.encode("utf-8")
+        ).decode("utf-8")
+
+        upload_opts = upload_options.FileUploadOptions(
+            file_path=file_path,
+            file_name=file_name,
+            file_size=file_size,
+            mime_type=mime_type,
+            entity_type=entity_type,
+            entity_id=entity_id,
+            project_name=project_name,
+            encoded_url_override=encoded_url_override,
+            delete_after_upload=False,
+        )
+
+        file_uploader.upload_attachment(
+            upload_options=upload_opts,
+            rest_client=self._rest_client,
+            upload_httpx_client=self._rest_httpx_client,
+        )

sdks/python/src/opik/api_objects/attachment/converters.py

@@ -0,0 +1,164 @@
+import base64
+import binascii
+import logging
+import os
+import shutil
+import tempfile
+from typing import Literal, Optional
+
+from ...file_upload import mime_type
+from ...message_processing import messages
+from . import attachment
+
+LOGGER = logging.getLogger(__name__)
+
+
+def attachment_to_message(
+    attachment_data: attachment.Attachment,
+    entity_type: Literal["trace", "span"],
+    entity_id: str,
+    project_name: str,
+    url_override: str,
+    delete_after_upload: bool = False,
+) -> messages.CreateAttachmentMessage:
+    if attachment_data.data is None:
+        raise ValueError("Attachment data cannot be None")
+
+    mimetype = guess_attachment_type(attachment_data)
+    base_url_path = base64.b64encode(url_override.encode("utf-8")).decode("utf-8")
+
+    if isinstance(attachment_data.data, bytes):
+        file_path = _write_file_like_to_temp_file(attachment_data.data)
+        # make sure the temporary file is deleted after upload
+        attachment_data.create_temp_copy = False
+        delete_after_upload = True
+    elif os.path.exists(attachment_data.data):
+        file_path = attachment_data.data
+    else:
+        try:
+            decoded_bytes = base64.b64decode(attachment_data.data, validate=True)
+            file_path = _write_file_like_to_temp_file(decoded_bytes)
+            attachment_data.create_temp_copy = False
+            delete_after_upload = True
+        except binascii.Error as e:
+            raise ValueError(
+                "Attachment data must be bytes, an existing file path, or a valid base64-encoded string."
+            ) from e
+
+    file_name = attachment_data.file_name
+    should_delete_after_upload = delete_after_upload
+
+    if file_name is None:
+        file_name = os.path.basename(file_path)
+
+    # Try to create a temporary copy if requested
+    if attachment_data.create_temp_copy:
+        tmp_file_path = _try_create_temp_copy(file_path)
+        if tmp_file_path is not None:
+            file_path = tmp_file_path
+            should_delete_after_upload = True
+        else:
+            should_delete_after_upload = False
+
+    return messages.CreateAttachmentMessage(
+        file_path=file_path,
+        file_name=file_name,
+        mime_type=mimetype,
+        entity_type=entity_type,
+        entity_id=entity_id,
+        project_name=project_name,
+        encoded_url_override=base_url_path,
+        delete_after_upload=should_delete_after_upload,
+    )
+
+
+def _write_file_like_to_temp_file(file_like: bytes) -> str:
+    """
+    Writes a bytes-like file object to a temporary file on the filesystem.
+
+    This function accepts a file-like object in the form of bytes and writes its
+    contents to a temporary file. The temporary file is not deleted automatically
+    on program termination. The path to the created temporary file is returned.
+
+    Args:
+        file_like: The bytes-like object containing the file content
+            to be written to a temporary file.
+
+    Returns:
+        The full path to the created temporary file.
+    """
+    temp_file = tempfile.NamedTemporaryFile(mode="wb", delete=False)
+    try:
+        temp_file.write(file_like)
+        temp_file.flush()
+        LOGGER.debug(
+            "Created temporary copy of file-like attachment to file: %s",
+            temp_file.name,
+        )
+    except Exception as e:
+        LOGGER.error(
+            "Failed to write file-like attachment to temp file: %s",
+            e,
+            exc_info=True,
+        )
+        raise
+    finally:
+        temp_file.close()
+
+    return temp_file.name
+
+
+def _try_create_temp_copy(file_path: str) -> Optional[str]:
+    """
+    Create a temporary copy of a file.
+
+    This ensures the file remains available for upload even if the user
+    deletes the original file. The temp file is created with delete=False
+    so it persists until the upload manager processes and deletes it.
+
+    Args:
+        file_path: Path to the original file.
+
+    Returns:
+        Path to the temporary copy.
+    """
+    _, extension = os.path.splitext(file_path)
+    temp_file = tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=extension)
+    try:
+        with open(file_path, "rb") as original_file:
+            shutil.copyfileobj(original_file, temp_file)
+        temp_file.flush()
+        temp_file.close()
+        LOGGER.debug(
+            "Created temporary copy of attachment: %s -> %s",
+            file_path,
+            temp_file.name,
+        )
+        return temp_file.name
+    except Exception:
+        temp_file.close()
+        LOGGER.error(
+            "Failed to create temporary copy of attachment: %s. Opik will try to use the original file.",
+            file_path,
+            exc_info=True,
+        )
+        return None
+
+
+def guess_attachment_type(attachment_data: attachment.Attachment) -> Optional[str]:
+    if attachment_data.content_type is not None:
+        return attachment_data.content_type
+
+    mimetype = None
+    if attachment_data.file_name is not None:
+        mimetype = mime_type.guess_mime_type(file=attachment_data.file_name)
+
+    if mimetype is not None:
+        return mimetype
+
+    if isinstance(attachment_data.data, str):
+        mimetype = mime_type.guess_mime_type(file=attachment_data.data)
+    elif isinstance(attachment_data.data, bytes):
+        mimetype = mime_type.BINARY_MIME_TYPE
+
+    return mimetype

sdks/python/src/opik/api_objects/attachment/decoder.py

@@ -0,0 +1,18 @@
+import abc
+from typing import Any, Optional
+
+from . import attachment
+
+
+class AttachmentDecoder(abc.ABC):
+    """
+    Abstract base class for decoding file attachments.
+
+    This class serves as an interface for decoding raw attachment data into
+    an `Attachment` object. Implementing classes should define the specific
+    logic to handle various attachment decoding formats.
+    """
+
+    @abc.abstractmethod
+    def decode(self, raw_data: str, **kwargs: Any) -> Optional[attachment.Attachment]:
+        pass

sdks/python/src/opik/api_objects/attachment/decoder_base64.py

@@ -0,0 +1,83 @@
+import base64
+import binascii
+import logging
+import tempfile
+from typing import Any, Optional, Literal
+
+from . import attachment, decoder, decoder_helpers
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Base64AttachmentDecoder(decoder.AttachmentDecoder):
+    """Decodes base64 encoded attachment data.
+
+    This decoder decodes base64 strings, detects MIME types from content, and creates Attachment objects.
+    """
+
+    def decode(
+        self,
+        raw_data: str,
+        context: Literal["input", "output", "metadata"] = "input",
+        **kwargs: Any,
+    ) -> Optional[attachment.Attachment]:
+        """Decode base64 encoded data into an Attachment object.
+
+        Args:
+            raw_data: Base64 encoded string data
+            context: Context string for filename generation.
+
+        Returns:
+            Attachment object with decoded data, or None if decoding fails or type is not recognizable
+        """
+        if not isinstance(raw_data, str):
+            LOGGER.warning("Attachment data is not a string, skipping.")
+            return None
+
+        try:
+            # Decode base64 string to bytes
+            decoded_bytes = base64.b64decode(raw_data, validate=True)
+
+            # Detect MIME type from content
+            mime_type = decoder_helpers.detect_mime_type(decoded_bytes)
+
+            # Skip if not a recognizable file type
+            if not mime_type or mime_type in ("application/octet-stream", "text/plain"):
+                LOGGER.debug("Attachment type is not recognized, skipping.")
+                return None
+
+            # Get file extension from the MIME type
+            extension = decoder_helpers.get_file_extension(mime_type)
+
+            # Generate filename
+            file_name = decoder_helpers.create_attachment_filename(
+                context, extension=extension
+            )
+
+            # Save decoded bytes to a temporary file
+            temp_file = tempfile.NamedTemporaryFile(
+                mode="wb", delete=False, suffix=extension
+            )
+            temp_file.write(decoded_bytes)
+            temp_file.flush()
+            temp_file.close()
+
+            # Return Attachment object with a file path
+            return attachment.Attachment(
+                data=temp_file.name, file_name=file_name, content_type=mime_type
+            )
+
+        except (ValueError, binascii.Error) as e:
+            LOGGER.debug(
+                "Failed to decode attachment data, reason: invalid base64. Reason: %s",
+                e,
+                exc_info=True,
+            )
+            # Not valid base64, return None
+            return None
+        except Exception as ex:
+            LOGGER.warning(
+                "Failed to decode attachment data, reason: %s", ex, exc_info=True
+            )
+            # Unexpected error, return None to avoid crashing the pipeline
+            return None

sdks/python/src/opik/api_objects/attachment/decoder_helpers.py

@@ -0,0 +1,156 @@
+import mimetypes
+import random
+import time
+from typing import Optional
+
+
+# The attachment file name regex
+ATTACHMENT_FILE_NAME_REGEX = r"(?:input|output|metadata)-attachment-\d+-\d+-sdk\.\w+"
+ATTACHMENT_FILE_NAME_PLACEHOLDER_REGEX = (
+    r"\[((?:input|output|metadata)-attachment-\d+-\d+-sdk\.\w+)\]"
+)
+
+
+def get_file_extension(mime_type: str) -> str:
+    """Convert MIME type to file extension.
+
+    Mirrors the Java getFileExtension() method in AttachmentStripperService.
+
+    Args:
+        mime_type: The MIME type (e.g., "image/png", "application/pdf")
+
+    Returns:
+        File extension without a leading dot (e.g., "png", "pdf")
+    """
+    if not mime_type:
+        return "bin"
+
+    # Try to get extension from mimetypes module
+    extension = mimetypes.guess_extension(mime_type, strict=False)
+
+    if extension:
+        # Remove the leading dot
+        extension = extension.lstrip(".")
+        # Handle special cases where mimetypes returns less common extensions
+        if mime_type == "image/jpeg" and extension == "jpe":
+            return "jpg"
+        return extension
+
+    # Fallback: extract from the MIME type (e.g., "image/png" -> "png")
+    if "/" in mime_type:
+        subtype = mime_type.split("/")[1]
+        # Handle special cases like "svg+xml" -> "svg"
+        if "+" in subtype:
+            subtype = subtype.split("+")[0]
+        # Remove any parameters (e.g., "jpeg; charset=utf-8" -> "jpeg")
+        subtype = subtype.split(";")[0].strip()
+        return subtype
+
+    return "bin"
+
+
+# Image signatures keyed by header prefix. The WebP entry is handled
+# separately because it lives inside a RIFF container (the "WEBP" marker is
+# at offset 8, not at the start).
+_IMAGE_HEADER_SIGNATURES = (
+    (b"\x89PNG\r\n\x1a\n", "image/png"),
+    (b"\xff\xd8\xff", "image/jpeg"),
+    (b"GIF87a", "image/gif"),
+    (b"GIF89a", "image/gif"),
+)
+
+
+def detect_image_mime_type_from_header(data: bytes) -> Optional[str]:
+    """Return the image MIME type implied by the leading bytes of ``data``,
+    or None if no known image signature matches.
+
+    Header-only — works on partial payloads (e.g. the first 16 bytes decoded
+    from a base64 string). Use this when you don't have the full file and
+    only need to know "is this an image, and which kind?".
+    """
+    for prefix, mime in _IMAGE_HEADER_SIGNATURES:
+        if data.startswith(prefix):
+            return mime
+    if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        return "image/webp"
+    return None
+
+
+def detect_mime_type(data: bytes) -> Optional[str]:
+    """Detect MIME type from byte content using magic bytes.
+
+    This provides basic MIME type detection similar to Apache Tika in the Java implementation.
+    It checks common file format magic bytes.
+
+    Args:
+        data: The byte data to analyze
+
+    Returns:
+        Detected MIME type string, or "application/octet-stream" if unknown
+    """
+    if len(data) < 4:
+        return "application/octet-stream"
+
+    image_mime_type = detect_image_mime_type_from_header(data)
+    if image_mime_type is not None:
+        # Historical behavior: JPEG also requires the FFD9 end marker, so we
+        # only treat a payload as JPEG when the full file is intact.
+        if image_mime_type == "image/jpeg" and data[-2:] != b"\xff\xd9":
+            pass
+        else:
+            return image_mime_type
+
+    # PDF
+    if data[:4] == b"%PDF":
+        return "application/pdf"
+
+    # SVG (XML-based, check for SVG tag)
+    try:
+        text = data[:1024].decode("utf-8", errors="ignore")
+        if "<svg" in text.lower():
+            return "image/svg+xml"
+    except Exception:
+        pass
+
+    # MP4
+    if len(data) >= 12 and data[4:8] == b"ftyp":
+        return "video/mp4"
+
+    # JSON
+    try:
+        text = data[:100].decode("utf-8", errors="strict").strip()
+        if text.startswith("{") or text.startswith("["):
+            return "application/json"
+    except Exception:
+        pass
+
+    # Default to octet-stream for unknown types
+    return "application/octet-stream"
+
+
+def create_attachment_filename(context: str, extension: str) -> str:
+    """
+    Generates a unique attachment filename based on the provided context and file extension.
+
+    This function creates a filename by combining the given context, a randomly generated
+    prefix to ensure uniqueness, the current timestamp in milliseconds, and the provided
+    file extension. The generated filename aligns with the backend convention for naming
+    attachments, which includes specific formatting and structure.
+
+    Args:
+        context: The context to use as the base for the filename (e.g., "input",
+            "output", or "metadata").
+        extension: The file extension to use for the filename (e.g., "png",
+            "jpg", "txt").
+
+    Returns:
+        A generated filename string in the format
+            "{context}-attachment-{random_prefix}-{timestamp}.{extension}".
+    """
+    # The backend has the following naming convention: r"\\[((?:input|output|metadata)-attachment-\\d+-\\d+\\.\\w+)\\]"
+    # Example: [input-attachment-1-1704067200000.png]
+
+    timestamp = int(round(time.time() * 1000))
+    # we need to generate a large enough random prefix to avoid collisions
+    random_prefix = random.randint(1, 99999999)
+    return f"{context}-attachment-{random_prefix}-{timestamp}-sdk.{extension}"

sdks/python/src/opik/api_objects/connection_resources.py

@@ -0,0 +1,412 @@
+"""Connection-scoped transport resources shared by :class:`opik.Opik` handles.
+
+The objects built here (httpx pool, REST client, message-processing chain, file
+upload manager, replay manager + connection monitor, and the streamer with its
+consumer threads) are properties of the *connection* ``(url, workspace, api_key,
+...)`` rather than of an individual client.
+
+Responsibilities are split so each type does one thing:
+
+- :class:`SharedConnectionResourcesBundle` — the value object: holds the live
+  transport objects and knows how to dispose them (``close``).
+- :class:`ConnectionResourceManager` — the lifecycle authority: derives the
+  connection identity, builds-or-reuses a bundle, ref-counts it, and decides
+  when to tear it down (including at process exit).
+- :class:`Lease` — a per-handle, release-once token that delegates all lifecycle
+  decisions back to the manager.
+"""
+
+import atexit
+import hashlib
+import json
+import logging
+import threading
+from typing import Callable, Dict, Optional, Tuple
+
+import httpx
+
+from .. import config as opik_config
+from .. import httpx_client, rest_client_configurator
+from ..file_upload import upload_manager
+from ..healthcheck import connection_monitor, connection_probe
+from ..message_processing import (
+    message_queue,
+    permissions,
+    streamer,
+    streamer_constructors,
+)
+from ..message_processing.processors import message_processors, message_processors_chain
+from ..message_processing.replay import replay_manager
+from ..rest_api import client as rest_api_client
+
+LOGGER = logging.getLogger(__name__)
+
+
+class SharedConnectionResourcesBundle:
+    """Owns the expensive transport objects for one connection identity.
+
+    Connection-scoped: it carries no ``project_name`` or per-call state, so it
+    can back multiple :class:`opik.Opik` handles. ``close`` disposes what the
+    bundle owns — the streamer's threads and the file-upload worker pool, plus
+    the httpx connection pool on a durable (``flush=True``) close — so evicting a
+    bundle never leaks threads. ``flush_timeout`` is the connection's configured
+    drain budget, used when the process-exit hook closes the bundle.
+    """
+
+    def __init__(
+        self,
+        httpx_client: httpx.Client,
+        rest_client: rest_api_client.OpikApi,
+        message_processor: message_processors.ChainedMessageProcessor,
+        file_upload_manager: upload_manager.FileUploadManager,
+        replay_manager: replay_manager.ReplayManager,
+        streamer: streamer.Streamer,
+        flush_timeout: Optional[int],
+    ) -> None:
+        self.httpx_client = httpx_client
+        self.rest_client = rest_client
+        self.message_processor = message_processor
+        self.file_upload_manager = file_upload_manager
+        self.replay_manager = replay_manager
+        self.streamer = streamer
+        self.flush_timeout = flush_timeout
+
+    def close(self, timeout: Optional[int], *, flush: bool) -> None:
+        # Drain/stop the streamer (consumer threads, replay, batch preprocessor);
+        # on flush=True it also flushes pending file uploads.
+        # Closing the streamer also stops and joins the replay manager (its own
+        # daemon thread), so there is no separate replay teardown to do here.
+        self.streamer.close(timeout, flush=flush)
+        # Stop the upload worker pool too, so eviction doesn't leave its threads
+        # running. wait=flush mirrors the streamer: block for in-flight uploads
+        # on a durable close, return immediately on fire-and-forget teardown.
+        self.file_upload_manager.close(wait=flush)
+        if flush:
+            # Close the httpx pool only on a durable close, and last — after the
+            # streamer has joined the replay thread and uploads have drained, so
+            # no request is in flight. Each bundle owns a dedicated client (built
+            # per connection identity), so this never affects another bundle.
+            # flush=False is fire-and-forget: the streamer deliberately leaves
+            # daemon threads to finish in-flight requests, so closing the pool
+            # here would race them — leave it for GC / process-exit close_all.
+            self.httpx_client.close()
+
+    def flush(self, timeout: Optional[int]) -> None:
+        """Drain the shared message queue without tearing the bundle down.
+
+        Used when a handle releases with ``flush=True`` while other handles still
+        share the bundle: the queued data is persisted now, but the transport
+        stays alive for the remaining handles.
+        """
+        self.streamer.flush(timeout)
+
+
+def _create_replay_manager(
+    config: opik_config.OpikConfig, httpx_client: httpx.Client
+) -> replay_manager.ReplayManager:
+    probe = connection_probe.ConnectionProbe(
+        base_url=config.url_override,
+        client=httpx_client,
+    )
+    monitor = connection_monitor.OpikConnectionMonitor(
+        ping_interval=config.connection_monitor_ping_interval,
+        check_timeout=config.connection_monitor_check_timeout,
+        probe=probe,
+    )
+
+    return replay_manager.ReplayManager(
+        monitor=monitor,
+        batch_size=config.replay_batch_size,
+        batch_replay_delay=config.replay_batch_replay_delay,
+        tick_interval_seconds=config.replay_tick_interval,
+    )
+
+
+def create_connection_resources(
+    config: opik_config.OpikConfig, *, use_batching: bool
+) -> SharedConnectionResourcesBundle:
+    """Build a full transport stack for ``config``.
+
+    Pure construction with no cache awareness — this is the default builder that
+    :class:`ConnectionResourceManager` invokes on a cache miss.
+    """
+    httpx_client_ = httpx_client.get(
+        workspace=config.workspace,
+        api_key=config.api_key,
+        check_tls_certificate=config.check_tls_certificate,
+        compress_json_requests=config.enable_json_request_compression,
+    )
+    rest_client = rest_api_client.OpikApi(
+        base_url=config.url_override,
+        httpx_client=httpx_client_,
+    )
+    rest_client._client_wrapper._timeout = (
+        httpx.USE_CLIENT_DEFAULT
+    )  # See https://github.com/fern-api/fern/issues/5321
+    rest_client_configurator.configure(rest_client)
+
+    max_queue_size = message_queue.calculate_max_queue_size(
+        maximal_queue_size=config.maximal_queue_size,
+        batch_factor=config.maximal_queue_size_batch_factor,
+    )
+
+    file_uploader = upload_manager.FileUploadManager(
+        rest_client=rest_client,
+        httpx_client=httpx_client_,
+        worker_count=config.file_upload_background_workers,
+    )
+
+    fallback_replay = _create_replay_manager(config, httpx_client_)
+
+    message_processor = message_processors_chain.create_message_processors_chain(
+        rest_client=rest_client,
+        file_upload_manager=file_uploader,
+        fallback_replay_manager=fallback_replay,
+        unauthorized_message_types_registry=permissions.UnauthorizedMessageTypeRegistry(
+            retry_interval_seconds=config.unauthorized_message_type_retry_interval,
+            max_retry_count=config.unauthorized_message_type_max_retry_count,
+        ),
+    )
+    streamer_ = streamer_constructors.construct_online_streamer(
+        file_uploader=file_uploader,
+        n_consumers=config.background_workers,
+        use_batching=use_batching,
+        use_attachment_extraction=config.is_attachment_extraction_active,
+        min_base64_embedded_attachment_size=config.min_base64_embedded_attachment_size,
+        max_queue_size=max_queue_size,
+        message_processor=message_processor,
+        url_override=config.url_override,
+        fallback_replay_manager=fallback_replay,
+    )
+
+    return SharedConnectionResourcesBundle(
+        httpx_client=httpx_client_,
+        rest_client=rest_client,
+        message_processor=message_processor,
+        file_upload_manager=file_uploader,
+        replay_manager=fallback_replay,
+        streamer=streamer_,
+        flush_timeout=config.default_flush_timeout,
+    )
+
+
+# Opaque, hashable connection identity produced by ``_connection_key``.
+ConnectionKey = Tuple[str, bool]
+
+
+def _connection_key(
+    config: opik_config.OpikConfig, *, use_batching: bool
+) -> ConnectionKey:
+    # The whole config defines a connection's identity: any differing field
+    # yields a different bundle. Hashing the serialized config keeps the key
+    # compact and, by construction, never holds the api_key (or any field) in
+    # plaintext.
+    #
+    # Note this means clients that differ only by per-handle settings (e.g. a
+    # different default ``project_name`` or ``default_flush_timeout``) get
+    # separate bundles. That is safe — project is carried per trace and the flush
+    # timeout is a per-``end()`` argument — but to share one connection across
+    # projects, use a single client and pass ``project_name`` per call.
+    fingerprint = json.dumps(config.model_dump(mode="json"), sort_keys=True)
+    digest = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()
+    return (digest, use_batching)
+
+
+class Lease:
+    """Per-handle, release-once token over a bundle.
+
+    Each :class:`opik.Opik` handle holds its own lease. It carries the bundle so
+    the handle can delegate without re-looking it up, and guards a single
+    ``release`` so an explicit ``end()`` followed by the GC finalizer cannot
+    release twice. All lifecycle *decisions* (refcount, teardown) live on the
+    manager — the lease only forwards.
+    """
+
+    def __init__(
+        self,
+        manager: "ConnectionResourceManager",
+        key: ConnectionKey,
+        resources: SharedConnectionResourcesBundle,
+    ) -> None:
+        self._manager = manager
+        self._key = key
+        self.resources = resources
+        self._released = False
+        self._once_lock = threading.Lock()
+
+    def release(
+        self, timeout: Optional[int], *, flush: bool = True, close_on_zero: bool
+    ) -> None:
+        with self._once_lock:
+            if self._released:
+                return
+            self._released = True
+        self._manager.release(
+            self._key, timeout, flush=flush, close_on_zero=close_on_zero
+        )
+
+
+class _Entry:
+    def __init__(
+        self, resources: SharedConnectionResourcesBundle, refcount: int
+    ) -> None:
+        self.resources = resources
+        self.refcount = refcount
+
+
+class ConnectionResourceManager:
+    """Single owner of the shared connection-resource lifecycle.
+
+    Derives the connection identity from a config, builds-or-reuses a bundle
+    ref-counted by that identity, and tears a bundle down only when its last
+    lease is released *explicitly* (``Opik.end()``) — always after evicting it
+    under the lock, so a concurrent ``acquire`` never receives a closing bundle.
+    A reference dropped by a GC finalizer (``close_on_zero=False``) only
+    decrements the count and leaves the bundle cached; closing is never done in
+    garbage collection. Whatever survives to process exit is disposed by
+    ``close_all``. Disposal mechanics are delegated to the bundle's ``close``;
+    this class owns *when* it happens.
+    """
+
+    def __init__(
+        self,
+        builder: Callable[
+            ..., SharedConnectionResourcesBundle
+        ] = create_connection_resources,
+    ) -> None:
+        self._builder = builder
+        self._lock = threading.Lock()
+        self._entries: Dict[ConnectionKey, _Entry] = {}
+
+    def acquire(
+        self,
+        config: opik_config.OpikConfig,
+        *,
+        use_batching: bool,
+    ) -> Lease:
+        key = _connection_key(config, use_batching=use_batching)
+
+        # Fast path: an existing bundle is reused under the lock.
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is not None:
+                entry.refcount += 1
+                return Lease(manager=self, key=key, resources=entry.resources)
+
+        # No bundle yet — build outside the lock so a slow transport-stack
+        # construction does not serialize unrelated acquisitions.
+        bundle = self._builder(config, use_batching=use_batching)
+
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is None:
+                self._entries[key] = _Entry(resources=bundle, refcount=1)
+                return Lease(manager=self, key=key, resources=bundle)
+            # Lost the construction race: keep the bundle that won, take a
+            # reference on it, and drop ours below (outside the lock).
+            entry.refcount += 1
+            lease = Lease(manager=self, key=key, resources=entry.resources)
+
+        # Discard the bundle we lost the race with. A teardown failure here must
+        # not reject the caller — the winning lease is already valid — so log and
+        # move on.
+        try:
+            bundle.close(timeout=0, flush=False)
+        except Exception:
+            LOGGER.debug(
+                "Failed to close connection resources discarded after an acquire race",
+                exc_info=True,
+            )
+        return lease
+
+    def release(
+        self,
+        key: ConnectionKey,
+        timeout: Optional[int],
+        *,
+        flush: bool = True,
+        close_on_zero: bool,
+    ) -> None:
+        # Durability under sharing: an explicit ``end(flush=True)`` on a handle
+        # that still shares its bundle must drain the shared queue *before* this
+        # handle gives up its reference. Flushing while our reference is still
+        # counted keeps refcount >= 1, so a concurrent last-release cannot evict
+        # and ``close(flush=False)`` the bundle — which would clear the message
+        # queue out from under this flush and lose the data the ``flush=True``
+        # caller was promised. Pre-flush only when another handle also shares the
+        # bundle; a sole holder's ``close(flush=True)`` below already drains
+        # durably. A GC finalizer (``close_on_zero=False``) never does network
+        # I/O, so it never pre-flushes.
+        if flush and close_on_zero:
+            with self._lock:
+                entry = self._entries.get(key)
+                shared_bundle = (
+                    entry.resources
+                    if entry is not None and entry.refcount > 1
+                    else None
+                )
+            if shared_bundle is not None:
+                shared_bundle.flush(timeout)
+
+        # Now drop our reference. Because we only decrement here — after any
+        # pre-flush above has completed — a close can never run while another
+        # handle is mid pre-flush: that handle still holds its reference, so the
+        # count cannot reach zero until its flush returns.
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is None:
+                return
+            entry.refcount -= 1
+            if entry.refcount > 0:
+                return
+            if not close_on_zero:
+                # The last reference was dropped by a GC finalizer (see
+                # ``Opik._acquire_shared_resources``). Only the refcount
+                # decrement above is safe to run there; closing — streamer
+                # thread joins, file-upload pool shutdown, network flush — must
+                # never happen inside garbage collection. Leave the bundle
+                # cached so a later same-identity ``acquire`` reuses it, or
+                # ``close_all`` disposes it at process exit.
+                return
+            # Evict before close, under the lock, so a concurrent acquire never
+            # receives a bundle that is being torn down.
+            del self._entries[key]
+            bundle = entry.resources
+
+        bundle.close(timeout, flush=flush)
+
+    def close_all(self, *, flush: bool = True) -> None:
+        """Close and evict every cached bundle. Registered as the process
+        ``atexit`` hook (``flush=True``), where each bundle is drained within its
+        own connection's configured ``flush_timeout`` rather than unbounded;
+        ``flush=False`` resets the registry without network I/O."""
+        with self._lock:
+            entries = list(self._entries.values())
+            self._entries.clear()
+
+        for entry in entries:
+            try:
+                entry.resources.close(entry.resources.flush_timeout, flush=flush)
+            except Exception:
+                LOGGER.debug(
+                    "Failed to close shared connection resources",
+                    exc_info=True,
+                )
+
+    def active_connection_count(self) -> int:
+        """Number of live cached bundles. For tests and debugging."""
+        with self._lock:
+            return len(self._entries)
+
+    def reference_count(
+        self, config: opik_config.OpikConfig, *, use_batching: bool
+    ) -> int:
+        """Number of handles currently sharing ``config``'s bundle (0 if none)."""
+        key = _connection_key(config, use_batching=use_batching)
+        with self._lock:
+            entry = self._entries.get(key)
+            return 0 if entry is None else entry.refcount
+
+
+MANAGER = ConnectionResourceManager()
+atexit.register(MANAGER.close_all)

sdks/python/src/opik/api_objects/constants.py

@@ -0,0 +1,10 @@
+FEEDBACK_SCORE_SOURCE_SDK = "sdk"
+DATASET_SOURCE_SDK = "sdk"
+
+FEEDBACK_SCORES_MAX_BATCH_SIZE = 1000
+EXPERIMENT_ITEMS_MAX_BATCH_SIZE = 1000
+DATASET_ITEMS_MAX_BATCH_SIZE = 1000
+ANNOTATION_QUEUE_ITEMS_MAX_BATCH_SIZE = 1000
+DELETE_TRACE_BATCH_SIZE = 1000
+
+DATASET_STREAM_BATCH_SIZE = 2000

sdks/python/src/opik/api_objects/conversation/conversation_factory.py

@@ -0,0 +1,43 @@
+from typing import List, Callable
+
+from opik.rest_api import TracePublic, JsonListStringPublic
+from . import conversation_thread
+
+
+def create_conversation_from_traces(
+    traces: List[TracePublic],
+    input_transform: Callable[[JsonListStringPublic], str],
+    output_transform: Callable[[JsonListStringPublic], str],
+) -> conversation_thread.ConversationThread:
+    """
+    Creates a conversation object from given traces, transforming inputs and outputs using
+    provided transformation functions. The method processes each trace to compose a complete
+    conversation by consecutively adding user messages and assistant messages.
+
+    Args:
+        traces: A list of TracePublic objects representing trace data for user
+            and assistant interaction flows.
+        input_transform: A callable function that transforms the input data
+            from a JsonListStringPublic format to a string.
+        output_transform: A callable function that transforms the output data
+            from a JsonListStringPublic format to a string.
+
+    Returns:
+        A Conversation object that contains user and assistant message
+        sequences derived from the provided traces.
+    """
+    # Sort traces by start time to ensure they are processed in the correct order -
+    # the first user message should be first recorded
+    traces.sort(key=lambda trace_: trace_.start_time)
+
+    discussion = conversation_thread.ConversationThread()
+    for trace in traces:
+        trace_input = input_transform(trace.input)
+        if trace_input is not None:
+            discussion.add_user_message(trace_input)
+
+        trace_output = output_transform(trace.output)
+        if trace_output is not None:
+            discussion.add_assistant_message(trace_output)
+
+    return discussion

sdks/python/src/opik/api_objects/conversation/conversation_thread.py

@@ -0,0 +1,49 @@
+from typing import Dict, List
+
+import pydantic
+
+
+class ConversationThreadItem(pydantic.BaseModel):
+    """
+    Represents a single message within a conversation thread.
+
+    Each ConversationItem contains the role of the sender (e.g., 'user', 'assistant', 'system')
+    and the content of the message. This structured format allows for consistent representation
+    of messages across different conversation interfaces and evaluation systems.
+    """
+
+    role: str
+    content: str
+
+
+class ConversationThread(pydantic.BaseModel):
+    """
+    Represents a conversation thread composed of multiple conversation items.
+
+    This class is built using Pydantic's BaseModel to ensure type validation and data
+    integrity. It maintains a list of conversation items, where each item is an
+    instance of the `ConversationThreadItem` class. The conversation thread allows
+    adding messages from various roles, such as assistant, user, and system, and
+    provides the ability to export the conversation data as a JSON-serializable list.
+
+    Attributes:
+        discussion (List[ConversationThreadItem]): A list of conversation items
+            representing the dialogue between the roles.
+    """
+
+    discussion: List[ConversationThreadItem] = pydantic.Field(default_factory=list)
+
+    def add_item(self, item: ConversationThreadItem) -> None:
+        self.discussion.append(item)
+
+    def add_assistant_message(self, message: str) -> None:
+        self.add_item(ConversationThreadItem(role="assistant", content=message))
+
+    def add_user_message(self, message: str) -> None:
+        self.add_item(ConversationThreadItem(role="user", content=message))
+
+    def add_system_message(self, message: str) -> None:
+        self.add_item(ConversationThreadItem(role="system", content=message))
+
+    def as_json_list(self) -> List[Dict[str, str]]:
+        return [item.model_dump() for item in self.discussion]

sdks/python/src/opik/api_objects/dashboard/__init__.py

@@ -0,0 +1,43 @@
+from .dashboard import Dashboard
+from .types import (
+    BreakdownConfig,
+    BreakdownField,
+    ChartType,
+    DashboardLayoutItem,
+    DashboardSection,
+    DashboardState,
+    DashboardType,
+    DashboardWidget,
+    ExperimentLeaderboardConfig,
+    ExperimentsFeedbackScoresConfig,
+    ProjectMetricsConfig,
+    ProjectMetricType,
+    ProjectStatsCardConfig,
+    StatsCardMetric,
+    TextMarkdownConfig,
+    TraceDataType,
+    WidgetConfig,
+    WidgetType,
+)
+
+__all__ = [
+    "Dashboard",
+    "BreakdownConfig",
+    "BreakdownField",
+    "ChartType",
+    "DashboardLayoutItem",
+    "DashboardSection",
+    "DashboardState",
+    "DashboardType",
+    "DashboardWidget",
+    "ExperimentLeaderboardConfig",
+    "ExperimentsFeedbackScoresConfig",
+    "ProjectMetricsConfig",
+    "ProjectMetricType",
+    "ProjectStatsCardConfig",
+    "StatsCardMetric",
+    "TextMarkdownConfig",
+    "TraceDataType",
+    "WidgetConfig",
+    "WidgetType",
+]

sdks/python/src/opik/api_objects/dashboard/dashboard.py

@@ -0,0 +1,365 @@
+import copy
+import logging
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union
+
+from opik import exceptions, id_helpers
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.types import dashboard_public as rest_dashboard_public
+
+from . import layout, types, validation
+
+if TYPE_CHECKING:
+    from opik.api_objects import opik_client
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Dashboard:
+    """A high-level wrapper around an Opik dashboard.
+
+    Do not instantiate directly; use :meth:`opik.Opik.create_dashboard`,
+    :meth:`opik.Opik.get_dashboard`, or :meth:`opik.Opik.get_dashboards`.
+
+    The wrapper holds the raw ``config`` blob read from the backend as its source
+    of truth. Mutators edit that blob in place and PATCH the whole document back
+    (the backend replaces the column wholesale), so fields the SDK does not model
+    are preserved. Mutations are last-writer-wins: there is no optimistic
+    concurrency control, so concurrent edits to the same dashboard can clobber
+    each other. Call :meth:`reload` to re-sync before mutating if needed.
+    """
+
+    def __init__(
+        self,
+        dashboard_public: rest_dashboard_public.DashboardPublic,
+        rest_client: rest_api_client.OpikApi,
+        client: "opik_client.Opik",
+    ) -> None:
+        self._rest_client = rest_client
+        self.client = client
+        self._absorb(dashboard_public)
+
+    @classmethod
+    def from_public(
+        cls,
+        dashboard_public: rest_dashboard_public.DashboardPublic,
+        rest_client: rest_api_client.OpikApi,
+        client: "opik_client.Opik",
+    ) -> "Dashboard":
+        return cls(
+            dashboard_public=dashboard_public, rest_client=rest_client, client=client
+        )
+
+    def _absorb(self, dashboard_public: rest_dashboard_public.DashboardPublic) -> None:
+        if dashboard_public.id is None:
+            raise exceptions.DashboardValidationError(
+                "Backend returned a dashboard without an id"
+            )
+        self._id: str = dashboard_public.id
+        self._name = dashboard_public.name
+        self._description = dashboard_public.description
+        self._type = dashboard_public.type
+        self._scope = dashboard_public.scope
+        self._project_id: Optional[str] = dashboard_public.project_id
+        self._config: Dict[str, Any] = copy.deepcopy(dashboard_public.config or {})
+        self._created_at = dashboard_public.created_at
+        self._last_updated_at = dashboard_public.last_updated_at
+
+    @property
+    def id(self) -> str:
+        """Unique identifier of the dashboard."""
+        return self._id
+
+    @property
+    def name(self) -> str:
+        """Display name shown in the Opik UI."""
+        return self._name
+
+    @property
+    def description(self) -> Optional[str]:
+        """Optional free-text description of the dashboard's purpose."""
+        return self._description
+
+    @property
+    def type(self) -> Optional[str]:
+        """Dashboard type: ``"multi_project"`` for project-metric charts,
+        or ``"experiments"`` for experiment evaluation charts.
+
+        The type constrains which widget types are allowed (see
+        :class:`~opik.api_objects.dashboard.types.WidgetType`).
+        """
+        return self._type
+
+    @property
+    def project_id(self) -> Optional[str]:
+        """Project this dashboard is scoped to, or ``None`` for workspace-level dashboards.
+
+        Project-scoped widget types (``project_metrics``, ``project_stats_card``) can
+        only be added to dashboards that have a project. Pass ``project_name`` or
+        ``project_id`` to :meth:`opik.Opik.create_dashboard` to associate a project.
+        """
+        return self._project_id
+
+    @property
+    def scope(self) -> Optional[str]:
+        """Visibility scope set by the backend: ``"workspace"`` (shared with all
+        workspace members) or ``"insights"`` (platform-managed insight dashboard).
+        """
+        return self._scope
+
+    @property
+    def config(self) -> Dict[str, Any]:
+        """A deep copy of the raw JSON config blob persisted by the backend.
+
+        The schema is owned by the frontend; use :attr:`sections` for a typed view.
+        Direct mutation of this copy has no effect — use the mutator methods instead.
+        """
+        return copy.deepcopy(self._config)
+
+    @property
+    def state(self) -> types.DashboardState:
+        """The config parsed into a typed :class:`~opik.api_objects.dashboard.types.DashboardState`
+        (version, sections, lastModified). Returns a new object on every access.
+        """
+        return types.DashboardState.model_validate(self._config)
+
+    @property
+    def sections(self) -> List[types.DashboardSection]:
+        """Typed sections of the dashboard, each containing widgets and their grid layout.
+        Returns a new list on every access; mutations do not affect the dashboard — call
+        :meth:`replace_sections` to persist changes.
+        """
+        return self.state.sections
+
+    def add_widget(
+        self,
+        widget: Union[types.DashboardWidget, Dict[str, Any]],
+        *,
+        section_id: Optional[str] = None,
+        size: Optional[Dict[str, int]] = None,
+    ) -> str:
+        """Add a widget to a section and auto-place it on the grid.
+
+        The widget is positioned using the same algorithm as the Opik UI, appending
+        it to the first available slot that fits its default (or overridden) size.
+
+        Args:
+            widget: A :class:`~opik.api_objects.dashboard.types.DashboardWidget`
+                instance or a raw config dict.
+            section_id: ID of the section to add the widget to. Defaults to the
+                first section on the dashboard.
+            size: Optional ``{"w": int, "h": int}`` to override the widget's
+                default grid dimensions (columns × rows).
+
+        Returns:
+            The ID of the newly added widget.
+        """
+        self._assert_config_writable()
+        widget_dict = copy.deepcopy(validation.as_widget_dict(widget))
+        if not widget_dict.get("id"):
+            widget_dict["id"] = id_helpers.generate_id()
+        if size is not None and ("w" not in size or "h" not in size):
+            raise exceptions.DashboardValidationError(
+                "size must contain both 'w' and 'h' keys"
+            )
+        validation.validate_widget_for_dashboard(widget_dict, self._type)
+        validation.inject_project_id(widget_dict, self._project_id)
+
+        resolved_section_id = (
+            self._default_section_id() if section_id is None else section_id
+        )
+        with self._atomic_config():
+            section = self._get_section(resolved_section_id)
+            section.setdefault("widgets", []).append(widget_dict)
+            typed_layout = [
+                types.DashboardLayoutItem.model_validate(i)
+                for i in section.get("layout", [])
+            ]
+            section["layout"] = [
+                item.to_jsonable()
+                for item in layout.calculate_layout_for_adding_widget(
+                    typed_layout,
+                    widget_type=str(widget_dict["type"]),
+                    widget_id=str(widget_dict["id"]),
+                    size=size,
+                )
+            ]
+            self._commit_config()
+        return str(widget_dict["id"])
+
+    def remove_widget(self, widget_id: str) -> None:
+        """Remove a widget and its grid layout entry from whichever section holds it.
+
+        Raises :class:`~opik.exceptions.DashboardValidationError` if the widget ID
+        is not found.
+        """
+        self._assert_config_writable()
+        removed = False
+        for section in self._config.get("sections", []):
+            widgets = section.get("widgets", [])
+            kept = [w for w in widgets if w.get("id") != widget_id]
+            if len(kept) != len(widgets):
+                removed = True
+
+        if not removed:
+            raise exceptions.DashboardValidationError(
+                f"Widget with id {widget_id!r} not found in dashboard"
+            )
+
+        with self._atomic_config():
+            for section in self._config.get("sections", []):
+                widgets = section.get("widgets", [])
+                kept = [w for w in widgets if w.get("id") != widget_id]
+                if len(kept) != len(widgets):
+                    section["widgets"] = kept
+                    typed_layout = [
+                        types.DashboardLayoutItem.model_validate(i)
+                        for i in section.get("layout", [])
+                    ]
+                    section["layout"] = [
+                        item.to_jsonable()
+                        for item in layout.remove_widget_from_layout(
+                            typed_layout, widget_id
+                        )
+                    ]
+            self._commit_config()
+
+    def update_widget(
+        self,
+        widget_id: str,
+        *,
+        title: Optional[str] = None,
+        subtitle: Optional[str] = None,
+        config: Optional[Union[types.WidgetConfig, Dict[str, Any]]] = None,
+    ) -> None:
+        """Update a widget's display properties or configuration.
+
+        Only the supplied keyword arguments are changed; omitted ones are left
+        as-is. ``config`` is *merged* into the widget's existing config dict
+        (not replaced), so partial updates are safe.
+        """
+        self._assert_config_writable()
+        with self._atomic_config():
+            widget = self._find_widget(widget_id)
+
+            if title is not None:
+                widget["title"] = title
+            if subtitle is not None:
+                widget["subtitle"] = subtitle
+            if config is not None:
+                config_dict = (
+                    config.to_jsonable()
+                    if isinstance(config, types._DashboardModel)
+                    else config
+                )
+                if not isinstance(widget.get("config"), dict):
+                    widget["config"] = {}
+                widget["config"].update(config_dict)
+
+            validation.validate_widget_for_dashboard(widget, self._type)
+            self._commit_config()
+
+    def add_section(self, title: str) -> str:
+        """Append a new empty section to the dashboard and return its ID.
+
+        Sections group related widgets visually; each dashboard starts with one
+        default section created by the backend.
+        """
+        self._assert_config_writable()
+        section = types.DashboardSection(title=title).to_jsonable()
+        with self._atomic_config():
+            self._config.setdefault("sections", []).append(section)
+            self._commit_config()
+        return str(section["id"])
+
+    def replace_sections(
+        self,
+        sections: List[Union[types.DashboardSection, Dict[str, Any]]],
+    ) -> None:
+        """Replace all sections of the dashboard at once.
+
+        This is the primary way to reorder sections, move widgets between sections,
+        or adjust widget grid positions (x, y, w, h) — mutate the list returned by
+        :attr:`sections` and pass it back here.
+        """
+        self._assert_config_writable()
+        section_dicts = copy.deepcopy(validation.as_section_dicts(sections))
+        for section in section_dicts:
+            for widget in section.get("widgets", []):
+                validation.validate_widget_for_dashboard(widget, self._type)
+                validation.inject_project_id(widget, self._project_id)
+        with self._atomic_config():
+            self._config["sections"] = section_dicts
+            self._commit_config()
+
+    def rename(self, name: str) -> None:
+        """Change the dashboard's display name."""
+        response = self._rest_client.dashboards.update_dashboard(self._id, name=name)
+        self._absorb(response)
+
+    def set_description(self, description: str) -> None:
+        """Set the dashboard's free-text description."""
+        response = self._rest_client.dashboards.update_dashboard(
+            self._id, description=description
+        )
+        self._absorb(response)
+
+    def reload(self) -> None:
+        """Re-fetch the dashboard from the backend, replacing all local state.
+
+        Useful before a mutating operation when another client may have modified
+        the dashboard since it was last loaded (last-writer-wins semantics).
+        """
+        response = self._rest_client.dashboards.get_dashboard_by_id(self._id)
+        self._absorb(response)
+
+    def delete(self) -> None:
+        """Permanently delete the dashboard from the workspace."""
+        self._rest_client.dashboards.delete_dashboard(self._id)
+
+    @contextmanager
+    def _atomic_config(self) -> Generator[None, None, None]:
+        """Snapshot _config before a mutation; restore it if the commit fails."""
+        old_config = copy.deepcopy(self._config)
+        try:
+            yield
+        except Exception:
+            self._config = old_config
+            raise
+
+    def _assert_config_writable(self) -> None:
+        validation.validate_writable_version(self._config.get("version"))
+
+    def _commit_config(self) -> None:
+        self._config["version"] = types.DASHBOARD_VERSION
+        self._config["lastModified"] = types.now_ms()
+        validation.validate_structure(self._config)
+        response = self._rest_client.dashboards.update_dashboard(
+            self._id, config=self._config
+        )
+        self._absorb(response)
+
+    def _default_section_id(self) -> str:
+        sections = self._config.get("sections", [])
+        if not sections:
+            raise exceptions.DashboardValidationError(
+                "Dashboard has no sections. Add a section first with add_section()."
+            )
+        return str(sections[0]["id"])
+
+    def _get_section(self, section_id: str) -> Dict[str, Any]:
+        for section in self._config.get("sections", []):
+            if section.get("id") == section_id:
+                return section
+        raise exceptions.DashboardValidationError(
+            f"Section with id {section_id!r} not found in dashboard"
+        )
+
+    def _find_widget(self, widget_id: str) -> Dict[str, Any]:
+        for section in self._config.get("sections", []):
+            for widget in section.get("widgets", []):
+                if widget.get("id") == widget_id:
+                    return widget
+        raise exceptions.DashboardValidationError(
+            f"Widget with id {widget_id!r} not found in dashboard"
+        )

sdks/python/src/opik/api_objects/dashboard/layout.py

@@ -0,0 +1,152 @@
+"""Grid auto-layout, ported from apps/opik-frontend/src/lib/dashboard/layout.ts.
+
+Operates on :class:`~opik.api_objects.dashboard.types.DashboardLayoutItem` objects.
+Keeping the algorithm in sync with the frontend means widgets the SDK adds are
+positioned the same way the UI would position them.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from . import types
+from .types import DashboardLayoutItem
+
+GRID_COLUMNS = types.GRID_COLUMNS
+MAX_WIDGET_HEIGHT = types.MAX_WIDGET_HEIGHT
+MIN_WIDGET_WIDTH = types.MIN_WIDGET_WIDTH
+MIN_WIDGET_HEIGHT = types.MIN_WIDGET_HEIGHT
+
+_WIDGET_SIZE_CONFIG: Dict[str, Dict[str, int]] = {
+    types.WidgetType.PROJECT_METRICS.value: {"w": 2, "h": 4, "minW": 2, "minH": 4},
+    types.WidgetType.PROJECT_STATS_CARD.value: {"w": 1, "h": 2, "minW": 1, "minH": 2},
+    types.WidgetType.TEXT_MARKDOWN.value: {"w": 2, "h": 4, "minW": 1, "minH": 4},
+    types.WidgetType.EXPERIMENTS_FEEDBACK_SCORES.value: {
+        "w": 2,
+        "h": 4,
+        "minW": 2,
+        "minH": 4,
+    },
+    types.WidgetType.EXPERIMENT_LEADERBOARD.value: {
+        "w": 6,
+        "h": 6,
+        "minW": 4,
+        "minH": 4,
+    },
+}
+
+_DEFAULT_SIZE_CONFIG = {
+    "w": 2,
+    "h": 2,
+    "minW": MIN_WIDGET_WIDTH,
+    "minH": MIN_WIDGET_HEIGHT,
+}
+
+
+def get_widget_size_config(widget_type: str) -> Dict[str, int]:
+    return _WIDGET_SIZE_CONFIG.get(widget_type, _DEFAULT_SIZE_CONFIG)
+
+
+def get_column_heights(layout: List[DashboardLayoutItem]) -> List[int]:
+    heights = [0] * GRID_COLUMNS
+
+    for item in layout:
+        start_col = item.x
+        end_col = min(item.x + item.w, GRID_COLUMNS)
+        item_bottom = item.y + item.h
+
+        for col in range(start_col, end_col):
+            heights[col] = max(heights[col], item_bottom)
+
+    return heights
+
+
+def find_first_available_position(
+    w: int, h: int, column_heights: List[int]
+) -> Dict[str, int]:
+    # The frontend leaves w unclamped, which makes the search range empty (and y
+    # Infinity) for w > GRID_COLUMNS. Clamp here so the result is always a finite
+    # placement; real widget sizes never exceed the grid width.
+    w = min(w, GRID_COLUMNS)
+
+    min_height = float("inf")
+    best_x = 0
+
+    for x in range(0, GRID_COLUMNS - w + 1):
+        max_height_in_range = max(column_heights[x : x + w])
+        if max_height_in_range < min_height:
+            min_height = max_height_in_range
+            best_x = x
+
+    return {"x": best_x, "y": int(min_height)}
+
+
+def calculate_layout_for_adding_widget(
+    layout: List[DashboardLayoutItem],
+    widget_type: str,
+    widget_id: str,
+    size: Optional[Dict[str, int]] = None,
+) -> List[DashboardLayoutItem]:
+    size_config = get_widget_size_config(widget_type)
+    raw_w = size["w"] if size else size_config["w"]
+    raw_h = size["h"] if size else size_config["h"]
+    w = max(size_config["minW"], min(raw_w, GRID_COLUMNS))
+    h = max(size_config["minH"], min(raw_h, MAX_WIDGET_HEIGHT))
+
+    new_item = DashboardLayoutItem(
+        id=widget_id,
+        x=0,
+        y=0,
+        w=w,
+        h=h,
+        min_w=size_config["minW"],
+        min_h=size_config["minH"],
+        max_w=GRID_COLUMNS,
+        max_h=MAX_WIDGET_HEIGHT,
+    )
+
+    if not layout:
+        return [new_item]
+
+    column_heights = get_column_heights(layout)
+    position = find_first_available_position(w, h, column_heights)
+    new_item.x = position["x"]
+    new_item.y = position["y"]
+
+    return [*layout, new_item]
+
+
+def normalize_layout(
+    layout: List[DashboardLayoutItem],
+    widgets: Optional[List[Dict[str, Any]]] = None,
+) -> List[DashboardLayoutItem]:
+    widgets_by_id = {w["id"]: w for w in (widgets or [])}
+
+    normalized: List[DashboardLayoutItem] = []
+    for item in layout:
+        widget = widgets_by_id.get(item.id)
+        if widget is not None:
+            size_config = get_widget_size_config(str(widget["type"]))
+            min_w, min_h = size_config["minW"], size_config["minH"]
+        else:
+            min_w, min_h = MIN_WIDGET_WIDTH, MIN_WIDGET_HEIGHT
+
+        normalized.append(
+            DashboardLayoutItem(
+                id=item.id,
+                x=max(0, min(item.x, GRID_COLUMNS - item.w)),
+                y=max(0, item.y),
+                w=max(min_w, min(item.w, GRID_COLUMNS)),
+                h=max(min_h, min(item.h, MAX_WIDGET_HEIGHT)),
+                min_w=min_w,
+                min_h=min_h,
+                max_w=GRID_COLUMNS,
+                max_h=MAX_WIDGET_HEIGHT,
+            )
+        )
+
+    return normalized
+
+
+def remove_widget_from_layout(
+    layout: List[DashboardLayoutItem], widget_id: str
+) -> List[DashboardLayoutItem]:
+    return [item for item in layout if item.id != widget_id]

sdks/python/src/opik/api_objects/dashboard/rest_operations.py

@@ -0,0 +1,53 @@
+from typing import TYPE_CHECKING, List, Optional
+
+from opik.rest_api import client as rest_api_client
+
+from . import dashboard
+
+if TYPE_CHECKING:
+    from opik.api_objects import opik_client
+
+_PAGE_SIZE = 100
+
+
+def find_dashboards(
+    rest_client: rest_api_client.OpikApi,
+    client: "opik_client.Opik",
+    name: Optional[str] = None,
+    project_id: Optional[str] = None,
+    max_results: int = 100,
+    sorting: Optional[str] = None,
+    filters: Optional[str] = None,
+) -> List[dashboard.Dashboard]:
+    dashboards: List[dashboard.Dashboard] = []
+    page = 1
+
+    while len(dashboards) < max_results:
+        page_data = rest_client.dashboards.find_dashboards(
+            page=page,
+            size=_PAGE_SIZE,
+            name=name,
+            project_id=project_id,
+            sorting=sorting,
+            filters=filters,
+        )
+
+        content = page_data.content or []
+        if not content:
+            break
+
+        for dashboard_public in content[: max_results - len(dashboards)]:
+            dashboards.append(
+                dashboard.Dashboard.from_public(
+                    dashboard_public=dashboard_public,
+                    rest_client=rest_client,
+                    client=client,
+                )
+            )
+
+        if len(content) < _PAGE_SIZE:
+            break
+
+        page += 1
+
+    return dashboards

sdks/python/src/opik/api_objects/dashboard/types.py

@@ -0,0 +1,296 @@
+"""Typed models for the Opik dashboard ``config`` blob.
+
+The dashboard ``config`` is stored by the backend as an opaque JSON document; its
+real schema is owned by the frontend. These models mirror that schema (schema
+``version`` 4) and are used purely as builders/validators — the canonical
+representation carried by :class:`opik.api_objects.dashboard.dashboard.Dashboard`
+is always the raw ``dict`` read back from the backend, so fields the SDK does not
+model survive a read-modify-write round trip untouched.
+
+Source of truth, re-sync these when the frontend schema changes:
+  - apps/opik-frontend/src/types/dashboard.ts        (state/section/widget/layout shapes, enums)
+  - apps/opik-frontend/src/lib/dashboard/utils.ts     (DASHBOARD_VERSION, experiment limits, widget/type rules)
+  - apps/opik-frontend/src/lib/dashboard/layout.ts    (grid constants, per-widget sizes)
+  - apps/opik-frontend/src/api/projects/useProjectMetric.ts                         (ProjectMetricType wire values)
+  - apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/metrics.ts  (StatsCardMetric ids)
+
+Enums are permissive: known values are exposed for discoverability/autocomplete,
+but the model fields accept arbitrary strings so the SDK keeps working when the
+frontend adds new widget types, metrics, or fields.
+"""
+
+import enum
+import time
+from typing import Any, Dict, List, Optional, Union
+
+import pydantic
+from pydantic.alias_generators import to_camel
+
+from opik import exceptions, id_helpers
+
+# Mirrors apps/opik-frontend/src/lib/dashboard/utils.ts and layout.ts
+DASHBOARD_VERSION = 4
+GRID_COLUMNS = 6
+MAX_WIDGET_HEIGHT = 12
+MIN_WIDGET_WIDTH = 1
+MIN_WIDGET_HEIGHT = 1
+MIN_MAX_EXPERIMENTS = 1
+MAX_MAX_EXPERIMENTS = 100
+DEFAULT_MAX_EXPERIMENTS = 10
+
+FEEDBACK_SCORES_PREFIX = "feedback_scores."
+
+
+def now_ms() -> int:
+    """Current epoch time in milliseconds (frontend stores ``lastModified`` as ms)."""
+    return int(time.time() * 1000)
+
+
+class DashboardType(str, enum.Enum):
+    MULTI_PROJECT = "multi_project"
+    EXPERIMENTS = "experiments"
+
+
+class WidgetType(str, enum.Enum):
+    PROJECT_METRICS = "project_metrics"
+    PROJECT_STATS_CARD = "project_stats_card"
+    TEXT_MARKDOWN = "text_markdown"
+    EXPERIMENTS_FEEDBACK_SCORES = "experiments_feedback_scores"
+    EXPERIMENT_LEADERBOARD = "experiment_leaderboard"
+
+
+class BreakdownField(str, enum.Enum):
+    NONE = "none"
+    TAGS = "tags"
+    METADATA = "metadata"
+    NAME = "name"
+    ERROR_INFO = "error_info"
+    ERROR_TYPE = "error_type"
+    MODEL = "model"
+    PROVIDER = "provider"
+    TYPE = "type"
+
+
+class ChartType(str, enum.Enum):
+    LINE = "line"
+    BAR = "bar"
+    RADAR = "radar"
+
+
+class TraceDataType(str, enum.Enum):
+    TRACES = "traces"
+    SPANS = "spans"
+
+
+class ProjectMetricType(str, enum.Enum):
+    """ALL-CAPS metric ids used by the ``project_metrics`` widget ``metricType`` field."""
+
+    FEEDBACK_SCORES = "FEEDBACK_SCORES"
+    TRACE_COUNT = "TRACE_COUNT"
+    DURATION = "DURATION"
+    TOKEN_USAGE = "TOKEN_USAGE"
+    COST = "COST"
+    GUARDRAILS_FAILED_COUNT = "GUARDRAILS_FAILED_COUNT"
+    THREAD_COUNT = "THREAD_COUNT"
+    THREAD_DURATION = "THREAD_DURATION"
+    THREAD_FEEDBACK_SCORES = "THREAD_FEEDBACK_SCORES"
+    SPAN_COUNT = "SPAN_COUNT"
+    SPAN_DURATION = "SPAN_DURATION"
+    SPAN_FEEDBACK_SCORES = "SPAN_FEEDBACK_SCORES"
+    SPAN_TOKEN_USAGE = "SPAN_TOKEN_USAGE"
+    TRACE_AVERAGE_DURATION = "TRACE_AVERAGE_DURATION"
+    SPAN_AVERAGE_DURATION = "SPAN_AVERAGE_DURATION"
+    THREAD_AVERAGE_DURATION = "THREAD_AVERAGE_DURATION"
+    TRACE_ERROR_RATE = "TRACE_ERROR_RATE"
+    SPAN_ERROR_RATE = "SPAN_ERROR_RATE"
+
+
+class StatsCardMetric(str, enum.Enum):
+    """Lowercase-dotted metric ids used by the ``project_stats_card`` widget ``metric`` field.
+
+    Dynamic feedback-score metrics use the ``feedback_scores.<score_name>`` form and
+    are not enumerated here; any such string is accepted.
+    """
+
+    DURATION_P50 = "duration.p50"
+    DURATION_P90 = "duration.p90"
+    DURATION_P99 = "duration.p99"
+    INPUT = "input"
+    OUTPUT = "output"
+    METADATA = "metadata"
+    TAGS = "tags"
+    TOTAL_ESTIMATED_COST_SUM = "total_estimated_cost_sum"
+    USAGE_COMPLETION_TOKENS = "usage.completion_tokens"
+    USAGE_PROMPT_TOKENS = "usage.prompt_tokens"
+    USAGE_TOTAL_TOKENS = "usage.total_tokens"
+    ERROR_COUNT = "error_count"
+    TRACE_COUNT = "trace_count"
+    THREAD_COUNT = "thread_count"
+    LLM_SPAN_COUNT = "llm_span_count"
+    SPAN_COUNT = "span_count"
+    TOTAL_ESTIMATED_COST = "total_estimated_cost"
+    GUARDRAILS_FAILED_COUNT = "guardrails_failed_count"
+
+
+class _DashboardModel(pydantic.BaseModel):
+    """Base for every dashboard model: snake_case in Python, camelCase on the wire."""
+
+    model_config = pydantic.ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        extra="allow",
+    )
+
+    def to_jsonable(self) -> Dict[str, Any]:
+        """Serialize to a pure-JSON dict with camelCase keys (drops unset optionals)."""
+        return self.model_dump(by_alias=True, exclude_none=True, mode="json")
+
+
+class BreakdownConfig(_DashboardModel):
+    field: str
+    metadata_key: Optional[str] = None
+    sub_metric: Optional[str] = None
+    aggregate_total: Optional[bool] = None
+
+    @pydantic.model_validator(mode="after")
+    def _check_metadata_key(self) -> "BreakdownConfig":
+        if self.field == BreakdownField.METADATA and not self.metadata_key:
+            raise exceptions.DashboardValidationError(
+                "breakdown.metadata_key is required when breakdown.field is 'metadata'"
+            )
+        return self
+
+
+class ProjectMetricsConfig(_DashboardModel):
+    metric_type: str = ProjectMetricType.TRACE_COUNT.value
+    chart_type: Optional[str] = ChartType.LINE.value
+    trace_filters: Optional[List[Dict[str, Any]]] = None
+    thread_filters: Optional[List[Dict[str, Any]]] = None
+    span_filters: Optional[List[Dict[str, Any]]] = None
+    feedback_scores: Optional[List[str]] = None
+    duration_metrics: Optional[List[str]] = None
+    usage_metrics: Optional[List[str]] = None
+    breakdown: Optional[BreakdownConfig] = None
+
+
+class ProjectStatsCardConfig(_DashboardModel):
+    source: str = TraceDataType.TRACES.value
+    metric: str = StatsCardMetric.TRACE_COUNT.value
+    trace_filters: Optional[List[Dict[str, Any]]] = None
+    span_filters: Optional[List[Dict[str, Any]]] = None
+
+
+class TextMarkdownConfig(_DashboardModel):
+    content: Optional[str] = None
+
+
+class ExperimentsFeedbackScoresConfig(_DashboardModel):
+    filters: Optional[List[Dict[str, Any]]] = None
+    groups: Optional[List[Dict[str, Any]]] = None
+    chart_type: Optional[str] = ChartType.BAR.value
+    feedback_scores: Optional[List[str]] = None
+    max_experiments_count: Optional[Union[int, str]] = DEFAULT_MAX_EXPERIMENTS
+
+    @pydantic.model_validator(mode="after")
+    def _check_max_experiments(self) -> "ExperimentsFeedbackScoresConfig":
+        _check_experiment_range(self.max_experiments_count, "max_experiments_count")
+        return self
+
+
+class ExperimentLeaderboardConfig(_DashboardModel):
+    selected_columns: List[str] = pydantic.Field(
+        default_factory=lambda: [
+            "dataset_id",
+            "created_at",
+            "duration.p50",
+            "pass_rate",
+        ]
+    )
+    enable_ranking: bool = False
+    filters: Optional[List[Dict[str, Any]]] = None
+    ranking_metric: Optional[str] = None
+    ranking_direction: Optional[bool] = None
+    columns_order: Optional[List[str]] = None
+    scores_columns_order: Optional[List[str]] = None
+    metadata_columns_order: Optional[List[str]] = None
+    columns_width: Optional[Dict[str, int]] = None
+    # Frontend stores this as either an int (default config) or a string (after editing).
+    max_rows: Optional[Union[int, str]] = None
+    sorting: Optional[List[Dict[str, Any]]] = None
+
+    @pydantic.model_validator(mode="after")
+    def _check_ranking_and_rows(self) -> "ExperimentLeaderboardConfig":
+        if self.enable_ranking and not self.ranking_metric:
+            raise exceptions.DashboardValidationError(
+                "ranking_metric is required when enable_ranking is True"
+            )
+        if self.max_rows is not None:
+            _check_experiment_range(self.max_rows, "max_rows")
+        return self
+
+
+def _check_experiment_range(value: Union[int, str, None], field_name: str) -> None:
+    if value is None:
+        return
+    try:
+        numeric = int(value)
+    except (TypeError, ValueError):
+        raise exceptions.DashboardValidationError(
+            f"{field_name} must be an integer between {MIN_MAX_EXPERIMENTS} and {MAX_MAX_EXPERIMENTS}"
+        )
+    if not MIN_MAX_EXPERIMENTS <= numeric <= MAX_MAX_EXPERIMENTS:
+        raise exceptions.DashboardValidationError(
+            f"{field_name} must be between {MIN_MAX_EXPERIMENTS} and {MAX_MAX_EXPERIMENTS}, got {numeric}"
+        )
+
+
+WidgetConfig = Union[
+    ProjectMetricsConfig,
+    ProjectStatsCardConfig,
+    TextMarkdownConfig,
+    ExperimentsFeedbackScoresConfig,
+    ExperimentLeaderboardConfig,
+]
+
+
+class DashboardWidget(_DashboardModel):
+    type: str
+    id: str = pydantic.Field(default_factory=id_helpers.generate_id)
+    title: str = ""
+    generated_title: Optional[str] = None
+    subtitle: Optional[str] = None
+    config: Dict[str, Any] = pydantic.Field(default_factory=dict)
+
+    @pydantic.field_validator("config", mode="before")
+    @classmethod
+    def _normalize_config(cls, value: Any) -> Dict[str, Any]:
+        if isinstance(value, _DashboardModel):
+            return value.to_jsonable()
+        if value is None:
+            return {}
+        return value
+
+
+class DashboardLayoutItem(_DashboardModel):
+    id: str = pydantic.Field(alias="i")
+    x: int
+    y: int
+    w: int
+    h: int
+    min_w: Optional[int] = None
+    max_w: Optional[int] = None
+    min_h: Optional[int] = None
+    max_h: Optional[int] = None
+
+
+class DashboardSection(_DashboardModel):
+    title: str
+    id: str = pydantic.Field(default_factory=id_helpers.generate_id)
+    widgets: List[DashboardWidget] = pydantic.Field(default_factory=list)
+    layout: List[DashboardLayoutItem] = pydantic.Field(default_factory=list)
+
+
+class DashboardState(_DashboardModel):
+    version: int = DASHBOARD_VERSION
+    sections: List[DashboardSection] = pydantic.Field(default_factory=list)
+    last_modified: int = pydantic.Field(default_factory=now_ms)

sdks/python/src/opik/api_objects/dashboard/validation.py

@@ -0,0 +1,177 @@
+"""Dashboard validation invariants.
+
+Structural invariants (always enforced, on every write) keep the config in a
+state the frontend can render. Semantic checks (enforced only when constructing
+or mutating a widget) mirror the frontend's product rules. Reads never validate —
+the SDK must always be able to load dashboards created by a newer frontend.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from opik import exceptions
+
+from . import types
+
+LOGGER = logging.getLogger(__name__)
+
+_WIDGET_TYPES_BY_DASHBOARD_TYPE = {
+    types.DashboardType.MULTI_PROJECT.value: {
+        types.WidgetType.PROJECT_METRICS.value,
+        types.WidgetType.PROJECT_STATS_CARD.value,
+        types.WidgetType.TEXT_MARKDOWN.value,
+    },
+    types.DashboardType.EXPERIMENTS.value: {
+        types.WidgetType.EXPERIMENTS_FEEDBACK_SCORES.value,
+        types.WidgetType.EXPERIMENT_LEADERBOARD.value,
+        types.WidgetType.TEXT_MARKDOWN.value,
+    },
+}
+
+_PROJECT_METRIC_TYPES = {m.value for m in types.ProjectMetricType}
+_STATS_CARD_METRICS = {m.value for m in types.StatsCardMetric}
+_PROJECT_SCOPED_WIDGET_TYPES = {
+    types.WidgetType.PROJECT_METRICS.value,
+    types.WidgetType.PROJECT_STATS_CARD.value,
+}
+
+
+def inject_project_id(widget_dict: Dict[str, Any], project_id: Optional[str]) -> None:
+    """Inject projectId into a project-scoped widget config.
+
+    Raises DashboardValidationError if the widget is project-scoped but the dashboard
+    has no project_id — callers should ensure the dashboard was created with a project.
+    """
+    if widget_dict.get("type") not in _PROJECT_SCOPED_WIDGET_TYPES:
+        return
+    if not project_id:
+        raise exceptions.DashboardValidationError(
+            f"Widget type {widget_dict.get('type')!r} requires a project-scoped dashboard. "
+            "Pass project_name or project_id to create_dashboard."
+        )
+    widget_dict.setdefault("config", {})["projectId"] = project_id
+
+
+def validate_structure(state: Dict[str, Any]) -> None:
+    """Enforce widget/layout cross-references and id uniqueness across the dashboard."""
+    seen_section_ids: set = set()
+    seen_widget_ids: set = set()
+
+    for section in state.get("sections", []):
+        section_id = section.get("id")
+        if section_id in seen_section_ids:
+            raise exceptions.DashboardValidationError(
+                f"Duplicate section id: {section_id!r}"
+            )
+        seen_section_ids.add(section_id)
+
+        widget_ids = []
+        for widget in section.get("widgets", []):
+            widget_id = widget.get("id")
+            if widget_id in seen_widget_ids:
+                raise exceptions.DashboardValidationError(
+                    f"Duplicate widget id: {widget_id!r}"
+                )
+            seen_widget_ids.add(widget_id)
+            widget_ids.append(widget_id)
+
+        widget_id_set = set(widget_ids)
+        layout_ids = [item.get("i") for item in section.get("layout", [])]
+        layout_id_set = set(layout_ids)
+
+        missing_layout = widget_id_set - layout_id_set
+        if missing_layout:
+            raise exceptions.DashboardValidationError(
+                f"Widgets without a layout item in section {section_id!r}: {missing_layout}"
+            )
+
+        orphan_layout = layout_id_set - widget_id_set
+        if orphan_layout:
+            raise exceptions.DashboardValidationError(
+                f"Layout items referencing missing widgets in section {section_id!r}: {orphan_layout}"
+            )
+
+
+def validate_widget_for_dashboard(
+    widget: Dict[str, Any], dashboard_type: Optional[str]
+) -> None:
+    """Construct-time semantic checks for a single widget being added/updated."""
+    widget_type = widget.get("type")
+
+    if dashboard_type is not None:
+        allowed = _WIDGET_TYPES_BY_DASHBOARD_TYPE.get(dashboard_type)
+        if allowed is not None and widget_type not in allowed:
+            raise exceptions.DashboardValidationError(
+                f"Widget type {widget_type!r} is not supported on a {dashboard_type!r} "
+                f"dashboard. Allowed types: {sorted(allowed)}"
+            )
+
+    _warn_on_unknown_metric(widget_type, widget.get("config", {}))
+
+
+def _warn_on_unknown_metric(widget_type: Optional[str], config: Dict[str, Any]) -> None:
+    if widget_type == types.WidgetType.PROJECT_METRICS.value:
+        metric = config.get("metricType")
+        if metric is not None and metric not in _PROJECT_METRIC_TYPES:
+            LOGGER.warning(
+                "Unknown project_metrics metricType %r. Expected one of the ALL-CAPS "
+                "ids in opik.dashboard.ProjectMetricType (e.g. 'TRACE_COUNT', 'DURATION').",
+                metric,
+            )
+    elif widget_type == types.WidgetType.PROJECT_STATS_CARD.value:
+        metric = config.get("metric")
+        if (
+            metric is not None
+            and metric not in _STATS_CARD_METRICS
+            and not str(metric).startswith(types.FEEDBACK_SCORES_PREFIX)
+        ):
+            LOGGER.warning(
+                "Unknown project_stats_card metric %r. Expected one of the "
+                "lowercase-dotted ids in opik.dashboard.StatsCardMetric (e.g. "
+                "'trace_count', 'duration.p50') or a 'feedback_scores.<name>' id.",
+                metric,
+            )
+
+
+def validate_writable_version(version: Optional[int]) -> None:
+    """Refuse to write back a config whose schema version the SDK does not know.
+
+    The SDK does not run the frontend migration chain, so re-stamping a config of
+    an unknown version risks silently corrupting it.
+    """
+    if version is not None and version != types.DASHBOARD_VERSION:
+        raise exceptions.DashboardValidationError(
+            f"Refusing to write a dashboard with schema version {version}; this SDK "
+            f"only understands version {types.DASHBOARD_VERSION}. Upgrade the Opik SDK "
+            f"to modify this dashboard."
+        )
+
+
+def as_widget_dict(
+    widget: Union[types.DashboardWidget, Dict[str, Any]],
+) -> Dict[str, Any]:
+    """Coerce a DashboardWidget model or a raw dict into a plain config dict."""
+    if isinstance(widget, types.DashboardWidget):
+        return widget.to_jsonable()
+    if isinstance(widget, dict):
+        return widget
+    raise exceptions.DashboardValidationError(
+        f"Expected a DashboardWidget or dict, got {type(widget).__name__}"
+    )
+
+
+def as_section_dicts(
+    sections: List[Union[types.DashboardSection, Dict[str, Any]]],
+) -> List[Dict[str, Any]]:
+    """Coerce a list of DashboardSection models or raw dicts into plain dicts."""
+    result: List[Dict[str, Any]] = []
+    for section in sections:
+        if isinstance(section, types.DashboardSection):
+            result.append(section.to_jsonable())
+        elif isinstance(section, dict):
+            result.append(section)
+        else:
+            raise exceptions.DashboardValidationError(
+                f"Expected a DashboardSection or dict, got {type(section).__name__}"
+            )
+    return result

sdks/python/src/opik/api_objects/data_helpers.py

@@ -0,0 +1,79 @@
+from typing import Optional, Any, Dict, List, Union
+
+import pydantic
+
+from opik import dict_utils
+
+
+def merge_tags(
+    existing_tags: Optional[List[str]], new_tags: Optional[List[str]]
+) -> Optional[List[str]]:
+    """Merge tag lists, preserving existing tags and adding new ones.
+
+    If both existing_tags and new_tags are None or empty, return None."""
+    if existing_tags is None and new_tags is None:
+        return None
+
+    result = list(existing_tags or [])
+    if new_tags:
+        for tag in new_tags:
+            if tag not in result:
+                result.append(tag)
+
+    return result if result else None
+
+
+def merge_metadata(
+    existing_metadata: Optional[Dict[str, Any]],
+    new_metadata: Optional[Union[Dict[str, Any], pydantic.BaseModel]],
+    prompts: Optional[List[Dict[str, Any]]] = None,
+) -> Optional[Dict[str, Any]]:
+    """Merge the existing metadata dictionary with new data, with new values taking precedence.
+
+    If both existing_metadata and new_metadata are None or empty, return None.
+    """
+    if prompts is not None:
+        new_metadata = new_metadata or {}
+        new_metadata["opik_prompts"] = prompts
+
+    return _merge_dictionary_with_data(existing_metadata, new_data=new_metadata)
+
+
+def merge_inputs(
+    existing_inputs: Optional[Dict[str, Any]],
+    new_inputs: Optional[Union[Dict[str, Any], pydantic.BaseModel]],
+) -> Optional[Dict[str, Any]]:
+    """Merge the existing input dictionary with new data, with new values taking precedence.
+
+    If both existing_inputs and new_inputs are None or empty, return None."""
+    return _merge_dictionary_with_data(existing_inputs, new_data=new_inputs)
+
+
+def merge_outputs(
+    existing_outputs: Optional[Dict[str, Any]],
+    new_outputs: Optional[Union[Dict[str, Any], pydantic.BaseModel]],
+) -> Optional[Dict[str, Any]]:
+    """Merge the existing output dictionary with new data, with new values taking precedence.
+
+    If both existing_outputs and new_outputs are None or empty, return None."""
+    return _merge_dictionary_with_data(existing_outputs, new_data=new_outputs)
+
+
+def _merge_dictionary_with_data(
+    existing_dict: Optional[Dict[str, Any]],
+    new_data: Optional[Union[Dict[str, Any], pydantic.BaseModel]],
+) -> Optional[Dict[str, Any]]:
+    """Merge the dictionary with new data, with new values taking precedence.
+
+    If both existing_dict and new_data are None or empty, return None."""
+    if existing_dict is None and new_data is None:
+        return None
+
+    if isinstance(new_data, pydantic.BaseModel):
+        new_data = new_data.model_dump()
+
+    result = dict(existing_dict or {})
+    if new_data:
+        result = dict_utils.deepmerge(result, new_data)
+
+    return result if result else None

sdks/python/src/opik/api_objects/dataset/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import Dataset, DatasetVersion
+
+
+__all__ = ["Dataset", "DatasetVersion"]

sdks/python/src/opik/api_objects/dataset/converters.py

@@ -0,0 +1,99 @@
+import json
+
+from typing import List, Callable, Any, Dict, TYPE_CHECKING
+import logging
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from . import dataset_item
+from . import helpers
+
+ItemConstructor = Callable[[Any], dataset_item.DatasetItem]
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def to_pandas(
+    items: List[dataset_item.DatasetItem], keys_mapping: Dict[str, str]
+) -> "pd.DataFrame":
+    helpers.raise_if_pandas_is_unavailable()
+
+    import pandas as pd
+
+    new_item_dicts = []
+
+    for item in items:
+        item_content = item.get_content(include_id=True)
+        new_item_dict = {
+            keys_mapping.get(key, key): value for key, value in item_content.items()
+        }
+        new_item_dicts.append(new_item_dict)
+
+    return pd.DataFrame(new_item_dicts)
+
+
+def from_jsonl_file(
+    file_path: str, keys_mapping: Dict[str, str], ignore_keys: List[str]
+) -> List[dataset_item.DatasetItem]:
+    items = []
+    with open(file_path, "r", encoding="utf-8") as file:
+        for line in file:
+            json_object = line.strip()
+            if json_object:  # Skip empty lines
+                items.append(json.loads(json_object))
+
+    json_str = json.dumps(items)
+    return from_json(json_str, keys_mapping, ignore_keys)
+
+
+def from_pandas(
+    dataframe: "pd.DataFrame",
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[dataset_item.DatasetItem]:
+    helpers.raise_if_pandas_is_unavailable()
+
+    result = []
+    ignore_keys = [] if ignore_keys is None else ignore_keys
+    for _, row in dataframe.iterrows():
+        item_kwargs = {
+            keys_mapping.get(key, key): value
+            for key, value in row.items()
+            if key not in ignore_keys
+        }
+        result.append(dataset_item.DatasetItem(**item_kwargs))
+
+    return result
+
+
+def to_json(items: List[dataset_item.DatasetItem], keys_mapping: Dict[str, str]) -> str:
+    new_item_dicts = []
+
+    for item in items:
+        item_content = item.get_content(include_id=True)
+        new_item_dict = {
+            keys_mapping.get(key, key): value for key, value in item_content.items()
+        }
+        new_item_dicts.append(new_item_dict)
+
+    result: str = json.dumps(new_item_dicts, indent=2)
+    return result
+
+
+def from_json(
+    value: str, keys_mapping: Dict[str, str], ignore_keys: List[str]
+) -> List[dataset_item.DatasetItem]:
+    result = []
+    item_dicts: List[Dict[str, Any]] = json.loads(value)
+
+    for item_dict in item_dicts:
+        item_kwargs = {
+            keys_mapping.get(key, key): value
+            for key, value in item_dict.items()
+            if key not in ignore_keys
+        }
+        result.append(dataset_item.DatasetItem(**item_kwargs))
+
+    return result

sdks/python/src/opik/api_objects/dataset/dataset.py

@@ -0,0 +1,923 @@
+import abc
+import datetime
+import logging
+import functools
+import sys
+from typing import (
+    Optional,
+    Any,
+    List,
+    Dict,
+    Sequence,
+    Set,
+    TYPE_CHECKING,
+    Iterator,
+)
+
+from opik.api_objects import rest_helpers
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.core.api_error import ApiError
+from opik.rest_api.types import (
+    dataset_item_write as rest_dataset_item,
+    dataset_public as rest_dataset_public,
+    dataset_version_public,
+    evaluator_item_write as rest_evaluator_item,
+    execution_policy_write as rest_execution_policy,
+)
+from opik.message_processing.batching import sequence_splitter
+from opik import id_helpers
+import opik.exceptions as exceptions
+import opik.config as config
+from .. import constants
+from . import dataset_item, converters, rest_operations, execution_policy
+
+if sys.version_info >= (3, 12):
+    from typing import override
+else:
+    from typing_extensions import override
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DatasetExportOperations(abc.ABC):
+    """
+    Abstract base class providing export operations for dataset items.
+
+    This class defines the common interface for exporting dataset items,
+    shared by both Dataset (current state) and DatasetVersion (specific version).
+    """
+
+    @abc.abstractmethod
+    def __internal_api__stream_items_as_dataclasses__(
+        self,
+        nb_samples: Optional[int] = None,
+        batch_size: Optional[int] = None,
+        dataset_item_ids: Optional[List[str]] = None,
+        filter_string: Optional[str] = None,
+    ) -> Iterator[dataset_item.DatasetItem]:
+        """
+        Stream dataset items as DatasetItem objects.
+
+        Args:
+            nb_samples: Maximum number of items to retrieve.
+            batch_size: Maximum number of items to fetch per batch.
+            dataset_item_ids: Optional list of specific item IDs to retrieve.
+            filter_string: Optional OQL filter string to filter dataset items.
+
+        Yields:
+            DatasetItem objects one at a time.
+        """
+        raise NotImplementedError
+
+    def to_pandas(self) -> "pd.DataFrame":
+        """
+        Convert the dataset items to a pandas DataFrame.
+
+        Requires the `pandas` library to be installed.
+
+        Returns:
+            A pandas DataFrame containing all items.
+        """
+        dataset_items = list(self.__internal_api__stream_items_as_dataclasses__())
+        return converters.to_pandas(dataset_items, keys_mapping={})
+
+    def to_json(self) -> str:
+        """
+        Convert the dataset items to a JSON string.
+
+        Returns:
+            A JSON string representation of all items.
+        """
+        dataset_items = list(self.__internal_api__stream_items_as_dataclasses__())
+        return converters.to_json(dataset_items, keys_mapping={})
+
+    def get_items(
+        self,
+        nb_samples: Optional[int] = None,
+        filter_string: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Retrieve dataset items as a list of dictionaries.
+
+        Args:
+            nb_samples: Maximum number of items to retrieve. If not set, all items are returned.
+            filter_string: Optional OQL filter string to filter dataset items.
+                Supports filtering by tags, data fields, metadata, etc.
+
+                Supported columns include:
+                - `id`, `source`, `trace_id`, `span_id`: String fields
+                - `data`: Dictionary field (use dot notation, e.g., "data.category")
+                - `tags`: List field (use "contains" operator)
+                - `created_at`, `last_updated_at`: DateTime fields (ISO 8601 format)
+                - `created_by`, `last_updated_by`: String fields
+
+                Examples:
+                - `tags contains "failed"` - Items with 'failed' tag
+                - `data.category = "test"` - Items with specific data field value
+                - `created_at >= "2024-01-01T00:00:00Z"` - Items created after date
+
+        Returns:
+            A list of dictionaries representing the dataset items.
+        """
+        dataset_items_as_dicts = [
+            {"id": item.id, **item.get_content()}
+            for item in self.__internal_api__stream_items_as_dataclasses__(
+                nb_samples=nb_samples, filter_string=filter_string
+            )
+        ]
+        return dataset_items_as_dicts
+
+    @abc.abstractmethod
+    def get_version_info(
+        self,
+    ) -> Optional[dataset_version_public.DatasetVersionPublic]:
+        """
+        Get version information for experiment association.
+
+        Returns:
+            DatasetVersionPublic containing version metadata (id, version_name, etc.).
+            For Dataset, returns info about the current/latest version, or None if no version exists.
+            For DatasetVersion, returns info about this specific version.
+        """
+        raise NotImplementedError
+
+
+class DatasetVersion(DatasetExportOperations):
+    """
+    A read-only view of a specific dataset version.
+
+    This class provides access to dataset items at a specific version point in time.
+    It supports reading version metadata and retrieving items, but does not allow
+    mutations to the dataset.
+
+    This object should not be created directly. Use :meth:`Dataset.get_dataset_version`
+    to obtain an instance.
+    """
+
+    def __init__(
+        self,
+        dataset_name: str,
+        dataset_id: str,
+        rest_client: rest_api_client.OpikApi,
+        version_info: dataset_version_public.DatasetVersionPublic,
+        project_name: Optional[str],
+        client: Optional[Any] = None,
+    ) -> None:
+        self._dataset_name = dataset_name
+        self._dataset_id = dataset_id
+        self._rest_client = rest_client
+        self._version_info = version_info
+        self._project_name = project_name
+        self.client = client
+
+    @property
+    def dataset_name(self) -> str:
+        """The name of the dataset this version belongs to."""
+        return self._dataset_name
+
+    @property
+    def project_name(self) -> Optional[str]:
+        """The name of the project this dataset belongs to."""
+        return self._project_name
+
+    @property
+    def name(self) -> str:
+        """The name of the dataset this version belongs to (alias for dataset_name)."""
+        return self._dataset_name
+
+    @property
+    def dataset_id(self) -> str:
+        """The unique identifier of the dataset this version belongs to."""
+        return self._dataset_id
+
+    @property
+    def id(self) -> str:
+        """The unique identifier of the dataset this version belongs to (alias for dataset_id)."""
+        return self._dataset_id
+
+    @property
+    def version_id(self) -> Optional[str]:
+        """The unique identifier of this specific version."""
+        return self._version_info.id
+
+    @property
+    def dataset_items_count(self) -> Optional[int]:
+        """Total number of items in this version (alias for items_total)."""
+        return self._version_info.items_total
+
+    @property
+    def version_hash(self) -> Optional[str]:
+        """The unique hash identifier of this version."""
+        return self._version_info.version_hash
+
+    @property
+    def version_name(self) -> Optional[str]:
+        """The sequential version name (e.g., 'v1', 'v2')."""
+        return self._version_info.version_name
+
+    @property
+    def tags(self) -> Optional[List[str]]:
+        """Tags associated with this version."""
+        return self._version_info.tags
+
+    @property
+    def is_latest(self) -> Optional[bool]:
+        """Whether this is the latest version of the dataset."""
+        return self._version_info.is_latest
+
+    @property
+    def items_total(self) -> Optional[int]:
+        """Total number of items in this version."""
+        return self._version_info.items_total
+
+    @property
+    def items_added(self) -> Optional[int]:
+        """Number of items added since the previous version."""
+        return self._version_info.items_added
+
+    @property
+    def items_modified(self) -> Optional[int]:
+        """Number of items modified since the previous version."""
+        return self._version_info.items_modified
+
+    @property
+    def items_deleted(self) -> Optional[int]:
+        """Number of items deleted since the previous version."""
+        return self._version_info.items_deleted
+
+    @property
+    def change_description(self) -> Optional[str]:
+        """Description of changes in this version."""
+        return self._version_info.change_description
+
+    @property
+    def created_at(self) -> Optional[datetime.datetime]:
+        """Timestamp when this version was created."""
+        return self._version_info.created_at
+
+    @property
+    def created_by(self) -> Optional[str]:
+        """User who created this version."""
+        return self._version_info.created_by
+
+    @override
+    def __internal_api__stream_items_as_dataclasses__(
+        self,
+        nb_samples: Optional[int] = None,
+        batch_size: Optional[int] = None,
+        dataset_item_ids: Optional[List[str]] = None,
+        filter_string: Optional[str] = None,
+    ) -> Iterator[dataset_item.DatasetItem]:
+        return rest_operations.stream_dataset_items(
+            rest_client=self._rest_client,
+            dataset_name=self._dataset_name,
+            project_name=self._project_name,
+            nb_samples=nb_samples,
+            batch_size=batch_size,
+            dataset_item_ids=dataset_item_ids,
+            filter_string=filter_string,
+            dataset_version=self._version_info.version_hash,
+        )
+
+    @override
+    def get_version_info(
+        self,
+    ) -> Optional[dataset_version_public.DatasetVersionPublic]:
+        """
+        Get version information for this specific dataset version.
+
+        Returns:
+            DatasetVersionPublic containing this version's metadata.
+        """
+        return self._version_info
+
+    def get_evaluators(
+        self,
+        evaluator_model: Optional[str] = None,
+    ) -> List[Any]:
+        """
+        Get suite-level evaluators for this dataset version.
+
+        DatasetVersion does not support suite-level evaluators, so this always
+        returns an empty list.
+
+        Returns:
+            Empty list.
+        """
+        return []
+
+    def get_execution_policy(self) -> execution_policy.ExecutionPolicy:
+        """
+        Get the execution policy for this dataset version.
+
+        DatasetVersion does not support suite-level execution policy, so this
+        returns the default execution policy.
+
+        Returns:
+            Default execution policy.
+        """
+        return execution_policy.DEFAULT_EXECUTION_POLICY.copy()
+
+
+class Dataset(DatasetExportOperations):
+    def __init__(
+        self,
+        name: str,
+        description: Optional[str],
+        project_name: Optional[str],
+        rest_client: rest_api_client.OpikApi,
+        dataset_items_count: Optional[int] = None,
+        client: Optional[Any] = None,
+    ) -> None:
+        """
+        A Dataset object. This object should not be created directly, instead use :meth:`opik.Opik.create_dataset` or :meth:`opik.Opik.get_dataset`.
+        """
+        self._name = name
+        self._description = description
+        self._rest_client = rest_client
+        self._dataset_items_count = dataset_items_count
+        self._project_name = project_name
+        self.client = client
+
+        self._id_to_hash: Dict[str, str] = {}
+        self._hashes: Set[str] = set()
+        # True when the local hash cache is consistent with the backend.
+        # Directly-constructed Datasets (create_dataset, test-suite helpers,
+        # unit tests) start synced — there's nothing on the backend we haven't
+        # seen locally. The backend-fetch factories (`from_public`,
+        # `rest_operations.get_datasets`) flip this to False so dedup does a
+        # one-shot sync on the first `insert()` instead of paying an N+1
+        # sync at list time.
+        self._hashes_synced: bool = True
+
+    @classmethod
+    def from_public(
+        cls,
+        dataset_fern: rest_dataset_public.DatasetPublic,
+        project_name: str,
+        rest_client: rest_api_client.OpikApi,
+        client: Optional[Any] = None,
+    ) -> "Dataset":
+        """Build a Dataset from a backend response, resolving the actual project.
+
+        The backend may find the dataset via workspace-wide fallback even when
+        the caller's project_name doesn't match the dataset's actual project.
+        This method uses project_id from the response to resolve the real
+        project name, so downstream calls target the correct project.
+        """
+        actual_project_name: Optional[str] = None
+        if dataset_fern.project_id is not None:
+            actual_project_name = rest_client.projects.get_project_by_id(
+                dataset_fern.project_id
+            ).name
+
+        dataset_ = cls(
+            name=dataset_fern.name,
+            description=dataset_fern.description,
+            project_name=actual_project_name or project_name,
+            rest_client=rest_client,
+            dataset_items_count=dataset_fern.dataset_items_count,
+            client=client,
+        )
+        # Backend may already hold items we haven't seen; lazy-sync on first
+        # insert so content-hash dedup still works without paying a sync now.
+        dataset_.__internal_api__hashes_synced__ = False
+        return dataset_
+
+    @functools.cached_property
+    def id(self) -> str:
+        """The id of the dataset"""
+        return self._rest_client.datasets.get_dataset_by_identifier(
+            dataset_name=self._name, project_name=self._project_name
+        ).id
+
+    @property
+    def name(self) -> str:
+        """The name of the dataset."""
+        return self._name
+
+    @property
+    def project_name(self) -> Optional[str]:
+        """The name of the project this dataset belongs to."""
+        return self._project_name
+
+    @property
+    def description(self) -> Optional[str]:
+        """The description of the dataset."""
+        return self._description
+
+    @property
+    def dataset_items_count(self) -> Optional[int]:
+        """
+        The total number of items in the dataset.
+
+        If the count is not cached locally, it will be fetched from the backend.
+        """
+        if self._dataset_items_count is None:
+            dataset_info = self._rest_client.datasets.get_dataset_by_id(id=self.id)
+            self._dataset_items_count = dataset_info.dataset_items_count
+
+        return self._dataset_items_count
+
+    def get_current_version_name(self) -> Optional[str]:
+        """
+        Get the current version name of the dataset.
+
+        The version name is fetched from the backend and reflects the latest
+        committed version after any mutation operations (insert, update, delete).
+
+        Returns:
+            The current version name (e.g., 'v1', 'v2'), or None if no version exists.
+        """
+        version_info = self.get_version_info()
+        return version_info.version_name if version_info else None
+
+    @override
+    def get_version_info(
+        self,
+    ) -> Optional[dataset_version_public.DatasetVersionPublic]:
+        """
+        Get version information for the current (latest) dataset version.
+
+        Returns:
+            DatasetVersionPublic containing the current version's metadata,
+            or None if no version exists yet.
+        """
+        versions_response = None
+        try:
+            versions_response = self._rest_client.datasets.list_dataset_versions(
+                id=self.id,
+                page=1,
+                size=1,
+            )
+        except ApiError as e:
+            if e.status_code == 403:
+                LOGGER.debug(
+                    "Versioning is not enabled for datasets get version info returning None"
+                )
+            else:
+                raise
+        if not versions_response or not versions_response.content:
+            return None
+        return versions_response.content[0]
+
+    def get_evaluators(
+        self,
+        evaluator_model: Optional[str] = None,
+    ) -> List[Any]:
+        """
+        Get suite-level evaluators from the current dataset version.
+
+        Converts EvaluatorItemPublic objects from the BE into LLMJudge instances.
+
+        Args:
+            evaluator_model: Optional model name to use for LLMJudge evaluators.
+
+        Returns:
+            List of LLMJudge instances extracted from the version.
+        """
+        from opik.evaluation.suite_evaluators import llm_judge
+        from opik.evaluation.suite_evaluators.llm_judge import (
+            config as llm_judge_config,
+        )
+
+        version_info = self.get_version_info()
+        if version_info is None or not version_info.evaluators:
+            return []
+
+        evaluators: List[Any] = []
+        for evaluator_item in version_info.evaluators:
+            try:
+                if evaluator_item.type == "llm_judge":
+                    cfg = llm_judge_config.LLMJudgeConfig(**evaluator_item.config)
+                    evaluator = llm_judge.LLMJudge.from_config(
+                        cfg, init_kwargs={"model": evaluator_model}
+                    )
+                    evaluators.append(evaluator)
+                else:
+                    LOGGER.warning(
+                        "Unsupported evaluator type in version: %s. Only 'llm_judge' is supported.",
+                        evaluator_item.type,
+                    )
+            except Exception:
+                LOGGER.error(
+                    "Failed to instantiate evaluator from version config: %s",
+                    evaluator_item.config,
+                    exc_info=True,
+                )
+                raise
+
+        return evaluators
+
+    def get_execution_policy(
+        self,
+    ) -> execution_policy.ExecutionPolicy:
+        """
+        Get suite-level execution policy from the current dataset version.
+
+        Returns:
+            ExecutionPolicy dict with runs_per_item and pass_threshold.
+        """
+        version_info = self.get_version_info()
+        if version_info is not None and version_info.execution_policy is not None:
+            ep = version_info.execution_policy
+            return {
+                "runs_per_item": ep.runs_per_item
+                if ep.runs_per_item is not None
+                else 1,
+                "pass_threshold": ep.pass_threshold
+                if ep.pass_threshold is not None
+                else 1,
+            }
+
+        return execution_policy.DEFAULT_EXECUTION_POLICY.copy()
+
+    def get_tags(self) -> List[str]:
+        """
+        Get the tags for this dataset.
+
+        Returns:
+            List of tag strings.
+        """
+        dataset_fern = self._rest_client.datasets.get_dataset_by_identifier(
+            dataset_name=self._name, project_name=self._project_name
+        )
+        return dataset_fern.tags or []
+
+    def _convert_to_rest_item(
+        self, item: dataset_item.DatasetItem
+    ) -> rest_dataset_item.DatasetItemWrite:
+        """Convert a DatasetItem to REST API format.
+
+        Args:
+            item: The DatasetItem to convert.
+
+        Returns:
+            DatasetItemWrite object ready for REST API.
+        """
+        evaluators = None
+        if item.evaluators:
+            evaluators = [
+                rest_evaluator_item.EvaluatorItemWrite(
+                    name=e.name,
+                    type=e.type,  # type: ignore
+                    config=e.config,
+                )
+                for e in item.evaluators
+            ]
+
+        execution_policy = None
+        if item.execution_policy:
+            execution_policy = rest_execution_policy.ExecutionPolicyWrite(
+                runs_per_item=item.execution_policy.runs_per_item,
+                pass_threshold=item.execution_policy.pass_threshold,
+            )
+
+        return rest_dataset_item.DatasetItemWrite(
+            id=item.id,  # type: ignore
+            trace_id=item.trace_id,  # type: ignore
+            span_id=item.span_id,  # type: ignore
+            source=item.source,  # type: ignore
+            data=item.get_content(),
+            description=item.description,
+            evaluators=evaluators,
+            execution_policy=execution_policy,
+        )
+
+    def _insert_batch_with_retry(
+        self,
+        batch: List[rest_dataset_item.DatasetItemWrite],
+        batch_group_id: str,
+    ) -> None:
+        """Insert a batch of dataset items with automatic retry on rate limit errors.
+
+        Args:
+            batch: List of dataset items to insert.
+            batch_group_id: UUIDv7 identifier that groups all batches from a single
+                user operation together. All batches sent as part of one insert/update
+                call share the same batch_group_id.
+        """
+        rest_helpers.ensure_rest_api_call_respecting_rate_limit(
+            lambda: self._rest_client.datasets.create_or_update_dataset_items(
+                dataset_name=self._name,
+                items=batch,
+                batch_group_id=batch_group_id,
+                project_name=self._project_name,
+            )
+        )
+        LOGGER.debug("Successfully sent dataset items batch of size %d", len(batch))
+
+    def __internal_api__insert_items_as_dataclasses__(
+        self, items: List[dataset_item.DatasetItem]
+    ) -> None:
+        # Lazy-sync against the backend the first time we insert into a
+        # dataset that was fetched from the backend (list or get-by-name
+        # factory), so content-hash dedup still works without paying an
+        # N+1 sync at list time.
+        if not self._hashes_synced:
+            self.__internal_api__sync_hashes__()
+
+        # Remove duplicates if they already exist
+        deduplicated_items: List[dataset_item.DatasetItem] = []
+        for item in items:
+            item_hash = item.content_hash()
+
+            if item_hash in self._hashes:
+                LOGGER.debug(
+                    "Duplicate item found with hash: %s - ignored the event",
+                    item_hash,
+                )
+                continue
+
+            deduplicated_items.append(item)
+            self._hashes.add(item_hash)
+            self._id_to_hash[item.id] = item_hash
+
+        rest_items = [self._convert_to_rest_item(item) for item in deduplicated_items]
+
+        batches = sequence_splitter.split_into_batches(
+            rest_items,
+            max_payload_size_MB=config.MAX_BATCH_SIZE_MB,
+            max_length=constants.DATASET_ITEMS_MAX_BATCH_SIZE,
+        )
+
+        batch_group_id = id_helpers.generate_id()
+
+        for batch in batches:
+            LOGGER.debug("Sending dataset items batch of size %d", len(batch))
+            self._insert_batch_with_retry(batch, batch_group_id=batch_group_id)
+
+        # Invalidate the cached count so it will be fetched from backend on next access
+        self._dataset_items_count = None
+
+    def insert(self, items: Sequence[Dict[str, Any]]) -> None:
+        """
+        Insert new items into the dataset. A new dataset version will be created.
+
+        Args:
+            items: List of dicts (which will be converted to dataset items)
+                to add to the dataset.
+        """
+        dataset_items: List[dataset_item.DatasetItem] = [  # type: ignore
+            (dataset_item.DatasetItem(**item) if isinstance(item, dict) else item)
+            for item in items
+        ]
+        self.__internal_api__insert_items_as_dataclasses__(dataset_items)
+
+    @property
+    def __internal_api__hashes_synced__(self) -> bool:
+        """Whether the local hash cache is in sync with the backend.
+
+        `__init__` defaults this to True (a freshly constructed Dataset
+        has no backend state to sync). Factory paths that construct a
+        Dataset from an existing backend state (`from_public`,
+        `rest_operations.get_datasets`) flip it to False so the first
+        :meth:`insert` triggers a one-shot sync instead of paying an
+        N+1 sync at list time.
+        """
+        return self._hashes_synced
+
+    @__internal_api__hashes_synced__.setter
+    def __internal_api__hashes_synced__(self, value: bool) -> None:
+        self._hashes_synced = value
+
+    def __internal_api__sync_hashes__(self) -> None:
+        """Updates all the hashes in the dataset"""
+        LOGGER.debug("Start hash sync in dataset")
+
+        self._id_to_hash = {}
+        self._hashes = set()
+
+        for item in self.__internal_api__stream_items_as_dataclasses__():
+            item_hash = item.content_hash()
+            self._id_to_hash[item.id] = item_hash  # type: ignore
+            self._hashes.add(item_hash)
+
+        self._hashes_synced = True
+        LOGGER.debug("Finish hash sync in dataset")
+
+    def update(self, items: List[Dict[str, Any]]) -> None:
+        """
+        Update existing items in the dataset.
+
+        Args:
+            items: List of DatasetItem objects to update in the dataset. You need to provide the full item object as it will override what has been supplied previously.
+
+        Raises:
+            DatasetItemUpdateOperationRequiresItemId: If any item in the list is missing an id.
+        """
+        for item in items:
+            if "id" not in item:
+                raise exceptions.DatasetItemUpdateOperationRequiresItemId(
+                    "Missing id for dataset item to update: %s", item
+                )
+
+        self.insert(items)
+
+    def _delete_batch_with_retry(
+        self,
+        batch: List[str],
+        batch_group_id: str,
+    ) -> None:
+        """Delete a batch of dataset items with automatic retry on rate limit errors.
+
+        Args:
+            batch: List of item IDs to delete.
+            batch_group_id: UUIDv7 identifier that groups all batches from a single
+                user operation together. All batches sent as part of one delete
+                call share the same batch_group_id.
+        """
+        rest_helpers.ensure_rest_api_call_respecting_rate_limit(
+            lambda: self._rest_client.datasets.delete_dataset_items(
+                item_ids=batch, batch_group_id=batch_group_id
+            )
+        )
+        LOGGER.debug("Successfully deleted dataset items batch of size %d", len(batch))
+
+    def delete(self, items_ids: List[str]) -> None:
+        """
+        Delete items from the dataset. A new dataset version will be created.
+
+        Args:
+            items_ids: List of item ids to delete.
+        """
+        batches = sequence_splitter.split_into_batches(
+            items_ids, max_length=constants.DATASET_ITEMS_MAX_BATCH_SIZE
+        )
+
+        batch_group_id = id_helpers.generate_id()
+
+        for batch in batches:
+            LOGGER.debug("Deleting dataset items batch: %s", batch)
+            self._delete_batch_with_retry(batch, batch_group_id=batch_group_id)
+
+            for item_id in batch:
+                if item_id in self._id_to_hash:
+                    hash = self._id_to_hash[item_id]
+                    self._hashes.discard(hash)
+                    del self._id_to_hash[item_id]
+
+        # Invalidate the cached count so it will be fetched from backend on next access
+        self._dataset_items_count = None
+
+    def clear(self) -> None:
+        """
+        Delete all items from the given dataset. A new dataset version will be created.
+        """
+        item_ids = [
+            item.id
+            for item in self.__internal_api__stream_items_as_dataclasses__()
+            if item.id is not None
+        ]
+
+        self.delete(item_ids)
+
+    @override
+    def __internal_api__stream_items_as_dataclasses__(
+        self,
+        nb_samples: Optional[int] = None,
+        batch_size: Optional[int] = None,
+        dataset_item_ids: Optional[List[str]] = None,
+        filter_string: Optional[str] = None,
+    ) -> Iterator[dataset_item.DatasetItem]:
+        """
+        Stream dataset items as a generator instead of loading all at once.
+
+        This method yields dataset items one at a time, enabling evaluation to start
+        processing items before the entire dataset is downloaded. This is particularly
+        useful for large datasets with heavy payloads (images, videos, audio).
+
+        Args:
+            nb_samples: Maximum number of items to retrieve. If None, all items are streamed.
+            batch_size: Maximum number of items to fetch per batch from the backend.
+                        If None, uses the default value from constants.DATASET_STREAM_BATCH_SIZE.
+            dataset_item_ids: Optional list of specific item IDs to retrieve. If provided,
+                            only items with matching IDs will be yielded.
+            filter_string: Optional OQL filter string to filter dataset items.
+
+        Yields:
+            DatasetItem objects one at a time
+        """
+        return rest_operations.stream_dataset_items(
+            rest_client=self._rest_client,
+            dataset_name=self._name,
+            project_name=self._project_name,
+            nb_samples=nb_samples,
+            batch_size=batch_size,
+            dataset_item_ids=dataset_item_ids,
+            filter_string=filter_string,
+            dataset_version=None,
+        )
+
+    def insert_from_json(
+        self,
+        json_array: str,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Args:
+            json_array: json string of format: "[{...}, {...}, {...}]" where every dictionary
+                is to be transformed into dataset item
+            keys_mapping: dictionary that maps json keys to item fields names
+                Example: {'Expected output': 'expected_output'}
+            ignore_keys: if your json dicts contain keys that are not needed for DatasetItem
+                construction - pass them as ignore_keys argument
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        new_items = converters.from_json(
+            json_array, keys_mapping=keys_mapping, ignore_keys=ignore_keys
+        )
+
+        self.insert(new_items)
+
+    def read_jsonl_from_file(
+        self,
+        file_path: str,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Read JSONL from a file and insert it into the dataset.
+
+        Args:
+            file_path: Path to the JSONL file
+            keys_mapping: dictionary that maps json keys to item fields names
+                Example: {'Expected output': 'expected_output'}
+            ignore_keys: if your json dicts contain keys that are not needed for DatasetItem
+                construction - pass them as ignore_keys argument
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+        new_items = converters.from_jsonl_file(file_path, keys_mapping, ignore_keys)
+        self.insert(new_items)
+
+    def insert_from_pandas(
+        self,
+        dataframe: "pd.DataFrame",
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Requires: `pandas` library to be installed.
+
+        Args:
+            dataframe: pandas dataframe
+            keys_mapping: Dictionary that maps dataframe column names to dataset item field names.
+                Example: {'Expected output': 'expected_output'}
+            ignore_keys: if your dataframe contains columns that are not needed for DatasetItem
+                construction - pass them as ignore_keys argument
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        new_items = converters.from_pandas(dataframe, keys_mapping, ignore_keys)
+
+        self.insert(new_items)
+
+    def get_version_view(self, version_name: str) -> DatasetVersion:
+        """
+        Get a read-only view of a specific dataset version.
+
+        The returned DatasetVersion object allows reading version metadata and
+        retrieving items via :meth:`DatasetVersion.get_items`, but does not support
+        mutations.
+
+        Args:
+            version_name: The version name (e.g., 'v1', 'v2').
+
+        Returns:
+            A read-only DatasetVersion object for accessing the specified version.
+
+        Raises:
+            opik.exceptions.DatasetVersionNotFound: If the specified version does not exist.
+
+        Example:
+            >>> dataset = client.get_dataset("my_dataset")
+            >>> version = dataset.get_version_view("v1")
+            >>> items = version.get_items()
+        """
+        version_info = rest_operations.find_version_by_name(
+            rest_client=self._rest_client,
+            dataset_id=self.id,
+            version_name=version_name,
+        )
+
+        if version_info is None:
+            raise exceptions.DatasetVersionNotFound(
+                f"Dataset version '{version_name}' not found in dataset '{self._name}'"
+            )
+
+        return DatasetVersion(
+            dataset_name=self._name,
+            dataset_id=self.id,
+            rest_client=self._rest_client,
+            version_info=version_info,
+            project_name=self._project_name,
+            client=self.client,
+        )

sdks/python/src/opik/api_objects/dataset/dataset_item.py

@@ -0,0 +1,106 @@
+from typing import Optional, Dict, Any, List
+import pydantic
+import json
+import hashlib
+from .. import constants, helpers
+
+
+class EvaluatorItem(pydantic.BaseModel):
+    """
+    An evaluator configuration for a dataset item.
+    """
+
+    model_config = pydantic.ConfigDict(extra="allow", strict=False)
+
+    name: str
+    """The name of the evaluator."""
+
+    type: str
+    """The type of evaluator (e.g., 'llm_judge', 'code_metric')."""
+
+    config: Dict[str, Any]
+    """The evaluator configuration."""
+
+
+class ExecutionPolicyItem(pydantic.BaseModel):
+    """
+    Execution policy for a dataset item.
+    """
+
+    model_config = pydantic.ConfigDict(extra="allow", strict=False)
+
+    runs_per_item: Optional[int] = None
+    """Number of times to run the task for this item."""
+
+    pass_threshold: Optional[int] = None
+    """Minimum number of runs that must pass for the item to pass."""
+
+
+class DatasetItem(pydantic.BaseModel):
+    """
+    A DatasetItem object representing an item in a dataset.
+    The format is flexible.
+    """
+
+    model_config = pydantic.ConfigDict(extra="allow", strict=False)
+
+    id: pydantic.SkipValidation[str] = pydantic.Field(
+        default_factory=helpers.generate_id
+    )
+    """The unique identifier for this dataset item."""
+
+    trace_id: Optional[str] = None
+    """The ID of the trace associated with this dataset item."""
+
+    span_id: Optional[str] = None
+    """The ID of the span associated with this dataset item."""
+
+    source: str = constants.DATASET_SOURCE_SDK
+    """The source of the dataset item. Defaults to DATASET_SOURCE_SDK."""
+
+    description: Optional[str] = None
+    """Optional description of the dataset item."""
+
+    evaluators: Optional[List[EvaluatorItem]] = None
+    """List of evaluators configured for this dataset item."""
+
+    execution_policy: Optional[ExecutionPolicyItem] = None
+    """Execution policy for this dataset item."""
+
+    def get_content(
+        self,
+        include_id: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Get the data content of the dataset item (extra fields).
+
+        Note: evaluators and execution_policy are not included in data content
+
+        Args:
+            include_id: Whether to include the item ID in the content.
+
+        Returns:
+            Dictionary containing the item's extra fields.
+        """
+        content = {**self.model_extra}
+        if include_id:
+            content["id"] = self.id
+
+        return content
+
+    def content_hash(self) -> str:
+        content = self.get_content()
+
+        if self.description is not None:
+            content["description"] = self.description
+
+        if self.evaluators is not None:
+            content["evaluators"] = [e.model_dump() for e in self.evaluators]
+
+        if self.execution_policy is not None:
+            content["execution_policy"] = self.execution_policy.model_dump()
+
+        json_string = json.dumps(content, sort_keys=True)
+        hash_object = hashlib.sha256(json_string.encode())
+
+        return hash_object.hexdigest()

sdks/python/src/opik/api_objects/dataset/execution_policy.py

@@ -0,0 +1,22 @@
+"""Execution policy type and default for dataset evaluation."""
+
+from typing import TypedDict
+
+
+class ExecutionPolicy(TypedDict, total=False):
+    """
+    Execution policy for test suite items.
+
+    Attributes:
+        runs_per_item: Number of times to run evaluation per item.
+        pass_threshold: Minimum number of passing runs required for item to pass.
+    """
+
+    runs_per_item: int
+    pass_threshold: int
+
+
+DEFAULT_EXECUTION_POLICY: ExecutionPolicy = {
+    "runs_per_item": 1,
+    "pass_threshold": 1,
+}

sdks/python/src/opik/api_objects/dataset/helpers.py

@@ -0,0 +1,9 @@
+import importlib.util
+
+
+def raise_if_pandas_is_unavailable() -> None:
+    if importlib.util.find_spec("pandas") is None:
+        raise ImportError(
+            "The Python library Pandas is required for this method. "
+            "You can install it with `pip install pandas`."
+        )

sdks/python/src/opik/api_objects/dataset/rest_operations.py

@@ -0,0 +1,495 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, Iterator, List, Optional, Set, TYPE_CHECKING
+
+from opik.rest_api import OpikApi
+from opik.rest_api.types import (
+    dataset_item as rest_dataset_item_read,
+    dataset_version_public,
+)
+import opik.exceptions as exceptions
+from opik.message_processing import streamer
+from opik.rest_client_configurator import retry_decorator
+from opik.api_objects import opik_query_language, rest_stream_parser
+from . import dataset, dataset_item, execution_policy
+from .. import experiment, constants, rest_helpers
+from ..experiment import experiments_client
+from ...rest_api.core.api_error import ApiError
+
+if TYPE_CHECKING:
+    from opik.evaluation.suite_evaluators import llm_judge
+    from .test_suite.test_suite import TestSuite
+
+LOGGER = logging.getLogger(__name__)
+
+
+def stream_dataset_items(
+    rest_client: OpikApi,
+    dataset_name: str,
+    project_name: Optional[str],
+    nb_samples: Optional[int] = None,
+    batch_size: Optional[int] = None,
+    dataset_item_ids: Optional[List[str]] = None,
+    filter_string: Optional[str] = None,
+    dataset_version: Optional[str] = None,
+) -> Iterator[dataset_item.DatasetItem]:
+    """
+    Stream dataset items from the backend as a generator.
+
+    Args:
+        rest_client: The REST API client.
+        dataset_name: Name of the dataset to stream items from.
+        project_name: Name of the project to stream items from.
+        nb_samples: Maximum number of items to retrieve. If None, all items are streamed.
+        batch_size: Maximum number of items to fetch per batch from the backend.
+        dataset_item_ids: Optional list of specific item IDs to retrieve.
+        filter_string: Optional OQL filter string to filter dataset items.
+        dataset_version: Optional dataset version hash to filter items by a specific version.
+
+    Yields:
+        DatasetItem objects one at a time.
+    """
+    if batch_size is None:
+        batch_size = constants.DATASET_STREAM_BATCH_SIZE
+
+    last_retrieved_id: Optional[str] = None
+    should_retrieve_more_items = True
+    items_yielded = 0
+    dataset_items_ids_left: Optional[Set[str]] = (
+        set(dataset_item_ids) if dataset_item_ids else None
+    )
+    _conflicting_keys_warned = False
+
+    filters: Optional[str] = None
+    if filter_string:
+        oql = opik_query_language.OpikQueryLanguage.for_dataset_items(filter_string)
+        filter_expressions = oql.get_filter_expressions()
+        if filter_expressions:
+            filters = json.dumps(filter_expressions)
+
+    while should_retrieve_more_items:
+
+        @retry_decorator.opik_rest_retry
+        def _fetch_batch() -> List[rest_dataset_item_read.DatasetItem]:
+            return rest_stream_parser.read_and_parse_stream(
+                stream=rest_client.datasets.stream_dataset_items(
+                    dataset_name=dataset_name,
+                    project_name=project_name,
+                    last_retrieved_id=last_retrieved_id,
+                    steam_limit=batch_size,
+                    filters=filters,
+                    dataset_version=dataset_version,
+                ),
+                item_class=rest_dataset_item_read.DatasetItem,
+                nb_samples=nb_samples,
+            )
+
+        dataset_items = _fetch_batch()
+
+        if len(dataset_items) == 0:
+            should_retrieve_more_items = False
+            break
+
+        for item in dataset_items:
+            item_id = item.id
+            last_retrieved_id = item_id
+
+            if dataset_items_ids_left is not None:
+                if item_id not in dataset_items_ids_left:
+                    continue
+                else:
+                    dataset_items_ids_left.remove(item_id)
+
+            # Convert evaluators from REST format to DatasetItem format
+            evaluators = None
+            if item.evaluators:
+                evaluators = [
+                    dataset_item.EvaluatorItem(
+                        name=e.name,
+                        type=e.type,
+                        config=e.config,
+                    )
+                    for e in item.evaluators
+                ]
+
+            # Convert execution_policy from REST format to DatasetItem format
+            execution_policy = None
+            if item.execution_policy:
+                execution_policy = dataset_item.ExecutionPolicyItem(
+                    runs_per_item=item.execution_policy.runs_per_item,
+                    pass_threshold=item.execution_policy.pass_threshold,
+                )
+
+            # Strip DatasetItem field names from user data before unpacking to avoid
+            # "multiple values for keyword argument" errors. This happens when user data
+            # contains a key that matches a DatasetItem field (e.g. 'id' in HotpotQA).
+            conflicting = (
+                item.data.keys() & dataset_item.DatasetItem.model_fields.keys()
+            )
+            if conflicting and not _conflicting_keys_warned:
+                _conflicting_keys_warned = True
+                LOGGER.warning(
+                    "Dataset item data contains keys that shadow DatasetItem fields and will be ignored: %s. "
+                    "Rename these keys in your dataset to preserve them.",
+                    sorted(conflicting),
+                )
+            extra_data = {
+                k: v
+                for k, v in item.data.items()
+                if k not in dataset_item.DatasetItem.model_fields
+            }
+
+            reconstructed_item = dataset_item.DatasetItem(
+                id=item.id,
+                trace_id=item.trace_id,
+                span_id=item.span_id,
+                source=item.source,
+                description=item.description,
+                evaluators=evaluators,
+                execution_policy=execution_policy,
+                **extra_data,
+            )
+
+            yield reconstructed_item
+            items_yielded += 1
+
+            if nb_samples is not None and items_yielded >= nb_samples:
+                should_retrieve_more_items = False
+                break
+
+            if dataset_items_ids_left is not None and len(dataset_items_ids_left) == 0:
+                should_retrieve_more_items = False
+                break
+
+    if dataset_items_ids_left and len(dataset_items_ids_left) > 0:
+        LOGGER.warning(
+            "The following dataset items were not found in the dataset: %s",
+            dataset_items_ids_left,
+        )
+
+
+def find_version_by_name(
+    rest_client: OpikApi,
+    dataset_id: str,
+    version_name: str,
+) -> Optional[dataset_version_public.DatasetVersionPublic]:
+    """
+    Find a dataset version by version name.
+
+    Args:
+        rest_client: The REST API client.
+        dataset_id: The dataset ID to search versions in.
+        version_name: Version name to search for (e.g., 'v1', 'v2').
+
+    Returns:
+        The DatasetVersionPublic if found, None otherwise.
+    """
+    try:
+        return rest_client.datasets.retrieve_dataset_version(
+            id=dataset_id, version_name=version_name
+        )
+    except ApiError as e:
+        if e.status_code == 404:
+            return None
+        raise
+
+
+def get_datasets(
+    project_name: Optional[str],
+    rest_client: OpikApi,
+    max_results: int = 1000,
+    sync_items: bool = False,
+) -> List[dataset.Dataset]:
+    page_size = 100
+    datasets: List[dataset.Dataset] = []
+    page = 1
+
+    project_id = rest_helpers.resolve_project_id_by_name_optional(
+        rest_client, project_name=project_name
+    )
+
+    while len(datasets) < max_results:
+        page_datasets = rest_client.datasets.find_datasets(
+            page=page,
+            size=page_size,
+            project_id=project_id,
+        )
+
+        if len(page_datasets.content) == 0:
+            break
+
+        for dataset_fern in page_datasets.content[: (max_results - len(datasets))]:
+            dataset_ = dataset.Dataset(
+                name=dataset_fern.name,
+                description=dataset_fern.description,
+                project_name=project_name,
+                rest_client=rest_client,
+                dataset_items_count=dataset_fern.dataset_items_count,
+            )
+
+            if sync_items:
+                dataset_.__internal_api__sync_hashes__()
+            else:
+                # Backend holds items we haven't seen locally; defer the sync
+                # until the first `insert()` so dedup still works without
+                # paying an N+1 sync right now.
+                dataset_.__internal_api__hashes_synced__ = False
+
+            datasets.append(dataset_)
+
+        page += 1
+
+    return datasets
+
+
+def get_test_suites(
+    project_name: Optional[str],
+    rest_client: OpikApi,
+    max_results: int = 1000,
+    client: Optional[Any] = None,
+) -> List[TestSuite]:
+    from .test_suite import test_suite as test_suite_module
+
+    page_size = 100
+    suites: List[test_suite_module.TestSuite] = []
+    page = 1
+
+    project_id = rest_helpers.resolve_project_id_by_name_optional(
+        rest_client, project_name=project_name
+    )
+
+    while len(suites) < max_results:
+        page_datasets = rest_client.datasets.find_datasets(
+            page=page,
+            size=page_size,
+            project_id=project_id,
+        )
+
+        if len(page_datasets.content) == 0:
+            break
+
+        for dataset_fern in page_datasets.content:
+            if len(suites) >= max_results:
+                break
+            if dataset_fern.type != "evaluation_suite":
+                continue
+
+            suite_dataset = dataset.Dataset(
+                name=dataset_fern.name,
+                description=dataset_fern.description,
+                project_name=project_name,
+                rest_client=rest_client,
+                dataset_items_count=dataset_fern.dataset_items_count,
+                client=client,
+            )
+
+            suites.append(
+                test_suite_module.TestSuite(
+                    name=dataset_fern.name,
+                    dataset_=suite_dataset,
+                    client=client,
+                )
+            )
+
+        page += 1
+
+    return suites
+
+
+def get_dataset_id(
+    rest_client: OpikApi, dataset_name: str, project_name: Optional[str]
+) -> str:
+    try:
+        dataset_id = rest_client.datasets.get_dataset_by_identifier(
+            dataset_name=dataset_name, project_name=project_name
+        ).id
+    except ApiError as e:
+        if e.status_code == 404:
+            raise exceptions.DatasetNotFound(
+                f"Dataset with the name {dataset_name} not found."
+            ) from e
+        raise
+
+    return dataset_id
+
+
+def get_dataset_experiments(
+    rest_client: OpikApi,
+    dataset_id: str,
+    max_results: int,
+    streamer: streamer.Streamer,
+    experiments_client: experiments_client.ExperimentsClient,
+) -> List[experiment.Experiment]:
+    page_size = 100
+    experiments: List[experiment.Experiment] = []
+
+    page = 1
+    while len(experiments) < max_results:
+        page_experiments = rest_client.experiments.find_experiments(
+            page=page,
+            size=page_size,
+            dataset_id=dataset_id,
+        )
+
+        if len(page_experiments.content) == 0:
+            break
+
+        for experiment_ in page_experiments.content[: max_results - len(experiments)]:
+            experiments.append(
+                experiment.Experiment(
+                    id=experiment_.id,
+                    name=experiment_.name,
+                    dataset_name=experiment_.dataset_name,
+                    rest_client=rest_client,
+                    streamer=streamer,
+                    experiments_client=experiments_client,
+                    tags=experiment_.tags,
+                )
+            )
+
+        page += 1
+
+    return experiments
+
+
+def create_test_suite_dataset(
+    rest_client: OpikApi,
+    dataset_name: str,
+    project_name: Optional[str],
+    description: Optional[str],
+    evaluators: Optional[List[llm_judge.LLMJudge]],
+    exec_policy: Optional[execution_policy.ExecutionPolicy],
+    tags: Optional[List[str]] = None,
+) -> str:
+    """
+    Create a dataset of type 'test_suite' and its initial version
+    with evaluators and execution_policy persisted to the backend.
+
+    Args:
+        rest_client: The REST API client.
+        dataset_name: The name of the dataset/suite.
+        project_name: The name of the project.
+        description: Optional description.
+        evaluators: LLMJudge evaluators.
+        exec_policy: Execution policy dict.
+        tags: Optional list of tags for the suite.
+
+    Returns:
+        The dataset ID.
+    """
+    rest_client.datasets.create_dataset(
+        name=dataset_name,
+        description=description,
+        project_name=project_name,
+        # TODO: OPIK-5795 - migrate DB value from 'evaluation_suite' to 'test_suite'
+        type="evaluation_suite",
+        tags=tags,
+    )
+
+    dataset_fern = rest_client.datasets.get_dataset_by_identifier(
+        dataset_name=dataset_name,
+        project_name=project_name,
+    )
+
+    # Skip initial version when there is no metadata to persist.
+    # This avoids an empty v1 that the TS SDK doesn't create (OPIK-5815).
+    if not evaluators and not exec_policy:
+        return dataset_fern.id
+
+    resolved_policy = exec_policy or execution_policy.DEFAULT_EXECUTION_POLICY.copy()
+    request: Dict[str, Any] = {
+        "change_description": "Suite created via SDK",
+    }
+    if evaluators:
+        request["evaluators"] = [
+            {
+                "name": e.name,
+                "type": "llm_judge",
+                "config": e.to_config().model_dump(by_alias=True),
+            }
+            for e in evaluators
+        ]
+    request["execution_policy"] = {
+        "runs_per_item": resolved_policy.get("runs_per_item", 1),
+        "pass_threshold": resolved_policy.get("pass_threshold", 1),
+    }
+    rest_client.datasets.apply_dataset_item_changes(
+        id=dataset_fern.id, request=request, override=True
+    )
+
+    return dataset_fern.id
+
+
+def create_initial_test_suite_version(
+    rest_client: OpikApi,
+    dataset_id: str,
+    evaluators: List[llm_judge.LLMJudge],
+    exec_policy: execution_policy.ExecutionPolicy,
+) -> None:
+    """
+    Create the first version for a test suite that has no versions yet.
+    Uses override=True since there is no base version to build on.
+    """
+    request: Dict[str, Any] = {
+        "change_description": "Suite created via SDK",
+    }
+    if evaluators:
+        request["evaluators"] = [
+            {
+                "name": e.name,
+                "type": "llm_judge",
+                "config": e.to_config().model_dump(by_alias=True),
+            }
+            for e in evaluators
+        ]
+    request["execution_policy"] = {
+        "runs_per_item": exec_policy.get("runs_per_item", 1),
+        "pass_threshold": exec_policy.get("pass_threshold", 1),
+    }
+    rest_client.datasets.apply_dataset_item_changes(
+        id=dataset_id, request=request, override=True
+    )
+
+
+def update_test_suite_dataset(
+    rest_client: OpikApi,
+    dataset_id: str,
+    base_version_id: str,
+    evaluators: List[llm_judge.LLMJudge],
+    exec_policy: execution_policy.ExecutionPolicy,
+    change_description: Optional[str] = None,
+) -> None:
+    """
+    Update suite-level evaluators and execution_policy by creating a new
+    dataset version based on the current latest version.
+
+    Args:
+        rest_client: The REST API client.
+        dataset_id: The dataset ID.
+        base_version_id: The current latest version UUID to base the update on.
+        evaluators: Suite-level LLMJudge evaluators.
+        exec_policy: Execution policy dict.
+        change_description: Optional description of the change for the new version.
+    """
+    request: Dict[str, Any] = {
+        "base_version": base_version_id,
+        "evaluators": [
+            {
+                "name": e.name,
+                "type": "llm_judge",
+                "config": e.to_config().model_dump(by_alias=True),
+            }
+            for e in evaluators
+        ],
+        "execution_policy": {
+            "runs_per_item": exec_policy.get("runs_per_item", 1),
+            "pass_threshold": exec_policy.get("pass_threshold", 1),
+        },
+    }
+    if change_description:
+        request["change_description"] = change_description
+    rest_client.datasets.apply_dataset_item_changes(
+        id=dataset_id, request=request, override=False
+    )

sdks/python/src/opik/api_objects/dataset/test_suite/__init__.py

@@ -0,0 +1,8 @@
+from .test_suite import TestSuite, TestSuiteVersion
+from .types import TestSuiteItem
+
+__all__ = [
+    "TestSuite",
+    "TestSuiteVersion",
+    "TestSuiteItem",
+]

sdks/python/src/opik/api_objects/dataset/test_suite/converters.py

@@ -0,0 +1,226 @@
+"""Converters between raw dataset/REST formats and test suite formats.
+
+The two key adapter functions provide bidirectional conversion:
+
+* :func:`dataset_item_to_suite_item_dict` — DatasetItem → TestSuiteItem (exports)
+* :func:`suite_item_dict_to_dataset_item` — TestSuiteItem → DatasetItem (imports)
+
+These adapters bridge a **structural** gap between the flat DatasetItem model
+(extra fields stored via pydantic ``model_extra``) and the nested TestSuiteItem
+format (``data``, ``assertions``, ``execution_policy``).  Because of this
+structural difference the generic ``dataset/converters`` serialisation helpers
+cannot be reused directly for test-suite I/O.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from opik.evaluation.suite_evaluators.llm_judge import LLMJudge
+
+from opik import id_helpers
+from opik.api_objects.dataset import dataset_item, validators, helpers
+from opik.api_objects.dataset.test_suite import types as suite_types
+from opik.rest_api.types import (
+    evaluator_item_public as rest_evaluator_item_public,
+    execution_policy_public as rest_execution_policy_public,
+)
+from .. import execution_policy
+
+
+def evaluators_to_assertions(evaluators: List[LLMJudge]) -> List[str]:
+    """Extract assertion strings from a list of LLMJudge instances."""
+    assertions: List[str] = []
+    for evaluator in evaluators:
+        assertions.extend(evaluator.assertions)
+    return assertions
+
+
+def version_evaluators_to_assertions(
+    evaluators: Optional[List[rest_evaluator_item_public.EvaluatorItemPublic]],
+) -> List[str]:
+    """Extract assertion strings from REST evaluator items on a dataset version."""
+    from opik.evaluation.suite_evaluators import llm_judge
+    from opik.evaluation.suite_evaluators.llm_judge import config as llm_judge_config
+
+    assertions: List[str] = []
+    if evaluators:
+        for evaluator in evaluators:
+            if evaluator.type == "llm_judge":
+                cfg = llm_judge_config.LLMJudgeConfig(**evaluator.config)
+                judge = llm_judge.LLMJudge.from_config(cfg)
+                assertions.extend(judge.assertions)
+    return assertions
+
+
+def version_policy_to_execution_policy(
+    policy: Optional[rest_execution_policy_public.ExecutionPolicyPublic],
+) -> execution_policy.ExecutionPolicy:
+    """Convert a REST execution policy object to an ExecutionPolicy dict."""
+    if policy:
+        return execution_policy.ExecutionPolicy(
+            runs_per_item=policy.runs_per_item or 1,
+            pass_threshold=policy.pass_threshold or 1,
+        )
+    return execution_policy.DEFAULT_EXECUTION_POLICY.copy()
+
+
+def dataset_item_to_suite_item_dict(
+    item: dataset_item.DatasetItem,
+) -> suite_types.TestSuiteItem:
+    """Convert a DatasetItem into a TestSuiteItem dict with decoded assertions."""
+    result = suite_types.TestSuiteItem(
+        id=item.id,
+        data=item.get_content(),
+        assertions=version_evaluators_to_assertions(item.evaluators),
+    )
+    if item.description is not None:
+        result["description"] = item.description
+    if item.execution_policy is not None:
+        result["execution_policy"] = {
+            "runs_per_item": item.execution_policy.runs_per_item or 1,
+            "pass_threshold": item.execution_policy.pass_threshold or 1,
+        }
+    return result
+
+
+def suite_item_dict_to_dataset_item(
+    item: suite_types.TestSuiteItem,
+) -> dataset_item.DatasetItem:
+    """Convert a TestSuiteItem dict into a DatasetItem with evaluators.
+
+    This is the inverse of :func:`dataset_item_to_suite_item_dict`.
+    """
+    evaluators = validators.resolve_evaluators(
+        item.get("assertions"), None, "item-level assertions"
+    )
+
+    evaluator_items = None
+    if evaluators:
+        evaluator_items = [
+            dataset_item.EvaluatorItem(
+                name=e.name,
+                type="llm_judge",
+                config=e.to_config().model_dump(by_alias=True),
+            )
+            for e in evaluators
+        ]
+
+    ep = item.get("execution_policy")
+    execution_policy_item = None
+    if ep:
+        execution_policy_item = dataset_item.ExecutionPolicyItem(
+            runs_per_item=ep.get("runs_per_item"),
+            pass_threshold=ep.get("pass_threshold"),
+        )
+
+    return dataset_item.DatasetItem(
+        id=item.get("id", id_helpers.generate_id()),
+        description=item.get("description"),
+        evaluators=evaluator_items,
+        execution_policy=execution_policy_item,
+        **item["data"],
+    )
+
+
+# ---------------------------------------------------------------------------
+# Export
+# ---------------------------------------------------------------------------
+
+
+def to_json(items: List[suite_types.TestSuiteItem]) -> str:
+    """Serialise a list of TestSuiteItem dicts to a JSON string."""
+    return json.dumps(items, indent=2)
+
+
+def to_pandas(items: List[suite_types.TestSuiteItem]) -> "pd.DataFrame":
+    """Convert a list of TestSuiteItem dicts to a pandas DataFrame."""
+    helpers.raise_if_pandas_is_unavailable()
+
+    import pandas as pd
+
+    return pd.DataFrame(items)
+
+
+# ---------------------------------------------------------------------------
+# Import
+# ---------------------------------------------------------------------------
+
+
+def _apply_key_mapping(
+    item_dict: Dict[str, Any],
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> Dict[str, Any]:
+    return {
+        keys_mapping.get(key, key): value
+        for key, value in item_dict.items()
+        if key not in ignore_keys
+    }
+
+
+def _from_dicts(
+    item_dicts: List[Dict[str, Any]],
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[suite_types.TestSuiteItem]:
+    return [
+        _apply_key_mapping(d, keys_mapping, ignore_keys)  # type: ignore[misc]
+        for d in item_dicts
+    ]
+
+
+def from_json(
+    value: str,
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[suite_types.TestSuiteItem]:
+    """Parse a JSON array string into a list of TestSuiteItem dicts."""
+    parsed = json.loads(value)
+    if not isinstance(parsed, list):
+        raise ValueError(
+            f"JSON input must be an array of objects, got {type(parsed).__name__}."
+        )
+    return _from_dicts(parsed, keys_mapping, ignore_keys)
+
+
+def from_pandas(
+    dataframe: "pd.DataFrame",
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[suite_types.TestSuiteItem]:
+    """Convert pandas DataFrame rows into a list of TestSuiteItem dicts."""
+    helpers.raise_if_pandas_is_unavailable()
+
+    items: List[suite_types.TestSuiteItem] = []
+    for record in dataframe.to_dict(orient="records"):
+        mapped: Dict[str, Any] = {}
+        for key, value in record.items():
+            if key in ignore_keys:
+                continue
+            # pandas stores missing optional fields as float NaN
+            if isinstance(value, float) and value != value:
+                continue
+            mapped[keys_mapping.get(key, key)] = value
+        items.append(mapped)  # type: ignore[arg-type]
+    return items
+
+
+def from_jsonl_file(
+    file_path: str,
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[suite_types.TestSuiteItem]:
+    """Read a JSONL file into a list of TestSuiteItem dicts."""
+    raw_items: List[Dict[str, Any]] = []
+    with open(file_path, "r", encoding="utf-8") as file:
+        for line in file:
+            line = line.strip()
+            if line:
+                raw_items.append(json.loads(line))
+
+    return _from_dicts(raw_items, keys_mapping, ignore_keys)

sdks/python/src/opik/api_objects/dataset/test_suite/report_processors/displayer.py

@@ -0,0 +1,163 @@
+"""Console display for test suite results."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Dict, Optional, TYPE_CHECKING
+
+from rich import align, console, panel, table, text
+
+from ..test_suite_result import is_score_passed
+
+if TYPE_CHECKING:
+    from .. import test_suite_result as _result_mod
+
+
+def _format_time(seconds: float) -> str:
+    hours, remainder = divmod(seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}"
+
+
+def display_suite_results(
+    suite_result: _result_mod.TestSuiteResult,
+    verbose: int,
+    report_path: Optional[str],
+) -> None:
+    suite_name = suite_result.suite_name or "Test Suite"
+    total_time = suite_result.total_time or 0.0
+    experiment_url = suite_result.experiment_url
+    test_results = [
+        test_result
+        for item_result in suite_result.item_results.values()
+        for test_result in item_result.test_results
+    ]
+    nb_runs = len(test_results)
+    nb_items = suite_result.items_total
+    items_passed = suite_result.items_passed
+    items_total = suite_result.items_total
+    suite_passed = suite_result.all_items_passed
+
+    assertion_passed_count: Dict[str, int] = defaultdict(int)
+    assertion_total_count: Dict[str, int] = defaultdict(int)
+
+    for test_result_ in test_results:
+        for score in test_result_.score_results:
+            assertion_total_count[score.name] += 1
+            if is_score_passed(score):
+                assertion_passed_count[score.name] += 1
+
+    time_text = text.Text(f"Total time:        {_format_time(total_time)}")
+    time_text.stylize("bold", 0, 18)
+    time_text = align.Align.left(time_text)
+
+    nb_samples_text = text.Text(f"Number of items:   {nb_items:,} ({nb_runs:,} runs)")
+    nb_samples_text.stylize("bold", 0, 18)
+    nb_samples_text = align.Align.left(nb_samples_text)
+
+    pass_rate = items_passed / items_total if items_total > 0 else None
+    if items_total == 0:
+        pass_style = "yellow bold"
+        pass_label = "NO ITEMS"
+    elif suite_passed:
+        pass_style = "green bold"
+        pass_label = "PASSED"
+    else:
+        pass_style = "red bold"
+        pass_label = "FAILED"
+    pass_text = text.Text(f"Suite result:      {pass_label}", style=pass_style)
+    pass_text.stylize("bold", 0, 18)
+    pass_text = align.Align.left(pass_text)
+
+    items_text = text.Text(f"Items passed:      {items_passed}/{items_total}")
+    items_text.stylize("bold", 0, 18)
+    items_text = align.Align.left(items_text)
+
+    rate_value = f"{pass_rate:.1%}" if pass_rate is not None else "N/A"
+    rate_text = text.Text(f"Pass rate:         {rate_value}")
+    rate_text.stylize("bold", 0, 18)
+    rate_text = align.Align.left(rate_text)
+
+    task_times = [
+        tr.task_execution_time
+        for tr in test_results
+        if tr.task_execution_time is not None
+    ]
+    scoring_times = [
+        tr.scoring_time for tr in test_results if tr.scoring_time is not None
+    ]
+
+    content = table.Table.grid()
+    content.add_row(text.Text(""))
+
+    if experiment_url is not None:
+        link_text = text.Text.from_markup(
+            f"[bold cyan][link={experiment_url}]View results in Opik dashboard[/link][/bold cyan]"
+        )
+        content.add_row(align.Align.left(link_text))
+
+    if report_path is not None:
+        report_text = text.Text.from_markup(
+            f"[bold green][link=file://{report_path}]View local detailed report file[/link][/bold green]"
+        )
+        content.add_row(align.Align.left(report_text))
+
+    if experiment_url is not None or report_path is not None:
+        content.add_row(text.Text(""))
+
+    content.add_row(time_text)
+    content.add_row(nb_samples_text)
+    content.add_row(pass_text)
+    content.add_row(items_text)
+    content.add_row(rate_text)
+
+    if task_times and scoring_times:
+        avg_task = sum(task_times) / len(task_times)
+        avg_scoring = sum(scoring_times) / len(scoring_times)
+        avg_total = avg_task + avg_scoring
+
+        avg_task_text = text.Text(f"Avg task time:     {avg_task:.2f}s")
+        avg_task_text.stylize("bold", 0, 18)
+        avg_scoring_text = text.Text(f"Avg scoring time:  {avg_scoring:.2f}s")
+        avg_scoring_text.stylize("bold", 0, 18)
+        avg_total_text = text.Text(f"Avg total time:    {avg_total:.2f}s")
+        avg_total_text.stylize("bold", 0, 18)
+
+        content.add_row(text.Text(""))
+        content.add_row(align.Align.left(avg_task_text))
+        content.add_row(align.Align.left(avg_scoring_text))
+        content.add_row(align.Align.left(avg_total_text))
+
+    if verbose >= 2 and assertion_total_count:
+        sorted_assertions = sorted(
+            assertion_total_count.keys(),
+            key=lambda n: (
+                assertion_passed_count[n] / assertion_total_count[n]
+                if assertion_total_count[n] > 0
+                else 0.0
+            ),
+        )
+
+        score_strings = text.Text("")
+        for name in sorted_assertions:
+            passed = assertion_passed_count[name]
+            total = assertion_total_count[name]
+            rate = passed / total if total > 0 else 0.0
+            style = "green bold" if passed == total else "red bold"
+            score_strings += text.Text(
+                f"{name}: {rate:.0%} passed ({passed}/{total})\n",
+                style=style,
+            )
+
+        content.add_row(text.Text(""))
+        content.add_row(align.Align.left(score_strings))
+
+    panel_content = panel.Panel(
+        content,
+        title=f"{suite_name} ({nb_items} items, {nb_runs} runs)",
+        title_align="left",
+        expand=False,
+    )
+
+    console_container = console.Console()
+    console_container.print(panel_content)

sdks/python/src/opik/api_objects/dataset/test_suite/report_processors/file_writer.py

@@ -0,0 +1,59 @@
+"""Save structured JSON report files for test suite results."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .. import test_suite_result as _result_mod
+
+LOGGER = logging.getLogger(__name__)
+
+DEFAULT_REPORT_DIR = "opik_test_suite_reports"
+
+
+def save_report(
+    suite_result: _result_mod.TestSuiteResult,
+    output_path: Optional[str] = None,
+) -> str:
+    """Save a test suite result as a structured JSON report file.
+
+    Args:
+        suite_result: The test suite result to serialize.
+        output_path: Optional file path. If not provided, a default path
+            is generated under the ``opik_test_suite_reports/`` directory.
+
+    Returns:
+        The absolute path to the written report file.
+    """
+    report_dict = suite_result.to_report_dict()
+
+    if output_path is None:
+        output_path = build_default_report_path(
+            suite_result.experiment_name or suite_result.experiment_id
+        )
+
+    output_path = os.path.abspath(output_path)
+    parent_dir = os.path.dirname(output_path)
+    if parent_dir:
+        os.makedirs(parent_dir, exist_ok=True)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(report_dict, f, indent=2, default=str, ensure_ascii=False)
+
+    LOGGER.debug("Test suite report saved to %s", output_path)
+    return output_path
+
+
+def build_default_report_path(experiment_name: str) -> str:
+    """Build a default report file path from an experiment name."""
+    safe_name = _sanitize_filename(experiment_name)
+    return os.path.abspath(os.path.join(DEFAULT_REPORT_DIR, f"{safe_name}.json"))
+
+
+def _sanitize_filename(name: str) -> str:
+    """Replace characters that are unsafe in file names."""
+    return "".join(c if c.isalnum() or c in "-_." else "_" for c in name)

sdks/python/src/opik/api_objects/dataset/test_suite/suite_result_constructor.py

@@ -0,0 +1,93 @@
+"""
+Suite result construction logic for test suites.
+
+This module handles building TestSuiteResult from raw evaluation results,
+including pass/fail determination based on execution policies.
+"""
+
+from collections import defaultdict
+from typing import Dict, List, Optional
+
+from opik.api_objects.dataset import dataset_item
+from opik.evaluation import evaluation_result, test_result
+
+from . import types as suite_types
+from .test_suite_result import is_score_passed
+
+
+def build_suite_result(
+    eval_result: evaluation_result.EvaluationResult,
+    suite_name: Optional[str] = None,
+    total_time: Optional[float] = None,
+) -> suite_types.TestSuiteResult:
+    """
+    Build a TestSuiteResult from an EvaluationResult.
+
+    Groups test results by dataset item and computes pass/fail status
+    based on execution policies stored in each item.
+
+    Pass/fail logic:
+    - A RUN passes if all its assertion scores pass (value=True or value=1)
+    - An ITEM passes if runs_passed >= pass_threshold
+    - The SUITE passes if all items pass
+
+    Args:
+        eval_result: The raw evaluation result from the evaluation engine.
+
+    Returns:
+        TestSuiteResult with pass/fail status for each item and the suite.
+    """
+    results_by_item: Dict[str, List[test_result.TestResult]] = defaultdict(list)
+    items_cache: Dict[str, Optional[dataset_item.DatasetItem]] = {}
+
+    for result in eval_result.test_results:
+        item_id = result.test_case.dataset_item_id
+        results_by_item[item_id].append(result)
+        if item_id not in items_cache:
+            items_cache[item_id] = result.test_case.dataset_item
+
+    item_results: Dict[str, suite_types.ItemResult] = {}
+    items_passed = 0
+
+    for item_id, item_test_results in results_by_item.items():
+        item = items_cache.get(item_id)
+        pass_threshold = 1
+        configured_runs_per_item = 1
+        if item is not None and item.execution_policy is not None:
+            if item.execution_policy.pass_threshold is not None:
+                pass_threshold = item.execution_policy.pass_threshold
+            if item.execution_policy.runs_per_item is not None:
+                configured_runs_per_item = item.execution_policy.runs_per_item
+
+        has_assertions = any(r.score_results for r in item_test_results)
+
+        runs_passed = sum(
+            1
+            for r in item_test_results
+            if not r.score_results or all(is_score_passed(s) for s in r.score_results)
+        )
+
+        passed = runs_passed >= pass_threshold
+
+        if passed:
+            items_passed += 1
+
+        item_results[item_id] = suite_types.ItemResult(
+            dataset_item_id=item_id,
+            passed=passed,
+            has_assertions=has_assertions,
+            runs_passed=runs_passed,
+            runs_total=len(item_test_results),
+            configured_runs_per_item=configured_runs_per_item,
+            pass_threshold=pass_threshold,
+            test_results=sorted(item_test_results, key=lambda r: r.trial_id),
+        )
+
+    return suite_types.TestSuiteResult(
+        items_passed=items_passed,
+        items_total=len(results_by_item),
+        item_results=item_results,
+        evaluation_result_=eval_result,
+        suite_name=suite_name,
+        total_time=total_time,
+    )

sdks/python/src/opik/api_objects/dataset/test_suite/test_suite.py

@@ -0,0 +1,741 @@
+"""
+Test Suite API for regression testing LLM applications.
+
+Test Suites are pre-configured regression test suites that let you
+validate that prompt changes, model updates, or code modifications don't
+break existing functionality.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from opik.api_objects import opik_client as opik_client_module
+    from opik.evaluation.suite_evaluators.llm_judge import LLMJudge
+
+import datetime
+
+from opik import exceptions as opik_exceptions
+from opik.api_objects.prompt import base_prompt
+from opik.api_objects.dataset import dataset
+from opik.rest_api.types import dataset_version_public
+from . import types as suite_types, converters
+from .. import validators, execution_policy, rest_operations
+
+
+LOGGER = logging.getLogger(__name__)
+
+LLMTask = Callable[[Dict[str, Any]], Any]
+
+
+def _evaluators_equal(a: List[LLMJudge], b: List[LLMJudge]) -> bool:
+    """Compare two lists of LLMJudge evaluators by their assertion sets."""
+    a_assertions = sorted(assertion for e in a for assertion in e.assertions)
+    b_assertions = sorted(assertion for e in b for assertion in e.assertions)
+    return a_assertions == b_assertions
+
+
+def validate_task_result(
+    result: Any,
+    input_data: Any = None,
+) -> Dict[str, Any]:
+    """Normalise the value returned by a task function into a result dict.
+
+    If *result* is already a :class:`dict`, it is returned as-is (the
+    supported keys are ``"input"`` and ``"output"``).
+
+    For any other type the value is wrapped automatically::
+
+        {"output": result}
+
+    When *input_data* is also provided the wrapper becomes::
+
+        {"input": input_data, "output": result}
+
+    Args:
+        result: Value returned by the task callable.
+        input_data: Optional input that was passed to the task. Included in
+            the wrapper dict as ``"input"`` when *result* is not a dict.
+
+    Returns:
+        A dict suitable for use as an experiment trace result.
+    """
+    if isinstance(result, dict):
+        missing = {"input", "output"} - result.keys()
+        if missing:
+            raise ValueError(
+                f"The task function must return a dict with 'input' and "
+                f"'output' keys, but the returned dict is missing: "
+                f"{missing}. Got keys: {set(result.keys())}. "
+                f"Example: return {{'input': data, 'output': response}}"
+            )
+        return result
+
+    wrapped: Dict[str, Any] = {"output": result}
+    if input_data is not None:
+        wrapped["input"] = input_data
+    return wrapped
+
+
+class TestSuiteVersion:
+    """
+    A read-only view of a specific test suite version.
+
+    Provides access to suite items, assertions, and execution policy at a
+    specific version point in time. Does not allow mutations.
+
+    Obtain an instance via :meth:`TestSuite.get_version_view`.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        dataset_version: dataset.DatasetVersion,
+        version_info: dataset_version_public.DatasetVersionPublic,
+    ) -> None:
+        self._name = name
+        self._dataset_version = dataset_version
+        self._version_info = version_info
+
+    @property
+    def name(self) -> str:
+        """The name of the test suite this version belongs to."""
+        return self._name
+
+    @property
+    def id(self) -> str:
+        """The dataset ID of the test suite."""
+        return self._dataset_version.dataset_id
+
+    @property
+    def version_name(self) -> Optional[str]:
+        """The sequential version name (e.g., 'v1', 'v2')."""
+        return self._version_info.version_name
+
+    @property
+    def version_id(self) -> Optional[str]:
+        """The unique identifier of this specific version."""
+        return self._version_info.id
+
+    @property
+    def is_latest(self) -> Optional[bool]:
+        """Whether this is the latest version."""
+        return self._version_info.is_latest
+
+    @property
+    def items_total(self) -> Optional[int]:
+        """Total number of items in this version."""
+        return self._version_info.items_total
+
+    @property
+    def version_hash(self) -> Optional[str]:
+        """The unique hash identifier of this version."""
+        return self._version_info.version_hash
+
+    @property
+    def tags(self) -> Optional[List[str]]:
+        """Tags associated with this version."""
+        return self._version_info.tags
+
+    @property
+    def items_added(self) -> Optional[int]:
+        """Number of items added since the previous version."""
+        return self._version_info.items_added
+
+    @property
+    def items_modified(self) -> Optional[int]:
+        """Number of items modified since the previous version."""
+        return self._version_info.items_modified
+
+    @property
+    def items_deleted(self) -> Optional[int]:
+        """Number of items deleted since the previous version."""
+        return self._version_info.items_deleted
+
+    @property
+    def change_description(self) -> Optional[str]:
+        """Description of changes in this version."""
+        return self._version_info.change_description
+
+    @property
+    def created_at(self) -> Optional[datetime.datetime]:
+        """Timestamp when this version was created."""
+        return self._version_info.created_at
+
+    @property
+    def created_by(self) -> Optional[str]:
+        """User who created this version."""
+        return self._version_info.created_by
+
+    @property
+    def project_name(self) -> Optional[str]:
+        """The project name associated with the test suite."""
+        return self._dataset_version.project_name
+
+    @property
+    def __internal_api__dataset_version__(self) -> dataset.DatasetVersion:
+        """Internal access to the underlying dataset version. Not part of the public API."""
+        return self._dataset_version
+
+    def get_items(
+        self,
+        nb_samples: Optional[int] = None,
+        filter_string: Optional[str] = None,
+    ) -> List[suite_types.TestSuiteItem]:
+        """
+        Retrieve suite items at this version as a list of dictionaries.
+
+        Args:
+            nb_samples: Maximum number of items to retrieve.
+            filter_string: Optional OQL filter string.
+
+        Returns:
+            A list of item dicts with keys: id, data, description,
+            assertions, execution_policy.
+        """
+        return [
+            converters.dataset_item_to_suite_item_dict(item)
+            for item in self._dataset_version.__internal_api__stream_items_as_dataclasses__(
+                nb_samples=nb_samples,
+                filter_string=filter_string,
+            )
+        ]
+
+    def get_global_assertions(self) -> List[str]:
+        """
+        Get the suite-level assertions stored in this version.
+
+        Returns:
+            List of assertion strings.
+        """
+        return converters.version_evaluators_to_assertions(
+            self._version_info.evaluators
+        )
+
+    def get_global_execution_policy(self) -> execution_policy.ExecutionPolicy:
+        """
+        Get the suite-level execution policy stored in this version.
+
+        Returns:
+            ExecutionPolicy dict with runs_per_item and pass_threshold.
+        """
+        return converters.version_policy_to_execution_policy(
+            self._version_info.execution_policy
+        )
+
+
+class TestSuite:
+    """
+    A pre-configured regression test suite for LLM applications.
+
+    Test Suites let you:
+    - Define test cases with inputs and context
+    - Configure assertions that will be checked by an LLM
+    - Run tests against any task function
+
+    Suite-level assertions and execution policy are stored in the dataset's
+    metadata and read by the evaluation engine when running the suite.
+
+    Example:
+        >>> import opik
+        >>>
+        >>> client = opik.Opik()
+        >>>
+        >>> suite = client.create_test_suite(
+        ...     name="Refund Policy Tests",
+        ...     description="Regression tests for refund scenarios",
+        ...     global_assertions=[
+        ...         "Response does not contain hallucinated information",
+        ...         "Response is helpful to the user",
+        ...     ],
+        ... )
+        >>>
+        >>> suite.insert([
+        ...     {
+        ...         "data": {"user_input": "How do I get a refund?", "user_tier": "premium"},
+        ...         "assertions": ["Response is polite"],
+        ...     },
+        ... ])
+        >>>
+        >>> results = opik.run_tests(test_suite=suite, task=my_llm_function)
+    """
+
+    def __init__(
+        self,
+        name: str,
+        dataset_: dataset.Dataset,
+        client: Optional["opik_client_module.Opik"] = None,
+    ):
+        """
+        Internal constructor — not part of the public API.
+
+        Use :meth:`opik.Opik.create_test_suite` or
+        :meth:`opik.Opik.get_or_create_test_suite` instead.
+        """
+        self._name = name
+        self._dataset = dataset_
+        self._client = client
+
+    @property
+    def id(self) -> str:
+        """The ID of the test suite."""
+        return self._dataset.id
+
+    @property
+    def name(self) -> str:
+        """The name of the test suite."""
+        return self._name
+
+    @property
+    def description(self) -> Optional[str]:
+        """The description of the test suite."""
+        return self._dataset.description
+
+    @property
+    def project_name(self) -> Optional[str]:
+        """The project name associated with the test suite."""
+        return self._dataset.project_name
+
+    @property
+    def __internal_api__dataset__(self) -> dataset.Dataset:
+        """Internal access to the underlying dataset. Not part of the public API."""
+        return self._dataset
+
+    @property
+    def items_count(self) -> Optional[int]:
+        """The total number of items in the test suite."""
+        return self._dataset.dataset_items_count
+
+    def get_tags(self) -> List[str]:
+        """
+        Get the tags for the suite.
+
+        Returns:
+            List of tag strings.
+        """
+        return self._dataset.get_tags()
+
+    def get_current_version_name(self) -> Optional[str]:
+        """
+        Get the current version name of the test suite.
+
+        Returns:
+            The current version name (e.g., 'v1', 'v2'), or None if
+            no version exists.
+        """
+        return self._dataset.get_current_version_name()
+
+    def get_version_info(
+        self,
+    ) -> Optional[dataset_version_public.DatasetVersionPublic]:
+        """
+        Get version information for the current (latest) version.
+
+        Returns:
+            DatasetVersionPublic containing the current version's metadata,
+            or None if no version exists yet.
+        """
+        return self._dataset.get_version_info()
+
+    def get_version_view(self, version_name: str) -> TestSuiteVersion:
+        """
+        Get a read-only view of a specific version.
+
+        Args:
+            version_name: The version name (e.g., 'v1', 'v2').
+
+        Returns:
+            A read-only TestSuiteVersion for accessing the specified
+            version's items, assertions, and execution policy.
+
+        Raises:
+            opik.exceptions.DatasetVersionNotFound: If the version does not
+                exist.
+        """
+        version_info = rest_operations.find_version_by_name(
+            rest_client=self._dataset._rest_client,
+            dataset_id=self._dataset.id,
+            version_name=version_name,
+        )
+        if version_info is None:
+            raise opik_exceptions.DatasetVersionNotFound(
+                f"Version '{version_name}' not found in test suite '{self._name}'"
+            )
+
+        dataset_version = dataset.DatasetVersion(
+            dataset_name=self._name,
+            dataset_id=self._dataset.id,
+            rest_client=self._dataset._rest_client,
+            version_info=version_info,
+            project_name=self._dataset.project_name,
+            client=self._dataset.client,
+        )
+        return TestSuiteVersion(
+            name=self._name,
+            dataset_version=dataset_version,
+            version_info=version_info,
+        )
+
+    def get_items(
+        self,
+        nb_samples: Optional[int] = None,
+        filter_string: Optional[str] = None,
+    ) -> List[suite_types.TestSuiteItem]:
+        """
+        Retrieve suite items as a list of dictionaries.
+
+        Each item dict has keys: ``id``, ``data``, ``description``,
+        ``assertions``, ``execution_policy``.
+
+        Args:
+            nb_samples: Maximum number of items to retrieve.
+                If None, all items are returned.
+            filter_string: Optional OQL filter string to filter items.
+
+        Returns:
+            A list of item dictionaries.
+        """
+        return [
+            converters.dataset_item_to_suite_item_dict(item)
+            for item in self._dataset.__internal_api__stream_items_as_dataclasses__(
+                nb_samples=nb_samples,
+                filter_string=filter_string,
+            )
+        ]
+
+    def to_pandas(self) -> "pd.DataFrame":
+        """
+        Convert the test suite items to a pandas DataFrame.
+
+        Requires the ``pandas`` library to be installed.
+
+        Returns:
+            A pandas DataFrame containing all items with columns
+            such as ``id``, ``data``, ``assertions``, ``description``,
+            and ``execution_policy``.
+        """
+        return converters.to_pandas(self.get_items())
+
+    def to_json(self) -> str:
+        """
+        Convert the test suite items to a JSON string.
+
+        Returns:
+            A JSON string representation of all items.
+        """
+        return converters.to_json(self.get_items())
+
+    def insert_from_json(
+        self,
+        json_array: str,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Insert test suite items from a JSON string.
+
+        Each JSON object must map to valid test suite item keys after
+        applying ``keys_mapping``:
+
+        - ``data`` (required) — dict of test case inputs
+        - ``assertions`` — list of assertion strings
+        - ``description`` — item description
+        - ``execution_policy`` — dict with ``runs_per_item`` and ``pass_threshold``
+        - ``id`` — item identifier (auto-generated if omitted)
+
+        Args:
+            json_array: JSON string of format ``[{...}, {...}]``.
+            keys_mapping: Maps JSON keys to the target keys listed above.
+                Example: ``{"test_data": "data", "checks": "assertions"}``
+            ignore_keys: Keys in the JSON dicts to skip during import.
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        self.insert(converters.from_json(json_array, keys_mapping, ignore_keys))
+
+    def insert_from_pandas(
+        self,
+        dataframe: "pd.DataFrame",
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Insert test suite items from a pandas DataFrame.
+
+        Requires the ``pandas`` library to be installed.
+
+        Each DataFrame row must map to valid test suite item keys after
+        applying ``keys_mapping``:
+
+        - ``data`` (required) — dict of test case inputs
+        - ``assertions`` — list of assertion strings
+        - ``description`` — item description
+        - ``execution_policy`` — dict with ``runs_per_item`` and ``pass_threshold``
+        - ``id`` — item identifier (auto-generated if omitted)
+
+        Args:
+            dataframe: pandas DataFrame.
+            keys_mapping: Maps column names to the target keys listed above.
+                Example: ``{"test_data": "data", "checks": "assertions"}``
+            ignore_keys: Column names in the DataFrame to skip during import.
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        self.insert(converters.from_pandas(dataframe, keys_mapping, ignore_keys))
+
+    def insert_from_jsonl_file(
+        self,
+        file_path: str,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Read JSONL from a file and insert items into the test suite.
+
+        Each line must be a JSON object that maps to valid test suite item
+        keys after applying ``keys_mapping``:
+
+        - ``data`` (required) — dict of test case inputs
+        - ``assertions`` — list of assertion strings
+        - ``description`` — item description
+        - ``execution_policy`` — dict with ``runs_per_item`` and ``pass_threshold``
+        - ``id`` — item identifier (auto-generated if omitted)
+
+        Args:
+            file_path: Path to the JSONL file.
+            keys_mapping: Maps JSON keys to the target keys listed above.
+                Example: ``{"test_data": "data", "checks": "assertions"}``
+            ignore_keys: Keys in the JSON objects to skip during import.
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        self.insert(converters.from_jsonl_file(file_path, keys_mapping, ignore_keys))
+
+    def update_test_settings(
+        self,
+        *,
+        global_execution_policy: Optional[execution_policy.ExecutionPolicy] = None,
+        global_assertions: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Update the suite-level assertions and/or execution policy.
+
+        Supports partial updates: any parameter not provided will retain
+        its current value. If the new values are identical to the current
+        values, no new version is created.
+
+        Args:
+            global_execution_policy: New execution policy for the suite.
+                If not provided, the current policy is kept.
+            global_assertions: New suite-level assertions. Each string
+                describes an expected behavior that will be checked by an
+                LLM. If not provided, the current assertions are kept.
+
+        Raises:
+            ValueError: If nothing to update is provided.
+        """
+        if global_execution_policy is not None:
+            validators.validate_execution_policy(global_execution_policy)
+
+        resolved = validators.resolve_evaluators(
+            global_assertions, None, "suite-level assertions"
+        )
+
+        if resolved is None and global_execution_policy is None:
+            raise ValueError(
+                "At least one of 'global_assertions' or "
+                "'global_execution_policy' must be provided."
+            )
+
+        version_info = self._dataset.get_version_info()
+
+        if version_info is None:
+            new_evaluators = resolved or []
+            new_policy = (
+                global_execution_policy
+                or execution_policy.DEFAULT_EXECUTION_POLICY.copy()
+            )
+            rest_operations.create_initial_test_suite_version(
+                rest_client=self._dataset._rest_client,
+                dataset_id=self._dataset.id,
+                evaluators=new_evaluators,
+                exec_policy=new_policy,
+            )
+            return
+
+        current_evaluators = self._dataset.get_evaluators()
+        current_policy = self.get_global_execution_policy()
+
+        new_evaluators = resolved if resolved is not None else current_evaluators
+        new_policy = (
+            global_execution_policy
+            if global_execution_policy is not None
+            else current_policy
+        )
+
+        if (
+            _evaluators_equal(new_evaluators, current_evaluators)
+            and new_policy == current_policy
+        ):
+            return
+
+        change_parts: List[str] = []
+        if resolved is not None:
+            change_parts.append("assertions")
+        if global_execution_policy is not None:
+            change_parts.append("execution policy")
+
+        rest_operations.update_test_suite_dataset(
+            rest_client=self._dataset._rest_client,
+            dataset_id=self._dataset.id,
+            base_version_id=version_info.id,
+            evaluators=new_evaluators,
+            exec_policy=new_policy,
+            change_description=f"Updated {' and '.join(change_parts)} via SDK",
+        )
+
+    def delete(self, items_ids: List[str]) -> None:
+        """
+        Delete items from the test suite by their IDs.
+
+        Args:
+            items_ids: List of item IDs to delete.
+        """
+        self._dataset.delete(items_ids)
+
+    def clear(self) -> None:
+        """
+        Delete all items from the test suite.
+        """
+        item_ids = [
+            item.id
+            for item in self._dataset.__internal_api__stream_items_as_dataclasses__()
+            if item.id is not None
+        ]
+        if item_ids:
+            self._dataset.delete(item_ids)
+
+    def get_global_execution_policy(self) -> execution_policy.ExecutionPolicy:
+        """
+        Get the suite-level execution policy.
+
+        Returns:
+            ExecutionPolicy dict with runs_per_item and pass_threshold.
+        """
+        return self._dataset.get_execution_policy()
+
+    def get_global_assertions(self) -> List[str]:
+        """
+        Get the suite-level assertions.
+
+        Returns:
+            List of assertion strings.
+        """
+        return converters.evaluators_to_assertions(self._dataset.get_evaluators())
+
+    def update(
+        self,
+        items: List[suite_types.TestSuiteItem],
+    ) -> None:
+        """
+        Update existing items in the test suite.
+
+        Each item dict must include an ``"id"`` key identifying the item to
+        update.  The remaining keys (``"data"``, ``"assertions"``,
+        ``"description"``, ``"execution_policy"``) replace the previous values.
+
+        Args:
+            items: List of item dicts to update. Each must contain ``"id"``.
+
+        Raises:
+            DatasetItemUpdateOperationRequiresItemId: If any item is missing
+                an ``"id"`` key.
+        """
+        for item in items:
+            if "id" not in item:
+                raise opik_exceptions.DatasetItemUpdateOperationRequiresItemId(
+                    "Missing id for test suite item to update: %s", item
+                )
+
+        self.insert(items)
+
+    def insert(
+        self,
+        items: List[suite_types.TestSuiteItem],
+    ) -> None:
+        """
+        Insert test cases into the test suite.
+
+        Args:
+            items: List of test case items to add.
+
+        Example:
+            >>> suite.insert([
+            ...     {"data": {"question": "How do I get a refund?"}},
+            ...     {
+            ...         "data": {"question": "Is my account hacked?"},
+            ...         "assertions": ["Response treats the concern with urgency"],
+            ...         "execution_policy": {"runs_per_item": 5, "pass_threshold": 4},
+            ...     },
+            ... ])
+        """
+        validators.validate_suite_items(items)
+
+        ds_items = [converters.suite_item_dict_to_dataset_item(item) for item in items]
+        self._dataset.__internal_api__insert_items_as_dataclasses__(ds_items)
+
+    def __internal_api__run_optimization_suite__(
+        self,
+        task: LLMTask,
+        *,
+        experiment_name_prefix: Optional[str] = None,
+        experiment_name: Optional[str] = None,
+        project_name: Optional[str] = None,
+        experiment_config: Optional[Dict[str, Any]] = None,
+        prompts: Optional[List[base_prompt.BasePrompt]] = None,
+        experiment_tags: Optional[List[str]] = None,
+        verbose: int = 2,
+        worker_threads: int = 16,
+        model: Optional[str] = None,
+        optimization_id: Optional[str] = None,
+        experiment_type: Optional[str] = None,
+        dataset_item_ids: Optional[List[str]] = None,
+        dataset_filter_string: Optional[str] = None,
+        client: Optional["opik_client_module.Opik"] = None,
+        generate_report: bool = True,
+        report_output_path: Optional[str] = None,
+        blueprint_id: Optional[str] = None,
+    ) -> suite_types.TestSuiteResult:
+        """
+        Internal entry point used by the optimizer framework.
+        """
+        from opik.evaluation.evaluator import __internal_api__run_test_suite__
+
+        return __internal_api__run_test_suite__(
+            suite_dataset=self._dataset,
+            task=task,
+            client=client,
+            dataset_item_ids=dataset_item_ids,
+            dataset_filter_string=dataset_filter_string,
+            experiment_name_prefix=experiment_name_prefix,
+            experiment_name=experiment_name,
+            project_name=project_name,
+            experiment_config=experiment_config,
+            prompts=prompts,
+            experiment_tags=experiment_tags,
+            verbose=verbose,
+            task_threads=worker_threads,
+            evaluator_model=model,
+            optimization_id=optimization_id,
+            experiment_type=experiment_type,
+            generate_report=generate_report,
+            report_output_path=report_output_path,
+            blueprint_id=blueprint_id,
+        )

sdks/python/src/opik/api_objects/dataset/test_suite/test_suite_result.py

@@ -0,0 +1,218 @@
+"""TestSuiteResult and ItemResult types."""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from opik.evaluation import evaluation_result, test_result
+    from opik.evaluation.metrics.score_result import ScoreResult
+
+
+def is_score_passed(score: ScoreResult) -> bool:
+    """Determine whether a score result represents a passing assertion."""
+    if score.scoring_failed:
+        return False
+    return (isinstance(score.value, bool) and score.value) or score.value == 1
+
+
+@dataclasses.dataclass
+class ItemResult:
+    """Result for a single test suite item."""
+
+    dataset_item_id: str
+    """The ID of the dataset item."""
+
+    passed: bool
+    """Whether this item passed based on its execution policy."""
+
+    has_assertions: bool
+    """Whether this item had any assertions evaluated."""
+
+    runs_passed: int
+    """Number of runs that passed for this item."""
+
+    runs_total: int
+    """Total number of runs completed for this item."""
+
+    configured_runs_per_item: int
+    """Configured runs_per_item from the execution policy."""
+
+    pass_threshold: int
+    """Minimum passing runs required (from execution policy)."""
+
+    test_results: List[test_result.TestResult]
+    """Individual test results for each run of this item."""
+
+
+class TestSuiteResult:
+    """
+    Result of running a test suite.
+
+    Contains pass/fail status for each item based on execution policy,
+    as well as overall suite pass/fail status.
+    """
+
+    def __init__(
+        self,
+        items_passed: int,
+        items_total: int,
+        item_results: Dict[str, ItemResult],
+        evaluation_result_: evaluation_result.EvaluationResult,
+        suite_name: Optional[str] = None,
+        total_time: Optional[float] = None,
+    ) -> None:
+        self._items_passed = items_passed
+        self._items_total = items_total
+        self._item_results = item_results
+        self._evaluation_result = evaluation_result_
+        self._suite_name = suite_name
+        self._total_time = total_time
+
+    @property
+    def all_items_passed(self) -> bool:
+        """Whether all items in the suite passed."""
+        return self._items_passed == self._items_total
+
+    @property
+    def items_passed(self) -> int:
+        """Number of items that passed."""
+        return self._items_passed
+
+    @property
+    def items_total(self) -> int:
+        """Total number of items evaluated."""
+        return self._items_total
+
+    @property
+    def item_results(self) -> Dict[str, ItemResult]:
+        """Results for each item, keyed by dataset_item_id."""
+        return self._item_results
+
+    @property
+    def pass_rate(self) -> Optional[float]:
+        """Pass rate among items that had assertions.
+
+        Items without any assertions are excluded from the calculation.
+        Returns None if no items had assertions.
+        """
+        items_with_assertions = [
+            r for r in self._item_results.values() if r.has_assertions
+        ]
+        if not items_with_assertions:
+            return None
+        passed = sum(1 for r in items_with_assertions if r.passed)
+        return passed / len(items_with_assertions)
+
+    @property
+    def suite_name(self) -> Optional[str]:
+        """The name of the test suite."""
+        return self._suite_name
+
+    @property
+    def total_time(self) -> Optional[float]:
+        """Total evaluation time in seconds."""
+        return self._total_time
+
+    @property
+    def experiment_id(self) -> str:
+        """The experiment ID."""
+        return self._evaluation_result.experiment_id
+
+    @property
+    def experiment_name(self) -> Optional[str]:
+        """The experiment name."""
+        return self._evaluation_result.experiment_name
+
+    @property
+    def experiment_url(self) -> Optional[str]:
+        """URL to view the experiment."""
+        return self._evaluation_result.experiment_url
+
+    def to_report_dict(self) -> Dict[str, Any]:
+        """Convert the result to a structured report dictionary."""
+        from datetime import datetime, timezone
+
+        items: List[Dict[str, Any]] = []
+
+        for item_id, item_result in self._item_results.items():
+            runs: List[Dict[str, Any]] = []
+
+            for test_result_ in item_result.test_results:
+                assertions: List[Dict[str, Any]] = []
+                for score in test_result_.score_results:
+                    assertion: Dict[str, Any] = {
+                        "name": score.name,
+                        "passed": is_score_passed(score),
+                        "value": score.value,
+                        "scoring_failed": score.scoring_failed,
+                    }
+                    if score.reason is not None:
+                        assertion["reason"] = score.reason
+                    if score.metadata is not None:
+                        assertion["metadata"] = score.metadata
+                    assertions.append(assertion)
+
+                run_passed = (
+                    all(a["passed"] for a in assertions) if assertions else True
+                )
+
+                run: Dict[str, Any] = {
+                    "trial_id": test_result_.trial_id,
+                    "passed": run_passed,
+                    "input": test_result_.test_case.task_output.get("input"),
+                    "output": test_result_.test_case.task_output.get("output"),
+                    "assertions": assertions,
+                }
+                if test_result_.test_case.trace_id:
+                    run["trace_id"] = test_result_.test_case.trace_id
+                if test_result_.task_execution_time is not None:
+                    run["task_execution_time_seconds"] = round(
+                        test_result_.task_execution_time, 3
+                    )
+                if test_result_.scoring_time is not None:
+                    run["scoring_time_seconds"] = round(test_result_.scoring_time, 3)
+                runs.append(run)
+
+            items.append(
+                {
+                    "dataset_item_id": item_id,
+                    "passed": item_result.passed,
+                    "runs_passed": item_result.runs_passed,
+                    "execution_policy": {
+                        "runs_per_item": item_result.configured_runs_per_item,
+                        "pass_threshold": item_result.pass_threshold,
+                    },
+                    "runs": runs,
+                }
+            )
+
+        report: Dict[str, Any] = {
+            "suite_passed": self.all_items_passed,
+            "items_passed": self._items_passed,
+            "items_total": self._items_total,
+            "pass_rate": self.pass_rate,
+            "experiment_id": self.experiment_id,
+        }
+
+        if self._suite_name is not None:
+            report["suite_name"] = self._suite_name
+
+        if self.experiment_name is not None:
+            report["experiment_name"] = self.experiment_name
+
+        if self.experiment_url is not None:
+            report["experiment_url"] = self.experiment_url
+
+        if self._total_time is not None:
+            report["total_time_seconds"] = round(self._total_time, 3)
+
+        report["generated_at"] = datetime.now(timezone.utc).isoformat()
+        report["items"] = items
+
+        return report
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Alias for to_report_dict()."""
+        return self.to_report_dict()

sdks/python/src/opik/api_objects/dataset/test_suite/types.py

@@ -0,0 +1,25 @@
+"""Type definitions for test suite."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, TypedDict
+
+from typing_extensions import Required
+
+from ..execution_policy import ExecutionPolicy
+from .test_suite_result import TestSuiteResult, ItemResult
+
+__all__ = ["TestSuiteItem", "TestSuiteResult", "ItemResult"]
+
+
+class TestSuiteItem(TypedDict, total=False):
+    """A test case item for a test suite.
+
+    Used for both inserting new items and reading existing ones.
+    """
+
+    id: str
+    data: Required[Dict[str, Any]]
+    assertions: List[str]
+    description: str
+    execution_policy: ExecutionPolicy

sdks/python/src/opik/api_objects/dataset/validators.py

@@ -0,0 +1,135 @@
+"""Validators for dataset and test suite operations."""
+
+from __future__ import annotations
+
+from typing import Any, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from opik.evaluation.suite_evaluators.llm_judge import LLMJudge
+
+
+def validate_evaluators(evaluators: List[Any], context: str) -> None:
+    """
+    Validate that all evaluators are LLMJudge instances.
+
+    Args:
+        evaluators: List of evaluators to validate.
+        context: Description of where the evaluators are being used (for error message).
+
+    Raises:
+        TypeError: If any evaluator is not an LLMJudge instance.
+    """
+    from opik.evaluation.suite_evaluators import llm_judge
+
+    for evaluator in evaluators:
+        if not isinstance(evaluator, llm_judge.LLMJudge):
+            raise TypeError(
+                f"Test suites only support LLMJudge evaluators. "
+                f"Got {type(evaluator).__name__} in {context}. "
+                f"Use LLMJudge from opik.evaluation.suite_evaluators instead."
+            )
+
+
+def resolve_evaluators(
+    assertions: Optional[List[str]],
+    evaluators: Optional[List[Any]],
+    context: str,
+) -> Optional[List[LLMJudge]]:
+    """
+    Resolve assertions shorthand and/or evaluators into a list of LLMJudge instances.
+
+    Args:
+        assertions: List of assertion strings to build an LLMJudge from.
+        evaluators: List of pre-built LLMJudge evaluators.
+        context: Description of where this is used (for error messages).
+
+    Returns:
+        A list of LLMJudge instances, or None if neither was provided.
+
+    Raises:
+        ValueError: If both assertions and evaluators are provided.
+        TypeError: If any evaluator is not an LLMJudge instance.
+    """
+    if assertions is not None and evaluators is not None:
+        raise ValueError(
+            f"Cannot specify both 'assertions' and 'evaluators' for {context}. "
+            f"Use 'assertions' for a shorthand or 'evaluators' for full control, "
+            f"but not both."
+        )
+
+    if assertions is not None:
+        if not assertions:
+            return []
+        from opik.evaluation.suite_evaluators import llm_judge
+
+        return [llm_judge.LLMJudge(assertions=assertions)]
+
+    if evaluators is not None:
+        validate_evaluators(evaluators, context)
+        return evaluators
+
+    return None
+
+
+_VALID_ITEM_KEYS = {"id", "data", "assertions", "description", "execution_policy"}
+_VALID_EXECUTION_POLICY_KEYS = {"runs_per_item", "pass_threshold"}
+
+
+def validate_execution_policy(ep: Any, context: str = "execution_policy") -> None:
+    if not isinstance(ep, dict):
+        raise TypeError(f"'{context}' must be a dict, got {type(ep).__name__}")
+    unknown_keys = set(ep.keys()) - _VALID_EXECUTION_POLICY_KEYS
+    if unknown_keys:
+        raise ValueError(
+            f"'{context}' has unknown keys: {unknown_keys}. "
+            f"Valid keys are: {_VALID_EXECUTION_POLICY_KEYS}"
+        )
+    missing_keys = _VALID_EXECUTION_POLICY_KEYS - set(ep.keys())
+    if missing_keys:
+        raise ValueError(
+            f"'{context}' is missing required keys: {missing_keys}. "
+            f"Both 'runs_per_item' and 'pass_threshold' must be provided."
+        )
+    for key in ep:
+        if not isinstance(ep[key], int):
+            raise TypeError(
+                f"'{context}.{key}' must be an int, got {type(ep[key]).__name__}"
+            )
+
+
+def validate_suite_items(items: List[Any]) -> None:
+    for i, item in enumerate(items):
+        if not isinstance(item, dict):
+            raise TypeError(
+                f"Item at index {i} must be a dict, got {type(item).__name__}"
+            )
+
+        unknown_keys = set(item.keys()) - _VALID_ITEM_KEYS
+        if unknown_keys:
+            raise ValueError(
+                f"Item at index {i} has unknown keys: {unknown_keys}. "
+                f"Valid keys are: {_VALID_ITEM_KEYS}"
+            )
+
+        if "data" not in item:
+            raise ValueError(f"Item at index {i} is missing required key 'data'")
+        if not isinstance(item["data"], dict):
+            raise TypeError(
+                f"Item at index {i} 'data' must be a dict, "
+                f"got {type(item['data']).__name__}"
+            )
+
+        if "assertions" in item:
+            assertions = item["assertions"]
+            if not isinstance(assertions, list) or not all(
+                isinstance(a, str) for a in assertions
+            ):
+                raise TypeError(
+                    f"Item at index {i} 'assertions' must be a list of strings"
+                )
+
+        if "execution_policy" in item:
+            validate_execution_policy(
+                item["execution_policy"],
+                context=f"Item at index {i} 'execution_policy'",
+            )

sdks/python/src/opik/api_objects/experiment/__init__.py

@@ -0,0 +1,8 @@
+from .experiment import Experiment
+from .helpers import build_metadata_and_prompt_versions, handle_prompt_args
+
+__all__ = [
+    "Experiment",
+    "build_metadata_and_prompt_versions",
+    "handle_prompt_args",
+]

sdks/python/src/opik/api_objects/experiment/experiment.py

@@ -0,0 +1,171 @@
+import functools
+import logging
+from typing import List, Optional, TYPE_CHECKING
+
+from opik.message_processing.batching import sequence_splitter
+from opik.message_processing import messages, streamer
+from opik.rest_api import client as rest_api_client
+from opik.rest_api import types as rest_api_types
+from . import experiment_item, experiments_client
+from .. import constants, helpers
+from ...api_objects.prompt import base_prompt
+
+if TYPE_CHECKING:
+    from opik.evaluation.metrics import score_result
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Experiment:
+    def __init__(
+        self,
+        id: str,
+        name: Optional[str],
+        dataset_name: str,
+        rest_client: rest_api_client.OpikApi,
+        streamer: streamer.Streamer,
+        experiments_client: experiments_client.ExperimentsClient,
+        prompts: Optional[List[base_prompt.BasePrompt]] = None,
+        tags: Optional[List[str]] = None,
+        project_name: Optional[str] = None,
+    ) -> None:
+        self._id = id
+        self._name = name
+        self._dataset_name = dataset_name
+        self._rest_client = rest_client
+        self._prompts = prompts
+        self._streamer = streamer
+        self._experiments_client = experiments_client
+        self._tags = tags
+        self._project_name = project_name
+
+    @property
+    def project_name(self) -> Optional[str]:
+        return self._project_name
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def dataset_name(self) -> str:
+        return self._dataset_name
+
+    @property
+    def name(self) -> str:
+        if self._name is not None:
+            return self._name
+
+        name = self._rest_client.experiments.get_experiment_by_id(id=self.id).name
+        self._name = name
+
+        return name
+
+    @property
+    def tags(self) -> Optional[List[str]]:
+        return self._tags
+
+    @property
+    def prompts(self) -> Optional[List[base_prompt.BasePrompt]]:
+        return self._prompts
+
+    @functools.cached_property
+    def dataset_id(self) -> str:
+        return self._rest_client.datasets.get_dataset_by_identifier(
+            dataset_name=self._dataset_name
+        ).id
+
+    @property
+    def experiments_rest_client(self) -> rest_api_client.ExperimentsClient:
+        return self._rest_client.experiments
+
+    def get_experiment_data(self) -> rest_api_types.experiment_public.ExperimentPublic:
+        return self._rest_client.experiments.get_experiment_by_id(id=self.id)
+
+    def insert(
+        self,
+        experiment_items_references: List[experiment_item.ExperimentItemReferences],
+    ) -> None:
+        """
+        Creates a new experiment item by linking the existing trace and dataset item.
+
+        Args:
+            experiment_items_references: The list of ExperimentItemReferences objects, containing
+                trace id and dataset item id to link together into experiment item.
+
+        Returns:
+            None
+        """
+
+        experiment_item_messages = [
+            messages.ExperimentItemMessage(
+                id=helpers.generate_id(),
+                experiment_id=self.id,
+                dataset_item_id=item.dataset_item_id,
+                trace_id=item.trace_id,
+                project_name=item.project_name,
+                execution_policy=item.execution_policy,
+            )
+            for item in experiment_items_references
+        ]
+
+        # Split into batches for the streamer
+        batches = sequence_splitter.split_into_batches(
+            experiment_item_messages,
+            max_length=constants.FEEDBACK_SCORES_MAX_BATCH_SIZE,
+        )
+
+        for batch in batches:
+            create_experiment_items_batch_message = (
+                messages.CreateExperimentItemsBatchMessage(batch=batch)
+            )
+            self._streamer.put(create_experiment_items_batch_message)
+
+    def get_items(
+        self,
+        max_results: Optional[int] = 10000,
+        truncate: bool = False,
+    ) -> List[experiment_item.ExperimentItemContent]:
+        """
+        Retrieves and returns a list of experiment items for this experiment.
+
+        Args:
+            max_results: Maximum number of experiment items to retrieve. Defaults to 10000 if not specified.
+            truncate: Whether to truncate the items returned by the backend. Defaults to False.
+
+        Returns:
+            List of ExperimentItemContent objects for this experiment.
+        """
+        if max_results is None:
+            max_results = 10000  # TODO: remove this once we have a proper way to get all experiment items
+
+        return self._experiments_client.find_experiment_items_for_dataset(
+            dataset_name=self.dataset_name,
+            experiment_ids=[self.id],
+            truncate=truncate,
+            max_results=max_results,
+            project_name=self._project_name,
+        )
+
+    def log_experiment_scores(
+        self,
+        score_results: List["score_result.ScoreResult"],
+    ) -> None:
+        """Log experiment-level scores to the backend."""
+        experiment_scores: List[rest_api_types.ExperimentScore] = []
+
+        for score_result_ in score_results:
+            if score_result_.scoring_failed:
+                continue
+
+            experiment_score = rest_api_types.ExperimentScore(
+                name=score_result_.name,
+                value=score_result_.value,
+            )
+            experiment_scores.append(experiment_score)
+
+        if experiment_scores:
+            self._rest_client.experiments.update_experiment(
+                id=self.id,
+                experiment_scores=experiment_scores,
+            )

sdks/python/src/opik/api_objects/experiment/experiment_item.py

@@ -0,0 +1,67 @@
+import dataclasses
+
+from typing import Dict, Any, List, Optional
+from opik.types import FeedbackScoreDict
+from opik.rest_api.types import experiment_item_compare
+
+AssertionResultDict = Dict[str, Any]
+
+
+@dataclasses.dataclass
+class ExperimentItemReferences:
+    dataset_item_id: str
+    trace_id: str
+    project_name: Optional[str] = None
+    execution_policy: Optional[Dict[str, Any]] = None
+
+
+@dataclasses.dataclass
+class ExperimentItemContent:
+    id: str
+    dataset_item_id: str
+    trace_id: str
+    dataset_item_data: Optional[Dict[str, Any]]
+    evaluation_task_output: Optional[Dict[str, Any]]
+    feedback_scores: List[FeedbackScoreDict]
+    assertion_results: List[AssertionResultDict] = dataclasses.field(
+        default_factory=list
+    )
+
+    @classmethod
+    def from_rest_experiment_item_compare(
+        cls,
+        value: experiment_item_compare.ExperimentItemCompare,
+        dataset_item_data: Optional[Dict[str, Any]] = None,
+    ) -> "ExperimentItemContent":
+        if value.feedback_scores is None:
+            feedback_scores: List[FeedbackScoreDict] = []
+        else:
+            feedback_scores = [
+                {
+                    "category_name": rest_feedback_score.category_name,
+                    "name": rest_feedback_score.name,
+                    "reason": rest_feedback_score.reason,
+                    "value": rest_feedback_score.value,
+                }
+                for rest_feedback_score in value.feedback_scores
+            ]
+
+        if value.assertion_results is None:
+            assertion_results: List[AssertionResultDict] = []
+        else:
+            assertion_results = [
+                ar
+                if isinstance(ar, dict)
+                else {"value": ar.value, "passed": ar.passed, "reason": ar.reason}
+                for ar in value.assertion_results
+            ]
+
+        return ExperimentItemContent(
+            id=value.id,
+            trace_id=value.trace_id,
+            dataset_item_id=value.dataset_item_id,
+            dataset_item_data=dataset_item_data if dataset_item_data else value.input,
+            evaluation_task_output=value.output,
+            feedback_scores=feedback_scores,
+            assertion_results=assertion_results,
+        )