chore: import upstream snapshot with attribution

sdks/python/tests/e2e/runner/conftest.py

@@ -0,0 +1,285 @@
+import base64
+import dataclasses
+import hashlib
+import hmac as hmac_mod
+import os
+import re
+import shutil
+import subprocess
+import sys
+import threading
+import time
+import uuid
+
+import pytest
+
+import opik
+import opik.api_objects.opik_client
+from opik.api_objects import rest_helpers
+from opik.cli.local_runner.pairing import hkdf_sha256
+from opik.rest_api import core as rest_api_core
+
+
+ECHO_APP = os.path.join(os.path.dirname(__file__), "echo_app.py")
+OPIK_CLI = shutil.which("opik") or "opik"
+
+RUNNER_STARTUP_TIMEOUT = 30
+
+_phase_report_key = pytest.StashKey[dict]()
+
+
+@dataclasses.dataclass
+class RunnerInfo:
+    runner_id: str
+    process: subprocess.Popen
+    output_lines: list
+    bridge_key: bytes = b""
+
+
+@dataclasses.dataclass
+class TestProject:
+    id: str
+    name: str
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    rep = outcome.get_result()
+    item.stash.setdefault(_phase_report_key, {})[rep.when] = rep
+
+
+@pytest.fixture()
+def opik_client(configure_e2e_tests_env, shutdown_cached_client_after_test):
+    client = opik.api_objects.opik_client.Opik(batching=True)
+    yield client
+    client.end()
+
+
+@pytest.fixture()
+def api_client(opik_client):
+    return opik_client.rest_client
+
+
+@pytest.fixture()
+def subprocess_env(opik_client):
+    cfg = opik_client.config
+    env = os.environ.copy()
+    env["OPIK_URL_OVERRIDE"] = cfg.url_override
+    if cfg.api_key:
+        env["OPIK_API_KEY"] = cfg.api_key
+    if cfg.workspace:
+        env["OPIK_WORKSPACE"] = cfg.workspace
+    return env
+
+
+@pytest.fixture()
+def project(api_client, temporary_project_name) -> TestProject:
+    """Per-test project, created on setup. The shared ``temporary_project_name``
+    fixture handles name generation and teardown."""
+    try:
+        api_client.projects.create_project(name=temporary_project_name)
+    except rest_api_core.ApiError:
+        pass
+    project_id = rest_helpers.resolve_project_id_by_name(
+        api_client, temporary_project_name
+    )
+    return TestProject(id=project_id, name=temporary_project_name)
+
+
+def _drain_stdout(proc, output_lines):
+    for line in proc.stdout:
+        output_lines.append(line.rstrip())
+
+
+def _parse_pairing_url(output_lines, timeout):
+    """Wait for the CLI to print the pairing URL, then parse the fragment.
+
+    The URL may be split across multiple stdout lines by the terminal.
+    Join all output and search for the full URL.
+    """
+    deadline = time.monotonic() + timeout
+    url_pattern = re.compile(
+        r"(https?://\S+/opik/pair/v1(?:\?[^#\s]*)?#[A-Za-z0-9_\-]+)"
+    )
+
+    while time.monotonic() < deadline:
+        joined = "".join(list(output_lines))
+        match = url_pattern.search(joined)
+        if match:
+            url = match.group(1)
+            fragment = url.split("#", 1)[1]
+            padded = fragment + "=" * ((4 - len(fragment) % 4) % 4)
+            raw = base64.urlsafe_b64decode(padded)
+            if len(raw) < 65:
+                time.sleep(0.05)
+                continue
+            name_len = raw[64]
+            if len(raw) < 65 + name_len:
+                time.sleep(0.05)
+                continue
+            return {
+                "session_id": str(uuid.UUID(bytes=raw[0:16])),
+                "activation_key": raw[16:48],
+                "project_id": str(uuid.UUID(bytes=raw[48:64])),
+                "runner_name": raw[65 : 65 + name_len].decode("utf-8"),
+            }
+        time.sleep(0.05)
+    return None
+
+
+def _activate_session(api_client, activation_key, session_id, runner_name):
+    session_id_bytes = uuid.UUID(session_id).bytes
+    runner_name_hash = hashlib.sha256(runner_name.encode("utf-8")).digest()
+    message = session_id_bytes + runner_name_hash
+    sig = hmac_mod.new(activation_key, message, hashlib.sha256).digest()
+    hmac_b64 = base64.b64encode(sig).decode("ascii")
+
+    api_client.pairing.activate_pairing_session(
+        session_id, runner_name=runner_name, hmac=hmac_b64
+    )
+
+
+def _start_runner(
+    api_client, subprocess_env, project_id, opik_client, request, cli_args
+):
+    proc = subprocess.Popen(
+        cli_args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=subprocess_env,
+    )
+
+    output_lines = []
+    drain_thread = threading.Thread(
+        target=_drain_stdout, args=(proc, output_lines), daemon=True
+    )
+    drain_thread.start()
+
+    # Wait for the CLI to print its pairing URL, then parse the fragment
+    payload = _parse_pairing_url(output_lines, timeout=15)
+    if payload is None:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail(
+            "CLI did not print pairing URL within 15s.\n"
+            "Output:\n" + "\n".join(output_lines)
+        )
+
+    # Activate the CLI's session (simulates browser clicking Connect)
+    _activate_session(
+        api_client,
+        payload["activation_key"],
+        payload["session_id"],
+        payload["runner_name"],
+    )
+
+    # Wait for the CLI to detect connected status
+    deadline = time.monotonic() + RUNNER_STARTUP_TIMEOUT
+    connected = False
+    while time.monotonic() < deadline:
+        for line in list(output_lines):
+            if "Paired" in line:
+                connected = True
+                break
+        if connected:
+            break
+        if proc.poll() is not None:
+            break
+        time.sleep(0.1)
+
+    if not connected:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail(
+            f"Runner did not connect within {RUNNER_STARTUP_TIMEOUT}s.\n"
+            f"Output:\n" + "\n".join(output_lines)
+        )
+
+    # Get runner_id by polling the API
+    runners_page = api_client.runners.list_runners(project_id=project_id, size=10)
+    runner_id = None
+    if runners_page.content:
+        for r in runners_page.content:
+            if r.status == "connected":
+                runner_id = r.id
+                break
+
+    if runner_id is None:
+        proc.terminate()
+        proc.wait(timeout=5)
+        drain_thread.join(timeout=5)
+        pytest.fail("No connected runner found after activation")
+
+    bridge_key = hkdf_sha256(
+        ikm=payload["activation_key"],
+        salt=uuid.UUID(payload["session_id"]).bytes,
+        info=b"opik-bridge-v1",
+    )
+
+    info = RunnerInfo(
+        runner_id=runner_id,
+        process=proc,
+        output_lines=output_lines,
+        bridge_key=bridge_key,
+    )
+
+    yield info
+
+    # Runner CLI + echo_app don't need a graceful window in tests; SIGTERM →
+    # short wait → SIGKILL is fine. Drops per-test teardown from ~10s to ~2s.
+    proc.terminate()
+    try:
+        proc.wait(timeout=2)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait()
+    drain_thread.join(timeout=2)
+
+    report = request.node.stash.get(_phase_report_key, {})
+    if report.get("call") and report["call"].failed:
+        print(f"\n--- Runner output (runner_id={runner_id}) ---")
+        print("\n".join(output_lines))
+        print("--- End runner output ---")
+
+
+@pytest.fixture()
+def runner_process(api_client, subprocess_env, project, opik_client, request):
+    """Endpoint runner — for job/agent E2E tests."""
+    yield from _start_runner(
+        api_client,
+        subprocess_env,
+        project.id,
+        opik_client,
+        request,
+        cli_args=[
+            OPIK_CLI,
+            "endpoint",
+            "--project",
+            project.name,
+            "--",
+            sys.executable,
+            ECHO_APP,
+        ],
+    )
+
+
+@pytest.fixture()
+def bridge_runner_process(api_client, subprocess_env, project, opik_client, request):
+    """Connect runner — for bridge command E2E tests."""
+    yield from _start_runner(
+        api_client,
+        subprocess_env,
+        project.id,
+        opik_client,
+        request,
+        cli_args=[
+            OPIK_CLI,
+            "connect",
+            "--project",
+            project.name,
+        ],
+    )

sdks/python/tests/e2e/runner/echo_app.py

@@ -0,0 +1,36 @@
+"""Agent app used by the runner e2e tests.
+
+Contains two entrypoints:
+- echo: basic echo function
+- echo_config: echo with a configurable greeting (used in mask tests)
+"""
+
+import threading
+
+import opik
+
+
+@opik.track(entrypoint=True)
+def echo(message: str) -> str:
+    print(f"echo stdout: {message}")
+    return f"echo: {message}"
+
+
+class EchoConfig(opik.Config):
+    greeting: str
+
+
+@opik.track(entrypoint=True)
+def echo_config(message: str) -> str:
+    client = opik.Opik()
+    cfg = client.get_or_create_config(
+        fallback=EchoConfig(greeting="fallback-greeting"),
+    )
+    return f"{cfg.greeting}: {message}"
+
+
+if __name__ == "__main__":
+    # Analogous to uvicorn.run() in a FastAPI app: @opik.track(entrypoint=True) registers
+    # handlers at import time, and this blocks the main thread so the runner loop stays
+    # alive to process jobs.
+    threading.Event().wait()

sdks/python/tests/e2e/runner/test_bridge_e2e.py

@@ -0,0 +1,195 @@
+"""Happy-path e2e tests for the bridge command system.
+
+Exec tests: echo, nonzero exit, background process.
+File operation flow: write → list → search → edit → read, all via the API.
+"""
+
+import time
+
+from opik.rest_api.errors.conflict_error import ConflictError
+from opik.runner.bridge_auth import sign
+
+from .conftest import RunnerInfo
+
+_BRIDGE_READY_TIMEOUT = 15
+
+
+def _submit_and_wait(api_client, runner_id, cmd_type, args, bridge_key=b""):
+    if bridge_key:
+        args = dict(args)
+        args["_hmac"] = sign(bridge_key, cmd_type, args)
+
+    deadline = time.monotonic() + _BRIDGE_READY_TIMEOUT
+    while True:
+        try:
+            resp = api_client.runners.create_bridge_command(
+                runner_id,
+                type=cmd_type,
+                args=args,
+            )
+            break
+        except ConflictError:
+            if time.monotonic() > deadline:
+                raise
+            time.sleep(1)
+
+    return api_client.runners.get_bridge_command(
+        runner_id,
+        resp.command_id,
+        wait=True,
+        timeout=15,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Auth tests
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_command__unsigned__fails_with_auth_failed(
+    api_client, bridge_runner_process: RunnerInfo
+):
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "echo should-not-run"},
+    )
+    assert cmd.status == "failed"
+    assert cmd.error["code"] == "auth_failed"
+
+
+def test_bridge_command__invalid_hmac__fails_with_auth_failed(
+    api_client, bridge_runner_process: RunnerInfo
+):
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "echo should-not-run", "_hmac": "dGhpcyBpcyBub3QgYSB2YWxpZCBobWFj"},
+    )
+    assert cmd.status == "failed"
+    assert cmd.error["code"] == "auth_failed"
+
+
+# ---------------------------------------------------------------------------
+# Exec tests
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_exec_echo(api_client, bridge_runner_process: RunnerInfo):
+    """Submit a simple echo command and verify the result."""
+    marker = f"bridge-e2e-{int(time.time())}"
+
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": f"echo {marker}"},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert marker in cmd.result["stdout"]
+    assert cmd.result["exit_code"] == 0
+
+
+def test_bridge_exec_nonzero_exit(api_client, bridge_runner_process: RunnerInfo):
+    """Verify non-zero exit codes are returned correctly."""
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "exit 42"},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert cmd.result["exit_code"] == 42
+
+
+def test_bridge_exec_background(api_client, bridge_runner_process: RunnerInfo):
+    """Submit a background command, verify PID is returned."""
+    cmd = _submit_and_wait(
+        api_client,
+        bridge_runner_process.runner_id,
+        "Exec",
+        {"command": "sleep 30", "background": True},
+        bridge_key=bridge_runner_process.bridge_key,
+    )
+
+    assert cmd.status == "completed"
+    assert "pid" in cmd.result
+    assert cmd.result["status"] == "running"
+
+
+# ---------------------------------------------------------------------------
+# File operation flow: write → list → search → edit → read
+# ---------------------------------------------------------------------------
+
+
+def test_bridge_file_operations(api_client, bridge_runner_process: RunnerInfo):
+    """Write a file, find it with list/search, edit it, and read back."""
+    rid = bridge_runner_process.runner_id
+    bk = bridge_runner_process.bridge_key
+    marker = f"xyzzy_{int(time.time())}"
+    filename = f"bridge_e2e_{int(time.time())}.py"
+    original_content = f"# {marker}\n"
+
+    # 1. Write
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "WriteFile",
+        {"path": filename, "content": original_content},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert cmd.result["created"] is True
+
+    # 2. ListFiles — new file should appear in the root
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "ListFiles",
+        {"pattern": f"{filename}"},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert any(filename in f for f in cmd.result["files"]), (
+        f"{filename} not in {cmd.result['files']}"
+    )
+
+    # 3. EditFile — replace content
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "EditFile",
+        {
+            "path": filename,
+            "edits": [{"old_string": marker, "new_string": f"edited_{marker}"}],
+        },
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert cmd.result["edits_applied"] == 1
+
+    # 4. ReadFile — verify the edit took effect
+    cmd = _submit_and_wait(
+        api_client,
+        rid,
+        "ReadFile",
+        {"path": filename},
+        bridge_key=bk,
+    )
+    assert cmd.status == "completed"
+    assert f"edited_{marker}" in cmd.result["content"]
+
+    # 5. Cleanup via Exec
+    _submit_and_wait(
+        api_client,
+        rid,
+        "Exec",
+        {"command": f"rm {filename}"},
+        bridge_key=bk,
+    )

sdks/python/tests/e2e/runner/test_runner_e2e.py

@@ -0,0 +1,191 @@
+"""Happy-path e2e tests for the local runner.
+
+Test 1 (basic): register echo agent, create job, verify trace output.
+Test 2 (mask):  register echo_config agent, create mask, create job with
+                mask_id, verify the mask value appears in the trace output.
+"""
+
+import time
+from typing import Optional
+
+
+import opik
+import opik.rest_api.client as rest_api_client
+from opik.api_objects.agent_config.config import ConfigManager
+from .conftest import RunnerInfo
+
+
+JOB_COMPLETION_TIMEOUT = 30
+TRACE_PROPAGATION_TIMEOUT = 30
+AGENT_REGISTRATION_TIMEOUT = 10
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def submit_job(
+    api: rest_api_client.OpikApi,
+    agent_name: str,
+    message: str,
+    project_id: str,
+    mask_id: Optional[str] = None,
+) -> None:
+    """Create a job for the given agent."""
+    api.runners.create_job(
+        agent_name=agent_name,
+        inputs={"message": message},
+        project_id=project_id,
+        mask_id=mask_id,
+    )
+
+
+def wait_for_completed_job(
+    api: rest_api_client.OpikApi, runner_id: str, match_text: str
+):
+    """Poll list_jobs until a completed job whose inputs contain *match_text* appears."""
+    result = []
+
+    def _find():
+        page = api.runners.list_jobs(runner_id=runner_id, size=20)
+        if page.content:
+            for j in page.content:
+                if j.status == "completed" and j.inputs and match_text in str(j.inputs):
+                    result.clear()
+                    result.append(j)
+                    return True
+        return False
+
+    assert opik.synchronization.until(
+        _find,
+        max_try_seconds=JOB_COMPLETION_TIMEOUT,
+        allow_errors=True,
+    ), f"No completed job with '{match_text}' found within {JOB_COMPLETION_TIMEOUT}s"
+
+    return result[0]
+
+
+def find_trace_by_input(
+    api: rest_api_client.OpikApi, project_name: str, match_text: str
+):
+    """Poll until a trace whose input contains *match_text* appears and has output."""
+    result = []
+
+    def _find():
+        page = api.traces.get_traces_by_project(
+            project_name=project_name,
+            size=20,
+        )
+        if page.content:
+            for t in page.content:
+                if t.input and match_text in str(t.input) and t.output:
+                    result.clear()
+                    result.append(t)
+                    return True
+        return False
+
+    assert opik.synchronization.until(
+        _find,
+        max_try_seconds=TRACE_PROPAGATION_TIMEOUT,
+        allow_errors=True,
+    ), f"No trace with '{match_text}' found within {TRACE_PROPAGATION_TIMEOUT}s"
+
+    return result[0]
+
+
+def wait_for_agent_registration(
+    api: rest_api_client.OpikApi, agent_name: str, project_id: str
+) -> None:
+    """Poll until the agent is registered with any runner in the project."""
+
+    def _is_agent_registered():
+        runners_page = api.runners.list_runners(project_id=project_id, size=50)
+        if runners_page.content:
+            for runner in runners_page.content:
+                if runner.agents:
+                    for agent in runner.agents:
+                        if agent.name == agent_name:
+                            return True
+        return False
+
+    if not opik.synchronization.until(
+        _is_agent_registered,
+        max_try_seconds=AGENT_REGISTRATION_TIMEOUT,
+        allow_errors=True,
+    ):
+        raise AssertionError(
+            f"Agent '{agent_name}' was not registered within {AGENT_REGISTRATION_TIMEOUT}s"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_runner_happy_path(api_client, runner_process: RunnerInfo, project):
+    """Basic: register echo agent, run job, verify job result, trace output, and job logs."""
+    message = f"hello-e2e-{int(time.time())}"
+
+    wait_for_agent_registration(api_client, "echo", project.id)
+
+    submit_job(api_client, "echo", message, project.id)
+
+    job = wait_for_completed_job(api_client, runner_process.runner_id, message)
+    assert job.result is not None, "Completed job should have a result"
+    assert f"echo: {message}" in str(job.result)
+    assert job.trace_id is not None, "Completed job should have a trace_id"
+
+    trace = find_trace_by_input(api_client, project.name, message)
+    assert f"echo: {message}" in str(trace.output)
+
+    logs_result = []
+
+    def _find_logs():
+        logs = api_client.runners.get_job_logs(job.id)
+        if logs:
+            logs_result.clear()
+            logs_result.extend(logs)
+            return True
+        return False
+
+    assert opik.synchronization.until(
+        _find_logs,
+        max_try_seconds=5,
+        allow_errors=True,
+    ), f"Expected job logs for job {job.id}, got none"
+
+    log_text = " ".join(entry.text for entry in logs_result)
+    assert message in log_text, f"Expected '{message}' in job logs, got: {log_text}"
+
+
+def test_runner_with_mask(opik_client, api_client, runner_process: RunnerInfo, project):
+    """Mask: register echo_config agent, create mask, verify mask value in job result and trace."""
+    message = f"mask-e2e-{int(time.time())}"
+    custom_greeting = f"custom-greeting-{int(time.time())}"
+
+    wait_for_agent_registration(api_client, "echo_config", project.id)
+
+    manager = ConfigManager(
+        project_name=project.name,
+        rest_client_=opik_client.rest_client,
+    )
+    manager.create_blueprint(
+        parameters={"greeting": "default-greeting"},
+    )
+    mask_id = manager.create_mask(
+        parameters={"greeting": custom_greeting},
+    )
+
+    submit_job(api_client, "echo_config", message, project.id, mask_id=mask_id)
+
+    job = wait_for_completed_job(api_client, runner_process.runner_id, message)
+    assert job.result is not None, "Completed job should have a result"
+    assert custom_greeting in str(job.result)
+    assert job.trace_id is not None, "Completed job should have a trace_id"
+
+    trace = find_trace_by_input(api_client, project.name, message)
+    assert custom_greeting in str(trace.output), (
+        f"Expected '{custom_greeting}' in trace output, got: {trace.output}"
+    )