chore: import upstream snapshot with attribution
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,512 @@
|
||||
"""Tests for the interactive-shell CLI assistant.
|
||||
|
||||
Covers:
|
||||
|
||||
- terminology: the LLM is instructed to call this surface the "interactive
|
||||
shell" and is forbidden from using "REPL" in user-facing answers (#604);
|
||||
- formatting: assistant Markdown output is rendered through Rich's Markdown
|
||||
renderer so tables / **bold** / `code` display correctly in the terminal
|
||||
instead of leaking raw Markdown syntax (#604).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from core.agent_harness.prompts import prompt_context as default_prompt_context
|
||||
from core.agent_harness.prompts.assistant_agent_prompt import (
|
||||
_MARKDOWN_RULE,
|
||||
_TERMINOLOGY_RULE,
|
||||
_build_observation_block,
|
||||
_build_system_prompt,
|
||||
build_environment_block,
|
||||
)
|
||||
from core.agent_harness.prompts.prompt_context import DefaultPromptContextProvider
|
||||
from surfaces.interactive_shell.runtime import answer_turn as cli_agent
|
||||
from surfaces.interactive_shell.runtime.answer_turn import answer_shell_question
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _build_environment_block(session: Session) -> str:
|
||||
"""Adapter for the relocated, signature-changed environment-block builder."""
|
||||
return build_environment_block(
|
||||
integrations=tuple(session.configured_integrations),
|
||||
known=session.configured_integrations_known,
|
||||
)
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
# ``force_terminal=True`` so Rich emits its real renderer output (the
|
||||
# same path the user sees) rather than collapsing markdown into raw
|
||||
# text on a non-tty stream.
|
||||
return (
|
||||
Console(file=buf, force_terminal=True, color_system=None, width=80, highlight=False),
|
||||
buf,
|
||||
)
|
||||
|
||||
|
||||
class _FakeLLMClient:
|
||||
"""Streaming-aware fake.
|
||||
|
||||
``invoke_stream`` yields the canned content as a single chunk. ``content``
|
||||
accepts either a plain string or an Anthropic-style list of content blocks
|
||||
(objects with ``.text`` or dicts with a ``"text"`` key); blocks are flattened
|
||||
to the same text the real SDK's ``text_stream`` would surface.
|
||||
"""
|
||||
|
||||
def __init__(self, content: Any) -> None:
|
||||
self._content = content
|
||||
self.last_prompt: str | None = None
|
||||
|
||||
def invoke_stream(self, prompt: str) -> Iterator[str]:
|
||||
self.last_prompt = prompt
|
||||
if isinstance(self._content, list):
|
||||
parts: list[str] = []
|
||||
for block in self._content:
|
||||
if isinstance(block, dict):
|
||||
parts.append(block.get("text", ""))
|
||||
elif hasattr(block, "text"):
|
||||
parts.append(block.text)
|
||||
yield "\n".join(parts)
|
||||
return
|
||||
yield str(self._content)
|
||||
|
||||
|
||||
def _patch_llm(monkeypatch: Any, content: Any) -> _FakeLLMClient:
|
||||
client = _FakeLLMClient(content)
|
||||
# ``answer_shell_question`` imports ``get_llm_for_reasoning`` lazily from
|
||||
# ``core.llm.factory.get_llm``, so we patch the symbol on that module.
|
||||
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: client)
|
||||
return client
|
||||
|
||||
|
||||
def _patch_grounding(
|
||||
monkeypatch: Any,
|
||||
*,
|
||||
cli_reference: str = "(ref)",
|
||||
agents_md: str = "",
|
||||
investigation_flow: str = "",
|
||||
) -> None:
|
||||
"""Pin the shell grounding caches the prompt provider reads from.
|
||||
|
||||
The conversational assistant now sources grounding text through
|
||||
``DefaultPromptContextProvider`` (over ``session.grounding`` + the
|
||||
investigation-flow reference), so tests patch the provider methods rather
|
||||
than module-level builders.
|
||||
"""
|
||||
provider = DefaultPromptContextProvider
|
||||
monkeypatch.setattr(provider, "cli_reference", lambda _self: cli_reference)
|
||||
monkeypatch.setattr(provider, "agents_md", lambda _self: agents_md)
|
||||
monkeypatch.setattr(provider, "investigation_flow", lambda _self: investigation_flow)
|
||||
|
||||
|
||||
class TestSystemPromptTerminology:
|
||||
"""The LLM grounding must steer answers away from the word 'REPL'."""
|
||||
|
||||
def test_conversational_prompt_uses_interactive_shell_not_repl(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)")
|
||||
assert "interactive shell" in prompt
|
||||
assert "argv" in prompt
|
||||
assert "!" in prompt
|
||||
# The prompt must explicitly forbid the "REPL" jargon so the model
|
||||
# does not echo it back in answers (#604).
|
||||
assert _TERMINOLOGY_RULE in prompt
|
||||
assert "Never use the word 'REPL'" in prompt
|
||||
|
||||
def test_prompt_requests_markdown_formatting(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)")
|
||||
assert _MARKDOWN_RULE in prompt
|
||||
assert "Markdown" in prompt
|
||||
|
||||
def test_conversational_prompt_does_not_expose_action_json_contract(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)")
|
||||
|
||||
assert '"action"' not in prompt
|
||||
assert "switch_llm_provider" not in prompt
|
||||
assert "run_interactive" not in prompt
|
||||
|
||||
def test_prompt_gives_generic_integration_setup_guidance(self) -> None:
|
||||
"""If a setup request reaches the assistant, it gives guidance only."""
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)")
|
||||
assert "/integrations setup <service>" in prompt
|
||||
assert "/mcp connect <server>" in prompt
|
||||
assert "Do not emit JSON" in prompt
|
||||
|
||||
|
||||
class TestSystemPromptAgentsMdGrounding:
|
||||
"""The conversational shell wires AGENTS.md repo-map content (#1442)."""
|
||||
|
||||
def test_section_present_in_conversational_prompt_when_agents_md_provided(self) -> None:
|
||||
prompt = _build_system_prompt(
|
||||
reference="(ref)",
|
||||
history="(hist)",
|
||||
agents_md="repo map content",
|
||||
)
|
||||
assert "--- Repo map (AGENTS.md) ---" in prompt
|
||||
assert "repo map content" in prompt
|
||||
|
||||
def test_section_omitted_when_agents_md_empty(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)", agents_md="")
|
||||
assert "--- Repo map (AGENTS.md) ---" not in prompt
|
||||
|
||||
def test_section_omitted_by_default_for_callers_that_dont_pass_it(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)")
|
||||
assert "--- Repo map (AGENTS.md) ---" not in prompt
|
||||
|
||||
|
||||
class TestSystemPromptInvestigationFlowGrounding:
|
||||
"""The conversational shell includes the investigation-flow reference block."""
|
||||
|
||||
def test_investigation_flow_section_present_when_reference_provided(self) -> None:
|
||||
prompt = _build_system_prompt(
|
||||
reference="(ref)",
|
||||
history="(hist)",
|
||||
investigation_flow="resolve → extract → investigate → deliver",
|
||||
)
|
||||
|
||||
assert "--- Investigation flow reference ---" in prompt
|
||||
assert "resolve → extract → investigate → deliver" in prompt
|
||||
assert "do not claim the pipeline definition is unavailable" in prompt
|
||||
|
||||
def test_investigation_flow_section_omitted_when_reference_empty(self) -> None:
|
||||
prompt = _build_system_prompt(reference="(ref)", history="(hist)", investigation_flow="")
|
||||
|
||||
assert "--- Investigation flow reference ---" not in prompt
|
||||
|
||||
def test_answer_shell_question_injects_investigation_flow_reference(
|
||||
self, monkeypatch: Any
|
||||
) -> None:
|
||||
client = _patch_llm(monkeypatch, "Yes, I can describe the pipeline.")
|
||||
_patch_grounding(
|
||||
monkeypatch,
|
||||
investigation_flow="resolve → extract → investigate → deliver",
|
||||
)
|
||||
|
||||
console, _ = _capture()
|
||||
answer_shell_question("Can you see how investigations are structured?", Session(), console)
|
||||
|
||||
assert client.last_prompt is not None
|
||||
assert "--- Investigation flow reference ---" in client.last_prompt
|
||||
assert "resolve → extract → investigate → deliver" in client.last_prompt
|
||||
|
||||
|
||||
class TestEnvironmentIntegrationGrounding:
|
||||
"""The assistant must be told which integrations are configured (#sentry-context)."""
|
||||
|
||||
def test_block_lists_configured_services_when_known(self) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("gitlab", "datadog")
|
||||
block = _build_environment_block(session)
|
||||
assert "--- Environment (current shell state) ---" in block
|
||||
assert "gitlab" in block
|
||||
assert "datadog" in block
|
||||
assert "not in that list is NOT configured" in block
|
||||
|
||||
def test_block_states_none_when_known_and_empty(self) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ()
|
||||
block = _build_environment_block(session)
|
||||
assert "No integrations are configured" in block
|
||||
|
||||
def test_block_lists_active_llm_settings_when_available(self) -> None:
|
||||
block = build_environment_block(
|
||||
integrations=("github",),
|
||||
known=True,
|
||||
llm_provider="openai",
|
||||
reasoning_model="gpt-5.5",
|
||||
toolcall_model="gpt-5.4-mini",
|
||||
llm_settings_available=True,
|
||||
)
|
||||
|
||||
assert "--- Environment (current shell state) ---" in block
|
||||
assert "Configured integrations in this session: github" in block
|
||||
assert "provider openai" in block
|
||||
assert "reasoning model gpt-5.5" in block
|
||||
assert "tool-call model gpt-5.4-mini" in block
|
||||
assert "answer directly from these values" in block
|
||||
assert "`/model`, `/status`, or `opensre config show`" in block
|
||||
|
||||
def test_block_states_llm_settings_unavailable(self) -> None:
|
||||
block = build_environment_block(
|
||||
integrations=(),
|
||||
known=False,
|
||||
llm_settings_available=False,
|
||||
)
|
||||
|
||||
assert "Active LLM settings are unavailable" in block
|
||||
assert "could not be read" in block
|
||||
|
||||
def test_block_omitted_when_unknown(self) -> None:
|
||||
session = Session()
|
||||
assert session.configured_integrations_known is False
|
||||
assert _build_environment_block(session) == ""
|
||||
|
||||
def test_answer_shell_question_injects_configured_integrations(self, monkeypatch: Any) -> None:
|
||||
client = _patch_llm(monkeypatch, "No, Sentry is not configured.")
|
||||
_patch_grounding(monkeypatch)
|
||||
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("gitlab",)
|
||||
console, _ = _capture()
|
||||
answer_shell_question("is sentry installed?", session, console)
|
||||
|
||||
assert client.last_prompt is not None
|
||||
assert "--- Environment (current shell state) ---" in client.last_prompt
|
||||
assert "gitlab" in client.last_prompt
|
||||
|
||||
def test_answer_shell_question_injects_active_llm_settings(self, monkeypatch: Any) -> None:
|
||||
client = _patch_llm(monkeypatch, "You are using OpenAI.")
|
||||
_patch_grounding(monkeypatch)
|
||||
|
||||
class _Settings:
|
||||
provider = "openai"
|
||||
|
||||
monkeypatch.setattr(default_prompt_context, "load_llm_settings", lambda: _Settings())
|
||||
monkeypatch.setattr(
|
||||
default_prompt_context,
|
||||
"resolve_provider_models",
|
||||
lambda _settings, _provider: ("gpt-5.5", "gpt-5.4-mini"),
|
||||
)
|
||||
|
||||
console, _ = _capture()
|
||||
answer_shell_question("what model am I using now?", Session(), console)
|
||||
|
||||
assert client.last_prompt is not None
|
||||
assert "Active LLM settings in this session" in client.last_prompt
|
||||
assert "provider openai" in client.last_prompt
|
||||
assert "reasoning model gpt-5.5" in client.last_prompt
|
||||
assert "tool-call model gpt-5.4-mini" in client.last_prompt
|
||||
|
||||
|
||||
class TestObservationSummaryBlock:
|
||||
"""The observe→answer loop feeds discovery output back for summarization."""
|
||||
|
||||
def test_block_empty_without_observation(self) -> None:
|
||||
assert _build_observation_block(None) == ""
|
||||
assert _build_observation_block(" ") == ""
|
||||
|
||||
def test_block_wraps_command_output_with_summarize_instruction(self) -> None:
|
||||
block = _build_observation_block("- sentry: missing (Not configured.)")
|
||||
assert "tool_results" in block
|
||||
assert "- sentry: missing (Not configured.)" in block
|
||||
assert "summarize" in block.lower()
|
||||
# The summary turn must not kick off more tool calls; offers are prose only.
|
||||
assert "not request, plan, or emit any further tool calls" in block.lower()
|
||||
|
||||
def test_answer_shell_question_injects_observation(self, monkeypatch: Any) -> None:
|
||||
client = _patch_llm(monkeypatch, "No — Sentry is not configured.")
|
||||
_patch_grounding(monkeypatch)
|
||||
|
||||
session = Session()
|
||||
console, _ = _capture()
|
||||
observation = (
|
||||
"Integration status from `/integrations`:\n- sentry: missing (Not configured.)"
|
||||
)
|
||||
answer_shell_question(
|
||||
"is sentry installed?", session, console, tool_observation=observation
|
||||
)
|
||||
|
||||
assert client.last_prompt is not None
|
||||
assert "tool_results" in client.last_prompt
|
||||
assert "sentry: missing" in client.last_prompt
|
||||
|
||||
|
||||
class TestAssistantOutputRendering:
|
||||
"""The assistant reply must be rendered, not printed as raw Markdown."""
|
||||
|
||||
def test_bold_markdown_is_rendered(self, monkeypatch: Any) -> None:
|
||||
# End-of-stream force-flush renders the buffered text as
|
||||
# Markdown — ``**`` delimiters are stripped.
|
||||
_patch_llm(monkeypatch, "Hello **world**")
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("hi", session, console)
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "**world**" not in output
|
||||
assert "world" in output
|
||||
assert "Hello" in output
|
||||
assert session.tokens.totals.get("output", 0) > 0
|
||||
|
||||
def test_table_markdown_is_rendered_as_table(self, monkeypatch: Any) -> None:
|
||||
markdown = (
|
||||
"| Command | What it does |\n|---|---|\n"
|
||||
"| `opensre` | Start the interactive shell (TTY) |\n"
|
||||
)
|
||||
_patch_llm(monkeypatch, markdown)
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("show commands", session, console)
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
# Rich's Markdown table renderer replaces the ``|---|---|``
|
||||
# separator with box-drawing chars — the literal must not leak.
|
||||
assert "|---|---|" not in output
|
||||
assert "Command" in output
|
||||
assert "What it does" in output
|
||||
assert "opensre" in output
|
||||
|
||||
def test_response_is_recorded_in_session_history(self, monkeypatch: Any) -> None:
|
||||
_patch_llm(monkeypatch, "Sure thing.")
|
||||
session = Session()
|
||||
console, _ = _capture()
|
||||
answer_shell_question("hello", session, console)
|
||||
assert session.cli_agent_messages[-2:] == [
|
||||
("user", "hello"),
|
||||
("assistant", "Sure thing."),
|
||||
]
|
||||
|
||||
def test_command_selection_prompt_uses_llm_response(self, monkeypatch: Any) -> None:
|
||||
_patch_llm(monkeypatch, "Use `opensre investigate` for incidents.")
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("what command do I use?", session, console)
|
||||
output = _strip_ansi(buf.getvalue()).casefold()
|
||||
assert "opensre investigate" in output
|
||||
assert session.cli_agent_messages[-2:] == [
|
||||
("user", "what command do I use?"),
|
||||
("assistant", "Use `opensre investigate` for incidents."),
|
||||
]
|
||||
assert session.tokens.call_count == 1
|
||||
|
||||
def test_structured_content_blocks_are_rendered(self, monkeypatch: Any) -> None:
|
||||
class _Block:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
_patch_llm(monkeypatch, [_Block("First line"), {"text": "Second line"}])
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("hello", session, console)
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "First line" in output
|
||||
assert "Second line" in output
|
||||
assert session.cli_agent_messages[-1] == ("assistant", "First line\nSecond line")
|
||||
|
||||
def test_llm_failure_prints_red_error_and_does_not_record(self, monkeypatch: Any) -> None:
|
||||
captured_errors: list[BaseException] = []
|
||||
|
||||
class _Boom:
|
||||
def invoke_stream(self, _prompt: str) -> Iterator[str]:
|
||||
raise RuntimeError("upstream 503")
|
||||
yield # pragma: no cover -- generator marker
|
||||
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: _Boom())
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.error_reporting.capture_exception",
|
||||
lambda exc, **_kwargs: captured_errors.append(exc),
|
||||
)
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("hi", session, console)
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "assistant failed" in output
|
||||
assert "upstream 503" in output
|
||||
assert len(captured_errors) == 1
|
||||
assert isinstance(captured_errors[0], RuntimeError)
|
||||
# On failure the turn must NOT be appended to the cli-agent history,
|
||||
# otherwise the next turn's prompt would carry a phantom assistant
|
||||
# message.
|
||||
assert session.cli_agent_messages == []
|
||||
|
||||
|
||||
class TestStreamingMigration:
|
||||
"""cli_agent must consume invoke_stream and send output through the shared streaming renderer."""
|
||||
|
||||
def test_response_uses_invoke_stream_not_invoke(self, monkeypatch: Any) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class _Recording:
|
||||
def invoke(self, _prompt: str) -> Any:
|
||||
calls.append("invoke")
|
||||
raise AssertionError("cli_agent must not call invoke after streaming migration")
|
||||
|
||||
def invoke_stream(self, _prompt: str) -> Iterator[str]:
|
||||
calls.append("invoke_stream")
|
||||
yield "ok"
|
||||
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: _Recording())
|
||||
|
||||
console, _ = _capture()
|
||||
answer_shell_question("hi", Session(), console)
|
||||
|
||||
assert calls == ["invoke_stream"]
|
||||
|
||||
def test_json_like_response_is_plain_assistant_text(self, monkeypatch: Any) -> None:
|
||||
_patch_llm(
|
||||
monkeypatch,
|
||||
'{"actions":[{"action":"switch_llm_provider","provider":"anthropic"}]}',
|
||||
)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
answer_shell_question("switch to anthropic", session, console)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert '"switch_llm_provider"' in output
|
||||
assert "Requested actions" not in output
|
||||
assert "$ /model set anthropic" not in output
|
||||
assert session.history == []
|
||||
assert session.cli_agent_messages[-1] == (
|
||||
"assistant",
|
||||
'{"actions":[{"action":"switch_llm_provider","provider":"anthropic"}]}',
|
||||
)
|
||||
|
||||
|
||||
def test_answer_shell_question_injects_synthetic_observation_on_why_failed(
|
||||
tmp_path: Path,
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
obs = tmp_path / "latest.json"
|
||||
obs.write_text(
|
||||
'{"scenario_id": "008-storage-full-missing-metric", "score": {"passed": false}}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
session = Session()
|
||||
session.last_synthetic_observation_path = str(obs.resolve())
|
||||
console, _buf = _capture()
|
||||
client = _patch_llm(monkeypatch, "The synthetic run failed the scoring gate.")
|
||||
answer_shell_question("why did it fail?", session, console)
|
||||
assert client.last_prompt is not None
|
||||
assert "observation_json" in client.last_prompt
|
||||
assert "008-storage-full-missing-metric" in client.last_prompt
|
||||
|
||||
|
||||
def test_answer_shell_question_skips_observation_without_failure_question(
|
||||
tmp_path: Path,
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
obs = tmp_path / "latest.json"
|
||||
obs.write_text("{}", encoding="utf-8")
|
||||
session = Session()
|
||||
session.last_synthetic_observation_path = str(obs.resolve())
|
||||
console, _buf = _capture()
|
||||
client = _patch_llm(monkeypatch, "hi")
|
||||
answer_shell_question("hello", session, console)
|
||||
assert client.last_prompt is not None
|
||||
assert "observation_json" not in client.last_prompt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
"""Remove ANSI escape sequences so assertions test the visible output."""
|
||||
import re
|
||||
|
||||
# Standard CSI-sequence regex; covers Rich's bold / color escapes.
|
||||
return re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", text)
|
||||
|
||||
|
||||
def test_module_exports_answer_shell_question() -> None:
|
||||
assert "answer_shell_question" in cli_agent.__all__
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Read-only discovery commands stash a compact observation for the agent.
|
||||
|
||||
This is what lets the observe→answer loop summarize "is sentry installed?" from
|
||||
the actual ``/integrations`` output instead of leaving the user with a raw table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from surfaces.interactive_shell.command_registry.integrations import (
|
||||
_MAX_OBSERVATION_DETAIL_CHARS,
|
||||
_record_integration_show_observation,
|
||||
_record_integrations_observation,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def test_records_status_lines_for_each_service() -> None:
|
||||
session = Session()
|
||||
results = [
|
||||
{"service": "datadog", "source": "local store", "status": "passed", "detail": "Connected."},
|
||||
{"service": "sentry", "source": "-", "status": "missing", "detail": "Not configured."},
|
||||
]
|
||||
|
||||
_record_integrations_observation(session, results)
|
||||
|
||||
assert session.agent.last_observation is not None
|
||||
obs = session.agent.last_observation
|
||||
assert "datadog: passed" in obs
|
||||
assert "sentry: missing" in obs
|
||||
assert "Not configured." in obs
|
||||
|
||||
|
||||
def test_truncates_long_detail() -> None:
|
||||
session = Session()
|
||||
long_detail = "x" * (_MAX_OBSERVATION_DETAIL_CHARS + 50)
|
||||
_record_integrations_observation(
|
||||
session, [{"service": "datadog", "status": "passed", "detail": long_detail}]
|
||||
)
|
||||
|
||||
assert session.agent.last_observation is not None
|
||||
# Detail is bounded (ellipsis substituted for the tail) so prompts stay cheap.
|
||||
assert "…" in session.agent.last_observation
|
||||
assert long_detail not in session.agent.last_observation
|
||||
|
||||
|
||||
def test_skips_rows_without_a_service_name() -> None:
|
||||
session = Session()
|
||||
_record_integrations_observation(session, [{"service": "", "status": "missing"}])
|
||||
assert session.agent.last_observation is None
|
||||
|
||||
|
||||
def test_show_observation_renders_key_values() -> None:
|
||||
session = Session()
|
||||
_record_integration_show_observation(
|
||||
session, {"service": "datadog", "status": "passed", "monitors": "14"}
|
||||
)
|
||||
assert session.agent.last_observation is not None
|
||||
assert "service: datadog" in session.agent.last_observation
|
||||
assert "monitors: 14" in session.agent.last_observation
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Configuring an integration mid-session refreshes the session's known state.
|
||||
|
||||
Regression: after ``/integrations setup sentry`` saved sentry to the local
|
||||
store, the assistant still answered "Sentry is not integrated" because the
|
||||
session's ``configured_integrations`` snapshot was only taken at REPL boot and
|
||||
never refreshed. The setup/remove (and /mcp connect|disconnect) paths must
|
||||
re-resolve the env + store integration set so the same session reflects the
|
||||
change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry import integrations as _integrations
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _console() -> Console:
|
||||
return Console(force_terminal=False, no_color=True)
|
||||
|
||||
|
||||
def _noop_cli_command(*_args: Any, **_kwargs: Any) -> bool:
|
||||
"""Stand in for the setup/remove subprocess so no child process is spawned."""
|
||||
return True
|
||||
|
||||
|
||||
def test_refresh_integration_state_rehydrates_and_clears_cache(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: ["gitlab", "sentry"],
|
||||
)
|
||||
refreshed = {"gitlab": {"token": "x"}, "sentry": {"dsn": "y"}}
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: refreshed,
|
||||
)
|
||||
session = Session()
|
||||
# Stale boot-time snapshot + a cached resolution from an earlier turn.
|
||||
session.configured_integrations = ("gitlab",)
|
||||
session.configured_integrations_known = True
|
||||
session.resolved_integrations_cache = {"gitlab": {}}
|
||||
|
||||
session.refresh_integration_state()
|
||||
|
||||
assert session.resolved_integrations_cache == refreshed
|
||||
assert session.configured_integrations_known is True
|
||||
assert session.configured_integrations == ("gitlab", "sentry")
|
||||
|
||||
|
||||
def test_setup_subcommand_refreshes_configured_integrations(monkeypatch: Any) -> None:
|
||||
# The setup subprocess is replaced with a no-op; the store mutation is
|
||||
# simulated by flipping what effective resolution returns afterwards.
|
||||
monkeypatch.setattr(_integrations, "run_cli_command", _noop_cli_command)
|
||||
|
||||
store: dict[str, dict[str, Any]] = {"gitlab": {}}
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: list(store),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: dict(store),
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert session.configured_integrations == ("gitlab",)
|
||||
|
||||
# The user runs `/integrations setup sentry`; setup writes sentry to store.
|
||||
store["sentry"] = {}
|
||||
handled = _integrations._cmd_integrations(session, _console(), ["setup", "sentry"])
|
||||
|
||||
assert handled is True
|
||||
assert "sentry" in session.configured_integrations
|
||||
|
||||
|
||||
def test_remove_subcommand_refreshes_configured_integrations(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(_integrations, "run_cli_command", _noop_cli_command)
|
||||
|
||||
store: dict[str, dict[str, Any]] = {"gitlab": {}, "sentry": {}}
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: list(store),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: dict(store),
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert "sentry" in session.configured_integrations
|
||||
|
||||
store.pop("sentry")
|
||||
handled = _integrations._cmd_integrations(session, _console(), ["remove", "sentry"])
|
||||
|
||||
assert handled is True
|
||||
assert "sentry" not in session.configured_integrations
|
||||
|
||||
|
||||
def test_mcp_connect_refreshes_configured_integrations(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(_integrations, "run_cli_command", _noop_cli_command)
|
||||
|
||||
store: dict[str, dict[str, Any]] = {"gitlab": {}}
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: list(store),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: dict(store),
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
|
||||
store["github_mcp"] = {}
|
||||
handled = _integrations._cmd_mcp(session, _console(), ["connect", "github_mcp"])
|
||||
|
||||
assert handled is True
|
||||
assert "github_mcp" in session.configured_integrations
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from surfaces.interactive_shell.command_registry import repl_data
|
||||
|
||||
|
||||
def test_configured_integration_names_reads_catalog_without_verify(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"integrations.verify.resolve_effective_integrations",
|
||||
lambda: {"aws": {}, "grafana": {}},
|
||||
)
|
||||
verify_calls: list[str | None] = []
|
||||
monkeypatch.setattr(
|
||||
"integrations.verify.verify_integrations",
|
||||
lambda service=None, **_kwargs: verify_calls.append(service) or [],
|
||||
)
|
||||
|
||||
assert repl_data.configured_integration_names() == ["aws", "grafana"]
|
||||
assert verify_calls == []
|
||||
|
||||
|
||||
def test_verify_integration_checks_one_service(monkeypatch) -> None:
|
||||
calls: list[str | None] = []
|
||||
|
||||
def _verify(service: str | None = None, **kwargs: object) -> list[dict[str, str]]:
|
||||
calls.append(service)
|
||||
return [{"service": service or "", "source": "env", "status": "ok", "detail": "ok"}]
|
||||
|
||||
monkeypatch.setattr("integrations.verify.verify_integrations", _verify)
|
||||
|
||||
row = repl_data.verify_integration("aws")
|
||||
|
||||
assert calls == ["aws"]
|
||||
assert row is not None
|
||||
assert row["service"] == "aws"
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for slash-command MCP catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.agent_harness.tools.action_tools import get_action_tool
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS
|
||||
from surfaces.interactive_shell.command_registry.slash_catalog import (
|
||||
_MCP_BY_COMMAND,
|
||||
build_slash_command_specs,
|
||||
format_slash_catalog_text,
|
||||
slash_invoke_input_schema,
|
||||
slash_invoke_tool_description,
|
||||
)
|
||||
|
||||
_MIN_LLM_DESCRIPTION_LEN = 20
|
||||
|
||||
|
||||
def test_slash_catalog_covers_all_registered_commands() -> None:
|
||||
registered = set(SLASH_COMMANDS.keys())
|
||||
catalogued = set(_MCP_BY_COMMAND)
|
||||
missing = sorted(registered - catalogued)
|
||||
stale = sorted(catalogued - registered)
|
||||
assert not missing, (
|
||||
"Add _MCP_BY_COMMAND entries in surfaces/interactive_shell/command_registry/"
|
||||
f"slash_catalog.py for: {missing}. See surfaces/interactive_shell/AGENTS.md "
|
||||
"(Slash commands → REPL + CLI parity)."
|
||||
)
|
||||
assert not stale, (
|
||||
"Remove stale _MCP_BY_COMMAND keys from slash_catalog.py (no longer in "
|
||||
f"SLASH_COMMANDS): {stale}"
|
||||
)
|
||||
specs = build_slash_command_specs()
|
||||
assert len(specs) == len(SLASH_COMMANDS)
|
||||
|
||||
|
||||
def test_slash_command_specs_have_mcp_metadata() -> None:
|
||||
for spec in build_slash_command_specs():
|
||||
assert len(spec.llm_description) >= _MIN_LLM_DESCRIPTION_LEN, spec.name
|
||||
assert spec.use_cases, spec.name
|
||||
|
||||
|
||||
def test_slash_invoke_tool_description_lists_every_command() -> None:
|
||||
description = slash_invoke_tool_description()
|
||||
for name in SLASH_COMMANDS:
|
||||
assert name in description
|
||||
|
||||
|
||||
def test_slash_invoke_description_is_not_a_natural_language_router() -> None:
|
||||
description = slash_invoke_tool_description().lower()
|
||||
|
||||
assert "explicit slash-command operations" in description
|
||||
assert "do not use this as a natural-language router" in description
|
||||
assert "assistant_handoff" in description
|
||||
assert "read-only discovery" in description
|
||||
|
||||
|
||||
def test_slash_invoke_schema_enum_matches_slash_commands() -> None:
|
||||
schema = slash_invoke_input_schema()
|
||||
command = schema["properties"]["command"]
|
||||
assert set(command["enum"]) == set(SLASH_COMMANDS.keys())
|
||||
|
||||
|
||||
def test_registered_slash_invoke_uses_catalog() -> None:
|
||||
entry = get_action_tool("slash_invoke")
|
||||
assert entry is not None
|
||||
assert len(entry.description) > 200
|
||||
assert set(entry.input_schema["properties"]["command"]["enum"]) == set(SLASH_COMMANDS.keys())
|
||||
|
||||
|
||||
def test_format_slash_catalog_text_compact_is_non_empty() -> None:
|
||||
text = format_slash_catalog_text(compact=True)
|
||||
assert text
|
||||
assert "**/health**" in text
|
||||
|
||||
|
||||
def test_model_catalog_excludes_natural_language_status_questions() -> None:
|
||||
spec = next(spec for spec in build_slash_command_specs() if spec.name == "/model")
|
||||
|
||||
assert "explicit /model command operations" in spec.llm_description.lower()
|
||||
assert "asks to run /model show" in " ".join(spec.use_cases).lower()
|
||||
|
||||
anti_examples = " ".join(spec.anti_examples).lower()
|
||||
assert "which model is being used now" in anti_examples
|
||||
assert "what model/provider" in anti_examples
|
||||
assert "openai is configured" in anti_examples
|
||||
assert "assistant_handoff" in anti_examples
|
||||
|
||||
|
||||
def test_status_and_tools_catalog_exclude_natural_language_status_questions() -> None:
|
||||
specs = {spec.name: spec for spec in build_slash_command_specs()}
|
||||
|
||||
status = specs["/status"]
|
||||
assert "explicit /status command operation" in status.llm_description.lower()
|
||||
assert "explicitly types /status" in " ".join(status.use_cases).lower()
|
||||
assert "current session status" in " ".join(status.anti_examples).lower()
|
||||
assert "assistant_handoff" in " ".join(status.anti_examples).lower()
|
||||
|
||||
tools = specs["/tools"]
|
||||
assert "explicit /tools command operation" in tools.llm_description.lower()
|
||||
assert "explicitly types /tools" in " ".join(tools.use_cases).lower()
|
||||
assert "what tools or capabilities" in " ".join(tools.anti_examples).lower()
|
||||
assert "assistant_handoff" in " ".join(tools.anti_examples).lower()
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Tests for slash typo suggestion helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS, dispatch_slash
|
||||
from surfaces.interactive_shell.command_registry.suggestions import (
|
||||
format_invalid_subcommand_message,
|
||||
format_unknown_slash_message,
|
||||
resolve_literal_slash_typo,
|
||||
subcommand_hints,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.action_turn import run_action_tool_turn
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def test_format_unknown_slash_message_without_suggestion_points_to_help() -> None:
|
||||
message = format_unknown_slash_message(
|
||||
"/made-up",
|
||||
command_names=tuple(SLASH_COMMANDS),
|
||||
)
|
||||
assert message == "Unknown command: /made-up. Type /help for the full command list."
|
||||
|
||||
|
||||
def test_format_unknown_slash_message_with_suggestion() -> None:
|
||||
message = format_unknown_slash_message(
|
||||
"/modle",
|
||||
command_names=tuple(SLASH_COMMANDS),
|
||||
)
|
||||
assert "Did you mean /model?" in message
|
||||
assert "Type /help" in message
|
||||
|
||||
|
||||
def test_resolve_literal_slash_typo_unknown_root() -> None:
|
||||
typo = resolve_literal_slash_typo("/invest", SLASH_COMMANDS)
|
||||
assert typo is not None
|
||||
assert typo.outcome == "unknown_command"
|
||||
assert "Did you mean /investigate?" in typo.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command_line",
|
||||
[
|
||||
"/resume redis",
|
||||
"/help model",
|
||||
"/help /model",
|
||||
"/integrations ls",
|
||||
"/tools ls",
|
||||
"/tools tool",
|
||||
"/mcp ls",
|
||||
],
|
||||
)
|
||||
def test_resolve_literal_slash_typo_allows_free_form_first_args(command_line: str) -> None:
|
||||
assert resolve_literal_slash_typo(command_line, SLASH_COMMANDS) is None
|
||||
|
||||
|
||||
def test_dispatch_invalid_subcommand_is_handled_by_command_handler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.integrations.repl_data.load_verified_integrations",
|
||||
lambda: [],
|
||||
)
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/integrations bogus", session, console) is True
|
||||
assert "unknown subcommand" in buf.getvalue().lower()
|
||||
assert resolve_literal_slash_typo("/integrations bogus", SLASH_COMMANDS) is None
|
||||
|
||||
|
||||
def test_subcommand_hints_ignores_usage_placeholders() -> None:
|
||||
resume = SLASH_COMMANDS["/resume"]
|
||||
assert subcommand_hints(resume) == ()
|
||||
help_cmd = SLASH_COMMANDS["/help"]
|
||||
assert subcommand_hints(help_cmd) == ()
|
||||
|
||||
|
||||
def test_format_invalid_subcommand_message_lists_known_subcommands() -> None:
|
||||
cmd = SLASH_COMMANDS["/integrations"]
|
||||
message = format_invalid_subcommand_message(cmd, ["bogus"])
|
||||
assert "Invalid subcommand: bogus" in message
|
||||
assert "/integrations list" in message
|
||||
|
||||
|
||||
def test_dispatch_unknown_command_records_full_response_and_outcome() -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/modle", session, console) is True
|
||||
output = buf.getvalue()
|
||||
assert "Unknown command" in output
|
||||
latest = session.history[-1]
|
||||
assert latest["ok"] is False
|
||||
assert latest["slash_outcome"] == "unknown_command"
|
||||
assert latest["response_text"] == latest["response_text"].strip()
|
||||
assert "Type /help" in latest["response_text"]
|
||||
|
||||
|
||||
def test_run_action_tool_turn_handles_unknown_literal_slash_before_tool_validation() -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
result = run_action_tool_turn("/invest", session, console)
|
||||
assert result.handled is True
|
||||
assert result.response_text
|
||||
assert "Unknown command" in result.response_text
|
||||
assert "Unknown command" in buf.getvalue()
|
||||
latest = session.history[-1]
|
||||
assert latest["type"] == "slash"
|
||||
assert latest["slash_outcome"] == "unknown_command"
|
||||
assert latest["response_text"] == result.response_text
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Shared fixtures for interactive-shell tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _repl_execution_policy_auto_yes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Elevated REPL actions prompt for confirmation; stdin is non-TTY under pytest."""
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.execution_confirm.DEFAULT_CONFIRM_FN",
|
||||
lambda _prompt: "y",
|
||||
)
|
||||
monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_active_theme() -> None:
|
||||
"""Reset the active theme to green before each test.
|
||||
|
||||
``set_active_theme()`` mutates module-level state in
|
||||
``platform.terminal.theme``, which persists across tests
|
||||
and can cause order-dependent failures.
|
||||
"""
|
||||
from platform.terminal.theme import set_active_theme
|
||||
|
||||
set_active_theme("green")
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Tests for interactive shell command history helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.prompt_history import load_command_history_entries
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _capture() -> tuple[object, object]:
|
||||
from io import StringIO
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
buf = StringIO()
|
||||
console = Console(file=buf, force_terminal=True, color_system="truecolor", highlight=False)
|
||||
return console, buf
|
||||
|
||||
|
||||
def test_load_command_history_entries_returns_empty_on_mkdir_oserror(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mock_path = MagicMock(spec=Path)
|
||||
mock_parent = MagicMock()
|
||||
mock_path.parent = mock_parent
|
||||
mock_parent.mkdir.side_effect = OSError(13, "Permission denied")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.prompt_history.storage.prompt_history_path",
|
||||
lambda: mock_path,
|
||||
)
|
||||
|
||||
assert load_command_history_entries() == []
|
||||
|
||||
|
||||
def test_history_slash_command_does_not_raise_when_history_dir_unwritable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Regression: /history must not crash the REPL when OPENSRE_HOME_DIR is not writable."""
|
||||
mock_path = MagicMock(spec=Path)
|
||||
mock_parent = MagicMock()
|
||||
mock_path.parent = mock_parent
|
||||
mock_parent.mkdir.side_effect = OSError(30, "Read-only file system")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.prompt_history.storage.prompt_history_path",
|
||||
lambda: mock_path,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/history", session, console) is True
|
||||
assert "no history yet" in buf.getvalue()
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tests for /history subcommands and /privacy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from prompt_toolkit.history import FileHistory, InMemoryHistory
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell import prompt_history as history_module
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.prompt_history.policy import RedactingFileHistory
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def _redirect_history_path(monkeypatch: pytest.MonkeyPatch, target: Path) -> None:
|
||||
"""Point ``prompt_history_path()`` at a tmp file across all importers."""
|
||||
from surfaces.interactive_shell.command_registry import privacy_cmds as privacy_cmds_module
|
||||
from surfaces.interactive_shell.prompt_history import storage as history_storage
|
||||
|
||||
monkeypatch.setattr(history_module, "prompt_history_path", lambda: target)
|
||||
monkeypatch.setattr(privacy_cmds_module, "prompt_history_path", lambda: target)
|
||||
monkeypatch.setattr(history_storage, "prompt_history_path", lambda: target)
|
||||
|
||||
|
||||
class TestHistoryClear:
|
||||
def test_truncates_existing_file(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
history_file.write_text("# 2026-01-01 00:00:00\n+older\n", encoding="utf-8")
|
||||
_redirect_history_path(monkeypatch, history_file)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/history clear", session, console) is True
|
||||
|
||||
assert history_file.read_text(encoding="utf-8") == ""
|
||||
assert "cleared" in buf.getvalue()
|
||||
|
||||
def test_handles_missing_file(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
history_file = tmp_path / "history" # never created
|
||||
_redirect_history_path(monkeypatch, history_file)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/history clear", session, console) is True
|
||||
assert "cleared" in buf.getvalue()
|
||||
|
||||
|
||||
class TestHistoryToggle:
|
||||
def test_off_then_on_flips_paused_flag(self, tmp_path: Path) -> None:
|
||||
backend = RedactingFileHistory(str(tmp_path / "history"))
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
|
||||
console, _ = _capture()
|
||||
dispatch_slash("/history off", session, console)
|
||||
assert backend.paused is True
|
||||
|
||||
dispatch_slash("/history on", session, console)
|
||||
assert backend.paused is False
|
||||
|
||||
def test_off_with_in_memory_backend_is_noop_message(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = InMemoryHistory()
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/history off", session, console)
|
||||
assert "in-memory" in buf.getvalue() or "not persisting" in buf.getvalue()
|
||||
|
||||
def test_file_history_backend_reports_runtime_pause_is_unavailable(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = FileHistory(str(tmp_path / "history"))
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/history off", session, console)
|
||||
assert "without redaction" in buf.getvalue()
|
||||
|
||||
def test_file_history_backend_reports_persistence_already_on(self, tmp_path: Path) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = FileHistory(str(tmp_path / "history"))
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/history on", session, console)
|
||||
assert "already on" in buf.getvalue()
|
||||
|
||||
|
||||
class TestHistoryRetention:
|
||||
def test_sets_cap_and_prunes(self, tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file), max_entries=10)
|
||||
for i in range(5):
|
||||
backend.store_string(f"entry-{i}")
|
||||
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, _ = _capture()
|
||||
dispatch_slash("/history retention 2", session, console)
|
||||
|
||||
persisted = list(reversed(list(backend.load_history_strings())))
|
||||
assert persisted == ["entry-3", "entry-4"]
|
||||
assert backend._max_entries == 2
|
||||
|
||||
def test_rejects_non_integer(self, tmp_path: Path) -> None:
|
||||
backend = RedactingFileHistory(str(tmp_path / "history"))
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/history retention oops", session, console)
|
||||
assert "non-negative integer" in buf.getvalue()
|
||||
|
||||
def test_rejects_negative(self, tmp_path: Path) -> None:
|
||||
backend = RedactingFileHistory(str(tmp_path / "history"))
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/history retention -1", session, console)
|
||||
assert "non-negative integer" in buf.getvalue()
|
||||
|
||||
def test_zero_sets_unlimited_without_crashing(self, tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file), max_entries=2)
|
||||
for i in range(4):
|
||||
backend.store_string(f"entry-{i}")
|
||||
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, buf = _capture()
|
||||
|
||||
assert dispatch_slash("/history retention 0", session, console) is True
|
||||
persisted = list(reversed(list(backend.load_history_strings())))
|
||||
assert persisted == ["entry-2", "entry-3"]
|
||||
assert backend.max_entries == 0
|
||||
assert "retention cap set to 0" in buf.getvalue()
|
||||
|
||||
|
||||
class TestHistoryUnknownSubcommand:
|
||||
def test_prints_usage(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/history bogus", session, console)
|
||||
assert "usage:" in buf.getvalue()
|
||||
|
||||
|
||||
class TestPrivacyCommand:
|
||||
def test_shows_redacting_state(self, tmp_path: Path) -> None:
|
||||
backend = RedactingFileHistory(str(tmp_path / "history"))
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/privacy", session, console)
|
||||
out = buf.getvalue()
|
||||
assert "Privacy settings" in out
|
||||
assert "persistence" in out
|
||||
assert "redaction" in out
|
||||
assert "threat model" in out
|
||||
|
||||
def test_shows_paused_state_when_off(self, tmp_path: Path) -> None:
|
||||
backend = RedactingFileHistory(str(tmp_path / "history"))
|
||||
backend.paused = True
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = backend
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/privacy", session, console)
|
||||
assert "paused" in buf.getvalue()
|
||||
|
||||
def test_in_memory_backend_reports_off(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = InMemoryHistory()
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/privacy", session, console)
|
||||
assert "in-memory" in buf.getvalue()
|
||||
|
||||
def test_file_history_backend_reports_no_redaction(self, tmp_path: Path) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_history_backend = FileHistory(str(tmp_path / "history"))
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/privacy", session, console)
|
||||
out = buf.getvalue()
|
||||
assert "on (no redaction)" in out
|
||||
assert "redaction" in out
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Tests for history-redaction patterns, retention, and policy resolution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.prompt_history.policy import (
|
||||
DEFAULT_MAX_ENTRIES,
|
||||
DEFAULT_REDACTION_RULES,
|
||||
HistoryPolicy,
|
||||
RedactingFileHistory,
|
||||
redact_text,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected_marker"),
|
||||
[
|
||||
("AKIAIOSFODNN7EXAMPLE", "[REDACTED:aws_key]"),
|
||||
(
|
||||
"aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY1",
|
||||
"[REDACTED:aws_secret]",
|
||||
),
|
||||
("ghp_" + "a" * 36, "[REDACTED:github_pat]"),
|
||||
("github_pat_" + "x" * 82, "[REDACTED:github_pat]"),
|
||||
("sk-ant-" + "y" * 90, "[REDACTED:anthropic_key]"),
|
||||
("sk-" + "z" * 48, "[REDACTED:openai_key]"),
|
||||
("xoxb-12345-67890-abcdefghijklmn", "[REDACTED:slack_token]"),
|
||||
("sk_live_" + "Q" * 24, "[REDACTED:stripe_key]"),
|
||||
(
|
||||
"Authorization: Bearer abc123abc123abc123abc123",
|
||||
"Bearer [REDACTED]",
|
||||
),
|
||||
(
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3OD.SflKxwRJSMeKKF2QT4",
|
||||
"[REDACTED:jwt]",
|
||||
),
|
||||
("psql --password=hunter2 -h db", "[REDACTED:password]"),
|
||||
(
|
||||
"-----BEGIN RSA PRIVATE KEY-----\nMIIEvQIBADANBgkq\n-----END RSA PRIVATE KEY-----",
|
||||
"[REDACTED:private_key]",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_each_default_pattern_redacts_known_token(raw: str, expected_marker: str) -> None:
|
||||
assert expected_marker in redact_text(raw)
|
||||
|
||||
|
||||
def test_full_pem_block_is_redacted_including_body_and_footer() -> None:
|
||||
pem = (
|
||||
"-----BEGIN RSA PRIVATE KEY-----\n"
|
||||
"MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDaaaa\n"
|
||||
"bbbbbCCCCCdddddEEEEEfffffGGGGGhhhhhIIIIIjjjjjKKKKKlllll\n"
|
||||
"mmmmmNNNNNoooooPPPPPqqqqqRRRRRsssssTTTTTuuuuuVVVVVwwwww\n"
|
||||
"-----END RSA PRIVATE KEY-----"
|
||||
)
|
||||
redacted = redact_text(pem)
|
||||
assert "[REDACTED:private_key]" in redacted
|
||||
# The body and footer must be gone — no base64 or END marker can leak to disk.
|
||||
assert "MIIEvQIBADANBgkq" not in redacted
|
||||
assert "-----END" not in redacted
|
||||
assert "-----BEGIN" not in redacted
|
||||
|
||||
|
||||
def test_openssh_pem_block_is_redacted_end_to_end() -> None:
|
||||
pem = (
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----\n"
|
||||
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFw\n"
|
||||
"-----END OPENSSH PRIVATE KEY-----"
|
||||
)
|
||||
redacted = redact_text(pem)
|
||||
assert "[REDACTED:private_key]" in redacted
|
||||
assert "b3BlbnNzaC1rZXktdjEA" not in redacted
|
||||
|
||||
|
||||
def test_pem_block_inside_history_entry_does_not_leak_to_disk(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file))
|
||||
pem = (
|
||||
"ssh-add - <<'EOF'\n"
|
||||
"-----BEGIN EC PRIVATE KEY-----\n"
|
||||
"MHcCAQEEIBcccDDDDDeeeeeFFFFFggggghhhhhIIIIIjjjjjKKKKKlllll\n"
|
||||
"mmmmmNNNNNoooooPPPPPqqqqq\n"
|
||||
"-----END EC PRIVATE KEY-----\n"
|
||||
"EOF"
|
||||
)
|
||||
backend.store_string(pem)
|
||||
contents = history_file.read_text(encoding="utf-8")
|
||||
assert "[REDACTED:private_key]" in contents
|
||||
assert "MHcCAQEEIBcccDDDDD" not in contents
|
||||
assert "-----END EC PRIVATE KEY-----" not in contents
|
||||
assert "-----BEGIN EC PRIVATE KEY-----" not in contents
|
||||
|
||||
|
||||
def test_natural_language_is_left_alone() -> None:
|
||||
text = "investigate api errors after the redis cluster restarted at 12:30"
|
||||
assert redact_text(text) == text
|
||||
|
||||
|
||||
def test_only_secret_segment_is_replaced() -> None:
|
||||
raw = "kubectl describe pod and AKIAIOSFODNN7EXAMPLE was in the env"
|
||||
out = redact_text(raw)
|
||||
assert "[REDACTED:aws_key]" in out
|
||||
assert out.startswith("kubectl describe pod and ")
|
||||
assert out.endswith(" was in the env")
|
||||
|
||||
|
||||
def test_redacting_file_history_writes_redacted_only(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file))
|
||||
backend.store_string("AKIAIOSFODNN7EXAMPLE plus other tokens")
|
||||
contents = history_file.read_text(encoding="utf-8")
|
||||
assert "AKIAIOSFODNN7EXAMPLE" not in contents
|
||||
assert "[REDACTED:aws_key]" in contents
|
||||
|
||||
|
||||
def test_paused_backend_does_not_persist(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file))
|
||||
backend.paused = True
|
||||
backend.store_string("any string")
|
||||
assert not history_file.exists() or history_file.read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_retention_cap_drops_oldest_entries(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file), max_entries=3)
|
||||
for i in range(5):
|
||||
backend.store_string(f"entry-{i}")
|
||||
|
||||
persisted = list(reversed(list(backend.load_history_strings())))
|
||||
assert persisted == ["entry-2", "entry-3", "entry-4"]
|
||||
|
||||
|
||||
def test_zero_retention_keeps_unlimited(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file), max_entries=0)
|
||||
for i in range(5):
|
||||
backend.store_string(f"entry-{i}")
|
||||
|
||||
persisted = list(reversed(list(backend.load_history_strings())))
|
||||
assert persisted == [f"entry-{i}" for i in range(5)]
|
||||
|
||||
|
||||
def test_policy_defaults_match_documented_shape() -> None:
|
||||
policy = HistoryPolicy.load()
|
||||
assert policy.enabled is True
|
||||
assert policy.redact is True
|
||||
assert policy.max_entries == DEFAULT_MAX_ENTRIES
|
||||
|
||||
|
||||
def test_env_var_disables_persistence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENSRE_HISTORY_ENABLED", "0")
|
||||
policy = HistoryPolicy.load()
|
||||
assert policy.enabled is False
|
||||
|
||||
|
||||
def test_env_var_disables_redaction(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENSRE_HISTORY_REDACT", "false")
|
||||
policy = HistoryPolicy.load()
|
||||
assert policy.redact is False
|
||||
|
||||
|
||||
def test_env_var_overrides_max_entries(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENSRE_HISTORY_MAX_ENTRIES", "42")
|
||||
policy = HistoryPolicy.load()
|
||||
assert policy.max_entries == 42
|
||||
|
||||
|
||||
def test_env_var_garbage_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENSRE_HISTORY_MAX_ENTRIES", "not-a-number")
|
||||
policy = HistoryPolicy.load()
|
||||
assert policy.max_entries == DEFAULT_MAX_ENTRIES
|
||||
|
||||
|
||||
def test_file_settings_used_when_env_absent() -> None:
|
||||
policy = HistoryPolicy.load({"enabled": False, "redact": False, "max_entries": 100})
|
||||
assert policy.enabled is False
|
||||
assert policy.redact is False
|
||||
assert policy.max_entries == 100
|
||||
|
||||
|
||||
def test_quoted_false_like_file_settings_are_parsed_as_disabled() -> None:
|
||||
policy = HistoryPolicy.load({"enabled": "false", "redact": "0"})
|
||||
assert policy.enabled is False
|
||||
assert policy.redact is False
|
||||
|
||||
|
||||
def test_prune_to_cap_is_safe_when_cap_is_zero(tmp_path: Path) -> None:
|
||||
history_file = tmp_path / "history"
|
||||
backend = RedactingFileHistory(str(history_file), max_entries=0)
|
||||
backend.store_string("entry-0")
|
||||
backend._prune_to_cap()
|
||||
|
||||
persisted = list(reversed(list(backend.load_history_strings())))
|
||||
assert persisted == ["entry-0"]
|
||||
|
||||
|
||||
def test_default_pattern_set_size_is_stable() -> None:
|
||||
# Catch accidental rule deletions in PRs.
|
||||
assert len(DEFAULT_REDACTION_RULES) >= 12
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Tests for interactive-shell LLM grounding reference corpora."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Tests for the AGENTS.md grounding helpers used by the interactive shell."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding import agents_md_reference
|
||||
from core.agent_harness.grounding._cache import excerpt
|
||||
from core.agent_harness.grounding.agents_md_reference import (
|
||||
AgentsMdFile,
|
||||
AgentsMdReference,
|
||||
)
|
||||
|
||||
|
||||
def _write(root: Path, relpath: str, content: str) -> None:
|
||||
path = root / relpath
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _seed_agents_md(root: Path) -> None:
|
||||
_write(root, "AGENTS.md", "# Repo map\n\nTop-level guidance.\n")
|
||||
_write(root, "core/llm/AGENTS.md", "# Hosted LLM Runtime\n\nLLM API clients.\n")
|
||||
_write(root, "integrations/llm_cli/AGENTS.md", "# llm_cli\n\nSubprocess CLIs.\n")
|
||||
# Files that MUST be skipped — tests, vcs, caches, virtualenv, vendored deps.
|
||||
_write(root, "tests/AGENTS.md", "should be skipped")
|
||||
_write(root, ".git/AGENTS.md", "should be skipped")
|
||||
_write(root, "__pycache__/AGENTS.md", "should be skipped")
|
||||
_write(root, "node_modules/some-pkg/AGENTS.md", "should be skipped")
|
||||
_write(root, ".venv/lib/python3.13/site-packages/foo/AGENTS.md", "should be skipped")
|
||||
|
||||
|
||||
class TestDiscoverAgentsMdFiles:
|
||||
def test_returns_empty_list_when_root_missing(self, tmp_path: Path) -> None:
|
||||
missing = tmp_path / "no-such-dir"
|
||||
assert AgentsMdReference().discover(missing) == []
|
||||
|
||||
def test_walks_root_and_skips_excluded_dirs(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
relpaths = {f.relpath for f in files}
|
||||
assert "AGENTS.md" in relpaths
|
||||
assert "core/llm/AGENTS.md" in relpaths
|
||||
assert "integrations/llm_cli/AGENTS.md" in relpaths
|
||||
# None of the skip-dir paths should leak into the index.
|
||||
for r in relpaths:
|
||||
assert not r.startswith("tests/")
|
||||
assert not r.startswith(".git/")
|
||||
assert not r.startswith("__pycache__/")
|
||||
assert not r.startswith("node_modules/")
|
||||
assert not r.startswith(".venv/")
|
||||
|
||||
def test_results_are_sorted_by_relpath(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
relpaths = [f.relpath for f in files]
|
||||
assert relpaths == sorted(relpaths)
|
||||
|
||||
def test_body_is_verbatim_no_frontmatter_stripping(self, tmp_path: Path) -> None:
|
||||
# Unlike docs_reference, AGENTS.md files are plain Markdown — frontmatter
|
||||
# delimiters (if a contributor adds them) must be preserved verbatim.
|
||||
body = "---\nfoo: bar\n---\n\n# Title\n\nBody.\n"
|
||||
_write(tmp_path, "AGENTS.md", body)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
assert len(files) == 1
|
||||
assert files[0].body == body
|
||||
|
||||
|
||||
class TestBuildAgentsMdReferenceText:
|
||||
def test_returns_empty_when_no_files_present(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path / "missing")
|
||||
assert AgentsMdReference().build_text() == ""
|
||||
|
||||
def test_concatenates_each_file_with_labelled_header(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path)
|
||||
text = AgentsMdReference().build_text()
|
||||
# Root file must be disambiguated from per-package files.
|
||||
assert "=== AGENTS.md (root) ===" in text
|
||||
assert "=== core/llm/AGENTS.md ===" in text
|
||||
assert "=== integrations/llm_cli/AGENTS.md ===" in text
|
||||
# Bodies must appear in the concatenated block.
|
||||
assert "Top-level guidance." in text
|
||||
assert "LLM API clients." in text
|
||||
assert "Subprocess CLIs." in text
|
||||
|
||||
def test_caps_total_to_max_chars(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path)
|
||||
text = AgentsMdReference().build_text(max_chars=100)
|
||||
# Allow a small margin for the truncation marker appended at the cap.
|
||||
assert len(text) <= 200
|
||||
assert "truncated" in text
|
||||
|
||||
|
||||
class TestExcerpt:
|
||||
def test_returns_full_body_when_short(self) -> None:
|
||||
assert excerpt("short body", max_chars=100) == "short body"
|
||||
|
||||
def test_truncates_long_body_at_paragraph_boundary(self) -> None:
|
||||
body = ("paragraph one. " * 10) + "\n\n" + ("paragraph two. " * 10)
|
||||
out = excerpt(body, max_chars=80)
|
||||
assert "truncated" in out
|
||||
# The cut should happen at or before the second paragraph, since the
|
||||
# rfind for "\n\n" before max_chars is the preferred cutoff.
|
||||
assert "paragraph two" not in out
|
||||
|
||||
|
||||
class TestAgentsMdFileDataclass:
|
||||
def test_is_hashable_and_immutable(self) -> None:
|
||||
f = AgentsMdFile(relpath="AGENTS.md", body="hello")
|
||||
assert f in {f}
|
||||
|
||||
|
||||
class TestAgentsMdGroundingCache:
|
||||
def test_cache_maxsize_matches_implementation(self) -> None:
|
||||
stats = AgentsMdReference().stats()
|
||||
assert stats.maxsize == 32
|
||||
|
||||
def test_repeated_discover_hits_parse_cache(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
ref = AgentsMdReference()
|
||||
ref.discover(tmp_path)
|
||||
info1 = ref.stats()
|
||||
ref.discover(tmp_path)
|
||||
info2 = ref.stats()
|
||||
assert info2.hits == info1.hits + 1
|
||||
assert info2.misses == info1.misses
|
||||
|
||||
def test_invalidate_resets_stats(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
ref = AgentsMdReference()
|
||||
ref.discover(tmp_path)
|
||||
ref.discover(tmp_path)
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
cleared = ref.stats()
|
||||
assert cleared.hits == 0
|
||||
assert cleared.misses == 0
|
||||
assert cleared.currsize == 0
|
||||
|
||||
def test_file_edit_invalidates_and_refreshes_content(self, tmp_path: Path) -> None:
|
||||
ref = AgentsMdReference()
|
||||
_write(tmp_path, "AGENTS.md", "# Repo map\n\nOld content.\n")
|
||||
files1 = ref.discover(tmp_path)
|
||||
assert any("Old content" in f.body for f in files1)
|
||||
|
||||
# Bump mtime well beyond filesystem mtime resolution so the
|
||||
# fingerprint definitely changes on the next discover call.
|
||||
target = tmp_path / "AGENTS.md"
|
||||
target.write_text("# Repo map\n\nNew refreshed content.\n", encoding="utf-8")
|
||||
st = target.stat()
|
||||
os.utime(target, ns=(st.st_atime_ns, st.st_mtime_ns + 2_000_000_000))
|
||||
|
||||
files2 = ref.discover(tmp_path)
|
||||
assert any("New refreshed content" in f.body for f in files2)
|
||||
assert not any("Old content" in f.body for f in files2)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Tests for interactive-shell CLI reference grounding cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import click
|
||||
import pytest
|
||||
|
||||
import surfaces.interactive_shell.grounding.cli_reference as cli_reference_module
|
||||
from surfaces.interactive_shell.session.session import Session
|
||||
|
||||
|
||||
def _reference_with_cli() -> cli_reference_module.CliReference:
|
||||
"""Return a :class:`CliReference` wired to the shell's CLI command group.
|
||||
|
||||
CLI catalog assembly lives in ``surfaces/`` (T-05 — see issue #3538); tests
|
||||
inject the group the same way ``ShellPromptContextProvider`` does.
|
||||
"""
|
||||
from surfaces.cli.__main__ import cli
|
||||
|
||||
ref = cli_reference_module.CliReference()
|
||||
ref.set_command_group_provider(lambda: cli)
|
||||
return ref
|
||||
|
||||
|
||||
def test_second_build_is_cache_hit() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
s1 = ref.stats()
|
||||
ref.build_text()
|
||||
s2 = ref.stats()
|
||||
assert s2.hits == s1.hits + 1
|
||||
assert s2.misses == s1.misses
|
||||
|
||||
|
||||
def test_cold_build_is_silent(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
from surfaces.cli.__main__ import cli
|
||||
|
||||
text = _reference_with_cli().build_text()
|
||||
captured = capsys.readouterr()
|
||||
first_command = sorted(cli.commands.keys())[0]
|
||||
|
||||
assert captured.out == ""
|
||||
assert captured.err == ""
|
||||
assert "=== opensre --help ===" in text
|
||||
assert f"=== opensre {first_command} --help ===" in text
|
||||
assert f"Usage: opensre {first_command}" in text
|
||||
|
||||
|
||||
def test_invalidate_forces_rebuild_miss() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
s1 = ref.stats()
|
||||
assert s1.misses == 1
|
||||
ref.invalidate()
|
||||
assert ref.stats().misses == 0
|
||||
ref.build_text()
|
||||
s2 = ref.stats()
|
||||
assert s2.misses == 1
|
||||
assert s2.cached is True
|
||||
|
||||
|
||||
def test_signature_change_busts_cli_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
ref = _reference_with_cli()
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_current_cli_signature",
|
||||
lambda *_args, **_kwargs: "sig-a",
|
||||
)
|
||||
ref.build_text()
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_current_cli_signature",
|
||||
lambda *_args, **_kwargs: "sig-b",
|
||||
)
|
||||
ref.build_text()
|
||||
stats = ref.stats()
|
||||
assert stats.misses >= 2
|
||||
assert stats.signature == "sig-b"
|
||||
|
||||
|
||||
def test_invalidate_resets_hit_miss_counters() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
ref.build_text()
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
s = ref.stats()
|
||||
assert s.hits == 0
|
||||
assert s.misses == 0
|
||||
|
||||
|
||||
def test_non_cacheable_short_output_skips_store(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_build_cli_reference_text_uncached",
|
||||
lambda *_args, **_kwargs: "too short",
|
||||
)
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
ref.build_text()
|
||||
stats = ref.stats()
|
||||
assert stats.cached is False
|
||||
assert stats.misses >= 2
|
||||
|
||||
|
||||
def test_non_cacheable_long_without_sentinel_skips_store(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
filler = "x" * 120
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_build_cli_reference_text_uncached",
|
||||
lambda *_args, **_kwargs: filler,
|
||||
)
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
assert ref.stats().cached is False
|
||||
|
||||
|
||||
def test_reference_without_provider_returns_placeholder() -> None:
|
||||
"""Without a command-group provider the cache emits a placeholder — no surface imports."""
|
||||
ref = cli_reference_module.CliReference()
|
||||
text = ref.build_text()
|
||||
assert "=== opensre --help ===" in text
|
||||
assert "not available in this runtime" in text
|
||||
# Placeholder is intentionally short-lived: it must not populate the cache.
|
||||
assert ref.stats().cached is False
|
||||
|
||||
|
||||
def test_command_group_provider_is_bound_lazily() -> None:
|
||||
"""The provider is invoked only when :meth:`build_text` runs, not at bind time."""
|
||||
calls: list[int] = []
|
||||
|
||||
def _provider() -> click.Command:
|
||||
calls.append(1)
|
||||
group = click.Group("opensre")
|
||||
group.add_command(click.Command("noop"))
|
||||
return group
|
||||
|
||||
ref = cli_reference_module.CliReference()
|
||||
ref.set_command_group_provider(_provider)
|
||||
assert not calls
|
||||
ref.build_text()
|
||||
assert calls == [1]
|
||||
|
||||
|
||||
def test_shell_prompt_context_provider_includes_cli_reference() -> None:
|
||||
provider = cli_reference_module.shell_prompt_context_provider(Session())
|
||||
text = provider.cli_reference()
|
||||
assert "=== opensre --help ===" in text
|
||||
assert "Usage: opensre" in text
|
||||
|
||||
|
||||
def test_shell_prompt_context_provider_reuses_session_cli_cache() -> None:
|
||||
session = Session()
|
||||
first = cli_reference_module.shell_prompt_context_provider(session)
|
||||
second = cli_reference_module.shell_prompt_context_provider(session)
|
||||
first.cli_reference()
|
||||
second.cli_reference()
|
||||
assert second._cli.stats().hits >= 1 # noqa: SLF001 - session-scoped cache reuse
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Tests for the documentation-grounding helpers used by the interactive shell."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding import docs_reference
|
||||
from core.agent_harness.grounding._cache import excerpt
|
||||
from core.agent_harness.grounding.docs_reference import (
|
||||
DocPage,
|
||||
DocsReference,
|
||||
_query_tokens,
|
||||
build_docs_index,
|
||||
find_relevant_docs,
|
||||
)
|
||||
|
||||
|
||||
def _write_doc(root: Path, relpath: str, content: str) -> None:
|
||||
path = root / relpath
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _seed_docs(root: Path) -> None:
|
||||
_write_doc(
|
||||
root,
|
||||
"datadog.mdx",
|
||||
'---\ntitle: "Datadog"\n---\n\n'
|
||||
"### Step 1: Create API Key\n\n"
|
||||
"In Datadog, create an API Key under organizational settings.\n\n"
|
||||
"### Step 2: Configure OpenSRE\n\n"
|
||||
"Set DD_API_KEY and DD_APP_KEY in your environment.\n",
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"deployment.mdx",
|
||||
'---\ntitle: "Deployment"\n---\n\n'
|
||||
"OpenSRE can deploy to Railway or EC2.\n\n"
|
||||
"Use `opensre remote` to connect to a deployed agent.\n",
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"quickstart.mdx",
|
||||
'---\ntitle: "Quickstart"\n---\n\nInstall OpenSRE and run your first investigation.\n',
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"tutorials/investigating-task-failures.mdx",
|
||||
"# Investigating task failures\n\n"
|
||||
"Walk through how to investigate a failed task using OpenSRE.\n",
|
||||
)
|
||||
# Asset content under skip dirs MUST be excluded from the index.
|
||||
_write_doc(root, "images/datadog.mdx", "should be skipped")
|
||||
_write_doc(root, "assets/anything.mdx", "should be skipped")
|
||||
|
||||
|
||||
class TestDiscoverDocs:
|
||||
def test_returns_empty_list_when_root_missing(self, tmp_path: Path) -> None:
|
||||
missing = tmp_path / "no-docs"
|
||||
assert DocsReference().discover(missing) == []
|
||||
|
||||
def test_walks_root_and_skips_asset_dirs(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
slugs = {p.slug for p in pages}
|
||||
assert "datadog" in slugs
|
||||
assert "deployment" in slugs
|
||||
assert "quickstart" in slugs
|
||||
assert "investigating-task-failures" in slugs
|
||||
# Anything under images/ or assets/ must be skipped.
|
||||
relpaths = {p.relpath for p in pages}
|
||||
assert all(not r.startswith("images/") for r in relpaths)
|
||||
assert all(not r.startswith("assets/") for r in relpaths)
|
||||
|
||||
def test_extracts_title_from_frontmatter(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
assert by_slug["datadog"].title == "Datadog"
|
||||
assert by_slug["deployment"].title == "Deployment"
|
||||
|
||||
def test_falls_back_to_first_heading_when_no_frontmatter(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
assert by_slug["investigating-task-failures"].title == "Investigating task failures"
|
||||
|
||||
def test_strips_frontmatter_from_body(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
# Frontmatter delimiters and the title line must NOT appear in the body
|
||||
# (they would otherwise leak into the LLM grounding context).
|
||||
assert "title:" not in by_slug["datadog"].body
|
||||
assert by_slug["datadog"].body.lstrip().startswith("###")
|
||||
|
||||
|
||||
class TestQueryTokens:
|
||||
def test_strips_stopwords_and_short_tokens(self) -> None:
|
||||
tokens = _query_tokens("How do I configure Datadog?")
|
||||
# Stopwords are removed, 'datadog' / 'configure' remain.
|
||||
assert "datadog" in tokens
|
||||
assert "configure" in tokens
|
||||
assert "how" not in tokens
|
||||
assert "do" not in tokens
|
||||
assert "i" not in tokens
|
||||
|
||||
def test_drops_opensre_brand_token(self) -> None:
|
||||
# Every doc mentions "opensre" so it would otherwise dominate ranking.
|
||||
tokens = _query_tokens("how do I install opensre")
|
||||
assert "opensre" not in tokens
|
||||
assert "install" in tokens
|
||||
|
||||
def test_keeps_two_letter_tokens(self) -> None:
|
||||
tokens = _query_tokens("how do I tune ai vm sizing")
|
||||
assert "ai" in tokens
|
||||
assert "vm" in tokens
|
||||
|
||||
|
||||
class TestFindRelevantDocs:
|
||||
def test_empty_query_returns_empty(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
# Query with only stopwords should not match anything.
|
||||
assert find_relevant_docs("how do I", pages) == []
|
||||
|
||||
def test_ranks_datadog_page_first_for_datadog_query(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("how do I configure Datadog?", pages)
|
||||
assert results, "expected at least one match"
|
||||
assert results[0].slug == "datadog"
|
||||
|
||||
def test_ranks_deployment_page_first_for_deploy_query(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("how do I deploy this?", pages)
|
||||
assert results
|
||||
assert results[0].slug == "deployment"
|
||||
|
||||
def test_caps_results_at_top_n(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("install configure deploy investigate", pages, top_n=2)
|
||||
assert len(results) <= 2
|
||||
|
||||
def test_nested_page_with_weak_match_is_not_dropped_by_depth(self, tmp_path: Path) -> None:
|
||||
"""A page whose only match is a single body token, nested deep enough
|
||||
that the depth penalty equals or exceeds its raw score, must still
|
||||
surface as a lower-ranked result instead of being excluded entirely.
|
||||
|
||||
Regression: previously the depth penalty was applied unconditionally
|
||||
before the score>0 filter, so a page with raw_score=1 at depth=2
|
||||
scored -1 and was dropped from results.
|
||||
"""
|
||||
# Page nested 2 levels deep whose only match for "masking" is a single
|
||||
# body-token mention. raw_score == 1, depth == 2, so without clamping
|
||||
# the final score would be -1 and the page would be filtered out.
|
||||
_write_doc(
|
||||
tmp_path,
|
||||
"tutorials/advanced/notes.mdx",
|
||||
"# Notes\n\nWe briefly mention masking in this tutorial.\n",
|
||||
)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("masking", pages)
|
||||
slugs = [p.slug for p in results]
|
||||
assert "notes" in slugs, (
|
||||
"weak nested match must still surface, not be dropped by depth alone"
|
||||
)
|
||||
|
||||
|
||||
class TestBuildDocsReferenceText:
|
||||
def test_returns_empty_when_no_docs_present(self, tmp_path: Path) -> None:
|
||||
# Point at a non-existent docs root.
|
||||
assert DocsReference().build_text("anything", root=tmp_path / "missing") == ""
|
||||
|
||||
def test_includes_relevant_doc_excerpt_and_index(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
text = DocsReference().build_text("how do I configure Datadog?", root=tmp_path)
|
||||
assert "datadog.mdx" in text
|
||||
assert "API Key" in text
|
||||
# The compact index of all pages must always be appended so the LLM
|
||||
# can suggest other relevant pages even when one ranked highest.
|
||||
assert "docs index" in text
|
||||
assert "deployment.mdx" in text
|
||||
|
||||
def test_truncates_to_max_chars(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
text = DocsReference().build_text("Datadog", max_chars=120, root=tmp_path)
|
||||
assert len(text) <= 200
|
||||
assert "truncated" in text
|
||||
|
||||
|
||||
class TestBuildDocsIndex:
|
||||
def test_lists_all_pages_with_titles(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
index = build_docs_index(pages)
|
||||
assert "datadog.mdx: Datadog" in index
|
||||
assert "deployment.mdx: Deployment" in index
|
||||
|
||||
def test_returns_empty_string_for_no_pages(self) -> None:
|
||||
assert build_docs_index([]) == ""
|
||||
|
||||
|
||||
class TestExcerpt:
|
||||
def test_returns_full_body_when_short(self) -> None:
|
||||
body = "Short body."
|
||||
assert excerpt(body, max_chars=100) == "Short body."
|
||||
|
||||
def test_truncates_long_body_with_marker(self) -> None:
|
||||
body = ("paragraph one. " * 10) + "\n\n" + ("paragraph two. " * 10)
|
||||
out = excerpt(body, max_chars=80)
|
||||
assert "truncated" in out
|
||||
|
||||
|
||||
class TestDocPageDataclass:
|
||||
def test_is_hashable_and_immutable(self) -> None:
|
||||
page = DocPage(slug="x", relpath="x.mdx", title="X", body="hello")
|
||||
# frozen dataclasses are hashable, so they can be stored in sets.
|
||||
assert page in {page}
|
||||
|
||||
|
||||
class TestDocsGroundingCache:
|
||||
def test_cache_maxsize_matches_implementation(self) -> None:
|
||||
stats = DocsReference().stats()
|
||||
assert stats.maxsize == 32
|
||||
|
||||
def test_repeated_discover_hits_parse_cache(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
info1 = ref.stats()
|
||||
ref.discover(tmp_path)
|
||||
info2 = ref.stats()
|
||||
assert info2.hits == info1.hits + 1
|
||||
assert info2.misses == info1.misses
|
||||
|
||||
def test_invalidate_resets_stats(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
ref.discover(tmp_path)
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
cleared = ref.stats()
|
||||
assert cleared.hits == 0
|
||||
assert cleared.misses == 0
|
||||
assert cleared.currsize == 0
|
||||
|
||||
def test_file_edit_invalidates_and_refreshes_content(self, tmp_path: Path) -> None:
|
||||
ref = DocsReference()
|
||||
_write_doc(
|
||||
tmp_path,
|
||||
"datadog.mdx",
|
||||
'---\ntitle: "Datadog"\n---\n\nOld content.\n',
|
||||
)
|
||||
pages1 = ref.discover(tmp_path)
|
||||
assert any("Old content" in p.body for p in pages1)
|
||||
|
||||
datadog = tmp_path / "datadog.mdx"
|
||||
datadog.write_text(
|
||||
'---\ntitle: "Datadog"\n---\n\nNew refreshed content.\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
pages2 = ref.discover(tmp_path)
|
||||
assert any("New refreshed content" in p.body for p in pages2)
|
||||
assert not any("Old content" in p.body for p in pages2)
|
||||
|
||||
def test_single_tree_walk_per_discover_call(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Regression: avoid a second full walk inside the parse path on a miss."""
|
||||
_seed_docs(tmp_path)
|
||||
calls = 0
|
||||
real_iter = docs_reference._iter_doc_files
|
||||
|
||||
def _spy(root: Path) -> list[Path]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return real_iter(root)
|
||||
|
||||
monkeypatch.setattr(docs_reference, "_iter_doc_files", _spy)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
assert calls == 1
|
||||
ref.discover(tmp_path)
|
||||
assert calls == 2
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Tests for optional grounding-cache diagnostic logging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding.diagnostics import (
|
||||
GroundingSource,
|
||||
log_grounding_cache_diagnostics,
|
||||
)
|
||||
from core.agent_harness.grounding.models import CacheStats
|
||||
|
||||
|
||||
def _sources() -> list[GroundingSource]:
|
||||
return [GroundingSource(name="unit", stats_fn=lambda: CacheStats(name="unit", hits=1))]
|
||||
|
||||
|
||||
def test_log_skips_when_tracer_verbose_unset(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.delenv("TRACER_VERBOSE", raising=False)
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test")
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_log_skips_when_tracer_verbose_not_one(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setenv("TRACER_VERBOSE", "0")
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test")
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_log_emits_debug_when_tracer_verbose_on(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setenv("TRACER_VERBOSE", "1")
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test_reason")
|
||||
assert any("unit_test_reason" in r.message for r in caplog.records)
|
||||
assert any("grounding cache" in r.message.lower() for r in caplog.records)
|
||||
@@ -0,0 +1,23 @@
|
||||
# REPL watchdog — PR demo (copy into “Demo/Screenshot”)
|
||||
|
||||
Paste the block below into your pull request so reviewers see the demo **in the PR** (see [docs/DEVELOPMENT.md](../../docs/DEVELOPMENT.md#interactive-shell-repl-watchdog-demo)).
|
||||
|
||||
```text
|
||||
$ uv run opensre
|
||||
> /trust on
|
||||
> /watch <PID> --max-cpu 80
|
||||
task <id> started.
|
||||
> /watches
|
||||
(table: id, pid, kind, status, thresholds, last sample)
|
||||
> /unwatch <id>
|
||||
> /watches
|
||||
(status: cancelled)
|
||||
```
|
||||
|
||||
Replace `<PID>` with a real process id (for example your REPL’s Python pid). Replace `<id>` with the task id from `/watches`.
|
||||
|
||||
Optional (Telegram): set `TELEGRAM_BOT_TOKEN` and `TELEGRAM_DEFAULT_CHAT_ID`, use a lower `--max-cpu` so the threshold trips; expect one line: `[task …] alarm fired: … (telegram delivered)`.
|
||||
|
||||
---
|
||||
|
||||
Automated proof (CI): `uv run pytest tests/interactive_shell/test_watchdog_repl_e2e_demo.py -v --tb=short`
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Unit tests for the miss triage store and benchmark scenario conversion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.domain.feedback import (
|
||||
MissRecord,
|
||||
MissTaxonomy,
|
||||
compute_recurrence,
|
||||
compute_stats,
|
||||
load_misses,
|
||||
record_miss,
|
||||
to_benchmark_scenario,
|
||||
)
|
||||
from core.domain.feedback.misses import (
|
||||
export_scenarios,
|
||||
filter_top_misses,
|
||||
misses_path,
|
||||
parse_since,
|
||||
taxonomy_choices,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opensre_home(monkeypatch, tmp_path: Path) -> Path:
|
||||
home = tmp_path / ".opensre"
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", home)
|
||||
return home
|
||||
|
||||
|
||||
# ── taxonomy enum surface ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_taxonomy_choices_includes_all_buckets() -> None:
|
||||
keys = {key for key, _ in taxonomy_choices()}
|
||||
assert keys == {t.value for t in MissTaxonomy}
|
||||
|
||||
|
||||
# ── record_miss / load_misses round-trip ──────────────────────────────────────
|
||||
|
||||
|
||||
def _feedback_dict(**overrides) -> dict:
|
||||
base = {
|
||||
"feedback_id": "fb-001",
|
||||
"timestamp": "2026-06-02T10:00:00+00:00",
|
||||
"run_id": "run-001",
|
||||
"alert_name": "checkout-api 5xx",
|
||||
"rating": "inaccurate",
|
||||
"note": "missed the canary deploy from 09:42",
|
||||
"root_cause": "The investigation concluded a DB outage.",
|
||||
"root_cause_category": "database",
|
||||
"validity_score": 0.45,
|
||||
"investigation_loop_count": 6,
|
||||
"user_id": "u-1",
|
||||
"user_email": "u@example.com",
|
||||
"org_id": "org-1",
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def test_record_miss_returns_none_and_warns_on_write_failure(
|
||||
opensre_home: Path, monkeypatch, capsys
|
||||
) -> None:
|
||||
def _explode(*_args, **_kwargs):
|
||||
raise PermissionError("disk full")
|
||||
|
||||
monkeypatch.setattr("pathlib.Path.open", _explode)
|
||||
|
||||
result = record_miss(_feedback_dict(), taxonomy=MissTaxonomy.RETRIEVAL_GAP)
|
||||
|
||||
assert result is None
|
||||
err = capsys.readouterr().err
|
||||
assert "could not record miss" in err
|
||||
assert "disk full" in err
|
||||
|
||||
|
||||
def test_record_miss_persists_to_jsonl(opensre_home: Path) -> None:
|
||||
feedback = _feedback_dict()
|
||||
final_state = {"pipeline_name": "checkout/api", "severity": "critical"}
|
||||
|
||||
rec = record_miss(
|
||||
feedback,
|
||||
taxonomy=MissTaxonomy.RETRIEVAL_GAP,
|
||||
final_state=final_state,
|
||||
)
|
||||
|
||||
assert rec is not None
|
||||
assert rec["taxonomy"] == "retrieval_gap"
|
||||
assert rec["pipeline_name"] == "checkout/api"
|
||||
assert rec["severity"] == "critical"
|
||||
assert rec["feedback_id"] == "fb-001"
|
||||
|
||||
path = misses_path()
|
||||
assert path.exists()
|
||||
written = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
||||
assert len(written) == 1
|
||||
assert written[0]["miss_id"] == rec["miss_id"]
|
||||
|
||||
|
||||
def test_record_miss_accepts_string_taxonomy(opensre_home: Path) -> None:
|
||||
rec = record_miss(_feedback_dict(), taxonomy="reasoning_gap")
|
||||
assert rec is not None
|
||||
assert rec["taxonomy"] == "reasoning_gap"
|
||||
|
||||
|
||||
def test_load_misses_skips_corrupt_lines(opensre_home: Path) -> None:
|
||||
path = misses_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
valid = {
|
||||
"miss_id": "ok",
|
||||
"alert_name": "a",
|
||||
"taxonomy": "tool_failure",
|
||||
"timestamp": "2026-06-02T00:00:00+00:00",
|
||||
}
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
json.dumps(valid),
|
||||
"not json at all",
|
||||
"",
|
||||
'{"miss_id": "partial"', # malformed
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
rows = load_misses()
|
||||
assert [r["miss_id"] for r in rows] == ["ok"]
|
||||
|
||||
|
||||
def test_load_misses_filters_by_since_and_taxonomy(opensre_home: Path) -> None:
|
||||
now = datetime.now(UTC)
|
||||
record_miss(
|
||||
_feedback_dict(timestamp=(now - timedelta(days=10)).isoformat(), alert_name="old"),
|
||||
taxonomy=MissTaxonomy.RETRIEVAL_GAP,
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(
|
||||
timestamp=(now - timedelta(hours=2)).isoformat(), alert_name="fresh-retrieval"
|
||||
),
|
||||
taxonomy=MissTaxonomy.RETRIEVAL_GAP,
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(timestamp=(now - timedelta(hours=1)).isoformat(), alert_name="fresh-tool"),
|
||||
taxonomy=MissTaxonomy.TOOL_FAILURE,
|
||||
)
|
||||
|
||||
recent = load_misses(since=now - timedelta(days=1))
|
||||
assert {r["alert_name"] for r in recent} == {"fresh-retrieval", "fresh-tool"}
|
||||
|
||||
retrieval = load_misses(taxonomy=MissTaxonomy.RETRIEVAL_GAP)
|
||||
assert {r["alert_name"] for r in retrieval} == {"old", "fresh-retrieval"}
|
||||
|
||||
|
||||
# ── recurrence + stats ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_compute_recurrence_groups_by_alert_and_taxonomy(opensre_home: Path) -> None:
|
||||
record_miss(_feedback_dict(alert_name="A"), taxonomy=MissTaxonomy.RETRIEVAL_GAP)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="A", feedback_id="fb-002"), taxonomy=MissTaxonomy.RETRIEVAL_GAP
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="A", feedback_id="fb-003"), taxonomy=MissTaxonomy.REASONING_GAP
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="B", feedback_id="fb-004"), taxonomy=MissTaxonomy.TOOL_FAILURE
|
||||
)
|
||||
|
||||
rec = compute_recurrence(load_misses())
|
||||
assert rec[("A", "retrieval_gap")] == 2
|
||||
assert rec[("A", "reasoning_gap")] == 1
|
||||
assert rec[("B", "tool_failure")] == 1
|
||||
|
||||
|
||||
def test_compute_stats_reports_totals_and_recurring(opensre_home: Path) -> None:
|
||||
record_miss(_feedback_dict(alert_name="A"), taxonomy=MissTaxonomy.RETRIEVAL_GAP)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="A", feedback_id="fb-002"), taxonomy=MissTaxonomy.RETRIEVAL_GAP
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="B", feedback_id="fb-003"), taxonomy=MissTaxonomy.TOOL_FAILURE
|
||||
)
|
||||
|
||||
stats = compute_stats(load_misses())
|
||||
assert stats["total"] == 3
|
||||
assert stats["by_taxonomy"]["retrieval_gap"] == 2
|
||||
assert stats["by_taxonomy"]["tool_failure"] == 1
|
||||
assert stats["unique_alerts"] == 2
|
||||
assert stats["recurring"] == [("A", "retrieval_gap", 2)]
|
||||
|
||||
|
||||
# ── filter_top_misses prioritises recurring pairs ─────────────────────────────
|
||||
|
||||
|
||||
def test_filter_top_misses_dedupes_and_prefers_recurrence(opensre_home: Path) -> None:
|
||||
now = datetime.now(UTC)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="A", timestamp=(now - timedelta(days=3)).isoformat()),
|
||||
taxonomy=MissTaxonomy.RETRIEVAL_GAP,
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(
|
||||
alert_name="A", feedback_id="fb-002", timestamp=(now - timedelta(days=1)).isoformat()
|
||||
),
|
||||
taxonomy=MissTaxonomy.RETRIEVAL_GAP,
|
||||
)
|
||||
record_miss(
|
||||
_feedback_dict(alert_name="B", feedback_id="fb-003", timestamp=now.isoformat()),
|
||||
taxonomy=MissTaxonomy.TOOL_FAILURE,
|
||||
)
|
||||
|
||||
top = filter_top_misses(load_misses(), top=10)
|
||||
|
||||
pairs = [(r["alert_name"], r["taxonomy"]) for r in top]
|
||||
# Deduped to one per (alert, taxonomy)
|
||||
assert pairs == [("A", "retrieval_gap"), ("B", "tool_failure")]
|
||||
|
||||
|
||||
def test_filter_top_misses_handles_zero_and_empty() -> None:
|
||||
assert filter_top_misses([], top=5) == []
|
||||
assert filter_top_misses([{"alert_name": "x"}], top=0) == []
|
||||
|
||||
|
||||
# ── scenario conversion ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_to_benchmark_scenario_carries_rubric() -> None:
|
||||
miss = MissRecord(
|
||||
miss_id="m-1",
|
||||
run_id="r-1",
|
||||
alert_name="checkout-api 5xx",
|
||||
pipeline_name="checkout/api",
|
||||
severity="critical",
|
||||
timestamp="2026-06-02T10:00:00+00:00",
|
||||
taxonomy="retrieval_gap",
|
||||
taxonomy_detail="missed the canary deploy",
|
||||
root_cause="Bad deploy at 09:42",
|
||||
root_cause_category="deploy",
|
||||
)
|
||||
scenario = to_benchmark_scenario(miss)
|
||||
|
||||
assert scenario["alert_name"] == "checkout-api 5xx"
|
||||
assert scenario["title"].startswith("[Regression]")
|
||||
assert scenario["pipeline_name"] == "checkout/api"
|
||||
assert scenario["severity"] == "critical"
|
||||
assert scenario["_meta"]["miss_id"] == "m-1"
|
||||
assert scenario["_meta"]["taxonomy"] == "retrieval_gap"
|
||||
|
||||
# The rubric MUST live at commonAnnotations.scoring_points — that is
|
||||
# where extract_scoring_points reads it and where
|
||||
# strip_scoring_points_from_alert removes it before the agent sees the
|
||||
# alert. Anywhere else and either the judge misses the rubric or the
|
||||
# agent gets handed the answer.
|
||||
rubric = scenario["commonAnnotations"]["scoring_points"]
|
||||
assert rubric["expected_root_cause"] == "Bad deploy at 09:42"
|
||||
assert rubric["expected_category"] == "deploy"
|
||||
assert rubric["miss_notes"] == "missed the canary deploy"
|
||||
assert "scoring_points" not in scenario["_meta"]
|
||||
|
||||
|
||||
def test_to_benchmark_scenario_is_strippable_for_blind_agent_runs() -> None:
|
||||
"""Confirm the produced scenario is compatible with the existing
|
||||
strip_scoring_points_from_alert helper — i.e. the rubric ends up where
|
||||
the helper expects and is actually removed for non-evaluate runs."""
|
||||
from integrations.opensre import (
|
||||
extract_scoring_points,
|
||||
strip_scoring_points_from_alert,
|
||||
)
|
||||
|
||||
miss = MissRecord(
|
||||
miss_id="m-1",
|
||||
alert_name="checkout-api 5xx",
|
||||
taxonomy="retrieval_gap",
|
||||
taxonomy_detail="missed the canary deploy",
|
||||
root_cause="Bad deploy at 09:42",
|
||||
root_cause_category="deploy",
|
||||
)
|
||||
scenario = to_benchmark_scenario(miss)
|
||||
|
||||
# Judge can read the rubric.
|
||||
rubric_text = extract_scoring_points(scenario)
|
||||
assert "Bad deploy at 09:42" in rubric_text
|
||||
|
||||
# Strip removes it cleanly — agent does not see the answer.
|
||||
blind = strip_scoring_points_from_alert(scenario)
|
||||
assert "scoring_points" not in blind["commonAnnotations"]
|
||||
assert extract_scoring_points(blind) == ""
|
||||
|
||||
|
||||
def test_export_scenarios_handles_json_null_alert_name_and_taxonomy(tmp_path: Path) -> None:
|
||||
"""A JSON null in misses.jsonl returns Python None from dict.get — the
|
||||
export path must not crash _slugify's re.sub on those values."""
|
||||
misses: list[MissRecord] = [
|
||||
MissRecord(
|
||||
miss_id="m-1",
|
||||
alert_name=None, # type: ignore[typeddict-item]
|
||||
taxonomy=None, # type: ignore[typeddict-item]
|
||||
timestamp="2026-06-02T10:00:00+00:00",
|
||||
root_cause="rc",
|
||||
),
|
||||
]
|
||||
out = tmp_path / "scenarios"
|
||||
|
||||
written = export_scenarios(misses, out)
|
||||
|
||||
assert len(written) == 1
|
||||
# Slug falls back to the index + "unknown" taxonomy rather than crashing.
|
||||
assert written[0].parent.name == "0001_miss-0001_unknown"
|
||||
|
||||
|
||||
def test_export_scenarios_writes_per_case_directories(tmp_path: Path) -> None:
|
||||
misses: list[MissRecord] = [
|
||||
MissRecord(
|
||||
miss_id="m-1",
|
||||
alert_name="checkout 5xx",
|
||||
taxonomy="retrieval_gap",
|
||||
timestamp="2026-06-02T10:00:00+00:00",
|
||||
root_cause="rc",
|
||||
),
|
||||
MissRecord(
|
||||
miss_id="m-2",
|
||||
alert_name="checkout 5xx",
|
||||
taxonomy="reasoning_gap",
|
||||
timestamp="2026-06-02T11:00:00+00:00",
|
||||
root_cause="rc2",
|
||||
),
|
||||
]
|
||||
out = tmp_path / "scenarios"
|
||||
|
||||
written = export_scenarios(misses, out)
|
||||
|
||||
assert len(written) == 2
|
||||
assert all(p.name == "alert.json" for p in written)
|
||||
parents = [p.parent.name for p in written]
|
||||
assert any("retrieval_gap" in name for name in parents)
|
||||
assert any("reasoning_gap" in name for name in parents)
|
||||
|
||||
payload = json.loads(written[0].read_text())
|
||||
assert payload["_meta"]["taxonomy"] in {"retrieval_gap", "reasoning_gap"}
|
||||
|
||||
|
||||
# ── parse_since input parsing ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_since_accepts_relative_units() -> None:
|
||||
now = datetime.now(UTC)
|
||||
one_day_ago = parse_since("1d")
|
||||
assert (now - one_day_ago) < timedelta(seconds=5) + timedelta(days=1)
|
||||
assert (now - one_day_ago) > timedelta(hours=23, minutes=55)
|
||||
|
||||
assert parse_since("12h") < now
|
||||
assert parse_since("2w") < now - timedelta(days=13)
|
||||
|
||||
|
||||
def test_parse_since_accepts_iso_timestamp() -> None:
|
||||
parsed = parse_since("2026-06-02T10:00:00+00:00")
|
||||
assert parsed == datetime(2026, 6, 2, 10, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def test_parse_since_rejects_garbage() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
parse_since("")
|
||||
with pytest.raises(ValueError):
|
||||
parse_since("forever ago")
|
||||
@@ -0,0 +1,36 @@
|
||||
"""ShellOutputSink.render_error appends ``/model`` and ``/auth login`` hints on a credit-exhausted error."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.llm.shared.llm_retry import CREDIT_EXHAUSTED_MARKER
|
||||
from surfaces.interactive_shell.runtime.agent_harness_adapters import ShellOutputSink
|
||||
|
||||
|
||||
class _RecordingConsole:
|
||||
def __init__(self) -> None:
|
||||
self.lines: list[str] = []
|
||||
|
||||
def print(self, message: str = "") -> None:
|
||||
self.lines.append(str(message))
|
||||
|
||||
|
||||
def _render_error(message: str) -> str:
|
||||
console = _RecordingConsole()
|
||||
ShellOutputSink(console).render_error(message) # type: ignore[arg-type]
|
||||
return "\n".join(console.lines)
|
||||
|
||||
|
||||
def test_render_error_shows_model_hint_on_credit_exhaustion() -> None:
|
||||
output = _render_error(f"Anthropic {CREDIT_EXHAUSTED_MARKER}. Original error: 400")
|
||||
assert "/model" in output
|
||||
|
||||
|
||||
def test_render_error_shows_auth_login_hint_on_credit_exhaustion() -> None:
|
||||
output = _render_error(f"Anthropic {CREDIT_EXHAUSTED_MARKER}. Original error: 400")
|
||||
assert "/auth login" in output
|
||||
|
||||
|
||||
def test_render_error_no_hint_for_generic_error() -> None:
|
||||
output = _render_error("some other failure")
|
||||
assert "/model" not in output
|
||||
assert "/auth login" not in output
|
||||
@@ -0,0 +1,51 @@
|
||||
"""The turn-error render adds ``/model`` and ``/auth login`` recovery hints on a credit-exhausted error."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.llm.shared.llm_retry import LLMCreditExhaustedError
|
||||
from surfaces.interactive_shell.runtime.agent_presentation import (
|
||||
AgentEvent,
|
||||
AgentPresentationState,
|
||||
_render_agent_presentation_transition,
|
||||
)
|
||||
|
||||
|
||||
class _RecordingConsole:
|
||||
def __init__(self) -> None:
|
||||
self.lines: list[str] = []
|
||||
|
||||
def print(self, text: str = "") -> None:
|
||||
self.lines.append(text)
|
||||
|
||||
|
||||
def _render_turn_error(error: Exception) -> str:
|
||||
console = _RecordingConsole()
|
||||
asyncio.run(
|
||||
_render_agent_presentation_transition(
|
||||
previous=AgentPresentationState(),
|
||||
current=AgentPresentationState(),
|
||||
event=AgentEvent(type="turn_error", error=error),
|
||||
console=console, # type: ignore[arg-type]
|
||||
spinner=MagicMock(),
|
||||
)
|
||||
)
|
||||
return "\n".join(console.lines)
|
||||
|
||||
|
||||
def test_credit_exhausted_turn_error_shows_model_hint() -> None:
|
||||
output = _render_turn_error(LLMCreditExhaustedError("Anthropic credit exhausted"))
|
||||
assert "/model" in output
|
||||
|
||||
|
||||
def test_credit_exhausted_turn_error_shows_auth_login_hint() -> None:
|
||||
output = _render_turn_error(LLMCreditExhaustedError("Anthropic credit exhausted"))
|
||||
assert "/auth login" in output
|
||||
|
||||
|
||||
def test_other_turn_error_has_no_model_hint() -> None:
|
||||
output = _render_turn_error(RuntimeError("something else broke"))
|
||||
assert "/model" not in output
|
||||
assert "/auth login" not in output
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from surfaces.interactive_shell.runtime.background.notifications import (
|
||||
deliver_background_notifications,
|
||||
)
|
||||
from surfaces.interactive_shell.session.background_investigations import (
|
||||
BackgroundInvestigationRecord,
|
||||
)
|
||||
|
||||
|
||||
def test_deliver_background_notifications_sends_email_when_smtp_is_configured(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"integrations.catalog.resolve_effective_integrations",
|
||||
lambda: {
|
||||
"smtp": {
|
||||
"source": "local env",
|
||||
"config": {
|
||||
"host": "smtp.example.com",
|
||||
"port": 587,
|
||||
"security": "starttls",
|
||||
"from_address": "opensre@example.com",
|
||||
"default_to": "team@example.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _fake_send_smtp_report(
|
||||
*, report: str, subject: str, smtp_ctx: dict[str, object]
|
||||
) -> tuple[bool, str]:
|
||||
captured["report"] = report
|
||||
captured["subject"] = subject
|
||||
captured["smtp_ctx"] = smtp_ctx
|
||||
return True, ""
|
||||
|
||||
monkeypatch.setattr(
|
||||
"integrations.smtp.delivery.send_smtp_report",
|
||||
_fake_send_smtp_report,
|
||||
)
|
||||
|
||||
record = BackgroundInvestigationRecord(
|
||||
task_id="bg-123",
|
||||
status="completed",
|
||||
command="/investigate checkout-latency",
|
||||
root_cause="postgres connection pool saturation",
|
||||
top_analysis=("rds cpu spike",),
|
||||
next_steps=("raise pool size",),
|
||||
stats={"tool_call_count": 4, "investigation_loop_count": 2, "validity_score": 0.8},
|
||||
)
|
||||
|
||||
results = deliver_background_notifications(record=record, channels=("email",))
|
||||
|
||||
assert results == {"email": "sent"}
|
||||
assert captured["subject"] == "OpenSRE RCA complete: bg-123"
|
||||
assert "Root cause" in str(captured["report"])
|
||||
|
||||
|
||||
def test_deliver_background_notifications_skips_when_no_channels_configured() -> None:
|
||||
record = BackgroundInvestigationRecord(
|
||||
task_id="bg-123", status="completed", command="free-text"
|
||||
)
|
||||
results = deliver_background_notifications(record=record, channels=())
|
||||
assert results == {}
|
||||
|
||||
|
||||
def test_deliver_background_notifications_marks_missing_smtp(monkeypatch) -> None:
|
||||
monkeypatch.setattr("integrations.catalog.resolve_effective_integrations", lambda: {})
|
||||
record = BackgroundInvestigationRecord(
|
||||
task_id="bg-123", status="completed", command="free-text"
|
||||
)
|
||||
results = deliver_background_notifications(record=record, channels=("email",))
|
||||
assert results == {"email": "missing smtp integration"}
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.runtime.background.runner import (
|
||||
drain_background_notices,
|
||||
start_background_template_investigation,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def test_enqueue_and_drain_background_notices() -> None:
|
||||
import io
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
session = Session()
|
||||
session.terminal.enqueue_background_notice("[bold]done[/bold]")
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
drain_background_notices(session, console)
|
||||
|
||||
assert session.terminal.drain_background_notices() == []
|
||||
assert "done" in console.file.getvalue()
|
||||
|
||||
|
||||
def test_start_background_template_investigation_assigns_fresh_investigation_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import io
|
||||
|
||||
session = Session()
|
||||
session.last_investigation_id = "inv-stale"
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
def _fake_run(
|
||||
*,
|
||||
cancel_requested,
|
||||
template_name: str,
|
||||
context_overrides,
|
||||
) -> dict[str, object]:
|
||||
_ = (cancel_requested, template_name, context_overrides)
|
||||
return {"root_cause": "done"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.runtime.investigation_adapter.run_sample_alert_for_session_background",
|
||||
_fake_run,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.runtime.background.runner.track_investigation",
|
||||
lambda **_kwargs: MagicMock(
|
||||
__enter__=MagicMock(return_value=MagicMock()),
|
||||
__exit__=MagicMock(return_value=False),
|
||||
),
|
||||
)
|
||||
|
||||
task_id = start_background_template_investigation(
|
||||
template_name="generic",
|
||||
session=session,
|
||||
console=console,
|
||||
display_command="/investigate generic",
|
||||
investigation_target="generic",
|
||||
)
|
||||
|
||||
assert session.last_investigation_id
|
||||
assert session.last_investigation_id != "inv-stale"
|
||||
record = session.terminal.background_investigations[task_id]
|
||||
assert record.investigation_id == session.last_investigation_id
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Tests for validated interactive-shell runtime context assembly."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from prompt_toolkit import PromptSession
|
||||
from prompt_toolkit.application import create_app_session
|
||||
from prompt_toolkit.history import InMemoryHistory
|
||||
from prompt_toolkit.input import DummyInput
|
||||
from prompt_toolkit.output import DummyOutput
|
||||
from pydantic import ValidationError
|
||||
|
||||
from platform.common.task_registry import TaskRegistry
|
||||
from surfaces.interactive_shell.controller import InteractiveShellController
|
||||
from surfaces.interactive_shell.runtime.context import (
|
||||
ReplRuntimeContext,
|
||||
SessionBootstrapSpec,
|
||||
create_repl_runtime_context,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.core.state import (
|
||||
ReplState,
|
||||
SpinnerState,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _prompt_session() -> PromptSession[str]:
|
||||
return PromptSession(history=InMemoryHistory())
|
||||
|
||||
|
||||
def test_create_context_applies_canonical_session_bootstrap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
registry = TaskRegistry()
|
||||
hydrate_calls: list[str] = []
|
||||
|
||||
def _hydrate(self: Session) -> None:
|
||||
hydrate_calls.append(self.session_id)
|
||||
self.configured_integrations = ("github",)
|
||||
self.configured_integrations_known = True
|
||||
|
||||
monkeypatch.setattr(Session, "hydrate_configured_integrations", _hydrate)
|
||||
monkeypatch.setattr(TaskRegistry, "persistent", staticmethod(lambda: registry))
|
||||
|
||||
with create_app_session(input=DummyInput(), output=DummyOutput()):
|
||||
prompt = _prompt_session()
|
||||
session = Session()
|
||||
context = create_repl_runtime_context(
|
||||
session=session,
|
||||
pt_session=prompt,
|
||||
active_theme_name="pink",
|
||||
)
|
||||
|
||||
assert context.session is session
|
||||
assert session.terminal.active_theme_name == "pink"
|
||||
assert session.configured_integrations == ("github",)
|
||||
assert session.configured_integrations_known is True
|
||||
assert hydrate_calls == [session.session_id]
|
||||
assert session.task_registry is registry
|
||||
assert session.terminal.prompt_history_backend is prompt.history
|
||||
|
||||
|
||||
def test_context_supports_lightweight_bootstrap_for_unit_seams(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
Session,
|
||||
"hydrate_configured_integrations",
|
||||
lambda _self: (_ for _ in ()).throw(AssertionError("hydrated")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
TaskRegistry,
|
||||
"persistent",
|
||||
staticmethod(lambda: (_ for _ in ()).throw(AssertionError("persistent"))),
|
||||
)
|
||||
|
||||
context = create_repl_runtime_context(
|
||||
active_theme_name="green",
|
||||
hydrate_integrations=False,
|
||||
persistent_tasks=False,
|
||||
)
|
||||
|
||||
assert context.session.terminal.active_theme_name == "green"
|
||||
assert isinstance(context.state, ReplState)
|
||||
assert isinstance(context.spinner, SpinnerState)
|
||||
|
||||
|
||||
def test_create_context_registers_jsonl_session_trace_sink(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""REPL boot wires the JSONL session-trace sink for ATM / span metrics."""
|
||||
from platform.observability.trace.spans import (
|
||||
NoopSessionTraceSink,
|
||||
get_session_trace_sink,
|
||||
is_session_trace_active,
|
||||
set_session_trace_sink,
|
||||
)
|
||||
from surfaces.interactive_shell.session.trace_sink import JsonlSessionTraceSink
|
||||
|
||||
monkeypatch.setattr(
|
||||
Session,
|
||||
"hydrate_configured_integrations",
|
||||
lambda _self: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
TaskRegistry,
|
||||
"persistent",
|
||||
staticmethod(TaskRegistry),
|
||||
)
|
||||
set_session_trace_sink(NoopSessionTraceSink())
|
||||
try:
|
||||
create_repl_runtime_context(
|
||||
hydrate_integrations=False,
|
||||
persistent_tasks=False,
|
||||
)
|
||||
assert is_session_trace_active()
|
||||
assert isinstance(get_session_trace_sink(), JsonlSessionTraceSink)
|
||||
finally:
|
||||
set_session_trace_sink(NoopSessionTraceSink())
|
||||
|
||||
|
||||
def test_create_context_uses_noop_trace_sink_for_in_memory_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from core.agent_harness.session import InMemorySessionStorage
|
||||
from platform.observability.trace.spans import (
|
||||
NoopSessionTraceSink,
|
||||
get_session_trace_sink,
|
||||
is_session_trace_active,
|
||||
set_session_trace_sink,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
Session,
|
||||
"hydrate_configured_integrations",
|
||||
lambda _self: None,
|
||||
)
|
||||
set_session_trace_sink(NoopSessionTraceSink())
|
||||
try:
|
||||
create_repl_runtime_context(
|
||||
session=Session(storage=InMemorySessionStorage()),
|
||||
hydrate_integrations=False,
|
||||
persistent_tasks=False,
|
||||
)
|
||||
assert not is_session_trace_active()
|
||||
assert isinstance(get_session_trace_sink(), NoopSessionTraceSink)
|
||||
finally:
|
||||
set_session_trace_sink(NoopSessionTraceSink())
|
||||
|
||||
|
||||
def test_context_uses_canonical_initial_mutable_state() -> None:
|
||||
context = ReplRuntimeContext(session=Session())
|
||||
|
||||
assert isinstance(context.state, ReplState)
|
||||
assert isinstance(context.spinner, SpinnerState)
|
||||
assert context.state.exit_requested is False
|
||||
assert context.spinner.streaming is False
|
||||
|
||||
|
||||
def test_context_preserves_explicit_partial_mutable_state() -> None:
|
||||
state = ReplState()
|
||||
context = ReplRuntimeContext(session=Session(), state=state)
|
||||
|
||||
assert context.state is state
|
||||
assert isinstance(context.spinner, SpinnerState)
|
||||
|
||||
|
||||
def test_context_rejects_invalid_state_contracts() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
ReplRuntimeContext(session=object()) # type: ignore[arg-type]
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
SessionBootstrapSpec(active_theme_name=" ")
|
||||
|
||||
|
||||
def test_context_assignment_validates_inbox_type() -> None:
|
||||
context = ReplRuntimeContext(session=Session())
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
context.inbox = object() # type: ignore[assignment]
|
||||
|
||||
|
||||
def test_controller_reuses_validated_runtime_context() -> None:
|
||||
session = Session()
|
||||
state = ReplState()
|
||||
spinner = SpinnerState()
|
||||
context = ReplRuntimeContext(session=session, state=state, spinner=spinner)
|
||||
|
||||
controller = InteractiveShellController(context)
|
||||
|
||||
assert controller.runtime_context is context
|
||||
assert controller.session is session
|
||||
assert controller.state is state
|
||||
assert controller.spinner is spinner
|
||||
assert controller.input_reader.state is state
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Pure input-action decisions for the interactive shell controller."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.runtime.input import (
|
||||
InputCancelled,
|
||||
InputClosed,
|
||||
InputEvent,
|
||||
InputSubmitted,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.input.actions import (
|
||||
QUEUE_DURING_CONFIRMATION_WARNING,
|
||||
CancelTurn,
|
||||
CloseShell,
|
||||
DeliverConfirmation,
|
||||
IgnoreInput,
|
||||
ShellInputSnapshot,
|
||||
SubmitTurn,
|
||||
decide_input_action,
|
||||
)
|
||||
|
||||
|
||||
def _decide(
|
||||
event: InputEvent,
|
||||
*,
|
||||
exit_requested: bool = False,
|
||||
dispatch_running: bool = False,
|
||||
awaiting_confirmation: bool = False,
|
||||
needs_exclusive_stdin: bool = False,
|
||||
) -> object:
|
||||
return decide_input_action(
|
||||
event,
|
||||
ShellInputSnapshot(
|
||||
exit_requested=exit_requested,
|
||||
dispatch_running=dispatch_running,
|
||||
awaiting_confirmation=awaiting_confirmation,
|
||||
),
|
||||
needs_exclusive_stdin=lambda _text: needs_exclusive_stdin,
|
||||
)
|
||||
|
||||
|
||||
def test_decide_closes_on_input_closed() -> None:
|
||||
assert _decide(InputClosed()) == CloseShell()
|
||||
|
||||
|
||||
def test_decide_cancels_on_input_cancelled() -> None:
|
||||
assert _decide(InputCancelled()) == CancelTurn()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["", " "])
|
||||
def test_decide_ignores_empty_or_blank_submissions(text: str) -> None:
|
||||
assert _decide(InputSubmitted(text)) == IgnoreInput()
|
||||
|
||||
|
||||
def test_decide_ignores_submitted_input_after_exit_requested() -> None:
|
||||
assert _decide(InputSubmitted("/status"), exit_requested=True) == IgnoreInput()
|
||||
|
||||
|
||||
def test_decide_cancels_when_cancel_request_is_typed_during_dispatch() -> None:
|
||||
assert _decide(InputSubmitted(" /cancel "), dispatch_running=True) == CancelTurn(
|
||||
submitted_text="/cancel"
|
||||
)
|
||||
|
||||
|
||||
def test_decide_delivers_stripped_confirmation_answer() -> None:
|
||||
assert _decide(
|
||||
InputSubmitted(" yes "),
|
||||
awaiting_confirmation=True,
|
||||
) == DeliverConfirmation(text="yes")
|
||||
|
||||
|
||||
def test_decide_submits_non_confirmation_input_while_confirmation_is_pending() -> None:
|
||||
assert _decide(
|
||||
InputSubmitted("run /status"),
|
||||
awaiting_confirmation=True,
|
||||
) == SubmitTurn(
|
||||
text="run /status",
|
||||
warning=QUEUE_DURING_CONFIRMATION_WARNING,
|
||||
)
|
||||
|
||||
|
||||
def test_decide_submits_normal_turn_without_wait_by_default() -> None:
|
||||
assert _decide(InputSubmitted(" show status ")) == SubmitTurn(text="show status")
|
||||
|
||||
|
||||
def test_decide_submits_normal_turn_with_exclusive_stdin_wait() -> None:
|
||||
assert _decide(InputSubmitted("/integrations"), needs_exclusive_stdin=True) == SubmitTurn(
|
||||
text="/integrations",
|
||||
wait_until_idle=True,
|
||||
)
|
||||
@@ -0,0 +1,337 @@
|
||||
"""Tests for hydrating configured integrations onto the REPL session at boot.
|
||||
|
||||
Without this the agent cannot answer "is X installed?" and the integration
|
||||
guards stay dead because ``configured_integrations_known`` never flips to True.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
import surfaces.interactive_shell.main as main_entrypoint
|
||||
from core.agent_harness.session.integration_resolution import IntegrationResolutionResult
|
||||
from surfaces.interactive_shell.runtime.startup import first_launch_github as flg
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _console() -> Console:
|
||||
return Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
|
||||
def test_hydrate_populates_session_from_effective_resolution(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: ["gitlab", "datadog"],
|
||||
)
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert session.configured_integrations_known is True
|
||||
# Metadata discovery covers env + local store and is returned in sorted order.
|
||||
assert session.configured_integrations == ("datadog", "gitlab")
|
||||
|
||||
|
||||
def test_hydrate_marks_known_even_when_none_configured(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
list,
|
||||
)
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert session.configured_integrations_known is True
|
||||
assert session.configured_integrations == ()
|
||||
|
||||
|
||||
def test_warm_resolved_integrations_populates_cache(monkeypatch: Any) -> None:
|
||||
resolved = {"datadog": {"site": "datadoghq.com"}, "grafana": {"url": "http://localhost"}}
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: resolved,
|
||||
)
|
||||
session = Session()
|
||||
session.warm_resolved_integrations()
|
||||
assert session.resolved_integrations_cache == resolved
|
||||
|
||||
|
||||
def test_warm_resolved_integrations_is_idempotent(monkeypatch: Any) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def _resolve() -> dict[str, Any]:
|
||||
calls.append("resolve")
|
||||
return {"github": {}}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
_resolve,
|
||||
)
|
||||
session = Session()
|
||||
session.warm_resolved_integrations()
|
||||
session.warm_resolved_integrations()
|
||||
assert calls == ["resolve"]
|
||||
|
||||
|
||||
def test_warm_resolved_integrations_skips_empty_cache(monkeypatch: Any) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def _resolve() -> dict[str, Any]:
|
||||
calls.append("resolve")
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
_resolve,
|
||||
)
|
||||
session = Session()
|
||||
session.warm_resolved_integrations()
|
||||
assert session.resolved_integrations_cache is None
|
||||
session.warm_resolved_integrations()
|
||||
assert calls == ["resolve", "resolve"]
|
||||
|
||||
|
||||
def test_warm_resolved_integrations_uses_quiet_resolve(monkeypatch: Any) -> None:
|
||||
progress_calls: list[str] = []
|
||||
quiet_calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.investigation.stages.resolve_integrations.resolve_integrations",
|
||||
lambda _state: progress_calls.append("progress") or {"resolved_integrations": {}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: quiet_calls.append("quiet") or {"datadog": {}},
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.warm_resolved_integrations()
|
||||
|
||||
assert quiet_calls == ["quiet"]
|
||||
assert progress_calls == []
|
||||
assert session.resolved_integrations_cache == {"datadog": {}}
|
||||
|
||||
|
||||
def test_get_integrations_returns_pydantic_cached_result(monkeypatch: Any) -> None:
|
||||
def _unexpected_resolve() -> dict[str, Any]:
|
||||
raise AssertionError("cached integrations should not re-resolve")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
_unexpected_resolve,
|
||||
)
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {"datadog": {"site": "datadoghq.com"}}
|
||||
|
||||
result = session.get_integrations()
|
||||
|
||||
assert isinstance(result, IntegrationResolutionResult)
|
||||
assert result.resolved_integrations == {"datadog": {"site": "datadoghq.com"}}
|
||||
assert result.services == ("datadog",)
|
||||
|
||||
|
||||
def test_get_integrations_respects_explicit_empty_cache(monkeypatch: Any) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: calls.append("resolve") or {"datadog": {}},
|
||||
)
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
result = session.get_integrations()
|
||||
|
||||
assert result.resolved_integrations == {}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_get_integrations_warms_metadata_only_cache(monkeypatch: Any) -> None:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
lambda: calls.append("resolve") or {"datadog": {"site": "datadoghq.com"}},
|
||||
)
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {"_gateway_chat_id": "chat-1"}
|
||||
|
||||
result = session.get_integrations()
|
||||
|
||||
assert calls == ["resolve"]
|
||||
assert result.resolved_integrations == {
|
||||
"_gateway_chat_id": "chat-1",
|
||||
"datadog": {"site": "datadoghq.com"},
|
||||
}
|
||||
|
||||
|
||||
def test_stale_background_warm_does_not_overwrite_refreshed_cache() -> None:
|
||||
session = Session()
|
||||
stale_generation = session.integrations._warm_generation
|
||||
session.integrations._warm_generation += 1
|
||||
session.integrations._store(
|
||||
{"fresh": {"token": "new"}}, generation=session.integrations._warm_generation
|
||||
)
|
||||
session.integrations._store({"stale": {"token": "old"}}, generation=stale_generation)
|
||||
assert session.resolved_integrations_cache == {"fresh": {"token": "new"}}
|
||||
|
||||
|
||||
def test_hydrate_entrypoint_does_not_warm_before_prompt(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
lambda: ["datadog"],
|
||||
)
|
||||
resolve_calls: list[str] = []
|
||||
|
||||
def _resolve() -> dict[str, Any]:
|
||||
resolve_calls.append("resolve")
|
||||
return {"datadog": {"site": "datadoghq.com"}}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.agent_harness.session.integration_resolution.resolve_integrations",
|
||||
_resolve,
|
||||
)
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert session.configured_integrations_known is True
|
||||
assert session.resolved_integrations_cache is None
|
||||
assert resolve_calls == []
|
||||
|
||||
|
||||
def test_hydrate_leaves_unknown_on_failure(monkeypatch: Any) -> None:
|
||||
def _boom() -> list[str]:
|
||||
raise RuntimeError("catalog blew up")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"platform.harness_ports.configured_integration_services",
|
||||
_boom,
|
||||
)
|
||||
session = Session()
|
||||
session.hydrate_configured_integrations()
|
||||
assert session.configured_integrations_known is False
|
||||
assert session.configured_integrations == ()
|
||||
|
||||
|
||||
def test_gate_error_blocks_startup_without_bypass(monkeypatch: Any) -> None:
|
||||
"""On an unexpected gate error we must NOT fail open into the REPL unless an
|
||||
explicit bypass applies."""
|
||||
monkeypatch.setattr(
|
||||
flg,
|
||||
"should_require_github_login",
|
||||
lambda: (_ for _ in ()).throw(RuntimeError("gate broke")),
|
||||
)
|
||||
monkeypatch.setattr(flg, "_github_login_explicitly_bypassed", lambda: False)
|
||||
|
||||
assert flg.require_startup_github_login(_console()) is False
|
||||
|
||||
|
||||
def test_gate_error_allows_startup_with_bypass(monkeypatch: Any) -> None:
|
||||
monkeypatch.setattr(
|
||||
flg,
|
||||
"should_require_github_login",
|
||||
lambda: (_ for _ in ()).throw(RuntimeError("gate broke")),
|
||||
)
|
||||
monkeypatch.setattr(flg, "_github_login_explicitly_bypassed", lambda: True)
|
||||
|
||||
assert flg.require_startup_github_login(_console()) is True
|
||||
|
||||
|
||||
def test_run_repl_async_identifies_saved_github_username(monkeypatch: Any) -> None:
|
||||
identified: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"platform.analytics.cli.identify_saved_github_username",
|
||||
lambda: identified.append("called"),
|
||||
)
|
||||
|
||||
def _run_initial_input(*_args: Any, **_kwargs: Any) -> int:
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(main_entrypoint, "run_initial_input", _run_initial_input)
|
||||
|
||||
class _Session:
|
||||
active_theme_name = None
|
||||
|
||||
def hydrate_configured_integrations(self) -> None:
|
||||
return None
|
||||
|
||||
def warm_resolved_integrations(self) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
main_entrypoint,
|
||||
"create_repl_runtime_context",
|
||||
lambda **_kwargs: SimpleNamespace(session=_Session(), inbox=None),
|
||||
)
|
||||
|
||||
class _PromptSession:
|
||||
history = None
|
||||
|
||||
monkeypatch.setattr(
|
||||
main_entrypoint,
|
||||
"build_prompt_session",
|
||||
lambda: _PromptSession(),
|
||||
)
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main_entrypoint.run_repl_async(initial_input="hello"))
|
||||
|
||||
assert identified == ["called"]
|
||||
|
||||
|
||||
def test_run_repl_async_failed_resume_flushes_starter_session(
|
||||
monkeypatch: Any, tmp_path: Path
|
||||
) -> None:
|
||||
import asyncio
|
||||
|
||||
sessions_dir = tmp_path / "sessions"
|
||||
sessions_dir.mkdir()
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"platform.analytics.cli.identify_saved_github_username",
|
||||
lambda: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.session_cmds.resume.resume_session_by_prefix",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
flushed: list[str] = []
|
||||
original_flush = session.storage.flush
|
||||
|
||||
def _track_flush(current_session: Session) -> None:
|
||||
flushed.append(current_session.session_id)
|
||||
original_flush(current_session)
|
||||
|
||||
monkeypatch.setattr(session.storage, "flush", _track_flush)
|
||||
|
||||
class _PromptSession:
|
||||
history = None
|
||||
|
||||
monkeypatch.setattr(
|
||||
main_entrypoint,
|
||||
"build_prompt_session",
|
||||
lambda: _PromptSession(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
main_entrypoint,
|
||||
"create_repl_runtime_context",
|
||||
lambda **_kwargs: SimpleNamespace(session=session, inbox=None),
|
||||
)
|
||||
|
||||
exit_code = asyncio.run(main_entrypoint.run_repl_async(resume_session_id="missing-session"))
|
||||
|
||||
assert exit_code == 1
|
||||
assert flushed == [session.session_id]
|
||||
assert not (sessions_dir / f"{session.session_id}.jsonl").exists()
|
||||
|
||||
|
||||
def test_explicit_bypass_detects_skip_env(monkeypatch: Any) -> None:
|
||||
monkeypatch.setenv("OPENSRE_SKIP_GITHUB_LOGIN", "1")
|
||||
assert flg._github_login_explicitly_bypassed() is True
|
||||
|
||||
|
||||
def test_explicit_bypass_detects_ci_environment(monkeypatch: Any) -> None:
|
||||
monkeypatch.delenv("OPENSRE_SKIP_GITHUB_LOGIN", raising=False)
|
||||
monkeypatch.setenv("CI", "true")
|
||||
assert flg._github_login_explicitly_bypassed() is True
|
||||
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from integrations.github import login as github_login_mod
|
||||
from integrations.github.login import GitHubLoginResult
|
||||
from integrations.github.mcp import DEFAULT_GITHUB_MCP_TOOLSETS, DEFAULT_GITHUB_MCP_URL
|
||||
from integrations.github.mcp_oauth import GitHubDeviceCode
|
||||
from platform.analytics import source as analytics_source
|
||||
from platform.terminal import theme as ui_theme
|
||||
from surfaces.interactive_shell.runtime.startup import first_launch_github as flg
|
||||
|
||||
|
||||
def _console() -> Console:
|
||||
return Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
|
||||
def _terminal_console(output: io.StringIO) -> Console:
|
||||
return Console(file=output, force_terminal=True, color_system="truecolor", highlight=False)
|
||||
|
||||
|
||||
def _force_required(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Set every gate input so that login would be required."""
|
||||
monkeypatch.delenv("OPENSRE_SKIP_GITHUB_LOGIN", raising=False)
|
||||
monkeypatch.setattr(flg, "is_test_run", lambda: False)
|
||||
monkeypatch.setattr(flg, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(flg, "_github_already_configured", lambda: False)
|
||||
monkeypatch.setattr(flg, "read_github_login_deferred", lambda: False)
|
||||
|
||||
|
||||
def test_gate_required_when_all_conditions_met(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
assert flg.should_require_github_login() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["1", "true", "YES", "on"])
|
||||
def test_gate_skipped_by_env(monkeypatch: pytest.MonkeyPatch, value: str) -> None:
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setenv("OPENSRE_SKIP_GITHUB_LOGIN", value)
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_skipped_in_test_run(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(flg, "is_test_run", lambda: True)
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_required_on_linux_when_interactive(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
assert flg.should_require_github_login() is True
|
||||
|
||||
|
||||
def test_gate_skipped_when_not_tty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(flg, "repl_tty_interactive", lambda: False)
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_skipped_when_github_already_configured(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(flg, "_github_already_configured", lambda: True)
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_required_when_github_not_configured(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Regression: GitHub config is authoritative. A prior completed login (no
|
||||
longer recorded via any standalone marker) must not let the REPL start once
|
||||
the GitHub integration has been removed."""
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(flg, "_github_already_configured", lambda: False)
|
||||
assert flg.should_require_github_login() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ci_env",
|
||||
[
|
||||
("CI", "true"),
|
||||
("GITHUB_ACTIONS", "true"),
|
||||
("OPENSRE_IS_TEST", "1"),
|
||||
],
|
||||
)
|
||||
def test_gate_skipped_in_ci_like_environment(
|
||||
monkeypatch: pytest.MonkeyPatch, ci_env: tuple[str, str]
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENSRE_SKIP_GITHUB_LOGIN", raising=False)
|
||||
monkeypatch.delenv("CI", raising=False)
|
||||
monkeypatch.delenv("GITHUB_ACTIONS", raising=False)
|
||||
monkeypatch.delenv("OPENSRE_IS_TEST", raising=False)
|
||||
monkeypatch.delenv("OPENSRE_INVESTIGATION_SOURCE", raising=False)
|
||||
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
|
||||
monkeypatch.setattr(flg, "is_test_run", analytics_source.is_test_run)
|
||||
monkeypatch.setattr(flg, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(flg, "_github_already_configured", lambda: False)
|
||||
monkeypatch.setenv(ci_env[0], ci_env[1])
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_skipped_when_github_login_deferred(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(flg, "read_github_login_deferred", lambda: True)
|
||||
assert flg.should_require_github_login() is False
|
||||
|
||||
|
||||
def test_gate_required_when_stale_github_store_record_has_no_token(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Legacy installs may have an abandoned github store row without credentials."""
|
||||
_force_required(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"integrations.store.get_integration",
|
||||
lambda service: (
|
||||
{
|
||||
"credentials": {
|
||||
"mode": "streamable-http",
|
||||
"url": DEFAULT_GITHUB_MCP_URL,
|
||||
"toolsets": list(DEFAULT_GITHUB_MCP_TOOLSETS),
|
||||
}
|
||||
}
|
||||
if service == "github"
|
||||
else None
|
||||
),
|
||||
)
|
||||
assert flg.should_require_github_login() is True
|
||||
|
||||
|
||||
def test_device_code_prompt_highlights_user_code(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("NO_COLOR", raising=False)
|
||||
monkeypatch.delenv("FORCE_COLOR", raising=False)
|
||||
ui_theme.set_active_theme("blue")
|
||||
output = io.StringIO()
|
||||
code = GitHubDeviceCode(
|
||||
device_code="dev-123",
|
||||
user_code="WXYZ-1234",
|
||||
verification_uri="https://github.com/login/device",
|
||||
expires_in=900,
|
||||
interval=5,
|
||||
)
|
||||
|
||||
flg._show_device_code(_terminal_console(output), code)
|
||||
|
||||
rendered = output.getvalue()
|
||||
assert f"{ui_theme.DEVICE_CODE_ANSI}WXYZ-1234" in rendered
|
||||
|
||||
|
||||
def test_orchestrator_success_proceeds_and_propagates(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "_offer_github_login", lambda _console: True)
|
||||
monkeypatch.setattr(
|
||||
github_login_mod,
|
||||
"authenticate_and_configure_github",
|
||||
lambda **_kwargs: GitHubLoginResult(ok=True, username="octocat", detail="OK"),
|
||||
)
|
||||
completed: list[str] = []
|
||||
monkeypatch.setattr(flg, "capture_github_login_completed", completed.append)
|
||||
cleared: list[bool] = []
|
||||
monkeypatch.setattr(flg, "clear_github_login_deferral", lambda: cleared.append(True))
|
||||
|
||||
proceed = flg.require_github_login_on_first_launch(_console())
|
||||
|
||||
assert proceed is True
|
||||
assert completed == ["octocat"]
|
||||
assert cleared == [True]
|
||||
|
||||
|
||||
def test_orchestrator_skip_at_menu_proceeds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "_offer_github_login", lambda _console: False)
|
||||
deferred: list[bool] = []
|
||||
monkeypatch.setattr(flg, "write_github_login_deferred", deferred.append)
|
||||
|
||||
proceed = flg.require_github_login_on_first_launch(_console())
|
||||
|
||||
assert proceed is True
|
||||
assert deferred == [True]
|
||||
|
||||
|
||||
def test_orchestrator_escape_during_wait_proceeds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "_offer_github_login", lambda _console: True)
|
||||
|
||||
def _raise_cancel(**_kwargs: object) -> GitHubLoginResult:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
monkeypatch.setattr(github_login_mod, "authenticate_and_configure_github", _raise_cancel)
|
||||
deferred: list[bool] = []
|
||||
monkeypatch.setattr(flg, "write_github_login_deferred", deferred.append)
|
||||
|
||||
proceed = flg.require_github_login_on_first_launch(_console())
|
||||
|
||||
assert proceed is True
|
||||
assert deferred == [True]
|
||||
|
||||
|
||||
def test_orchestrator_failure_then_decline_retry_skips(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "_offer_github_login", lambda _console: True)
|
||||
monkeypatch.setattr(
|
||||
github_login_mod,
|
||||
"authenticate_and_configure_github",
|
||||
lambda **_kwargs: GitHubLoginResult(ok=False, detail="cannot verify"),
|
||||
)
|
||||
monkeypatch.setattr(flg, "_ask_retry", lambda _console: False)
|
||||
deferred: list[bool] = []
|
||||
monkeypatch.setattr(flg, "write_github_login_deferred", deferred.append)
|
||||
|
||||
proceed = flg.require_github_login_on_first_launch(_console())
|
||||
|
||||
assert proceed is True
|
||||
assert deferred == [True]
|
||||
|
||||
|
||||
def test_orchestrator_retries_until_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "_offer_github_login", lambda _console: True)
|
||||
calls = {"n": 0}
|
||||
|
||||
def _login(**_kwargs: object) -> GitHubLoginResult:
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
return GitHubLoginResult(ok=False, detail="cannot verify")
|
||||
return GitHubLoginResult(ok=True, username="octocat", detail="OK")
|
||||
|
||||
monkeypatch.setattr(github_login_mod, "authenticate_and_configure_github", _login)
|
||||
monkeypatch.setattr(flg, "_ask_retry", lambda _console: True)
|
||||
monkeypatch.setattr(flg, "capture_github_login_completed", lambda _username: None)
|
||||
monkeypatch.setattr(flg, "clear_github_login_deferral", lambda: None)
|
||||
|
||||
proceed = flg.require_github_login_on_first_launch(_console())
|
||||
|
||||
assert proceed is True
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_clear_github_login_deferral_noop_when_not_deferred(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(flg, "read_github_login_deferred", lambda: False)
|
||||
writes: list[bool] = []
|
||||
monkeypatch.setattr(flg, "write_github_login_deferred", writes.append)
|
||||
|
||||
flg.clear_github_login_deferral()
|
||||
|
||||
assert writes == []
|
||||
|
||||
|
||||
def test_clear_github_login_deferral_clears_when_deferred(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg, "read_github_login_deferred", lambda: True)
|
||||
writes: list[bool] = []
|
||||
monkeypatch.setattr(flg, "write_github_login_deferred", writes.append)
|
||||
|
||||
flg.clear_github_login_deferral()
|
||||
|
||||
assert writes == [False]
|
||||
|
||||
|
||||
def test_sleep_until_or_cancel_raises_on_escape(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(flg.os, "name", "posix")
|
||||
monkeypatch.setattr(flg.sys.stdin, "isatty", lambda: True)
|
||||
monkeypatch.setattr(flg.sys.stdin, "fileno", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.key_reader.read_key_unix",
|
||||
lambda **_kwargs: "cancel",
|
||||
)
|
||||
monkeypatch.setattr("termios.tcgetattr", lambda _fd: [0] * 7)
|
||||
monkeypatch.setattr("tty.setraw", lambda _fd: None)
|
||||
restored: list[bool] = []
|
||||
monkeypatch.setattr(
|
||||
"termios.tcsetattr",
|
||||
lambda _fd, _when, _attrs: restored.append(True),
|
||||
)
|
||||
|
||||
def _ready(
|
||||
_fd: int, *_args: object, **_kwargs: object
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
return ([0], [], [])
|
||||
|
||||
monkeypatch.setattr("select.select", _ready)
|
||||
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
flg._sleep_until_or_cancel(1.0)
|
||||
|
||||
assert restored == [True]
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Tests for prompt input event conversion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.runtime.core.state import ReplState
|
||||
from surfaces.interactive_shell.runtime.input import (
|
||||
InputCancelled,
|
||||
InputClosed,
|
||||
InputSubmitted,
|
||||
PromptInputReader,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.input import prompt_input_reader as reader_module
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
class FakePrompt:
|
||||
def __init__(self, read: Callable[[], str]) -> None:
|
||||
self._read = read
|
||||
|
||||
async def read_prompt_text(self) -> str:
|
||||
return self._read()
|
||||
|
||||
|
||||
class SequencePrompt:
|
||||
def __init__(self, values: list[str]) -> None:
|
||||
self._values = values
|
||||
|
||||
async def read_prompt_text(self) -> str:
|
||||
return self._values.pop(0)
|
||||
|
||||
|
||||
def _reader(
|
||||
prompt: Any,
|
||||
state: ReplState | None = None,
|
||||
session: Session | None = None,
|
||||
console: Console | None = None,
|
||||
) -> PromptInputReader:
|
||||
return PromptInputReader(
|
||||
prompt,
|
||||
state or ReplState(),
|
||||
session or Session(),
|
||||
console or Console(file=io.StringIO(), force_terminal=False),
|
||||
)
|
||||
|
||||
|
||||
def _running_state() -> tuple[ReplState, asyncio.Task[None]]:
|
||||
state = ReplState()
|
||||
task = asyncio.create_task(asyncio.sleep(60))
|
||||
state.start_dispatch(task=task, cancel_event=threading.Event())
|
||||
return state, task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_submits_normal_text(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
reset_calls = 0
|
||||
|
||||
def _reset() -> None:
|
||||
nonlocal reset_calls
|
||||
reset_calls += 1
|
||||
|
||||
monkeypatch.setattr(reader_module, "repl_reset_ctrl_c_gate", _reset)
|
||||
|
||||
event = await _reader(FakePrompt(lambda: "show incidents")).read()
|
||||
|
||||
assert event == InputSubmitted("show incidents")
|
||||
assert reset_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_eof_with_dispatch_running_returns_cancelled() -> None:
|
||||
state, task = _running_state()
|
||||
try:
|
||||
event = await _reader(FakePrompt(lambda: (_ for _ in ()).throw(EOFError)), state).read()
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
assert event == InputCancelled()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_eof_without_dispatch_renders_resume_hint() -> None:
|
||||
output = io.StringIO()
|
||||
session = Session()
|
||||
session.session_id = "session-123"
|
||||
console = Console(file=output, force_terminal=False, color_system=None)
|
||||
|
||||
event = await _reader(
|
||||
FakePrompt(lambda: (_ for _ in ()).throw(EOFError)),
|
||||
session=session,
|
||||
console=console,
|
||||
).read()
|
||||
|
||||
assert event == InputClosed()
|
||||
assert "Resume this session with:" in output.getvalue()
|
||||
assert "/resume session-123" in output.getvalue()
|
||||
assert "--resume session-123" in output.getvalue()
|
||||
assert "Goodbye!" in output.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_keyboard_interrupt_with_dispatch_running_returns_cancelled() -> (
|
||||
None
|
||||
):
|
||||
state, task = _running_state()
|
||||
try:
|
||||
event = await _reader(
|
||||
FakePrompt(lambda: (_ for _ in ()).throw(KeyboardInterrupt)),
|
||||
state,
|
||||
).read()
|
||||
finally:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
assert event == InputCancelled()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_keyboard_interrupt_without_dispatch_uses_ctrl_c_gate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(reader_module, "repl_prompt_note_ctrl_c", lambda _console, _sid: False)
|
||||
|
||||
event = await _reader(FakePrompt(lambda: (_ for _ in ()).throw(KeyboardInterrupt))).read()
|
||||
|
||||
assert event == InputCancelled()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_keyboard_interrupt_without_dispatch_can_close(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(reader_module, "repl_prompt_note_ctrl_c", lambda _console, _sid: True)
|
||||
|
||||
event = await _reader(FakePrompt(lambda: (_ for _ in ()).throw(KeyboardInterrupt))).read()
|
||||
|
||||
assert event == InputClosed()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_strips_cpr_sequences() -> None:
|
||||
event = await _reader(FakePrompt(lambda: "status\x1b[12;80R now")).read()
|
||||
|
||||
assert event == InputSubmitted("status now")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_input_reader_ignores_cpr_only_input() -> None:
|
||||
event = await _reader(
|
||||
SequencePrompt(["\x1b[12;80R", "show status"]),
|
||||
).read()
|
||||
|
||||
assert event == InputSubmitted("show status")
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Tests for ReplSubprocessPresenter Rich markup escaping."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from io import StringIO
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.runtime.subprocess_runner.repl_presenter import (
|
||||
ReplSubprocessPresenter,
|
||||
_escape_markup_message,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _presenter() -> tuple[ReplSubprocessPresenter, StringIO]:
|
||||
buffer = StringIO()
|
||||
console = Console(file=buffer, force_terminal=True, width=80, color_system=None)
|
||||
session = Session()
|
||||
return ReplSubprocessPresenter(session, console), buffer
|
||||
|
||||
|
||||
def test_escape_markup_message_preserves_intentional_tags() -> None:
|
||||
escaped = _escape_markup_message("[error]failed[/]")
|
||||
buffer = StringIO()
|
||||
Console(file=buffer, force_terminal=True, width=80, color_system=None).print(escaped)
|
||||
output = buffer.getvalue()
|
||||
assert "failed" in output
|
||||
|
||||
|
||||
def test_escape_markup_message_escapes_plain_markup() -> None:
|
||||
escaped = _escape_markup_message("Grafana [prod] rate[5m]")
|
||||
buffer = StringIO()
|
||||
Console(file=buffer, force_terminal=True, width=80, color_system=None).print(escaped)
|
||||
output = buffer.getvalue()
|
||||
assert "[prod]" in output
|
||||
assert "rate[5m]" in output
|
||||
|
||||
|
||||
def test_escape_markup_message_escapes_dynamic_text_inside_tags() -> None:
|
||||
escaped = _escape_markup_message("[bold]task [critical][/bold]")
|
||||
assert "task" in escaped
|
||||
assert "[critical]" in escaped
|
||||
|
||||
|
||||
def test_print_error_escapes_dynamic_markup() -> None:
|
||||
presenter, buffer = _presenter()
|
||||
presenter.print_error("failed: [bold]bad[/]")
|
||||
output = buffer.getvalue()
|
||||
assert "[bold]bad[/]" in output
|
||||
assert "bad" in output
|
||||
|
||||
|
||||
def test_print_escapes_untrusted_suffix_via_print_error_pattern() -> None:
|
||||
presenter, buffer = _presenter()
|
||||
presenter.print_error("command failed to start: [bold]bad[/]")
|
||||
output = buffer.getvalue()
|
||||
assert "[bold]bad[/]" in output
|
||||
|
||||
|
||||
def test_print_preserves_task_id_markup_with_escaped_brackets() -> None:
|
||||
presenter, buffer = _presenter()
|
||||
task_id = "task-[critical]"
|
||||
presenter.print(
|
||||
f"[dim]synthetic test started — task[/] [bold]{task_id}[/bold]. "
|
||||
"[highlight]/tasks[/] [dim]to monitor.[/]"
|
||||
)
|
||||
output = buffer.getvalue()
|
||||
assert task_id in output
|
||||
assert "/tasks" in output
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Tests for Session state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import config.constants as const_module
|
||||
from config.constants.prompts import SUGGESTED_PROMPT_AFTER_FAILED_SYNTHETIC_TEST
|
||||
from platform.common.task_registry import TaskRegistry
|
||||
from platform.common.task_types import TaskKind
|
||||
from surfaces.interactive_shell.session import (
|
||||
Session,
|
||||
)
|
||||
from surfaces.interactive_shell.session.session import _scenario_id_from_synthetic_label
|
||||
|
||||
|
||||
class TestSession:
|
||||
def test_defaults(self) -> None:
|
||||
session = Session()
|
||||
assert session.history == []
|
||||
assert session.last_state is None
|
||||
assert session.accumulated_context == {}
|
||||
assert session.terminal.trust_mode is False
|
||||
assert session.task_registry.list_recent() == []
|
||||
assert session.terminal.metrics.turn_count == 0
|
||||
assert session.terminal.metrics.fallback_count == 0
|
||||
assert session.terminal.metrics.ctrl_c_intervention_count == 0
|
||||
assert session.terminal.metrics.correction_intervention_count == 0
|
||||
assert session.terminal.pending_prompt_default is None
|
||||
assert session.last_synthetic_observation_path is None
|
||||
|
||||
def test_take_pending_prompt_default_returns_and_clears(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.pending_prompt_default = "why did it fail?"
|
||||
assert session.terminal.pop_pending_prompt_default() == "why did it fail?"
|
||||
assert session.terminal.pending_prompt_default is None
|
||||
assert session.terminal.pop_pending_prompt_default() == ""
|
||||
|
||||
def test_clear_resets_pending_prompt_default(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.pending_prompt_default = "why did it fail?"
|
||||
session.clear()
|
||||
assert session.terminal.pending_prompt_default is None
|
||||
|
||||
def test_queue_auto_command_sets_pending_and_notifies(self) -> None:
|
||||
session = Session()
|
||||
calls: list[bool] = []
|
||||
session.terminal.prompt_refresh_fn = lambda: calls.append(True)
|
||||
session.terminal.set_auto_command("/integrations setup sentry")
|
||||
assert session.terminal.pending_prompt_default == "/integrations setup sentry"
|
||||
assert session.terminal.pending_prompt_autosubmit is True
|
||||
assert calls == [True]
|
||||
|
||||
def test_take_pending_autosubmit_returns_and_clears(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.pending_prompt_autosubmit = True
|
||||
assert session.terminal.pop_pending_autosubmit() is True
|
||||
assert session.terminal.pending_prompt_autosubmit is False
|
||||
assert session.terminal.pop_pending_autosubmit() is False
|
||||
|
||||
def test_clear_resets_pending_autosubmit(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.set_auto_command("/integrations setup sentry")
|
||||
session.clear()
|
||||
assert session.terminal.pending_prompt_autosubmit is False
|
||||
assert session.terminal.pending_prompt_default is None
|
||||
|
||||
def test_scenario_id_from_synthetic_label(self) -> None:
|
||||
assert (
|
||||
_scenario_id_from_synthetic_label(
|
||||
"opensre tests synthetic --scenario 001-replication-lag"
|
||||
)
|
||||
== "001-replication-lag"
|
||||
)
|
||||
assert _scenario_id_from_synthetic_label("rds_postgres:001-replication-lag") == (
|
||||
"001-replication-lag"
|
||||
)
|
||||
assert _scenario_id_from_synthetic_label("opensre tests synthetic --scenario ./evil") == ""
|
||||
assert _scenario_id_from_synthetic_label("rds_postgres:not-a-scenario") == ""
|
||||
|
||||
def test_suggest_synthetic_failure_follow_up_sets_pending(self) -> None:
|
||||
session = Session()
|
||||
session.suggest_synthetic_failure_follow_up(
|
||||
label="opensre tests synthetic --scenario 001-replication-lag",
|
||||
)
|
||||
assert (
|
||||
session.terminal.pending_prompt_default == SUGGESTED_PROMPT_AFTER_FAILED_SYNTHETIC_TEST
|
||||
)
|
||||
|
||||
def test_record_appends_entry(self) -> None:
|
||||
session = Session()
|
||||
session.record("alert", "cpu high")
|
||||
session.record("slash", "/status", ok=True)
|
||||
session.record("alert", "bad one", ok=False)
|
||||
assert len(session.history) == 3
|
||||
assert session.history[-1]["type"] == "alert"
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_mark_latest_updates_most_recent_matching_kind(self) -> None:
|
||||
session = Session()
|
||||
session.record("slash", "/investigate missing.json")
|
||||
session.record("alert", "missing.json", ok=False)
|
||||
|
||||
session.mark_latest(ok=False, kind="slash")
|
||||
|
||||
assert session.history[0]["ok"] is False
|
||||
assert session.history[1]["ok"] is False
|
||||
|
||||
def test_clear_preserves_trust_mode(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
session.terminal.background_notification_preferences.set_channels(["email"])
|
||||
session.accumulated_context["service"] = "api"
|
||||
session.record("alert", "something")
|
||||
session.last_state = {"foo": "bar"}
|
||||
session.agent.messages.append(("user", "hey"))
|
||||
session.terminal.metrics.record_intervention("ctrl_c")
|
||||
session.terminal.metrics.record_intervention("correction")
|
||||
|
||||
assert session.terminal.history_generation == 0
|
||||
session.clear()
|
||||
assert session.terminal.history_generation == 1
|
||||
|
||||
assert session.history == []
|
||||
assert session.last_state is None
|
||||
assert session.accumulated_context == {}
|
||||
assert session.agent.messages == []
|
||||
assert session.task_registry.list_recent() == []
|
||||
assert session.terminal.metrics.ctrl_c_intervention_count == 0
|
||||
assert session.terminal.metrics.correction_intervention_count == 0
|
||||
assert session.terminal.background_notification_preferences.channels == ("email",)
|
||||
assert session.terminal.trust_mode is True # preserved intentionally
|
||||
|
||||
def test_clear_keeps_persisted_task_history_file(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
session = Session()
|
||||
monkeypatch.setattr(const_module, "OPENSRE_HOME_DIR", tmp_path)
|
||||
session.task_registry = TaskRegistry.persistent()
|
||||
task = session.task_registry.create(
|
||||
TaskKind.SYNTHETIC_TEST, command="opensre tests synthetic"
|
||||
)
|
||||
task.mark_running()
|
||||
|
||||
session.clear()
|
||||
|
||||
reloaded = TaskRegistry.persistent()
|
||||
loaded = reloaded.get(task.task_id)
|
||||
assert loaded is not None
|
||||
assert loaded.task_id == task.task_id
|
||||
|
||||
def test_accumulate_from_state_extracts_known_keys(self) -> None:
|
||||
session = Session()
|
||||
session.accumulate_from_state(
|
||||
{
|
||||
"service": "orders-api",
|
||||
"pipeline_name": "events_fact",
|
||||
"cluster_name": "prod-us-east",
|
||||
"region": "us-east-1",
|
||||
"environment": "production",
|
||||
"root_cause": "disk full", # not accumulated
|
||||
"evidence": {"ev-1": "x"}, # not accumulated
|
||||
}
|
||||
)
|
||||
assert session.accumulated_context == {
|
||||
"service": "orders-api",
|
||||
"pipeline_name": "events_fact",
|
||||
"cluster_name": "prod-us-east",
|
||||
"region": "us-east-1",
|
||||
"environment": "production",
|
||||
}
|
||||
|
||||
def test_accumulate_from_state_skips_empty_and_none(self) -> None:
|
||||
session = Session()
|
||||
session.accumulate_from_state(
|
||||
{
|
||||
"service": "",
|
||||
"cluster_name": None,
|
||||
"region": "us-east-1",
|
||||
}
|
||||
)
|
||||
assert session.accumulated_context == {"region": "us-east-1"}
|
||||
|
||||
def test_accumulate_from_state_merges_across_calls(self) -> None:
|
||||
"""Subsequent investigations fill in context the earlier one didn't have."""
|
||||
session = Session()
|
||||
session.accumulate_from_state({"service": "orders-api"})
|
||||
session.accumulate_from_state({"cluster_name": "prod-us-east"})
|
||||
assert session.accumulated_context == {
|
||||
"service": "orders-api",
|
||||
"cluster_name": "prod-us-east",
|
||||
}
|
||||
|
||||
def test_accumulate_from_state_handles_none_and_empty_state(self) -> None:
|
||||
session = Session()
|
||||
session.accumulate_from_state(None)
|
||||
session.accumulate_from_state({})
|
||||
assert session.accumulated_context == {}
|
||||
|
||||
def test_record_terminal_turn_updates_aggregates(self) -> None:
|
||||
session = Session()
|
||||
|
||||
first = session.terminal.metrics.record_turn(
|
||||
executed_count=2,
|
||||
executed_success_count=1,
|
||||
fallback_to_llm=True,
|
||||
)
|
||||
second = session.terminal.metrics.record_turn(
|
||||
executed_count=1,
|
||||
executed_success_count=1,
|
||||
fallback_to_llm=False,
|
||||
)
|
||||
|
||||
assert first.turn_index == 1
|
||||
assert first.fallback_count == 1
|
||||
assert first.action_success_percent == 50.0
|
||||
assert first.fallback_rate_percent == 100.0
|
||||
|
||||
assert second.turn_index == 2
|
||||
assert second.fallback_count == 1
|
||||
assert round(second.action_success_percent, 2) == 66.67
|
||||
assert second.fallback_rate_percent == 50.0
|
||||
|
||||
def test_record_intervention_increments_per_kind(self) -> None:
|
||||
session = Session()
|
||||
|
||||
session.terminal.metrics.record_intervention("ctrl_c")
|
||||
session.terminal.metrics.record_intervention("ctrl_c")
|
||||
session.terminal.metrics.record_intervention("correction")
|
||||
|
||||
assert session.terminal.metrics.ctrl_c_intervention_count == 2
|
||||
assert session.terminal.metrics.correction_intervention_count == 1
|
||||
|
||||
def test_record_intervention_kinds_are_independent(self) -> None:
|
||||
"""Incrementing one kind does not touch the other."""
|
||||
session = Session()
|
||||
|
||||
session.terminal.metrics.record_intervention("correction")
|
||||
|
||||
assert session.terminal.metrics.ctrl_c_intervention_count == 0
|
||||
assert session.terminal.metrics.correction_intervention_count == 1
|
||||
|
||||
def test_fresh_session_starts_with_zero_intervention_counts(self) -> None:
|
||||
"""A new Session does not inherit any prior session's counters."""
|
||||
first = Session()
|
||||
first.terminal.metrics.record_intervention("ctrl_c")
|
||||
first.terminal.metrics.record_intervention("correction")
|
||||
|
||||
second = Session()
|
||||
|
||||
assert second.terminal.metrics.ctrl_c_intervention_count == 0
|
||||
assert second.terminal.metrics.correction_intervention_count == 0
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Tests for the inline turn spinner in runtime.core.state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
|
||||
from surfaces.interactive_shell.runtime.core.state import SpinnerState
|
||||
|
||||
_GLYPHS = SpinnerState._SPINNER_FRAMES
|
||||
|
||||
|
||||
def _glyph(spinner: SpinnerState) -> str:
|
||||
rendered = spinner.inline_spinner_ansi()
|
||||
match = re.search("|".join(map(re.escape, _GLYPHS)), rendered)
|
||||
assert match is not None, f"no spinner glyph in {rendered!r}"
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def test_spinner_frame_is_a_function_of_elapsed_time_not_call_count() -> None:
|
||||
"""Regression: the glyph must animate no matter how often it is rendered.
|
||||
|
||||
prompt_toolkit evaluates the prompt message callable several times per
|
||||
render pass. A per-call frame counter advanced exactly one full cycle per
|
||||
visible render (10 evals x 10 frames), so the on-screen glyph never
|
||||
changed. Deriving the frame from elapsed time makes repeated evaluations
|
||||
idempotent and guarantees the animation advances between renders.
|
||||
"""
|
||||
spinner = SpinnerState()
|
||||
spinner.start()
|
||||
|
||||
# Repeated evaluations inside one render pass: same frame every time.
|
||||
first = _glyph(spinner)
|
||||
assert all(_glyph(spinner) == first for _ in range(10))
|
||||
|
||||
# A frame interval later the glyph must have advanced.
|
||||
spinner.started_at -= SpinnerState._FRAME_INTERVAL_S * 1.01
|
||||
assert _glyph(spinner) != first
|
||||
|
||||
# Over a full cycle of elapsed time, every frame is visited in order.
|
||||
spinner.start()
|
||||
seen = []
|
||||
for step in range(len(_GLYPHS)):
|
||||
spinner.started_at = time.monotonic() - step * SpinnerState._FRAME_INTERVAL_S * 1.001
|
||||
seen.append(_glyph(spinner))
|
||||
assert seen == list(_GLYPHS)
|
||||
|
||||
|
||||
def test_spinner_renders_elapsed_seconds_and_cancel_hint() -> None:
|
||||
spinner = SpinnerState()
|
||||
spinner.start()
|
||||
spinner.started_at = time.monotonic() - 8.2
|
||||
|
||||
rendered = spinner.inline_spinner_ansi()
|
||||
|
||||
assert "(8s)" in rendered
|
||||
assert "esc to cancel" in rendered
|
||||
|
||||
|
||||
def test_spinner_empty_when_not_streaming() -> None:
|
||||
spinner = SpinnerState()
|
||||
assert spinner.inline_spinner_ansi() == ""
|
||||
spinner.start()
|
||||
spinner.stop()
|
||||
assert spinner.inline_spinner_ansi() == ""
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.runtime.background.workers import BackgroundTaskManager
|
||||
from surfaces.interactive_shell.runtime.core.state import ReplState, SpinnerState
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spinner_ticker_invalidates_once_after_streaming_stops(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The ticker repaints while streaming and once more on the streaming->idle edge.
|
||||
|
||||
Without the trailing invalidation a stopped investigation would leave the
|
||||
last phase label rendered on the prompt until unrelated I/O redraws it.
|
||||
"""
|
||||
state = ReplState()
|
||||
spinner = SpinnerState()
|
||||
calls: list[int] = []
|
||||
manager = BackgroundTaskManager(
|
||||
cast(Session, cast(Any, object())),
|
||||
state,
|
||||
spinner,
|
||||
None,
|
||||
lambda: calls.append(1),
|
||||
)
|
||||
|
||||
ticks = 0
|
||||
real_sleep = asyncio.sleep
|
||||
|
||||
async def fake_sleep(_seconds: float) -> None:
|
||||
nonlocal ticks
|
||||
ticks += 1
|
||||
if ticks == 1:
|
||||
spinner.set_phase("Investigation") # streaming on
|
||||
elif ticks == 3:
|
||||
spinner.stop() # streaming off (investigation done)
|
||||
elif ticks >= 5:
|
||||
state.exit_requested = True
|
||||
await real_sleep(0)
|
||||
|
||||
monkeypatch.setattr(asyncio, "sleep", fake_sleep)
|
||||
await manager._spinner_ticker()
|
||||
|
||||
# ticks 1-2 stream (2 invalidations) + one trailing edge at tick 3, then
|
||||
# silence while idle at tick 4.
|
||||
assert len(calls) == 3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,565 @@
|
||||
"""Tests for REPL task registry and /tasks · /cancel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Callable, Iterator
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from config.constants.prompts import SUGGESTED_PROMPT_AFTER_FAILED_SYNTHETIC_TEST
|
||||
from platform.common.task_registry import TaskRegistry
|
||||
from platform.common.task_types import TaskKind, TaskStatus
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.session import (
|
||||
Session,
|
||||
)
|
||||
from tools.interactive_shell.synthetic.runner import watch_synthetic_subprocess
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stderr_buf() -> Iterator[tempfile.SpooledTemporaryFile]: # type: ignore[type-arg]
|
||||
"""Stderr buffer for synthetic-watcher tests.
|
||||
|
||||
The watcher's ``finally`` block closes the buffer; the ``with`` here
|
||||
is the belt-and-braces close that runs when a test exits before the
|
||||
watcher does (e.g. deferred-thread tests where ``pending[0]()`` is
|
||||
never invoked).
|
||||
"""
|
||||
with tempfile.SpooledTemporaryFile() as buf:
|
||||
yield buf
|
||||
|
||||
|
||||
class TestTaskRecord:
|
||||
def test_lifecycle_completed(self) -> None:
|
||||
reg = TaskRegistry()
|
||||
t = reg.create(TaskKind.INVESTIGATION)
|
||||
assert t.status == TaskStatus.PENDING
|
||||
t.mark_running()
|
||||
assert t.status == TaskStatus.RUNNING
|
||||
t.mark_completed(result="done")
|
||||
assert t.status == TaskStatus.COMPLETED
|
||||
assert t.result == "done"
|
||||
assert t.ended_at is not None
|
||||
t.mark_failed("x")
|
||||
assert t.status == TaskStatus.COMPLETED
|
||||
|
||||
def test_mark_cancelled_idempotent_after_terminal(self) -> None:
|
||||
reg = TaskRegistry()
|
||||
t = reg.create(TaskKind.INVESTIGATION)
|
||||
t.mark_running()
|
||||
t.mark_cancelled()
|
||||
assert t.status == TaskStatus.CANCELLED
|
||||
t.mark_completed(result="nope")
|
||||
assert t.status == TaskStatus.CANCELLED
|
||||
|
||||
def test_request_cancel_sets_event_even_when_pending(self) -> None:
|
||||
reg = TaskRegistry()
|
||||
t = reg.create(TaskKind.INVESTIGATION)
|
||||
assert t.request_cancel() is False
|
||||
assert t.cancel_requested.is_set()
|
||||
assert t.status == TaskStatus.PENDING
|
||||
|
||||
def test_request_cancel_sets_event_and_terminates_process(self) -> None:
|
||||
reg = TaskRegistry()
|
||||
t = reg.create(TaskKind.SYNTHETIC_TEST)
|
||||
t.mark_running()
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
t.attach_process(proc)
|
||||
assert t.request_cancel() is True
|
||||
proc.terminate.assert_called_once()
|
||||
assert t.cancel_requested.is_set()
|
||||
|
||||
|
||||
class TestTaskRegistry:
|
||||
def test_get_single_prefix_match(self) -> None:
|
||||
reg = TaskRegistry()
|
||||
t = reg.create(TaskKind.INVESTIGATION)
|
||||
assert reg.get(t.task_id[:4]) == t
|
||||
assert reg.get("") is None
|
||||
|
||||
def test_candidates_ambiguous_prefix(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ids = iter(["11111111", "11112222"])
|
||||
|
||||
def _fake_hex(_nbytes: int) -> str:
|
||||
return next(_ids)
|
||||
|
||||
monkeypatch.setattr("platform.common.task_registry.secrets.token_hex", _fake_hex)
|
||||
session = Session()
|
||||
session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/cancel 1111", session, console)
|
||||
assert "ambiguous" in buf.getvalue().lower()
|
||||
|
||||
def test_ring_buffer_drops_oldest(self) -> None:
|
||||
reg = TaskRegistry(max_tasks=3)
|
||||
first = reg.create(TaskKind.INVESTIGATION)
|
||||
reg.create(TaskKind.INVESTIGATION)
|
||||
reg.create(TaskKind.INVESTIGATION)
|
||||
reg.create(TaskKind.INVESTIGATION)
|
||||
recent_ids = [t.task_id for t in reg.list_recent(10)]
|
||||
assert first.task_id not in recent_ids
|
||||
assert len(recent_ids) == 3
|
||||
|
||||
def test_persistent_registry_reloads_running_pid(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import config.constants as const_module
|
||||
|
||||
monkeypatch.setattr(const_module, "OPENSRE_HOME_DIR", tmp_path)
|
||||
reg = TaskRegistry.persistent()
|
||||
task = reg.create(TaskKind.SYNTHETIC_TEST, command="opensre tests synthetic")
|
||||
task.mark_running()
|
||||
task.attach_pid(os.getpid())
|
||||
|
||||
reloaded = TaskRegistry.persistent()
|
||||
[loaded] = reloaded.list_recent()
|
||||
assert loaded.task_id == task.task_id
|
||||
assert loaded.status == TaskStatus.RUNNING
|
||||
assert loaded.pid == os.getpid()
|
||||
assert loaded.command == "opensre tests synthetic"
|
||||
|
||||
def test_persistent_registry_marks_missing_pid_finished(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import config.constants as const_module
|
||||
|
||||
monkeypatch.setattr(const_module, "OPENSRE_HOME_DIR", tmp_path)
|
||||
store_path = tmp_path / "interactive_tasks.json"
|
||||
store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
store_path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"task_id": "abc12345",
|
||||
"kind": "synthetic_test",
|
||||
"status": "running",
|
||||
"started_at": 1.0,
|
||||
"ended_at": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
"pid": 999_999,
|
||||
"command": "opensre tests synthetic",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"platform.common.task_types.os.kill",
|
||||
lambda _pid, _sig: (_ for _ in ()).throw(ProcessLookupError()),
|
||||
)
|
||||
|
||||
reloaded = TaskRegistry.persistent()
|
||||
[loaded] = reloaded.list_recent()
|
||||
assert loaded.status == TaskStatus.COMPLETED
|
||||
assert loaded.result == "process exited while shell was closed"
|
||||
|
||||
def test_cancel_rehydrated_task_does_not_signal_pid(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import config.constants as const_module
|
||||
|
||||
monkeypatch.setattr(const_module, "OPENSRE_HOME_DIR", tmp_path)
|
||||
calls: list[tuple[int, int]] = []
|
||||
|
||||
def _fake_kill(pid: int, sig: int) -> None:
|
||||
calls.append((pid, sig))
|
||||
|
||||
monkeypatch.setattr("platform.common.task_types.os.kill", _fake_kill)
|
||||
reg = TaskRegistry.persistent()
|
||||
task = reg.create(TaskKind.SYNTHETIC_TEST, command="opensre tests synthetic")
|
||||
task.mark_running()
|
||||
task.attach_pid(12345)
|
||||
|
||||
reloaded = TaskRegistry.persistent()
|
||||
loaded = reloaded.get(task.task_id)
|
||||
assert loaded is not None
|
||||
assert loaded.request_cancel() is True
|
||||
assert loaded.status == TaskStatus.CANCELLED
|
||||
assert (12345, 15) not in calls
|
||||
|
||||
def test_session_new_does_not_truncate_persistent_task_store(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import config.constants as const_module
|
||||
|
||||
monkeypatch.setattr(const_module, "OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
session.task_registry = TaskRegistry.persistent()
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST, command="opensre tests")
|
||||
task.mark_running()
|
||||
task.mark_completed(result="ok")
|
||||
|
||||
session.clear()
|
||||
|
||||
# /new must keep the on-disk store intact: a fresh persistent registry
|
||||
# still finds the task, and the session's swapped-in registry continues
|
||||
# to surface persisted history via its disk-backed merge so /tasks does
|
||||
# not "forget" the user's prior runs after /new.
|
||||
reloaded = TaskRegistry.persistent()
|
||||
[loaded] = reloaded.list_recent()
|
||||
assert loaded.task_id == task.task_id
|
||||
assert loaded.command == "opensre tests"
|
||||
[visible_after_new] = session.task_registry.list_recent()
|
||||
assert visible_after_new.task_id == task.task_id
|
||||
|
||||
|
||||
class TestSlashTaskCommands:
|
||||
def test_tasks_empty_message(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/tasks", session, console)
|
||||
assert "no tasks" in buf.getvalue().lower()
|
||||
|
||||
def test_tasks_shows_recent_rows(self) -> None:
|
||||
session = Session()
|
||||
t = session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
t.mark_running()
|
||||
t.mark_completed(result="rc")
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/tasks", session, console)
|
||||
out = buf.getvalue()
|
||||
assert t.task_id in out
|
||||
assert "investigation" in out
|
||||
assert "completed" in out
|
||||
|
||||
def test_cancel_usage_without_id(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/cancel", session, console)
|
||||
assert "usage" in buf.getvalue().lower()
|
||||
|
||||
def test_cancel_unknown_id(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
dispatch_slash("/cancel deadbeef", session, console)
|
||||
assert "no task" in buf.getvalue().lower()
|
||||
|
||||
def test_cancel_completed_task_message(self) -> None:
|
||||
session = Session()
|
||||
t = session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
t.mark_running()
|
||||
t.mark_completed(result="x")
|
||||
console, buf = _capture()
|
||||
dispatch_slash(f"/cancel {t.task_id}", session, console)
|
||||
assert "already finished" in buf.getvalue().lower()
|
||||
|
||||
def test_cancel_running_investigation_signals(self) -> None:
|
||||
session = Session()
|
||||
t = session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
t.mark_running()
|
||||
console, buf = _capture()
|
||||
dispatch_slash(f"/cancel {t.task_id}", session, console)
|
||||
out = buf.getvalue()
|
||||
assert "cancellation" in out.lower()
|
||||
assert "Ctrl+C" in out
|
||||
|
||||
def test_cancel_running_synthetic_signals_and_terminates_process(self) -> None:
|
||||
session = Session()
|
||||
t = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
t.mark_running()
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = None
|
||||
t.attach_process(proc)
|
||||
console, buf = _capture()
|
||||
dispatch_slash(f"/cancel {t.task_id}", session, console)
|
||||
assert t.cancel_requested.is_set()
|
||||
proc.terminate.assert_called_once()
|
||||
out = buf.getvalue()
|
||||
assert "stop requested" in out.lower()
|
||||
assert t.task_id in out
|
||||
|
||||
|
||||
class _ImmediateThread:
|
||||
"""Run ``target`` synchronously inside ``start()`` (for deterministic tests)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: object = None,
|
||||
target: Callable[[], None] | None = None,
|
||||
name: object = None,
|
||||
args: tuple[object, ...] = (),
|
||||
kwargs: dict[str, object] | None = None,
|
||||
*,
|
||||
daemon: object = None,
|
||||
) -> None:
|
||||
del group, args, kwargs, daemon, name # threaded API baggage
|
||||
if target is None:
|
||||
raise TypeError("target required")
|
||||
self._target = target
|
||||
|
||||
def start(self) -> None:
|
||||
self._target()
|
||||
|
||||
|
||||
class _DeferredSyntheticThread:
|
||||
"""Queue ``target`` via ``start()``; tests invoke queued callables explicitly."""
|
||||
|
||||
pending: list[Callable[[], None]] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: object = None,
|
||||
target: Callable[[], None] | None = None,
|
||||
name: object = None,
|
||||
args: tuple[object, ...] = (),
|
||||
kwargs: dict[str, object] | None = None,
|
||||
*,
|
||||
daemon: object = None,
|
||||
) -> None:
|
||||
del group, args, kwargs, daemon, name
|
||||
if target is None:
|
||||
raise TypeError("target required")
|
||||
self._target = target
|
||||
|
||||
def start(self) -> None:
|
||||
_DeferredSyntheticThread.pending.append(self._target)
|
||||
|
||||
|
||||
def _synthetic_presenter(session: Session) -> MagicMock:
|
||||
presenter = MagicMock()
|
||||
presenter.session = session
|
||||
presenter.start_task_output_streams.return_value = []
|
||||
presenter.join_task_output_streams.return_value = None
|
||||
presenter.report_exception.return_value = None
|
||||
presenter.print_error.return_value = None
|
||||
return presenter
|
||||
|
||||
|
||||
class TestSyntheticSubprocessWatcher:
|
||||
def test_watch_marks_completed_when_process_already_done(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_ImmediateThread,
|
||||
)
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
task.attach_process(proc)
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
assert task.status == TaskStatus.COMPLETED
|
||||
hist = session.history[-1]
|
||||
assert hist["type"] == "synthetic_test"
|
||||
assert hist["ok"] is True
|
||||
assert "task:" in hist["text"]
|
||||
|
||||
def test_watch_honours_exit_code_when_cancel_races_loop_exit(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
"""Process exits naturally (code 0) in the same poll tick that /cancel fires.
|
||||
|
||||
The poll loop exits because proc.poll() returns non-None *before* the
|
||||
cancel_requested branch runs, so terminated_by_watcher stays False.
|
||||
The task must be COMPLETED, not CANCELLED — the process succeeded.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_ImmediateThread,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
proc = MagicMock()
|
||||
|
||||
# poll() returns None once (enter loop body), then cancel fires via
|
||||
# sleep, which also makes poll return 0 — loop exits via while condition.
|
||||
pending: list[int | None] = [None]
|
||||
|
||||
def _poll_side() -> int | None:
|
||||
return pending[0]
|
||||
|
||||
proc.poll.side_effect = _poll_side
|
||||
proc.returncode = 0
|
||||
|
||||
sleeps: list[float] = []
|
||||
|
||||
def _fake_sleep(_secs: float) -> None:
|
||||
sleeps.append(_secs)
|
||||
task.cancel_requested.set()
|
||||
pending[0] = 0 # process finishes naturally in the same window
|
||||
|
||||
monkeypatch.setattr("tools.interactive_shell.subprocess.time.sleep", _fake_sleep)
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
# terminated_by_watcher is False → honour exit code 0 → COMPLETED
|
||||
assert task.status == TaskStatus.COMPLETED
|
||||
assert sleeps
|
||||
|
||||
def test_watch_marks_cancelled_when_watcher_kills_process(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
"""cancel_requested is set while proc is still running; watcher terminates it."""
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_ImmediateThread,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
proc = MagicMock()
|
||||
|
||||
# poll() always returns None so the watcher's cancel branch runs and
|
||||
# calls _terminate_child_process; returncode is set after that.
|
||||
proc.poll.return_value = None
|
||||
proc.returncode = -15
|
||||
|
||||
task.cancel_requested.set() # cancel already set before first loop check
|
||||
|
||||
# Skip the sleep so the loop iterates immediately to the cancel branch.
|
||||
monkeypatch.setattr("tools.interactive_shell.subprocess.time.sleep", lambda _: None)
|
||||
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
assert task.status == TaskStatus.CANCELLED
|
||||
hist = session.history[-1]
|
||||
assert hist["type"] == "synthetic_test"
|
||||
assert hist["ok"] is False
|
||||
|
||||
def test_watch_honours_exit_code_when_cancel_races_natural_exit(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
"""Process exits naturally (code 0) while /cancel fires concurrently.
|
||||
|
||||
The watcher should mark the task COMPLETED, not CANCELLED, because we
|
||||
never called _terminate_child_process — the process was already gone.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_ImmediateThread,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
proc = MagicMock()
|
||||
# Process already finished; poll returns non-None immediately so the
|
||||
# while-loop body never executes — terminated_by_watcher stays False.
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
|
||||
# Simulate /cancel arriving just as the watcher reads poll()
|
||||
task.cancel_requested.set()
|
||||
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
assert task.status == TaskStatus.COMPLETED
|
||||
|
||||
def test_watch_captures_stderr_on_failure(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
"""Diagnostic stderr output is included in mark_failed message."""
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_ImmediateThread,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 1
|
||||
proc.returncode = 1
|
||||
|
||||
stderr_buf.write(b"ConnectionError: database unreachable\n")
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
assert task.status == TaskStatus.FAILED
|
||||
assert "exit code 1" in (task.error or "")
|
||||
assert "ConnectionError" in (task.error or "")
|
||||
assert (
|
||||
session.terminal.pending_prompt_default == SUGGESTED_PROMPT_AFTER_FAILED_SYNTHETIC_TEST
|
||||
)
|
||||
|
||||
def test_watch_skips_synthetic_history_after_reset(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
_DeferredSyntheticThread.pending.clear()
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_DeferredSyntheticThread,
|
||||
)
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
task.attach_process(proc)
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
assert len(_DeferredSyntheticThread.pending) == 1
|
||||
session.clear()
|
||||
_DeferredSyntheticThread.pending[0]()
|
||||
assert session.history == []
|
||||
_DeferredSyntheticThread.pending.clear()
|
||||
|
||||
def test_deferred_watcher_writes_history_when_no_reset(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stderr_buf: tempfile.SpooledTemporaryFile, # type: ignore[type-arg]
|
||||
) -> None:
|
||||
_DeferredSyntheticThread.pending.clear()
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.synthetic.runner.threading.Thread",
|
||||
_DeferredSyntheticThread,
|
||||
)
|
||||
|
||||
proc = MagicMock()
|
||||
proc.poll.return_value = 0
|
||||
proc.returncode = 0
|
||||
|
||||
session = Session()
|
||||
presenter = _synthetic_presenter(session)
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
task.attach_process(proc)
|
||||
watch_synthetic_subprocess(task, proc, presenter, "rds_postgres", stderr_buf)
|
||||
_DeferredSyntheticThread.pending[0]()
|
||||
assert session.history[-1]["type"] == "synthetic_test"
|
||||
_DeferredSyntheticThread.pending.clear()
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Tests for interactive-shell session token accounting."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from core.agent_harness.accounting.token_accounting import (
|
||||
build_llm_run_info,
|
||||
estimate_tokens,
|
||||
format_token_total,
|
||||
record_llm_turn,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.answer_turn import answer_shell_question
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.streaming import _CHARS_PER_TOKEN
|
||||
|
||||
|
||||
def test_estimate_tokens_uses_chars_per_token_ratio() -> None:
|
||||
text = "x" * (8 * _CHARS_PER_TOKEN)
|
||||
assert estimate_tokens(text) == 8
|
||||
|
||||
|
||||
def test_record_token_usage_accumulates_on_session() -> None:
|
||||
session = Session()
|
||||
record_llm_turn(session, prompt="a" * 40, response="b" * 20)
|
||||
assert session.tokens.totals == {
|
||||
"input": 10,
|
||||
"output": 5,
|
||||
"input_estimated": 10,
|
||||
"output_estimated": 5,
|
||||
}
|
||||
assert session.tokens.call_count == 1
|
||||
assert session.tokens.has_estimates is True
|
||||
record_llm_turn(session, prompt="c" * 8, response="d" * 4)
|
||||
assert session.tokens.totals == {
|
||||
"input": 12,
|
||||
"output": 6,
|
||||
"input_estimated": 12,
|
||||
"output_estimated": 6,
|
||||
}
|
||||
assert session.tokens.call_count == 2
|
||||
|
||||
|
||||
def test_record_llm_turn_uses_provider_counts_when_present() -> None:
|
||||
session = Session()
|
||||
inp, out, estimated = record_llm_turn(
|
||||
session,
|
||||
prompt="ignored when counts supplied",
|
||||
response="also ignored",
|
||||
input_tokens=1200,
|
||||
output_tokens=80,
|
||||
)
|
||||
assert (inp, out, estimated) == (1200, 80, False)
|
||||
assert session.tokens.totals == {
|
||||
"input": 1200,
|
||||
"output": 80,
|
||||
"input_measured": 1200,
|
||||
"output_measured": 80,
|
||||
}
|
||||
assert session.tokens.has_estimates is False
|
||||
|
||||
|
||||
def test_format_token_total_shows_mixed_breakdown() -> None:
|
||||
session = Session()
|
||||
record_llm_turn(
|
||||
session,
|
||||
prompt="x",
|
||||
response="y",
|
||||
input_tokens=300,
|
||||
output_tokens=40,
|
||||
)
|
||||
record_llm_turn(session, prompt="a" * 400, response="b" * 80)
|
||||
assert format_token_total(session, direction="input") == (
|
||||
"input tokens",
|
||||
"400 (300 provider + 100 est.)",
|
||||
)
|
||||
assert format_token_total(session, direction="output") == (
|
||||
"output tokens",
|
||||
"60 (40 provider + 20 est.)",
|
||||
)
|
||||
|
||||
|
||||
def test_build_llm_run_info_records_tokens_and_metadata() -> None:
|
||||
session = Session()
|
||||
|
||||
class _Client:
|
||||
_model = "claude-test"
|
||||
_provider_label = "Anthropic"
|
||||
|
||||
run = build_llm_run_info(
|
||||
session=session,
|
||||
prompt="a" * 40,
|
||||
response_text="b" * 20,
|
||||
client=_Client(),
|
||||
started=time.monotonic() - 0.05,
|
||||
)
|
||||
assert run.model == "claude-test"
|
||||
assert run.provider == "anthropic"
|
||||
assert run.input_tokens == 10
|
||||
assert run.output_tokens == 5
|
||||
assert run.response_text == "b" * 20
|
||||
assert run.latency_ms is not None and run.latency_ms >= 0
|
||||
|
||||
|
||||
def test_coerce_usage_tokens_accepts_float_counts() -> None:
|
||||
from core.llm.shared.usage import coerce_usage_tokens
|
||||
|
||||
assert coerce_usage_tokens(
|
||||
{"input_tokens": 512.0, "output_tokens": 64.0},
|
||||
input_key="input_tokens",
|
||||
output_key="output_tokens",
|
||||
) == (512, 64)
|
||||
|
||||
|
||||
def test_record_token_usage_skips_zero_counts() -> None:
|
||||
session = Session()
|
||||
session.tokens.record()
|
||||
assert session.tokens.totals == {}
|
||||
assert session.tokens.call_count == 0
|
||||
|
||||
|
||||
class _FakeLLMClient:
|
||||
def __init__(self, content: str) -> None:
|
||||
self._content = content
|
||||
self.last_prompt: str | None = None
|
||||
|
||||
def invoke_stream(self, prompt: str) -> Iterator[str]:
|
||||
self.last_prompt = prompt
|
||||
yield self._content
|
||||
|
||||
|
||||
def test_answer_shell_question_records_session_token_usage(monkeypatch: Any) -> None:
|
||||
client = _FakeLLMClient("assistant reply")
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: client)
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False)
|
||||
answer_shell_question("hello", session, console)
|
||||
assert session.tokens.totals["input"] > 0
|
||||
assert session.tokens.totals["output"] == estimate_tokens("assistant reply")
|
||||
assert session.tokens.has_estimates is True
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Tests for ShellTurnAccounting's pending turn LLM/error consumption."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.agent_harness.accounting.token_accounting import LlmRunInfo
|
||||
from core.agent_harness.turns.turn_results import ShellTurnResult, ToolCallingTurnResult
|
||||
from surfaces.interactive_shell.runtime.core.turn_accounting import ShellTurnAccounting
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
class _FakeRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.errors: list[tuple[str, str]] = []
|
||||
self.responses: list[tuple[str, Any]] = []
|
||||
self.flushed = 0
|
||||
|
||||
def set_error(self, kind: str, message: str) -> None:
|
||||
self.errors.append((kind, message))
|
||||
|
||||
def set_response(self, text: str, run: Any | None = None) -> None:
|
||||
self.responses.append((text, run))
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushed += 1
|
||||
|
||||
|
||||
def _result(*, llm_run: Any | None = None, text: str = "done") -> ShellTurnResult:
|
||||
return ShellTurnResult(
|
||||
final_intent="slash",
|
||||
action_result=ToolCallingTurnResult(
|
||||
planned_count=0,
|
||||
executed_count=0,
|
||||
executed_success_count=0,
|
||||
has_unhandled_clause=False,
|
||||
handled=True,
|
||||
),
|
||||
assistant_response_text=text,
|
||||
llm_run=llm_run,
|
||||
)
|
||||
|
||||
|
||||
def test_finalize_applies_pending_turn_llm_when_no_conversational_run() -> None:
|
||||
session = Session()
|
||||
pending = LlmRunInfo(model="claude-sonnet-4-5", provider="anthropic", input_tokens=100)
|
||||
session.terminal.set_pending_turn_llm(pending)
|
||||
recorder = _FakeRecorder()
|
||||
accounting = ShellTurnAccounting(session=session, text="/investigate", recorder=recorder) # type: ignore[arg-type]
|
||||
|
||||
accounting.finalize(_result())
|
||||
|
||||
assert recorder.responses == [("done", pending)]
|
||||
assert recorder.flushed == 1
|
||||
assert session.terminal.pop_pending_turn_llm() is None
|
||||
|
||||
|
||||
def test_finalize_prefers_conversational_run_over_pending() -> None:
|
||||
session = Session()
|
||||
session.terminal.set_pending_turn_llm(LlmRunInfo(model="stale"))
|
||||
conversational = LlmRunInfo(model="fresh")
|
||||
recorder = _FakeRecorder()
|
||||
accounting = ShellTurnAccounting(session=session, text="hi", recorder=recorder) # type: ignore[arg-type]
|
||||
|
||||
accounting.finalize(_result(llm_run=conversational))
|
||||
|
||||
assert recorder.responses[0][1] is conversational
|
||||
# The stale pending run was still consumed so it cannot leak later.
|
||||
assert session.terminal.pop_pending_turn_llm() is None
|
||||
|
||||
|
||||
def test_finalize_sets_structured_error_from_pending_turn_error() -> None:
|
||||
session = Session()
|
||||
session.terminal.set_pending_turn_error("config", "ANTHROPIC_API_KEY not set")
|
||||
recorder = _FakeRecorder()
|
||||
accounting = ShellTurnAccounting(session=session, text="/investigate", recorder=recorder) # type: ignore[arg-type]
|
||||
|
||||
accounting.finalize(_result(text="investigation_failed"))
|
||||
|
||||
assert recorder.errors == [("config", "ANTHROPIC_API_KEY not set")]
|
||||
assert session.terminal.pop_pending_turn_error() is None
|
||||
|
||||
|
||||
def test_finalize_consumes_pending_state_even_without_recorder() -> None:
|
||||
session = Session()
|
||||
session.terminal.set_pending_turn_llm(LlmRunInfo(model="m"))
|
||||
session.terminal.set_pending_turn_error("llm", "boom")
|
||||
accounting = ShellTurnAccounting(session=session, text="hi", recorder=None)
|
||||
|
||||
accounting.finalize(_result())
|
||||
|
||||
assert session.terminal.pop_pending_turn_llm() is None
|
||||
assert session.terminal.pop_pending_turn_error() is None
|
||||
|
||||
|
||||
def test_stage_investigation_turn_telemetry_populates_pending_state() -> None:
|
||||
from surfaces.interactive_shell.command_registry.investigation import (
|
||||
_stage_investigation_turn_telemetry,
|
||||
)
|
||||
from surfaces.interactive_shell.ui.investigation_outcome import InvestigationOutcome
|
||||
|
||||
session = Session()
|
||||
outcome = InvestigationOutcome(
|
||||
status="failed",
|
||||
target="generic",
|
||||
investigation_id="inv-1",
|
||||
error_message="Anthropic authentication failed. Check ANTHROPIC_API_KEY.",
|
||||
failure_category="llm",
|
||||
llm_model="claude-sonnet-4-5",
|
||||
llm_provider="anthropic",
|
||||
llm_input_tokens=500,
|
||||
llm_output_tokens=120,
|
||||
duration_ms=4200,
|
||||
)
|
||||
|
||||
_stage_investigation_turn_telemetry(session, outcome)
|
||||
|
||||
run = session.terminal.pop_pending_turn_llm()
|
||||
assert run is not None
|
||||
assert run.model == "claude-sonnet-4-5"
|
||||
assert run.provider == "anthropic"
|
||||
assert run.input_tokens == 500
|
||||
assert run.output_tokens == 120
|
||||
assert run.latency_ms == 4200
|
||||
assert session.terminal.pop_pending_turn_error() == (
|
||||
"llm",
|
||||
"Anthropic authentication failed. Check ANTHROPIC_API_KEY.",
|
||||
)
|
||||
# Provider-measured tokens also count toward the session totals.
|
||||
assert session.tokens.totals["input"] == 500
|
||||
assert session.tokens.totals["output"] == 120
|
||||
|
||||
|
||||
def test_stage_investigation_turn_telemetry_completed_run_has_no_error() -> None:
|
||||
from surfaces.interactive_shell.command_registry.investigation import (
|
||||
_stage_investigation_turn_telemetry,
|
||||
)
|
||||
from surfaces.interactive_shell.ui.investigation_outcome import InvestigationOutcome
|
||||
|
||||
session = Session()
|
||||
_stage_investigation_turn_telemetry(
|
||||
session,
|
||||
InvestigationOutcome(
|
||||
status="completed",
|
||||
target="generic",
|
||||
investigation_id="inv-2",
|
||||
llm_model="claude-sonnet-4-5",
|
||||
),
|
||||
)
|
||||
|
||||
assert session.terminal.pop_pending_turn_llm() is not None
|
||||
assert session.terminal.pop_pending_turn_error() is None
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Tests for the in-memory session storage backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.agent_harness.session import InMemorySessionStorage
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _session(storage: InMemorySessionStorage) -> Session:
|
||||
return Session(storage=storage)
|
||||
|
||||
|
||||
def test_open_then_record_appends_turn() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
session.record("chat", "hello world")
|
||||
|
||||
records = storage.read(session.session_id)
|
||||
assert records[0]["type"] == "session"
|
||||
assert records[0]["version"] == 2
|
||||
turns = [r for r in records if r["type"] == "custom_message"]
|
||||
assert turns[0]["custom_type"] == "turn_stub"
|
||||
assert turns[0]["kind"] == "chat"
|
||||
assert turns[0]["text"] == "hello world"
|
||||
|
||||
|
||||
def test_record_noop_when_not_opened() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
session.record("chat", "hi") # no open_session
|
||||
assert storage.read(session.session_id) == []
|
||||
|
||||
|
||||
def test_flush_writes_session_end_with_counts() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
session.record("chat", "q1")
|
||||
session.record("alert", "boom")
|
||||
storage.flush(session)
|
||||
|
||||
leaf = storage.read(session.session_id)[-1]
|
||||
assert leaf["type"] == "leaf"
|
||||
assert leaf["total_turns"] == 2
|
||||
|
||||
|
||||
def test_flush_deletes_empty_session() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
storage.flush(session)
|
||||
assert storage.read(session.session_id) == []
|
||||
|
||||
|
||||
def test_flush_is_idempotent() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
session.record("chat", "hi")
|
||||
storage.flush(session)
|
||||
storage.flush(session)
|
||||
leaves = [r for r in storage.read(session.session_id) if r["type"] == "leaf"]
|
||||
assert len(leaves) == 1
|
||||
|
||||
|
||||
def test_append_turn_detail_writes_message_entries() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
storage.append_turn_detail(session.session_id, "chat", "hello", response="hi")
|
||||
|
||||
records = storage.read(session.session_id)
|
||||
messages = [r for r in records if r["type"] == "message"]
|
||||
assert [(r["role"], r["content"]) for r in messages] == [("user", "hello"), ("assistant", "hi")]
|
||||
|
||||
|
||||
def test_append_tool_call_reopens_finalized_session() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
session.record("chat", "do a thing")
|
||||
storage.flush(session)
|
||||
storage.append_tool_call(session.session_id, tool="t", arguments={}, result="{}", ok=True)
|
||||
|
||||
records = storage.read(session.session_id)
|
||||
assert any(r["type"] == "tool_call" for r in records)
|
||||
assert any(r["type"] == "tool_result" for r in records)
|
||||
|
||||
|
||||
def test_append_investigation_result_returns_id() -> None:
|
||||
storage = InMemorySessionStorage()
|
||||
session = _session(storage)
|
||||
storage.open_session(session)
|
||||
inv_id = storage.append_investigation_result(
|
||||
session.session_id, {"root_cause": "leak", "problem_md": "report"}, trigger="t"
|
||||
)
|
||||
inv = next(r for r in storage.read(session.session_id) if r["type"] == "investigation_result")
|
||||
assert inv["investigation_id"] == inv_id
|
||||
assert inv["root_cause"] == "leak"
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Shell-session invariants: the ``Session`` subclass and its terminal/alerts facets.
|
||||
|
||||
Pins that the shell ``Session`` composes ``SessionCore`` plus the ``terminal`` and
|
||||
``alerts`` facets, that each relocated field cluster lives on its facet, and that the
|
||||
facet delegation (analytics staging, incoming-alert cap) behaves. Core-only invariants
|
||||
live in ``tests/core/agent_harness/session/test_session_characterization.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from core.agent_harness.session.persistence.memory import InMemorySessionStorage
|
||||
from core.agent_harness.session.session_core import SessionCore
|
||||
from core.domain.alerts.inbox import IncomingAlert
|
||||
from surfaces.interactive_shell.session.session import Session
|
||||
|
||||
# Core fields are inherited from SessionCore; the shell adds these two facets.
|
||||
_CORE_FIELD_COUNT = 19
|
||||
_FACET_FIELDS = ("alerts", "terminal")
|
||||
|
||||
|
||||
def _session() -> Session:
|
||||
return Session(storage=InMemorySessionStorage())
|
||||
|
||||
|
||||
def test_session_is_a_session_core_with_two_facets() -> None:
|
||||
assert issubclass(Session, SessionCore)
|
||||
field_names = {f.name for f in dataclasses.fields(Session)}
|
||||
assert "terminal" in field_names
|
||||
assert "alerts" in field_names
|
||||
assert len(field_names) == _CORE_FIELD_COUNT + len(_FACET_FIELDS)
|
||||
|
||||
|
||||
def test_alert_inbox_facet_holds_the_relocated_alert_state() -> None:
|
||||
inbox = _session().alerts
|
||||
assert hasattr(inbox, "entries") # was Session.incoming_alerts
|
||||
assert hasattr(inbox, "_max") # was Session._INCOMING_ALERTS_MAX
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_theme_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in ("active_theme_name", "pending_theme_refresh", "trust_mode"):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_prompt_toolkit_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in (
|
||||
"prompt_history_backend",
|
||||
"prompt_app",
|
||||
"main_loop",
|
||||
"prompt_refresh_fn",
|
||||
"fleet_sampler_starter",
|
||||
):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_pending_prompt_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in (
|
||||
"pending_prompt_default",
|
||||
"pending_prompt_autosubmit",
|
||||
"exclusive_stdin_active",
|
||||
"agent_turn_executed_slashes",
|
||||
):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_background_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in (
|
||||
"background_mode_enabled",
|
||||
"background_investigations",
|
||||
"background_notification_preferences",
|
||||
"background_notices",
|
||||
"_background_notices_lock",
|
||||
):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_metrics_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in ("metrics", "history_generation"):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_terminal_facet_holds_the_analytics_staging_cluster() -> None:
|
||||
terminal = _session().terminal
|
||||
for f in ("_turn_outcome_hint", "_pending_turn_llm", "_pending_turn_error"):
|
||||
assert hasattr(terminal, f)
|
||||
|
||||
|
||||
def test_analytics_staging_pop_methods_consume_exactly_once() -> None:
|
||||
terminal = _session().terminal
|
||||
terminal.set_turn_outcome_hint("handled")
|
||||
assert terminal.pop_turn_outcome_hint() == "handled"
|
||||
assert terminal.pop_turn_outcome_hint() is None
|
||||
terminal.set_turn_outcome_hint(" ")
|
||||
assert terminal.pop_turn_outcome_hint() is None
|
||||
|
||||
|
||||
def test_incoming_alerts_are_capped_and_drop_oldest_first() -> None:
|
||||
session = _session()
|
||||
cap = session.alerts._max
|
||||
for i in range(cap + 5):
|
||||
session.record_incoming_alert(IncomingAlert(text=f"alert-{i}"))
|
||||
assert len(session.alerts.entries) == cap
|
||||
assert session.alerts.entries[0].text == "alert-5"
|
||||
assert session.alerts.entries[-1].text == f"alert-{cap + 4}"
|
||||
@@ -0,0 +1,443 @@
|
||||
"""End-to-end /resume scenario tests: session identity, JSONL turns, slash recording."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from rich.console import Console
|
||||
|
||||
from config.config import (
|
||||
get_configured_llm_provider,
|
||||
get_llm_provider_api_key_env,
|
||||
resolve_llm_settings_verbose,
|
||||
)
|
||||
from config.llm_auth.credentials import status as credential_status
|
||||
from core.agent_harness.session import JsonlSessionStorage
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
SessionStore = JsonlSessionStorage()
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def _read_turns(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
turns: list[dict] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
rec = json.loads(line)
|
||||
if rec.get("type") == "custom_message" and rec.get("custom_type") == "turn_stub":
|
||||
turns.append({"type": "turn", "kind": rec.get("kind"), "text": rec.get("text")})
|
||||
return turns
|
||||
|
||||
|
||||
def _write_finalized_session(
|
||||
sessions_dir: Path,
|
||||
session_id: str,
|
||||
*,
|
||||
chat_text: str = "why is redis slow?",
|
||||
messages: list[tuple[str, str]] | None = None,
|
||||
context: dict[str, str] | None = None,
|
||||
) -> Path:
|
||||
"""Create a closed session file that /resume can load."""
|
||||
path = sessions_dir / f"{session_id}.jsonl"
|
||||
msgs = messages or [("user", chat_text), ("assistant", "check connection pool")]
|
||||
ctx = context or {"service": "redis"}
|
||||
ids = [f"entry{i}" for i in range(1, 6)]
|
||||
lines = [
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session",
|
||||
"version": 2,
|
||||
"id": session_id,
|
||||
"created_at": "2026-05-29T10:00:00+00:00",
|
||||
"cwd": "",
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": ids[0],
|
||||
"parent_id": None,
|
||||
"timestamp": "2026-05-29T10:00:01+00:00",
|
||||
"type": "custom_message",
|
||||
"custom_type": "turn_stub",
|
||||
"kind": "chat",
|
||||
"text": chat_text,
|
||||
"display": False,
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": ids[1],
|
||||
"parent_id": ids[0],
|
||||
"timestamp": "2026-05-29T10:00:02+00:00",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": msgs[0][1],
|
||||
"metadata": {"kind": "chat"},
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": ids[2],
|
||||
"parent_id": ids[1],
|
||||
"timestamp": "2026-05-29T10:00:03+00:00",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": msgs[1][1],
|
||||
"metadata": {"kind": "chat"},
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": ids[3],
|
||||
"parent_id": ids[2],
|
||||
"timestamp": "2026-05-29T10:00:04+00:00",
|
||||
"type": "custom_message",
|
||||
"custom_type": "accumulated_context",
|
||||
"content": ctx,
|
||||
"display": False,
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": ids[4],
|
||||
"parent_id": ids[3],
|
||||
"timestamp": "2026-05-29T10:00:05+00:00",
|
||||
"type": "leaf",
|
||||
"total_turns": 1,
|
||||
}
|
||||
),
|
||||
]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_sessions(tmp_path: Path) -> Path:
|
||||
"""Sessions directory with SessionStore patched for the test."""
|
||||
directory = tmp_path / "sessions"
|
||||
directory.mkdir()
|
||||
with patch("config.constants.OPENSRE_HOME_DIR", tmp_path):
|
||||
yield directory
|
||||
|
||||
|
||||
def _open_current(session: Session) -> None:
|
||||
SessionStore.open_session(session)
|
||||
|
||||
|
||||
def _require_live_llm_for_repl_planner() -> None:
|
||||
explicit_pin = os.environ.get("LLM_PROVIDER", "").strip().lower()
|
||||
resolution = None
|
||||
try:
|
||||
resolution = resolve_llm_settings_verbose()
|
||||
except ValidationError as exc:
|
||||
provider = get_configured_llm_provider()
|
||||
env_var = get_llm_provider_api_key_env(provider)
|
||||
msg = exc.errors()[0].get("msg", str(exc)) if exc.errors() else str(exc)
|
||||
hint = f" configured provider={provider!r}"
|
||||
if env_var is not None:
|
||||
hint += f", required key={env_var}"
|
||||
pytest.skip(f"Skipping live REPL planner smoke; missing LLM configuration:{hint}. {msg}")
|
||||
|
||||
if resolution is None:
|
||||
pytest.skip("Skipping live REPL planner smoke; LLM configuration was not resolved.")
|
||||
|
||||
if explicit_pin and resolution.resolved_provider != explicit_pin:
|
||||
pytest.skip(
|
||||
f"Skipping live REPL planner smoke; LLM_PROVIDER={explicit_pin!r} resolved as "
|
||||
f"{resolution.resolved_provider!r}."
|
||||
)
|
||||
|
||||
auth = credential_status(resolution.resolved_provider)
|
||||
if not auth.configured or auth.stale:
|
||||
pytest.skip(
|
||||
"Skipping live REPL planner smoke; missing LLM credentials:"
|
||||
f" provider={resolution.resolved_provider!r}, auth={auth.source}, detail={auth.detail}"
|
||||
)
|
||||
|
||||
env_var = get_llm_provider_api_key_env(resolution.resolved_provider)
|
||||
if env_var is not None and not os.environ.get(env_var):
|
||||
pytest.skip(
|
||||
f"Skipping live REPL planner smoke; {env_var} is not exported to the child REPL."
|
||||
)
|
||||
|
||||
|
||||
class TestResumeScenarioMatrix:
|
||||
"""Scenario coverage for /resume session adoption and JSONL persistence."""
|
||||
|
||||
def test_scenario_fresh_repl_resumes_prior_session(self, isolated_sessions: Path) -> None:
|
||||
"""Fresh REPL session resumes a prior session: adopt ID, slash on target only."""
|
||||
target_id = "aaaa1111-2222-3333-4444-555566667777"
|
||||
_write_finalized_session(isolated_sessions, target_id, chat_text="why is redis slow?")
|
||||
|
||||
session = Session()
|
||||
current_id = session.session_id
|
||||
_open_current(session)
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/resume {target_id[:8]}", session, console) is True
|
||||
|
||||
assert session.session_id == target_id
|
||||
assert "resumed session" in buf.getvalue()
|
||||
|
||||
current_path = isolated_sessions / f"{current_id}.jsonl"
|
||||
if current_path.exists():
|
||||
assert not any(
|
||||
t.get("kind") == "slash" and "/resume" in t.get("text", "")
|
||||
for t in _read_turns(current_path)
|
||||
)
|
||||
|
||||
target_turns = _read_turns(isolated_sessions / f"{target_id}.jsonl")
|
||||
assert any(
|
||||
t.get("kind") == "slash" and t.get("text", "").startswith("/resume")
|
||||
for t in target_turns
|
||||
)
|
||||
assert session.agent.messages[0] == ("user", "why is redis slow?")
|
||||
assert session.accumulated_context == {"service": "redis"}
|
||||
|
||||
def test_scenario_post_resume_slash_and_chat_append_to_target(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
"""After /resume, further slash commands and chat turns land on the same file."""
|
||||
target_id = "bbbb2222-3333-4444-5555-666677778888"
|
||||
_write_finalized_session(isolated_sessions, target_id)
|
||||
|
||||
session = Session()
|
||||
_open_current(session)
|
||||
console, _ = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {target_id[:8]}", session, console)
|
||||
dispatch_slash("/status", session, console)
|
||||
session.record("chat", "follow-up question after resume")
|
||||
|
||||
target_path = isolated_sessions / f"{target_id}.jsonl"
|
||||
kinds = [t["kind"] for t in _read_turns(target_path)]
|
||||
assert "slash" in kinds
|
||||
assert kinds.count("slash") >= 2
|
||||
assert "chat" in kinds
|
||||
assert "leaf" in target_path.read_text(encoding="utf-8")
|
||||
|
||||
def test_scenario_empty_starter_session_removed_on_resume(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
"""A brand-new REPL with no turns should not leave a junk file after /resume."""
|
||||
target_id = "cccc3333-4444-5555-6666-777788889999"
|
||||
_write_finalized_session(isolated_sessions, target_id)
|
||||
|
||||
session = Session()
|
||||
starter_id = session.session_id
|
||||
_open_current(session)
|
||||
console, _ = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {target_id[:8]}", session, console)
|
||||
|
||||
starter_path = isolated_sessions / f"{starter_id}.jsonl"
|
||||
assert not starter_path.exists()
|
||||
assert session.session_id == target_id
|
||||
|
||||
def test_scenario_resume_by_name_substring(self, isolated_sessions: Path) -> None:
|
||||
target_id = "dddd4444-5555-6666-7777-888899990000"
|
||||
_write_finalized_session(isolated_sessions, target_id, chat_text="investigate OOM killer")
|
||||
|
||||
session = Session()
|
||||
_open_current(session)
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/resume OOM", session, console)
|
||||
|
||||
assert session.session_id == target_id
|
||||
assert "resumed session" in buf.getvalue()
|
||||
target_turns = _read_turns(isolated_sessions / f"{target_id}.jsonl")
|
||||
assert any(t.get("text") == "/resume OOM" for t in target_turns)
|
||||
|
||||
def test_resume_session_by_prefix_matches_name_substring(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
from surfaces.interactive_shell.command_registry.session_cmds.resume import (
|
||||
resume_session_by_prefix,
|
||||
)
|
||||
|
||||
target_id = "eeee5555-6666-7777-8888-999900001111"
|
||||
_write_finalized_session(isolated_sessions, target_id, chat_text="investigate OOM killer")
|
||||
|
||||
session = Session()
|
||||
_open_current(session)
|
||||
console, buf = _capture()
|
||||
|
||||
assert resume_session_by_prefix("OOM", session, console)
|
||||
assert session.session_id == target_id
|
||||
assert "resumed session" in buf.getvalue()
|
||||
|
||||
def test_scenario_resume_not_found_records_on_current(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
session = Session()
|
||||
current_id = session.session_id
|
||||
_open_current(session)
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash("/resume deadbeef", session, console)
|
||||
|
||||
assert session.session_id == current_id
|
||||
assert "not found" in buf.getvalue()
|
||||
turns = _read_turns(isolated_sessions / f"{current_id}.jsonl")
|
||||
assert turns[-1]["kind"] == "slash"
|
||||
assert turns[-1]["text"] == "/resume deadbeef"
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_scenario_resume_current_session_guard(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
session = Session()
|
||||
_open_current(session)
|
||||
session.record("chat", "still working here")
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {session.session_id[:8]}", session, console)
|
||||
|
||||
assert "current session" in buf.getvalue()
|
||||
assert session.session_id # unchanged
|
||||
turns = _read_turns(isolated_sessions / f"{session.session_id}.jsonl")
|
||||
assert any(t.get("text", "").startswith("/resume") for t in turns)
|
||||
|
||||
def test_scenario_resume_empty_target_does_not_switch(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
empty_id = "eeee5555-6666-7777-8888-999900001111"
|
||||
path = isolated_sessions / f"{empty_id}.jsonl"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session",
|
||||
"version": 2,
|
||||
"id": empty_id,
|
||||
"created_at": "2026-05-29T10:00:00+00:00",
|
||||
"cwd": "",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
session = Session()
|
||||
current_id = session.session_id
|
||||
_open_current(session)
|
||||
console, buf = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {empty_id[:8]}", session, console)
|
||||
|
||||
assert session.session_id == current_id
|
||||
assert "no conversation to resume" in buf.getvalue()
|
||||
|
||||
def test_scenario_chain_resume_two_targets(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
"""Resume session A, then resume session B — each gets its own /resume slash turn."""
|
||||
id_a = "ffff6666-7777-8888-9999-000011112222"
|
||||
id_b = "11117777-8888-9999-0000-111122223333"
|
||||
_write_finalized_session(isolated_sessions, id_a, chat_text="session A question")
|
||||
_write_finalized_session(isolated_sessions, id_b, chat_text="session B question")
|
||||
|
||||
session = Session()
|
||||
_open_current(session)
|
||||
console, _ = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {id_a[:8]}", session, console)
|
||||
assert session.session_id == id_a
|
||||
|
||||
dispatch_slash(f"/resume {id_b[:8]}", session, console)
|
||||
assert session.session_id == id_b
|
||||
|
||||
turns_a = _read_turns(isolated_sessions / f"{id_a}.jsonl")
|
||||
turns_b = _read_turns(isolated_sessions / f"{id_b}.jsonl")
|
||||
assert any("/resume" in t.get("text", "") for t in turns_a)
|
||||
assert any("/resume" in t.get("text", "") for t in turns_b)
|
||||
assert session.agent.messages[0] == ("user", "session B question")
|
||||
|
||||
def test_scenario_active_session_with_turns_flushed_without_resume_slash(
|
||||
self,
|
||||
isolated_sessions: Path,
|
||||
) -> None:
|
||||
"""Switching away from a session that had real turns preserves them without /resume."""
|
||||
target_id = "22228888-9999-0000-1111-222233334444"
|
||||
_write_finalized_session(isolated_sessions, target_id)
|
||||
|
||||
session = Session()
|
||||
current_id = session.session_id
|
||||
_open_current(session)
|
||||
session.record("chat", "work in progress")
|
||||
console, _ = _capture()
|
||||
|
||||
dispatch_slash(f"/resume {target_id[:8]}", session, console)
|
||||
|
||||
old_turns = _read_turns(isolated_sessions / f"{current_id}.jsonl")
|
||||
assert any(t["kind"] == "chat" for t in old_turns)
|
||||
assert not any(t.get("kind") == "slash" for t in old_turns)
|
||||
assert (isolated_sessions / f"{current_id}.jsonl").read_text(encoding="utf-8").find(
|
||||
'"type": "leaf"'
|
||||
) != -1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.live_llm
|
||||
class TestResumeLiveRepl:
|
||||
"""Live REPL smoke test via ReplDriver with isolated HOME and real planner."""
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Flaky live-LLM REPL round-trip (subprocess + 60s waits); resume load "
|
||||
"logic is covered offline. Run manually via ReplDriver."
|
||||
)
|
||||
def test_live_resume_round_trip(self, tmp_path: Path) -> None:
|
||||
_require_live_llm_for_repl_planner()
|
||||
|
||||
from tests.utils.repl_driver import ReplDriver
|
||||
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
sessions_dir = home / ".opensre" / "sessions"
|
||||
sessions_dir.mkdir(parents=True)
|
||||
target_id = "live9999-aaaa-bbbb-cccc-ddddeeeeffff"
|
||||
_write_finalized_session(sessions_dir, target_id, chat_text="live redis investigation")
|
||||
|
||||
with ReplDriver(home=home, startup_wait=10.0) as repl:
|
||||
if repl.contains("Press Enter to continue"):
|
||||
repl.send("", wait=3.0)
|
||||
repl.reset_output()
|
||||
|
||||
repl.send("/sessions", wait=1.0)
|
||||
assert repl.wait_until_contains("live9999", "live redis", timeout=60.0)
|
||||
|
||||
repl.reset_output()
|
||||
repl.send(f"/resume {target_id[:8]}", wait=1.0)
|
||||
assert repl.wait_until_contains("resumed session", timeout=60.0)
|
||||
assert repl.wait_until_contains("live redis investigation", timeout=10.0)
|
||||
|
||||
repl.reset_output()
|
||||
repl.send("/status", wait=1.0)
|
||||
assert repl.wait_until_contains("interactions", timeout=60.0)
|
||||
|
||||
target_path = sessions_dir / f"{target_id}.jsonl"
|
||||
assert target_path.exists()
|
||||
turns = _read_turns(target_path)
|
||||
assert any(t.get("kind") == "slash" and "/resume" in t.get("text", "") for t in turns)
|
||||
assert any(t.get("kind") == "slash" and "/status" in t.get("text", "") for t in turns)
|
||||
@@ -0,0 +1,966 @@
|
||||
"""Tests for SessionStore: incremental writes (open_session, append_turn, flush, load_recent)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.session import (
|
||||
JsonlSessionRepo,
|
||||
JsonlSessionStorage,
|
||||
)
|
||||
from core.agent_harness.session.persistence.paths import sessions_dir as _sessions_dir
|
||||
from surfaces.interactive_shell.session import (
|
||||
Session,
|
||||
)
|
||||
|
||||
|
||||
class _SessionStoreFacade(JsonlSessionStorage, JsonlSessionRepo):
|
||||
"""Test facade exposing both the storage and repo APIs on one object."""
|
||||
|
||||
|
||||
SessionStore = _SessionStoreFacade()
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_session() -> Session:
|
||||
return Session()
|
||||
|
||||
|
||||
def _read_lines(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _turn_stubs(records: list[dict]) -> list[dict]:
|
||||
return [
|
||||
record
|
||||
for record in records
|
||||
if record.get("type") == "custom_message" and record.get("custom_type") == "turn_stub"
|
||||
]
|
||||
|
||||
|
||||
def _write_v2_session(path: Path, sid: str, *, started_at: str, text: str = "hi") -> None:
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session",
|
||||
"version": 2,
|
||||
"id": sid,
|
||||
"created_at": started_at,
|
||||
"cwd": "",
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": "entry1",
|
||||
"parent_id": None,
|
||||
"timestamp": started_at,
|
||||
"type": "custom_message",
|
||||
"custom_type": "turn_stub",
|
||||
"kind": "chat",
|
||||
"text": text,
|
||||
"display": False,
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def _patch_dir(tmp_path: Path):
|
||||
return patch("core.agent_harness.session.persistence.paths.sessions_dir", return_value=tmp_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""Redirect OpenSRE's home to a real temp dir so SessionStore reads/writes real files.
|
||||
|
||||
No mocking: ``_sessions_dir()`` resolves the real ``OPENSRE_HOME_DIR`` constant
|
||||
on every call, so pointing it at a temp directory exercises the genuine
|
||||
filesystem path end to end.
|
||||
"""
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
return tmp_path
|
||||
|
||||
|
||||
# ── open_session ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_open_session_creates_file_with_session_start(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
|
||||
files = list(tmp_path.glob("*.jsonl"))
|
||||
assert len(files) == 1
|
||||
records = _read_lines(files[0])
|
||||
assert records[0]["type"] == "session"
|
||||
assert records[0]["version"] == 2
|
||||
assert records[0]["id"] == session.session_id
|
||||
|
||||
|
||||
def test_open_session_uses_session_id_as_filename(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
assert (tmp_path / f"{session.session_id}.jsonl").exists()
|
||||
|
||||
|
||||
def test_open_session_never_raises_on_bad_path() -> None:
|
||||
session = _make_session()
|
||||
with patch(
|
||||
"core.agent_harness.session.persistence.paths.sessions_dir",
|
||||
return_value=Path("/nonexistent/cannot/write"),
|
||||
):
|
||||
SessionStore.open_session(session) # must not raise
|
||||
|
||||
|
||||
# ── append_turn ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_append_turn_adds_record_to_existing_file(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hello world")
|
||||
SessionStore.append_turn(session, "alert", "HighCPU on prod")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
turns = _turn_stubs(records)
|
||||
assert len(turns) == 2
|
||||
assert turns[0]["kind"] == "chat"
|
||||
assert turns[0]["text"] == "hello world"
|
||||
assert turns[1]["kind"] == "alert"
|
||||
assert turns[1]["text"] == "HighCPU on prod"
|
||||
|
||||
|
||||
def test_append_turn_stores_full_text_without_truncation(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
long_text = "x" * 500
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", long_text)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
turn = next(r for r in records if r["type"] == "custom_message")
|
||||
assert len(turn["text"]) == 500
|
||||
|
||||
|
||||
def test_append_turn_noop_when_file_missing(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
# Do NOT call open_session — file doesn't exist
|
||||
SessionStore.append_turn(session, "chat", "hello")
|
||||
assert not list(tmp_path.glob("*.jsonl")), "no file should be created"
|
||||
|
||||
|
||||
# ── append_turn_detail ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_append_turn_detail_writes_full_prompt_and_response(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn_detail(
|
||||
session.session_id,
|
||||
"chat",
|
||||
"how do I debug high CPU?",
|
||||
response="Root cause is a memory leak.",
|
||||
turn_id="abc-123",
|
||||
model="claude-3-5",
|
||||
provider="anthropic",
|
||||
latency_ms=1500,
|
||||
)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
messages = [r for r in records if r["type"] == "message"]
|
||||
assert [(m["role"], m["content"]) for m in messages] == [
|
||||
("user", "how do I debug high CPU?"),
|
||||
("assistant", "Root cause is a memory leak."),
|
||||
]
|
||||
assert messages[0]["metadata"]["kind"] == "chat"
|
||||
assert messages[0]["metadata"]["turn_id"] == "abc-123"
|
||||
assert messages[0]["metadata"]["model"] == "claude-3-5"
|
||||
assert messages[0]["metadata"]["provider"] == "anthropic"
|
||||
assert messages[0]["metadata"]["latency_ms"] == 1500
|
||||
|
||||
|
||||
def test_append_turn_detail_stores_system_prompt_metadata(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn_detail(
|
||||
session.session_id,
|
||||
"chat",
|
||||
"question",
|
||||
response="answer",
|
||||
system_prompt="assistant system block",
|
||||
)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
messages = [r for r in records if r["type"] == "message"]
|
||||
assert messages[0]["metadata"]["system_prompt"] == "assistant system block"
|
||||
|
||||
|
||||
def test_append_turn_detail_omits_none_fields(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn_detail(session.session_id, "chat", "hi")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
detail = next(r for r in records if r["type"] == "message")
|
||||
assert detail["role"] == "user"
|
||||
assert detail["content"] == "hi"
|
||||
assert "turn_id" not in detail["metadata"]
|
||||
assert "model" not in detail["metadata"]
|
||||
|
||||
|
||||
def test_append_turn_detail_noop_when_file_missing(tmp_path: Path) -> None:
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.append_turn_detail("nonexistent-id", "chat", "hi")
|
||||
assert not list(tmp_path.glob("*.jsonl"))
|
||||
|
||||
|
||||
# ── append_tool_call ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_append_tool_call_writes_record(tmp_home: Path) -> None:
|
||||
session = _make_session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_tool_call(
|
||||
session.session_id,
|
||||
tool="call_posthog_tool",
|
||||
arguments={"tool": "execute-sql", "args": {"query": "select 1"}},
|
||||
result='{"rows": []}',
|
||||
ok=True,
|
||||
source="posthog_mcp",
|
||||
)
|
||||
|
||||
records = _read_lines(_sessions_dir() / f"{session.session_id}.jsonl")
|
||||
call = next(r for r in records if r["type"] == "tool_call")
|
||||
result = next(r for r in records if r["type"] == "tool_result")
|
||||
assert call["tool"] == "call_posthog_tool"
|
||||
assert call["arguments"] == {"tool": "execute-sql", "args": {"query": "select 1"}}
|
||||
assert result["content"] == '{"rows": []}'
|
||||
assert result["ok"] is True
|
||||
assert call["source"] == "posthog_mcp"
|
||||
assert "timestamp" in call
|
||||
|
||||
|
||||
def test_append_tool_call_omits_source_when_none(tmp_home: Path) -> None:
|
||||
session = _make_session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_tool_call(
|
||||
session.session_id,
|
||||
tool="list_posthog_tools",
|
||||
arguments={},
|
||||
result="error: boom",
|
||||
ok=False,
|
||||
)
|
||||
|
||||
records = _read_lines(_sessions_dir() / f"{session.session_id}.jsonl")
|
||||
result = next(r for r in records if r["type"] == "tool_result")
|
||||
assert result["ok"] is False
|
||||
call = next(r for r in records if r["type"] == "tool_call")
|
||||
assert "source" not in call
|
||||
|
||||
|
||||
def test_append_tool_call_noop_when_file_missing(tmp_home: Path) -> None:
|
||||
SessionStore.append_tool_call("nonexistent-id", tool="t", arguments={}, result="r", ok=True)
|
||||
assert not list(_sessions_dir().glob("*.jsonl"))
|
||||
|
||||
|
||||
def test_append_tool_call_reopens_finalized_session(tmp_home: Path) -> None:
|
||||
session = _make_session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "cli_agent", "events for davincios in posthog")
|
||||
SessionStore.flush(session)
|
||||
# A late tool-call write (e.g. background gather) must reopen the file.
|
||||
SessionStore.append_tool_call(
|
||||
session.session_id, tool="call_posthog_tool", arguments={}, result="{}", ok=True
|
||||
)
|
||||
|
||||
records = _read_lines(_sessions_dir() / f"{session.session_id}.jsonl")
|
||||
assert any(r["type"] == "tool_call" for r in records)
|
||||
|
||||
|
||||
# ── flush ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_flush_writes_session_end(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "q1")
|
||||
SessionStore.append_turn(session, "alert", "alert1")
|
||||
SessionStore.flush(session)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
leaf = records[-1]
|
||||
assert leaf["type"] == "leaf"
|
||||
assert leaf["total_turns"] == 2
|
||||
assert leaf["chat_turns"] == 1
|
||||
assert leaf["investigation_turns"] == 1
|
||||
|
||||
|
||||
def test_flush_counts_cli_agent_turns_as_chat(tmp_path: Path) -> None:
|
||||
"""Chat turns recorded as kind='cli_agent' must count as chat_turns."""
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "cli_agent", "why is redis slow?")
|
||||
SessionStore.append_turn(session, "chat", "how do I use /resume?")
|
||||
SessionStore.append_turn(session, "follow_up", "what else?")
|
||||
SessionStore.flush(session)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
leaf = records[-1]
|
||||
assert leaf["chat_turns"] == 3
|
||||
assert leaf["investigation_turns"] == 0
|
||||
|
||||
|
||||
def test_flush_writes_conversation_snapshot_when_messages_present(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
session.agent.messages = [("user", "hello"), ("assistant", "hi there")]
|
||||
session.accumulated_context = {"service": "api", "cluster": "prod"}
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hello")
|
||||
SessionStore.flush(session)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
messages = [r for r in records if r["type"] == "message"]
|
||||
context = next(
|
||||
r
|
||||
for r in records
|
||||
if r.get("type") == "custom_message" and r.get("custom_type") == "accumulated_context"
|
||||
)
|
||||
assert [(m["role"], m["content"]) for m in messages] == [
|
||||
("user", "hello"),
|
||||
("assistant", "hi there"),
|
||||
]
|
||||
assert context["content"] == {"service": "api", "cluster": "prod"}
|
||||
# persisted branch entries must come before leaf
|
||||
types = [r["type"] for r in records]
|
||||
assert types.index("message") < types.index("leaf")
|
||||
|
||||
|
||||
def test_flush_skips_snapshot_when_no_messages(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
assert not any(r["type"] == "message" for r in records)
|
||||
|
||||
|
||||
def test_flush_deletes_file_when_no_turns(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.flush(session)
|
||||
|
||||
assert not (tmp_path / f"{session.session_id}.jsonl").exists()
|
||||
|
||||
|
||||
def test_flush_keeps_file_when_only_turn_details(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
# turn_detail only (no stub) — should NOT delete the file
|
||||
SessionStore.append_turn_detail(session.session_id, "chat", "hello", response="hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
assert (tmp_path / f"{session.session_id}.jsonl").exists()
|
||||
|
||||
|
||||
def test_flush_noop_when_file_missing(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.flush(session) # no open_session called — must not raise
|
||||
|
||||
|
||||
def test_flush_is_idempotent(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
SessionStore.flush(session) # second call must not append another session_end
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
leaf_records = [r for r in records if r["type"] == "leaf"]
|
||||
assert len(leaf_records) == 1, "flush() must be idempotent — only one leaf"
|
||||
|
||||
|
||||
def test_append_turn_reopens_finalized_session(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "hello")
|
||||
SessionStore.flush(session)
|
||||
session.record("slash", "/status")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
types = [r["type"] for r in records]
|
||||
assert types.count("leaf") == 1
|
||||
assert records[-1]["type"] == "custom_message"
|
||||
assert records[-1]["kind"] == "slash"
|
||||
assert records[-1]["text"] == "/status"
|
||||
|
||||
|
||||
def test_reopen_session_strips_trailing_end_and_snapshot(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "hello")
|
||||
SessionStore.flush(session)
|
||||
SessionStore.reopen_session(session.session_id)
|
||||
session.record("chat", "continued")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
types = [r["type"] for r in records]
|
||||
assert types.count("leaf") == 1
|
||||
assert "conversation_snapshot" not in types
|
||||
assert types[-1] == "custom_message"
|
||||
assert records[-1]["text"] == "continued"
|
||||
|
||||
|
||||
def test_reopen_session_noop_for_open_session(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "hello")
|
||||
SessionStore.reopen_session(session.session_id)
|
||||
session.record("chat", "still open")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
assert records[-1]["text"] == "still open"
|
||||
assert all(r["type"] != "leaf" for r in records)
|
||||
|
||||
|
||||
# ── session.record() wiring ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_session_record_calls_append_turn(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "what's wrong with prod?")
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
turns = _turn_stubs(records)
|
||||
assert len(turns) == 1
|
||||
assert turns[0]["kind"] == "chat"
|
||||
assert turns[0]["text"] == "what's wrong with prod?"
|
||||
|
||||
|
||||
# ── load_recent ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_load_recent_returns_empty_when_no_dir(tmp_path: Path) -> None:
|
||||
with _patch_dir(tmp_path / "missing"):
|
||||
result = SessionStore.load_recent()
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_load_recent_counts_turns_for_in_progress_session(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "cli_agent", "hi")
|
||||
SessionStore.append_turn(session, "chat", "follow-up")
|
||||
SessionStore.append_turn(session, "alert", "OOM")
|
||||
# No flush — session still in progress
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["total_turns"] == 3
|
||||
assert results[0]["chat_turns"] == 2
|
||||
assert results[0]["investigation_turns"] == 1
|
||||
assert results[0]["duration_secs"] is None
|
||||
assert results[0]["is_ended"] is False
|
||||
|
||||
|
||||
def test_load_recent_uses_session_end_stats_when_available(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["is_ended"] is True
|
||||
assert results[0]["total_turns"] == 1
|
||||
assert results[0]["duration_secs"] is not None
|
||||
|
||||
|
||||
def test_load_recent_reports_has_snapshot_true(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
session.agent.messages = [("user", "hi"), ("assistant", "hello")]
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["has_snapshot"] is True
|
||||
|
||||
|
||||
def test_load_recent_reports_has_snapshot_false_without_conversation(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["has_snapshot"] is False
|
||||
|
||||
|
||||
def test_load_recent_returns_newest_first(tmp_path: Path) -> None:
|
||||
for started in ["2024-01-01T10:00:00+00:00", "2024-01-02T10:00:00+00:00"]:
|
||||
sid = str(uuid.uuid4())
|
||||
_write_v2_session(tmp_path / f"{sid}.jsonl", sid, started_at=started)
|
||||
|
||||
with _patch_dir(tmp_path):
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["started_at"] > results[1]["started_at"]
|
||||
|
||||
|
||||
def test_load_recent_skips_malformed_files(tmp_path: Path) -> None:
|
||||
(tmp_path / "bad.jsonl").write_text("not json\n")
|
||||
(tmp_path / "empty.jsonl").write_text("")
|
||||
|
||||
sid = str(uuid.uuid4())
|
||||
_write_v2_session(
|
||||
tmp_path / f"{sid}.jsonl",
|
||||
sid,
|
||||
started_at="2024-01-01T10:00:00+00:00",
|
||||
text="ok",
|
||||
)
|
||||
|
||||
with _patch_dir(tmp_path):
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["session_id"] == sid
|
||||
|
||||
|
||||
def test_load_recent_respects_n_limit(tmp_path: Path) -> None:
|
||||
for _ in range(5):
|
||||
sid = str(uuid.uuid4())
|
||||
_write_v2_session(
|
||||
tmp_path / f"{sid}.jsonl",
|
||||
sid,
|
||||
started_at="2024-01-01T10:00:00+00:00",
|
||||
)
|
||||
|
||||
with _patch_dir(tmp_path):
|
||||
assert len(SessionStore.load_recent(n=3)) == 3
|
||||
|
||||
|
||||
# ── load_session ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_load_session_returns_none_for_missing_prefix(tmp_path: Path) -> None:
|
||||
with _patch_dir(tmp_path):
|
||||
assert SessionStore.load_session("nonexistent") is None
|
||||
|
||||
|
||||
def test_load_session_returns_none_when_no_dir(tmp_path: Path) -> None:
|
||||
with _patch_dir(tmp_path / "missing"):
|
||||
assert SessionStore.load_session("abc") is None
|
||||
|
||||
|
||||
def test_load_session_restores_from_conversation_snapshot(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
session.agent.messages = [("user", "how is prod?"), ("assistant", "prod is healthy")]
|
||||
session.accumulated_context = {"service": "api"}
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "how is prod?")
|
||||
SessionStore.flush(session)
|
||||
|
||||
data = SessionStore.load_session(session.session_id[:8])
|
||||
|
||||
assert data is not None
|
||||
assert data["has_snapshot"] is False
|
||||
assert data["cli_agent_messages"] == [
|
||||
("user", "how is prod?"),
|
||||
("assistant", "prod is healthy"),
|
||||
]
|
||||
assert data["accumulated_context"] == {"service": "api"}
|
||||
|
||||
|
||||
def test_load_session_fallback_to_turn_details_when_no_snapshot(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn_detail(
|
||||
session.session_id, "chat", "debug high CPU", response="It's a leak"
|
||||
)
|
||||
# Flush without cli_agent_messages — no snapshot written
|
||||
SessionStore.flush(session)
|
||||
|
||||
data = SessionStore.load_session(session.session_id[:8])
|
||||
|
||||
assert data is not None
|
||||
assert data["has_snapshot"] is False
|
||||
messages = data["cli_agent_messages"]
|
||||
assert ("user", "debug high CPU") in messages
|
||||
assert ("assistant", "It's a leak") in messages
|
||||
|
||||
|
||||
def test_load_session_ambiguous_prefix_returns_none(tmp_path: Path) -> None:
|
||||
# Two sessions sharing the same prefix
|
||||
for _ in range(2):
|
||||
sid = "aaaabbbb" + str(uuid.uuid4())[8:]
|
||||
_write_v2_session(
|
||||
tmp_path / f"{sid}.jsonl",
|
||||
sid,
|
||||
started_at="2024-01-01T10:00:00+00:00",
|
||||
)
|
||||
with _patch_dir(tmp_path):
|
||||
result = SessionStore.load_session("aaaa")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_load_session_includes_history_and_turn_details(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hello")
|
||||
SessionStore.append_turn_detail(session.session_id, "chat", "hello", response="hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
data = SessionStore.load_session(session.session_id)
|
||||
|
||||
assert data is not None
|
||||
assert len(data["history"]) == 1
|
||||
assert data["history"][0]["kind"] == "chat"
|
||||
assert len(data["turn_details"]) == 1
|
||||
assert data["turn_details"][0]["response"] == "hi"
|
||||
|
||||
|
||||
# ── session name derivation ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_load_recent_derives_name_from_turn_detail(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "why is redis slow?")
|
||||
SessionStore.append_turn_detail(
|
||||
session.session_id, "chat", "why is redis slow?", response="It's a memory issue"
|
||||
)
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["name"] == "why is redis slow?"
|
||||
|
||||
|
||||
def test_load_recent_derives_name_from_cli_agent_turn(tmp_path: Path) -> None:
|
||||
"""Real chat turns recorded as kind='cli_agent' are reconstructed."""
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "cli_agent", "debug the OOM killer on prod")
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["name"] == "debug the OOM killer on prod"
|
||||
|
||||
|
||||
def test_load_recent_derives_name_from_turn_stub_when_no_detail(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "alert", "HighCPU on prod-api-1")
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["name"] == "HighCPU on prod-api-1"
|
||||
|
||||
|
||||
def test_load_recent_name_truncated_at_50_chars(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
long_prompt = "a" * 60
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", long_prompt)
|
||||
|
||||
results = SessionStore.load_recent()
|
||||
|
||||
assert results[0]["name"] == "a" * 50 + "…"
|
||||
|
||||
|
||||
def test_load_recent_name_empty_for_slash_only_session(tmp_path: Path) -> None:
|
||||
sid = str(uuid.uuid4())
|
||||
(tmp_path / f"{sid}.jsonl").write_text(
|
||||
"\n".join(
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session",
|
||||
"version": 2,
|
||||
"id": sid,
|
||||
"created_at": "2024-01-01T10:00:00+00:00",
|
||||
"cwd": "",
|
||||
}
|
||||
),
|
||||
json.dumps(
|
||||
{
|
||||
"id": "entry1",
|
||||
"parent_id": None,
|
||||
"timestamp": "2024-01-01T10:00:01+00:00",
|
||||
"type": "custom_message",
|
||||
"custom_type": "turn_stub",
|
||||
"kind": "slash",
|
||||
"text": "/status",
|
||||
"display": False,
|
||||
}
|
||||
),
|
||||
]
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
with _patch_dir(tmp_path):
|
||||
results = SessionStore.load_recent()
|
||||
assert results[0]["name"] == ""
|
||||
|
||||
|
||||
def test_load_session_includes_name(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "debug the OOM killer")
|
||||
SessionStore.flush(session)
|
||||
|
||||
data = SessionStore.load_session(session.session_id[:8])
|
||||
|
||||
assert data is not None
|
||||
assert data["name"] == "debug the OOM killer"
|
||||
|
||||
|
||||
def test_count_prefix_matches_returns_correct_count(tmp_path: Path) -> None:
|
||||
for _ in range(3):
|
||||
sid = str(uuid.uuid4())
|
||||
_write_v2_session(
|
||||
tmp_path / f"{sid}.jsonl",
|
||||
sid,
|
||||
started_at="2024-01-01T10:00:00+00:00",
|
||||
)
|
||||
with _patch_dir(tmp_path):
|
||||
# Full UUID prefix matches exactly one
|
||||
first_sid = list(tmp_path.glob("*.jsonl"))[0].stem
|
||||
assert SessionStore.count_prefix_matches(first_sid[:8]) == 1
|
||||
# Very short prefix may match multiple — no assertion on count, just that it doesn't raise
|
||||
count = SessionStore.count_prefix_matches("")
|
||||
assert count == 3
|
||||
|
||||
|
||||
def test_count_prefix_matches_returns_zero_for_missing_dir(tmp_path: Path) -> None:
|
||||
with _patch_dir(tmp_path / "missing"):
|
||||
assert SessionStore.count_prefix_matches("abc") == 0
|
||||
|
||||
|
||||
def test_load_session_matches_full_id(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_turn(session, "chat", "hi")
|
||||
SessionStore.flush(session)
|
||||
|
||||
data = SessionStore.load_session(session.session_id)
|
||||
|
||||
assert data is not None
|
||||
assert data["session_id"] == session.session_id
|
||||
|
||||
|
||||
# ── flush resilience ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_flush_writes_session_end_even_when_snapshot_serialization_fails(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""P1: a snapshot serialization failure must not prevent session_end from being written."""
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "hello")
|
||||
# Inject a non-JSON-serializable value into accumulated_context so
|
||||
# json.dumps(snapshot) raises TypeError inside the inner suppress block.
|
||||
session.accumulated_context["bad"] = object() # type: ignore[assignment]
|
||||
|
||||
SessionStore.flush(session)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
types = [r["type"] for r in records]
|
||||
# context entry may degrade through default=str, but leaf must be present.
|
||||
assert "leaf" in types
|
||||
assert records[-1]["type"] == "leaf"
|
||||
|
||||
|
||||
# ── /new lifecycle ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_new_closes_old_session_and_opens_new(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
session.record("chat", "pre-new question")
|
||||
sid1 = session.session_id
|
||||
|
||||
# Simulate /new (flush → clear → open_session)
|
||||
SessionStore.flush(session)
|
||||
session.clear()
|
||||
SessionStore.open_session(session)
|
||||
sid2 = session.session_id
|
||||
|
||||
session.record("chat", "post-new question")
|
||||
|
||||
assert sid1 != sid2
|
||||
# Old session file has a leaf marker
|
||||
old_records = _read_lines(tmp_path / f"{sid1}.jsonl")
|
||||
assert old_records[-1]["type"] == "leaf"
|
||||
# New session file exists with turn but no leaf yet
|
||||
new_records = _read_lines(tmp_path / f"{sid2}.jsonl")
|
||||
assert new_records[0]["type"] == "session"
|
||||
assert any(r["type"] == "custom_message" for r in new_records)
|
||||
assert new_records[-1]["type"] != "leaf"
|
||||
|
||||
|
||||
# ── Session field behaviour ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_repl_session_has_stable_session_id() -> None:
|
||||
s = _make_session()
|
||||
assert isinstance(s.session_id, str) and len(s.session_id) > 0
|
||||
assert s.started_at <= time.time()
|
||||
|
||||
|
||||
def test_session_rotates_id_on_clear() -> None:
|
||||
s = _make_session()
|
||||
original_id = s.session_id
|
||||
s.history.append({"type": "chat", "text": "hi", "ok": True})
|
||||
time.sleep(0.01)
|
||||
s.clear()
|
||||
assert s.session_id != original_id
|
||||
assert s.started_at <= time.time()
|
||||
|
||||
|
||||
# ── investigation_result / RCA history ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_append_investigation_result_writes_record(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
state = {
|
||||
"root_cause": "connection pool exhausted",
|
||||
"problem_md": "## Summary\nPool leak in checkout-api",
|
||||
"root_cause_category": "resource",
|
||||
"alert_name": "checkout latency",
|
||||
}
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
state,
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
inv = next(r for r in records if r["type"] == "investigation_result")
|
||||
assert inv["investigation_id"] == inv_id
|
||||
assert inv["root_cause"] == "connection pool exhausted"
|
||||
assert "Pool leak" in inv["report"]
|
||||
assert inv["trigger"] == "/investigate generic"
|
||||
|
||||
|
||||
def test_append_investigation_result_uses_report_fallback(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "api error", "report": "report-only payload"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
records = _read_lines(tmp_path / f"{session.session_id}.jsonl")
|
||||
inv = next(r for r in records if r["type"] == "investigation_result")
|
||||
assert inv["report"] == "report-only payload"
|
||||
|
||||
|
||||
def test_load_investigation_history_returns_newest_first(tmp_path: Path) -> None:
|
||||
session_a = _make_session()
|
||||
session_b = Session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session_a)
|
||||
SessionStore.append_investigation_result(
|
||||
session_a.session_id,
|
||||
{"root_cause": "older issue", "problem_md": "old report"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
SessionStore.open_session(session_b)
|
||||
SessionStore.append_investigation_result(
|
||||
session_b.session_id,
|
||||
{"root_cause": "newer issue", "problem_md": "new report"},
|
||||
trigger="/investigate datadog",
|
||||
)
|
||||
|
||||
history = SessionStore.load_investigation_history()
|
||||
|
||||
assert len(history) == 2
|
||||
assert history[0]["root_cause"] == "newer issue"
|
||||
assert history[1]["root_cause"] == "older issue"
|
||||
|
||||
|
||||
def test_load_investigation_by_prefix(tmp_path: Path) -> None:
|
||||
session = _make_session()
|
||||
with _patch_dir(tmp_path):
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "disk full", "problem_md": "report body"},
|
||||
trigger="/investigate alert.json",
|
||||
)
|
||||
loaded = SessionStore.load_investigation(inv_id[:4])
|
||||
|
||||
assert loaded is not None
|
||||
assert loaded["investigation_id"] == inv_id
|
||||
assert loaded["root_cause"] == "disk full"
|
||||
|
||||
|
||||
def test_apply_investigation_result_persists_record(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
session.apply_investigation_result(
|
||||
{"root_cause": "OOM killer", "problem_md": "memory spike"},
|
||||
trigger="sample:generic",
|
||||
)
|
||||
|
||||
history = SessionStore.load_investigation_history()
|
||||
assert len(history) == 1
|
||||
assert history[0]["root_cause"] == "OOM killer"
|
||||
assert history[0]["trigger"] == "sample:generic"
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for structured REPL shell execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from typing import NoReturn
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.interactive_shell.shell.execution import (
|
||||
ShellExecutionResult,
|
||||
execute_shell_command,
|
||||
)
|
||||
from tools.interactive_shell.shell.parsing import parse_shell_command
|
||||
|
||||
|
||||
def test_execute_shell_command_reports_timeout_argv_mode(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def _raise(*_args: object, **_kwargs: object) -> NoReturn: # pragma: no cover
|
||||
raise subprocess.TimeoutExpired(
|
||||
cmd=["sleep", "999"],
|
||||
timeout=1,
|
||||
output="partial out\n",
|
||||
stderr="partial err\n",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.shell.execution.subprocess.run",
|
||||
_raise,
|
||||
)
|
||||
|
||||
result = execute_shell_command(
|
||||
command="ignored",
|
||||
argv=["sleep", "999"],
|
||||
use_shell=False,
|
||||
timeout_seconds=1,
|
||||
max_output_chars=10_000,
|
||||
)
|
||||
|
||||
assert result == ShellExecutionResult(
|
||||
command="ignored",
|
||||
argv=["sleep", "999"],
|
||||
stdout="partial out\n",
|
||||
stderr="partial err\n",
|
||||
exit_code=None,
|
||||
timed_out=True,
|
||||
truncated=False,
|
||||
executed_with_shell=False,
|
||||
)
|
||||
|
||||
|
||||
def test_execute_shell_command_reports_timeout_shell_mode(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def _raise(*_args: object, **_kwargs: object) -> NoReturn: # pragma: no cover
|
||||
raise subprocess.TimeoutExpired(
|
||||
cmd="sleep 999",
|
||||
timeout=1,
|
||||
output="out\n",
|
||||
stderr="err\n",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.interactive_shell.shell.execution.subprocess.run",
|
||||
_raise,
|
||||
)
|
||||
|
||||
result = execute_shell_command(
|
||||
command="sleep 999",
|
||||
argv=None,
|
||||
use_shell=True,
|
||||
timeout_seconds=1,
|
||||
max_output_chars=10_000,
|
||||
)
|
||||
|
||||
assert result == ShellExecutionResult(
|
||||
command="sleep 999",
|
||||
argv=None,
|
||||
stdout="out\n",
|
||||
stderr="err\n",
|
||||
exit_code=None,
|
||||
timed_out=True,
|
||||
truncated=False,
|
||||
executed_with_shell=True,
|
||||
)
|
||||
|
||||
|
||||
def test_execute_quoted_heredoc_through_shell() -> None:
|
||||
command = """python3 - <<'PY'
|
||||
print("hello-heredoc")
|
||||
PY"""
|
||||
parsed = parse_shell_command(command, is_windows=False)
|
||||
assert parsed.use_shell is True
|
||||
|
||||
result = execute_shell_command(
|
||||
command=parsed.command,
|
||||
argv=parsed.argv,
|
||||
use_shell=parsed.use_shell,
|
||||
timeout_seconds=10,
|
||||
max_output_chars=10_000,
|
||||
)
|
||||
|
||||
assert result.timed_out is False
|
||||
assert result.exit_code == 0
|
||||
assert "hello-heredoc" in result.stdout
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Tests for shell-specific execution policy.
|
||||
|
||||
Alpha mode allows every shell command (read-only, mutating, restricted,
|
||||
operators, substitution) and only rejects genuinely empty input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.interactive_shell.shell.policy import evaluate_shell_command
|
||||
|
||||
|
||||
def test_read_only_shell_is_allow() -> None:
|
||||
r = evaluate_shell_command("pwd")
|
||||
assert r.verdict == "allow"
|
||||
assert r.tool_type == "shell"
|
||||
|
||||
|
||||
def test_restricted_shell_is_allow() -> None:
|
||||
"""Alpha mode removed the restricted deny floor; ``sudo`` now runs."""
|
||||
r = evaluate_shell_command("sudo ls /")
|
||||
assert r.verdict == "allow"
|
||||
assert r.shell_classification == "unrestricted"
|
||||
|
||||
|
||||
def test_operator_shell_is_allow() -> None:
|
||||
"""Shell operators run through a shell instead of being blocked."""
|
||||
r = evaluate_shell_command("ls | grep x")
|
||||
assert r.verdict == "allow"
|
||||
|
||||
|
||||
def test_mutating_shell_is_allow() -> None:
|
||||
r = evaluate_shell_command("rm -rf /tmp/x")
|
||||
assert r.verdict == "allow"
|
||||
assert r.shell_classification == "unrestricted"
|
||||
|
||||
|
||||
def test_passthrough_shell_is_allow() -> None:
|
||||
r = evaluate_shell_command("!echo hi")
|
||||
assert r.verdict == "allow"
|
||||
|
||||
|
||||
def test_empty_shell_input_is_deny() -> None:
|
||||
"""Only genuinely empty input is rejected (input validation, not a guardrail)."""
|
||||
r = evaluate_shell_command("!")
|
||||
assert r.verdict == "deny"
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Tests for REPL shell command display formatting."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.interactive_shell.shell.display import format_shell_command_for_display
|
||||
|
||||
|
||||
def test_single_line_command_is_unchanged() -> None:
|
||||
assert format_shell_command_for_display("ls -la") == "ls -la"
|
||||
|
||||
|
||||
def test_quoted_heredoc_body_is_collapsed() -> None:
|
||||
command = """python3 - <<'PY'
|
||||
import json
|
||||
print(1)
|
||||
PY"""
|
||||
assert format_shell_command_for_display(command) == "python3 - <<'PY' … (2 lines)"
|
||||
|
||||
|
||||
def test_unquoted_heredoc_body_is_collapsed() -> None:
|
||||
command = """cat <<EOF
|
||||
alpha
|
||||
beta
|
||||
EOF"""
|
||||
assert format_shell_command_for_display(command) == "cat <<EOF … (2 lines)"
|
||||
|
||||
|
||||
def test_runner_display_hides_github_stars_script() -> None:
|
||||
command = """python3 - <<'PY'
|
||||
import json, urllib.request
|
||||
url='https://api.github.com/repos/tracer-cloud/opensre'
|
||||
with urllib.request.urlopen(url, timeout=10) as r:
|
||||
data=json.load(r)
|
||||
print(data.get('stargazers_count'))
|
||||
PY"""
|
||||
display = format_shell_command_for_display(command)
|
||||
assert display.startswith("python3 - <<'PY' … (")
|
||||
assert "import json" not in display
|
||||
assert "stargazers_count" not in display
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Tests for interactive-shell command parsing.
|
||||
|
||||
Alpha mode removed the shell-command safety policy (allowlist / restricted /
|
||||
mutating classification and the deny floor). Parsing now only decides *how* a
|
||||
command runs — via ``argv`` or through a shell — and never blocks a command for
|
||||
safety. The only non-execution outcome is empty input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.interactive_shell.shell.parsing import (
|
||||
argv_for_repl_builtin_detection,
|
||||
parse_shell_command,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_shell_command_detects_passthrough_prefix() -> None:
|
||||
parsed = parse_shell_command("!echo hello", is_windows=False)
|
||||
|
||||
assert parsed.passthrough is True
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.command == "echo hello"
|
||||
assert parsed.argv is None
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_plain_command_uses_argv_without_shell() -> None:
|
||||
parsed = parse_shell_command("rm -rf /tmp/x", is_windows=False)
|
||||
|
||||
assert parsed.passthrough is False
|
||||
assert parsed.use_shell is False
|
||||
assert parsed.argv == ["rm", "-rf", "/tmp/x"]
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_restricted_command_is_no_longer_blocked() -> None:
|
||||
"""``sudo`` and friends used to be a hard deny; alpha mode just runs them."""
|
||||
parsed = parse_shell_command("sudo systemctl restart nginx", is_windows=False)
|
||||
|
||||
assert parsed.use_shell is False
|
||||
assert parsed.argv == ["sudo", "systemctl", "restart", "nginx"]
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_operators_run_through_shell_without_block() -> None:
|
||||
parsed = parse_shell_command("ls | wc -l", is_windows=False)
|
||||
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.passthrough is False
|
||||
assert parsed.argv is None
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_command_substitution_runs_through_shell() -> None:
|
||||
parsed = parse_shell_command("echo $(date)", is_windows=False)
|
||||
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_quoted_heredoc_runs_through_shell() -> None:
|
||||
command = """python3 - <<'PY'
|
||||
print("hello-heredoc")
|
||||
PY"""
|
||||
parsed = parse_shell_command(command, is_windows=False)
|
||||
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.argv is None
|
||||
assert parsed.command == command.strip()
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_unquoted_heredoc_runs_through_shell() -> None:
|
||||
command = """cat <<EOF
|
||||
line one
|
||||
EOF"""
|
||||
parsed = parse_shell_command(command, is_windows=False)
|
||||
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.argv is None
|
||||
|
||||
|
||||
def test_unbalanced_quotes_fall_back_to_shell() -> None:
|
||||
parsed = parse_shell_command('echo "unterminated', is_windows=False)
|
||||
|
||||
assert parsed.use_shell is True
|
||||
assert parsed.parse_error is None
|
||||
|
||||
|
||||
def test_empty_passthrough_is_parse_error() -> None:
|
||||
parsed = parse_shell_command("!", is_windows=False)
|
||||
|
||||
assert parsed.parse_error is not None
|
||||
assert parsed.passthrough is True
|
||||
|
||||
|
||||
def test_empty_command_is_parse_error() -> None:
|
||||
parsed = parse_shell_command(" ", is_windows=False)
|
||||
|
||||
assert parsed.parse_error == "empty command."
|
||||
|
||||
|
||||
def test_argv_for_repl_builtin_detection_splits_passthrough_for_cd() -> None:
|
||||
parsed = parse_shell_command("!cd /tmp", is_windows=False)
|
||||
assert argv_for_repl_builtin_detection(parsed=parsed, is_windows=False) == ["cd", "/tmp"]
|
||||
|
||||
|
||||
def test_argv_for_repl_builtin_detection_returns_plain_argv() -> None:
|
||||
parsed = parse_shell_command("pwd", is_windows=False)
|
||||
assert argv_for_repl_builtin_detection(parsed=parsed, is_windows=False) == ["pwd"]
|
||||
|
||||
|
||||
def test_argv_for_repl_builtin_detection_skips_operator_command() -> None:
|
||||
"""A leading ``cd`` in an operator command must not be hijacked as a builtin."""
|
||||
parsed = parse_shell_command("cd /tmp && ls", is_windows=False)
|
||||
assert argv_for_repl_builtin_detection(parsed=parsed, is_windows=False) is None
|
||||
@@ -0,0 +1,836 @@
|
||||
"""Unit tests for /fleet slash command and conflict renderer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS, dispatch_slash
|
||||
from surfaces.interactive_shell.command_registry.agents import core as agents_core
|
||||
from surfaces.interactive_shell.command_registry.agents import trace as agents_trace
|
||||
from surfaces.interactive_shell.command_registry.agents.conflicts_view import render_conflicts
|
||||
from surfaces.interactive_shell.command_registry.agents.trace import _slice_to_utf8_boundary
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from tools.system.fleet_monitoring import config as config_mod
|
||||
from tools.system.fleet_monitoring.conflicts import DEFAULT_WINDOW_SECONDS, FileWriteConflict
|
||||
from tools.system.fleet_monitoring.registry import AgentRecord, AgentRegistry
|
||||
from tools.system.fleet_monitoring.tail import AttachUnsupported, TailBuffer
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False, width=120), buf
|
||||
|
||||
|
||||
def _isolate_registry(monkeypatch: pytest.MonkeyPatch, path: Path) -> AgentRegistry:
|
||||
"""Point the slash command's ``AgentRegistry()`` constructor at
|
||||
``path`` so tests don't read the developer's real
|
||||
``~/.opensre/agents.jsonl``. Returns the registry instance
|
||||
that the test can populate.
|
||||
"""
|
||||
registry = AgentRegistry(path=path)
|
||||
|
||||
monkeypatch.setattr(agents_core, "AgentRegistry", lambda: AgentRegistry(path=path))
|
||||
monkeypatch.setattr(agents_trace, "AgentRegistry", lambda: AgentRegistry(path=path))
|
||||
monkeypatch.setattr(
|
||||
agents_core,
|
||||
"registered_and_discovered_agents",
|
||||
lambda _registry=None: AgentRegistry(path=path).list(),
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_agents_yaml(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""Autouse: redirect ``agents_config_path()`` to a per-test tmp path so
|
||||
``/fleet`` (which now reads ``agents.yaml`` for the ``$/hr`` cell)
|
||||
and ``/fleet budget`` never touch the developer's real
|
||||
``~/.opensre/agents.yaml``.
|
||||
"""
|
||||
target = tmp_path / "agents.yaml"
|
||||
monkeypatch.setattr(config_mod, "agents_config_path", lambda: target)
|
||||
return target
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_sampler_module_state() -> None:
|
||||
"""Reset module-level state in the sampler before each test (#2023).
|
||||
|
||||
Same isolation pattern as ``tests/interactive_shell/ui/test_agents_view.py``:
|
||||
probe snapshots, the token rate tracker, and the per-tick caches
|
||||
all live as module globals and can leak across test files.
|
||||
"""
|
||||
from tools.system.fleet_monitoring import sampler as sampler_mod
|
||||
from tools.system.fleet_monitoring.token_rate import TOKEN_RATE_TRACKER
|
||||
|
||||
sampler_mod._latest.clear()
|
||||
sampler_mod._TickCache.registry_snapshot = {}
|
||||
sampler_mod._TickCache.agents_config = None
|
||||
for pid in list(TOKEN_RATE_TRACKER.known_pids()):
|
||||
TOKEN_RATE_TRACKER.forget(pid)
|
||||
|
||||
|
||||
class TestAgentsRegistration:
|
||||
def test_agents_command_is_registered(self) -> None:
|
||||
assert "/fleet" in SLASH_COMMANDS
|
||||
|
||||
def test_agents_first_arg_completions_include_conflicts(self) -> None:
|
||||
cmd = SLASH_COMMANDS["/fleet"]
|
||||
keywords = [pair[0] for pair in cmd.first_arg_completions]
|
||||
assert "conflicts" in keywords
|
||||
|
||||
def test_agents_first_arg_completions_include_trace(self) -> None:
|
||||
cmd = SLASH_COMMANDS["/fleet"]
|
||||
keywords = [pair[0] for pair in cmd.first_arg_completions]
|
||||
assert "trace" in keywords
|
||||
|
||||
def test_agents_first_arg_completions_include_wait_and_graph(self) -> None:
|
||||
cmd = SLASH_COMMANDS["/fleet"]
|
||||
keywords = [pair[0] for pair in cmd.first_arg_completions]
|
||||
assert "wait" in keywords
|
||||
assert "graph" in keywords
|
||||
|
||||
def test_default_window_constant_is_ten_seconds(self) -> None:
|
||||
assert DEFAULT_WINDOW_SECONDS == 10.0
|
||||
|
||||
|
||||
class TestAgentsDispatch:
|
||||
def test_conflicts_with_empty_event_source_renders_empty_state(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet conflicts", session, console) is True
|
||||
assert "no conflicts detected" in buf.getvalue()
|
||||
|
||||
def test_no_subcommand_with_empty_registry_renders_empty_state(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
|
||||
assert dispatch_slash("/fleet", session, console) is True
|
||||
|
||||
out = buf.getvalue()
|
||||
# Caption from agents_view.render_agents_table:
|
||||
assert "no agents discovered or registered" in out
|
||||
# Header row still rendered with the dashboard column structure:
|
||||
assert "agent" in out
|
||||
assert "pid" in out
|
||||
|
||||
def test_no_subcommand_renders_registered_agents(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="cursor-tab", pid=9133, command="cursor"))
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet", session, console) is True
|
||||
|
||||
out = buf.getvalue()
|
||||
assert "claude-code" in out
|
||||
assert "8421" in out
|
||||
assert "cursor-tab" in out
|
||||
assert "9133" in out
|
||||
|
||||
def test_no_subcommand_renders_discovered_agents(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
agents_core,
|
||||
"registered_and_discovered_agents",
|
||||
lambda _registry=None: [
|
||||
AgentRecord(
|
||||
name="cursor-claude-code",
|
||||
pid=80435,
|
||||
command="claude --output-format stream-json",
|
||||
source="discovered",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet", session, console) is True
|
||||
|
||||
out = buf.getvalue()
|
||||
assert "cursor-claude-code" in out
|
||||
assert "80435" in out
|
||||
|
||||
def test_unknown_subcommand_prints_error(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet bogus", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "unknown subcommand" in out.lower()
|
||||
assert "bogus" in out
|
||||
|
||||
def test_unknown_subcommand_message_lists_trace(self) -> None:
|
||||
# When the user types a bogus subcommand the help string should
|
||||
# advertise every supported one, including ``bus`` and ``trace``.
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet bogus", session, console) is True
|
||||
out = buf.getvalue().lower()
|
||||
assert "bus" in out
|
||||
assert "trace" in out
|
||||
|
||||
def test_dollar_hr_cell_does_not_read_budget_from_agents_yaml(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Regression for #2023's semantic split: ``$/hr`` is observed
|
||||
cost (sampler-driven), not the configured budget.
|
||||
|
||||
Setting ``/fleet budget`` persists ``hourly_budget_usd`` for
|
||||
a future alarm feature, but it must NOT appear in the ``$/hr``
|
||||
column of ``/fleet``. The cell renders ``-`` for a
|
||||
not-yet-observed PID, regardless of any configured budget.
|
||||
"""
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
|
||||
# Seed a budget — exercises the full slash write path.
|
||||
session = Session()
|
||||
write_console, _ = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code 5", session, write_console) is True
|
||||
|
||||
list_console, list_buf = _capture()
|
||||
assert dispatch_slash("/fleet", session, list_console) is True
|
||||
out = list_buf.getvalue()
|
||||
# The budget is persisted in agents.yaml (verified by the
|
||||
# ``/fleet budget`` round-trip test above), but the ``$/hr``
|
||||
# cell intentionally does not surface it. No ``$5.00`` should
|
||||
# appear anywhere in the rendered dashboard.
|
||||
assert "$5.00" not in out
|
||||
# The dashboard does render the row though.
|
||||
assert "claude-code" in out
|
||||
|
||||
def test_bare_agents_does_not_crash_on_schema_invalid_config(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolated_agents_yaml: Path
|
||||
) -> None:
|
||||
# Hand-edited agents.yaml with a typo'd field used to crash bare
|
||||
# /fleet with a raw ValidationError traceback. The dashboard
|
||||
# must degrade gracefully (render with $/hr = '-') so the user
|
||||
# can still see their fleet while /fleet budget surfaces the
|
||||
# actual error message.
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
isolated_agents_yaml.parent.mkdir(parents=True, exist_ok=True)
|
||||
isolated_agents_yaml.write_text(
|
||||
"agents:\n claude-code:\n hourly_budegt_usd: 5.0\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet", session, console) is True
|
||||
out = buf.getvalue()
|
||||
# Dashboard still renders the agent row.
|
||||
assert "claude-code" in out
|
||||
assert "8421" in out
|
||||
|
||||
|
||||
class TestAgentsBudget:
|
||||
def test_no_args_empty_state_when_no_config(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget", session, console) is True
|
||||
assert "no per-agent budgets" in buf.getvalue().lower()
|
||||
|
||||
def test_writes_and_round_trips_through_load(self, isolated_agents_yaml: Path) -> None:
|
||||
session = Session()
|
||||
write_console, write_buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code 5", session, write_console) is True
|
||||
|
||||
# Confirmation message references the agent and amount.
|
||||
write_out = write_buf.getvalue()
|
||||
assert "claude-code" in write_out
|
||||
assert "$5.00" in write_out
|
||||
|
||||
# Subsequent /fleet budget lists the just-written entry.
|
||||
read_console, read_buf = _capture()
|
||||
assert dispatch_slash("/fleet budget", session, read_console) is True
|
||||
read_out = read_buf.getvalue()
|
||||
assert "claude-code" in read_out
|
||||
assert "$5.00" in read_out
|
||||
|
||||
# File on disk has the expected shape.
|
||||
assert isolated_agents_yaml.exists()
|
||||
|
||||
def test_rejects_negative_budget(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code -3", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "invalid budget" in out.lower()
|
||||
# Latest slash invocation should be marked failed.
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_rejects_zero_budget(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code 0", session, console) is True
|
||||
assert "invalid budget" in buf.getvalue().lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_rejects_non_numeric_budget(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code five", session, console) is True
|
||||
assert "invalid budget" in buf.getvalue().lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_rejects_nan_budget(self, isolated_agents_yaml: Path) -> None:
|
||||
# ``float("nan") <= 0`` is ``False``, so without ``math.isfinite``
|
||||
# ``nan`` would slip past the guard, hit set_agent_budget, and
|
||||
# poison agents.yaml so the next load raises ValidationError.
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code nan", session, console) is True
|
||||
assert "invalid budget" in buf.getvalue().lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
# The file must not exist — a single non-finite write can't be
|
||||
# allowed to leave agents.yaml in an unreadable state.
|
||||
assert not isolated_agents_yaml.exists()
|
||||
|
||||
def test_rejects_inf_budget(self, isolated_agents_yaml: Path) -> None:
|
||||
# ``float("inf") <= 0`` is ``False`` and ``gt=0`` alone accepts
|
||||
# ``inf`` (``inf > 0`` is ``True``); only ``isfinite`` blocks it.
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code inf", session, console) is True
|
||||
assert "invalid budget" in buf.getvalue().lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
assert not isolated_agents_yaml.exists()
|
||||
|
||||
def test_single_arg_prints_usage(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget claude-code", session, console) is True
|
||||
assert "usage" in buf.getvalue().lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
def test_first_arg_completions_include_budget(self) -> None:
|
||||
cmd = SLASH_COMMANDS["/fleet"]
|
||||
keywords = [pair[0] for pair in cmd.first_arg_completions]
|
||||
assert "budget" in keywords
|
||||
|
||||
def test_corrupt_config_surfaces_friendly_error(self, isolated_agents_yaml: Path) -> None:
|
||||
# Hand-edit an agents.yaml with a typo'd field. The loader
|
||||
# raises ValidationError; the slash handler catches it and
|
||||
# renders a "agents.yaml has invalid contents" message rather
|
||||
# than crashing the REPL.
|
||||
isolated_agents_yaml.parent.mkdir(parents=True, exist_ok=True)
|
||||
isolated_agents_yaml.write_text(
|
||||
"agents:\n claude-code:\n hourly_budegt_usd: 5.0\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet budget", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "invalid contents" in out.lower()
|
||||
assert session.history[-1]["ok"] is False
|
||||
|
||||
|
||||
class TestRenderConflicts:
|
||||
def test_empty_list_returns_empty_state_string(self) -> None:
|
||||
assert render_conflicts([]) == "no conflicts detected"
|
||||
|
||||
def test_non_empty_list_returns_table_with_paths_and_agents(self) -> None:
|
||||
conflicts = [
|
||||
FileWriteConflict(
|
||||
path="/repo/auth.py",
|
||||
agents=("claude-code:1", "cursor:2"),
|
||||
first_seen=100.0,
|
||||
last_seen=110.0,
|
||||
),
|
||||
]
|
||||
result = render_conflicts(conflicts)
|
||||
assert isinstance(result, Table)
|
||||
|
||||
buf = io.StringIO()
|
||||
Console(file=buf, force_terminal=False, highlight=False, width=120).print(result)
|
||||
out = buf.getvalue()
|
||||
assert "/repo/auth.py" in out
|
||||
assert "claude-code:1" in out
|
||||
assert "cursor:2" in out
|
||||
|
||||
def test_multiple_conflicts_each_rendered(self) -> None:
|
||||
conflicts = [
|
||||
FileWriteConflict(
|
||||
path="/new.py",
|
||||
agents=("claude-code:1", "cursor:2"),
|
||||
first_seen=140.0,
|
||||
last_seen=150.0,
|
||||
),
|
||||
FileWriteConflict(
|
||||
path="/old.py",
|
||||
agents=("aider:3", "cursor:2"),
|
||||
first_seen=100.0,
|
||||
last_seen=105.0,
|
||||
),
|
||||
]
|
||||
result = render_conflicts(conflicts)
|
||||
assert isinstance(result, Table)
|
||||
|
||||
buf = io.StringIO()
|
||||
Console(file=buf, force_terminal=False, highlight=False, width=120).print(result)
|
||||
out = buf.getvalue()
|
||||
assert "/new.py" in out
|
||||
assert "/old.py" in out
|
||||
assert "aider:3" in out
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Minimal :class:`AttachSession` stand-in for slash-command tests.
|
||||
|
||||
Lets us drive ``_render_live_tail`` through ``dispatch_slash`` without
|
||||
spawning a reader thread or touching the filesystem.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
chunks: list[bytes] | None = None,
|
||||
*,
|
||||
raise_ki: bool = False,
|
||||
producer_exited: bool = False,
|
||||
) -> None:
|
||||
self.buffer = TailBuffer()
|
||||
self._chunks = list(chunks or [])
|
||||
self._raise_ki = raise_ki
|
||||
self.producer_exited = producer_exited
|
||||
self.closed = False
|
||||
|
||||
def __iter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
def __next__(self) -> bytes:
|
||||
if self._raise_ki:
|
||||
self._raise_ki = False # raise once, like a real Ctrl+C
|
||||
raise KeyboardInterrupt
|
||||
if not self._chunks:
|
||||
raise StopIteration
|
||||
chunk = self._chunks.pop(0)
|
||||
# Mirror :class:`AttachSession.__next__`: append-on-yield so the
|
||||
# slash-command renderer only ever needs to read ``sess.buffer``.
|
||||
self.buffer.append(chunk)
|
||||
return chunk
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
def __enter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
class TestSliceToUtf8Boundary:
|
||||
"""``_slice_to_utf8_boundary`` enforces the same chunk-edge guarantee
|
||||
on the render side that :class:`TailBuffer` enforces on the consumer
|
||||
side. Without it, the 64 KiB cap in ``_render_live_tail`` would slice
|
||||
mid-codepoint and surface a U+FFFD at the top of the live view."""
|
||||
|
||||
def test_returns_input_when_under_cap(self) -> None:
|
||||
data = b"\xf0\x9f\xa6\x80hello"
|
||||
assert _slice_to_utf8_boundary(data, max_bytes=64) is data
|
||||
|
||||
def test_returns_input_when_exactly_at_cap(self) -> None:
|
||||
data = b"X" * 64
|
||||
assert _slice_to_utf8_boundary(data, max_bytes=64) is data
|
||||
|
||||
def test_walks_past_continuation_bytes_at_slice_start(self) -> None:
|
||||
# 🦀 = b"\xf0\x9f\xa6\x80". Slicing at byte 2 of that codepoint
|
||||
# would leave two stray continuation bytes (\xa6\x80) at the
|
||||
# head; the boundary walk must skip them so the decoded string
|
||||
# has no leading replacement character.
|
||||
prefix = b"X" * 1000
|
||||
data = prefix + b"\xf0\x9f\xa6\x80 done"
|
||||
# max_bytes drops the first 998 bytes of prefix but keeps the
|
||||
# last two ASCII bytes of prefix plus the partial 4-byte 🦀
|
||||
# plus " done" — i.e. it lands inside the codepoint.
|
||||
sliced = _slice_to_utf8_boundary(data, max_bytes=8)
|
||||
# The decoded suffix must not start with U+FFFD.
|
||||
decoded = sliced.decode("utf-8", errors="replace")
|
||||
assert not decoded.startswith("�")
|
||||
# And it must end with the trailing ASCII so we know we kept
|
||||
# the suffix, not over-trimmed.
|
||||
assert decoded.endswith(" done")
|
||||
|
||||
def test_drops_at_most_three_continuation_bytes(self) -> None:
|
||||
# If for some reason the slice starts with more than 3
|
||||
# continuation bytes (corrupt input), we don't loop forever.
|
||||
# The bound is the max UTF-8 codepoint length (4 bytes).
|
||||
bad = b"\x80\x80\x80\x80\x80hello"
|
||||
sliced = _slice_to_utf8_boundary(bad, max_bytes=len(bad))
|
||||
assert sliced is bad # under cap, returned unchanged
|
||||
|
||||
sliced = _slice_to_utf8_boundary(b"prefix" + bad, max_bytes=len(bad))
|
||||
# We walk forward at most 4 bytes; if all of those are still
|
||||
# continuation bytes, decoding will surface U+FFFD — the helper
|
||||
# is bounded, not magic.
|
||||
assert len(sliced) >= len(bad) - 4
|
||||
|
||||
def test_pure_ascii_unchanged_after_slice(self) -> None:
|
||||
data = b"abcdefghij"
|
||||
sliced = _slice_to_utf8_boundary(data, max_bytes=5)
|
||||
assert sliced == b"fghij"
|
||||
|
||||
def test_multibyte_decode_clean_after_slice(self) -> None:
|
||||
# Snapshot ends with several 2-byte codepoints (é = b"\xc3\xa9");
|
||||
# the slice must land on a leading byte so the decoded string
|
||||
# has no leading replacement character.
|
||||
prefix = b"a" * 100
|
||||
data = prefix + b"\xc3\xa9\xc3\xa9\xc3\xa9"
|
||||
sliced = _slice_to_utf8_boundary(data, max_bytes=5)
|
||||
decoded = sliced.decode("utf-8", errors="replace")
|
||||
assert "�" not in decoded
|
||||
|
||||
|
||||
class TestAgentsTrace:
|
||||
def test_no_args_prints_usage(self) -> None:
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace", sess_obj, console) is True
|
||||
out = buf.getvalue().lower()
|
||||
assert "usage" in out
|
||||
assert "<pid>" in out
|
||||
assert sess_obj.history[-1]["ok"] is False
|
||||
|
||||
def test_non_numeric_pid_rejected(self) -> None:
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace abc", sess_obj, console) is True
|
||||
out = buf.getvalue().lower()
|
||||
assert "invalid pid" in out
|
||||
assert sess_obj.history[-1]["ok"] is False
|
||||
|
||||
def test_too_many_args_rejected(self) -> None:
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 1 2", sess_obj, console) is True
|
||||
assert "usage" in buf.getvalue().lower()
|
||||
assert sess_obj.history[-1]["ok"] is False
|
||||
|
||||
def test_attach_unsupported_renders_reason(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def _refuse(_pid: int) -> _FakeSession:
|
||||
raise AttachUnsupported("stdout is on a terminal; live tail not supported")
|
||||
|
||||
monkeypatch.setattr(agents_trace, "attach", _refuse)
|
||||
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "cannot trace" in out
|
||||
assert "stdout is on a terminal" in out
|
||||
assert sess_obj.history[-1]["ok"] is False
|
||||
|
||||
def test_unknown_pid_falls_back_to_pid_label(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Header says "pid <n>" when the pid is not in the registry.
|
||||
_isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
monkeypatch.setattr(agents_trace, "attach", lambda _pid: _FakeSession(chunks=[]))
|
||||
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "pid 8421" in out
|
||||
assert "trace ended" in out
|
||||
|
||||
def test_known_pid_uses_registered_name_in_header(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
monkeypatch.setattr(agents_trace, "attach", lambda _pid: _FakeSession(chunks=[]))
|
||||
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "claude-code" in out
|
||||
assert "8421" in out
|
||||
|
||||
def test_renders_chunks_through_live(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Two chunks then StopIteration: the handler should append them
|
||||
# to the buffer and feed Live.update; the rendered text should
|
||||
# land in the captured console output.
|
||||
monkeypatch.setattr(
|
||||
agents_trace,
|
||||
"attach",
|
||||
lambda _pid: _FakeSession(chunks=[b"hello ", b"world\n"]),
|
||||
)
|
||||
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "hello world" in out
|
||||
assert "trace ended" in out
|
||||
|
||||
def test_swallows_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Single Ctrl+C inside the Live block must not propagate out of
|
||||
# ``dispatch_slash`` — the REPL should return to its prompt.
|
||||
# This is the kubectl-logs-style UX, deliberately different from
|
||||
# ``stream_to_console``'s double-press pattern.
|
||||
monkeypatch.setattr(agents_trace, "attach", lambda _pid: _FakeSession(raise_ki=True))
|
||||
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
# Must not raise:
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
assert "trace ended" in buf.getvalue()
|
||||
|
||||
def test_session_is_closed_even_on_keyboard_interrupt(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Lifecycle guard: the ``with`` block in the handler must close
|
||||
# the AttachSession (and thereby the reader thread) regardless
|
||||
# of whether the iteration completed naturally or was stopped
|
||||
# by a Ctrl+C.
|
||||
fake = _FakeSession(raise_ki=True)
|
||||
monkeypatch.setattr(agents_trace, "attach", lambda _pid: fake)
|
||||
|
||||
sess_obj = Session()
|
||||
console, _ = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
assert fake.closed is True
|
||||
|
||||
def test_renders_process_exited_when_producer_died(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# When the agent process dies during a trace, the trailer should
|
||||
# surface that explicitly so an unattended trace doesn't look
|
||||
# the same as a Ctrl+C abort.
|
||||
monkeypatch.setattr(
|
||||
agents_trace,
|
||||
"attach",
|
||||
lambda _pid: _FakeSession(chunks=[b"goodbye\n"], producer_exited=True),
|
||||
)
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "process exited" in out
|
||||
assert "trace ended" in out
|
||||
|
||||
def test_does_not_render_process_exited_on_user_stop(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Producer is alive, user pressed Ctrl+C: only "trace ended"
|
||||
# should appear; "process exited" would be misleading.
|
||||
monkeypatch.setattr(
|
||||
agents_trace,
|
||||
"attach",
|
||||
lambda _pid: _FakeSession(raise_ki=True, producer_exited=False),
|
||||
)
|
||||
sess_obj = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet trace 8421", sess_obj, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "process exited" not in out
|
||||
assert "trace ended" in out
|
||||
|
||||
|
||||
class TestAgentsWait:
|
||||
def test_usage_when_missing_on_flag(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 1234 5678", session, console) is True
|
||||
assert "usage:" in buf.getvalue().lower()
|
||||
|
||||
def test_rejects_non_numeric_pid(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait abc --on 5678", session, console) is True
|
||||
assert "invalid pid" in buf.getvalue().lower()
|
||||
|
||||
def test_rejects_non_numeric_on_pid(self) -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 1234 --on xyz", session, console) is True
|
||||
assert "invalid other-pid" in buf.getvalue().lower()
|
||||
|
||||
def test_rejects_self_wait(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 1234 --on 1234", session, console) is True
|
||||
assert "waiting for itself" in buf.getvalue()
|
||||
|
||||
def test_rejects_unknown_waiter(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 9999 --on 8421", session, console) is True
|
||||
assert "pid 9999 is not in the agent registry" in buf.getvalue()
|
||||
|
||||
def test_rejects_unknown_target(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider"))
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 7702 --on 9999", session, console) is True
|
||||
assert "pid 9999 is not in the agent registry" in buf.getvalue()
|
||||
|
||||
def test_happy_path_persists_dependency(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# End-to-end: wait announces the edge in output AND survives a
|
||||
# fresh AgentRegistry load (i.e. the rewrite step actually fired).
|
||||
registry_path = tmp_path / "agents.jsonl"
|
||||
registry = _isolate_registry(monkeypatch, registry_path)
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider"))
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet wait 7702 --on 8421", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "aider" in out
|
||||
assert "claude-code" in out
|
||||
assert "now waits on" in out
|
||||
|
||||
reloaded = AgentRegistry(path=registry_path).get(7702)
|
||||
assert reloaded is not None
|
||||
assert reloaded.waits_on == (8421,)
|
||||
|
||||
def test_repeated_wait_is_idempotent(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Two wait calls with the same pair must not produce [8421, 8421].
|
||||
# Guards against a regression where `add_waits_on`'s membership
|
||||
# check is removed and duplicates leak into the JSONL.
|
||||
registry_path = tmp_path / "agents.jsonl"
|
||||
registry = _isolate_registry(monkeypatch, registry_path)
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider"))
|
||||
|
||||
session = Session()
|
||||
for _ in range(2):
|
||||
console, _ = _capture()
|
||||
assert dispatch_slash("/fleet wait 7702 --on 8421", session, console) is True
|
||||
|
||||
reloaded = AgentRegistry(path=registry_path).get(7702)
|
||||
assert reloaded is not None
|
||||
assert reloaded.waits_on == (8421,)
|
||||
|
||||
|
||||
class TestAgentsGraph:
|
||||
def test_empty_registry_renders_empty_state(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
assert "no registered agents" in buf.getvalue()
|
||||
|
||||
def test_acyclic_chain_renders_all_agents(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# The issue's example tree:
|
||||
# claude-code (8421) [active]
|
||||
# └── aider (7702) [waiting on claude-code]
|
||||
# └── cursor-tab (9133) [waiting on aider]
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider", waits_on=(8421,)))
|
||||
registry.register(
|
||||
AgentRecord(name="cursor-tab", pid=9133, command="cursor", waits_on=(7702,))
|
||||
)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "claude-code" in out
|
||||
assert "aider" in out
|
||||
assert "cursor-tab" in out
|
||||
assert "claude-code (8421) [active]" in out
|
||||
assert "aider (7702) [waiting on claude-code]" in out
|
||||
assert "cursor-tab (9133) [waiting on aider]" in out.lower()
|
||||
|
||||
def test_cycle_prints_warning(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="alpha", pid=1, command="a", waits_on=(2,)))
|
||||
registry.register(AgentRecord(name="beta", pid=2, command="b", waits_on=(1,)))
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
out = buf.getvalue()
|
||||
|
||||
assert "agent dependency cycle detected" in out
|
||||
assert "alpha" in out
|
||||
assert "beta" in out
|
||||
assert "alpha (1) -> beta (2) -> alpha (1)" in out
|
||||
|
||||
def test_acyclic_chain_multiple_roots(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider", waits_on=(8421,)))
|
||||
registry.register(AgentRecord(name="cursor-tab", pid=9133, command="cursor"))
|
||||
registry.register(AgentRecord(name="aider", pid=8491, command="aider", waits_on=(9133,)))
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "cursor-tab (9133) [active]" in out
|
||||
assert "claude-code (8421) [active]" in out
|
||||
assert "aider (7702) [waiting on claude-code]" in out
|
||||
assert "aider (8491) [waiting on cursor-tab]" in out
|
||||
|
||||
def test_acyclic_chain_multiple_blockers(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="aider", pid=7702, command="aider", waits_on=(8421,)))
|
||||
registry.register(
|
||||
AgentRecord(name="cursor-tab", pid=9133, command="cursor", waits_on=(8421,))
|
||||
)
|
||||
registry.register(
|
||||
AgentRecord(name="aider", pid=8491, command="aider", waits_on=(9133, 7702))
|
||||
)
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "claude-code (8421) [active]" in out
|
||||
assert "aider (8491) [waiting on aider]" in out
|
||||
assert "aider (8491) [waiting on cursor-tab]" in out
|
||||
|
||||
def test_acyclic_chain_multiple_roots_blockers(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
registry = _isolate_registry(monkeypatch, tmp_path / "agents.jsonl")
|
||||
registry.register(AgentRecord(name="claude-code", pid=8421, command="claude"))
|
||||
registry.register(AgentRecord(name="cursor-tab", pid=9133, command="cursor"))
|
||||
registry.register(
|
||||
AgentRecord(name="aider", pid=7702, command="aider", waits_on=(8421, 9133))
|
||||
)
|
||||
registry.register(
|
||||
AgentRecord(name="cursor-tab", pid=9134, command="cursor", waits_on=(9133, 8421))
|
||||
)
|
||||
registry.register(AgentRecord(name="aider", pid=8491, command="aider", waits_on=(9134,)))
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/fleet graph", session, console) is True
|
||||
out = buf.getvalue()
|
||||
assert "claude-code (8421) [active]" in out
|
||||
assert "cursor-tab (9133) [active]" in out
|
||||
assert "cursor-tab (9134) [waiting on claude-code]" in out
|
||||
assert "cursor-tab (9134) [waiting on cursor-tab]" in out
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Tests for REPL alert listener wiring in :mod:`surfaces.interactive_shell.controller`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from config.repl_config import ReplConfig
|
||||
from surfaces.interactive_shell.controller import _alert_listener
|
||||
|
||||
|
||||
def test_alert_listener_replaces_stale_process_token(monkeypatch) -> None:
|
||||
os.environ["OPENSRE_ALERT_LISTENER_TOKEN"] = "stale"
|
||||
captured: list[str | None] = []
|
||||
|
||||
def _fake_serve(**_kwargs: object) -> MagicMock:
|
||||
captured.append(os.environ.get("OPENSRE_ALERT_LISTENER_TOKEN"))
|
||||
handle = MagicMock()
|
||||
handle.bound_address = "127.0.0.1:8765"
|
||||
return handle
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gateway.web_server.serve_webapp_in_thread",
|
||||
_fake_serve,
|
||||
)
|
||||
cfg = ReplConfig(alert_listener_enabled=True, alert_listener_token="fresh")
|
||||
|
||||
with _alert_listener(cfg, Console(force_terminal=False)) as inbox:
|
||||
assert inbox is not None
|
||||
assert captured == ["fresh"]
|
||||
|
||||
assert os.environ.get("OPENSRE_ALERT_LISTENER_TOKEN") == "stale"
|
||||
|
||||
|
||||
def test_alert_listener_clears_token_when_unconfigured(monkeypatch) -> None:
|
||||
os.environ["OPENSRE_ALERT_LISTENER_TOKEN"] = "stale"
|
||||
captured: list[str | None] = []
|
||||
|
||||
def _fake_serve(**_kwargs: object) -> MagicMock:
|
||||
captured.append(os.environ.get("OPENSRE_ALERT_LISTENER_TOKEN"))
|
||||
handle = MagicMock()
|
||||
handle.bound_address = "127.0.0.1:8765"
|
||||
return handle
|
||||
|
||||
monkeypatch.setattr(
|
||||
"gateway.web_server.serve_webapp_in_thread",
|
||||
_fake_serve,
|
||||
)
|
||||
cfg = ReplConfig(alert_listener_enabled=True, alert_listener_token=None)
|
||||
|
||||
with _alert_listener(cfg, Console(force_terminal=False)) as inbox:
|
||||
assert inbox is not None
|
||||
assert captured == [None]
|
||||
|
||||
assert os.environ.get("OPENSRE_ALERT_LISTENER_TOKEN") == "stale"
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Tests for Bedrock-specific model validation and custom ID handling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.command_registry.model.command import (
|
||||
_prompt_custom_model_id,
|
||||
_reasoning_model_menu_choices,
|
||||
_toolcall_model_menu_choices,
|
||||
)
|
||||
from surfaces.interactive_shell.command_registry.model.switching import _is_model_allowed
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeModelOption:
|
||||
value: str
|
||||
label: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeProvider:
|
||||
value: str
|
||||
models: tuple[_FakeModelOption, ...] = ()
|
||||
allow_custom_models: bool = False
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# _is_model_allowed — custom model bypass
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestIsModelSupportedBedrock:
|
||||
"""Bedrock must accept any non-empty model string (ARNs, regional prefixes, etc.)."""
|
||||
|
||||
def test_bedrock_accepts_inference_profile_id(self) -> None:
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "us.anthropic.claude-sonnet-4-6") is True
|
||||
|
||||
def test_bedrock_accepts_eu_prefix(self) -> None:
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "eu.anthropic.claude-sonnet-4-6") is True
|
||||
|
||||
def test_bedrock_accepts_global_prefix(self) -> None:
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "global.anthropic.claude-opus-4-7") is True
|
||||
|
||||
def test_bedrock_accepts_on_demand_model(self) -> None:
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "mistral.mistral-large-3-675b-instruct") is True
|
||||
|
||||
def test_bedrock_accepts_full_arn(self) -> None:
|
||||
arn = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/my-profile"
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, arn) is True
|
||||
|
||||
def test_bedrock_rejects_empty_string(self) -> None:
|
||||
provider = _FakeProvider(value="bedrock", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "") is False
|
||||
|
||||
def test_ollama_also_accepts_any_model(self) -> None:
|
||||
"""Ollama shares the same bypass pattern — verify it still works."""
|
||||
provider = _FakeProvider(value="ollama", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "llama3.2") is True
|
||||
|
||||
def test_other_provider_requires_match(self) -> None:
|
||||
"""Non-bypass providers must match the curated model list."""
|
||||
models = (_FakeModelOption(value="gpt-5.4"),)
|
||||
provider = _FakeProvider(value="anthropic", models=models)
|
||||
assert _is_model_allowed(provider, "gpt-5.4") is True
|
||||
assert _is_model_allowed(provider, "gpt-unknown") is False
|
||||
|
||||
def test_custom_provider_accepts_unknown_non_empty_model(self) -> None:
|
||||
provider = _FakeProvider(value="openai", allow_custom_models=True)
|
||||
assert _is_model_allowed(provider, "gpt-5.5") is True
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Menu choices — __custom__ sentinel
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMenuChoicesCustomOption:
|
||||
"""Bedrock menus must include the __custom__ escape hatch."""
|
||||
|
||||
def test_reasoning_menu_includes_custom_for_bedrock(self) -> None:
|
||||
provider = _FakeProvider(
|
||||
value="bedrock",
|
||||
models=(_FakeModelOption(value="us.anthropic.claude-sonnet-4-6"),),
|
||||
allow_custom_models=True,
|
||||
)
|
||||
choices = _reasoning_model_menu_choices(provider)
|
||||
values = [v for v, _ in choices]
|
||||
assert "__custom__" in values
|
||||
|
||||
def test_toolcall_menu_includes_custom_for_bedrock(self) -> None:
|
||||
provider = _FakeProvider(
|
||||
value="bedrock",
|
||||
models=(_FakeModelOption(value="us.anthropic.claude-sonnet-4-6"),),
|
||||
allow_custom_models=True,
|
||||
)
|
||||
choices = _toolcall_model_menu_choices(provider)
|
||||
values = [v for v, _ in choices]
|
||||
assert "__custom__" in values
|
||||
|
||||
def test_reasoning_menu_includes_custom_for_openai(self) -> None:
|
||||
provider = _FakeProvider(
|
||||
value="openai",
|
||||
models=(_FakeModelOption(value="gpt-5.4"),),
|
||||
allow_custom_models=True,
|
||||
)
|
||||
choices = _reasoning_model_menu_choices(provider)
|
||||
values = [v for v, _ in choices]
|
||||
assert "__custom__" in values
|
||||
|
||||
def test_toolcall_menu_includes_custom_for_openai(self) -> None:
|
||||
provider = _FakeProvider(
|
||||
value="openai",
|
||||
models=(_FakeModelOption(value="gpt-5.4"),),
|
||||
allow_custom_models=True,
|
||||
)
|
||||
choices = _toolcall_model_menu_choices(provider)
|
||||
values = [v for v, _ in choices]
|
||||
assert "__custom__" in values
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# _prompt_custom_model_id
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestPromptCustomModelId:
|
||||
"""Custom model ID prompt edge cases."""
|
||||
|
||||
def test_returns_stripped_value(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
monkeypatch.setattr(console, "input", lambda _prompt: " us.anthropic.claude-opus-4-7 ")
|
||||
result = _prompt_custom_model_id(console)
|
||||
assert result == "us.anthropic.claude-opus-4-7"
|
||||
|
||||
def test_returns_none_on_empty_input(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
monkeypatch.setattr(console, "input", lambda _prompt: " ")
|
||||
result = _prompt_custom_model_id(console)
|
||||
assert result is None
|
||||
|
||||
def test_returns_none_on_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
|
||||
def _raise_interrupt(_prompt: str) -> str:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
monkeypatch.setattr(console, "input", _raise_interrupt)
|
||||
result = _prompt_custom_model_id(console)
|
||||
assert result is None
|
||||
|
||||
def test_returns_none_on_eof(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from rich.console import Console
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
|
||||
def _raise_eof(_prompt: str) -> str:
|
||||
raise EOFError
|
||||
|
||||
monkeypatch.setattr(console, "input", _raise_eof)
|
||||
result = _prompt_custom_model_id(console)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Bedrock ProviderOption in wizard config
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBedrockProviderConfig:
|
||||
"""Verify Bedrock is registered correctly in the wizard config."""
|
||||
|
||||
def test_bedrock_in_supported_providers(self) -> None:
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
assert "bedrock" in PROVIDER_BY_VALUE
|
||||
|
||||
def test_bedrock_credential_kind_is_none(self) -> None:
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
assert provider.credential_kind == "none"
|
||||
|
||||
def test_bedrock_has_curated_models(self) -> None:
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
assert len(provider.models) >= 10
|
||||
|
||||
def test_bedrock_curated_models_use_inference_profiles(self) -> None:
|
||||
"""All Claude models in the curated list must use us.* inference profile IDs."""
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
claude_models = [m for m in provider.models if "anthropic" in str(getattr(m, "value", ""))]
|
||||
for model in claude_models:
|
||||
value = str(getattr(model, "value", ""))
|
||||
assert value.startswith("us."), (
|
||||
f"Claude model '{value}' must use a us.* inference profile ID"
|
||||
)
|
||||
|
||||
def test_bedrock_has_toolcall_model_env(self) -> None:
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
assert provider.toolcall_model_env == "BEDROCK_TOOLCALL_MODEL"
|
||||
|
||||
def test_bedrock_api_key_env_is_empty(self) -> None:
|
||||
"""api_key_env="" is intentional — Bedrock uses IAM auth, not an API key."""
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
assert provider.api_key_env == ""
|
||||
# Empty string must be falsy so downstream ``bool(provider.api_key_env)``
|
||||
# checks correctly skip API-key validation for Bedrock.
|
||||
assert not provider.api_key_env
|
||||
|
||||
def test_bedrock_no_credential_default(self) -> None:
|
||||
"""credential_default must use the dataclass default (empty string).
|
||||
|
||||
Region is picked up from AWS_DEFAULT_REGION / ~/.aws/config, not from
|
||||
the wizard credential prompt (which is skipped for credential_kind="none").
|
||||
"""
|
||||
from surfaces.cli.wizard.config import PROVIDER_BY_VALUE
|
||||
|
||||
provider = PROVIDER_BY_VALUE["bedrock"]
|
||||
assert provider.credential_default == ""
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# _interactive_set_toolcall + __custom__ integration
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestInteractiveSetToolcallCustom:
|
||||
"""Verify the __custom__ branch inside _interactive_set_toolcall wires through correctly."""
|
||||
|
||||
def test_custom_toolcall_calls_switch_with_typed_id(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Selecting __custom__ in toolcall menu should prompt and call switch_toolcall_model."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry.model import command as model_mod
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
custom_id = "eu.anthropic.claude-sonnet-4-6"
|
||||
|
||||
# First call: pick provider "bedrock"; second call: pick "__custom__"
|
||||
choose_returns = iter(["bedrock", "__custom__"])
|
||||
monkeypatch.setattr(model_mod, "repl_choose_one", lambda **_kw: next(choose_returns))
|
||||
monkeypatch.setattr(model_mod, "_prompt_custom_model_id", lambda *_args: custom_id)
|
||||
|
||||
with patch.object(model_mod, "switch_toolcall_model", return_value=True) as mock_switch:
|
||||
result = model_mod._interactive_set_toolcall(console)
|
||||
|
||||
assert result is True
|
||||
mock_switch.assert_called_once_with(custom_id, console, provider_name="bedrock")
|
||||
|
||||
def test_custom_toolcall_returns_none_on_cancel(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""If user cancels the custom prompt, _interactive_set_toolcall returns None."""
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry.model import command as model_mod
|
||||
|
||||
console = Console(force_terminal=False)
|
||||
|
||||
choose_returns = iter(["bedrock", "__custom__"])
|
||||
monkeypatch.setattr(model_mod, "repl_choose_one", lambda **_kw: next(choose_returns))
|
||||
monkeypatch.setattr(model_mod, "_prompt_custom_model_id", lambda *_args: None)
|
||||
|
||||
result = model_mod._interactive_set_toolcall(console)
|
||||
assert result is None
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Boot-time import-budget guardrails for the interactive shell.
|
||||
|
||||
These lock in the lazy-loading wins so a future eager import in the boot path
|
||||
(controller construction, banner, background workers, prompt/completion) fails
|
||||
loudly instead of silently re-inflating REPL startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Modules that must NOT be pulled into the graph merely by importing the REPL
|
||||
# entrypoint. Each maps to the lazy-loading change that keeps it out.
|
||||
_FORBIDDEN_AT_BOOT: tuple[str, ...] = (
|
||||
# Harness/turn-execution stack — deferred until the first turn is dispatched.
|
||||
"surfaces.interactive_shell.runtime.shell_turn_execution",
|
||||
"core.agent.agent",
|
||||
"core.agent_harness.turns.action_driver",
|
||||
# Email delivery — only loads on background-RCA completion.
|
||||
"integrations.smtp.delivery",
|
||||
)
|
||||
|
||||
|
||||
def _modules_loaded_by(import_target: str, candidates: tuple[str, ...]) -> list[str]:
|
||||
check = "; ".join(f"print('{name}', '{name}' in sys.modules)" for name in candidates)
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", f"import sys; import {import_target}; {check}"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
loaded: list[str] = []
|
||||
for line in result.stdout.splitlines():
|
||||
name, _, flag = line.partition(" ")
|
||||
if flag.strip() == "True":
|
||||
loaded.append(name)
|
||||
return loaded
|
||||
|
||||
|
||||
def test_repl_boot_import_stays_lazy() -> None:
|
||||
loaded = _modules_loaded_by(
|
||||
"surfaces.interactive_shell.main",
|
||||
_FORBIDDEN_AT_BOOT,
|
||||
)
|
||||
assert loaded == [], (
|
||||
"importing surfaces.interactive_shell.main eagerly pulled modules that "
|
||||
f"must load lazily: {loaded}"
|
||||
)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Unit tests for modular slash-command registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS, dispatch_slash
|
||||
from surfaces.interactive_shell.command_registry.integrations import (
|
||||
_INTEGRATIONS_FIRST_ARGS,
|
||||
_MCP_FIRST_ARGS,
|
||||
)
|
||||
from surfaces.interactive_shell.command_registry.investigation import (
|
||||
_INVESTIGATE_FIRST_ARGS,
|
||||
_TEMPLATE_FIRST_ARGS,
|
||||
)
|
||||
from surfaces.interactive_shell.command_registry.model.command import _MODEL_FIRST_ARGS
|
||||
from surfaces.interactive_shell.command_registry.settings_cmds import (
|
||||
_TRUST_FIRST_ARGS,
|
||||
_VERBOSE_FIRST_ARGS,
|
||||
)
|
||||
from surfaces.interactive_shell.command_registry.tools_cmds import _TOOLS_FIRST_ARGS
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def test_slash_registry_includes_modular_commands() -> None:
|
||||
for name in (
|
||||
"/help",
|
||||
"/?",
|
||||
"/exit",
|
||||
"/model",
|
||||
"/tools",
|
||||
"/integrations",
|
||||
"/investigate",
|
||||
"/tasks",
|
||||
"/watch",
|
||||
"/watches",
|
||||
"/unwatch",
|
||||
"/health",
|
||||
):
|
||||
assert name in SLASH_COMMANDS
|
||||
|
||||
|
||||
def test_dispatch_unknown_command_stays_in_repl() -> None:
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/not-a-real-slash", session, console) is True
|
||||
assert "Unknown command" in buf.getvalue()
|
||||
|
||||
|
||||
def test_registry_first_arg_completion_hints_co_located_with_handlers() -> None:
|
||||
"""Merged registry exposes the same first-arg tab tuples defined in each module."""
|
||||
expected: dict[str, tuple[tuple[str, str], ...]] = {
|
||||
"/model": _MODEL_FIRST_ARGS,
|
||||
"/tools": _TOOLS_FIRST_ARGS,
|
||||
"/integrations": _INTEGRATIONS_FIRST_ARGS,
|
||||
"/mcp": _MCP_FIRST_ARGS,
|
||||
"/investigate": _INVESTIGATE_FIRST_ARGS,
|
||||
"/template": _TEMPLATE_FIRST_ARGS,
|
||||
"/trust": _TRUST_FIRST_ARGS,
|
||||
"/verbose": _VERBOSE_FIRST_ARGS,
|
||||
}
|
||||
for name, tup in expected.items():
|
||||
assert SLASH_COMMANDS[name].first_arg_completions == tup
|
||||
|
||||
assert SLASH_COMMANDS["/help"].first_arg_completions == ()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
"""Tests for the /gateway slash command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.command_registry.gateway_cmds import _cmd_gateway
|
||||
|
||||
_MODULE = "surfaces.interactive_shell.command_registry.gateway_cmds"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def console() -> Console:
|
||||
return Console(record=True, force_terminal=False, width=200)
|
||||
|
||||
|
||||
def _output(console: Console) -> str:
|
||||
return console.export_text()
|
||||
|
||||
|
||||
def test_status_shows_running_daemon_and_components(
|
||||
monkeypatch: pytest.MonkeyPatch, console: Console
|
||||
) -> None:
|
||||
monkeypatch.setattr(f"{_MODULE}.gateway_daemon_pid", lambda: 4242)
|
||||
monkeypatch.setattr(
|
||||
f"{_MODULE}.read_component_status",
|
||||
lambda: {"web": "serving :8000", "telegram": "polling for messages"},
|
||||
)
|
||||
|
||||
assert _cmd_gateway(MagicMock(), console, ["status"]) is True
|
||||
|
||||
out = _output(console)
|
||||
assert "running (pid 4242)" in out
|
||||
assert "web: serving :8000" in out
|
||||
assert "telegram: polling for messages" in out
|
||||
|
||||
|
||||
def test_bare_gateway_defaults_to_status(monkeypatch: pytest.MonkeyPatch, console: Console) -> None:
|
||||
monkeypatch.setattr(f"{_MODULE}.gateway_daemon_pid", lambda: None)
|
||||
monkeypatch.setattr(f"{_MODULE}.read_component_status", dict)
|
||||
|
||||
assert _cmd_gateway(MagicMock(), console, []) is True
|
||||
assert "stopped" in _output(console)
|
||||
|
||||
|
||||
def test_start_reports_outcome_and_log_path(
|
||||
monkeypatch: pytest.MonkeyPatch, console: Console
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
f"{_MODULE}.start_gateway_daemon",
|
||||
lambda: (True, "OpenSRE gateway started (pid 7)."),
|
||||
)
|
||||
|
||||
assert _cmd_gateway(MagicMock(), console, ["start"]) is True
|
||||
|
||||
out = _output(console)
|
||||
assert "started (pid 7)" in out
|
||||
assert "gateway.log" in out
|
||||
|
||||
|
||||
def test_logs_prints_tail(monkeypatch: pytest.MonkeyPatch, console: Console) -> None:
|
||||
monkeypatch.setattr(f"{_MODULE}.read_gateway_log_tail", lambda lines: f"tail of {lines}")
|
||||
|
||||
assert _cmd_gateway(MagicMock(), console, ["logs", "7"]) is True
|
||||
assert "tail of 7" in _output(console)
|
||||
|
||||
|
||||
def test_unknown_subcommand_prints_usage(console: Console) -> None:
|
||||
assert _cmd_gateway(MagicMock(), console, ["restart"]) is True
|
||||
assert "usage:" in _output(console)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Tests for the session-scoped grounding context and its diagnostics sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.agent_harness.grounding.context import GroundingContext
|
||||
from core.agent_harness.grounding.diagnostics import (
|
||||
GroundingSource,
|
||||
log_grounding_cache_diagnostics,
|
||||
)
|
||||
from core.agent_harness.grounding.models import CacheStats
|
||||
from surfaces.interactive_shell.grounding.cli_reference import ShellPromptContextProvider
|
||||
from surfaces.interactive_shell.session.session import Session
|
||||
|
||||
|
||||
def _make_source(name: str, hits: int = 0) -> GroundingSource:
|
||||
return GroundingSource(name=name, stats_fn=lambda: CacheStats(name=name, hits=hits))
|
||||
|
||||
|
||||
def test_context_exposes_one_source_per_cache() -> None:
|
||||
"""A GroundingContext yields a diagnostics source for each repo-level cache."""
|
||||
ctx = GroundingContext()
|
||||
names = [s.name for s in ctx.iter_sources()]
|
||||
assert names == ["docs", "agents_md"]
|
||||
|
||||
|
||||
def test_context_sources_are_isolated_per_instance() -> None:
|
||||
"""Two contexts own independent caches (no shared module-level state)."""
|
||||
ctx_a = GroundingContext()
|
||||
ctx_b = GroundingContext()
|
||||
assert ctx_a.docs is not ctx_b.docs
|
||||
assert ctx_a.agents_md is not ctx_b.agents_md
|
||||
|
||||
|
||||
def test_invalidate_clears_every_cache() -> None:
|
||||
ctx = GroundingContext()
|
||||
ctx.agents_md.build_text()
|
||||
assert ctx.agents_md.stats().misses >= 1
|
||||
ctx.invalidate()
|
||||
assert ctx.agents_md.stats().misses == 0
|
||||
|
||||
|
||||
def test_shell_prompt_provider_cli_cache_is_session_scoped() -> None:
|
||||
session_a = Session()
|
||||
session_b = Session()
|
||||
provider_a = ShellPromptContextProvider(session_a)
|
||||
provider_b = ShellPromptContextProvider(session_a)
|
||||
provider_c = ShellPromptContextProvider(session_b)
|
||||
provider_a.cli_reference()
|
||||
provider_b.cli_reference()
|
||||
assert provider_b._cli.stats().hits >= 1 # noqa: SLF001 - same session shares cache
|
||||
provider_c.cli_reference()
|
||||
assert provider_c._cli.stats().misses >= 1 # noqa: SLF001 - different session is isolated
|
||||
|
||||
|
||||
def test_log_grounding_iterates_provided_sources(monkeypatch: object) -> None:
|
||||
"""log_grounding_cache_diagnostics logs each provided source when verbose."""
|
||||
import os
|
||||
|
||||
from core.agent_harness.grounding import diagnostics as _gd
|
||||
|
||||
logged: list[str] = []
|
||||
try:
|
||||
monkeypatch.setenv("TRACER_VERBOSE", "1") # type: ignore[attr-defined]
|
||||
monkeypatch.setattr( # type: ignore[attr-defined]
|
||||
_gd._logger,
|
||||
"debug",
|
||||
lambda msg, *args: logged.append(msg % args),
|
||||
)
|
||||
log_grounding_cache_diagnostics([_make_source("mock", hits=5)], "test_reason")
|
||||
assert any("mock" in entry for entry in logged)
|
||||
finally:
|
||||
os.environ.pop("TRACER_VERBOSE", None)
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def test_model_switch_resets_runtime_llm_caches(monkeypatch: Any) -> None:
|
||||
import surfaces.interactive_shell.command_registry.model.switching as model_module
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr("core.llm.factory.reset_llm_clients", lambda: calls.append("reset"))
|
||||
|
||||
model_module._reset_runtime_llm_caches()
|
||||
|
||||
assert calls == ["reset"]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""/model set offers inline API-key entry instead of dead-ending on a missing key."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import surfaces.interactive_shell.command_registry.model.switching as switching
|
||||
|
||||
|
||||
class _Console:
|
||||
is_terminal = True # switch_llm_provider only prompts on an interactive console
|
||||
|
||||
def __init__(self, key: str = "") -> None:
|
||||
self._key = key
|
||||
self.printed: list[str] = []
|
||||
|
||||
def print(self, message: str = "") -> None:
|
||||
self.printed.append(str(message))
|
||||
|
||||
def input(self, prompt: str = "", *, password: bool = False) -> str:
|
||||
_ = (prompt, password)
|
||||
return self._key
|
||||
|
||||
|
||||
def _credential_status_sequence(monkeypatch: Any, *, configured_after: bool) -> None:
|
||||
"""First check reports missing; the re-check after a save reports the result."""
|
||||
import config.llm_auth.credentials as credentials
|
||||
|
||||
statuses = iter(
|
||||
[
|
||||
SimpleNamespace(configured=False, stale=False, detail=""),
|
||||
SimpleNamespace(configured=configured_after, stale=False, detail=""),
|
||||
]
|
||||
)
|
||||
fallback = SimpleNamespace(configured=configured_after, stale=False, detail="")
|
||||
monkeypatch.setattr(credentials, "status", lambda _p: next(statuses, fallback))
|
||||
|
||||
|
||||
def test_blank_key_cancels_without_saving(monkeypatch: Any) -> None:
|
||||
import surfaces.cli.llm_auth.service as service
|
||||
|
||||
_credential_status_sequence(monkeypatch, configured_after=False)
|
||||
saves: list[Any] = []
|
||||
monkeypatch.setattr(service, "configure_api_key_provider", lambda **kw: saves.append(kw))
|
||||
|
||||
console = _Console(key="") # blank input = cancel
|
||||
assert switching.switch_llm_provider("openai", console) is False # type: ignore[arg-type]
|
||||
assert saves == []
|
||||
assert any("cancelled" in line for line in console.printed)
|
||||
|
||||
|
||||
def test_pasted_key_is_saved_and_switch_proceeds(monkeypatch: Any) -> None:
|
||||
import surfaces.cli.llm_auth.providers as providers
|
||||
import surfaces.cli.llm_auth.service as service
|
||||
import surfaces.cli.wizard.env_sync as env_sync
|
||||
|
||||
_credential_status_sequence(monkeypatch, configured_after=True)
|
||||
monkeypatch.setattr(providers, "resolve_auth_profile", lambda _p: object())
|
||||
saves: list[Any] = []
|
||||
monkeypatch.setattr(service, "configure_api_key_provider", lambda **kw: saves.append(kw))
|
||||
monkeypatch.setattr(env_sync, "sync_provider_env", lambda **_: "/tmp/.env")
|
||||
monkeypatch.setattr(switching, "_reset_runtime_llm_caches", lambda: None)
|
||||
monkeypatch.setattr(switching, "render_models_table", lambda *_: None)
|
||||
monkeypatch.setattr(switching.repl_data, "load_llm_settings", lambda: {})
|
||||
|
||||
console = _Console(key="sk-test-123")
|
||||
assert switching.switch_llm_provider("openai", console) is True # type: ignore[arg-type]
|
||||
assert saves and saves[0]["api_key"] == "sk-test-123"
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Programmatic parity validation between the Click CLI and the REPL slash commands."""
|
||||
|
||||
from surfaces.cli.__main__ import cli
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS
|
||||
|
||||
# Commands that are intentionally excluded from the REPL (e.g. they don't make sense in session).
|
||||
# 'agent' is excluded because the REPL itself is the agent entry point.
|
||||
# 'gateway' is excluded because it starts a standalone local HTTP server process.
|
||||
EXCLUDED_COMMANDS = {"agent", "gateway"}
|
||||
|
||||
|
||||
def test_cli_slash_command_parity():
|
||||
"""Ensure every top-level Click command has a corresponding slash command in the REPL."""
|
||||
# Get all registered top-level commands from the main Click group
|
||||
cli_commands = set(cli.commands.keys())
|
||||
|
||||
# Filter out excluded commands
|
||||
expected_commands = cli_commands - EXCLUDED_COMMANDS
|
||||
|
||||
# Get all registered slash commands (strip leading slash for comparison)
|
||||
registered_slash_names = {name.lstrip("/") for name in SLASH_COMMANDS}
|
||||
|
||||
# Find missing commands
|
||||
missing = expected_commands - registered_slash_names
|
||||
|
||||
assert not missing, (
|
||||
f"The following CLI commands are missing from the REPL slash-command registry: {missing}"
|
||||
)
|
||||
|
||||
|
||||
def test_slash_command_help_parity():
|
||||
"""Ensure slash command descriptions are concise and usage is structured."""
|
||||
for name, cmd in SLASH_COMMANDS.items():
|
||||
assert len(cmd.description) > 10, f"Description for {name} is too short or missing."
|
||||
assert "(" not in cmd.description, f"Description for {name} should not contain usage."
|
||||
if name in {"/integrations", "/remote", "/tests", "/guardrails"}:
|
||||
assert cmd.usage, f"Usage for {name} should list common subcommands."
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Tests for /rca history and /rca show."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from core.agent_harness.session import JsonlSessionStorage
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
SessionStore = JsonlSessionStorage()
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def test_rca_history_lists_persisted_reports(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "redis timeout", "problem_md": "cache unavailable"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca history", Session(), console) is True
|
||||
output = buf.getvalue()
|
||||
assert "RCA history" in output
|
||||
assert "redis timeout" in output
|
||||
assert "/investigate generic" in output
|
||||
|
||||
|
||||
def test_rca_show_renders_full_report(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "bad deploy", "problem_md": "## Report\nRollback required"},
|
||||
trigger="/investigate grafana",
|
||||
)
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca show {inv_id[:4]}", Session(), console) is True
|
||||
output = buf.getvalue()
|
||||
assert "bad deploy" in output
|
||||
assert "Rollback required" in output
|
||||
|
||||
|
||||
def test_bare_rca_defaults_to_history(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca", Session(), console) is True
|
||||
assert "no persisted RCA reports yet" in buf.getvalue()
|
||||
|
||||
|
||||
def test_tty_rca_menu_latest_shows_report(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from surfaces.interactive_shell.command_registry import rca_cmds
|
||||
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "connection pool exhausted", "problem_md": "## Report\nPool at max"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rca_cmds, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(rca_cmds, "repl_choose_one", lambda **_: rca_cmds._RCA_LATEST)
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca", session, console) is True
|
||||
output = buf.getvalue()
|
||||
assert "connection pool exhausted" in output
|
||||
assert "Pool at max" in output
|
||||
assert inv_id[:8] in output
|
||||
|
||||
|
||||
def test_tty_rca_history_menu_picks_report_directly(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from surfaces.interactive_shell.command_registry import rca_cmds
|
||||
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
older_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "older issue", "problem_md": "older report"},
|
||||
trigger="/investigate grafana",
|
||||
)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "newer issue", "problem_md": "newer report"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rca_cmds, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(rca_cmds, "repl_choose_one", lambda **_: older_id)
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca history", session, console) is True
|
||||
output = buf.getvalue()
|
||||
assert "older issue" in output
|
||||
assert "older report" in output
|
||||
assert "newer issue" not in output
|
||||
|
||||
|
||||
def test_tty_rca_root_menu_history_picks_report(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from surfaces.interactive_shell.command_registry import rca_cmds
|
||||
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
older_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "older issue", "problem_md": "older report"},
|
||||
trigger="/investigate grafana",
|
||||
)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "newer issue", "problem_md": "newer report"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
picks = iter([rca_cmds._RCA_HISTORY, older_id])
|
||||
monkeypatch.setattr(rca_cmds, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(rca_cmds, "repl_choose_one", lambda **_: next(picks))
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca", session, console) is True
|
||||
output = buf.getvalue()
|
||||
assert "older issue" in output
|
||||
assert "older report" in output
|
||||
assert "newer issue" not in output
|
||||
|
||||
|
||||
def test_rca_save_writes_markdown(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "redis timeout", "problem_md": "cache unavailable"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
dest = tmp_path / "report.md"
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save {dest}", Session(), console) is True
|
||||
assert (
|
||||
dest.read_text(encoding="utf-8")
|
||||
== "## Root Cause\n\nredis timeout\n\n## Report\n\ncache unavailable\n"
|
||||
)
|
||||
assert "saved:" in buf.getvalue()
|
||||
|
||||
|
||||
def test_rca_save_by_id_writes_json(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "bad deploy", "problem_md": "rollback required"},
|
||||
trigger="/investigate grafana",
|
||||
)
|
||||
|
||||
dest = tmp_path / "report.json"
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save {inv_id[:4]} {dest}", Session(), console) is True
|
||||
payload = json.loads(dest.read_text(encoding="utf-8"))
|
||||
assert payload["root_cause"] == "bad deploy"
|
||||
assert payload["problem_md"] == "rollback required"
|
||||
assert payload["investigation_id"] == inv_id
|
||||
assert "saved:" in buf.getvalue()
|
||||
|
||||
|
||||
def test_rca_save_strips_quoted_path(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "quoted path", "problem_md": "saved body"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
dest = tmp_path / "quoted.md"
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save {inv_id[:4]} '{dest}'", Session(), console) is True
|
||||
assert dest.read_text(encoding="utf-8").startswith("## Root Cause")
|
||||
assert "saved:" in buf.getvalue()
|
||||
|
||||
|
||||
def test_rca_save_to_new_folder_adds_default_filename(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "folder save", "problem_md": "in subfolder"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
folder = tmp_path / "rca_reports"
|
||||
folder.mkdir()
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save {inv_id[:4]} {folder}/", Session(), console) is True
|
||||
saved = folder / f"rca-{inv_id[:8]}.md"
|
||||
assert saved.exists()
|
||||
assert "folder save" in saved.read_text(encoding="utf-8")
|
||||
assert "saved:" in buf.getvalue()
|
||||
|
||||
|
||||
def test_rca_save_to_new_folder_trailing_slash_creates_subdirectory(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
inv_id = SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "nested save", "problem_md": "inside new folder"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
folder = tmp_path / "new_rca_folder"
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save {inv_id[:4]} {folder}/", Session(), console) is True
|
||||
saved = folder / f"rca-{inv_id[:8]}.md"
|
||||
assert saved.exists()
|
||||
assert "nested save" in saved.read_text(encoding="utf-8")
|
||||
assert "saved:" in buf.getvalue()
|
||||
|
||||
|
||||
def test_rca_save_unknown_id_reports_not_found(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "existing issue", "problem_md": "existing report"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
dest = tmp_path / "report.md"
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash(f"/rca save badid {dest}", Session(), console) is True
|
||||
output = buf.getvalue()
|
||||
assert "RCA report not found" in output
|
||||
assert "no persisted RCA reports yet" not in output
|
||||
assert not dest.exists()
|
||||
|
||||
|
||||
def test_normalize_rca_save_path_strips_quotes() -> None:
|
||||
from surfaces.interactive_shell.command_registry import rca_cmds
|
||||
|
||||
dest = rca_cmds._normalize_rca_save_path("'/tmp/report.md'", investigation_id="abcd1234")
|
||||
assert dest == Path("/tmp/report.md")
|
||||
|
||||
|
||||
def test_tty_rca_save_menu_picks_latest_and_prompts_path(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from surfaces.interactive_shell.command_registry import rca_cmds
|
||||
|
||||
monkeypatch.setattr("config.constants.OPENSRE_HOME_DIR", tmp_path)
|
||||
session = Session()
|
||||
SessionStore.open_session(session)
|
||||
SessionStore.append_investigation_result(
|
||||
session.session_id,
|
||||
{"root_cause": "latest root cause", "problem_md": "latest body"},
|
||||
trigger="/investigate generic",
|
||||
)
|
||||
|
||||
dest = tmp_path / "picked.md"
|
||||
monkeypatch.setattr(rca_cmds, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(rca_cmds, "repl_choose_one", lambda **_: rca_cmds._RCA_LATEST)
|
||||
monkeypatch.setattr(rca_cmds, "_prompt_rca_save_path", lambda _console: str(dest))
|
||||
|
||||
console, buf = _capture()
|
||||
assert dispatch_slash("/rca save", session, console) is True
|
||||
assert "latest root cause" in dest.read_text(encoding="utf-8")
|
||||
assert "saved:" in buf.getvalue()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,333 @@
|
||||
"""Turn-focused tests for interactive shell terminal runtime dispatch helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from core.llm.types import AgentLLMResponse, ToolCall
|
||||
from surfaces.interactive_shell.runtime.core.turn_accounting import (
|
||||
ToolCallingTurnResult,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.shell_turn_execution import execute_shell_turn
|
||||
from surfaces.interactive_shell.runtime.turn_host import run_agent_turn_queue
|
||||
from surfaces.interactive_shell.runtime.utils import input_policy as loop_input_policy
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from tests.core.agent.orchestration.action_execution_test_harness import (
|
||||
FakeActionLLM,
|
||||
)
|
||||
from tools.interactive_shell.actions import (
|
||||
investigation as _investigation_tool,
|
||||
)
|
||||
from tools.interactive_shell.actions import (
|
||||
slash as _slash_tool,
|
||||
)
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_bare_integration_menu(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/integrations", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/investigate", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/mcp", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/model", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/theme", session) is True
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/integrations list", session) is False
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/theme blue", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/verify", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/verify datadog", session) is False
|
||||
|
||||
# Gating is literal-/slash only: bare command words are not recognized.
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("integrations", session) is False
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("integrations list", session) is False
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("verify", session) is False
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_false_for_investigate_with_target(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Queued menu selections run as ``/investigate <target>`` without blocking the prompt."""
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/investigate generic", session) is False
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/investigate alert.json", session) is False
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_exit_commands(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/exit", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/quit", session) is True
|
||||
# Bare command words are not recognized under literal-/slash gating.
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("quit", session) is False
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_update(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``/update`` hits the network; block the next prompt until output is printed."""
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/update", session) is True
|
||||
# Bare command words are not recognized under literal-/slash gating.
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("update", session) is False
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_integration_setup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/integrations setup", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/mcp connect github", session) is True
|
||||
# Bare command words are not recognized under literal-/slash gating.
|
||||
assert (
|
||||
loop_input_policy.turn_needs_exclusive_stdin("integrations setup datadog", session) is False
|
||||
)
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_integration_remove(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``remove``/``disconnect`` drive a native inline picker that reads raw
|
||||
stdin; the REPL must block the next prompt so keystrokes and CPR responses
|
||||
do not leak into the prompt buffer."""
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/integrations remove", session) is True
|
||||
assert (
|
||||
loop_input_policy.turn_needs_exclusive_stdin("/integrations remove github", session) is True
|
||||
)
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/mcp disconnect", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/mcp disconnect github", session) is True
|
||||
# Bare command words are not recognized under literal-/slash gating.
|
||||
assert (
|
||||
loop_input_policy.turn_needs_exclusive_stdin("integrations remove github", session) is False
|
||||
)
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_onboard(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``/onboard`` is an interactive wizard; the REPL must wait for it to
|
||||
finish before reading the next prompt so the wizard subprocess has
|
||||
exclusive stdin and can drive its own questionary widgets.
|
||||
"""
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/onboard", session) is True
|
||||
# Args don't change the exclusive-stdin requirement.
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/onboard local_llm", session) is True
|
||||
# Bare command words are not recognized under literal-/slash gating.
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("onboard", session) is False
|
||||
|
||||
|
||||
def test_turn_needs_exclusive_stdin_for_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``/config`` delegates to a subprocess; block the next prompt until output
|
||||
is printed so config lines do not overlap the pinned input bar.
|
||||
"""
|
||||
monkeypatch.setattr(loop_input_policy, "repl_tty_interactive", lambda: True)
|
||||
session = Session()
|
||||
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/config", session) is True
|
||||
assert loop_input_policy.turn_needs_exclusive_stdin("/config show", session) is True
|
||||
assert (
|
||||
loop_input_policy.turn_needs_exclusive_stdin(
|
||||
"/config set interactive.layout pinned",
|
||||
session,
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_queued_literal_quit_requests_runtime_exit() -> None:
|
||||
async def _scenario() -> None:
|
||||
from surfaces.interactive_shell.runtime.core.state import ReplState
|
||||
|
||||
state = ReplState()
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
async def _run_turn(text: str) -> None:
|
||||
await asyncio.to_thread(
|
||||
execute_shell_turn,
|
||||
text,
|
||||
session,
|
||||
console,
|
||||
recorder=None,
|
||||
confirm_fn=None,
|
||||
is_tty=None,
|
||||
request_exit=state.request_exit,
|
||||
)
|
||||
|
||||
worker = asyncio.create_task(run_agent_turn_queue(state=state, run_turn=_run_turn))
|
||||
await state.queue.put("/quit")
|
||||
await asyncio.wait_for(state.queue.join(), timeout=1)
|
||||
await asyncio.wait_for(worker, timeout=1)
|
||||
|
||||
assert state.exit_requested is True
|
||||
|
||||
asyncio.run(_scenario())
|
||||
|
||||
|
||||
def test_execute_shell_turn_nitro_prompt_uses_cli_agent_actions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
nitro_prompt = (
|
||||
"I want to deploy OpenSRE on a remote EC2 Nitro instance, and then I want to send\n"
|
||||
'it an investigation. Can you please deploy the instance and send it "hello world"?'
|
||||
)
|
||||
action_calls: list[str] = []
|
||||
llm_calls: list[str] = []
|
||||
|
||||
def _fake_execute_cli_actions(
|
||||
text: str,
|
||||
_session: Session,
|
||||
_console: Console,
|
||||
**kwargs: object,
|
||||
) -> ToolCallingTurnResult:
|
||||
action_calls.append(text)
|
||||
return ToolCallingTurnResult(
|
||||
planned_count=2,
|
||||
executed_count=2,
|
||||
executed_success_count=2,
|
||||
has_unhandled_clause=False,
|
||||
handled=True,
|
||||
)
|
||||
|
||||
def _fake_answer_shell_question(
|
||||
text: str,
|
||||
_session: Session,
|
||||
_console: Console,
|
||||
**kwargs: object,
|
||||
) -> None:
|
||||
llm_calls.append(text)
|
||||
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
execute_shell_turn(
|
||||
nitro_prompt,
|
||||
session,
|
||||
console,
|
||||
recorder=None,
|
||||
confirm_fn=None,
|
||||
is_tty=None,
|
||||
execute_actions=_fake_execute_cli_actions,
|
||||
answer_agent=_fake_answer_shell_question,
|
||||
)
|
||||
|
||||
assert action_calls == [nitro_prompt]
|
||||
assert llm_calls == []
|
||||
|
||||
|
||||
def test_execute_shell_turn_nitro_prompt_executes_remote_then_investigation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
nitro_prompt = (
|
||||
"I want to deploy OpenSRE on a remote EC2 Nitro instance, and then I want to send\n"
|
||||
'it an investigation. Can you please deploy the instance and send it "hello world"?'
|
||||
)
|
||||
call_order: list[str] = []
|
||||
|
||||
def _fake_dispatch(
|
||||
command: str,
|
||||
session: Session,
|
||||
console: Console,
|
||||
**_kwargs: object,
|
||||
) -> bool:
|
||||
call_order.append(f"slash:{command}")
|
||||
session.record("slash", command, ok=True)
|
||||
console.print(f"ran {command}")
|
||||
return True
|
||||
|
||||
def _fake_run_text_investigation(
|
||||
alert_text: str,
|
||||
_session: Session,
|
||||
_console: Console,
|
||||
**_kwargs: object,
|
||||
) -> None:
|
||||
call_order.append(f"investigation:{alert_text}")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.runtime.action_turn.default_llm_factory",
|
||||
lambda: FakeActionLLM(
|
||||
[
|
||||
AgentLLMResponse(
|
||||
content="",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="call_remote",
|
||||
name="slash_invoke",
|
||||
input={"command": "/remote", "args": []},
|
||||
),
|
||||
ToolCall(
|
||||
id="call_investigate",
|
||||
name="investigation_start",
|
||||
input={"alert_text": "hello world"},
|
||||
),
|
||||
],
|
||||
raw_content=None,
|
||||
)
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(_slash_tool, "dispatch_slash", _fake_dispatch)
|
||||
monkeypatch.setattr(_investigation_tool, "run_text_investigation", _fake_run_text_investigation)
|
||||
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
execute_shell_turn(
|
||||
nitro_prompt,
|
||||
session,
|
||||
console,
|
||||
recorder=None,
|
||||
confirm_fn=None,
|
||||
is_tty=None,
|
||||
)
|
||||
|
||||
assert call_order == ["slash:/remote", "investigation:hello world"]
|
||||
|
||||
|
||||
class TestDispatchSpinnerBehavior:
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"/history",
|
||||
"/tests",
|
||||
"/model show",
|
||||
],
|
||||
)
|
||||
def test_slash_dispatches_do_not_show_assistant_spinner(self, text: str) -> None:
|
||||
assert loop_input_policy.turn_should_show_spinner(text, Session()) is False
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"why did this fail?",
|
||||
"explain deploy",
|
||||
# Bare command words and opensre passthrough are no longer treated as
|
||||
# literal commands, so the spinner shows while the planner runs.
|
||||
"tests",
|
||||
"help",
|
||||
"opensre investigate -i alert.json",
|
||||
],
|
||||
)
|
||||
def test_non_slash_dispatches_show_assistant_spinner(self, text: str) -> None:
|
||||
assert loop_input_policy.turn_should_show_spinner(text, Session()) is True
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Tests for /watch, /watches, /unwatch slash commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from integrations.telegram.credentials import TelegramCredentials
|
||||
from platform.common.task_types import TaskKind, TaskStatus
|
||||
from surfaces.interactive_shell.command_registry import SLASH_COMMANDS, dispatch_slash
|
||||
from surfaces.interactive_shell.command_registry.watch_cmds import (
|
||||
WatchdogStartSpec,
|
||||
parse_watch_argv,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def test_slash_registry_includes_watchdog_commands() -> None:
|
||||
assert "/watch" in SLASH_COMMANDS
|
||||
assert "/watches" in SLASH_COMMANDS
|
||||
assert "/unwatch" in SLASH_COMMANDS
|
||||
|
||||
|
||||
def test_parse_watch_argv_rejects_non_numeric_pid() -> None:
|
||||
out = parse_watch_argv(["not-a-pid"])
|
||||
assert isinstance(out, str)
|
||||
assert "invalid pid" in out
|
||||
|
||||
|
||||
def test_parse_watch_argv_rejects_non_positive_pid() -> None:
|
||||
out = parse_watch_argv(["0"])
|
||||
assert isinstance(out, str)
|
||||
assert "positive" in out
|
||||
|
||||
|
||||
def test_parse_watch_argv_parses_flags() -> None:
|
||||
raw = parse_watch_argv(["999", "--max-cpu", "80", "--max-runtime", "10s", "--once"])
|
||||
assert isinstance(raw, WatchdogStartSpec)
|
||||
assert raw.pid == 999
|
||||
assert raw.max_cpu == 80.0
|
||||
assert raw.max_runtime_seconds == 10.0
|
||||
assert raw.once is True
|
||||
|
||||
|
||||
def test_dispatch_watch_creates_watchdog_task(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.load_credentials_from_env",
|
||||
lambda *_a, **_kw: TelegramCredentials(bot_token="x", chat_id="1"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.pid_exists",
|
||||
lambda _pid: True,
|
||||
)
|
||||
|
||||
def _fake_start(**_kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.start_watchdog_daemon_thread",
|
||||
_fake_start,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
console, buf = _capture()
|
||||
dispatch_slash(
|
||||
f"/watch {__import__('os').getpid()} --max-cpu 80",
|
||||
session,
|
||||
console,
|
||||
is_tty=True,
|
||||
)
|
||||
|
||||
watchdogs = [t for t in session.task_registry.list_recent(20) if t.kind == TaskKind.WATCHDOG]
|
||||
assert len(watchdogs) == 1
|
||||
assert watchdogs[0].status == TaskStatus.RUNNING
|
||||
assert "max_cpu=80" in (watchdogs[0].command or "")
|
||||
assert "started" in buf.getvalue()
|
||||
|
||||
|
||||
def test_unwatch_marks_watchdog_cancelled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.load_credentials_from_env",
|
||||
lambda *_a, **_kw: TelegramCredentials(bot_token="x", chat_id="1"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.pid_exists",
|
||||
lambda _pid: True,
|
||||
)
|
||||
|
||||
barrier = threading.Event()
|
||||
|
||||
def _slow_watchdog(**kwargs: object) -> None:
|
||||
task = kwargs["task"]
|
||||
barrier.set()
|
||||
while not task.cancel_requested.is_set():
|
||||
time.sleep(0.02)
|
||||
task.mark_cancelled()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.start_watchdog_daemon_thread",
|
||||
lambda **kw: threading.Thread(target=_slow_watchdog, kwargs=kw, daemon=True).start(),
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
console, _ = _capture()
|
||||
dispatch_slash(
|
||||
f"/watch {__import__('os').getpid()} --interval 1s",
|
||||
session,
|
||||
console,
|
||||
is_tty=True,
|
||||
)
|
||||
assert barrier.wait(timeout=2.0), "watchdog thread should start"
|
||||
task = next(t for t in session.task_registry.list_recent(20) if t.kind == TaskKind.WATCHDOG)
|
||||
dispatch_slash(f"/unwatch {task.task_id}", session, console, is_tty=True)
|
||||
for _ in range(200):
|
||||
task.refresh_rehydrated_status()
|
||||
if task.status == TaskStatus.CANCELLED:
|
||||
break
|
||||
time.sleep(0.02)
|
||||
assert task.status == TaskStatus.CANCELLED
|
||||
|
||||
|
||||
def test_unwatch_rejects_non_watchdog_task() -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
inv = session.task_registry.create(TaskKind.INVESTIGATION, command="x")
|
||||
inv.mark_running()
|
||||
console, buf = _capture()
|
||||
dispatch_slash(f"/unwatch {inv.task_id}", session, console, is_tty=True)
|
||||
assert "not a watchdog" in buf.getvalue()
|
||||
|
||||
|
||||
def test_run_watchdog_respects_cancel(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from platform.common.task_registry import TaskRegistry
|
||||
from tools.system.fleet_monitoring.probe import ProcessSnapshot
|
||||
from tools.system.watch_dog.monitor import run_watchdog
|
||||
|
||||
reg = TaskRegistry()
|
||||
task = reg.create(TaskKind.WATCHDOG, command="watchdog pid=1")
|
||||
task.mark_running()
|
||||
dispatcher = MagicMock()
|
||||
dispatcher.dispatch = MagicMock(return_value=True)
|
||||
|
||||
started_at = datetime.now(UTC) - timedelta(seconds=5)
|
||||
snap = ProcessSnapshot(
|
||||
pid=1,
|
||||
cpu_percent=1.0,
|
||||
rss_mb=10.0,
|
||||
num_fds=None,
|
||||
num_connections=None,
|
||||
status="running",
|
||||
started_at=started_at,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("tools.system.watch_dog.monitor.probe", lambda *_a, **_kw: snap)
|
||||
|
||||
thread = threading.Thread(
|
||||
target=run_watchdog,
|
||||
kwargs={
|
||||
"task": task,
|
||||
"watched_pid": 1,
|
||||
"interval_seconds": 0.15,
|
||||
"max_cpu": None,
|
||||
"max_runtime_seconds": None,
|
||||
"max_rss_mib": None,
|
||||
"once": False,
|
||||
"dispatcher": dispatcher,
|
||||
"on_alarm": None,
|
||||
},
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
time.sleep(0.05)
|
||||
task.request_cancel()
|
||||
thread.join(timeout=3.0)
|
||||
assert task.status == TaskStatus.CANCELLED
|
||||
|
||||
|
||||
def test_run_watchdog_once_without_thresholds_exits(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``--once`` with no threshold flags must finish after one sample (Greptile #1969)."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from platform.common.task_registry import TaskRegistry
|
||||
from tools.system.fleet_monitoring.probe import ProcessSnapshot
|
||||
from tools.system.watch_dog.monitor import run_watchdog
|
||||
|
||||
reg = TaskRegistry()
|
||||
task = reg.create(TaskKind.WATCHDOG, command="watchdog pid=1")
|
||||
task.mark_running()
|
||||
dispatcher = MagicMock()
|
||||
dispatcher.dispatch = MagicMock(return_value=True)
|
||||
|
||||
started_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
snap = ProcessSnapshot(
|
||||
pid=1,
|
||||
cpu_percent=1.0,
|
||||
rss_mb=10.0,
|
||||
num_fds=None,
|
||||
num_connections=None,
|
||||
status="running",
|
||||
started_at=started_at,
|
||||
)
|
||||
monkeypatch.setattr("tools.system.watch_dog.monitor.probe", lambda *_a, **_kw: snap)
|
||||
|
||||
run_watchdog(
|
||||
task=task,
|
||||
watched_pid=1,
|
||||
interval_seconds=0.1,
|
||||
max_cpu=None,
|
||||
max_runtime_seconds=None,
|
||||
max_rss_mib=None,
|
||||
once=True,
|
||||
dispatcher=dispatcher,
|
||||
on_alarm=None,
|
||||
)
|
||||
assert task.status == TaskStatus.COMPLETED
|
||||
assert task.result == "single sample (once)"
|
||||
dispatcher.dispatch.assert_not_called()
|
||||
@@ -0,0 +1,113 @@
|
||||
"""End-to-end style demo for REPL watchdog slash commands.
|
||||
|
||||
This file is the reviewer-facing "proper e2e demo": it drives the same
|
||||
``dispatch_slash`` path as the live REPL (trust, /watch, /watches, /tasks,
|
||||
/unwatch) with a real background watchdog thread and a stubbed ``probe`` so
|
||||
CI stays deterministic and offline.
|
||||
|
||||
For the GitHub **Demo/Screenshot** box, paste the steps from ``docs/DEVELOPMENT.md``
|
||||
(**Interactive shell: REPL watchdog demo**) or ``repl_watchdog_demo.md`` in this directory.
|
||||
|
||||
Run only this module::
|
||||
|
||||
uv run pytest tests/interactive_shell/test_watchdog_repl_e2e_demo.py -v --tb=short
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from integrations.telegram.credentials import TelegramCredentials
|
||||
from platform.common.task_types import TaskKind, TaskStatus
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from tools.system.fleet_monitoring.probe import ProcessSnapshot
|
||||
|
||||
|
||||
def _capture() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
|
||||
def test_repl_watchdog_end_to_end_demo_script(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Drive the full REPL slash pipeline for watchdogs (deterministic probe stub)."""
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.load_credentials_from_env",
|
||||
lambda *_a, **_kw: TelegramCredentials(bot_token="demo-token", chat_id="1"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.command_registry.watch_cmds.pid_exists",
|
||||
lambda _pid: True,
|
||||
)
|
||||
|
||||
pid = os.getpid()
|
||||
started_at = datetime.now(UTC) - timedelta(seconds=30)
|
||||
snap = ProcessSnapshot(
|
||||
pid=pid,
|
||||
cpu_percent=5.0,
|
||||
rss_mb=64.0,
|
||||
num_fds=None,
|
||||
num_connections=None,
|
||||
status="running",
|
||||
started_at=started_at,
|
||||
)
|
||||
monkeypatch.setattr("tools.system.watch_dog.monitor.probe", lambda *_a, **_kw: snap)
|
||||
|
||||
session = Session()
|
||||
console, buf = _capture()
|
||||
|
||||
assert dispatch_slash("/trust on", session, console, is_tty=True) is True
|
||||
|
||||
assert (
|
||||
dispatch_slash(
|
||||
f"/watch {pid} --max-cpu 80 --interval 0.15",
|
||||
session,
|
||||
console,
|
||||
is_tty=True,
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert "started" in buf.getvalue()
|
||||
|
||||
task = next(t for t in session.task_registry.list_recent(50) if t.kind == TaskKind.WATCHDOG)
|
||||
task_id = task.task_id
|
||||
|
||||
for _ in range(40):
|
||||
if task.progress and "cpu=" in (task.progress or ""):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert task.progress, "watchdog thread should publish at least one sample line"
|
||||
|
||||
buf.truncate(0)
|
||||
buf.seek(0)
|
||||
assert dispatch_slash("/watches", session, console, is_tty=True) is True
|
||||
watches_out = buf.getvalue()
|
||||
assert task_id in watches_out
|
||||
assert "Watchdogs" in watches_out
|
||||
assert "running" in watches_out.lower()
|
||||
assert "watchdog" in watches_out.lower()
|
||||
|
||||
buf.truncate(0)
|
||||
buf.seek(0)
|
||||
assert dispatch_slash("/tasks", session, console, is_tty=True) is True
|
||||
tasks_out = buf.getvalue()
|
||||
assert task_id in tasks_out
|
||||
assert "watchdog" in tasks_out.lower()
|
||||
|
||||
assert dispatch_slash(f"/unwatch {task_id}", session, console, is_tty=True) is True
|
||||
for _ in range(80):
|
||||
if task.status == TaskStatus.CANCELLED:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert task.status == TaskStatus.CANCELLED
|
||||
|
||||
buf.truncate(0)
|
||||
buf.seek(0)
|
||||
assert dispatch_slash("/watches", session, console, is_tty=True) is True
|
||||
assert "cancelled" in buf.getvalue().lower()
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Unit tests for the shared REPL execution policy.
|
||||
|
||||
Alpha mode: policy helpers resolve to ``allow`` with no confirmation prompt and
|
||||
there is no command guardrail. The ``ask`` verdict is retained for
|
||||
``trust_mode`` / future opt-in stricter policy, so those paths are covered here
|
||||
by exercising :func:`resolve_confirmation` with explicitly-constructed ``ask`` /
|
||||
``deny`` results.
|
||||
|
||||
This module is pure (no console, no ``input``, no analytics). The interaction
|
||||
layer (``execution_allowed``) and its terminal/analytics behavior are covered by
|
||||
``tests/interactive_shell/ui/test_execution_confirm.py``. Shell-specific policy
|
||||
lives in ``tools.interactive_shell.shell.policy`` and is covered by
|
||||
``tests/interactive_shell/shell/test_policy.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tools.interactive_shell.shared import (
|
||||
ConfirmationOutcome,
|
||||
ExecutionPolicyResult,
|
||||
ToolExecutionMode,
|
||||
ToolExecutionPlan,
|
||||
allow_tool,
|
||||
plan_foreground_tool,
|
||||
resolve_confirmation,
|
||||
)
|
||||
|
||||
|
||||
def _ask_result() -> ExecutionPolicyResult:
|
||||
"""An explicit ``ask`` verdict (the default policy no longer emits these)."""
|
||||
return ExecutionPolicyResult(
|
||||
verdict="ask",
|
||||
tool_type="slash",
|
||||
reason="this command may change configuration or run heavy work",
|
||||
)
|
||||
|
||||
|
||||
# --- Default-allow policy decisions -----------------------------------------
|
||||
|
||||
|
||||
def test_allow_tool_is_allow() -> None:
|
||||
r = allow_tool("slash")
|
||||
assert r.verdict == "allow"
|
||||
assert r.tool_type == "slash"
|
||||
assert r.reason is None
|
||||
|
||||
|
||||
def test_allow_tool_carries_arbitrary_tool_type() -> None:
|
||||
for tool_type in ("investigation", "sample_alert", "synthetic_test", "code_agent"):
|
||||
r = allow_tool(tool_type)
|
||||
assert r.verdict == "allow"
|
||||
assert r.tool_type == tool_type
|
||||
|
||||
|
||||
# --- plan_foreground_tool ---------------------------------------------------
|
||||
|
||||
|
||||
def test_plan_foreground_tool_defaults_classification_to_tool_type() -> None:
|
||||
plan = plan_foreground_tool("slash")
|
||||
assert isinstance(plan, ToolExecutionPlan)
|
||||
assert plan.tool_type == "slash"
|
||||
assert plan.classification == "slash"
|
||||
assert plan.execution_mode is ToolExecutionMode.FOREGROUND
|
||||
assert plan.policy.verdict == "allow"
|
||||
|
||||
|
||||
def test_plan_foreground_tool_accepts_explicit_classification() -> None:
|
||||
plan = plan_foreground_tool("investigation", "investigation_launch")
|
||||
assert plan.tool_type == "investigation"
|
||||
assert plan.classification == "investigation_launch"
|
||||
assert plan.execution_mode is ToolExecutionMode.FOREGROUND
|
||||
assert plan.policy.verdict == "allow"
|
||||
|
||||
|
||||
# --- resolve_confirmation: pure decision (no side effects) ------------------
|
||||
|
||||
|
||||
def test_resolve_allow_verdict_proceeds() -> None:
|
||||
plan = resolve_confirmation(allow_tool("slash"), trust_mode=False, is_tty=True)
|
||||
assert plan.outcome == ConfirmationOutcome.ALLOW
|
||||
assert plan.analytics_outcome == "allowed"
|
||||
|
||||
|
||||
def test_resolve_deny_verdict_blocks() -> None:
|
||||
result = ExecutionPolicyResult(
|
||||
verdict="deny",
|
||||
tool_type="shell",
|
||||
reason="empty command.",
|
||||
hint="Enter a command to run.",
|
||||
)
|
||||
plan = resolve_confirmation(result, trust_mode=False, is_tty=True)
|
||||
assert plan.outcome == ConfirmationOutcome.DENY
|
||||
assert plan.analytics_outcome == "blocked"
|
||||
assert plan.analytics_reason == "empty command."
|
||||
|
||||
|
||||
def test_resolve_ask_trust_mode_allows_without_prompt() -> None:
|
||||
plan = resolve_confirmation(_ask_result(), trust_mode=True, is_tty=True)
|
||||
assert plan.outcome == ConfirmationOutcome.ALLOW
|
||||
assert plan.analytics_outcome == "allowed"
|
||||
assert plan.analytics_reason == "trust_mode_skipped_prompt"
|
||||
|
||||
|
||||
def test_resolve_ask_non_tty_blocks() -> None:
|
||||
plan = resolve_confirmation(_ask_result(), trust_mode=False, is_tty=False)
|
||||
assert plan.outcome == ConfirmationOutcome.BLOCK_NON_TTY
|
||||
assert plan.analytics_outcome == "blocked"
|
||||
assert plan.analytics_reason == "non_interactive_stdin"
|
||||
|
||||
|
||||
def test_resolve_ask_tty_needs_confirmation() -> None:
|
||||
plan = resolve_confirmation(_ask_result(), trust_mode=False, is_tty=True)
|
||||
assert plan.outcome == ConfirmationOutcome.NEEDS_CONFIRMATION
|
||||
# The analytics outcome for a prompt is decided by the interaction layer.
|
||||
assert plan.analytics_outcome is None
|
||||
assert plan.analytics_reason is None
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Tests for the interactive-shell tool-gathering pass.
|
||||
|
||||
``gather_integration_tool_evidence`` runs a bounded tool-calling loop over the same
|
||||
registered tools the investigation uses and returns the collected outputs as a
|
||||
formatted observation block (or ``None`` when there is nothing to add). These
|
||||
tests exercise the no-tools, executed-results, no-executed, and exception paths
|
||||
without any live LLM by stubbing ``agent_factory`` and monkeypatching tool
|
||||
discovery / LLM load where needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
import core as runtime_module
|
||||
import platform.harness_ports as harness_ports
|
||||
from core.agent_harness.turns.evidence_driver import GatherAgentFactory
|
||||
from core.llm.types import ToolCall
|
||||
from surfaces.interactive_shell.runtime.integration_tool_gathering import (
|
||||
_format_gathering_progress_line,
|
||||
_resolve_gather_integrations,
|
||||
_tool_input_hint,
|
||||
gather_integration_tool_evidence,
|
||||
)
|
||||
from surfaces.interactive_shell.session import Session
|
||||
|
||||
_FakeRun = Callable[[dict[str, Any], list[dict[str, Any]]], runtime_module.AgentRunResult]
|
||||
|
||||
|
||||
def _console() -> Console:
|
||||
return Console(file=io.StringIO(), force_terminal=False, color_system=None, width=80)
|
||||
|
||||
|
||||
class _DummyTool:
|
||||
def __init__(self, name: str, source: str = "github") -> None:
|
||||
self.name = name
|
||||
self.source = source
|
||||
|
||||
|
||||
def _stub_agent_factory(run: _FakeRun) -> GatherAgentFactory:
|
||||
"""Return a factory that runs real gather setup but stubs ``Agent.run``."""
|
||||
|
||||
class _StubAgent:
|
||||
def __init__(self, on_runtime_event: Any) -> None:
|
||||
self._on_runtime_event = on_runtime_event
|
||||
|
||||
def run(self, initial_messages: list[dict[str, Any]]) -> runtime_module.AgentRunResult:
|
||||
kwargs = {"on_runtime_event": self._on_runtime_event}
|
||||
return run(kwargs, initial_messages)
|
||||
|
||||
def factory(
|
||||
*,
|
||||
llm: Any,
|
||||
session: Session,
|
||||
gather_tools: list[Any],
|
||||
resolved: dict[str, Any],
|
||||
on_progress: Any,
|
||||
) -> _StubAgent:
|
||||
_ = (llm, session, gather_tools, resolved)
|
||||
from core.events import runtime_event_callback_from_observer
|
||||
|
||||
return _StubAgent(runtime_event_callback_from_observer(on_progress))
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
def test_no_tools_available_returns_none(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
monkeypatch.setattr(harness_ports, "get_investigation_tools", lambda _resolved: [])
|
||||
|
||||
assert gather_integration_tool_evidence("any question", session, _console()) is None
|
||||
|
||||
|
||||
def test_secondary_only_tools_return_none(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("get_sre_guidance", source="knowledge")],
|
||||
)
|
||||
|
||||
def _unexpected_llm() -> Any:
|
||||
raise AssertionError("knowledge-only tools should not invoke the gather loop")
|
||||
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: _unexpected_llm())
|
||||
|
||||
assert gather_integration_tool_evidence("why did it fail?", session, _console()) is None
|
||||
|
||||
|
||||
def test_executed_results_return_formatted_observation(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("search_github_issues")],
|
||||
)
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: object())
|
||||
|
||||
executed = [
|
||||
(
|
||||
ToolCall(id="t1", name="search_github_issues", input={"owner": "o", "repo": "r"}),
|
||||
{"issues": ["#1", "#2"]},
|
||||
)
|
||||
]
|
||||
|
||||
def _fake_run(
|
||||
_kwargs: dict[str, Any], _initial_messages: list[dict[str, Any]]
|
||||
) -> runtime_module.AgentRunResult:
|
||||
return runtime_module.AgentRunResult(messages=[], final_text="", executed=executed)
|
||||
|
||||
observation = gather_integration_tool_evidence(
|
||||
"any open issues?",
|
||||
session,
|
||||
_console(),
|
||||
agent_factory=_stub_agent_factory(_fake_run),
|
||||
)
|
||||
|
||||
assert observation is not None
|
||||
assert "search_github_issues" in observation
|
||||
assert '"owner": "o"' in observation
|
||||
assert '"repo": "r"' in observation
|
||||
|
||||
|
||||
def test_no_executed_returns_none(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("search_github_issues")],
|
||||
)
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: object())
|
||||
|
||||
def _fake_run(
|
||||
_kwargs: dict[str, Any], _initial_messages: list[dict[str, Any]]
|
||||
) -> runtime_module.AgentRunResult:
|
||||
return runtime_module.AgentRunResult(messages=[], final_text="nothing to do", executed=[])
|
||||
|
||||
assert (
|
||||
gather_integration_tool_evidence(
|
||||
"any question",
|
||||
session,
|
||||
_console(),
|
||||
agent_factory=_stub_agent_factory(_fake_run),
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_exception_path_returns_none(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("search_github_issues")],
|
||||
)
|
||||
|
||||
def _boom() -> Any:
|
||||
raise RuntimeError("tool-calling client unavailable")
|
||||
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: _boom())
|
||||
|
||||
assert gather_integration_tool_evidence("any question", session, _console()) is None
|
||||
|
||||
|
||||
def test_tool_input_hint_prefers_distinguishing_fields() -> None:
|
||||
hint = _tool_input_hint(
|
||||
{
|
||||
"grafana_endpoint": "https://example.grafana.net",
|
||||
"metric_name": "sum(rate(http_requests_total[5m]))",
|
||||
"service_name": "checkout-api",
|
||||
}
|
||||
)
|
||||
assert hint == "sum(rate(http_requests_total[5m])) · checkout-api"
|
||||
|
||||
|
||||
def test_format_gathering_progress_line_shows_repeat_index_and_hint() -> None:
|
||||
line = _format_gathering_progress_line(
|
||||
"query_grafana_metrics",
|
||||
{"metric_name": "pipeline_runs_total"},
|
||||
repeat_index=2,
|
||||
)
|
||||
assert line.startswith("· gathering via Grafana · Mimir (2) — pipeline_runs_total…")
|
||||
|
||||
|
||||
def test_format_gathering_progress_line_escapes_display_and_hint_markup(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.runtime.integration_tool_gathering.tool_source_label",
|
||||
lambda _name: "Grafana [prod]",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.runtime.integration_tool_gathering.tool_short_label",
|
||||
lambda _name, _source: "Mimir",
|
||||
)
|
||||
|
||||
line = _format_gathering_progress_line(
|
||||
"query_grafana_metrics",
|
||||
{"metric_name": "[critical] rate[5m]"},
|
||||
repeat_index=1,
|
||||
)
|
||||
console = _console()
|
||||
console.print(f"[dim]{line}[/]")
|
||||
|
||||
output = console.file.getvalue()
|
||||
assert "Grafana [prod]" in output
|
||||
assert "[critical] rate[5m]" in output
|
||||
|
||||
|
||||
def test_gathering_progress_lines_print_on_tool_start(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
console = _console()
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("query_grafana_metrics", source="grafana")],
|
||||
)
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: object())
|
||||
|
||||
def _fake_run(
|
||||
kwargs: dict[str, Any], _initial_messages: list[dict[str, Any]]
|
||||
) -> runtime_module.AgentRunResult:
|
||||
on_runtime_event = kwargs.get("on_runtime_event")
|
||||
if on_runtime_event is not None:
|
||||
on_runtime_event(
|
||||
runtime_module.ToolExecutionStartEvent(
|
||||
tool_call_id="t1",
|
||||
tool_name="query_grafana_metrics",
|
||||
args={"metric_name": "pipeline_runs_total"},
|
||||
iteration=0,
|
||||
)
|
||||
)
|
||||
on_runtime_event(
|
||||
runtime_module.ToolExecutionStartEvent(
|
||||
tool_call_id="t2",
|
||||
tool_name="query_grafana_metrics",
|
||||
args={"metric_name": "http_errors_total"},
|
||||
iteration=0,
|
||||
)
|
||||
)
|
||||
return runtime_module.AgentRunResult(messages=[], final_text="", executed=[])
|
||||
|
||||
gather_integration_tool_evidence(
|
||||
"check metrics",
|
||||
session,
|
||||
console,
|
||||
agent_factory=_stub_agent_factory(_fake_run),
|
||||
)
|
||||
output = console.file.getvalue()
|
||||
assert "Grafana · Mimir — pipeline_runs_total" in output
|
||||
assert "Grafana · Mimir (2) — http_errors_total" in output
|
||||
|
||||
|
||||
def test_resolve_gather_integrations_enriches_github_from_repo_url() -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {
|
||||
"github": {"connection_verified": True, "url": "https://api.githubcopilot.com/mcp/"}
|
||||
}
|
||||
|
||||
resolved = _resolve_gather_integrations(
|
||||
session,
|
||||
"check github issues in https://github.com/Tracer-Cloud/opensre",
|
||||
)
|
||||
|
||||
gh = resolved["github"]
|
||||
assert gh["owner"] == "Tracer-Cloud"
|
||||
assert gh["repo"] == "opensre"
|
||||
assert session.github_repo_scope == ("Tracer-Cloud", "opensre")
|
||||
|
||||
|
||||
def test_resolve_gather_integrations_uses_session_cache_on_follow_up() -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {
|
||||
"github": {"connection_verified": True, "url": "https://api.githubcopilot.com/mcp/"}
|
||||
}
|
||||
session.github_repo_scope = ("Tracer-Cloud", "opensre")
|
||||
session.agent.messages = [
|
||||
("user", "https://github.com/Tracer-Cloud/opensre"),
|
||||
("assistant", "Got it."),
|
||||
]
|
||||
|
||||
resolved = _resolve_gather_integrations(session, "do these searches")
|
||||
|
||||
assert resolved["github"]["owner"] == "Tracer-Cloud"
|
||||
assert resolved["github"]["repo"] == "opensre"
|
||||
|
||||
|
||||
def test_resolve_gather_integrations_uses_passed_turn_view() -> None:
|
||||
"""When the turn's resolved view is supplied, it is the base — no session re-resolve."""
|
||||
session = Session()
|
||||
# The session cache holds a different integration than the turn resolved this turn.
|
||||
session.resolved_integrations_cache = {"datadog": {"connection_verified": True}}
|
||||
turn_resolved = {"slack": {"connection_verified": True}}
|
||||
|
||||
resolved = _resolve_gather_integrations(
|
||||
session, "post an update", resolved_integrations=turn_resolved
|
||||
)
|
||||
|
||||
assert resolved == {"slack": {"connection_verified": True}}
|
||||
|
||||
|
||||
def test_resolve_gather_integrations_applies_github_scope_over_passed_view() -> None:
|
||||
"""GitHub repo scope is still enriched on top of the passed turn view."""
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
turn_resolved = {
|
||||
"github": {"connection_verified": True, "url": "https://api.githubcopilot.com/mcp/"}
|
||||
}
|
||||
|
||||
resolved = _resolve_gather_integrations(
|
||||
session,
|
||||
"check github issues in https://github.com/Tracer-Cloud/opensre",
|
||||
resolved_integrations=turn_resolved,
|
||||
)
|
||||
|
||||
assert resolved["github"]["owner"] == "Tracer-Cloud"
|
||||
assert resolved["github"]["repo"] == "opensre"
|
||||
|
||||
|
||||
def test_gather_enriches_github_before_selecting_tools(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {
|
||||
"github": {"connection_verified": True, "url": "https://api.githubcopilot.com/mcp/"}
|
||||
}
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def _capture_tools(resolved: dict[str, Any]) -> list[_DummyTool]:
|
||||
seen["resolved"] = resolved
|
||||
gh = resolved.get("github", {})
|
||||
if isinstance(gh, dict) and gh.get("owner") and gh.get("repo"):
|
||||
return [_DummyTool("search_github_issues")]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(harness_ports, "get_investigation_tools", _capture_tools)
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: object())
|
||||
|
||||
def _fake_run(
|
||||
_kwargs: dict[str, Any], _initial_messages: list[dict[str, Any]]
|
||||
) -> runtime_module.AgentRunResult:
|
||||
return runtime_module.AgentRunResult(messages=[], final_text="", executed=[])
|
||||
|
||||
gather_integration_tool_evidence(
|
||||
"check github issues in https://github.com/Tracer-Cloud/opensre",
|
||||
session,
|
||||
_console(),
|
||||
agent_factory=_stub_agent_factory(_fake_run),
|
||||
)
|
||||
|
||||
gh = seen["resolved"]["github"]
|
||||
assert gh["owner"] == "Tracer-Cloud"
|
||||
assert gh["repo"] == "opensre"
|
||||
|
||||
|
||||
def test_gather_user_message_includes_recent_conversation(monkeypatch: Any) -> None:
|
||||
session = Session()
|
||||
session.resolved_integrations_cache = {}
|
||||
session.agent.messages = [("user", "prior question"), ("assistant", "prior answer")]
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
harness_ports,
|
||||
"get_investigation_tools",
|
||||
lambda _resolved: [_DummyTool("search_github_issues")],
|
||||
)
|
||||
monkeypatch.setattr("core.llm.factory.get_llm", lambda _role: object())
|
||||
|
||||
def _fake_run(
|
||||
_kwargs: dict[str, Any], initial_messages: list[dict[str, Any]]
|
||||
) -> runtime_module.AgentRunResult:
|
||||
captured["messages"] = initial_messages
|
||||
return runtime_module.AgentRunResult(messages=[], final_text="", executed=[])
|
||||
|
||||
gather_integration_tool_evidence(
|
||||
"follow up",
|
||||
session,
|
||||
_console(),
|
||||
agent_factory=_stub_agent_factory(_fake_run),
|
||||
)
|
||||
|
||||
content = captured["messages"][0]["content"]
|
||||
assert "Recent conversation:" in content
|
||||
assert "prior question" in content
|
||||
assert "Current question:\nfollow up" in content
|
||||
@@ -0,0 +1 @@
|
||||
# Package marker for mirrored tests.
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Tests for interactive-shell action rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
import tools.interactive_shell.actions.slash as slash_tool
|
||||
from core.agent_harness.turns.turn_results import ToolCallingTurnResult
|
||||
from surfaces.interactive_shell.runtime.action_turn import run_action_tool_turn
|
||||
from surfaces.interactive_shell.runtime.shell_turn_execution import execute_shell_turn
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.action_rendering import ActionRenderObserver
|
||||
from surfaces.interactive_shell.ui.input_prompt.rendering import _prompt_turn_number
|
||||
from tests.core.agent.orchestration.action_execution_test_harness import (
|
||||
ActionExecutionHarness,
|
||||
FakeActionLLM,
|
||||
no_tool_response,
|
||||
)
|
||||
|
||||
|
||||
def test_slash_invoke_tool_start_does_not_record_cli_agent() -> None:
|
||||
session = Session()
|
||||
buffer = io.StringIO()
|
||||
console = Console(file=buffer, force_terminal=False, highlight=False)
|
||||
observer = ActionRenderObserver(session=session, console=console, message="/model show")
|
||||
|
||||
observer(
|
||||
"tool_start",
|
||||
{"name": "slash_invoke", "input": {"command": "/model", "args": ["show"]}},
|
||||
)
|
||||
|
||||
assert session.history == []
|
||||
assert observer.planned_count == 1
|
||||
assert buffer.getvalue() == ""
|
||||
|
||||
|
||||
def test_shell_run_tool_start_does_not_record_cli_agent() -> None:
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
observer = ActionRenderObserver(session=session, console=console, message="!true")
|
||||
|
||||
observer("tool_start", {"name": "shell_run", "input": {"command": "true"}})
|
||||
|
||||
assert session.history == []
|
||||
assert observer.planned_count == 1
|
||||
|
||||
|
||||
def test_literal_slash_command_records_single_history_entry(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
dispatched: list[str] = []
|
||||
|
||||
def _fake_dispatch(
|
||||
command: str,
|
||||
session: Session,
|
||||
console: Console,
|
||||
**_kwargs: object,
|
||||
) -> bool:
|
||||
dispatched.append(command)
|
||||
session.record("slash", command, ok=True)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(slash_tool, "dispatch_slash", _fake_dispatch)
|
||||
session = Session()
|
||||
harness = ActionExecutionHarness(llm=FakeActionLLM([no_tool_response()]))
|
||||
|
||||
result = run_action_tool_turn(
|
||||
"/model show",
|
||||
session,
|
||||
harness.console,
|
||||
deps=harness.deps,
|
||||
)
|
||||
|
||||
assert result.handled is True
|
||||
assert dispatched == ["/model show"]
|
||||
assert session.history == [{"type": "slash", "text": "/model show", "ok": True}]
|
||||
assert _prompt_turn_number(session) == 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeLlmRun:
|
||||
response_text: str = "hello back"
|
||||
|
||||
|
||||
def test_chat_turn_records_single_cli_agent_history_entry() -> None:
|
||||
session = Session()
|
||||
console = Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
def _no_actions(
|
||||
_text: str,
|
||||
_session: Session,
|
||||
_console: Console,
|
||||
**kwargs: object,
|
||||
) -> ToolCallingTurnResult:
|
||||
return ToolCallingTurnResult(
|
||||
planned_count=0,
|
||||
executed_count=0,
|
||||
executed_success_count=0,
|
||||
has_unhandled_clause=False,
|
||||
handled=False,
|
||||
accounting_status="not_run",
|
||||
)
|
||||
|
||||
def _answer(
|
||||
_text: str,
|
||||
_session: Session,
|
||||
_console: Console,
|
||||
**kwargs: object,
|
||||
) -> _FakeLlmRun:
|
||||
return _FakeLlmRun()
|
||||
|
||||
execute_shell_turn(
|
||||
"what broke in prod?",
|
||||
session,
|
||||
console,
|
||||
recorder=None,
|
||||
execute_actions=_no_actions,
|
||||
answer_agent=_answer,
|
||||
)
|
||||
|
||||
assert session.history == [{"type": "cli_agent", "text": "what broke in prod?", "ok": True}]
|
||||
assert _prompt_turn_number(session) == 2
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Pure rendering tests for the ``/fleet`` dashboard table (issue #1488).
|
||||
|
||||
These cover ``render_agents_table`` in isolation — no slash-command
|
||||
dispatch, no real registry I/O. The integration tests in
|
||||
``test_agents_commands.py`` cover the dispatch path that consumes
|
||||
this function.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from surfaces.interactive_shell.ui.agents import agents_view as agents_view_mod
|
||||
from surfaces.interactive_shell.ui.agents.agents_view import _build_agents_table
|
||||
from tools.system.fleet_monitoring import config as config_mod
|
||||
from tools.system.fleet_monitoring.probe import ProcessSnapshot
|
||||
from tools.system.fleet_monitoring.registry import AgentRecord
|
||||
from tools.system.fleet_monitoring.token_rate import TOKEN_RATE_TRACKER
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_agents_yaml(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""Autouse: redirect ``agents_config_path`` to a per-test tmp file
|
||||
so the rendering tests don't read the developer's real
|
||||
``~/.opensre/agents.yaml`` (which would let real budgets
|
||||
leak into the placeholder assertions and create cross-machine
|
||||
flakes).
|
||||
"""
|
||||
target = tmp_path / "agents.yaml"
|
||||
monkeypatch.setattr(config_mod, "agents_config_path", lambda: target)
|
||||
return target
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_sampler_module_state() -> None:
|
||||
"""Reset module-level state in the sampler before each test.
|
||||
|
||||
Three globals can leak across test files:
|
||||
|
||||
- :data:`tools.system.fleet_monitoring.sampler._latest` (probe snapshots dict)
|
||||
- :data:`tools.system.fleet_monitoring.token_rate.TOKEN_RATE_TRACKER` (per-PID deque)
|
||||
- :data:`tools.system.fleet_monitoring.sampler._TickCache.registry_snapshot` and
|
||||
:data:`_TickCache.agents_config` (tick caches added in #2023 to
|
||||
avoid re-reading the disk on every render)
|
||||
|
||||
Sampler tests populate all of them; view tests must start clean
|
||||
so placeholder assertions are stable under ``pytest-xdist`` and
|
||||
in arbitrary alphabetical order.
|
||||
"""
|
||||
from tools.system.fleet_monitoring import sampler as sampler_mod
|
||||
|
||||
sampler_mod._latest.clear()
|
||||
sampler_mod._TickCache.registry_snapshot = {}
|
||||
sampler_mod._TickCache.agents_config = None
|
||||
for pid in list(TOKEN_RATE_TRACKER.known_pids()):
|
||||
TOKEN_RATE_TRACKER.forget(pid)
|
||||
|
||||
|
||||
# The columns this PR ships are the contract for #1490 and later
|
||||
# tickets that thread snapshot data into the rendering layer; pin
|
||||
# them here so a downstream reorder doesn't silently break the
|
||||
# dashboard preview.
|
||||
_DASHBOARD_COLUMNS: tuple[str, ...] = (
|
||||
"agent",
|
||||
"pid",
|
||||
"uptime",
|
||||
"cpu%",
|
||||
"tokens/min",
|
||||
"$/hr",
|
||||
"status",
|
||||
)
|
||||
|
||||
|
||||
def _render(records: list[AgentRecord]) -> tuple[Table, str]:
|
||||
"""Build the table and capture the printed form for substring assertions."""
|
||||
table = _build_agents_table(records)
|
||||
buf = io.StringIO()
|
||||
Console(file=buf, force_terminal=False, highlight=False, width=120).print(table)
|
||||
return table, buf.getvalue()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column structure — the contract downstream tickets lean on
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_table_has_full_dashboard_column_set_in_documented_order() -> None:
|
||||
table, _ = _render([])
|
||||
headers = tuple(str(col.header) for col in table.columns)
|
||||
assert headers == _DASHBOARD_COLUMNS
|
||||
|
||||
|
||||
def test_pid_column_is_right_justified_to_match_numeric_dashboard_preview() -> None:
|
||||
"""Numeric columns (pid, uptime, cpu%, tokens/min, $/hr) are
|
||||
right-justified; agent name and status are left-justified. This
|
||||
matches the spacing in the issue's mock and keeps later
|
||||
snapshot-injected cells aligned without re-styling."""
|
||||
table, _ = _render([])
|
||||
by_header = {str(col.header): col for col in table.columns}
|
||||
assert by_header["pid"].justify == "right"
|
||||
assert by_header["uptime"].justify == "right"
|
||||
assert by_header["cpu%"].justify == "right"
|
||||
assert by_header["tokens/min"].justify == "right"
|
||||
assert by_header["$/hr"].justify == "right"
|
||||
assert by_header["agent"].justify == "left"
|
||||
assert by_header["status"].justify == "left"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Empty state
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_empty_records_renders_table_with_zero_rows() -> None:
|
||||
table, _ = _render([])
|
||||
assert table.row_count == 0
|
||||
|
||||
|
||||
def test_empty_records_caption_announces_empty_state() -> None:
|
||||
"""Empty-state UX: the table caption tells the user the fleet
|
||||
is empty rather than leaving a blank table that looks like a bug.
|
||||
"""
|
||||
_, out = _render([])
|
||||
assert "no agents discovered or registered yet" in out
|
||||
|
||||
|
||||
def test_non_empty_records_have_no_caption() -> None:
|
||||
"""When the registry has rows, the caption is suppressed —
|
||||
the table content speaks for itself and a caption would be noise."""
|
||||
table, _ = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
assert table.caption is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Row content
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_row_contains_agent_name_and_pid() -> None:
|
||||
_, out = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
assert "claude-code" in out
|
||||
assert "8421" in out
|
||||
|
||||
|
||||
def test_metric_cells_are_placeholders_when_no_sampler_data() -> None:
|
||||
"""All five metric cells render ``-`` when the sampler has no
|
||||
data for the PID (REPL not running, non-interactive ``opensre
|
||||
agents list``, or fresh registration that has not yet been
|
||||
sampled). #2023 split ``tokens/min`` and ``$/hr`` from the
|
||||
yaml budget; both still fall back to ``-`` here for the same
|
||||
"no data" reason.
|
||||
"""
|
||||
table, _ = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
# row_count == 1, so iterate directly to inspect the rendered cells
|
||||
assert table.row_count == 1
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
# cells[0] = agent, cells[1] = pid, then metric cells and status.
|
||||
assert rendered_cells[2:] == ["-", "-", "-", "-", "-"]
|
||||
|
||||
|
||||
def test_table_shows_live_probe_data_when_snapshot_exists(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
fixed_now = datetime(2026, 5, 10, 14, 0, 0, tzinfo=UTC)
|
||||
started_at = datetime(2026, 5, 10, 12, 0, 0, tzinfo=UTC) # exactly 2h before
|
||||
|
||||
class FrozenDatetime(datetime):
|
||||
@classmethod
|
||||
def now(cls, _tz=None): # type: ignore[override]
|
||||
return fixed_now
|
||||
|
||||
monkeypatch.setattr(agents_view_mod, "datetime", FrozenDatetime)
|
||||
|
||||
fake_snapshot = ProcessSnapshot(
|
||||
pid=8421,
|
||||
cpu_percent=23.5,
|
||||
rss_mb=128.0,
|
||||
num_fds=42,
|
||||
num_connections=3,
|
||||
status="running",
|
||||
started_at=started_at,
|
||||
)
|
||||
monkeypatch.setattr(agents_view_mod, "get_snapshot", lambda _pid: fake_snapshot)
|
||||
|
||||
table, _ = _render([AgentRecord(name="cursor", pid=8444, command="cursor")])
|
||||
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
# ``tokens/min`` and ``$/hr`` are still ``-`` because no token
|
||||
# tracker entry exists for this PID — the snapshot fixture covers
|
||||
# only the resource side. The full live-data case is covered by
|
||||
# ``test_table_shows_tokens_and_cost_when_tracker_has_data``.
|
||||
# Without recent output activity, the status heuristic falls back
|
||||
# to the process start time and renders STUCK with a progress-time annotation.
|
||||
assert rendered_cells[2:] == ["2h0m", "23.5", "-", "-", "[red]stuck (2h0m no progress)[/red]"]
|
||||
|
||||
|
||||
def test_recent_agent_output_keeps_status_active(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
fixed_now = datetime(2026, 5, 10, 14, 0, 0, tzinfo=UTC)
|
||||
|
||||
class FrozenDatetime(datetime):
|
||||
@classmethod
|
||||
def now(cls, _tz=None): # type: ignore[override]
|
||||
return fixed_now
|
||||
|
||||
monkeypatch.setattr(agents_view_mod, "datetime", FrozenDatetime)
|
||||
|
||||
fake_snapshot = ProcessSnapshot(
|
||||
pid=8421,
|
||||
cpu_percent=23.5,
|
||||
rss_mb=128.0,
|
||||
num_fds=42,
|
||||
num_connections=3,
|
||||
status="running",
|
||||
started_at=fixed_now - timedelta(hours=2),
|
||||
last_output_at=fixed_now - timedelta(seconds=30),
|
||||
)
|
||||
monkeypatch.setattr(agents_view_mod, "get_snapshot", lambda _pid: fake_snapshot)
|
||||
|
||||
table, _ = _render([AgentRecord(name="cursor", pid=8444, command="cursor")])
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
|
||||
assert rendered_cells[6] == "[green]active[/green]"
|
||||
|
||||
|
||||
def test_stuck_message_anchors_to_last_output_at_not_started_at(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fixed_now = datetime(2026, 5, 10, 14, 0, 0, tzinfo=UTC)
|
||||
|
||||
class FrozenDatetime(datetime):
|
||||
@classmethod
|
||||
def now(cls, _tz=None): # type: ignore[override]
|
||||
return fixed_now
|
||||
|
||||
monkeypatch.setattr(agents_view_mod, "datetime", FrozenDatetime)
|
||||
|
||||
fake_snapshot = ProcessSnapshot(
|
||||
pid=8421,
|
||||
cpu_percent=5.0,
|
||||
rss_mb=64.0,
|
||||
num_fds=10,
|
||||
num_connections=1,
|
||||
status="running",
|
||||
started_at=fixed_now - timedelta(hours=3),
|
||||
last_output_at=fixed_now - timedelta(minutes=10),
|
||||
)
|
||||
monkeypatch.setattr(agents_view_mod, "get_snapshot", lambda _pid: fake_snapshot)
|
||||
|
||||
table, _ = _render([AgentRecord(name="cursor", pid=8444, command="cursor")])
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
|
||||
assert rendered_cells[6] == "[red]stuck (10m no progress)[/red]"
|
||||
|
||||
|
||||
def test_table_shows_tokens_and_cost_when_tracker_has_data(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""End-to-end of the #2023 wiring on the view side: when the
|
||||
sampler's accessors return live numbers, the view formats them
|
||||
into the ``tokens/min`` and ``$/hr`` columns.
|
||||
"""
|
||||
monkeypatch.setattr(agents_view_mod, "get_tokens_per_min", lambda _pid: 120.0)
|
||||
monkeypatch.setattr(agents_view_mod, "get_usd_per_hour", lambda _pid: 0.27)
|
||||
|
||||
table, _ = _render([AgentRecord(name="claude-code-8421", pid=8421, command="claude")])
|
||||
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
# cells[4] = tokens/min, cells[5] = $/hr
|
||||
assert rendered_cells[4] == "120"
|
||||
assert rendered_cells[5] == "$0.27"
|
||||
|
||||
|
||||
def test_tokens_per_min_formatter_uses_k_suffix_above_1000(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A busy session emitting > 1k tokens/min must not blow out the
|
||||
column width; the formatter falls back to ``1.2k`` shorthand.
|
||||
"""
|
||||
monkeypatch.setattr(agents_view_mod, "get_tokens_per_min", lambda _pid: 1234.0)
|
||||
monkeypatch.setattr(agents_view_mod, "get_usd_per_hour", lambda _pid: None)
|
||||
|
||||
table, _ = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
assert rendered_cells[4] == "1.2k"
|
||||
|
||||
|
||||
def test_idle_observed_agent_renders_zero_tokens_per_min_not_dash(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The ``0`` vs ``-`` distinction the tracker enforces must
|
||||
propagate to the cell: a known-but-idle agent renders ``0``,
|
||||
a never-observed agent renders ``-``.
|
||||
"""
|
||||
monkeypatch.setattr(agents_view_mod, "get_tokens_per_min", lambda _pid: 0.0)
|
||||
monkeypatch.setattr(agents_view_mod, "get_usd_per_hour", lambda _pid: 0.0)
|
||||
|
||||
table, _ = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
assert rendered_cells[4] == "0"
|
||||
assert rendered_cells[5] == "$0.00"
|
||||
|
||||
|
||||
def test_usd_per_hour_renders_dash_when_model_unknown(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""``get_usd_per_hour`` returns ``None`` when the model cannot
|
||||
be resolved (and therefore pricing cannot be applied). The cell
|
||||
must render ``-`` rather than ``$0.00`` so the user does not
|
||||
misread "unknown" as "free".
|
||||
"""
|
||||
monkeypatch.setattr(agents_view_mod, "get_tokens_per_min", lambda _pid: 500.0)
|
||||
monkeypatch.setattr(agents_view_mod, "get_usd_per_hour", lambda _pid: None)
|
||||
|
||||
table, _ = _render([AgentRecord(name="claude-code", pid=8421, command="claude")])
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
# tokens/min still shows the live figure …
|
||||
assert rendered_cells[4] == "500"
|
||||
# … but $/hr is honest about the missing rate.
|
||||
assert rendered_cells[5] == "-"
|
||||
|
||||
|
||||
def test_multiple_records_are_each_rendered_in_order() -> None:
|
||||
records = [
|
||||
AgentRecord(name="claude-code", pid=8421, command="claude"),
|
||||
AgentRecord(name="cursor-tab", pid=9133, command="cursor"),
|
||||
AgentRecord(name="aider", pid=7702, command="aider"),
|
||||
]
|
||||
table, out = _render(records)
|
||||
assert table.row_count == 3
|
||||
# Substring order in the rendered output preserves input order:
|
||||
pos_claude = out.index("claude-code")
|
||||
pos_cursor = out.index("cursor-tab")
|
||||
pos_aider = out.index("aider")
|
||||
assert pos_claude < pos_cursor < pos_aider
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defense against Rich-markup injection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_record_name_is_rich_escaped_so_markup_does_not_render() -> None:
|
||||
"""An adversarial agent name containing Rich markup tags must
|
||||
render literally, not interpreted. Without ``escape()``, a name
|
||||
like ``[bold red]ghost[/]`` would visually mimic a styled cell
|
||||
and could mask other dashboard content."""
|
||||
records = [
|
||||
AgentRecord(name="[bold red]ghost[/]", pid=1, command="bin"),
|
||||
]
|
||||
_, out = _render(records)
|
||||
# Literal brackets survive in the rendered output:
|
||||
assert "[bold red]ghost[/]" in out
|
||||
|
||||
|
||||
# Resilience to a schema-invalid ``agents.yaml`` is now enforced by
|
||||
# the sampler's catch-all around ``load_agents_config`` (see
|
||||
# ``test_sampler.py``); the view no longer touches the config.
|
||||
@@ -0,0 +1,318 @@
|
||||
"""End-to-end integration tests for the ``/fleet`` token wiring (#2023).
|
||||
|
||||
These tests drive the *real* pipeline — provider resolver, real
|
||||
:class:`ClaudeCodeJsonlSource` / :class:`CodexRolloutSource`, real
|
||||
:class:`ClaudeCodeMeter` / :class:`CodexMeter`, real
|
||||
:class:`TokenRateTracker`, real ``_resolved_model_for_pid``, real
|
||||
``_format_tokens_per_min`` / ``_format_usd_per_hour`` — against a
|
||||
fixture on disk.
|
||||
|
||||
The only stubs are :func:`tools.system.fleet_monitoring.probe.cwd_for_pid` and
|
||||
:func:`started_at_for_pid` (no real PIDs in a unit test) and the
|
||||
``AgentRegistry`` lookup (no real ``~/.opensre/agents.jsonl``).
|
||||
Everything else exercises production code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.ui.agents.agents_view import _build_agents_table
|
||||
from tools.system.fleet_monitoring import sampler as sampler_mod
|
||||
from tools.system.fleet_monitoring.probe import ProcessSnapshot
|
||||
from tools.system.fleet_monitoring.registry import AgentRecord
|
||||
from tools.system.fleet_monitoring.token_rate import TOKEN_RATE_TRACKER
|
||||
from tools.system.fleet_monitoring.token_sources import claude_code as claude_source_mod
|
||||
from tools.system.fleet_monitoring.token_sources import codex as codex_source_mod
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_sampler_globals() -> None:
|
||||
"""Same isolation pattern as the unit tests in this directory."""
|
||||
sampler_mod._latest.clear()
|
||||
sampler_mod._TickCache.registry_snapshot = {}
|
||||
sampler_mod._TickCache.agents_config = None
|
||||
for pid in list(TOKEN_RATE_TRACKER.known_pids()):
|
||||
TOKEN_RATE_TRACKER.forget(pid)
|
||||
|
||||
|
||||
def _render(records: list[AgentRecord]) -> str:
|
||||
table = _build_agents_table(records)
|
||||
buf = io.StringIO()
|
||||
Console(file=buf, force_terminal=False, highlight=False, width=140).print(table)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ---------- Claude Code end-to-end ----------------------------------
|
||||
|
||||
|
||||
def test_claude_code_end_to_end_renders_real_tokens_and_cost(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Real pipeline: psutil-stubbed cwd → real JSONL on disk →
|
||||
real source → real meter → real tracker → real view formatter.
|
||||
The view shows live ``tokens/min`` and ``$/hr`` cells, not ``-``.
|
||||
"""
|
||||
# Build a Claude Code project directory and session JSONL.
|
||||
fake_cwd = tmp_path / "repo"
|
||||
fake_cwd.mkdir()
|
||||
projects_root = tmp_path / "claude_projects"
|
||||
project_dir = projects_root / str(fake_cwd).replace("/", "-")
|
||||
project_dir.mkdir(parents=True)
|
||||
session = project_dir / "session-abc.jsonl"
|
||||
# Start with one historical event — the source seeks past it on
|
||||
# first read (no retro-pricing) so it should not contribute.
|
||||
session.write_text(
|
||||
'{"type":"system","subtype":"init","session_id":"abc","model":"claude-sonnet-4-5"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Stub psutil-fenced helpers — no real PID lookups in unit tests.
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.token_sources.claude_code.cwd_for_pid",
|
||||
lambda _pid: fake_cwd,
|
||||
)
|
||||
# After the Greptile-driven fix, the claude source requires
|
||||
# fd-level evidence to attribute a JSONL to a PID (no more
|
||||
# silent fallback to newest-by-mtime — mirrors codex).
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.token_sources.claude_code.open_files_for_pid",
|
||||
lambda _pid: (session,),
|
||||
)
|
||||
|
||||
# Wire a fresh source against the test projects root so the
|
||||
# production singleton (which would hit ``~/.claude/projects/``)
|
||||
# never enters this test's resolution path.
|
||||
isolated_source = claude_source_mod.ClaudeCodeJsonlSource(projects_root=projects_root)
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.sampler.get_token_source",
|
||||
lambda provider: isolated_source if provider == "claude-code" else None,
|
||||
)
|
||||
|
||||
# Registered agent: Claude Code at PID 8421.
|
||||
record = AgentRecord(
|
||||
name="claude-code-8421",
|
||||
pid=8421,
|
||||
command="claude --output-format stream-json",
|
||||
provider="claude-code",
|
||||
)
|
||||
|
||||
class _Registry:
|
||||
def list(self) -> list[AgentRecord]:
|
||||
return [record]
|
||||
|
||||
def get(self, pid: int) -> AgentRecord | None:
|
||||
return record if pid == 8421 else None
|
||||
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", _Registry)
|
||||
|
||||
# Stub probe so the snapshot path is also alive (uptime/cpu%).
|
||||
fake_snapshot = ProcessSnapshot(
|
||||
pid=8421,
|
||||
cpu_percent=18.4,
|
||||
rss_mb=128.0,
|
||||
num_fds=42,
|
||||
num_connections=3,
|
||||
status="running",
|
||||
started_at=datetime(2026, 5, 14, 10, 0, 0, tzinfo=UTC),
|
||||
)
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot)
|
||||
|
||||
# First sampler tick: cold-start the source (seeks to EOF) and
|
||||
# populate registry caches. No tokens yet — historical content
|
||||
# below EOF is correctly ignored.
|
||||
sampler_mod._TickCache.registry_snapshot[record.pid] = record
|
||||
sampler_mod._TickCache.agents_config = None
|
||||
sampler_mod._sample_tokens(record)
|
||||
|
||||
# Append a real Claude assistant turn with usage. This is the
|
||||
# delta the next sample call should pick up.
|
||||
with session.open("a", encoding="utf-8") as fh:
|
||||
fh.write(
|
||||
'{"type":"assistant","message":{"model":"claude-sonnet-4-5",'
|
||||
'"usage":{"input_tokens":200,"output_tokens":50}}}\n'
|
||||
)
|
||||
# Brief wait so mtime > started_at - 5s on every filesystem
|
||||
# (APFS sub-second mtime quirks rarely matter, but cheap).
|
||||
time.sleep(0.01)
|
||||
|
||||
sampler_mod._sample_tokens(record)
|
||||
|
||||
# Render through the real view.
|
||||
out = _render([record])
|
||||
|
||||
# ``tokens/min`` populated: 250 tokens in a 60 s window → 250.
|
||||
assert "250" in out
|
||||
|
||||
# ``$/hr`` populated with a real dollar figure for sonnet-4-5
|
||||
# (3 USD/M input × 0.7 + 15 USD/M output × 0.3 blend ≈ 6.6e-6
|
||||
# per token; 250 tok/min × 60 × 6.6e-6 ≈ $0.099). The actual
|
||||
# rendering rounds to two decimals, so look for the ``$`` prefix.
|
||||
assert "$0.0" in out or "$0.1" in out
|
||||
|
||||
# And the placeholder is gone for the metric columns.
|
||||
assert "claude-code-8421" in out
|
||||
|
||||
|
||||
def test_claude_code_end_to_end_shows_dash_when_source_resolves_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""When psutil cannot read the cwd (macOS hardened-runtime
|
||||
denials, cross-user processes), the dashboard stays honest:
|
||||
``tokens/min`` and ``$/hr`` render ``-`` rather than ``0`` or a
|
||||
bogus number.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.token_sources.claude_code.cwd_for_pid",
|
||||
lambda _pid: None,
|
||||
)
|
||||
isolated_source = claude_source_mod.ClaudeCodeJsonlSource(
|
||||
projects_root=tmp_path / "claude_projects",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.sampler.get_token_source",
|
||||
lambda provider: isolated_source if provider == "claude-code" else None,
|
||||
)
|
||||
|
||||
record = AgentRecord(
|
||||
name="claude-code-8421",
|
||||
pid=8421,
|
||||
command="claude",
|
||||
provider="claude-code",
|
||||
)
|
||||
|
||||
class _Registry:
|
||||
def list(self) -> list[AgentRecord]:
|
||||
return [record]
|
||||
|
||||
def get(self, pid: int) -> AgentRecord | None:
|
||||
return record if pid == 8421 else None
|
||||
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", _Registry)
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: None)
|
||||
|
||||
sampler_mod._TickCache.registry_snapshot[record.pid] = record
|
||||
sampler_mod._sample_tokens(record)
|
||||
sampler_mod._sample_tokens(record)
|
||||
|
||||
# Drive the real render and inspect the actual table cells rather
|
||||
# than substring-matching the printed form (which contains the
|
||||
# ``$/hr`` header literal).
|
||||
table = _build_agents_table([record])
|
||||
assert table.row_count == 1
|
||||
rendered_cells = [list(col.cells)[0] for col in table.columns]
|
||||
# cells[0]=agent, [1]=pid, [2..6]=uptime/cpu/tokens/min/$/hr/status.
|
||||
# Every metric cell — including ``$/hr`` — falls back to ``-``.
|
||||
assert rendered_cells[2:] == ["-", "-", "-", "-", "-"]
|
||||
# The printed form should still include the agent name (so
|
||||
# rendering didn't silently bail).
|
||||
out = _render([record])
|
||||
assert "claude-code-8421" in out
|
||||
|
||||
|
||||
# ---------- Codex end-to-end ----------------------------------------
|
||||
|
||||
|
||||
def test_codex_end_to_end_renders_real_tokens_and_cost(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Mirrors the Claude Code test for the Codex rollout source."""
|
||||
codex_home = tmp_path / "codex_home"
|
||||
started_at_epoch = time.time() - 30 # process started 30 s ago
|
||||
started_dt = datetime.fromtimestamp(started_at_epoch, tz=UTC)
|
||||
rollout_dir = (
|
||||
codex_home
|
||||
/ "sessions"
|
||||
/ started_dt.strftime("%Y")
|
||||
/ started_dt.strftime("%m")
|
||||
/ started_dt.strftime("%d")
|
||||
)
|
||||
rollout_dir.mkdir(parents=True)
|
||||
rollout = rollout_dir / "rollout-001-abc.jsonl"
|
||||
rollout.write_text(
|
||||
'{"type":"thread.started","model":"gpt-5-codex"}\n{"type":"turn.started"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
import os as _os
|
||||
|
||||
_os.utime(rollout, (started_at_epoch + 1, started_at_epoch + 1))
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.token_sources.codex.started_at_for_pid",
|
||||
lambda _pid: started_at_epoch,
|
||||
)
|
||||
# Source now requires fd-level confirmation that the PID owns
|
||||
# the rollout (production: each codex holds its rollout fd open).
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.token_sources.codex.open_files_for_pid",
|
||||
lambda _pid: (rollout,),
|
||||
)
|
||||
isolated_source = codex_source_mod.CodexRolloutSource(codex_home=codex_home)
|
||||
monkeypatch.setattr(
|
||||
"tools.system.fleet_monitoring.sampler.get_token_source",
|
||||
lambda provider: isolated_source if provider == "codex" else None,
|
||||
)
|
||||
|
||||
record = AgentRecord(
|
||||
name="codex-9999",
|
||||
pid=9999,
|
||||
command="codex exec",
|
||||
provider="codex",
|
||||
)
|
||||
|
||||
class _Registry:
|
||||
def list(self) -> list[AgentRecord]:
|
||||
return [record]
|
||||
|
||||
def get(self, pid: int) -> AgentRecord | None:
|
||||
return record if pid == 9999 else None
|
||||
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.AgentRegistry", _Registry)
|
||||
|
||||
fake_snapshot = ProcessSnapshot(
|
||||
pid=9999,
|
||||
cpu_percent=4.2,
|
||||
rss_mb=64.0,
|
||||
num_fds=20,
|
||||
num_connections=1,
|
||||
status="running",
|
||||
started_at=started_dt,
|
||||
)
|
||||
monkeypatch.setattr("tools.system.fleet_monitoring.sampler.probe", lambda _pid: fake_snapshot)
|
||||
|
||||
sampler_mod._TickCache.registry_snapshot[record.pid] = record
|
||||
sampler_mod._TickCache.agents_config = None
|
||||
sampler_mod._sample_tokens(record)
|
||||
|
||||
# Append a real per-turn token_count event (the on-disk format
|
||||
# codex-cli 0.130.0 writes — distinct from ``codex exec --json``
|
||||
# stdout). ``turn_context`` carries the model; ``event_msg`` with
|
||||
# ``payload.type == "token_count"`` carries ``last_token_usage``.
|
||||
with rollout.open("a", encoding="utf-8") as fh:
|
||||
fh.write(
|
||||
'{"type":"turn_context","payload":{"turn_id":"t_1","model":"gpt-5-codex"}}\n'
|
||||
'{"type":"event_msg","payload":{"type":"token_count","info":'
|
||||
'{"last_token_usage":'
|
||||
'{"input_tokens":175,"cached_input_tokens":0,'
|
||||
'"output_tokens":50,"reasoning_output_tokens":0,"total_tokens":225}}}}\n'
|
||||
)
|
||||
time.sleep(0.01)
|
||||
|
||||
sampler_mod._sample_tokens(record)
|
||||
|
||||
out = _render([record])
|
||||
|
||||
# 225 tokens (input + output) over a 60 s window.
|
||||
assert "225" in out
|
||||
# Real cost figure from the gpt-5-codex price table.
|
||||
assert "$0.0" in out or "$0.1" in out
|
||||
assert "codex-9999" in out
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Tests for the interactive-shell launch banner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.ui.banner import banner as banner_module
|
||||
from surfaces.interactive_shell.ui.banner import banner_state as banner_state_module
|
||||
from surfaces.interactive_shell.ui.components import rendering as rendering_module
|
||||
|
||||
|
||||
def test_banner_shows_ollama_model(monkeypatch: object) -> None:
|
||||
monkeypatch.setenv("LLM_PROVIDER", "ollama")
|
||||
monkeypatch.setenv("OLLAMA_MODEL", "qwen2.5:7b")
|
||||
console_file = io.StringIO()
|
||||
console = Console(file=console_file, force_terminal=False, highlight=False)
|
||||
|
||||
banner_module.render_banner(console)
|
||||
|
||||
output = console_file.getvalue()
|
||||
assert "ollama" in output
|
||||
assert "qwen2.5:7b" in output
|
||||
assert "ollama · default" not in output
|
||||
|
||||
|
||||
def test_ready_box_uses_active_theme_palette() -> None:
|
||||
from platform.terminal.theme import set_active_theme
|
||||
|
||||
set_active_theme("pink")
|
||||
pink_rgb = "255;179;217"
|
||||
green_rgb = "185;237;175"
|
||||
|
||||
console = Console(record=True, width=120)
|
||||
console.print(banner_module.build_ready_panel(console))
|
||||
|
||||
styled = console.export_text(styles=True)
|
||||
assert pink_rgb in styled
|
||||
assert green_rgb not in styled
|
||||
|
||||
|
||||
def test_splash_gradient_style_for_gradient_theme() -> None:
|
||||
from platform.terminal.theme import set_active_theme
|
||||
|
||||
set_active_theme("webflux")
|
||||
assert banner_module._splash_block_style(1, 3) == "bold #864C96"
|
||||
assert banner_module._splash_block_style(0, 1) == "bold #E23636"
|
||||
|
||||
|
||||
def test_splash_gradient_style_falls_back_to_highlight_without_gradient() -> None:
|
||||
from platform.terminal.theme import HIGHLIGHT, set_active_theme
|
||||
|
||||
set_active_theme("nord")
|
||||
assert banner_module._splash_block_style(0, 1) == f"bold {HIGHLIGHT}"
|
||||
|
||||
|
||||
def test_refresh_welcome_poster_uses_repl_safe_render(monkeypatch: object) -> None:
|
||||
console = Console(record=True, width=120)
|
||||
render_calls: list[dict[str, object | None]] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.rendering.repl_clear_screen",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
def _fake_render(
|
||||
_console: Console,
|
||||
*,
|
||||
session: object = None,
|
||||
theme_notice: str | None = None,
|
||||
) -> None:
|
||||
render_calls.append({"session": session, "theme_notice": theme_notice})
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.rendering.repl_render_launch_poster",
|
||||
_fake_render,
|
||||
)
|
||||
|
||||
rendering_module.refresh_welcome_poster(console, session="sess", theme_notice="pink")
|
||||
|
||||
assert render_calls == [{"session": "sess", "theme_notice": "pink"}]
|
||||
|
||||
|
||||
def test_get_username_prefers_github_handle(monkeypatch: object) -> None:
|
||||
monkeypatch.setattr(banner_module, "_github_username", lambda: "octocat")
|
||||
monkeypatch.setattr(banner_module.getpass, "getuser", lambda: "system-user")
|
||||
|
||||
assert banner_module._get_username() == "octocat"
|
||||
|
||||
|
||||
def test_get_username_falls_back_to_system_user(monkeypatch: object) -> None:
|
||||
monkeypatch.setattr(banner_module, "_github_username", lambda: "")
|
||||
monkeypatch.setattr(banner_module.getpass, "getuser", lambda: "system-user")
|
||||
|
||||
assert banner_module._get_username() == "system-user"
|
||||
|
||||
|
||||
def test_github_username_reads_saved_credential(monkeypatch: object) -> None:
|
||||
monkeypatch.setattr(
|
||||
"integrations.store.get_integration",
|
||||
lambda service: {"credentials": {"username": "octocat"}} if service == "github" else None,
|
||||
)
|
||||
|
||||
assert banner_module._github_username() == "octocat"
|
||||
|
||||
|
||||
def test_github_username_empty_when_not_configured(monkeypatch: object) -> None:
|
||||
monkeypatch.setattr("integrations.store.get_integration", lambda _service: None)
|
||||
|
||||
assert banner_module._github_username() == ""
|
||||
|
||||
|
||||
def test_github_username_survives_identity_import_failure(monkeypatch: object) -> None:
|
||||
import builtins
|
||||
|
||||
real_import = builtins.__import__
|
||||
|
||||
def _fail_github_identity(name: str, *args: object, **kwargs: object) -> object:
|
||||
if name == "integrations.github.identity":
|
||||
raise ImportError("simulated heavy import failure")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", _fail_github_identity)
|
||||
|
||||
assert banner_module._github_username() == ""
|
||||
|
||||
|
||||
def test_ambient_column_marks_incomplete_integration(monkeypatch: object) -> None:
|
||||
# A hosted MCP record saved without an API token is "present" but cannot
|
||||
# connect; the banner must mark it rather than imply it works.
|
||||
monkeypatch.setattr(
|
||||
banner_state_module,
|
||||
"_load_integration_health",
|
||||
lambda: [("Sentry", "ok"), ("Posthog_Mcp", "incomplete")],
|
||||
)
|
||||
monkeypatch.setattr(banner_state_module, "_is_alert_listener_active", lambda: False)
|
||||
|
||||
text = banner_state_module._build_ambient_right_column().plain
|
||||
|
||||
assert "Sentry" in text
|
||||
assert "Posthog_Mcp ⚠" in text
|
||||
assert "⚠ incomplete — run /integrations verify" in text
|
||||
|
||||
|
||||
def test_ambient_column_no_warning_when_all_healthy(monkeypatch: object) -> None:
|
||||
monkeypatch.setattr(
|
||||
banner_state_module,
|
||||
"_load_integration_health",
|
||||
lambda: [("Sentry", "ok"), ("GitHub", "ok")],
|
||||
)
|
||||
monkeypatch.setattr(banner_state_module, "_is_alert_listener_active", lambda: False)
|
||||
|
||||
text = banner_state_module._build_ambient_right_column().plain
|
||||
|
||||
assert "Sentry" in text
|
||||
assert "GitHub" in text
|
||||
assert "⚠" not in text
|
||||
|
||||
|
||||
def test_ready_box_expands_to_console_width() -> None:
|
||||
console_file = io.StringIO()
|
||||
console = Console(file=console_file, force_terminal=False, highlight=False, width=120)
|
||||
|
||||
banner_module.render_ready_box(console)
|
||||
|
||||
lines = [
|
||||
line for line in console_file.getvalue().splitlines() if line.startswith(("╭", "╰", "│"))
|
||||
]
|
||||
assert lines
|
||||
assert max(len(line) for line in lines) == 120
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Tests for inline raw-terminal choice menu rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
from surfaces.interactive_shell.ui.components import choice_menu
|
||||
|
||||
_ANSI_RE = re.compile(r"\x1b\[[0-9;:]*[A-Za-z]")
|
||||
|
||||
|
||||
def test_draw_menu_uses_carriage_return_newlines(monkeypatch) -> None:
|
||||
"""Raw-mode terminals do not translate LF to CRLF for us.
|
||||
|
||||
Plain ``\n`` makes each line begin at the previous line's ending column,
|
||||
which renders the picker as a diagonal staircase. The inline menu should
|
||||
write explicit ``\r\n`` newlines and reset to column zero for every row.
|
||||
"""
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(choice_menu, "_cols", lambda: 80)
|
||||
|
||||
choice_menu._draw_menu(
|
||||
title="integrations",
|
||||
crumb="/integrations",
|
||||
labels=["/integrations list", "/integrations verify"],
|
||||
index=0,
|
||||
erase_lines=0,
|
||||
)
|
||||
|
||||
rendered = out.getvalue()
|
||||
plain = _ANSI_RE.sub("", rendered)
|
||||
assert "\n" in rendered
|
||||
assert all(rendered[index - 1] == "\r" for index, char in enumerate(rendered) if char == "\n")
|
||||
assert "\rintegrations" in plain
|
||||
assert "\r/integrations" in plain
|
||||
assert "\r > /integrations list" in plain
|
||||
|
||||
|
||||
def test_erase_menu_block_resets_to_column_zero(monkeypatch) -> None:
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
|
||||
choice_menu._erase_menu("crumb", ["one", "two"])
|
||||
|
||||
rendered = out.getvalue()
|
||||
assert rendered.startswith("\r\x1b[")
|
||||
assert "A\r\x1b[J" in rendered
|
||||
assert rendered.endswith("\r")
|
||||
|
||||
|
||||
def test_reset_tty_column_writes_carriage_return(monkeypatch) -> None:
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
|
||||
choice_menu.reset_tty_column()
|
||||
|
||||
assert out.getvalue() == "\r"
|
||||
|
||||
|
||||
def test_pick_ignores_unmapped_keys(monkeypatch) -> None:
|
||||
out = io.StringIO()
|
||||
actions = iter(["ignore", "enter"])
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(choice_menu, "_cols", lambda: 80)
|
||||
monkeypatch.setattr(choice_menu, "_read_action", lambda: next(actions))
|
||||
|
||||
assert choice_menu._pick(title="test", crumb="", labels=["one"]) == 0
|
||||
|
||||
rendered = out.getvalue()
|
||||
assert rendered.count("test") == 2
|
||||
assert "A\r\x1b[J" in rendered
|
||||
|
||||
|
||||
def test_read_action_treats_space_as_enter(monkeypatch) -> None:
|
||||
monkeypatch.setattr(choice_menu.os, "name", "nt")
|
||||
monkeypatch.setitem(sys.modules, "msvcrt", SimpleNamespace(getch=lambda: b" "))
|
||||
|
||||
assert choice_menu._read_action() == "enter"
|
||||
|
||||
|
||||
def test_read_action_treats_right_arrow_as_enter(monkeypatch) -> None:
|
||||
keys = iter([b"\xe0", b"M"])
|
||||
monkeypatch.setattr(choice_menu.os, "name", "nt")
|
||||
monkeypatch.setitem(sys.modules, "msvcrt", SimpleNamespace(getch=lambda: next(keys)))
|
||||
|
||||
assert choice_menu._read_action() == "enter"
|
||||
|
||||
|
||||
def test_repl_choose_one_starts_at_initial_value(monkeypatch) -> None:
|
||||
out = io.StringIO()
|
||||
actions = iter(["enter"])
|
||||
monkeypatch.setattr(choice_menu, "repl_tty_interactive", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.cpr_stdin.drain_stale_cpr_bytes",
|
||||
lambda: None,
|
||||
)
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(choice_menu, "_cols", lambda: 80)
|
||||
monkeypatch.setattr(choice_menu, "_read_action", lambda: next(actions))
|
||||
|
||||
result = choice_menu.repl_choose_one(
|
||||
title="theme",
|
||||
breadcrumb="/theme",
|
||||
choices=[("green", "green"), ("blue", "blue (current)"), ("pink", "pink")],
|
||||
initial_value="blue",
|
||||
)
|
||||
|
||||
assert result == "blue"
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert "> blue (current)" in plain
|
||||
|
||||
|
||||
def test_read_action_ignores_left_arrow(monkeypatch) -> None:
|
||||
keys = iter([b"\xe0", b"K"])
|
||||
monkeypatch.setattr(choice_menu.os, "name", "nt")
|
||||
monkeypatch.setitem(sys.modules, "msvcrt", SimpleNamespace(getch=lambda: next(keys)))
|
||||
|
||||
assert choice_menu._read_action() == "ignore"
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Tests for CPR stdin hygiene helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.ui.components.cpr_stdin import (
|
||||
contains_cpr_sequence,
|
||||
strip_cpr_sequences,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
("\x1b[32;1R", ""),
|
||||
("[32;1R", ""),
|
||||
("\x9b32;1R", ""),
|
||||
("what is our current model?[32;1R", "what is our current model?"),
|
||||
("before \x1b[12;80R after", "before after"),
|
||||
("7R[25;57R23;57R", ""),
|
||||
("25;57R", ""),
|
||||
],
|
||||
)
|
||||
def test_strip_cpr_sequences_removes_terminal_cursor_replies(
|
||||
text: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert strip_cpr_sequences(text) == expected
|
||||
|
||||
|
||||
def test_contains_cpr_sequence_detects_leaked_bytes() -> None:
|
||||
assert contains_cpr_sequence("\x1b[12;80R")
|
||||
assert not contains_cpr_sequence("plain prompt text")
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Tests for the execution-policy interaction layer (``execution_allowed``).
|
||||
|
||||
These cover the terminal-facing half of the execution gate: console output and
|
||||
the confirmation prompt. The pure decision is tested in
|
||||
``tests/tools/interactive_shell/shared/test_execution_policy.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.execution_confirm import execution_allowed
|
||||
from tools.interactive_shell.shared import (
|
||||
ExecutionPolicyResult,
|
||||
allow_tool,
|
||||
)
|
||||
|
||||
|
||||
def _ask_result() -> ExecutionPolicyResult:
|
||||
"""An explicit ``ask`` verdict (the default policy no longer emits these)."""
|
||||
return ExecutionPolicyResult(
|
||||
verdict="ask",
|
||||
tool_type="slash",
|
||||
reason="this command may change configuration or run heavy work",
|
||||
)
|
||||
|
||||
|
||||
# --- execution_allowed: default-allow runs without prompting ----------------
|
||||
|
||||
|
||||
def test_allow_verdict_runs_without_prompt() -> None:
|
||||
session = Session()
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
|
||||
def _confirm(_: str) -> str: # pragma: no cover - must never be called
|
||||
raise AssertionError("default-allow must not prompt for confirmation")
|
||||
|
||||
r = allow_tool("slash")
|
||||
assert execution_allowed(
|
||||
r,
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/integrations verify foo",
|
||||
confirm_fn=_confirm,
|
||||
is_tty=True,
|
||||
)
|
||||
assert "Confirm" not in buf.getvalue()
|
||||
|
||||
|
||||
def test_non_tty_allows_default_policy() -> None:
|
||||
"""Default-allow no longer fails closed on non-interactive stdin."""
|
||||
session = Session()
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
r = allow_tool("slash")
|
||||
assert execution_allowed(
|
||||
r,
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/save out.md",
|
||||
is_tty=False,
|
||||
)
|
||||
|
||||
|
||||
def test_deny_verdict_blocks() -> None:
|
||||
session = Session()
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
# The default policy never emits a deny; construct one explicitly to cover
|
||||
# the execution_allowed deny path.
|
||||
r = ExecutionPolicyResult(
|
||||
verdict="deny",
|
||||
tool_type="shell",
|
||||
reason="empty command.",
|
||||
hint="Enter a command to run.",
|
||||
)
|
||||
assert not execution_allowed(
|
||||
r,
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="!",
|
||||
is_tty=True,
|
||||
)
|
||||
assert "blocked" in buf.getvalue()
|
||||
|
||||
|
||||
# --- Retained ask machinery (reachable only via explicit ask) ---------------
|
||||
|
||||
|
||||
def test_explicit_ask_trust_mode_allows() -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
assert execution_allowed(
|
||||
_ask_result(),
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/investigate x",
|
||||
confirm_fn=lambda _: "n",
|
||||
is_tty=True,
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_ask_non_tty_blocks() -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = False
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
assert not execution_allowed(
|
||||
_ask_result(),
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/save out.md",
|
||||
is_tty=False,
|
||||
)
|
||||
assert "not a TTY" in buf.getvalue()
|
||||
|
||||
|
||||
def test_explicit_ask_tty_accepts_empty_confirmation() -> None:
|
||||
session = Session()
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
captured: list[str] = []
|
||||
|
||||
def _confirm(prompt: str) -> str:
|
||||
captured.append(prompt)
|
||||
return ""
|
||||
|
||||
assert execution_allowed(
|
||||
_ask_result(),
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/integrations verify foo",
|
||||
confirm_fn=_confirm,
|
||||
is_tty=True,
|
||||
)
|
||||
assert captured == ["Proceed? [Y/n] "]
|
||||
|
||||
|
||||
def test_explicit_ask_tty_rejects_explicit_no() -> None:
|
||||
session = Session()
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
assert not execution_allowed(
|
||||
_ask_result(),
|
||||
session=session,
|
||||
console=console,
|
||||
action_summary="/integrations verify foo",
|
||||
confirm_fn=lambda _: "n",
|
||||
is_tty=True,
|
||||
)
|
||||
assert "cancelled" in buf.getvalue()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Tests for post-investigation feedback UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.ui.feedback import (
|
||||
_CHOICES,
|
||||
_format_root_cause_lines,
|
||||
_print_context,
|
||||
_root_cause_width,
|
||||
_run_select,
|
||||
)
|
||||
|
||||
|
||||
def _fixed_terminal_size(*_args: object, **_kwargs: object) -> os.terminal_size:
|
||||
return os.terminal_size((60, 24))
|
||||
|
||||
|
||||
def _wide_terminal_size(*_args: object, **_kwargs: object) -> os.terminal_size:
|
||||
return os.terminal_size((160, 24))
|
||||
|
||||
|
||||
def test_root_cause_width_uses_full_terminal_not_capped(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("shutil.get_terminal_size", _wide_terminal_size)
|
||||
|
||||
assert _root_cause_width(console=None) == 160
|
||||
|
||||
|
||||
def test_print_context_stdout_uses_full_terminal_width(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
monkeypatch.setattr("shutil.get_terminal_size", _wide_terminal_size)
|
||||
root = "Schema validation failed because payment_method is missing."
|
||||
|
||||
_print_context({"root_cause": root}, console=None)
|
||||
|
||||
captured = capsys.readouterr().out
|
||||
assert captured.startswith("\n" + "─" * 160 + "\n")
|
||||
assert captured.rstrip().endswith("─" * 160)
|
||||
|
||||
|
||||
def test_format_root_cause_lines_wraps_long_text_without_truncation() -> None:
|
||||
root = (
|
||||
"The Kubernetes job 'etl-transform-error' for pipeline "
|
||||
"'kubernetes_etl_pipeline' failed in namespace 'tracer-test' because "
|
||||
"schema validation requires 'payment_method'."
|
||||
)
|
||||
|
||||
lines = _format_root_cause_lines(root, cols=60)
|
||||
|
||||
assert len(lines) > 1
|
||||
assert all("…" not in line for line in lines)
|
||||
assert " ".join(line.strip() for line in lines) == f"Root cause: {root}"
|
||||
|
||||
|
||||
def test_print_context_shows_full_root_cause_in_rich_path() -> None:
|
||||
root = (
|
||||
"The Kubernetes job 'etl-transform-error' for pipeline "
|
||||
"'kubernetes_etl_pipeline' failed because schema validation requires "
|
||||
"'payment_method'."
|
||||
)
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False, width=60)
|
||||
|
||||
_print_context({"root_cause": root}, console=console)
|
||||
|
||||
output = buf.getvalue()
|
||||
assert root in output
|
||||
assert "…" not in output
|
||||
|
||||
|
||||
def test_print_context_escapes_rich_markup_in_root() -> None:
|
||||
root = "Pod restart [1/3] failed because schema validation requires 'payment_method'."
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False, width=80)
|
||||
|
||||
_print_context({"root_cause": root}, console=console)
|
||||
|
||||
output = buf.getvalue()
|
||||
assert "[1/3]" in output
|
||||
assert "…" not in output
|
||||
|
||||
|
||||
def test_print_context_shows_full_root_cause_in_stdout_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
monkeypatch.setattr("shutil.get_terminal_size", _fixed_terminal_size)
|
||||
root = (
|
||||
"The Kubernetes job 'etl-transform-error' for pipeline "
|
||||
"'kubernetes_etl_pipeline' failed because schema validation requires "
|
||||
"'payment_method'."
|
||||
)
|
||||
|
||||
_print_context({"root_cause": root}, console=None)
|
||||
|
||||
captured = capsys.readouterr().out
|
||||
assert "…" not in captured
|
||||
assert " ".join(captured.split()) == ("─" * 60 + " Root cause: " + root + " " + "─" * 60)
|
||||
|
||||
|
||||
def test_run_select_returns_highlighted_choice_on_enter(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
|
||||
monkeypatch.setattr("sys.stdout.isatty", lambda: True)
|
||||
calls = {"n": 0}
|
||||
|
||||
def read_then_enter(**_kwargs: object) -> str:
|
||||
calls["n"] += 1
|
||||
return "enter" if calls["n"] > 1 else "down"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.feedback.read_key_unix",
|
||||
read_then_enter,
|
||||
)
|
||||
monkeypatch.setattr("surfaces.interactive_shell.ui.feedback.flush_stdin_unix", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.feedback.restore_stdin_terminal",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
assert _run_select(_CHOICES) == "partial"
|
||||
captured = capsys.readouterr().out
|
||||
assert "Partially accurate" in captured
|
||||
assert "↑↓" in captured
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Tests for the grouped /help picker renderer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
|
||||
from platform.terminal import theme as ui_theme
|
||||
from surfaces.interactive_shell.command_registry.types import SlashCommand
|
||||
from surfaces.interactive_shell.ui.help import help_menu
|
||||
|
||||
_ANSI_RE = re.compile(r"\x1b\[[0-9;:]*[A-Za-z]")
|
||||
|
||||
|
||||
def _cmd(name: str, description: str | None = None) -> SlashCommand:
|
||||
return SlashCommand(name, description or f"{name} description", lambda *_args: True)
|
||||
|
||||
|
||||
def test_flatten_help_rows_preserves_category_headers() -> None:
|
||||
rows = help_menu._flatten_help_rows(
|
||||
[
|
||||
("Session", [_cmd("/status")]),
|
||||
("Tasks", [_cmd("/tasks"), _cmd("/cancel")]),
|
||||
]
|
||||
)
|
||||
|
||||
assert [row.section for row in rows if row.section is not None] == ["Session", "Tasks"]
|
||||
assert sum(1 for row in rows if row.separator) == 1
|
||||
assert [row.command.name for row in rows if row.command is not None] == [
|
||||
"/status",
|
||||
"/tasks",
|
||||
"/cancel",
|
||||
]
|
||||
|
||||
|
||||
def test_navigation_skips_category_headers() -> None:
|
||||
rows = help_menu._flatten_help_rows(
|
||||
[
|
||||
("Session", [_cmd("/status")]),
|
||||
("Tasks", [_cmd("/tasks")]),
|
||||
]
|
||||
)
|
||||
|
||||
assert help_menu._first_selectable_index(rows) == 1
|
||||
assert help_menu._next_selectable_index(rows, 1, 1) == 4
|
||||
assert help_menu._next_selectable_index(rows, 4, -1) == 1
|
||||
|
||||
|
||||
def test_draw_help_menu_renders_horizontal_category_dividers(monkeypatch) -> None:
|
||||
rows = help_menu._flatten_help_rows(
|
||||
[
|
||||
("Session", [_cmd("/status")]),
|
||||
("Tasks", [_cmd("/tasks")]),
|
||||
]
|
||||
)
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 40)
|
||||
|
||||
help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=1,
|
||||
expanded=None,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert "Session" in plain
|
||||
assert "Tasks" in plain
|
||||
assert plain.count(help_menu._separator_rule(40)) >= 2
|
||||
assert "┼" in plain
|
||||
assert help_menu._render_grid_row("Session", "", 40) in plain
|
||||
assert help_menu._render_grid_row("Tasks", "", 40) in plain
|
||||
|
||||
|
||||
def test_section_rows_keep_divider_dim() -> None:
|
||||
rendered = help_menu._render_help_row(
|
||||
help_menu.HelpRow(section="Investigation"),
|
||||
selected=False,
|
||||
expanded=False,
|
||||
width=40,
|
||||
)
|
||||
|
||||
assert f"{ui_theme.BOLD_BRAND_ANSI}Investigation" in rendered
|
||||
assert f"{ui_theme.DIM_COUNTER_ANSI}│" in rendered
|
||||
|
||||
|
||||
def test_detail_rows_use_text_labels_dim_values_and_dim_divider() -> None:
|
||||
label = help_menu._render_display_row(
|
||||
help_menu.HelpDisplayRow(detail=help_menu.HelpDetailLine("usage:", "label")),
|
||||
selected=False,
|
||||
expanded=False,
|
||||
width=40,
|
||||
)
|
||||
value = help_menu._render_display_row(
|
||||
help_menu.HelpDisplayRow(detail=help_menu.HelpDetailLine(" /help", "value")),
|
||||
selected=False,
|
||||
expanded=False,
|
||||
width=40,
|
||||
)
|
||||
|
||||
assert f"{ui_theme.DIM_COUNTER_ANSI}{' ' * help_menu._left_column_width(40)}│ " in label
|
||||
assert f"{ui_theme.TEXT_ANSI}usage:" in label
|
||||
assert f"{ui_theme.DIM_COUNTER_ANSI}{' ' * help_menu._left_column_width(40)}│ " in value
|
||||
assert f"{ui_theme.DIM_COUNTER_ANSI} /help" in value
|
||||
|
||||
|
||||
def test_draw_help_menu_centers_title(monkeypatch) -> None:
|
||||
rows = help_menu._flatten_help_rows([("Session", [_cmd("/status")])])
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 40)
|
||||
|
||||
help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=1,
|
||||
expanded=None,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
plain_lines = _ANSI_RE.sub("", out.getvalue()).splitlines()
|
||||
assert " Slash commands " in plain_lines
|
||||
|
||||
|
||||
def test_draw_help_menu_uses_bounded_viewport_and_category_context(monkeypatch) -> None:
|
||||
sections = [
|
||||
("Session", [_cmd(f"/session-{index}") for index in range(10)]),
|
||||
("Tasks", [_cmd(f"/task-{index}") for index in range(10)]),
|
||||
]
|
||||
rows = help_menu._flatten_help_rows(sections)
|
||||
selected = next(
|
||||
index for index, row in enumerate(rows) if row.command and row.command.name == "/task-2"
|
||||
)
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 90)
|
||||
|
||||
height = help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=selected,
|
||||
expanded=None,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
rendered = out.getvalue()
|
||||
plain = _ANSI_RE.sub("", rendered)
|
||||
assert height == 13
|
||||
assert "Tasks" in plain
|
||||
assert "/task-2" in plain
|
||||
assert "/session-0" not in plain
|
||||
assert "/task-9" not in plain
|
||||
|
||||
|
||||
def test_draw_help_menu_expands_selected_command_inline(monkeypatch) -> None:
|
||||
command = SlashCommand(
|
||||
"/new",
|
||||
"Start a new session keeping conversation context.",
|
||||
lambda *_args: True,
|
||||
usage=("/new",),
|
||||
notes=("Unlike /clear, /new preserves the LLM conversation context.",),
|
||||
)
|
||||
rows = help_menu._flatten_help_rows([("Session", [command])])
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 90)
|
||||
|
||||
help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=1,
|
||||
expanded=1,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert "/new" in plain
|
||||
assert "> ▾ /new" in plain
|
||||
assert "│ Start a new session" in plain
|
||||
assert help_menu._render_grid_row("", "usage:", 90) in plain
|
||||
assert "usage:" in plain
|
||||
assert "LLM conversation context" in plain
|
||||
|
||||
|
||||
def test_draw_help_menu_marks_expandable_and_plain_commands(monkeypatch) -> None:
|
||||
expandable = SlashCommand(
|
||||
"/trust",
|
||||
"Manage trust mode.",
|
||||
lambda *_args: True,
|
||||
usage=("/trust on", "/trust off"),
|
||||
)
|
||||
plain_command = _cmd("/status", "Show session status.")
|
||||
rows = help_menu._flatten_help_rows([("Session", [expandable, plain_command])])
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 90)
|
||||
|
||||
help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=1,
|
||||
expanded=None,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert "> ▸ /trust" in plain
|
||||
assert "│ Manage trust mode." in plain
|
||||
assert " /status" in plain
|
||||
assert "▸ /status" not in plain
|
||||
|
||||
|
||||
def test_command_rows_highlight_only_unselected_command_name() -> None:
|
||||
rendered = help_menu._render_command_row(
|
||||
SlashCommand(
|
||||
"/trust",
|
||||
"Manage trust mode.",
|
||||
lambda *_args: True,
|
||||
usage=("/trust on", "/trust off"),
|
||||
),
|
||||
selected=False,
|
||||
expanded=False,
|
||||
width=60,
|
||||
)
|
||||
|
||||
assert (
|
||||
f"{ui_theme.DIM_COUNTER_ANSI} ▸ {ui_theme.ANSI_RESET}{ui_theme.HIGHLIGHT_ANSI}/trust"
|
||||
) in rendered
|
||||
assert f"{ui_theme.HIGHLIGHT_ANSI}▸" not in rendered
|
||||
|
||||
|
||||
def test_help_menu_command_rows_track_active_theme() -> None:
|
||||
from platform.terminal.theme import set_active_theme
|
||||
|
||||
set_active_theme("green")
|
||||
green_highlight = ui_theme.HIGHLIGHT_ANSI
|
||||
set_active_theme("purple")
|
||||
rendered = help_menu._render_command_row(
|
||||
SlashCommand("/trust", "Manage trust mode.", lambda *_args: True),
|
||||
selected=False,
|
||||
expanded=False,
|
||||
width=60,
|
||||
)
|
||||
|
||||
assert ui_theme.HIGHLIGHT_ANSI in rendered
|
||||
assert green_highlight not in rendered
|
||||
|
||||
|
||||
def test_expanded_detail_lines_include_all_usage_examples_and_notes() -> None:
|
||||
command = SlashCommand(
|
||||
"/model",
|
||||
"Show model settings.",
|
||||
lambda *_args: True,
|
||||
usage=tuple(f"/model action {index}" for index in range(10)),
|
||||
examples=("/model set openai",),
|
||||
notes=("Provider defaults can be restored.",),
|
||||
)
|
||||
|
||||
lines = help_menu._expanded_detail_lines(command)
|
||||
text_lines = [line.text for line in lines]
|
||||
|
||||
assert lines[0].role == "label"
|
||||
assert lines[1].role == "value"
|
||||
assert "/model action 0" in lines[1].text
|
||||
assert " /model action 9" in text_lines
|
||||
assert "examples:" in text_lines
|
||||
assert " /model set openai" in text_lines
|
||||
assert "notes:" in text_lines
|
||||
assert " Provider defaults can be restored." in text_lines
|
||||
assert "…" not in text_lines
|
||||
|
||||
|
||||
def test_draw_help_menu_expands_viewport_to_show_all_details(monkeypatch) -> None:
|
||||
command = SlashCommand(
|
||||
"/model",
|
||||
"Show model settings.",
|
||||
lambda *_args: True,
|
||||
usage=tuple(f"/model action {index}" for index in range(10)),
|
||||
)
|
||||
rows = help_menu._flatten_help_rows([("Models", [command, _cmd("/status")])])
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 90)
|
||||
|
||||
height = help_menu._draw_help_menu(
|
||||
rows,
|
||||
selected=1,
|
||||
expanded=1,
|
||||
erase_lines=0,
|
||||
viewport_height=7,
|
||||
)
|
||||
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert height > 13
|
||||
assert "/model action 0" in plain
|
||||
assert "/model action 9" in plain
|
||||
assert "…" not in plain
|
||||
|
||||
|
||||
def test_choose_help_command_space_toggles_inline_details_and_exits(monkeypatch) -> None:
|
||||
sections = [
|
||||
(
|
||||
"Session",
|
||||
[
|
||||
_cmd("/status"),
|
||||
SlashCommand(
|
||||
"/trust",
|
||||
"Manage trust mode.",
|
||||
lambda *_args: True,
|
||||
usage=("/trust on", "/trust off"),
|
||||
),
|
||||
],
|
||||
)
|
||||
]
|
||||
out = io.StringIO()
|
||||
actions = iter(["down", "space", "cancel"])
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 80)
|
||||
monkeypatch.setattr(help_menu, "read_menu_action", lambda: next(actions))
|
||||
|
||||
selected = help_menu.choose_help_command(sections)
|
||||
|
||||
assert selected is None
|
||||
rendered = out.getvalue()
|
||||
assert "\x1b[" in rendered
|
||||
assert "A\r\x1b[J" in rendered
|
||||
plain = _ANSI_RE.sub("", rendered)
|
||||
assert "/trust on" in plain
|
||||
|
||||
|
||||
def test_choose_help_command_enter_selects_command_without_details(monkeypatch) -> None:
|
||||
sections = [("Session", [_cmd("/status")])]
|
||||
out = io.StringIO()
|
||||
actions = iter(["enter"])
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 80)
|
||||
monkeypatch.setattr(help_menu, "read_menu_action", lambda: next(actions))
|
||||
|
||||
assert help_menu.choose_help_command(sections) == "/status"
|
||||
|
||||
plain = _ANSI_RE.sub("", out.getvalue())
|
||||
assert "No additional usage." not in plain
|
||||
|
||||
|
||||
def test_choose_help_command_ignores_unknown_actions(monkeypatch) -> None:
|
||||
sections = [("Session", [_cmd("/status")])]
|
||||
out = io.StringIO()
|
||||
actions = iter(["ignore", "cancel"])
|
||||
monkeypatch.setattr(sys, "stdout", out)
|
||||
monkeypatch.setattr(help_menu, "menu_columns", lambda: 80)
|
||||
monkeypatch.setattr(help_menu, "read_menu_action", lambda: next(actions))
|
||||
|
||||
assert help_menu.choose_help_command(sections) is None
|
||||
|
||||
rendered = out.getvalue()
|
||||
assert rendered.count("Slash commands") == 2
|
||||
@@ -0,0 +1,316 @@
|
||||
"""Tests for incoming alert rendering in the REPL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from core.domain.alerts.inbox import AlertInbox, IncomingAlert
|
||||
from surfaces.interactive_shell.runtime import Session
|
||||
from surfaces.interactive_shell.ui.alerts import (
|
||||
drain_and_render_incoming,
|
||||
format_incoming_alert,
|
||||
time_ago,
|
||||
)
|
||||
|
||||
|
||||
class TestTimeAgo:
|
||||
"""Test time_ago helper."""
|
||||
|
||||
def test_seconds_ago(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
then = now - timedelta(seconds=5)
|
||||
result = time_ago(then)
|
||||
assert "5s ago" in result
|
||||
|
||||
def test_one_second_ago(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
then = now - timedelta(seconds=1)
|
||||
result = time_ago(then)
|
||||
assert "1s ago" in result
|
||||
|
||||
def test_minutes_ago(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
then = now - timedelta(minutes=3)
|
||||
result = time_ago(then)
|
||||
assert "3m ago" in result
|
||||
|
||||
def test_hours_ago(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
then = now - timedelta(hours=2)
|
||||
result = time_ago(then)
|
||||
assert "2h ago" in result
|
||||
|
||||
def test_days_ago(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
then = now - timedelta(days=1)
|
||||
result = time_ago(then)
|
||||
assert "1d ago" in result
|
||||
|
||||
def test_none_datetime(self) -> None:
|
||||
result = time_ago(None)
|
||||
assert result == "unknown"
|
||||
|
||||
|
||||
class TestFormatIncomingAlert:
|
||||
"""Test format_incoming_alert rendering."""
|
||||
|
||||
def test_renders_with_all_fields(self) -> None:
|
||||
alert = IncomingAlert(
|
||||
text="disk usage at 95%",
|
||||
alert_name="disk_alert",
|
||||
severity="critical",
|
||||
source="datadog-webhook",
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
renderable = format_incoming_alert(alert)
|
||||
# Just verify the alert object was created without error
|
||||
assert renderable is not None
|
||||
|
||||
def test_renders_with_minimal_fields(self) -> None:
|
||||
alert = IncomingAlert(text="something happened")
|
||||
renderable = format_incoming_alert(alert)
|
||||
# Just verify the alert object was created without error
|
||||
assert renderable is not None
|
||||
|
||||
def test_renders_without_source(self) -> None:
|
||||
alert = IncomingAlert(
|
||||
text="test alert",
|
||||
severity="warning",
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
renderable = format_incoming_alert(alert)
|
||||
# Just verify the alert object was created without error
|
||||
assert renderable is not None
|
||||
|
||||
def test_renders_without_severity(self) -> None:
|
||||
alert = IncomingAlert(
|
||||
text="test alert",
|
||||
source="custom",
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
renderable = format_incoming_alert(alert)
|
||||
# Just verify the alert object was created without error
|
||||
assert renderable is not None
|
||||
|
||||
def test_escapes_severity_markup(self) -> None:
|
||||
alert = IncomingAlert(
|
||||
text="payload",
|
||||
severity="critical] [red]pwned",
|
||||
source="webhook",
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
console = Console(record=True)
|
||||
console.print(format_incoming_alert(alert))
|
||||
output = console.export_text()
|
||||
|
||||
assert "[critical] [red]pwned]" in output
|
||||
assert "pwned]" in output
|
||||
|
||||
def test_severity_like_rich_style_tag_is_literal(self) -> None:
|
||||
"""Severity values resembling Rich markup must not apply styles."""
|
||||
alert = IncomingAlert(
|
||||
text="body",
|
||||
severity="bold red",
|
||||
received_at=datetime.now(UTC),
|
||||
)
|
||||
console = Console(record=True)
|
||||
console.print(format_incoming_alert(alert))
|
||||
output = console.export_text()
|
||||
|
||||
assert "[bold red]" in output
|
||||
|
||||
|
||||
class TestDrainAndRenderIncoming:
|
||||
"""Test drain_and_render_incoming functionality."""
|
||||
|
||||
def test_drains_fifo_order(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
# Add alerts in order
|
||||
alert1 = IncomingAlert(text="first")
|
||||
alert2 = IncomingAlert(text="second")
|
||||
alert3 = IncomingAlert(text="third")
|
||||
|
||||
inbox.put(alert1)
|
||||
inbox.put(alert2)
|
||||
inbox.put(alert3)
|
||||
|
||||
# Drain and render
|
||||
count = drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
assert count == 3
|
||||
assert len(session.alerts.entries) == 3
|
||||
assert session.alerts.entries[0].text == "first"
|
||||
assert session.alerts.entries[1].text == "second"
|
||||
assert session.alerts.entries[2].text == "third"
|
||||
|
||||
def test_records_in_history(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
alert = IncomingAlert(text="test alert")
|
||||
inbox.put(alert)
|
||||
|
||||
drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
# Check history
|
||||
assert len(session.history) == 1
|
||||
assert session.history[0]["type"] == "incoming_alert"
|
||||
assert session.history[0]["text"] == "test alert"
|
||||
assert session.history[0]["ok"] is True
|
||||
|
||||
def test_renders_to_console(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
alert = IncomingAlert(text="test message")
|
||||
inbox.put(alert)
|
||||
|
||||
# Just verify drain_and_render doesn't raise an exception
|
||||
count = drain_and_render_incoming(session, console, inbox)
|
||||
assert count == 1
|
||||
|
||||
def test_returns_count(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
inbox.put(IncomingAlert(text="alert1"))
|
||||
inbox.put(IncomingAlert(text="alert2"))
|
||||
|
||||
count = drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
assert count == 2
|
||||
|
||||
def test_drains_empty_inbox(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
count = drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
assert count == 0
|
||||
assert len(session.history) == 0
|
||||
|
||||
def test_caps_incoming_alerts_at_max(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox(maxsize=10)
|
||||
console = Console()
|
||||
|
||||
# Add more alerts than the session cap
|
||||
for i in range(300):
|
||||
alert = IncomingAlert(text=f"alert_{i}")
|
||||
inbox.put(alert)
|
||||
|
||||
drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
# Should be capped at _INCOMING_ALERTS_MAX (256)
|
||||
assert len(session.alerts.entries) <= session.alerts._max
|
||||
|
||||
|
||||
class TestSessionIncomingAlerts:
|
||||
"""Test Session handling of incoming alerts."""
|
||||
|
||||
def test_clear_resets_incoming_alerts(self) -> None:
|
||||
session = Session()
|
||||
inbox = AlertInbox()
|
||||
console = Console()
|
||||
|
||||
# Add some alerts
|
||||
inbox.put(IncomingAlert(text="alert1"))
|
||||
inbox.put(IncomingAlert(text="alert2"))
|
||||
drain_and_render_incoming(session, console, inbox)
|
||||
|
||||
assert len(session.alerts.entries) == 2
|
||||
|
||||
# Clear session
|
||||
session.clear()
|
||||
|
||||
assert len(session.alerts.entries) == 0
|
||||
assert len(session.history) == 0
|
||||
|
||||
def test_record_incoming_alert_kind(self) -> None:
|
||||
session = Session()
|
||||
alert = IncomingAlert(text="test alert")
|
||||
|
||||
session.record_incoming_alert(alert)
|
||||
|
||||
assert len(session.history) == 1
|
||||
assert session.history[0]["type"] == "incoming_alert"
|
||||
assert session.history[0]["text"] == "test alert"
|
||||
assert session.history[0]["ok"] is True
|
||||
assert len(session.alerts.entries) == 1
|
||||
assert session.alerts.entries[0].text == "test alert"
|
||||
|
||||
def test_record_incoming_alert_always_ok(self) -> None:
|
||||
session = Session()
|
||||
alert = IncomingAlert(text="test alert")
|
||||
|
||||
session.record_incoming_alert(alert)
|
||||
|
||||
assert session.history[0]["ok"] is True
|
||||
|
||||
def test_incoming_alerts_fifo_list(self) -> None:
|
||||
session = Session()
|
||||
|
||||
session.record_incoming_alert(IncomingAlert(text="first"))
|
||||
session.record_incoming_alert(IncomingAlert(text="second"))
|
||||
session.record_incoming_alert(IncomingAlert(text="third"))
|
||||
|
||||
assert len(session.alerts.entries) == 3
|
||||
assert session.alerts.entries[0].text == "first"
|
||||
assert session.alerts.entries[1].text == "second"
|
||||
assert session.alerts.entries[2].text == "third"
|
||||
|
||||
|
||||
class TestAlertInboxEventClearing:
|
||||
"""Test AlertInbox pending_event behavior."""
|
||||
|
||||
def test_event_set_on_put(self) -> None:
|
||||
inbox = AlertInbox()
|
||||
alert = IncomingAlert(text="test")
|
||||
|
||||
# Event should not be set initially
|
||||
assert not inbox.pending_event.is_set()
|
||||
|
||||
inbox.put(alert)
|
||||
|
||||
# Event should be set after put
|
||||
assert inbox.pending_event.is_set()
|
||||
|
||||
def test_event_cleared_on_iter_pending(self) -> None:
|
||||
inbox = AlertInbox()
|
||||
alert = IncomingAlert(text="test")
|
||||
|
||||
inbox.put(alert)
|
||||
assert inbox.pending_event.is_set()
|
||||
|
||||
inbox.iter_pending()
|
||||
|
||||
# Event should be cleared after draining
|
||||
assert not inbox.pending_event.is_set()
|
||||
|
||||
def test_event_not_cleared_if_queue_not_empty(self) -> None:
|
||||
inbox = AlertInbox()
|
||||
|
||||
inbox.put(IncomingAlert(text="first"))
|
||||
inbox.put(IncomingAlert(text="second"))
|
||||
|
||||
# Pop only one
|
||||
inbox.pop_nowait()
|
||||
|
||||
# Drain but queue still has one item
|
||||
# (this is artificial; normally iter_pending drains all)
|
||||
# Let's test with iter_pending which should drain all
|
||||
inbox2 = AlertInbox()
|
||||
inbox2.put(IncomingAlert(text="alert"))
|
||||
inbox2.iter_pending()
|
||||
|
||||
# After draining all, event should be cleared
|
||||
assert not inbox2.pending_event.is_set()
|
||||
@@ -0,0 +1,352 @@
|
||||
"""Tests for prompt placeholder and prefill behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
from prompt_toolkit.completion import Completion
|
||||
|
||||
from platform.common.task_types import TaskKind
|
||||
from surfaces.interactive_shell.runtime.core import state as loop_state
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.input_prompt import completion as prompt_completion
|
||||
from surfaces.interactive_shell.ui.input_prompt import rendering as prompt_rendering
|
||||
from surfaces.interactive_shell.ui.input_prompt.completion import completion_preview_hint_ansi
|
||||
from surfaces.interactive_shell.ui.input_prompt.refresh import wire_prompt_refresh
|
||||
from surfaces.interactive_shell.ui.input_prompt.rendering import (
|
||||
_DEFAULT_PLACEHOLDER_TEXT,
|
||||
_prompt_counter_text,
|
||||
_prompt_turn_number,
|
||||
resolve_idle_hint_ansi,
|
||||
resolve_prompt_placeholder,
|
||||
resolve_prompt_prefix_ansi,
|
||||
)
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
return re.sub(r"\x1b\[[0-9;]*m", "", text)
|
||||
|
||||
|
||||
def _placeholder_text(session: Session) -> str:
|
||||
return resolve_prompt_placeholder(session).value
|
||||
|
||||
|
||||
class _RefreshFakeBuffer:
|
||||
def __init__(self) -> None:
|
||||
self.text = ""
|
||||
self.submitted = False
|
||||
|
||||
def validate_and_handle(self) -> None:
|
||||
self.submitted = True
|
||||
|
||||
|
||||
class _RefreshFakeApp:
|
||||
is_running = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.current_buffer = _RefreshFakeBuffer()
|
||||
|
||||
def invalidate(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _RefreshFakeLoop:
|
||||
def call_soon_threadsafe(self, fn, *args) -> None: # type: ignore[no-untyped-def]
|
||||
fn(*args)
|
||||
|
||||
|
||||
class TestPromptRefreshAutoSubmit:
|
||||
def test_queue_auto_command_fills_and_submits_prompt(self) -> None:
|
||||
"""An agent-queued interactive command should be both prefilled and
|
||||
auto-submitted so it dispatches through the exclusive-stdin path."""
|
||||
session = Session()
|
||||
app = _RefreshFakeApp()
|
||||
wire_prompt_refresh(session, app, _RefreshFakeLoop())
|
||||
session.terminal.set_auto_command("/integrations setup sentry")
|
||||
assert app.current_buffer.text == "/integrations setup sentry"
|
||||
assert app.current_buffer.submitted is True
|
||||
|
||||
def test_plain_prefill_does_not_auto_submit(self) -> None:
|
||||
"""A prefill without the auto-submit flag must wait for the user (Enter)."""
|
||||
session = Session()
|
||||
app = _RefreshFakeApp()
|
||||
wire_prompt_refresh(session, app, _RefreshFakeLoop())
|
||||
session.terminal.pending_prompt_default = "why did it fail?"
|
||||
session.terminal.notify_prompt_changed()
|
||||
assert app.current_buffer.text == "why did it fail?"
|
||||
assert app.current_buffer.submitted is False
|
||||
|
||||
|
||||
class TestPromptTurnCounter:
|
||||
def test_first_turn_is_numbered_one(self) -> None:
|
||||
session = Session()
|
||||
assert _prompt_turn_number(session) == 1
|
||||
assert _prompt_counter_text(session) == "[1] "
|
||||
|
||||
def test_counter_advances_with_history(self) -> None:
|
||||
session = Session()
|
||||
session.record("chat", "hello")
|
||||
assert _prompt_turn_number(session) == 2
|
||||
assert _prompt_counter_text(session) == "[2] "
|
||||
|
||||
|
||||
class TestResolveIdleHint:
|
||||
def test_shows_connected_integrations_in_hint_bar(self) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog", "github", "grafana")
|
||||
rendered = _strip_ansi(resolve_idle_hint_ansi(session))
|
||||
assert "/ for commands" in rendered
|
||||
assert "Datadog" in rendered
|
||||
assert "GitHub" in rendered
|
||||
assert "Grafana" in rendered
|
||||
|
||||
def test_omits_integrations_when_none_configured(self) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ()
|
||||
rendered = _strip_ansi(resolve_idle_hint_ansi(session))
|
||||
assert "Datadog" not in rendered
|
||||
assert "/ for commands" in rendered
|
||||
|
||||
|
||||
class TestResolvePromptPlaceholder:
|
||||
def test_default_when_no_session_context(self) -> None:
|
||||
session = Session()
|
||||
assert _DEFAULT_PLACEHOLDER_TEXT in _placeholder_text(session)
|
||||
|
||||
def test_shows_trust_mode(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
text = _placeholder_text(session)
|
||||
assert "trust on" in text
|
||||
assert _DEFAULT_PLACEHOLDER_TEXT not in text
|
||||
|
||||
def test_shows_running_task_count(self) -> None:
|
||||
session = Session()
|
||||
task = session.task_registry.create(TaskKind.SYNTHETIC_TEST)
|
||||
task.mark_running()
|
||||
assert "1 task running" in _placeholder_text(session)
|
||||
|
||||
second = session.task_registry.create(TaskKind.INVESTIGATION)
|
||||
second.mark_running()
|
||||
assert "2 tasks running" in _placeholder_text(session)
|
||||
|
||||
def test_shows_resumed_session_name(self) -> None:
|
||||
session = Session()
|
||||
session.resumed_from_name = "redis-incident"
|
||||
text = _placeholder_text(session)
|
||||
assert "resumed: redis-incident" in text
|
||||
|
||||
def test_combines_multiple_state_segments(self) -> None:
|
||||
session = Session()
|
||||
session.terminal.trust_mode = True
|
||||
session.resumed_from_name = "redis-incident"
|
||||
task = session.task_registry.create(TaskKind.WATCHDOG)
|
||||
task.mark_running()
|
||||
text = _placeholder_text(session)
|
||||
assert "trust on" in text
|
||||
assert "1 task running" in text
|
||||
assert "resumed: redis-incident" in text
|
||||
assert " · " in text
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeCompleteState:
|
||||
completions: list[Completion]
|
||||
current_completion: Completion | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeBuffer:
|
||||
text: str
|
||||
complete_state: _FakeCompleteState | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeOutput:
|
||||
columns: int = 120
|
||||
|
||||
def get_size(self) -> _FakeOutput:
|
||||
return self
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeApp:
|
||||
current_buffer: _FakeBuffer
|
||||
output: _FakeOutput
|
||||
|
||||
|
||||
class TestCompletionPreviewHint:
|
||||
def test_returns_empty_when_no_app(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: None)
|
||||
assert completion_preview_hint_ansi() == ""
|
||||
|
||||
def test_shows_full_slash_command_description(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
completion = Completion(
|
||||
"/investigate",
|
||||
start_position=-1,
|
||||
display="/investigate",
|
||||
display_meta="Run an RCA investigation from a file or sample templa…",
|
||||
)
|
||||
app = _FakeApp(
|
||||
current_buffer=_FakeBuffer(
|
||||
text="/",
|
||||
complete_state=_FakeCompleteState(
|
||||
completions=[completion],
|
||||
current_completion=completion,
|
||||
),
|
||||
),
|
||||
output=_FakeOutput(),
|
||||
)
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: app)
|
||||
|
||||
rendered = _strip_ansi(completion_preview_hint_ansi())
|
||||
assert rendered.startswith("/investigate — ")
|
||||
assert len(rendered) > len("/investigate — " + completion.display_meta_text)
|
||||
assert "…" not in rendered
|
||||
|
||||
def test_unregistered_slash_completion_uses_display_label(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
completion = Completion(
|
||||
"/plugin-cmd",
|
||||
start_position=-1,
|
||||
display="/plugin-cmd",
|
||||
display_meta="Plugin-provided slash command.",
|
||||
)
|
||||
app = _FakeApp(
|
||||
current_buffer=_FakeBuffer(
|
||||
text="/",
|
||||
complete_state=_FakeCompleteState(
|
||||
completions=[completion],
|
||||
current_completion=completion,
|
||||
),
|
||||
),
|
||||
output=_FakeOutput(),
|
||||
)
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: app)
|
||||
|
||||
rendered = _strip_ansi(completion_preview_hint_ansi())
|
||||
assert rendered == "/plugin-cmd — Plugin-provided slash command."
|
||||
|
||||
def test_shows_subcommand_label_with_parent_command(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
completion = Completion(
|
||||
"high",
|
||||
start_position=-1,
|
||||
display="high",
|
||||
display_meta="favor more thorough reasoning",
|
||||
)
|
||||
app = _FakeApp(
|
||||
current_buffer=_FakeBuffer(
|
||||
text="/effort ",
|
||||
complete_state=_FakeCompleteState(
|
||||
completions=[completion],
|
||||
current_completion=completion,
|
||||
),
|
||||
),
|
||||
output=_FakeOutput(),
|
||||
)
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: app)
|
||||
|
||||
rendered = _strip_ansi(completion_preview_hint_ansi())
|
||||
assert rendered == "/effort high — favor more thorough reasoning"
|
||||
|
||||
def test_falls_back_to_first_completion_when_none_selected(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
first = Completion(
|
||||
"/plugin-cmd",
|
||||
start_position=-1,
|
||||
display="/plugin-cmd",
|
||||
display_meta="Plugin-provided slash command.",
|
||||
)
|
||||
app = _FakeApp(
|
||||
current_buffer=_FakeBuffer(
|
||||
text="/",
|
||||
complete_state=_FakeCompleteState(
|
||||
completions=[first],
|
||||
current_completion=None,
|
||||
),
|
||||
),
|
||||
output=_FakeOutput(),
|
||||
)
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: app)
|
||||
|
||||
rendered = _strip_ansi(completion_preview_hint_ansi())
|
||||
assert rendered == "/plugin-cmd — Plugin-provided slash command."
|
||||
|
||||
def test_clips_preview_to_terminal_width(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
long_meta = (
|
||||
"Plugin-provided slash command with a deliberately long description "
|
||||
"that must be clipped to the terminal width."
|
||||
)
|
||||
completion = Completion(
|
||||
"/plugin-cmd",
|
||||
start_position=-1,
|
||||
display="/plugin-cmd",
|
||||
display_meta=long_meta,
|
||||
)
|
||||
app = _FakeApp(
|
||||
current_buffer=_FakeBuffer(
|
||||
text="/",
|
||||
complete_state=_FakeCompleteState(
|
||||
completions=[completion],
|
||||
current_completion=completion,
|
||||
),
|
||||
),
|
||||
output=_FakeOutput(columns=40),
|
||||
)
|
||||
monkeypatch.setattr(prompt_completion, "get_app_or_none", lambda: app)
|
||||
|
||||
rendered = _strip_ansi(completion_preview_hint_ansi())
|
||||
assert rendered.endswith("…")
|
||||
assert len(rendered) <= 40
|
||||
assert rendered.startswith("/plugin-cmd — ")
|
||||
|
||||
|
||||
class TestResolvePromptPrefix:
|
||||
def test_prefers_inline_spinner_over_completion_preview(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
prompt_rendering,
|
||||
"completion_preview_hint_ansi",
|
||||
lambda: "preview line",
|
||||
)
|
||||
spinner = loop_state.SpinnerState()
|
||||
spinner.start()
|
||||
prefix = resolve_prompt_prefix_ansi(
|
||||
inline_spinner=spinner.inline_spinner_ansi(),
|
||||
idle_hint=spinner.idle_hint_ansi(),
|
||||
)
|
||||
assert "preview line" not in prefix
|
||||
assert "esc to cancel" in _strip_ansi(prefix)
|
||||
|
||||
def test_prefers_completion_preview_over_idle_hint(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
prompt_rendering,
|
||||
"completion_preview_hint_ansi",
|
||||
lambda: "preview line",
|
||||
)
|
||||
spinner = loop_state.SpinnerState()
|
||||
prefix = resolve_prompt_prefix_ansi(
|
||||
inline_spinner=spinner.inline_spinner_ansi(),
|
||||
idle_hint=spinner.idle_hint_ansi(),
|
||||
)
|
||||
assert prefix == "preview line"
|
||||
assert "/ for commands" not in prefix
|
||||
|
||||
def test_falls_back_to_idle_hint_when_no_preview(self) -> None:
|
||||
spinner = loop_state.SpinnerState()
|
||||
prefix = resolve_prompt_prefix_ansi(
|
||||
inline_spinner=spinner.inline_spinner_ansi(),
|
||||
idle_hint=spinner.idle_hint_ansi(),
|
||||
)
|
||||
assert "/ for commands" in _strip_ansi(prefix)
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Tests for structured investigation outcomes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from core.llm.shared.llm_retry import LLMCreditExhaustedError
|
||||
from platform.common.errors import OpenSREError
|
||||
from platform.common.task_types import TaskRecord
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.foreground_investigation import run_foreground_investigation
|
||||
from surfaces.interactive_shell.ui.investigation_outcome import (
|
||||
classify_investigation_failure,
|
||||
normalize_investigation_target,
|
||||
user_facing_error_message,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_investigation_target_template() -> None:
|
||||
assert normalize_investigation_target("generic") == "generic"
|
||||
assert normalize_investigation_target("template:datadog") == "datadog"
|
||||
|
||||
|
||||
def test_normalize_investigation_target_file_path() -> None:
|
||||
assert normalize_investigation_target(
|
||||
"alerts/checkout.json", path=Path("alerts/checkout.json")
|
||||
) == ("checkout.json")
|
||||
|
||||
|
||||
def test_classify_integration_failure() -> None:
|
||||
category, integration, _detail = classify_investigation_failure(
|
||||
RuntimeError("grafana query failed: 401 unauthorized")
|
||||
)
|
||||
assert category == "integration"
|
||||
assert integration == "grafana"
|
||||
|
||||
|
||||
def test_user_facing_error_message_includes_suggestion() -> None:
|
||||
message = user_facing_error_message(
|
||||
OpenSREError("jenkins is not configured", suggestion="Run /integrations setup jenkins")
|
||||
)
|
||||
assert "jenkins is not configured" in message
|
||||
assert "Suggestion:" in message
|
||||
|
||||
|
||||
def test_run_foreground_investigation_early_cancel_omits_stale_investigation_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.last_investigation_id = "inv-old"
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
|
||||
def _raise_interrupt(_task: TaskRecord) -> dict[str, object]:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
outcome = run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate generic",
|
||||
run=_raise_interrupt,
|
||||
exception_context="test",
|
||||
target="generic",
|
||||
)
|
||||
|
||||
assert outcome.status == "cancelled"
|
||||
assert outcome.investigation_id == ""
|
||||
task.mark_cancelled.assert_called_once()
|
||||
|
||||
|
||||
def test_run_foreground_investigation_credit_exhausted_shows_auth_login_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
session = Session()
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
|
||||
def _raise_credit_exhausted(_task: TaskRecord) -> dict[str, object]:
|
||||
raise LLMCreditExhaustedError(
|
||||
"Anthropic credit exhausted (provider billing/quota). Original error: 400"
|
||||
)
|
||||
|
||||
outcome = run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate alert.json",
|
||||
run=_raise_credit_exhausted,
|
||||
exception_context="test",
|
||||
target="alert.json",
|
||||
)
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert outcome.status == "failed"
|
||||
assert "/model" in output
|
||||
assert "/auth login" in output
|
||||
task.mark_failed.assert_called_once()
|
||||
|
||||
|
||||
def test_run_foreground_investigation_opensre_error_does_not_duplicate_auth_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
session = Session()
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
|
||||
def _raise_credit_exhausted_opensre_error(_task: TaskRecord) -> dict[str, object]:
|
||||
raise OpenSREError(
|
||||
"Anthropic credit exhausted (provider billing/quota). Original error: 400",
|
||||
suggestion=(
|
||||
"Run /auth login <provider> to re-authenticate or add a different provider."
|
||||
),
|
||||
)
|
||||
|
||||
outcome = run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate alert.json",
|
||||
run=_raise_credit_exhausted_opensre_error,
|
||||
exception_context="test",
|
||||
target="alert.json",
|
||||
)
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert outcome.status == "failed"
|
||||
assert output.count("/auth login") == 1
|
||||
task.mark_failed.assert_called_once()
|
||||
|
||||
|
||||
def test_run_foreground_investigation_skips_feedback_when_pt_app_running(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""#3690: prompt_app is on session.terminal; must not run raw stdin menu while active."""
|
||||
session = Session()
|
||||
session.terminal.prompt_app = MagicMock(is_running=True)
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
feedback = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.feedback.prompt_investigation_feedback",
|
||||
feedback,
|
||||
)
|
||||
|
||||
outcome = run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate grafana",
|
||||
run=lambda _task: {"root_cause": "sample"},
|
||||
exception_context="test",
|
||||
target="grafana",
|
||||
)
|
||||
|
||||
assert outcome.status == "completed"
|
||||
feedback.assert_not_called()
|
||||
|
||||
|
||||
def test_run_foreground_investigation_prompts_feedback_when_pt_app_idle(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.terminal.prompt_app = MagicMock(is_running=False)
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
feedback = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.feedback.prompt_investigation_feedback",
|
||||
feedback,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.choice_menu.repl_tty_interactive",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.key_reader.restore_stdin_terminal",
|
||||
lambda: None,
|
||||
)
|
||||
|
||||
run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate",
|
||||
run=lambda _task: {"root_cause": "sample"},
|
||||
exception_context="test",
|
||||
target="",
|
||||
)
|
||||
|
||||
feedback.assert_called_once_with({"root_cause": "sample"})
|
||||
|
||||
|
||||
def test_run_foreground_investigation_skips_feedback_on_headless_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Gateway SessionCore must not block on the RCA feedback picker."""
|
||||
from core.agent_harness.session import SessionCore
|
||||
from core.agent_harness.session.persistence.memory import InMemorySessionStorage
|
||||
|
||||
session = SessionCore(storage=InMemorySessionStorage())
|
||||
console = Console(force_terminal=False, color_system=None, highlight=False)
|
||||
task = MagicMock(spec=TaskRecord)
|
||||
task.cancel_requested = False
|
||||
monkeypatch.setattr(
|
||||
session.task_registry,
|
||||
"create",
|
||||
lambda *_args, **_kwargs: task,
|
||||
)
|
||||
feedback = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.feedback.prompt_investigation_feedback",
|
||||
feedback,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.choice_menu.repl_tty_interactive",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
outcome = run_foreground_investigation(
|
||||
session=session,
|
||||
console=console,
|
||||
task_command="/investigate grafana",
|
||||
run=lambda _task: {"root_cause": "sample"},
|
||||
exception_context="test",
|
||||
target="grafana",
|
||||
)
|
||||
|
||||
assert outcome.status == "completed"
|
||||
feedback.assert_not_called()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Tests for terminal cooked-mode restore after raw-mode menus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.ui.components import key_reader
|
||||
|
||||
|
||||
def test_restore_stdin_terminal_recooks_input_flags(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# A raw menu (tty.setraw) clears ICRNL, so Enter (CR) stops submitting until it
|
||||
# is restored. Pin that restore re-enables the cooked-mode flags on a raw snapshot.
|
||||
termios = pytest.importorskip("termios")
|
||||
|
||||
monkeypatch.setattr(key_reader.os, "name", "posix")
|
||||
monkeypatch.setattr(
|
||||
key_reader.sys, "stdin", SimpleNamespace(isatty=lambda: True, fileno=lambda: 0)
|
||||
)
|
||||
|
||||
raw_attrs = [0, 0, 0, 0, 0, 0, []] # iflag/oflag/cflag/lflag all cleared (raw mode)
|
||||
written: dict[str, list[object]] = {}
|
||||
monkeypatch.setattr(termios, "tcgetattr", lambda _fd: list(raw_attrs))
|
||||
monkeypatch.setattr(
|
||||
termios, "tcsetattr", lambda _fd, _when, attrs: written.__setitem__("attrs", attrs)
|
||||
)
|
||||
monkeypatch.setattr(termios, "tcflush", lambda _fd, _queue: None)
|
||||
|
||||
key_reader.restore_stdin_terminal()
|
||||
|
||||
attrs = written["attrs"]
|
||||
assert attrs[0] & termios.ICRNL # CR -> NL so Enter submits again
|
||||
assert attrs[1] & termios.OPOST # output newline post-processing
|
||||
assert attrs[3] & termios.ICANON # line editing
|
||||
assert attrs[3] & termios.ECHO # keystrokes visible
|
||||
|
||||
|
||||
def test_restore_stdin_terminal_is_a_no_op_when_not_a_tty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(key_reader.os, "name", "posix")
|
||||
monkeypatch.setattr(
|
||||
key_reader.sys,
|
||||
"stdin",
|
||||
SimpleNamespace(isatty=lambda: False, fileno=lambda: (_ for _ in ()).throw(AssertionError)),
|
||||
)
|
||||
key_reader.restore_stdin_terminal() # returns without touching termios
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Tests for the shared interactive-shell LLM loader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from typing import Any
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.ui.components import loaders
|
||||
from surfaces.interactive_shell.ui.components.loaders import llm_loader
|
||||
|
||||
|
||||
def _terminal_console() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return (
|
||||
Console(file=buf, force_terminal=True, color_system=None, width=80, highlight=False),
|
||||
buf,
|
||||
)
|
||||
|
||||
|
||||
def _plain_console() -> tuple[Console, io.StringIO]:
|
||||
"""Console that reports ``is_terminal == False`` — the CI / piped case."""
|
||||
buf = io.StringIO()
|
||||
return (Console(file=buf, force_terminal=False, color_system=None, width=80), buf)
|
||||
|
||||
|
||||
class TestLLMLoader:
|
||||
def test_yields_to_caller_and_runs_wrapped_block(self) -> None:
|
||||
console, _ = _terminal_console()
|
||||
ran: list[bool] = []
|
||||
with llm_loader(console):
|
||||
ran.append(True)
|
||||
assert ran == [True]
|
||||
|
||||
def test_skips_spinner_when_console_is_not_a_terminal(self) -> None:
|
||||
"""In CI / piped output we must NOT pollute logs with spinner frames."""
|
||||
console, buf = _plain_console()
|
||||
with llm_loader(console):
|
||||
pass
|
||||
# Nothing should be written — no label, no escape sequences, nothing.
|
||||
assert buf.getvalue() == ""
|
||||
|
||||
def test_loader_uses_subtle_spinner_style(self, monkeypatch: Any) -> None:
|
||||
"""The loader uses a dim, quiet spinner — less visual noise than a bright accent."""
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
# Fake context manager so we can introspect the kwargs without
|
||||
# triggering Rich's Live renderer in the test.
|
||||
class _FakeStatus:
|
||||
def __enter__(self) -> _FakeStatus:
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
def _fake_status(text: str, **kwargs: Any) -> _FakeStatus:
|
||||
captured["text"] = text
|
||||
captured["kwargs"] = kwargs
|
||||
return _FakeStatus()
|
||||
|
||||
console, _ = _terminal_console()
|
||||
monkeypatch.setattr(console, "status", _fake_status)
|
||||
|
||||
with llm_loader(console, label="consulting the model"):
|
||||
pass
|
||||
|
||||
from platform.terminal.theme import SECONDARY
|
||||
|
||||
assert SECONDARY in captured["text"]
|
||||
assert "consulting the model" in captured["text"]
|
||||
assert captured["kwargs"]["spinner"] == "dots"
|
||||
assert captured["kwargs"]["spinner_style"] == SECONDARY
|
||||
|
||||
|
||||
def test_module_exports_loader_and_default_label() -> None:
|
||||
assert "llm_loader" in loaders.__all__
|
||||
assert "DEFAULT_LOADER_LABEL" in loaders.__all__
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Tests for REPL provider/model resolution used by the welcome banner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from core.agent_harness.llm_resolution import resolve_provider_models
|
||||
from surfaces.interactive_shell.ui.tables import provider as ui_provider
|
||||
|
||||
|
||||
def test_ui_provider_reexports_core_resolver() -> None:
|
||||
assert ui_provider.resolve_provider_models is resolve_provider_models
|
||||
|
||||
|
||||
def test_antigravity_cli_reads_model_env(monkeypatch: object) -> None:
|
||||
monkeypatch.setenv("ANTIGRAVITY_CLI_MODEL", "gemini-2.5-pro")
|
||||
|
||||
reasoning, toolcall = resolve_provider_models(SimpleNamespace(), "antigravity-cli")
|
||||
|
||||
assert reasoning == "gemini-2.5-pro"
|
||||
assert toolcall == "gemini-2.5-pro"
|
||||
|
||||
|
||||
def test_antigravity_cli_falls_back_to_cli_default(monkeypatch: object) -> None:
|
||||
monkeypatch.delenv("ANTIGRAVITY_CLI_MODEL", raising=False)
|
||||
|
||||
reasoning, toolcall = resolve_provider_models(SimpleNamespace(), "antigravity-cli")
|
||||
|
||||
assert reasoning == "CLI default"
|
||||
assert toolcall == "CLI default"
|
||||
|
||||
|
||||
def test_antigravity_cli_does_not_use_hyphenated_settings_attr() -> None:
|
||||
"""Regression: hyphenated provider ids are not reachable via getattr(settings, ...)."""
|
||||
settings = SimpleNamespace(**{"antigravity-cli_model": "should-not-win"})
|
||||
|
||||
reasoning, toolcall = resolve_provider_models(settings, "antigravity-cli")
|
||||
|
||||
assert reasoning == "CLI default"
|
||||
assert toolcall == "CLI default"
|
||||
|
||||
|
||||
def test_openai_oauth_displays_codex_model(monkeypatch: object) -> None:
|
||||
monkeypatch.setenv("LLM_AUTH_METHOD", "oauth")
|
||||
monkeypatch.setenv("CODEX_MODEL", "gpt-5.5")
|
||||
|
||||
reasoning, toolcall = resolve_provider_models(SimpleNamespace(), "openai")
|
||||
|
||||
assert reasoning == "gpt-5.5"
|
||||
assert toolcall == "gpt-5.5"
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Tests for Rich rendering helpers used by the interactive shell."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.runtime.core.state import SpinnerState
|
||||
from surfaces.interactive_shell.ui.components.rendering import (
|
||||
_repl_write_buffer,
|
||||
print_repl_json,
|
||||
refresh_welcome_poster,
|
||||
repl_print,
|
||||
repl_render_launch_poster,
|
||||
repl_table,
|
||||
)
|
||||
from surfaces.interactive_shell.ui.streaming.console import StreamingConsole
|
||||
from surfaces.interactive_shell.ui.tables import (
|
||||
render_integrations_table,
|
||||
render_mcp_table,
|
||||
)
|
||||
|
||||
|
||||
def test_repl_table_minimal_box() -> None:
|
||||
t = repl_table(title="T")
|
||||
assert t.title == "T"
|
||||
|
||||
|
||||
def test_print_repl_json_tty_uses_single_buffered_write(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class _FakeStdout:
|
||||
def __init__(self) -> None:
|
||||
self.writes: list[str] = []
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
self.writes.append(text)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def isatty(self) -> bool:
|
||||
return True
|
||||
|
||||
fake_stdout = _FakeStdout()
|
||||
monkeypatch.setattr("sys.stdout", fake_stdout)
|
||||
|
||||
console = Console(file=fake_stdout, force_terminal=True, width=80)
|
||||
print_repl_json(console, '{"ok": true}')
|
||||
|
||||
assert len(fake_stdout.writes) == 1
|
||||
rendered = re.sub(r"\x1b\[[0-9;]*m", "", fake_stdout.writes[0])
|
||||
assert rendered.startswith("\r\n")
|
||||
assert '"ok": true' in rendered
|
||||
|
||||
|
||||
def test_render_integrations_table_empty_shows_hint() -> None:
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False)
|
||||
render_integrations_table(console, [])
|
||||
assert "opensre onboard" in buf.getvalue()
|
||||
|
||||
|
||||
def test_repl_print_resets_before_each_line(monkeypatch) -> None:
|
||||
resets: list[bool] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.choice_menu.prepare_repl_output_line",
|
||||
lambda: resets.append(True),
|
||||
)
|
||||
|
||||
buf = io.StringIO()
|
||||
console = Console(file=buf, force_terminal=False, width=80)
|
||||
repl_print(console, "line one")
|
||||
repl_print(console, "line two")
|
||||
|
||||
assert len(resets) == 2
|
||||
|
||||
|
||||
def test_repl_print_does_not_double_prepare_with_streaming_console(monkeypatch) -> None:
|
||||
resets: list[bool] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.choice_menu.prepare_repl_output_line",
|
||||
lambda: resets.append(True),
|
||||
)
|
||||
|
||||
console = StreamingConsole(
|
||||
SpinnerState(),
|
||||
threading.Event(),
|
||||
file=io.StringIO(),
|
||||
force_terminal=False,
|
||||
width=80,
|
||||
)
|
||||
repl_print(console, "line")
|
||||
|
||||
assert len(resets) == 1
|
||||
|
||||
|
||||
def test_repl_print_streaming_console_prepares_tty_once_when_interactive(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class _FakeStdout:
|
||||
def __init__(self) -> None:
|
||||
self.writes: list[str] = []
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
self.writes.append(text)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def isatty(self) -> bool:
|
||||
return True
|
||||
|
||||
fake_stdout = _FakeStdout()
|
||||
monkeypatch.setattr("sys.stdout", fake_stdout)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.choice_menu.repl_tty_interactive",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
console = StreamingConsole(
|
||||
SpinnerState(),
|
||||
threading.Event(),
|
||||
file=io.StringIO(),
|
||||
force_terminal=False,
|
||||
width=80,
|
||||
)
|
||||
repl_print(console, "line")
|
||||
|
||||
assert fake_stdout.writes == ["\r\n", "\r"]
|
||||
|
||||
|
||||
def test_repl_render_launch_poster_uses_crlf_on_tty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class _FakeStdout:
|
||||
def __init__(self) -> None:
|
||||
self.writes: list[str] = []
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
self.writes.append(text)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def isatty(self) -> bool:
|
||||
return True
|
||||
|
||||
fake_stdout = _FakeStdout()
|
||||
monkeypatch.setattr("sys.stdout", fake_stdout)
|
||||
|
||||
from platform.terminal.theme import set_active_theme
|
||||
|
||||
set_active_theme("blue")
|
||||
console = Console(
|
||||
file=fake_stdout,
|
||||
force_terminal=True,
|
||||
highlight=False,
|
||||
color_system="truecolor",
|
||||
width=120,
|
||||
)
|
||||
repl_render_launch_poster(console, theme_notice="blue")
|
||||
|
||||
written = "".join(fake_stdout.writes)
|
||||
assert "theme set:" in written
|
||||
assert "blue" in written
|
||||
assert "38;2;168;212;255" in written
|
||||
assert "185;237;175" not in written
|
||||
assert "opensre" in written
|
||||
assert "Welcome back" in written
|
||||
assert "\r\n" in written
|
||||
# REPL path must not emit bare \\n (causes double-spaced splash under patch_stdout).
|
||||
assert "\r" not in written.replace("\r\n", "")
|
||||
|
||||
|
||||
def test_repl_write_buffer_strips_only_escaped_cpr_sequences(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class _FakeStdout:
|
||||
def __init__(self) -> None:
|
||||
self.writes: list[str] = []
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
self.writes.append(text)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
fake_stdout = _FakeStdout()
|
||||
monkeypatch.setattr("sys.stdout", fake_stdout)
|
||||
|
||||
_repl_write_buffer("\x1b[1;1Rtheme set: pink 12;5R\r\n")
|
||||
|
||||
written = "".join(fake_stdout.writes)
|
||||
assert "theme set: pink" in written
|
||||
assert "12;5R" in written
|
||||
assert "\x1b[1;1R" not in written
|
||||
|
||||
|
||||
def test_refresh_welcome_poster_drains_cpr_after_clear(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
drains: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.rendering.repl_clear_screen",
|
||||
lambda: drains.append("clear"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.cpr_stdin.drain_stale_cpr_bytes",
|
||||
lambda: drains.append("drain"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.ui.components.rendering.repl_render_launch_poster",
|
||||
lambda *_args, **_kwargs: drains.append("render"),
|
||||
)
|
||||
|
||||
console = Console(file=io.StringIO(), force_terminal=False)
|
||||
refresh_welcome_poster(console)
|
||||
|
||||
assert drains == ["clear", "drain", "render"]
|
||||
|
||||
|
||||
def test_render_integrations_table_renders_content(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
"""print_repl_table on a non-TTY console writes via console.print to stdout.
|
||||
|
||||
The cursor-reset (prepare_repl_output_line) is no longer called from the
|
||||
table rendering path; on a real TTY the blank line and \\r\\n normalisation
|
||||
are folded into a single sys.stdout.write call in print_repl_table.
|
||||
"""
|
||||
console = Console(force_terminal=False, width=80)
|
||||
render_integrations_table(
|
||||
console,
|
||||
[
|
||||
{
|
||||
"service": "grafana",
|
||||
"source": "local store",
|
||||
"status": "passed",
|
||||
"detail": "Connected to https://example.grafana.net",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert "grafana" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_render_integrations_table_sorts_services_and_includes_mcp(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
console = Console(force_terminal=False, width=80)
|
||||
render_integrations_table(
|
||||
console,
|
||||
[
|
||||
{"service": "sentry", "source": "-", "status": "missing", "detail": "missing"},
|
||||
{"service": "github", "source": "-", "status": "missing", "detail": "missing"},
|
||||
{"service": "datadog", "source": "env", "status": "passed", "detail": "ok"},
|
||||
],
|
||||
)
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert output.index("datadog") < output.index("github") < output.index("sentry")
|
||||
assert "github" in output
|
||||
|
||||
|
||||
def test_render_mcp_table_renders_content(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
console = Console(force_terminal=False, width=80)
|
||||
render_mcp_table(
|
||||
console,
|
||||
[
|
||||
{
|
||||
"service": "github",
|
||||
"source": "local store",
|
||||
"status": "configured",
|
||||
"detail": "Connected",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert "github" in capsys.readouterr().out
|
||||
@@ -0,0 +1,983 @@
|
||||
"""Tests for the shared live-streaming renderer used by interactive-shell handlers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.ui.streaming import (
|
||||
format_token_count_short,
|
||||
render_response_header,
|
||||
stream_to_console,
|
||||
)
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
"""Drop ANSI escapes so assertions check the visible output."""
|
||||
return re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", text)
|
||||
|
||||
|
||||
def _tty_console() -> tuple[Console, io.StringIO]:
|
||||
"""Build a Console that thinks it is a terminal so Rich.Live actually renders."""
|
||||
buf = io.StringIO()
|
||||
return (
|
||||
Console(file=buf, force_terminal=True, color_system=None, width=80, highlight=False),
|
||||
buf,
|
||||
)
|
||||
|
||||
|
||||
def _non_tty_console() -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, color_system=None, width=80), buf
|
||||
|
||||
|
||||
def _yield_chunks(chunks: list[str]) -> Iterator[str]:
|
||||
yield from chunks
|
||||
|
||||
|
||||
class TestNonTtyFallback:
|
||||
"""On a non-terminal console the helper drains, prints, and returns the full text."""
|
||||
|
||||
def test_drains_stream_and_prints_without_live_artifacts(self) -> None:
|
||||
console, buf = _non_tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["Hel", "lo, ", "world"]),
|
||||
)
|
||||
|
||||
output = buf.getvalue()
|
||||
assert result == "Hello, world"
|
||||
# Bullet header + label + text reach piped output so captured
|
||||
# logs are useful. ``●`` is the row marker; ``assistant`` is the
|
||||
# dim label alongside it.
|
||||
assert "●" in output
|
||||
assert "assistant" in output
|
||||
assert "Hello, world" in output
|
||||
# No spinner / Live cursor-movement artifacts in non-TTY captures.
|
||||
assert "thinking" not in output
|
||||
|
||||
def test_suppression_drains_silently_in_non_tty(self) -> None:
|
||||
"""Suppressed payloads (machine-readable payloads) must not appear in piped output."""
|
||||
console, buf = _non_tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(['{"actions"', ":[]}"]),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
|
||||
assert result == '{"actions":[]}'
|
||||
output = buf.getvalue()
|
||||
# No bullet header for suppressed responses.
|
||||
assert "●" not in output
|
||||
assert '{"actions"' not in output
|
||||
|
||||
|
||||
class TestTtyParagraphRender:
|
||||
"""On a terminal console paragraphs render as Markdown the moment
|
||||
each ``\\n\\n`` boundary closes them; the final paragraph is
|
||||
force-flushed at end-of-stream. Code blocks are kept whole (we
|
||||
don't split mid-fence). The spinner indicator drives the live
|
||||
streaming feedback within a paragraph.
|
||||
"""
|
||||
|
||||
def test_renders_label_and_streamed_content_as_markdown(self) -> None:
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["Run **opensre", " investigate** to start."]),
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert result == "Run **opensre investigate** to start."
|
||||
# Bullet row marker pinned above the rendered paragraph.
|
||||
assert "●" in output
|
||||
# End-of-stream force-flush rendered Markdown — ``**`` stripped.
|
||||
assert "**opensre" not in output
|
||||
assert "opensre investigate" in output
|
||||
|
||||
def test_renders_first_paragraph_before_second_completes(self) -> None:
|
||||
"""A complete paragraph (``\\n\\n``) flushes immediately, even
|
||||
when more chunks would still arrive after it. The second
|
||||
paragraph stays buffered until its own boundary or EOS."""
|
||||
chunks: list[str] = []
|
||||
|
||||
def _capture_chunks() -> Iterator[str]:
|
||||
for c in ["First **para**.\n\n", "Second **para**."]:
|
||||
chunks.append(c)
|
||||
yield c
|
||||
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_capture_chunks(),
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert result == "First **para**.\n\nSecond **para**."
|
||||
# Both paragraphs are rendered (``**`` stripped).
|
||||
assert "First para." in output
|
||||
assert "Second para." in output
|
||||
assert "**para**" not in output
|
||||
|
||||
def test_paragraph_break_across_chunk_boundary_flushes(self) -> None:
|
||||
"""The cross-chunk seam — chunk N ends with ``\\n``, chunk N+1
|
||||
starts with ``\\n`` — must be detected as a paragraph break.
|
||||
|
||||
Without the seam check the fast-path skips the join (no
|
||||
``\\n\\n`` *inside* either chunk) and the boundary is missed
|
||||
until end-of-stream.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
# Patch only on this thread; restored by the test fixture's GC.
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
# ``"first.\n"`` then ``"\nsecond."`` — neither chunk
|
||||
# contains ``\n\n`` standalone, but joined they form a
|
||||
# paragraph break at the seam.
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["first.\n", "\nsecond."]),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 2 parses: first paragraph flushed at the seam, then second
|
||||
# tail force-flushed at end-of-stream.
|
||||
assert parse_count[0] == 2, (
|
||||
f"seam check missed the cross-chunk break — got {parse_count[0]} parses"
|
||||
)
|
||||
|
||||
def test_peeked_chunks_seed_prev_chunk_for_seam_detection(self) -> None:
|
||||
"""When ``suppress_if_starts_with`` peeks chunks but doesn't
|
||||
suppress, those peeked chunks become history for the seam
|
||||
check on the very first main-loop chunk.
|
||||
|
||||
Concretely: suppression-peek pulls ``"hello\\n"`` (didn't match
|
||||
``"{"``); main loop starts with ``"\\nworld"``. The seam should
|
||||
be detected — ``peeked[-1]`` is the initial ``prev_chunk``.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["hello\n", "\nworld"]),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 2 parses — peeked chunk + first main-loop chunk form a seam,
|
||||
# producing one paragraph; tail is force-flushed at EOS.
|
||||
assert parse_count[0] == 2
|
||||
|
||||
def test_open_code_block_is_not_split_mid_fence(self) -> None:
|
||||
"""``\\n\\n`` inside an open code block must NOT trigger a
|
||||
flush — splitting would render a partial fenced block whose
|
||||
formatting breaks. The fence stays whole until it closes."""
|
||||
chunks_with_open_fence = [
|
||||
"Header\n\n",
|
||||
"```python\n",
|
||||
"x = 1\n\n", # blank line inside code block — must not flush
|
||||
"y = 2\n",
|
||||
"```\n\n",
|
||||
"Trailing.",
|
||||
]
|
||||
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(chunks_with_open_fence),
|
||||
)
|
||||
|
||||
# Full text returned unchanged.
|
||||
assert "x = 1" in result
|
||||
assert "y = 2" in result
|
||||
# Both code lines must appear in the rendered output (i.e. the
|
||||
# fence wasn't split before its closing ``` was seen).
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "x = 1" in output
|
||||
assert "y = 2" in output
|
||||
assert "Trailing" in output
|
||||
|
||||
def test_post_fence_paragraph_renders_after_block_with_embedded_blank_line(
|
||||
self,
|
||||
) -> None:
|
||||
"""A code block containing a blank line must not bury later
|
||||
paragraphs at EOS. Once the closing fence arrives, the
|
||||
completed code-block paragraph flushes and any subsequent
|
||||
``\\n\\n``-terminated paragraph flushes mid-stream too.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(
|
||||
[
|
||||
"Intro.\n\n",
|
||||
"```python\n",
|
||||
"x = 1\n\n",
|
||||
"y = 2\n",
|
||||
"```\n\n",
|
||||
"Conclusion.\n\n",
|
||||
]
|
||||
),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 3 parses — intro flushes on its own ``\n\n``; the fenced block
|
||||
# flushes once the closing fence + ``\n\n`` arrive (skipping the
|
||||
# embedded blank line); conclusion flushes on its own ``\n\n``
|
||||
# before EOS. Without the skip-past-open-fence logic, the fenced
|
||||
# block + conclusion would defer to a single force-flush at EOS
|
||||
# → 2 parses.
|
||||
assert parse_count[0] == 3, (
|
||||
f"post-fence paragraph deferred to EOS — got {parse_count[0]} parses"
|
||||
)
|
||||
|
||||
def test_multiple_blank_lines_inside_single_fence_render_as_one_block(
|
||||
self,
|
||||
) -> None:
|
||||
"""A single code block with several embedded ``\\n\\n`` must render
|
||||
once when its fence closes. Exercises ``search_from`` advancing
|
||||
repeatedly within a single ``_flush_paragraphs`` call.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(
|
||||
[
|
||||
"Intro.\n\n",
|
||||
"```python\n",
|
||||
"a\n\n", # first embedded blank
|
||||
"b\n\n", # second embedded blank
|
||||
"c\n\n", # third embedded blank
|
||||
"d\n",
|
||||
"```\n\n",
|
||||
"After.\n\n",
|
||||
]
|
||||
),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 3 parses with the fix: intro + fenced block (rendered once when
|
||||
# fence closes, after 3 skip iterations advance search_from past
|
||||
# each embedded blank) + after. Without the fix, the inner loop
|
||||
# would break on the first odd-fence boundary and defer the block
|
||||
# + after to a single EOS force-flush → 2 parses.
|
||||
assert parse_count[0] == 3, (
|
||||
f"expected 3 parses (intro + block + after), got {parse_count[0]}"
|
||||
)
|
||||
|
||||
def test_two_consecutive_fences_each_with_blank_line_render_independently(
|
||||
self,
|
||||
) -> None:
|
||||
"""Two fenced blocks back-to-back, each containing an embedded
|
||||
``\\n\\n``. Each block must render as its own paragraph when its
|
||||
fence closes — ``search_from`` is reset to 0 after each render so
|
||||
the second block isn't blocked by stale state from the first.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(
|
||||
[
|
||||
"Intro.\n\n",
|
||||
"```py\nfoo\n\nbar\n```\n\n", # block 1, embedded blank
|
||||
"```py\nbaz\n\nqux\n```\n\n", # block 2, embedded blank
|
||||
"End.\n\n",
|
||||
]
|
||||
),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 4 parses with the fix: intro + block1 + block2 + end. Without
|
||||
# the fix, block1's leading embedded ``\n\n`` would lock the inner
|
||||
# loop on the odd-fence break for every subsequent chunk, so the
|
||||
# entire tail (block1 + block2 + end) collapses into one EOS
|
||||
# force-flush → 2 parses total.
|
||||
assert parse_count[0] == 4, (
|
||||
f"expected 4 parses (intro + 2 blocks + end), got {parse_count[0]}"
|
||||
)
|
||||
|
||||
def test_inline_triple_backtick_mention_does_not_block_paragraph(self) -> None:
|
||||
"""Single inline ``\\`\\`\\``` mention inside flowing text must not
|
||||
be miscounted as an open fence. The substring count would flip to
|
||||
odd (1), skipping the paragraph's ``\\n\\n`` boundary and deferring
|
||||
rendering to EOS. Only line-start fences count, so two paragraphs
|
||||
each render incrementally as their ``\\n\\n`` arrives.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, _ = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(
|
||||
[
|
||||
"The ``` marker opens a code block in markdown.\n\n",
|
||||
"Use it whenever you want to fence example code.\n\n",
|
||||
]
|
||||
),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 2 parses: each paragraph flushes on its own ``\n\n``. Without
|
||||
# the line-start fence check, the single inline ``` would flip
|
||||
# the count to odd, both ``\n\n`` boundaries would be skipped,
|
||||
# and the whole stream would force-flush as 1 parse at EOS.
|
||||
assert parse_count[0] == 2, (
|
||||
f"inline ``` mention blocked paragraph flush — got {parse_count[0]} parses"
|
||||
)
|
||||
|
||||
def test_mid_line_triple_backtick_does_not_count_as_fence(self) -> None:
|
||||
"""A ``\\`\\`\\``` that appears mid-line (not at line start) must
|
||||
NOT be counted as a fence boundary by the parity check. Real
|
||||
code blocks must keep accumulating across paragraphs until a
|
||||
line-start closing fence arrives — the mid-line backticks are
|
||||
inline content (often quoted/embedded in prose), not Markdown
|
||||
syntax.
|
||||
|
||||
Regression for the drive-by review point: a chunk like
|
||||
``\\nresult: ok\\`\\`\\`\\nmore text`` (closing-fence-shaped
|
||||
characters mid-line because the chunk boundary fell there)
|
||||
used to be a worry. The ``^\\`\\`\\``` regex with
|
||||
``re.MULTILINE`` matches only line-start fences, so this
|
||||
scenario stays correct.
|
||||
"""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
original_markdown = streaming_module.Markdown
|
||||
streaming_module.Markdown = _SpyMarkdown
|
||||
try:
|
||||
console, buf = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(
|
||||
[
|
||||
# Real fence opens at line start.
|
||||
"```py\n",
|
||||
"x = 1\n",
|
||||
# Mid-line backticks inside the still-open fence.
|
||||
# MUST be ignored by the parity check — fence
|
||||
# stays open until a real closing fence at line
|
||||
# start.
|
||||
"result: ok```\n",
|
||||
"y = 2\n",
|
||||
# Now the real closing fence at line start.
|
||||
"```\n\n",
|
||||
"After the block.\n\n",
|
||||
]
|
||||
),
|
||||
)
|
||||
finally:
|
||||
streaming_module.Markdown = original_markdown
|
||||
|
||||
# 2 parses: (1) the entire fenced code block as one Markdown
|
||||
# parse — proves the mid-line ``` didn't prematurely flush it;
|
||||
# (2) the "After the block." paragraph. Without the line-anchor,
|
||||
# the mid-line ``` would flip the parity, close the fence
|
||||
# early, and we'd see 3+ parses with broken code rendering.
|
||||
assert parse_count[0] == 2, f"mid-line ``` was miscounted — got {parse_count[0]} parses"
|
||||
# And the mid-line backticks must reach the rendered output as
|
||||
# plain text (inside the code block), not get eaten as syntax.
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "result: ok" in output
|
||||
|
||||
def test_unclosed_fence_with_embedded_blank_line_renders_at_eos(self) -> None:
|
||||
"""Unclosed fence containing an embedded ``\\n\\n`` must not hang
|
||||
the inner loop and must surface the partial buffer at end-of-stream.
|
||||
|
||||
With the skip-past-open-fence logic, the inner loop advances
|
||||
``search_from`` past each embedded ``\\n\\n``, eventually returns
|
||||
``-1`` from ``find``, and exits cleanly. The outer ``finally``
|
||||
then force-flushes the partial buffer so the user sees the
|
||||
truncated response rather than nothing.
|
||||
"""
|
||||
chunks = [
|
||||
"```py\n",
|
||||
"a = 1\n\n", # blank inside fence
|
||||
"b = 2\n", # stream ends without closing the fence
|
||||
]
|
||||
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(chunks),
|
||||
)
|
||||
|
||||
# Both code lines must appear in the rendered output — the
|
||||
# partial fence is force-flushed at EOS so the user sees what
|
||||
# was streamed before the LLM cut off.
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "a = 1" in output
|
||||
assert "b = 2" in output
|
||||
assert "a = 1" in result
|
||||
assert "b = 2" in result
|
||||
|
||||
def test_returns_empty_string_when_stream_is_empty(self) -> None:
|
||||
"""An empty stream must not leave a frozen spinner on screen."""
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks([]),
|
||||
)
|
||||
|
||||
assert result == ""
|
||||
# Bullet still printed (header fires before chunk processing),
|
||||
# but no spinner residue at finalize.
|
||||
assert "●" in _strip_ansi(buf.getvalue())
|
||||
|
||||
|
||||
class TestMidStreamError:
|
||||
"""Errors inside the stream propagate while the partial buffer stays on screen."""
|
||||
|
||||
def test_exception_propagates_with_partial_visible(self) -> None:
|
||||
def _broken_stream() -> Iterator[str]:
|
||||
yield "partial "
|
||||
yield "answer"
|
||||
raise RuntimeError("upstream 503")
|
||||
|
||||
console, buf = _tty_console()
|
||||
|
||||
with pytest.raises(RuntimeError, match="upstream 503"):
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_broken_stream(),
|
||||
)
|
||||
|
||||
# The partial response was rendered before the exception propagated,
|
||||
# so the caller can surface an error label below it.
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "partial answer" in output
|
||||
|
||||
def test_keyboard_interrupt_propagates_with_partial_visible(self) -> None:
|
||||
"""KeyboardInterrupt mid-stream propagates after the partial renders.
|
||||
|
||||
The double-press absorption logic that used to live here was moved
|
||||
to the prompt_toolkit cancel key bindings (see
|
||||
:func:`interactive_shell.ui.input_prompt.key_bindings.build_cancel_key_bindings`)
|
||||
— the streaming code just lets ``KeyboardInterrupt`` propagate,
|
||||
and the ``finally`` block in :func:`stream_to_console` ensures
|
||||
the partial buffer is rendered.
|
||||
"""
|
||||
|
||||
class _ChunksThenKbd:
|
||||
__slots__ = ("_i",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._i = 0
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return self
|
||||
|
||||
def __next__(self) -> str:
|
||||
parts = ("partial ", "answer")
|
||||
if self._i < len(parts):
|
||||
c = parts[self._i]
|
||||
self._i += 1
|
||||
return c
|
||||
raise KeyboardInterrupt
|
||||
|
||||
console, buf = _tty_console()
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=iter(_ChunksThenKbd()),
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
# Partial is rendered before the KI propagates — the ``finally``
|
||||
# in stream_to_console fires the Markdown render of the buffer.
|
||||
assert "partial answer" in output
|
||||
|
||||
|
||||
class TestTimingFooter:
|
||||
"""A small dim ``· Ns`` footer appears after a rendered live response."""
|
||||
|
||||
def test_footer_printed_after_streamed_response(self) -> None:
|
||||
console, buf = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["hello"]),
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert re.search(r"·\s+\d+\.\d+s", output) is not None
|
||||
|
||||
def test_footer_skipped_when_stream_is_empty(self) -> None:
|
||||
"""Empty stream must not print a timing footer under nothing."""
|
||||
console, buf = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks([]),
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert re.search(r"·\s+\d+\.\d+s", output) is None
|
||||
|
||||
def test_footer_skipped_when_response_is_suppressed(self) -> None:
|
||||
"""Suppressed machine-readable payloads should not get a timing footer either."""
|
||||
console, buf = _tty_console()
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(['{"actions"', ":[]}"]),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert re.search(r"·\s+\d+\.\d+s", output) is None
|
||||
|
||||
|
||||
class TestRenderResponseHeader:
|
||||
"""``render_response_header`` is the bullet-row marker shared with
|
||||
``action_turn.run_action_tool_turn`` — three call sites collapsed
|
||||
to one helper, so we lock in the visible output here.
|
||||
"""
|
||||
|
||||
def test_emits_bullet_glyph_and_label(self) -> None:
|
||||
console, buf = _tty_console()
|
||||
render_response_header(console, "assistant")
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "●" in output
|
||||
assert "assistant" in output
|
||||
|
||||
def test_label_is_passthrough(self) -> None:
|
||||
"""The function takes the label verbatim — callers pass either
|
||||
``STREAM_LABEL_ANSWER`` or ``STREAM_LABEL_ASSISTANT`` (or any
|
||||
free-form word). No filtering, no defaults."""
|
||||
console, buf = _tty_console()
|
||||
render_response_header(console, "answer")
|
||||
assert "answer" in _strip_ansi(buf.getvalue())
|
||||
|
||||
|
||||
class TestFormatTokenCountShort:
|
||||
"""Shared helper used by both the streaming footer and the live spinner."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("count", "expected"),
|
||||
[
|
||||
(0, "0"),
|
||||
(1, "1"),
|
||||
(999, "999"),
|
||||
(1000, "1.0k"),
|
||||
(1234, "1.2k"),
|
||||
(10000, "10.0k"),
|
||||
(123456, "123.5k"),
|
||||
],
|
||||
)
|
||||
def test_formats_at_boundaries(self, count: int, expected: str) -> None:
|
||||
assert format_token_count_short(count) == expected
|
||||
|
||||
|
||||
class _ProgressConsole(Console):
|
||||
"""Console with the loop's :class:`StreamingConsole` shape — exposes
|
||||
``update_streaming_progress`` and ``cancel_requested`` for the
|
||||
streaming layer's ``getattr`` dispatch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cancel_event: threading.Event | None = None,
|
||||
cancel_after_n_progress_calls: int | None = None,
|
||||
**kwargs: object,
|
||||
) -> None:
|
||||
super().__init__(**kwargs) # type: ignore[arg-type]
|
||||
self.progress_calls: list[int] = []
|
||||
self._cancel_event = cancel_event or threading.Event()
|
||||
self._cancel_after = cancel_after_n_progress_calls
|
||||
|
||||
def update_streaming_progress(self, bytes_received: int) -> None:
|
||||
self.progress_calls.append(bytes_received)
|
||||
if self._cancel_after is not None and len(self.progress_calls) >= self._cancel_after:
|
||||
self._cancel_event.set()
|
||||
|
||||
@property
|
||||
def cancel_requested(self) -> bool:
|
||||
return self._cancel_event.is_set()
|
||||
|
||||
|
||||
class TestProgressHook:
|
||||
"""``stream_to_console`` invokes the optional ``update_streaming_progress``
|
||||
hook on the console and throttles the call rate so worker-thread → UI
|
||||
cross-thread queueing isn't flooded on long streams.
|
||||
"""
|
||||
|
||||
def test_progress_hook_called_with_running_byte_count(self) -> None:
|
||||
buf = io.StringIO()
|
||||
console = _ProgressConsole(file=buf, force_terminal=True, color_system=None, width=80)
|
||||
chunks = ["Hello, ", "world", "!"]
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(chunks),
|
||||
)
|
||||
|
||||
assert result == "Hello, world!"
|
||||
assert console.progress_calls, "progress hook never fired"
|
||||
# Counts must be monotonically non-decreasing — the streaming
|
||||
# layer pushes a *running* byte total, never a per-chunk delta.
|
||||
assert console.progress_calls == sorted(console.progress_calls)
|
||||
# Each reported count must reflect bytes that *had* arrived by
|
||||
# that point in the stream — never exceed the final total.
|
||||
assert console.progress_calls[-1] <= len(result)
|
||||
|
||||
def test_progress_hook_throttled_on_burst_streams(self) -> None:
|
||||
"""A burst of 200 small chunks must not produce 200 hook calls.
|
||||
|
||||
Throttling target is ~10/s; the test stream finishes well under
|
||||
a second so we expect a small handful of calls (not one per
|
||||
chunk). The exact count is timing-dependent — assert ``<= 50``
|
||||
as a generous upper bound that still proves throttling fires.
|
||||
"""
|
||||
buf = io.StringIO()
|
||||
console = _ProgressConsole(file=buf, force_terminal=True, color_system=None, width=80)
|
||||
burst = ["x"] * 200
|
||||
stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(burst),
|
||||
)
|
||||
|
||||
assert len(console.progress_calls) <= 50, (
|
||||
f"throttle did not fire — got {len(console.progress_calls)} calls"
|
||||
)
|
||||
|
||||
def test_no_hook_when_console_lacks_method(self) -> None:
|
||||
"""Plain ``Console`` (no progress method) must stream cleanly."""
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["alpha", "beta"]),
|
||||
)
|
||||
assert result == "alphabeta"
|
||||
|
||||
def test_progress_hook_failure_does_not_truncate_response(self) -> None:
|
||||
"""A flaky status widget must never lose response content."""
|
||||
|
||||
class _BrokenConsole(Console):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
file=io.StringIO(),
|
||||
force_terminal=True,
|
||||
color_system=None,
|
||||
width=80,
|
||||
)
|
||||
|
||||
def update_streaming_progress(self, bytes_received: int) -> None: # noqa: ARG002
|
||||
raise RuntimeError("widget gone")
|
||||
|
||||
console = _BrokenConsole()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["full ", "answer"]),
|
||||
)
|
||||
assert result == "full answer"
|
||||
|
||||
|
||||
class TestParagraphFlushThrottle:
|
||||
"""Long single-paragraph streams must not pay O(n²) work re-joining
|
||||
the buffer on every chunk. The fast-path skips the join when no
|
||||
paragraph boundary could possibly land in the new chunk.
|
||||
"""
|
||||
|
||||
def _spy_markdown_parses(self, monkeypatch: pytest.MonkeyPatch) -> list[int]:
|
||||
"""Wrap ``streaming.Markdown`` so each construction increments a counter."""
|
||||
from surfaces.interactive_shell.ui import streaming as streaming_module
|
||||
|
||||
parse_count = [0]
|
||||
real_markdown = streaming_module.Markdown
|
||||
|
||||
class _SpyMarkdown(real_markdown): # type: ignore[misc, valid-type]
|
||||
def __init__(self, text: str, **kwargs) -> None:
|
||||
parse_count[0] += 1
|
||||
super().__init__(text, **kwargs)
|
||||
|
||||
monkeypatch.setattr(streaming_module, "Markdown", _SpyMarkdown)
|
||||
return parse_count
|
||||
|
||||
def test_long_single_paragraph_renders_once(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""No ``\\n\\n`` in any chunk → the only Markdown parse is the
|
||||
end-of-stream force-flush. Proves the fast-path skips the join
|
||||
on every intermediate chunk."""
|
||||
parse_count = self._spy_markdown_parses(monkeypatch)
|
||||
console, _ = _tty_console()
|
||||
|
||||
# 500 chunks, each a few words, no paragraph breaks.
|
||||
chunks = [f"word{i} " for i in range(500)]
|
||||
result = stream_to_console(console, label="assistant", chunks=_yield_chunks(chunks))
|
||||
|
||||
assert "word0" in result
|
||||
assert "word499" in result
|
||||
# End-of-stream force-flush is the only Markdown construction.
|
||||
assert parse_count[0] == 1, f"expected 1 parse (force-flush), got {parse_count[0]}"
|
||||
|
||||
def test_paragraph_boundary_per_chunk_renders_once_per_paragraph(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Each ``\\n\\n`` boundary triggers exactly one Markdown
|
||||
parse — the trailing tail is force-flushed at end."""
|
||||
parse_count = self._spy_markdown_parses(monkeypatch)
|
||||
console, _ = _tty_console()
|
||||
|
||||
chunks = [
|
||||
"para 1.\n\n",
|
||||
"para 2.\n\n",
|
||||
"para 3.\n\n",
|
||||
"trailing tail",
|
||||
]
|
||||
stream_to_console(console, label="assistant", chunks=_yield_chunks(chunks))
|
||||
|
||||
# 3 in-loop renders + 1 force-flush at end = 4 total.
|
||||
assert parse_count[0] == 4
|
||||
|
||||
def test_chunks_with_only_single_newlines_skip_flush(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Lists / code with single ``\\n`` separators don't trigger
|
||||
flush until a real ``\\n\\n`` boundary closes the block.
|
||||
Keeping multi-line list/table syntax intact is required for
|
||||
Rich's Markdown renderer to produce a proper Table / bullet
|
||||
list rather than rendering each row as a standalone block."""
|
||||
parse_count = self._spy_markdown_parses(monkeypatch)
|
||||
console, _ = _tty_console()
|
||||
|
||||
chunks = [
|
||||
"- item 1\n",
|
||||
"- item 2\n",
|
||||
"- item 3\n",
|
||||
# No \n\n — only force-flush at end.
|
||||
]
|
||||
stream_to_console(console, label="assistant", chunks=_yield_chunks(chunks))
|
||||
|
||||
assert parse_count[0] == 1
|
||||
|
||||
|
||||
class TestCancelPolling:
|
||||
"""``stream_to_console`` polls ``console.cancel_requested`` between
|
||||
chunks so an Esc-driven cancel signal stops the worker-thread stream
|
||||
before it drains the iterator.
|
||||
"""
|
||||
|
||||
def test_cancel_set_before_stream_returns_empty_partial(self) -> None:
|
||||
buf = io.StringIO()
|
||||
cancel_event = threading.Event()
|
||||
cancel_event.set() # cancel before any chunk is pulled
|
||||
console = _ProgressConsole(
|
||||
cancel_event=cancel_event,
|
||||
file=buf,
|
||||
force_terminal=True,
|
||||
color_system=None,
|
||||
width=80,
|
||||
)
|
||||
|
||||
# If the cancel poll didn't work, the iterator below would
|
||||
# raise (it's a single-use generator).
|
||||
chunks_iter = _yield_chunks(["a", "b", "c"])
|
||||
result = stream_to_console(console, label="assistant", chunks=chunks_iter)
|
||||
assert result == ""
|
||||
|
||||
def test_cancel_mid_stream_truncates_buffer(self) -> None:
|
||||
"""Cancel signalled mid-stream stops further chunk reads.
|
||||
|
||||
Uses a generator that flips the cancel flag from inside its own
|
||||
yield loop — that's deterministic regardless of throttling, since
|
||||
the next iteration of ``stream_to_console``'s loop checks the
|
||||
cancel flag *before* pulling the next chunk.
|
||||
"""
|
||||
buf = io.StringIO()
|
||||
cancel_event = threading.Event()
|
||||
console = _ProgressConsole(
|
||||
cancel_event=cancel_event,
|
||||
file=buf,
|
||||
force_terminal=True,
|
||||
color_system=None,
|
||||
width=80,
|
||||
)
|
||||
|
||||
chunks_yielded: list[int] = []
|
||||
|
||||
def _chunks_with_cancel() -> Iterator[str]:
|
||||
for i in range(20):
|
||||
chunks_yielded.append(i)
|
||||
if i == 3:
|
||||
cancel_event.set()
|
||||
yield f"chunk{i} "
|
||||
|
||||
result = stream_to_console(console, label="assistant", chunks=_chunks_with_cancel())
|
||||
|
||||
# The generator should not have been pumped through to chunk 19 —
|
||||
# ``stream_to_console`` should have broken out of its loop once
|
||||
# the cancel event was visible.
|
||||
assert max(chunks_yielded) < 19, (
|
||||
f"generator yielded too many chunks — got up to {max(chunks_yielded)}"
|
||||
)
|
||||
# The result must include chunks read before the cancel was
|
||||
# observed and must not include the trailing chunks.
|
||||
assert result.startswith("chunk0 ")
|
||||
assert "chunk19" not in result
|
||||
|
||||
def test_no_cancel_attr_means_stream_runs_to_completion(self) -> None:
|
||||
"""A console without ``cancel_requested`` must drain normally."""
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["one ", "two ", "three"]),
|
||||
)
|
||||
assert result == "one two three"
|
||||
|
||||
|
||||
class TestSuppressionPeek:
|
||||
"""``suppress_if_starts_with`` skips live rendering for content the caller will handle."""
|
||||
|
||||
def test_suppresses_and_drains_when_first_char_matches(self) -> None:
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(['{"actions"', ":[]", "}"]),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
|
||||
assert result == '{"actions":[]}'
|
||||
# No bullet header, no markdown, no live-region artifacts.
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "●" not in output
|
||||
assert '{"actions"' not in output
|
||||
|
||||
def test_renders_normally_when_first_char_does_not_match(self) -> None:
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks(["Hello, ", "world"]),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
|
||||
assert result == "Hello, world"
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "●" in output
|
||||
assert "Hello, world" in output
|
||||
|
||||
def test_skips_leading_whitespace_before_deciding(self) -> None:
|
||||
"""Leading whitespace must not block the suppression peek."""
|
||||
console, buf = _tty_console()
|
||||
result = stream_to_console(
|
||||
console,
|
||||
label="assistant",
|
||||
chunks=_yield_chunks([" \n", '{"action"', ':"slash"}']),
|
||||
suppress_if_starts_with="{",
|
||||
)
|
||||
|
||||
assert result == ' \n{"action":"slash"}'
|
||||
output = _strip_ansi(buf.getvalue())
|
||||
assert "●" not in output
|
||||
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from surfaces.interactive_shell.ui.components.time_format import (
|
||||
format_repl_duration,
|
||||
format_repl_timestamp,
|
||||
)
|
||||
|
||||
|
||||
def test_format_repl_duration() -> None:
|
||||
assert format_repl_duration(None) == "—"
|
||||
assert format_repl_duration(45) == "45s"
|
||||
assert format_repl_duration(125) == "2m 5s"
|
||||
assert format_repl_duration(3725) == "1h 2m"
|
||||
|
||||
|
||||
def test_format_repl_timestamp_iso_table_style() -> None:
|
||||
dt = datetime(2026, 5, 29, 10, 15, tzinfo=UTC)
|
||||
assert format_repl_timestamp(dt.isoformat(), style="table") == dt.astimezone().strftime(
|
||||
"%Y-%m-%d %H:%M"
|
||||
)
|
||||
|
||||
|
||||
def test_format_repl_timestamp_compact_style() -> None:
|
||||
dt = datetime(2026, 5, 29, 10, 15, tzinfo=UTC)
|
||||
assert format_repl_timestamp(dt, style="compact") == dt.astimezone().strftime("%m-%d %H:%M")
|
||||
|
||||
|
||||
def test_format_repl_timestamp_unix_utc_style() -> None:
|
||||
ts = datetime(2026, 5, 29, 10, 15, tzinfo=UTC).timestamp()
|
||||
assert format_repl_timestamp(ts, style="utc") == "2026-05-29 10:15:00 UTC"
|
||||
|
||||
|
||||
def test_format_repl_timestamp_invalid_iso_fallback() -> None:
|
||||
assert format_repl_timestamp("not-a-timestamp", style="table") == "not-a-timestamp"
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Tests for the registered-tool catalog used by the ``/tools list`` slash command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from core.tool_framework.registered_tool import RegisteredTool
|
||||
from surfaces.interactive_shell.command_registry import dispatch_slash
|
||||
from surfaces.interactive_shell.command_registry.tools_cmds import _TOOLS_FIRST_ARGS, _cmd_tools
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.ui.tables import tool_catalog
|
||||
from surfaces.interactive_shell.ui.tables.tool_catalog import (
|
||||
ToolCatalogEntry,
|
||||
_summarize_input_schema,
|
||||
build_tool_catalog,
|
||||
format_tool_catalog_text,
|
||||
)
|
||||
|
||||
|
||||
def _make_tool(
|
||||
name: str,
|
||||
*,
|
||||
description: str = "Tool description.",
|
||||
surfaces: tuple[str, ...] = ("investigation",),
|
||||
input_schema: dict[str, Any] | None = None,
|
||||
origin_module: str = "tools.registry",
|
||||
) -> RegisteredTool:
|
||||
"""Construct a minimal RegisteredTool stub for catalog rendering tests."""
|
||||
return RegisteredTool(
|
||||
name=name,
|
||||
description=description,
|
||||
input_schema=input_schema or {"type": "object", "properties": {}, "required": []},
|
||||
source="knowledge",
|
||||
run=lambda **_: None,
|
||||
surfaces=surfaces,
|
||||
origin_module=origin_module,
|
||||
origin_name=name,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_registry(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[RegisteredTool]]:
|
||||
"""Replace the live registry with a curated set so tests are deterministic."""
|
||||
tools: list[RegisteredTool] = []
|
||||
|
||||
def _fake_get_registered_tools(surface: str | None = None) -> list[RegisteredTool]:
|
||||
if surface is None:
|
||||
return list(tools)
|
||||
return [t for t in tools if surface in t.surfaces]
|
||||
|
||||
monkeypatch.setattr(tool_catalog, "get_registered_tools", _fake_get_registered_tools)
|
||||
yield tools
|
||||
|
||||
|
||||
class TestSummarizeInputSchema:
|
||||
def test_empty_schema_renders_no_params(self) -> None:
|
||||
assert _summarize_input_schema({}) == "(no params)"
|
||||
assert (
|
||||
_summarize_input_schema({"type": "object", "properties": {}, "required": []})
|
||||
== "(no params)"
|
||||
)
|
||||
|
||||
def test_required_params_render_without_question_mark(self) -> None:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}, "limit": {"type": "integer"}},
|
||||
"required": ["query", "limit"],
|
||||
}
|
||||
assert _summarize_input_schema(schema) == "query: string, limit: integer"
|
||||
|
||||
def test_optional_params_get_question_mark_suffix(self) -> None:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}, "limit": {"type": "integer"}},
|
||||
"required": ["query"],
|
||||
}
|
||||
assert _summarize_input_schema(schema) == "query: string, limit?: integer"
|
||||
|
||||
def test_untyped_property_renders_as_any(self) -> None:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"cloudops_backend": {}},
|
||||
"required": ["cloudops_backend"],
|
||||
}
|
||||
assert _summarize_input_schema(schema) == "cloudops_backend: any"
|
||||
|
||||
def test_overlong_summary_is_truncated_with_ellipsis(self) -> None:
|
||||
# Build a schema large enough to exceed the 200-char cap.
|
||||
properties = {f"param_{i}": {"type": "string"} for i in range(40)}
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": list(properties.keys()),
|
||||
}
|
||||
rendered = _summarize_input_schema(schema)
|
||||
assert len(rendered) <= 200
|
||||
assert rendered.endswith("…")
|
||||
|
||||
|
||||
class TestBuildToolCatalog:
|
||||
def test_returns_empty_list_when_no_tools(self, fake_registry: list[RegisteredTool]) -> None:
|
||||
del fake_registry # unused — registry is empty by default
|
||||
assert build_tool_catalog() == []
|
||||
|
||||
def test_projects_each_tool_into_a_catalog_entry(
|
||||
self, fake_registry: list[RegisteredTool]
|
||||
) -> None:
|
||||
fake_registry.append(
|
||||
_make_tool(
|
||||
"search_github",
|
||||
description="Search GitHub code.",
|
||||
surfaces=("investigation", "chat"),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
)
|
||||
)
|
||||
entries = build_tool_catalog()
|
||||
assert len(entries) == 1
|
||||
entry = entries[0]
|
||||
assert entry.name == "search_github"
|
||||
assert entry.surfaces == ("investigation", "chat")
|
||||
assert entry.description == "Search GitHub code."
|
||||
assert entry.input_schema_summary == "query: string"
|
||||
# Source file for the registry module resolves to its repo-relative path.
|
||||
assert entry.source_file == "tools/registry.py"
|
||||
|
||||
def test_filters_by_surface_when_provided(self, fake_registry: list[RegisteredTool]) -> None:
|
||||
fake_registry.extend(
|
||||
[
|
||||
_make_tool("inv_only", surfaces=("investigation",)),
|
||||
_make_tool("chat_only", surfaces=("chat",)),
|
||||
_make_tool("both", surfaces=("investigation", "chat")),
|
||||
]
|
||||
)
|
||||
chat_entries = build_tool_catalog(surface="chat")
|
||||
names = {e.name for e in chat_entries}
|
||||
assert names == {"chat_only", "both"}
|
||||
|
||||
def test_unresolvable_origin_module_yields_empty_source_file(
|
||||
self, fake_registry: list[RegisteredTool]
|
||||
) -> None:
|
||||
# A tool that registered but whose origin module no longer imports
|
||||
# cleanly (e.g. partial uninstall) must still surface in the catalog.
|
||||
fake_registry.append(_make_tool("orphan", origin_module="not.a.real.module"))
|
||||
entries = build_tool_catalog()
|
||||
assert len(entries) == 1
|
||||
assert entries[0].source_file == ""
|
||||
|
||||
def test_origin_module_outside_repo_returns_absolute_source_path(
|
||||
self, fake_registry: list[RegisteredTool], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
outside = Path("/virtual/site-packages/some_pkg/plugin.py")
|
||||
fake_mod = SimpleNamespace(__file__=str(outside))
|
||||
|
||||
monkeypatch.setattr(tool_catalog.importlib, "import_module", lambda _: fake_mod)
|
||||
fake_registry.append(_make_tool("ext_tool", origin_module="some_pkg.plugin"))
|
||||
entries = build_tool_catalog()
|
||||
assert len(entries) == 1
|
||||
assert entries[0].source_file == outside.as_posix()
|
||||
|
||||
|
||||
class TestFormatToolCatalogText:
|
||||
def test_returns_empty_string_for_empty_catalog(self) -> None:
|
||||
assert format_tool_catalog_text([]) == ""
|
||||
|
||||
def test_groups_by_surface_with_investigation_first(self) -> None:
|
||||
entries = [
|
||||
ToolCatalogEntry(
|
||||
name="alpha",
|
||||
surfaces=("investigation",),
|
||||
description="alpha desc",
|
||||
source_file="tools/alpha.py",
|
||||
input_schema_summary="(no params)",
|
||||
),
|
||||
ToolCatalogEntry(
|
||||
name="beta",
|
||||
surfaces=("chat",),
|
||||
description="beta desc",
|
||||
source_file="tools/beta.py",
|
||||
input_schema_summary="x: string",
|
||||
),
|
||||
]
|
||||
text = format_tool_catalog_text(entries)
|
||||
# Investigation header must precede chat header (canonical ordering).
|
||||
inv_pos = text.find("## investigation")
|
||||
chat_pos = text.find("## chat")
|
||||
assert inv_pos != -1 and chat_pos != -1
|
||||
assert inv_pos < chat_pos
|
||||
assert "alpha" in text and "beta" in text
|
||||
|
||||
def test_dual_surface_tool_appears_under_each_surface(self) -> None:
|
||||
entries = [
|
||||
ToolCatalogEntry(
|
||||
name="dual",
|
||||
surfaces=("investigation", "chat"),
|
||||
description="dual desc",
|
||||
source_file="tools/dual.py",
|
||||
input_schema_summary="(no params)",
|
||||
)
|
||||
]
|
||||
text = format_tool_catalog_text(entries)
|
||||
# Dual-surface tools surface in BOTH groups so the user can tell which
|
||||
# tools the chat agent vs the investigation pipeline can reach.
|
||||
assert text.count("**dual**") == 2
|
||||
assert "## investigation (1 tool)" in text
|
||||
assert "## chat (1 tool)" in text
|
||||
|
||||
def test_omits_source_line_when_source_file_unknown(self) -> None:
|
||||
entries = [
|
||||
ToolCatalogEntry(
|
||||
name="orphan",
|
||||
surfaces=("investigation",),
|
||||
description="orphan desc",
|
||||
source_file="",
|
||||
input_schema_summary="(no params)",
|
||||
)
|
||||
]
|
||||
text = format_tool_catalog_text(entries)
|
||||
assert "**orphan**" in text
|
||||
assert "source:" not in text
|
||||
|
||||
|
||||
class TestListToolsSlashCommand:
|
||||
"""``/tools list`` reaches the catalog and prints non-empty output."""
|
||||
|
||||
def _capture(self) -> tuple[Console, io.StringIO]:
|
||||
buf = io.StringIO()
|
||||
return Console(file=buf, force_terminal=False, highlight=False), buf
|
||||
|
||||
def test_list_tools_prints_grouped_catalog(self) -> None:
|
||||
console, buf = self._capture()
|
||||
session = Session()
|
||||
# Stub the catalog so the test stays decoupled from registry contents.
|
||||
fake = [
|
||||
ToolCatalogEntry(
|
||||
name="search_github",
|
||||
surfaces=("investigation", "chat"),
|
||||
description="Search GitHub code.",
|
||||
source_file="tools/search_github.py",
|
||||
input_schema_summary="query: string",
|
||||
)
|
||||
]
|
||||
with patch(
|
||||
"surfaces.interactive_shell.command_registry.tools_cmds.build_tool_catalog",
|
||||
return_value=fake,
|
||||
):
|
||||
assert _cmd_tools(session, console, ["list"]) is True
|
||||
out = buf.getvalue()
|
||||
assert "search_github" in out
|
||||
assert "investigation" in out
|
||||
assert "chat" in out
|
||||
assert "Search GitHub code." in out
|
||||
|
||||
def test_bare_tools_command_prints_full_registered_catalog(self) -> None:
|
||||
console, buf = self._capture()
|
||||
session = Session()
|
||||
fake = [
|
||||
ToolCatalogEntry(
|
||||
name="telegram_send_message",
|
||||
surfaces=("investigation", "chat"),
|
||||
description="Send a Telegram message.",
|
||||
source_file="tools/telegram_send_message_tool/tool.py",
|
||||
input_schema_summary="message: string",
|
||||
)
|
||||
]
|
||||
with patch(
|
||||
"surfaces.interactive_shell.command_registry.tools_cmds.build_tool_catalog",
|
||||
return_value=fake,
|
||||
) as catalog:
|
||||
assert dispatch_slash("/tools", session, console) is True
|
||||
catalog.assert_called_once_with()
|
||||
out = buf.getvalue()
|
||||
assert "telegram_send_message" in out
|
||||
assert "investigation" in out
|
||||
assert "chat" in out
|
||||
|
||||
def test_live_catalog_includes_telegram_send_message(self) -> None:
|
||||
names = {entry.name for entry in build_tool_catalog()}
|
||||
assert "telegram_send_message" in names
|
||||
|
||||
def test_list_tools_disables_markup_for_plain_catalog_text(self) -> None:
|
||||
console, buf = self._capture()
|
||||
session = Session()
|
||||
fake = [
|
||||
ToolCatalogEntry(
|
||||
name="risky_tool",
|
||||
surfaces=("investigation",),
|
||||
description="Payload [bold]injection[/bold] attempt",
|
||||
source_file="tools/risky.py",
|
||||
input_schema_summary="x: string",
|
||||
)
|
||||
]
|
||||
with patch(
|
||||
"surfaces.interactive_shell.command_registry.tools_cmds.build_tool_catalog",
|
||||
return_value=fake,
|
||||
):
|
||||
assert _cmd_tools(session, console, ["list"]) is True
|
||||
out = buf.getvalue()
|
||||
assert "[bold]injection[/bold]" in out
|
||||
|
||||
def test_list_tools_handles_empty_registry(self) -> None:
|
||||
console, buf = self._capture()
|
||||
session = Session()
|
||||
with patch(
|
||||
"surfaces.interactive_shell.command_registry.tools_cmds.build_tool_catalog",
|
||||
return_value=[],
|
||||
):
|
||||
assert _cmd_tools(session, console, ["list"]) is True
|
||||
assert "no tools registered" in buf.getvalue()
|
||||
|
||||
def test_tools_first_args_advertise_list_for_tab_completion(self) -> None:
|
||||
names = {arg for arg, _hint in _TOOLS_FIRST_ARGS}
|
||||
assert "list" in names
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
|
||||
|
||||
|
||||
def _no_file_settings(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
|
||||
lambda: {},
|
||||
)
|
||||
|
||||
|
||||
def test_load_defaults_to_redact_on(monkeypatch) -> None:
|
||||
"""Redaction must default on, matching HistoryPolicy — see issue #2804.
|
||||
|
||||
Prompt/response content can carry the same token shapes as typed command
|
||||
history and additionally leaves the machine via the PostHog sink, so it
|
||||
must not ship less guarded than history by default.
|
||||
"""
|
||||
_no_file_settings(monkeypatch)
|
||||
for var in (
|
||||
"OPENSRE_PROMPT_LOG_DISABLED",
|
||||
"OPENSRE_PROMPT_LOG_LOCAL_DISABLED",
|
||||
"OPENSRE_PROMPT_LOG_REDACT",
|
||||
"OPENSRE_PROMPT_LOG_PATH",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
config = PromptLogConfig.load()
|
||||
|
||||
assert config.redact is True
|
||||
assert config.posthog_enabled is True
|
||||
|
||||
|
||||
def test_load_respects_env_opt_out_of_redaction(monkeypatch) -> None:
|
||||
_no_file_settings(monkeypatch)
|
||||
monkeypatch.setenv("OPENSRE_PROMPT_LOG_REDACT", "0")
|
||||
|
||||
config = PromptLogConfig.load()
|
||||
|
||||
assert config.redact is False
|
||||
|
||||
|
||||
def test_load_respects_file_opt_out_of_redaction(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
|
||||
lambda: {"redact": False},
|
||||
)
|
||||
monkeypatch.delenv("OPENSRE_PROMPT_LOG_REDACT", raising=False)
|
||||
|
||||
config = PromptLogConfig.load()
|
||||
|
||||
assert config.redact is False
|
||||
|
||||
|
||||
def test_env_redact_overrides_file_setting(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.config.read_prompt_log_settings",
|
||||
lambda: {"redact": False},
|
||||
)
|
||||
monkeypatch.setenv("OPENSRE_PROMPT_LOG_REDACT", "1")
|
||||
|
||||
config = PromptLogConfig.load()
|
||||
|
||||
assert config.redact is True
|
||||
|
||||
|
||||
def test_dataclass_default_redact_is_on() -> None:
|
||||
"""Direct construction (e.g. in other tests/callers) should also default-redact."""
|
||||
assert PromptLogConfig().redact is True
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from surfaces.interactive_shell.runtime.core.turn_accounting import (
|
||||
ToolCallingTurnResult,
|
||||
)
|
||||
from surfaces.interactive_shell.runtime.shell_turn_execution import execute_shell_turn
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.utils.telemetry import LlmRunInfo
|
||||
|
||||
|
||||
class _FakeRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.responses: list[str] = []
|
||||
self.flushed = False
|
||||
|
||||
def set_response(self, text: str, _run: LlmRunInfo | None = None) -> None:
|
||||
self.responses.append(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushed = True
|
||||
|
||||
|
||||
def _console() -> Console:
|
||||
return Console(file=io.StringIO(), force_terminal=False, highlight=False)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Tool-gathering emits a progress line to the console; expectation of empty "
|
||||
"output needs revisiting. Skipped to unblock CI."
|
||||
)
|
||||
def test_execute_shell_turn_cli_agent_empty_response_is_recorded_empty() -> None:
|
||||
recorder = _FakeRecorder()
|
||||
|
||||
def fake_execute(*_args: object, **_kwargs: object) -> ToolCallingTurnResult:
|
||||
return ToolCallingTurnResult(
|
||||
planned_count=0,
|
||||
executed_count=0,
|
||||
executed_success_count=0,
|
||||
has_unhandled_clause=False,
|
||||
handled=False,
|
||||
)
|
||||
|
||||
def fake_answer(*_args: object, **_kwargs: object) -> LlmRunInfo:
|
||||
return LlmRunInfo(response_text="")
|
||||
|
||||
session = Session()
|
||||
output = io.StringIO()
|
||||
execute_shell_turn(
|
||||
"show datadog integration details",
|
||||
session,
|
||||
Console(file=output, force_terminal=False, highlight=False),
|
||||
recorder=recorder,
|
||||
confirm_fn=None,
|
||||
is_tty=None,
|
||||
execute_actions=fake_execute,
|
||||
answer_agent=fake_answer,
|
||||
)
|
||||
|
||||
assert output.getvalue() == ""
|
||||
assert recorder.responses == [""]
|
||||
assert session.last_assistant_intent == "cli_agent_fallback"
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Tests for per-turn integration snapshots on analytics capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.utils.telemetry.integration_snapshot import (
|
||||
build_turn_integration_snapshot,
|
||||
)
|
||||
|
||||
|
||||
def test_build_turn_integration_snapshot_empty_when_unconfigured() -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ()
|
||||
|
||||
snapshot = build_turn_integration_snapshot(session)
|
||||
|
||||
assert snapshot == {
|
||||
"connected_integrations": [],
|
||||
"connected_integrations_count": 0,
|
||||
"configured_integrations": [],
|
||||
"integration_snapshot_source": "runtime_config",
|
||||
}
|
||||
|
||||
|
||||
def test_build_turn_integration_snapshot_uses_session_configured_slugs(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog", "github")
|
||||
session.resolved_integrations_cache = {
|
||||
"datadog": {"api_key": "x", "app_key": "y", "connection_verified": True},
|
||||
"github": {"access_token": "token", "connection_verified": True},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
|
||||
lambda _resolved: [
|
||||
MagicMock(source="datadog"),
|
||||
MagicMock(source="github"),
|
||||
],
|
||||
)
|
||||
|
||||
snapshot = build_turn_integration_snapshot(session)
|
||||
|
||||
assert snapshot["configured_integrations"] == ["datadog", "github"]
|
||||
assert snapshot["connected_integrations"] == ["datadog", "github"]
|
||||
assert snapshot["connected_integrations_count"] == 2
|
||||
|
||||
|
||||
def test_build_turn_integration_snapshot_excludes_unavailable_tools(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog", "grafana")
|
||||
session.resolved_integrations_cache = {
|
||||
"datadog": {"api_key": "x", "app_key": "y", "connection_verified": True},
|
||||
"grafana": {"endpoint": "https://grafana.example.com", "api_key": "glsa"},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
|
||||
lambda _resolved: [MagicMock(source="datadog")],
|
||||
)
|
||||
|
||||
snapshot = build_turn_integration_snapshot(session)
|
||||
|
||||
assert snapshot["configured_integrations"] == ["datadog", "grafana"]
|
||||
assert snapshot["connected_integrations"] == ["datadog"]
|
||||
assert snapshot["connected_integrations_count"] == 1
|
||||
|
||||
|
||||
def test_build_turn_integration_snapshot_survives_tool_resolution_failure(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog",)
|
||||
session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
|
||||
|
||||
def _boom(_resolved: dict[str, Any]) -> list[MagicMock]:
|
||||
raise RuntimeError("tool registry blew up")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
|
||||
_boom,
|
||||
)
|
||||
|
||||
snapshot = build_turn_integration_snapshot(session)
|
||||
|
||||
assert snapshot["configured_integrations"] == ["datadog"]
|
||||
assert snapshot["connected_integrations"] == []
|
||||
assert snapshot["connected_integrations_count"] == 0
|
||||
|
||||
|
||||
def test_build_turn_integration_snapshot_survives_family_key_failure(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog",)
|
||||
session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
|
||||
lambda _resolved: [MagicMock(source="datadog")],
|
||||
)
|
||||
|
||||
def _boom(_service: str) -> str:
|
||||
raise RuntimeError("family key blew up")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.family_key",
|
||||
_boom,
|
||||
)
|
||||
|
||||
snapshot = build_turn_integration_snapshot(session)
|
||||
|
||||
assert snapshot["configured_integrations"] == ["datadog"]
|
||||
assert snapshot["connected_integrations"] == []
|
||||
assert snapshot["connected_integrations_count"] == 0
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for the investigation outcome analytics bridge."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from surfaces.interactive_shell.ui.investigation_outcome import InvestigationOutcome
|
||||
from surfaces.interactive_shell.utils.telemetry.investigation_analytics import (
|
||||
publish_investigation_outcome_analytics,
|
||||
)
|
||||
|
||||
|
||||
def _capture_outcome_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]:
|
||||
calls: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.investigation_analytics."
|
||||
"capture_investigation_outcome",
|
||||
lambda **kwargs: calls.append(kwargs),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.investigation_analytics."
|
||||
"capture_investigation_cancelled",
|
||||
lambda **_kwargs: None,
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
def test_completed_outcome_omits_placeholder_failure_properties(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_outcome_calls(monkeypatch)
|
||||
publish_investigation_outcome_analytics(
|
||||
InvestigationOutcome(
|
||||
status="completed",
|
||||
target="generic",
|
||||
investigation_id="inv-1",
|
||||
final_state={"root_cause": "disk full"},
|
||||
)
|
||||
)
|
||||
assert len(calls) == 1
|
||||
call = calls[0]
|
||||
assert call["status"] == "completed"
|
||||
assert call["root_cause_excerpt"] == "disk full"
|
||||
assert call["error_excerpt"] == ""
|
||||
assert call["failure_category"] is None
|
||||
assert call["integration_involved"] is None
|
||||
assert call["integration_failure_message"] is None
|
||||
assert call["failure_detail"] is None
|
||||
|
||||
|
||||
def test_failed_outcome_keeps_failure_properties(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls = _capture_outcome_calls(monkeypatch)
|
||||
publish_investigation_outcome_analytics(
|
||||
InvestigationOutcome(
|
||||
status="failed",
|
||||
target="generic",
|
||||
investigation_id="inv-2",
|
||||
error_message="grafana query failed: 401",
|
||||
error_detail="RuntimeError: grafana query failed: 401",
|
||||
failure_category="integration",
|
||||
integration_involved="grafana",
|
||||
integration_failure_message="grafana query failed: 401",
|
||||
)
|
||||
)
|
||||
assert len(calls) == 1
|
||||
call = calls[0]
|
||||
assert call["status"] == "failed"
|
||||
assert call["error_excerpt"] == "grafana query failed: 401"
|
||||
assert call["failure_category"] == "integration"
|
||||
assert call["integration_involved"] == "grafana"
|
||||
assert call["failure_detail"] == "RuntimeError: grafana query failed: 401"
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Tests for the investigation LLM usage observer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.llm.shared.usage import emit_usage, set_usage_hook
|
||||
from surfaces.interactive_shell.utils.telemetry.investigation_llm_usage import (
|
||||
observe_investigation_llm_usage,
|
||||
)
|
||||
|
||||
|
||||
def test_observe_accumulates_usage_and_clears_hook() -> None:
|
||||
with observe_investigation_llm_usage() as usage:
|
||||
emit_usage("claude-sonnet-4-5", 100, 20)
|
||||
emit_usage("claude-sonnet-4-5", 50, 10)
|
||||
assert usage.model == "claude-sonnet-4-5"
|
||||
assert usage.input_tokens == 150
|
||||
assert usage.output_tokens == 30
|
||||
assert usage.observed
|
||||
# Hook must be released so later owners can register.
|
||||
set_usage_hook(lambda *_args: None)
|
||||
set_usage_hook(None)
|
||||
|
||||
|
||||
def test_observe_tolerates_already_registered_hook() -> None:
|
||||
external: list[tuple[str, int, int]] = []
|
||||
set_usage_hook(lambda model, inp, out: external.append((model, inp, out)))
|
||||
try:
|
||||
with observe_investigation_llm_usage() as usage:
|
||||
emit_usage("m", 10, 5)
|
||||
assert not usage.observed
|
||||
assert external == [("m", 10, 5)]
|
||||
# The pre-existing hook must survive the observer.
|
||||
emit_usage("m", 1, 1)
|
||||
assert len(external) == 2
|
||||
finally:
|
||||
set_usage_hook(None)
|
||||
|
||||
|
||||
def test_observe_ignores_usage_outside_scope() -> None:
|
||||
with observe_investigation_llm_usage() as usage:
|
||||
pass
|
||||
emit_usage("m", 10, 5)
|
||||
assert not usage.observed
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks.local_jsonl import (
|
||||
append_prompt_log_record,
|
||||
)
|
||||
|
||||
|
||||
def test_append_prompt_log_record_writes_jsonl(tmp_path) -> None:
|
||||
log_path = tmp_path / "prompt_log.jsonl"
|
||||
append_prompt_log_record(path=log_path, record={"prompt": "hello", "response": "world"})
|
||||
lines = log_path.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) == 1
|
||||
payload = json.loads(lines[0])
|
||||
assert payload["prompt"] == "hello"
|
||||
assert payload["response"] == "world"
|
||||
|
||||
|
||||
def test_append_prompt_log_record_rotates_when_size_exceeded(tmp_path) -> None:
|
||||
log_path = tmp_path / "prompt_log.jsonl"
|
||||
log_path.write_text("x" * 200, encoding="utf-8")
|
||||
append_prompt_log_record(
|
||||
path=log_path,
|
||||
record={"prompt": "hello", "response": "world"},
|
||||
max_bytes=100,
|
||||
)
|
||||
backup = log_path.with_name(log_path.name + ".1")
|
||||
assert backup.exists()
|
||||
lines = log_path.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) == 1
|
||||
@@ -0,0 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from platform.analytics.events import Event
|
||||
from surfaces.interactive_shell.utils.telemetry.sinks import posthog_ai
|
||||
|
||||
|
||||
def test_capture_ai_generation_uses_analytics_capture(monkeypatch) -> None:
|
||||
calls: list[tuple[Event, dict[str, object]]] = []
|
||||
|
||||
class _FakeAnalytics:
|
||||
def capture(self, event: Event, properties: dict[str, object] | None = None) -> None:
|
||||
calls.append((event, properties or {}))
|
||||
|
||||
monkeypatch.setattr(posthog_ai, "get_analytics", lambda: _FakeAnalytics())
|
||||
posthog_ai.capture_ai_generation({"$ai_model": "gpt-test"})
|
||||
assert calls == [(Event.AI_GENERATION, {"$ai_model": "gpt-test"})]
|
||||
@@ -0,0 +1,609 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from surfaces.interactive_shell.session import Session
|
||||
from surfaces.interactive_shell.utils.telemetry.config import PromptLogConfig
|
||||
from surfaces.interactive_shell.utils.telemetry.recorder import LlmRunInfo, PromptRecorder
|
||||
|
||||
|
||||
def test_prompt_recorder_start_respects_supported_turns(monkeypatch, tmp_path: Path) -> None:
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=False,
|
||||
redact=False,
|
||||
max_chars=100,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
session = Session()
|
||||
assert PromptRecorder.start(session=session, text="hello", turn_kind="slash") is None
|
||||
assert PromptRecorder.start(session=session, text="hello", turn_kind="agent") is not None
|
||||
|
||||
|
||||
def test_prompt_recorder_for_background_task_uses_task_id_as_trace(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.for_background_task(
|
||||
session=session, command="opensre investigate --service api", task_id="ab247135"
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("command failed (exit 1)\nboom")
|
||||
recorder.flush()
|
||||
assert captured
|
||||
assert captured[0]["cli_turn_kind"] == "background_task"
|
||||
assert captured[0]["$ai_trace_id"] == "ab247135"
|
||||
assert captured[0]["$ai_input"][0]["content"] == "opensre investigate --service api"
|
||||
assert captured[0]["$ai_output_choices"][0]["content"] == "command failed (exit 1)\nboom"
|
||||
|
||||
|
||||
def test_prompt_recorder_for_background_task_disabled_returns_none(monkeypatch) -> None:
|
||||
cfg = PromptLogConfig(enabled=False)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
session = Session()
|
||||
assert PromptRecorder.for_background_task(session=session, command="x", task_id="t") is None
|
||||
|
||||
|
||||
def test_prompt_recorder_flush_writes_and_redacts(monkeypatch, tmp_path: Path) -> None:
|
||||
log_path = tmp_path / "prompt_log.jsonl"
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=True,
|
||||
posthog_enabled=False,
|
||||
redact=True,
|
||||
max_chars=1000,
|
||||
log_path=log_path,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="Bearer token-value-12345678901234567890",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response(
|
||||
"sk-ant-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456",
|
||||
LlmRunInfo(model="m", provider="p", latency_ms=10),
|
||||
)
|
||||
recorder.flush()
|
||||
payload = log_path.read_text(encoding="utf-8")
|
||||
assert "Bearer [REDACTED]" in payload
|
||||
assert "[REDACTED:anthropic_key]" in payload
|
||||
|
||||
|
||||
def test_prompt_recorder_sends_ai_generation(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {
|
||||
"connected_integrations": [],
|
||||
"connected_integrations_count": 0,
|
||||
"configured_integrations": [],
|
||||
"integration_snapshot_source": "runtime_config",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="hello",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
|
||||
recorder.flush()
|
||||
assert captured
|
||||
assert captured[0]["$ai_model"] == "gpt-test"
|
||||
assert captured[0]["$ai_input_tokens"] == 0
|
||||
assert captured[0]["connected_integrations"] == []
|
||||
assert captured[0]["connected_integrations_count"] == 0
|
||||
assert captured[0]["configured_integrations"] == []
|
||||
assert captured[0]["integration_snapshot_source"] == "runtime_config"
|
||||
|
||||
|
||||
def test_prompt_recorder_sends_connected_integrations(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {
|
||||
"connected_integrations": ["github"],
|
||||
"connected_integrations_count": 1,
|
||||
"configured_integrations": ["github"],
|
||||
"integration_snapshot_source": "runtime_config",
|
||||
},
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="hello",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
|
||||
recorder.flush()
|
||||
assert captured[0]["connected_integrations"] == ["github"]
|
||||
assert captured[0]["connected_integrations_count"] == 1
|
||||
|
||||
|
||||
def test_prompt_recorder_still_captures_when_tool_resolution_fails(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
|
||||
def _boom(_resolved: dict[str, object]) -> list[object]:
|
||||
raise RuntimeError("tool registry blew up")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.integration_snapshot.get_available_tools",
|
||||
_boom,
|
||||
)
|
||||
|
||||
session = Session()
|
||||
session.configured_integrations_known = True
|
||||
session.configured_integrations = ("datadog",)
|
||||
session.resolved_integrations_cache = {"datadog": {"api_key": "x", "app_key": "y"}}
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="hello",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("world", LlmRunInfo(model="gpt-test", provider="openai", latency_ms=50))
|
||||
recorder.flush()
|
||||
assert captured
|
||||
assert captured[0]["$ai_model"] == "gpt-test"
|
||||
assert captured[0]["configured_integrations"] == ["datadog"]
|
||||
assert captured[0]["connected_integrations"] == []
|
||||
|
||||
|
||||
def test_prompt_recorder_uses_no_conversational_agent_without_llm_run(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="/help",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("slash /help (succeeded)")
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_model"] == "no_conversational_agent"
|
||||
assert captured[0]["$ai_provider"] == "no_conversational_agent"
|
||||
|
||||
|
||||
def test_prompt_recorder_includes_investigation_id(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
session.last_investigation_id = "inv-abc"
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="/investigate generic",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response(
|
||||
"slash /investigate generic (failed)\ninvestigation_failed (generic):\nboom"
|
||||
)
|
||||
recorder.flush()
|
||||
assert captured[0]["investigation_id"] == "inv-abc"
|
||||
|
||||
|
||||
def test_prompt_recorder_omits_investigation_id_for_unrelated_turns(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
session.last_investigation_id = "inv-stale"
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="what integrations are configured?",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("github and datadog")
|
||||
recorder.flush()
|
||||
assert "investigation_id" not in captured[0]
|
||||
|
||||
|
||||
def test_prompt_recorder_uses_prompt_fallback_when_response_empty(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
captured: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
session.record("slash", "/help", ok=True, response_text="slash /help (succeeded)")
|
||||
recorder = PromptRecorder.start(session=session, text="/help", turn_kind="agent")
|
||||
assert recorder is not None
|
||||
recorder.set_response(" ")
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_output_choices"][0]["content"] == "terminal turn handled: /help"
|
||||
|
||||
|
||||
def test_prompt_recorder_background_task_uses_bound_investigation_id(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
session.last_investigation_id = "inv-stale"
|
||||
recorder = PromptRecorder.for_background_task(
|
||||
session=session,
|
||||
command="opensre investigate --service api",
|
||||
task_id="task-123",
|
||||
)
|
||||
assert recorder is not None
|
||||
session.last_investigation_id = "inv-other"
|
||||
recorder.set_response("command completed (exit 0)")
|
||||
recorder.flush()
|
||||
investigation_id = captured[0]["investigation_id"]
|
||||
assert isinstance(investigation_id, str)
|
||||
assert investigation_id not in {"", "inv-stale", "inv-other"}
|
||||
|
||||
|
||||
def test_prompt_recorder_set_error_adds_structured_properties(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(session=session, text="/investigate generic", turn_kind="agent")
|
||||
assert recorder is not None
|
||||
recorder.set_error("config", "ANTHROPIC_API_KEY not set")
|
||||
recorder.set_response(
|
||||
"slash /investigate generic (failed)\ninvestigation_failed (generic):\n"
|
||||
"ANTHROPIC_API_KEY not set"
|
||||
)
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_is_error"] is True
|
||||
assert captured[0]["$ai_error"] == "ANTHROPIC_API_KEY not set"
|
||||
assert captured[0]["error_kind"] == "config"
|
||||
# Investigation-style errors are terminal-path failures, not conversational
|
||||
# LLM provider failures: no ai_error_kind and the sentinel model stays.
|
||||
assert "ai_error_kind" not in captured[0]
|
||||
assert captured[0]["$ai_model"] == "no_conversational_agent"
|
||||
|
||||
|
||||
def test_prompt_recorder_omits_error_properties_by_default(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
recorder = PromptRecorder.start(session=session, text="hello", turn_kind="agent")
|
||||
assert recorder is not None
|
||||
recorder.set_response("world")
|
||||
recorder.flush()
|
||||
assert "$ai_is_error" not in captured[0]
|
||||
assert "$ai_error" not in captured[0]
|
||||
assert "error_kind" not in captured[0]
|
||||
|
||||
|
||||
def _posthog_recorder(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
*,
|
||||
text: str,
|
||||
captured: list[dict[str, object]],
|
||||
) -> PromptRecorder:
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
recorder = PromptRecorder.start(session=Session(), text=text, turn_kind="agent")
|
||||
assert recorder is not None
|
||||
return recorder
|
||||
|
||||
|
||||
def test_prompt_recorder_llm_provider_failure_never_uses_terminal_sentinel(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Conversational prompt + provider failure must not be tagged no_conversational_agent."""
|
||||
captured: list[dict[str, object]] = []
|
||||
recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
|
||||
error = (
|
||||
"Bedrock model 'us.anthropic.claude-sonnet-4-6' is not available for your account. "
|
||||
"Check Bedrock model access in the configured AWS region."
|
||||
)
|
||||
recorder.set_error("action_agent_error", error)
|
||||
recorder.set_response(error)
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_model"] == "unknown"
|
||||
assert captured[0]["$ai_provider"] == "unknown"
|
||||
assert captured[0]["ai_error_kind"] == "not_configured"
|
||||
assert "not available for your account" in captured[0]["$ai_output_choices"][0]["content"]
|
||||
|
||||
|
||||
def test_prompt_recorder_llm_provider_failure_reports_attempted_model(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
|
||||
recorder.set_error("assistant_error", "Anthropic authentication failed.")
|
||||
recorder.set_response(
|
||||
"",
|
||||
LlmRunInfo(model="claude-sonnet-4-6", provider="anthropic"),
|
||||
)
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_model"] == "claude-sonnet-4-6"
|
||||
assert captured[0]["$ai_provider"] == "anthropic"
|
||||
assert captured[0]["ai_error_kind"] == "auth"
|
||||
# Empty assistant text falls back to the error message, not the terminal fallback.
|
||||
assert captured[0]["$ai_output_choices"][0]["content"] == "Anthropic authentication failed."
|
||||
|
||||
|
||||
def test_prompt_recorder_flush_resolves_error_message_after_empty_set_response(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Flush-time fallback tolerates set_response before set_error."""
|
||||
captured: list[dict[str, object]] = []
|
||||
recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
|
||||
recorder.set_response("")
|
||||
recorder.set_error("assistant_error", "provider failed")
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_output_choices"][0]["content"] == "provider failed"
|
||||
|
||||
|
||||
def test_prompt_recorder_terminal_error_kinds_keep_terminal_sentinel(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Background-task style errors (e.g. subprocess timeout) stay terminal-action turns."""
|
||||
captured: list[dict[str, object]] = []
|
||||
recorder = _posthog_recorder(monkeypatch, tmp_path, text="hi", captured=captured)
|
||||
recorder.set_error("timeout", "command timed out after 60 seconds")
|
||||
recorder.set_response("command timed out after 60 seconds")
|
||||
recorder.flush()
|
||||
assert captured[0]["$ai_model"] == "no_conversational_agent"
|
||||
assert captured[0]["$ai_provider"] == "no_conversational_agent"
|
||||
assert "ai_error_kind" not in captured[0]
|
||||
|
||||
|
||||
def test_prompt_recorder_uses_only_latest_slash_outcome(monkeypatch, tmp_path: Path) -> None:
|
||||
captured: list[dict[str, object]] = []
|
||||
cfg = PromptLogConfig(
|
||||
enabled=True,
|
||||
local_enabled=False,
|
||||
posthog_enabled=True,
|
||||
redact=False,
|
||||
max_chars=1000,
|
||||
log_path=tmp_path / "prompt_log.jsonl",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.PromptLogConfig.load", lambda: cfg
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.build_turn_integration_snapshot",
|
||||
lambda _session: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"surfaces.interactive_shell.utils.telemetry.recorder.capture_ai_generation",
|
||||
lambda payload: captured.append(payload),
|
||||
)
|
||||
session = Session()
|
||||
session.record(
|
||||
"slash",
|
||||
"/modle",
|
||||
ok=False,
|
||||
response_text="Unknown command: /modle.",
|
||||
slash_outcome="unknown_command",
|
||||
)
|
||||
session.record("slash", "/help", ok=True, response_text="slash /help (succeeded)")
|
||||
recorder = PromptRecorder.start(
|
||||
session=session,
|
||||
text="what integrations are configured?",
|
||||
turn_kind="agent",
|
||||
)
|
||||
assert recorder is not None
|
||||
recorder.set_response("github and datadog")
|
||||
recorder.flush()
|
||||
assert "slash_outcome" not in captured[0]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user