"""CloudOpsBench-specific investigation agent. Subclasses :class:`tools.investigation.stages.gather_evidence.ConnectedInvestigationAgent` to enforce a minimum-tool-call floor before the agent is allowed to conclude. Production code is untouched — bench-only termination behavior lives here. Why we need a floor for the bench ---------------------------------- Production opensre lets the LLM decide when it has enough evidence. That's the right default for real incidents: latency matters, the LLM is usually right after a few tool calls, and forcing extra calls wastes tokens. CloudOpsBench cases are different: - The paper's protocol rewards deep multi-source evidence gathering (15-20 tool calls typical in winning runs). - The June-3 OpenAI bench showed gpt-4o median=7 steps and gpt-5 median=4 steps — both producing a1=0 despite the agent's structural advantage over plain LLM. - Tool coverage was 0.20 (gpt-4o) and 0.00 (gpt-5) — agents bailed before exercising the tools the paper measures against. We force the bench agent to gather more evidence before concluding. The loop's outer cap (``MAX_INVESTIGATION_LOOPS``) still bounds the worst case, so a stubborn model can't infinite-loop. """ from __future__ import annotations import logging import os from typing import Any, ClassVar from core.tool_framework.registered_tool import RegisteredTool from tools.investigation.stages.gather_evidence import ConnectedInvestigationAgent logger = logging.getLogger(__name__) # Default minimum-tool-call floor for the opensre+llm arm. Overridable via the # ``BENCH_MIN_TOOL_CALLS`` env var so the floor can be swept across runs WITHOUT # editing code — each sweep point is a fresh CLI process, so an import-time read # is sufficient. Tests still override the class attribute directly. # # Calibrated to 5 based on the 2026-06-06 floorsweep on 30 gpt-4o cases × 3 # seeds (.bench-results/cloudopsbench_floorsweep_openai/). Floor=5 produced the # highest single-shot A@1 mean (0.578) and the highest object_a1 (0.811) while # preserving a `rel` (0.374) much closer to the paper's gpt-4o reference (0.63) # than floor=8 (rel=0.306). Floor=8 (the prior default) over-explored — agents # averaged 9 tool calls per case, burning 3-4 calls on tools that didn't change # the diagnosis. See EXPERIMENTS.md in bench-results-openai/ for the full table. _DEFAULT_MIN_TOOL_CALLS = 5 _ENV_MIN_TOOL_CALLS = "BENCH_MIN_TOOL_CALLS" def _resolve_min_tool_calls() -> int: """Read the floor from the environment, falling back to the default. Invalid or negative values are ignored (with a warning) rather than crashing a long bench run; a 0 floor is legal and means "let the LLM decide", i.e. the same termination policy as the llm_alone control. """ raw = os.environ.get(_ENV_MIN_TOOL_CALLS) if raw is None or raw.strip() == "": return _DEFAULT_MIN_TOOL_CALLS try: value = int(raw) except ValueError: logger.warning( "Ignoring non-integer %s=%r; using default floor %d", _ENV_MIN_TOOL_CALLS, raw, _DEFAULT_MIN_TOOL_CALLS, ) return _DEFAULT_MIN_TOOL_CALLS if value < 0: logger.warning( "Ignoring negative %s=%d; using default floor %d", _ENV_MIN_TOOL_CALLS, value, _DEFAULT_MIN_TOOL_CALLS, ) return _DEFAULT_MIN_TOOL_CALLS return value # Tools available to the bench agent are exactly those registered by the # bench-specific package. Production opensre tools (real EKS API calls, # Hermes log tailing, etc.) would hit live infrastructure that the bench # task role intentionally cannot reach — burning calls on AccessDenied # instead of returning deterministic replay data. # # Trailing dot is deliberate: it matches anything UNDER the package, not # the package root itself. The registry only auto-discovers submodules # (via ``pkgutil.iter_modules``), so a tool whose ``origin_module`` is # exactly the root is theoretical — but if you register a single-file # bench tool module directly via :func:`register_external_tool_package`, # its ``origin_module`` will be the root and it'll be dropped here. Use # a submodule (e.g. ``tools/k8s/__init__.py``) instead. _BENCH_TOOL_MODULE_PREFIX = "tests.benchmarks.cloudopsbench.tools." class BenchInvestigationAgent(ConnectedInvestigationAgent): """Bench subclass that requires N tool calls before allowing conclusion. Threshold is a class attribute so subclasses or tests can override it without rebuilding the agent instance. Default 8 is calibrated for CloudOpsBench's median win-trajectory (~15-20 tool calls) while leaving headroom: even a perfect 8-call run is within the loop cap. Set ``BENCH_MIN_TOOL_CALLS`` to sweep the floor across runs. """ MIN_TOOL_CALLS = _resolve_min_tool_calls() ALLOWED_TOOL_MODULE_PREFIXES: ClassVar[tuple[str, ...]] = (_BENCH_TOOL_MODULE_PREFIX,) def _should_accept_conclusion( self, *, evidence_count: int, iteration: int, # noqa: ARG002 — base class signature ) -> tuple[bool, str | None]: if evidence_count >= self.MIN_TOOL_CALLS: return True, None return False, ( f"You've gathered {evidence_count} tool result(s) so far. Before " f"concluding, please continue investigating — what dimensions " f"of the system haven't you checked yet? Consider tool sources " f"you haven't queried, or evidence that would support OR " f"contradict your current hypothesis." ) def _filter_tools( self, tools: list[RegisteredTool], ) -> list[RegisteredTool]: """Restrict to bench-package tools by origin module. Filtering by ``origin_module`` instead of an explicit name list means a new bench tool added under ``tests/benchmarks/cloudopsbench/tools/`` is picked up automatically — no risk of the whitelist drifting out of sync with the tool registry. Silent-exclusion edge cases to know about (rare today, but possible if someone adds a tool in an unconventional way): - A tool whose ``origin_module`` is exactly the prefix root (no trailing submodule) is dropped — see the comment on ``_BENCH_TOOL_MODULE_PREFIX``. - A tool whose ``origin_module`` defaults to the empty string (e.g. directly-constructed ``RegisteredTool(...)`` without ``origin_module=`` set) is also dropped, and logged at WARNING so the registry bug surfaces in the run log instead of silently shrinking the bench tool set. """ return _filter_to_bench_package(tools, self.ALLOWED_TOOL_MODULE_PREFIXES) def _filter_to_bench_package( tools: list[RegisteredTool], allowed_prefixes: tuple[str, ...], ) -> list[RegisteredTool]: """Shared bench-package tool filter — same policy across all bench agents. Both :class:`BenchInvestigationAgent` (the opensre+llm path) and :class:`BaselineLLMAloneAgent` (the llm_alone control arm) must see the same tool surface; the comparison between modes is only fair when the tool inventory is identical. Extracting the filter into a free function keeps that contract enforced by reuse rather than by a "remember to keep these in sync" comment. """ kept: list[RegisteredTool] = [] dropped: list[str] = [] for tool in tools: if not tool.origin_module: logger.warning( "Bench filter dropping tool %r with empty origin_module — " "registry bug: tool was constructed without origin_module=. " "Set it explicitly so the bench can keep it.", tool.name, ) dropped.append(f"{tool.name} (no origin_module)") continue if tool.origin_module.startswith(allowed_prefixes): kept.append(tool) else: dropped.append(f"{tool.name} ({tool.origin_module})") if dropped: logger.debug("Bench filter dropped %d tool(s): %s", len(dropped), ", ".join(dropped)) return kept class BaselineLLMAloneAgent(ConnectedInvestigationAgent): """LLM-alone control arm for the bench. The audit identified this as the single biggest scientific gap in the cycle: without a matched in-harness baseline on the same cases, no "opensre helps" claim is attributable. This subclass is that control. What it inherits from :class:`ConnectedInvestigationAgent` (production): - The ReAct loop, evidence accumulation, context-budget enforcement - The default ``_should_accept_conclusion`` hook — accept whatever the LLM decides, no minimum-tool-call floor What it overrides: - ``_filter_tools`` — same bench-package whitelist :class:`BenchInvestigationAgent` uses, so the two modes see the IDENTICAL tool inventory and the only difference between them is the bench-specific termination policy (Lever #1's MIN_TOOL_CALLS=8) What this measures: the marginal lift from the bench-specific lever (MIN_TOOL_CALLS), not the full opensre-vs-bare-LLM gap. The system prompt and ReAct loop are still opensre's. A truly pure baseline (minimal SRE prompt, no opensre planning structure) is a follow-up; surface this limitation in the report rather than hiding it. """ ALLOWED_TOOL_MODULE_PREFIXES: ClassVar[tuple[str, ...]] = (_BENCH_TOOL_MODULE_PREFIX,) def _filter_tools( self, tools: list[RegisteredTool], ) -> list[RegisteredTool]: return _filter_to_bench_package(tools, self.ALLOWED_TOOL_MODULE_PREFIXES) # Minimal SRE-diagnostic system prompt for the pure baseline. # # Deliberately concise — no planner instructions, no stage-gate language, no # anti-hallucination scaffolding, no evidence-budget guidance. The point of # this control is to measure what a general-purpose LLM does with the same # tools and zero opensre-specific framing. Anything richer than this prompt # starts smuggling opensre's structural priors back into the "baseline." # # We DO ask for the same output shape (root cause + faulting component) # because the scorer needs to find those fields; that's a measurement # protocol requirement, not a reasoning prior. _PURE_BASELINE_SYSTEM_PROMPT = ( "You are an SRE diagnosing a Kubernetes incident. An alert has been raised. " "Use the available tools to investigate. When you have enough evidence to " "name a root cause, state your conclusion in two short fields: " "(1) the faulting component (Kubernetes object: deployment, pod, service, " "secret, etc.), and (2) the root cause in 1-2 sentences." ) class PureBaselineAgent(ConnectedInvestigationAgent): """Pure LLM-alone baseline — strips opensre's system prompt as well. The third arm the audit asked for. Comparison hierarchy: - ``opensre+llm`` → opensre prompt + Lever #1 floor (full opensre) - ``llm_alone`` → opensre prompt − Lever #1 floor (isolates Lever #1) - ``llm_alone_pure`` (this) → minimal prompt − Lever #1 floor (isolates opensre's PROMPT vs raw LLM+tools) Reading the contrasts: - (opensre+llm) − (llm_alone) = lift from Lever #1 - (opensre+llm) − (llm_alone_pure) = lift from full opensre stack (prompt + Lever #1) - (llm_alone) − (llm_alone_pure) = lift from opensre's PROMPT alone What this STILL inherits from :class:`ConnectedInvestigationAgent`: the ReAct loop scaffolding (tool execution, evidence accumulation, context-budget enforcement, retry-on-tool-error, etc.). Those are mechanical plumbing every baseline would need; they aren't "opensre's reasoning." The honest framing is "minimal-prompt LLM with tools," not "pure stdin/stdout LLM" — which would not be a meaningful comparison anyway. """ ALLOWED_TOOL_MODULE_PREFIXES: ClassVar[tuple[str, ...]] = (_BENCH_TOOL_MODULE_PREFIX,) def _filter_tools( self, tools: list[RegisteredTool], ) -> list[RegisteredTool]: # Same bench-package whitelist as Bench + Baseline arms — tool # surface is the methodological constant across all three modes. return _filter_to_bench_package(tools, self.ALLOWED_TOOL_MODULE_PREFIXES) def _build_system_prompt(self, state: dict[str, Any]) -> str: # noqa: ARG002 — interface contract return _PURE_BASELINE_SYSTEM_PROMPT # Trimmed bench prompt — sits between the full opensre prompt and the pure # baseline. The 2026-06-08 full-N floor=0 run loss diagnosis (n=353 paired # scenarios) showed 60% of opensre+llm losses against llm_alone_pure are # "predictor drift" cases: opensre's investigation correctly identifies the # fault_object (object_a1 is ~tied between the arms) but the predictor's # rank-1 root_cause is a token adjacent to the truth — e.g. # ``liveness_probe_incorrect_timing`` instead of ``..._protocol``, # ``image_registry_dns_failure`` instead of ``incorrect_image_reference``, # ``namespace_cpu_quota_exceeded`` instead of ``namespace_pod_quota_exceeded``. # # The predictor is faithful to its input; the wrong tokens come from # opensre's investigation TEXT itself. The full opensre system prompt's # hedging + validation + multi-stage scaffolding produces RCAs that lean on # adjacent vocabulary the predictor then formalizes. # # This trimmed variant keeps the parts that have customer value # (tool-output citation, structured component + root_cause output) and drops # the parts that empirically produce noise on cloudopsbench (hedging-by- # default language, multi-stage planner instructions, validation-of- # validation directives). It is BENCH-ONLY — production opensre's prompt is # unchanged. _TRIMMED_BENCH_SYSTEM_PROMPT = ( "You are an SRE agent investigating a Kubernetes incident. Use the " "available tools to gather evidence — typically pod state, error logs, " "recent events, and resource configuration.\n\n" "Dependency-traversal rule — INVESTIGATION-LAYER (does not affect " "your final localization decision; just expands what evidence you " "gather before concluding):\n" " When the failing service shows connection-related errors in its " "logs (connection refused, timeout, authentication failure, write " "failure, port unreachable), the actual fault may live in a stateful " "DEPENDENCY (database, cache, message queue) rather than in the " "service that reports the symptom. Before concluding, also call " "GetErrorLogs on the dependency pod itself. Stateful dependency pods " "(MySQL / MariaDB / Postgres / Redis / RabbitMQ / etc.) log their " "OWN internal failure modes — read-only mode enforcement, connection " "pool exhaustion, replication errors, slow queries, credential " "rejections — that are NOT visible from the caller's side. The same " "applies for namespace-scoped admission failures: when multiple pods " "in a namespace fail together, query for namespace-level resources " "(quotas, network policies, service accounts) rather than diagnosing " "from one victim service's logs.\n" " This rule expands the EVIDENCE you collect; it does NOT bias your " "localization. The final faulting component is whichever piece the " "evidence trajectory points at, including 'the dependency is healthy " "but the caller's config to reach it is wrong' — in which case the " "caller IS the fault.\n\n" "Alert-anchored upstream-attribution rule — INVESTIGATION-LAYER " "(applies when the alert itself names a specific service AND describes " "a performance, latency, network, or resource-saturation problem — " "e.g. 'network delay', 'CPU saturation', 'memory pressure', 'slow " "response'):\n" " Treat the service named in the alert as the primary suspect. A " "slow or saturated upstream service often produces NOISY error logs " "in downstream services that depend on it — timeouts, retries, " "'cannot reach', 'connection refused', 'request failed', 'order not " "found' — while the actual faulting service may show NO error logs " "at all (it is not crashing; it is just slow). The downstream " "services with the loudest logs are usually VICTIMS, not causes.\n" " When this pattern appears (alert names X with a performance / " "network / resource issue; X has few or no error logs; multiple other " "services Y, Z, W show timeout-shaped or 'cannot reach upstream' " "errors), the answer is almost always X with a performance / network " "/ resource root cause (pod_network_delay, pod_cpu_overload, " "pod_memory_pressure, etc.), not Y/Z/W with a runtime / config / " "database root cause. Investigate X's pod-level metrics — CPU, " "memory, network — rather than chasing the loud downstream logs.\n" " This rule does NOT override the dependency-traversal rule above. " "If the alert is about a connection-shaped failure (not a " "performance-shaped one), the dependency-traversal rule still " "applies. The two rules describe complementary patterns: " "connection-failure alerts often point DOWNSTREAM to a dependency; " "performance-shaped alerts usually point AT the named service " "itself.\n\n" "When you have identified the failing component and root cause, " "produce a concise conclusion:\n" " (1) the faulting component — Kubernetes object (deployment, pod, " "service, secret, namespace, etc.)\n" " (2) the root cause in 1-2 sentences naming the specific failure " "mode\n" " (3) cite the tool output that supports your conclusion.\n\n" "Do not hedge when the evidence is clear. Do not validate the same " "claim multiple ways. Do not break the investigation into stages " "unless the case genuinely requires multi-step escalation." ) class BenchInvestigationAgentTrimmedPrompt(BenchInvestigationAgent): """Bench-only ``BenchInvestigationAgent`` variant with a trimmed prompt. Inherits BenchInvestigationAgent's tool filter and the configurable ``MIN_TOOL_CALLS`` class attribute (set from config.min_tool_calls at CLI startup). Overrides only the system prompt. Selected by setting ``agent_variant: trimmed_prompt`` in a bench config. The CLI override (see ``_framework/cli.py``) swaps the adapter's investigation_agent_class to this when the field is set; default behavior (agent_variant unset / "default") returns the original ``BenchInvestigationAgent`` class. """ def _build_system_prompt(self, state: dict[str, Any]) -> str: # noqa: ARG002 — interface contract return _TRIMMED_BENCH_SYSTEM_PROMPT