"""Thin orchestration entrypoint for the synthetic RDS PostgreSQL benchmark suite. Pure scoring logic lives in scoring.py. Rendering/cross-axis reports live in reporting.py. Per-scenario observation building and Rich rendering live in observations.py. Trajectory policy types and evaluation live in trajectory_policy.py. """ from __future__ import annotations import argparse import difflib import json import os import sys import textwrap import time from collections.abc import Callable, Iterator from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import contextmanager, nullcontext from dataclasses import asdict, dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Any from rich.console import Console from rich.progress import ( BarColumn, Progress, TaskID, TaskProgressColumn, TextColumn, TimeElapsedColumn, ) from rich.table import Table from tests.synthetic.llm_provider_preflight import ( UnsupportedSyntheticLLMProviderError, validate_synthetic_llm_provider, ) from tests.synthetic.mock_aws_backend import FixtureAWSBackend from tests.synthetic.mock_grafana_backend.backend import FixtureGrafanaBackend from tests.synthetic.mock_grafana_backend.selective_backend import SelectiveGrafanaBackend from tests.synthetic.rds_postgres.observations import ( build_observation, compute_trajectory_metrics, render_report_to_console, write_observation, ) from tests.synthetic.rds_postgres.reporting import print_gap_report from tests.synthetic.rds_postgres.runner_api import ( LevelRunConfig, LevelRunResult, SuiteRunConfig, SuiteRunResult, default_parallel_workers, group_fixtures_by_level, parse_levels_csv, select_fixtures, ) from tests.synthetic.rds_postgres.scenario_loader import ( SUITE_DIR, GoldenTrajectoryConfig, ScenarioFixture, load_all_scenarios, ) from tests.synthetic.rds_postgres.scoring import ( FailureDetail, GateResult, ScenarioScore, _all_required_gates_pass, score_result, ) from tests.synthetic.rds_postgres.trajectory_policy import ( TrajectoryPolicy, TrajectoryPolicyResult, evaluate_trajectory_policy, ) __all__ = [ # orchestration "run_scenario", "run_synthetic_suite", "run_suite", "main", ] def run_investigation( raw_alert: str | dict[str, Any], *, resolved_integrations: dict[str, Any] | None = None, openclaw_context: dict[str, Any] | None = None, opensre_evaluate: bool = False, ) -> Any: """Lazy-import ``tools.investigation.capability.run_investigation`` (keeps monkeypatch target stable).""" from tools.investigation.capability import run_investigation as _impl return _impl( raw_alert, resolved_integrations=resolved_integrations, openclaw_context=openclaw_context, opensre_evaluate=opensre_evaluate, ) def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run the synthetic RDS PostgreSQL RCA suite.") parser.add_argument( "--scenario", default="", help="Run a single scenario directory name, e.g. 001-replication-lag.", ) parser.add_argument( "--levels", default="1,2,3,4", help=( "Comma-separated scenario_difficulty levels to execute (1-4). " "Ignored when --scenario is set." ), ) parser.add_argument( "--parallel-workers", type=int, default=None, dest="parallel_workers", help=( "Number of scenarios to execute in parallel. " "Defaults to min(8, cpu_count). " "Use 1 to run sequentially." ), ) parser.add_argument( "--parallel-levels", type=int, default=1, dest="parallel_levels", help="Deprecated alias for --parallel-workers (kept for back-compat).", ) parser.add_argument( "--json", action="store_true", help="Print machine-readable JSON results.", ) parser.add_argument( "--mock-grafana", action="store_true", dest="mock_grafana", help="Serve fixture data via FixtureGrafanaBackend instead of real Grafana calls.", ) parser.add_argument( "--axis2", action="store_true", help="Print Axis 1 vs Axis 2 gap report (requires results from both suites).", ) report_group = parser.add_mutually_exclusive_group() report_group.add_argument( "--report", action="store_true", dest="report", help="Print Rich observation report per scenario.", ) report_group.add_argument( "--no-report", action="store_false", dest="report", help="Disable Rich observation report output.", ) parser.set_defaults(report=None) parser.add_argument( "--observations-dir", default=str(SUITE_DIR / "_observations"), help="Directory where per-run observation JSON files are written.", ) parser.add_argument( "--baseline-out", default="", dest="baseline_out", help="Write per-scenario canonical_report_payload JSON snapshots into this directory.", ) parser.add_argument( "--baseline-check", default="", dest="baseline_check", help=( "Compare each scenario's canonical_report_payload against snapshots in this " "directory. Exits non-zero on any mismatch." ), ) return parser.parse_args(argv) def _build_run_config(args: argparse.Namespace) -> SuiteRunConfig: if args.parallel_workers is not None: workers = max(1, int(args.parallel_workers)) elif args.parallel_levels != 1: workers = max(1, int(args.parallel_levels)) else: workers = default_parallel_workers() return SuiteRunConfig( scenario=str(args.scenario or "").strip(), levels=parse_levels_csv(args.levels), parallel_workers=workers, parallel_levels=max(1, int(args.parallel_levels)), output_json=bool(args.json), mock_grafana=bool(args.mock_grafana), report=args.report, observations_dir=Path(args.observations_dir), baseline_out=Path(args.baseline_out) if args.baseline_out else None, baseline_check=Path(args.baseline_check) if args.baseline_check else None, ) def _build_resolved_integrations( fixture: ScenarioFixture, use_mock_grafana: bool, grafana_backend: Any = None, ) -> dict[str, Any] | None: """Build pre-resolved integrations for injection into run_investigation.""" integrations: dict[str, Any] = {} if use_mock_grafana or grafana_backend is not None: integrations["grafana"] = { "endpoint": "", "api_key": "", "_backend": grafana_backend or FixtureGrafanaBackend(fixture), } integrations["aws"] = { "region": fixture.metadata.region, "ec2_backend": FixtureAWSBackend(fixture), } return integrations def _resolved_golden_trajectory( fixture: ScenarioFixture, ) -> tuple[list[str], int | None, GoldenTrajectoryConfig | None]: golden_cfg = fixture.answer_key.golden_trajectory if golden_cfg is not None and golden_cfg.ordered_actions: if golden_cfg.max_loops is not None: return list(golden_cfg.ordered_actions), golden_cfg.max_loops, golden_cfg return ( list(golden_cfg.ordered_actions), fixture.answer_key.max_investigation_loops, golden_cfg, ) return ( list(fixture.answer_key.optimal_trajectory), fixture.answer_key.max_investigation_loops, None, ) def _trajectory_policy_for_fixture( *, max_loops: int | None, golden_cfg: GoldenTrajectoryConfig | None, ) -> TrajectoryPolicy | None: if golden_cfg is None: return None return TrajectoryPolicy( matching=golden_cfg.matching, max_edit_distance=golden_cfg.max_edit_distance, max_extra_actions=golden_cfg.max_extra_actions, max_redundancy=golden_cfg.max_redundancy, max_loops=max_loops, ) def _apply_trajectory_policy_to_score( score: ScenarioScore, trajectory_policy: TrajectoryPolicyResult | None, ) -> ScenarioScore: """Apply the trajectory policy result to the score, always recording the gate. The gate is recorded in ALL cases (pass, fail, not-applicable) so that ``_all_required_gates_pass`` acts as a true hard gate. """ gates = dict(score.gates) if trajectory_policy is None: gates["trajectory_policy"] = GateResult( status="pass", threshold="not_applicable — no golden trajectory configured", actual="not_applicable", ) return replace( score, passed=_all_required_gates_pass(gates) and not score.failure_reasons, gates=gates, ) gates["trajectory_policy"] = GateResult( status="pass" if trajectory_policy.passed else "fail", threshold="policy violations list must be empty", actual=f"violations={trajectory_policy.violations}", ) if trajectory_policy.passed: return replace( score, passed=_all_required_gates_pass(gates) and not score.failure_reasons, gates=gates, ) policy_reason = "trajectory policy failed: " + "; ".join( trajectory_policy.violations or ["unknown violation"] ) failures = list(score.failure_reasons) if not any(detail.code == "TRAJECTORY_POLICY_FAILED" for detail in failures): failures.append(FailureDetail(code="TRAJECTORY_POLICY_FAILED", detail=policy_reason)) combined_reason = "; ".join(detail.detail for detail in failures) return replace( score, passed=_all_required_gates_pass(gates) and not failures, gates=gates, failure_reasons=failures, failure_reason=combined_reason, ) def run_scenario( fixture: ScenarioFixture, use_mock_grafana: bool = False, grafana_backend: Any = None, ) -> tuple[dict[str, Any], ScenarioScore]: alert = fixture.alert resolved_integrations = _build_resolved_integrations( fixture, use_mock_grafana, grafana_backend=grafana_backend ) final_state = run_investigation( alert, resolved_integrations=resolved_integrations, ) state_dict = dict(final_state) queried_metrics: list[str] | None = None if grafana_backend is not None and hasattr(grafana_backend, "queried_metrics"): queried_metrics = list(grafana_backend.queried_metrics) return state_dict, score_result(fixture, state_dict, queried_metrics=queried_metrics) @dataclass(frozen=True) class _ScenarioExecution: fixture: ScenarioFixture score: ScenarioScore canonical_report_payload: dict[str, Any] observation_for_report: Any wall_time_s: float def _execute_fixture( fixture: ScenarioFixture, *, config: SuiteRunConfig, progress_hook: Callable[[str, int], None] | None = None, ) -> _ScenarioExecution: if progress_hook is not None: progress_hook(fixture.scenario_id, 1) started_at = datetime.now(UTC) started_monotonic = time.monotonic() final_state, score = run_scenario(fixture, use_mock_grafana=config.mock_grafana) wall_time_s = time.monotonic() - started_monotonic if progress_hook is not None: progress_hook(fixture.scenario_id, 2) executed_hypotheses = final_state.get("executed_hypotheses") or [] loops_used = len(executed_hypotheses) golden_trajectory, max_loops, golden_cfg = _resolved_golden_trajectory(fixture) trajectory_metrics = compute_trajectory_metrics( executed_hypotheses=executed_hypotheses, golden=golden_trajectory, loops_used=loops_used, max_loops=max_loops, ) trajectory_policy = ( evaluate_trajectory_policy( metrics=trajectory_metrics, golden_actions=golden_trajectory, policy=_trajectory_policy_for_fixture( max_loops=max_loops, golden_cfg=golden_cfg, ), ) if golden_cfg is not None else None ) score = _apply_trajectory_policy_to_score(score, trajectory_policy) if progress_hook is not None: progress_hook(fixture.scenario_id, 3) observation = build_observation( scenario_id=fixture.scenario_id, suite="axis1", backend="FixtureGrafanaBackend" if config.mock_grafana else "LiveGrafanaBackend", score=asdict(score), reasoning=asdict(score.reasoning) if score.reasoning is not None else None, trajectory=trajectory_metrics, evaluated_golden_actions=golden_trajectory, trajectory_policy=trajectory_policy, final_state=final_state, available_evidence_sources=list(fixture.metadata.available_evidence), required_evidence_sources=list(fixture.answer_key.required_evidence_sources), started_at=started_at, wall_time_s=wall_time_s, ) observation_path = write_observation(observation, config.observations_dir) relative_observation_path = str(observation_path.relative_to(config.observations_dir)) display_observation_path = str(observation_path.resolve()) observation_for_report = replace( observation, observation_path=f"{relative_observation_path} ({display_observation_path})", ) if progress_hook is not None: progress_hook(fixture.scenario_id, 4) return _ScenarioExecution( fixture=fixture, score=score, canonical_report_payload=observation.canonical_report_payload, observation_for_report=observation_for_report, wall_time_s=wall_time_s, ) def _run_level( level_config: LevelRunConfig, *, config: SuiteRunConfig, progress_hook: Callable[[str, int], None] | None = None, ) -> tuple[list[_ScenarioExecution], LevelRunResult]: started = time.monotonic() executions: list[_ScenarioExecution] = [] for fixture in level_config.fixtures: executions.append(_execute_fixture(fixture, config=config, progress_hook=progress_hook)) passed = sum(1 for execution in executions if execution.score.passed) level_result = LevelRunResult( level=level_config.level, scenario_ids=tuple(execution.fixture.scenario_id for execution in executions), passed=passed, failed=len(executions) - passed, wall_time_s=time.monotonic() - started, ) return executions, level_result @contextmanager def _suppress_investigation_rendering(enabled: bool) -> Iterator[None]: """Temporarily disable node-level investigation rendering.""" if not enabled: yield return previous_output_format = os.environ.get("TRACER_OUTPUT_FORMAT") os.environ["TRACER_OUTPUT_FORMAT"] = "none" from surfaces.interactive_shell.ui import output as output_module output_module.get_tracker(reset=True) try: yield finally: if previous_output_format is None: os.environ.pop("TRACER_OUTPUT_FORMAT", None) else: os.environ["TRACER_OUTPUT_FORMAT"] = previous_output_format output_module.get_tracker(reset=True) def _render_suite_overview( console: Console, *, config: SuiteRunConfig, level_configs: tuple[LevelRunConfig, ...], ) -> None: total = sum(len(level.fixtures) for level in level_configs) overview = Table(title="Synthetic Suite Overview", show_header=True) overview.add_column("Level", justify="right") overview.add_column("Scenarios", justify="right") overview.add_column("IDs") for level in level_configs: scenario_ids = ", ".join(fixture.scenario_id for fixture in level.fixtures) overview.add_row(str(level.level), str(len(level.fixtures)), scenario_ids) console.print(overview) console.print( "Run config: " f"total={total}, parallel_workers={config.parallel_workers}, " f"mock_grafana={config.mock_grafana}, observations_dir={config.observations_dir}" ) def _render_suite_summary( console: Console, *, executions: list[_ScenarioExecution], level_results: tuple[LevelRunResult, ...], ) -> None: summary = Table(title="Synthetic Suite Report", show_header=True) summary.add_column("Scenario") summary.add_column("Level", justify="right") summary.add_column("Status") summary.add_column("Category") summary.add_column("Wall(s)", justify="right") summary.add_column("Detail") for execution in executions: status = "PASS" if execution.score.passed else "FAIL" detail = execution.score.failure_reason or "-" summary.add_row( execution.fixture.scenario_id, str(execution.fixture.metadata.scenario_difficulty), status, execution.score.actual_category, f"{execution.wall_time_s:.2f}", detail, ) console.print(summary) level_table = Table(title="Level Summary", show_header=True) level_table.add_column("Level", justify="right") level_table.add_column("Passed", justify="right") level_table.add_column("Failed", justify="right") level_table.add_column("Wall(s)", justify="right") for level_result in level_results: level_table.add_row( str(level_result.level), str(level_result.passed), str(level_result.failed), f"{level_result.wall_time_s:.2f}", ) console.print(level_table) def _write_baseline(canonical_payloads: dict[str, Any], baseline_out_dir: Path) -> None: """Write per-scenario canonical_report_payload snapshots to *baseline_out_dir*.""" baseline_out_dir.mkdir(parents=True, exist_ok=True) for scenario_id, payload in canonical_payloads.items(): target = baseline_out_dir / f"{scenario_id}.json" target.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") def _check_baseline( canonical_payloads: dict[str, Any], baseline_check_dir: Path, ) -> list[str]: """Compare canonical payloads against committed baseline snapshots. Returns a list of human-readable mismatch descriptions (empty if all match). """ mismatches: list[str] = [] for scenario_id, actual_payload in canonical_payloads.items(): baseline_file = baseline_check_dir / f"{scenario_id}.json" if not baseline_file.exists(): mismatches.append(f"{scenario_id}: baseline file missing at {baseline_file}") continue expected = json.loads(baseline_file.read_text(encoding="utf-8")) actual_canonical = json.loads( json.dumps(actual_payload, sort_keys=True, separators=(",", ":")) ) expected_canonical = json.loads(json.dumps(expected, sort_keys=True, separators=(",", ":"))) if actual_canonical != expected_canonical: actual_str = json.dumps(actual_payload, indent=2, sort_keys=True) expected_str = json.dumps(expected, indent=2, sort_keys=True) diff_lines: list[str] = [] for line in difflib.unified_diff( expected_str.splitlines(), actual_str.splitlines(), fromfile=f"{scenario_id} (baseline)", tofile=f"{scenario_id} (actual)", lineterm="", ): diff_lines.append(line) mismatches.append( f"{scenario_id}: canonical payload differs from baseline\n" + "\n".join(diff_lines[:60]) ) return mismatches def run_synthetic_suite(config: SuiteRunConfig) -> SuiteRunResult: fixtures = load_all_scenarios(SUITE_DIR) try: selected_fixtures = select_fixtures(fixtures, config) except ValueError as exc: raise SystemExit(str(exc)) from exc level_order = ( tuple(sorted({fixture.metadata.scenario_difficulty for fixture in selected_fixtures})) if config.scenario else config.levels ) level_configs = group_fixtures_by_level(selected_fixtures, level_order) interactive_console = Console(highlight=False, soft_wrap=True) show_interactive = not config.output_json bulk_run = len(selected_fixtures) > 1 show_overview_only = show_interactive and bulk_run if show_interactive and level_configs: _render_suite_overview(interactive_console, config=config, level_configs=level_configs) level_executions: dict[int, list[_ScenarioExecution]] = {} level_results_map: dict[int, LevelRunResult] = {} task_map: dict[str, TaskID] = {} progress: Progress | None = None if show_interactive and level_configs and not show_overview_only: progress = Progress( TextColumn("[bold blue]{task.fields[level]}[/bold blue]"), TextColumn("{task.description}"), BarColumn(), TaskProgressColumn(), TimeElapsedColumn(), console=interactive_console, transient=False, ) for level_config in level_configs: for fixture in level_config.fixtures: task_id = progress.add_task( description=fixture.scenario_id, total=4, completed=0, level=f"L{level_config.level}", ) task_map[fixture.scenario_id] = task_id def _progress_hook(scenario_id: str, step: int) -> None: if progress is None: return task_id = task_map.get(scenario_id) if task_id is None: return progress.update(task_id, completed=step) all_fixtures = [f for lc in level_configs for f in lc.fixtures] max_workers = min(config.parallel_workers, len(all_fixtures)) if all_fixtures else 1 progress_context = progress if progress is not None else nullcontext() suppress_investigation_rendering = bulk_run or config.output_json with ( _suppress_investigation_rendering(suppress_investigation_rendering), progress_context, ): if max_workers > 1: with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_fixture = { executor.submit( _execute_fixture, fixture, config=config, progress_hook=_progress_hook, ): fixture for fixture in all_fixtures } for future in as_completed(future_to_fixture): execution = future.result() level = execution.fixture.metadata.scenario_difficulty level_executions.setdefault(level, []).append(execution) else: for fixture in all_fixtures: execution = _execute_fixture(fixture, config=config, progress_hook=_progress_hook) level = execution.fixture.metadata.scenario_difficulty level_executions.setdefault(level, []).append(execution) for level_config in level_configs: executions = level_executions.get(level_config.level, []) passed = sum(1 for e in executions if e.score.passed) level_results_map[level_config.level] = LevelRunResult( level=level_config.level, scenario_ids=tuple(e.fixture.scenario_id for e in executions), passed=passed, failed=len(executions) - passed, wall_time_s=sum(e.wall_time_s for e in executions), ) ordered_executions: list[_ScenarioExecution] = [] ordered_level_results: list[LevelRunResult] = [] for level in level_order: if level in level_results_map: ordered_executions.extend(level_executions[level]) ordered_level_results.append(level_results_map[level]) should_report = ( bool(config.report) if config.report is not None else len(selected_fixtures) == 1 ) if config.output_json: should_report = False if should_report: report_console = ( interactive_console if show_interactive else Console(highlight=False, soft_wrap=True) ) for execution in ordered_executions: render_report_to_console(execution.observation_for_report, report_console) if show_interactive and ordered_executions and not show_overview_only: _render_suite_summary( interactive_console, executions=ordered_executions, level_results=tuple(ordered_level_results), ) return SuiteRunResult( config=config, level_results=tuple(ordered_level_results), scores=tuple(execution.score for execution in ordered_executions), canonical_payloads={ execution.fixture.scenario_id: execution.canonical_report_payload for execution in ordered_executions }, ) def _run_axis2_suite( fixtures: list[ScenarioFixture], *, output_json: bool, ) -> list[ScenarioScore]: """Run every fixture twice (axis 1 and axis 2) and emit the gap report. Axis 1 uses ``FixtureGrafanaBackend`` (full mock data, the same backend the default suite uses with ``--mock-grafana``). Axis 2 uses ``SelectiveGrafanaBackend`` (query-aware adversarial mock). The combined result list is returned so :func:`main`'s exit code reflects failures on either axis — a fully-failing axis 2 run still surfaces as non-zero. """ axis1_results: list[ScenarioScore] = [] axis2_results: list[ScenarioScore] = [] for fixture in fixtures: _, score1 = run_scenario(fixture, use_mock_grafana=True) axis1_results.append(score1) _, score2 = run_scenario( fixture, use_mock_grafana=False, grafana_backend=SelectiveGrafanaBackend(fixture), ) axis2_results.append(score2) if output_json: print( json.dumps( { "axis1": [asdict(r) for r in axis1_results], "axis2": [asdict(r) for r in axis2_results], }, indent=2, ) ) else: print_gap_report(axis1_results, axis2_results, fixtures) return axis1_results + axis2_results def run_suite(argv: list[str] | None = None) -> list[ScenarioScore]: args = parse_args(argv) config = _build_run_config(args) validate_synthetic_llm_provider(suite_name="RDS PostgreSQL") # --axis2 short-circuits the default per-level orchestration: every selected # fixture is run twice (FixtureGrafanaBackend then SelectiveGrafanaBackend) # and the cross-axis gap is printed via ``print_gap_report``. This is the # canonical command documented in tests/synthetic/rds_postgres/README.md. if args.axis2: all_fixtures = load_all_scenarios(SUITE_DIR) try: selected_fixtures = select_fixtures(all_fixtures, config) except ValueError as exc: raise SystemExit(str(exc)) from exc return _run_axis2_suite(selected_fixtures, output_json=bool(args.json)) suite_result = run_synthetic_suite(config) results = list(suite_result.scores) canonical_payloads = dict(suite_result.canonical_payloads) if args.json: print(json.dumps([asdict(result) for result in results], indent=2)) if config.baseline_out: _write_baseline(canonical_payloads, config.baseline_out) if config.baseline_check: mismatches = _check_baseline(canonical_payloads, config.baseline_check) if mismatches: print("\n=== Baseline Check FAILED ===") for msg in mismatches: print(textwrap.indent(msg, " ")) raise SystemExit(1) return results def main(argv: list[str] | None = None) -> int: try: results = run_suite(argv) except UnsupportedSyntheticLLMProviderError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 return 0 if all(result.passed for result in results) else 1 if __name__ == "__main__": raise SystemExit(main())