tracer-cloud--opensre/tests/synthetic/rds_postgres/test_run_suite_observations.py

from __future__ import annotations

import json
import os
from dataclasses import replace
from pathlib import Path
from typing import Any

import pytest

import tests.synthetic.rds_postgres.run_suite as run_suite_module
from tests.synthetic.rds_postgres.scenario_loader import (
    SUITE_DIR,
    GoldenTrajectoryConfig,
    load_scenario,
)


def _fake_final_state() -> dict[str, Any]:
    return {
        "root_cause": "Replication lag due to write pressure.",
        "root_cause_category": "replication_lag",
        "validated_claims": [],
        "non_validated_claims": [],
        "causal_chain": [],
        "evidence": {
            "aws_cloudwatch_metrics": {
                "db_instance_identifier": "payments-prod",
                "metrics": [{"metric_name": "CPUUtilization"}],
                "observations": ["CPU elevated"],
            }
        },
        "executed_hypotheses": [
            {"actions": ["query_grafana_metrics"], "failed_actions": []},
        ],
        "investigation_loop_count": 1,
    }


def _fake_final_state_with_two_loops() -> dict[str, Any]:
    return {
        "root_cause": "Replication lag due to write pressure.",
        "root_cause_category": "replication_lag",
        "validated_claims": [],
        "non_validated_claims": [],
        "causal_chain": [],
        "evidence": {
            "aws_cloudwatch_metrics": {
                "db_instance_identifier": "payments-prod",
                "metrics": [{"metric_name": "CPUUtilization"}],
                "observations": ["CPU elevated"],
            }
        },
        "executed_hypotheses": [
            {"actions": ["query_grafana_metrics"], "failed_actions": []},
            {"actions": ["query_grafana_logs"], "failed_actions": []},
        ],
        "investigation_loop_count": 2,
    }


def _fake_write_observation(
    _observation: Any,
    observations_dir: Path,
) -> Path:
    target = observations_dir / "001-replication-lag" / "latest.json"
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_text("{}", encoding="utf-8")
    return target


def test_run_suite_applies_trajectory_policy_failure(
    monkeypatch,
    tmp_path: Path,
) -> None:
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")
    fixture = replace(
        fixture,
        answer_key=replace(
            fixture.answer_key,
            golden_trajectory=GoldenTrajectoryConfig(
                ordered_actions=[
                    "query_grafana_metrics",
                    "query_grafana_logs",
                ],
                matching="strict",
                max_extra_actions=0,
            ),
        ),
    )

    score = run_suite_module.ScenarioScore(
        scenario_id=fixture.scenario_id,
        passed=True,
        root_cause_present=True,
        expected_category=fixture.answer_key.root_cause_category,
        accepted_categories=tuple(sorted({fixture.answer_key.root_cause_category})),
        actual_category=fixture.answer_key.root_cause_category,
        missing_keywords=[],
        matched_keywords=list(fixture.answer_key.required_keywords),
        root_cause="Replication lag due to write pressure.",
    )

    render_calls: list[str] = []

    monkeypatch.setattr(run_suite_module, "load_all_scenarios", lambda _suite_dir: [fixture])
    monkeypatch.setattr(
        run_suite_module, "run_scenario", lambda *_args, **_kwargs: (_fake_final_state(), score)
    )
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation)
    monkeypatch.setattr(
        run_suite_module,
        "render_report_to_console",
        lambda *_args, **_kwargs: render_calls.append("called"),
    )

    results = run_suite_module.run_suite(
        [
            "--scenario",
            fixture.scenario_id,
            "--report",
            "--observations-dir",
            str(tmp_path),
        ]
    )

    assert len(results) == 1
    assert results[0].passed is False
    assert "trajectory policy failed" in results[0].failure_reason
    assert render_calls == ["called"]


def test_run_suite_enforces_resolved_loop_threshold_and_persists_score_consistency(
    monkeypatch,
    tmp_path: Path,
) -> None:
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")
    fixture = replace(
        fixture,
        answer_key=replace(
            fixture.answer_key,
            max_investigation_loops=1,
            golden_trajectory=GoldenTrajectoryConfig(
                ordered_actions=["query_grafana_metrics"],
                matching="set",
            ),
        ),
    )

    score = run_suite_module.ScenarioScore(
        scenario_id=fixture.scenario_id,
        passed=True,
        root_cause_present=True,
        expected_category=fixture.answer_key.root_cause_category,
        accepted_categories=tuple(sorted({fixture.answer_key.root_cause_category})),
        actual_category=fixture.answer_key.root_cause_category,
        missing_keywords=[],
        matched_keywords=list(fixture.answer_key.required_keywords),
        root_cause="Replication lag due to write pressure.",
    )

    persisted_scores: list[dict[str, Any]] = []

    def _capture_write_observation(observation: Any, observations_dir: Path) -> Path:
        persisted_scores.append(dict(observation.score))
        target = observations_dir / "001-replication-lag" / "latest.json"
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_text("{}", encoding="utf-8")
        return target

    monkeypatch.setattr(run_suite_module, "load_all_scenarios", lambda _suite_dir: [fixture])
    monkeypatch.setattr(
        run_suite_module,
        "run_scenario",
        lambda *_args, **_kwargs: (_fake_final_state_with_two_loops(), score),
    )
    monkeypatch.setattr(run_suite_module, "write_observation", _capture_write_observation)

    results = run_suite_module.run_suite(
        [
            "--scenario",
            fixture.scenario_id,
            "--observations-dir",
            str(tmp_path),
        ]
    )

    assert len(results) == 1
    assert results[0].passed is False
    assert "trajectory policy failed" in results[0].failure_reason
    assert "loops_used=2 > 1" in results[0].failure_reason
    assert len(persisted_scores) == 1
    assert persisted_scores[0]["passed"] is False
    assert persisted_scores[0]["failure_reason"] == results[0].failure_reason


def test_run_suite_json_mode_suppresses_report_render(
    monkeypatch,
    tmp_path: Path,
) -> None:
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")

    score = run_suite_module.ScenarioScore(
        scenario_id=fixture.scenario_id,
        passed=True,
        root_cause_present=True,
        expected_category=fixture.answer_key.root_cause_category,
        accepted_categories=tuple(sorted({fixture.answer_key.root_cause_category})),
        actual_category=fixture.answer_key.root_cause_category,
        missing_keywords=[],
        matched_keywords=list(fixture.answer_key.required_keywords),
        root_cause="Replication lag due to write pressure.",
    )

    render_calls: list[str] = []

    monkeypatch.setattr(run_suite_module, "load_all_scenarios", lambda _suite_dir: [fixture])
    monkeypatch.setattr(
        run_suite_module, "run_scenario", lambda *_args, **_kwargs: (_fake_final_state(), score)
    )
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation)
    monkeypatch.setattr(
        run_suite_module,
        "render_report_to_console",
        lambda *_args, **_kwargs: render_calls.append("called"),
    )

    _ = run_suite_module.run_suite(
        [
            "--scenario",
            fixture.scenario_id,
            "--report",
            "--json",
            "--observations-dir",
            str(tmp_path),
        ]
    )

    assert render_calls == []


def test_run_suite_bulk_execution_suppresses_investigation_rendering(
    monkeypatch: Any,
    tmp_path: Path,
) -> None:
    fixture_a = load_scenario(SUITE_DIR / "000-healthy")
    fixture_b = load_scenario(SUITE_DIR / "001-replication-lag")
    fixtures = [fixture_a, fixture_b]
    output_formats_seen: list[str | None] = []

    monkeypatch.setenv("TRACER_OUTPUT_FORMAT", "rich")
    monkeypatch.setattr(run_suite_module, "load_all_scenarios", lambda _suite_dir: fixtures)
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation)

    def _fake_run_scenario(
        fixture: Any,
        use_mock_grafana: bool = False,  # noqa: ARG001
        grafana_backend: Any = None,  # noqa: ARG001
    ) -> tuple[dict[str, Any], Any]:
        output_formats_seen.append(os.environ.get("TRACER_OUTPUT_FORMAT"))
        return _make_deterministic_final_state(), _make_base_score(fixture)

    monkeypatch.setattr(run_suite_module, "run_scenario", _fake_run_scenario)

    run_suite_module.run_suite(["--levels", "1", "--observations-dir", str(tmp_path)])

    assert output_formats_seen == ["none", "none"]
    assert os.environ.get("TRACER_OUTPUT_FORMAT") == "rich"


def test_run_suite_single_scenario_keeps_investigation_rendering_mode(
    monkeypatch: Any,
    tmp_path: Path,
) -> None:
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")
    output_formats_seen: list[str | None] = []

    monkeypatch.setattr(run_suite_module, "load_all_scenarios", lambda _suite_dir: [fixture])
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation)

    def _fake_run_scenario(
        fixture: Any,
        use_mock_grafana: bool = False,  # noqa: ARG001
        grafana_backend: Any = None,  # noqa: ARG001
    ) -> tuple[dict[str, Any], Any]:
        output_formats_seen.append(os.environ.get("TRACER_OUTPUT_FORMAT"))
        return _make_deterministic_final_state(), _make_base_score(fixture)

    monkeypatch.setattr(run_suite_module, "run_scenario", _fake_run_scenario)

    run_suite_module.run_suite(
        ["--scenario", fixture.scenario_id, "--observations-dir", str(tmp_path)]
    )

    assert output_formats_seen == [None]


# ---------------------------------------------------------------------------
# Baseline guardrail tests (Phase 0)
# ---------------------------------------------------------------------------

_BASELINE_DIR = Path(__file__).parent / "_baseline"


def _make_deterministic_final_state() -> dict[str, Any]:
    """Empty final state that produces stable canonical payloads without LLM calls."""
    return {
        "root_cause": "",
        "root_cause_category": "unknown",
        "validated_claims": [],
        "non_validated_claims": [],
        "causal_chain": [],
        "evidence": {},
        "executed_hypotheses": [],
        "investigation_loop_count": 0,
        "report": "",
    }


def _make_base_score(fixture: Any) -> Any:
    """Return a ScenarioScore identical to what score_result produces for an empty final state.

    The baseline was generated using score_result with an empty final state, so this
    must match exactly — bare ScenarioScore construction omits gates and is not equivalent.
    """
    return run_suite_module.score_result(fixture, _make_deterministic_final_state())


def _fake_write_observation_baseline(
    _observation: Any,
    observations_dir: Path,
) -> Path:
    target = observations_dir / "001-replication-lag" / "latest.json"
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_text("{}", encoding="utf-8")
    return target


def test_baseline_check_passes_against_committed_baseline(
    monkeypatch: Any,
    tmp_path: Path,
) -> None:
    """run_suite --baseline-check exits cleanly when payloads match the committed baseline."""
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")

    monkeypatch.setattr(
        run_suite_module,
        "load_all_scenarios",
        lambda _suite_dir: [fixture],
    )
    monkeypatch.setattr(
        run_suite_module,
        "run_scenario",
        lambda *_args, **_kwargs: (
            _make_deterministic_final_state(),
            _make_base_score(fixture),
        ),
    )
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation_baseline)

    # Should not raise SystemExit
    run_suite_module.run_suite(
        [
            "--scenario",
            fixture.scenario_id,
            "--observations-dir",
            str(tmp_path),
            "--baseline-check",
            str(_BASELINE_DIR),
        ]
    )


def test_baseline_check_detects_drift(
    monkeypatch: Any,
    tmp_path: Path,
) -> None:
    """run_suite --baseline-check exits non-zero when a canonical field differs from baseline."""
    fixture = load_scenario(SUITE_DIR / "001-replication-lag")

    # Tamper with the baseline to simulate drift
    tampered_dir = tmp_path / "tampered_baseline"
    tampered_dir.mkdir()
    original = json.loads((_BASELINE_DIR / "001-replication-lag.json").read_text())
    original["status"] = "pass"  # invert status to force mismatch
    (tampered_dir / "001-replication-lag.json").write_text(
        json.dumps(original, indent=2, sort_keys=True)
    )

    monkeypatch.setattr(
        run_suite_module,
        "load_all_scenarios",
        lambda _suite_dir: [fixture],
    )
    monkeypatch.setattr(
        run_suite_module,
        "run_scenario",
        lambda *_args, **_kwargs: (
            _make_deterministic_final_state(),
            _make_base_score(fixture),
        ),
    )
    monkeypatch.setattr(run_suite_module, "write_observation", _fake_write_observation_baseline)

    with pytest.raises(SystemExit) as exc_info:
        run_suite_module.run_suite(
            [
                "--scenario",
                fixture.scenario_id,
                "--observations-dir",
                str(tmp_path),
                "--baseline-check",
                str(tampered_dir),
            ]
        )
    assert exc_info.value.code == 1