tracer-cloud--opensre/tests/synthetic/rds_postgres/test_observation_metrics.py

from __future__ import annotations

import json
from dataclasses import asdict
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from tests.synthetic.rds_postgres.observations import (
    build_observation,
    compute_trajectory_metrics,
    edit_distance,
    lcs_length,
    render_report_to_string,
    write_observation,
)
from tests.synthetic.rds_postgres.run_suite import (
    _apply_trajectory_policy_to_score,
    _resolved_golden_trajectory,
    _trajectory_policy_for_fixture,
)
from tests.synthetic.rds_postgres.scenario_loader import SUITE_DIR, load_all_scenarios
from tests.synthetic.rds_postgres.scoring import score_result
from tests.synthetic.rds_postgres.trajectory_policy import (
    TrajectoryPolicy,
    evaluate_trajectory_policy,
)


def _sample_final_state() -> dict[str, Any]:
    return {
        "evidence": {
            "grafana_metrics": [{"metric_name": "CPUUtilization"}],
            "grafana_logs": [{"message": "replica lag detected"}],
            "aws_cloudwatch_metrics": {
                "db_instance_identifier": "db-1",
                "metrics": [{"metric_name": "CPUUtilization"}],
                "observations": ["CPU is elevated"],
            },
            "aws_performance_insights": {
                "observations": ["Top SQL Activity: select 1 | Avg Load: 2.0 AAS | Waits: CPU"],
                "top_sql": [{"sql": "select 1", "db_load": 2.0, "wait_event": "CPU"}],
                "wait_events": [],
            },
        },
        "executed_hypotheses": [
            {"actions": ["query_grafana_metrics", "query_grafana_logs"], "failed_actions": []}
        ],
        "investigation_loop_count": 1,
        "root_cause": "Replication lag from write-heavy workload.",
    }


def _sample_score_payload() -> dict[str, Any]:
    return {
        "scenario_id": "001-replication-lag",
        "passed": True,
        "expected_category": "resource_exhaustion",
        "actual_category": "resource_exhaustion",
        "missing_keywords": [],
        "matched_keywords": ["replication lag", "wal"],
        "exact_matched_keywords": ["replication lag", "wal"],
        "exact_missing_keywords": [],
        "semantic_matched_keywords": ["replication lag", "wal"],
        "semantic_missing_keywords": [],
        "exact_keyword_match": True,
        "semantic_keyword_match": True,
        "normalization_used": ["casefold_whitespace_normalization", "exact_phrase"],
        "gates": {
            "category_match": {
                "status": "pass",
                "threshold": "actual_category == 'resource_exhaustion'",
                "actual": "root_cause_present=True, actual_category='resource_exhaustion'",
            },
            "required_keyword_match": {
                "status": "pass",
                "threshold": "all required keywords matched (semantic)",
                "actual": "missing_semantic=[], missing_exact=[]",
            },
            "required_evidence_sources": {
                "status": "pass",
                "threshold": "all required evidence sources populated",
                "actual": "missing_required_evidence=[]",
            },
            "trajectory_budget": {
                "status": "pass",
                "threshold": "extra_actions_count == 0",
                "actual": "extra_actions_count=0",
            },
            "forbidden_category_clear": {
                "status": "pass",
                "threshold": "actual_category not in forbidden_categories",
                "actual": "actual_category='resource_exhaustion', forbidden=[]",
            },
            "forbidden_keyword_clear": {
                "status": "pass",
                "threshold": "no forbidden keywords appear in graded output text",
                "actual": "forbidden_hits=[]",
            },
            "failover_event_reasoning": {
                "status": "pass",
                "threshold": "not required unless failover sequence keywords are in answer key",
                "actual": "not_applicable",
            },
        },
        "failure_reasons": [],
        "failure_reason": "",
        "trajectory": {
            "expected_sequence": [
                "query_grafana_metrics",
                "query_grafana_logs",
                "query_grafana_alert_rules",
            ]
        },
    }


def test_lcs_length_and_edit_distance() -> None:
    a = ["query_grafana_metrics", "query_grafana_logs", "query_grafana_alert_rules"]
    b = ["query_grafana_metrics", "query_grafana_alert_rules"]
    assert lcs_length(a, b) == 2
    assert edit_distance(a, b) == 1


def test_compute_trajectory_metrics_detects_missing_and_redundancy() -> None:
    executed = [
        {
            "actions": [
                "query_grafana_metrics",
                "query_grafana_metrics",
                "query_grafana_logs",
            ],
            "failed_actions": [],
        }
    ]
    golden = [
        "query_grafana_metrics",
        "query_grafana_logs",
        "query_grafana_alert_rules",
    ]
    metrics = compute_trajectory_metrics(
        executed_hypotheses=executed,
        golden=golden,
        loops_used=1,
        max_loops=4,
    )
    assert metrics.missing_actions == ["query_grafana_alert_rules"]
    assert metrics.extra_actions == []
    assert metrics.redundancy_count == 1
    assert metrics.failed_action_count == 0
    assert metrics.loop_calibration_ok is True


def test_observation_roundtrip_and_report_rendering(tmp_path: Path) -> None:
    final_state = _sample_final_state()
    score = _sample_score_payload()
    trajectory = compute_trajectory_metrics(
        executed_hypotheses=final_state["executed_hypotheses"],
        golden=score["trajectory"]["expected_sequence"],
        loops_used=1,
        max_loops=4,
    )
    observation = build_observation(
        scenario_id="001-replication-lag",
        suite="axis1",
        backend="FixtureGrafanaBackend",
        score=score,
        reasoning=None,
        trajectory=trajectory,
        evaluated_golden_actions=list(score["trajectory"]["expected_sequence"]),
        trajectory_policy=evaluate_trajectory_policy(
            metrics=trajectory,
            golden_actions=list(score["trajectory"]["expected_sequence"]),
            policy=TrajectoryPolicy(matching="lcs"),
        ),
        final_state=final_state,
        available_evidence_sources=[
            "aws_cloudwatch_metrics",
            "aws_performance_insights",
            "aws_rds_events",
        ],
        required_evidence_sources=["aws_performance_insights", "aws_rds_events"],
        started_at=datetime.now(UTC),
        wall_time_s=1.2,
    )

    output_path = write_observation(observation, tmp_path)
    payload = json.loads(output_path.read_text(encoding="utf-8"))
    assert payload["scenario_id"] == "001-replication-lag"
    assert payload["report_schema_version"] == "report_v2"
    assert payload["scoring_formula_version"] == "v2_gated_semantic"
    assert "process_metrics" in payload["score"]
    assert payload["score"]["process_metrics"]["redundancy_count"] == 0
    assert payload["score"]["process_metrics"]["loop_count_consistent"] is True
    assert (
        "Duplicate action executions"
        in payload["score"]["process_metrics"]["definitions"]["redundancy_count"]
    )
    assert payload["score"]["actual_category"] == "resource_exhaustion"
    assert payload["observed_evidence_sources"] == [
        "aws_cloudwatch_metrics",
        "aws_performance_insights",
    ]
    assert payload["missing_required_evidence_sources"] == ["aws_rds_events"]
    assert payload["evidence_source_coverage"]["required_coverage"] == 0.5
    assert payload["evidence_source_coverage"]["available_coverage"] == 2 / 3
    assert payload["evidence_source_coverage"]["source_presence"] == {
        "aws_cloudwatch_metrics": True,
        "aws_performance_insights": True,
        "aws_rds_events": False,
    }
    assert payload["canonical_report_payload"]["status"] == "pass"
    assert payload["canonical_report_payload"]["report_schema_version"] == "report_v2"
    assert payload["canonical_report_payload"]["scoring_formula_version"] == "v2_gated_semantic"
    assert "failure_reason" not in payload["canonical_report_payload"]
    assert payload["canonical_report_payload"]["failure_reasons"] == []
    assert payload["canonical_report_payload"]["trajectory"]["golden"] == [
        "query_grafana_metrics",
        "query_grafana_logs",
        "query_grafana_alert_rules",
    ]
    assert payload["canonical_report_payload"]["trajectory"]["policy"] == {
        "passed": False,
        "matching": "lcs",
        "violations": ["lcs_ratio=0.67 < 1.00"],
    }
    assert payload["canonical_report_payload"]["evidence"]["missing_required_sources"] == [
        "aws_rds_events"
    ]
    assert payload["canonical_report_payload"]["observation_path"] == payload["observation_path"]
    assert payload["reasoning_status"] == "not_captured"
    assert "reasoning" not in payload
    assert payload["trajectory_policy_version"] == "default_v1"
    assert (tmp_path / "001-replication-lag" / "latest.json").exists()

    report_text = render_report_to_string(observation)
    assert "Synthetic RDS Run - 001-replication-lag" in report_text
    assert "PASS" in report_text
    assert "Observed evidence" in report_text
    assert "aws_performance_insights" in report_text
    assert "Missing evidence" in report_text
    assert "policy" in report_text
    assert "Trajectory" in report_text
    assert "lcs=0.67" in report_text
    assert "Observation:" in report_text


def test_compute_trajectory_metrics_handles_all_rds_scenarios() -> None:
    fixtures = load_all_scenarios()
    for fixture in fixtures:
        metrics = compute_trajectory_metrics(
            executed_hypotheses=[],
            golden=list(fixture.answer_key.optimal_trajectory),
            loops_used=0,
            max_loops=fixture.answer_key.max_investigation_loops,
        )
        assert metrics.loops_used == 0
        assert metrics.actions_per_loop == []


# ---------------------------------------------------------------------------
# Acceptance tests: canonical payload key stability (Phase 0)
# ---------------------------------------------------------------------------

_CANONICAL_CONTRACT_KEYS = (
    "report_schema_version",
    "scoring_formula_version",
    "status",
    "gates",
    "failure_reasons",
    "verdict_definitions",
    "trajectory",
)
_CANONICAL_EVIDENCE_KEYS = (
    "observed_sources",
    "required_sources",
    "missing_required_sources",
    "source_presence",
    "required_coverage",
    "available_coverage",
)
_CANONICAL_TRAJECTORY_KEYS = (
    "golden",
    "actual",
    "policy",
    "lcs_ratio",
    "edit_distance",
    "coverage",
    "extra_actions",
    "missing_actions",
    "redundancy_count",
    "failed_action_count",
    "strict_match",
)


def _build_empty_canonical_payload(fixture: Any) -> dict[str, Any]:
    """Return a canonical payload for the given fixture using an empty (no-LLM) final state."""
    final_state: dict[str, Any] = {
        "root_cause": "",
        "root_cause_category": "unknown",
        "validated_claims": [],
        "non_validated_claims": [],
        "causal_chain": [],
        "evidence": {},
        "executed_hypotheses": [],
        "investigation_loop_count": 0,
        "report": "",
    }
    score = score_result(fixture, final_state)
    golden_trajectory, max_loops, golden_cfg = _resolved_golden_trajectory(fixture)
    trajectory_metrics = compute_trajectory_metrics(
        executed_hypotheses=[],
        golden=golden_trajectory,
        loops_used=0,
        max_loops=max_loops,
    )
    trajectory_policy = (
        evaluate_trajectory_policy(
            metrics=trajectory_metrics,
            golden_actions=golden_trajectory,
            policy=_trajectory_policy_for_fixture(max_loops=max_loops, golden_cfg=golden_cfg),
        )
        if golden_cfg is not None
        else None
    )
    score = _apply_trajectory_policy_to_score(score, trajectory_policy)
    observation = build_observation(
        scenario_id=fixture.scenario_id,
        suite="axis1",
        backend="FixtureGrafanaBackend",
        score=asdict(score),
        reasoning=None,
        trajectory=trajectory_metrics,
        evaluated_golden_actions=golden_trajectory,
        trajectory_policy=trajectory_policy,
        final_state=final_state,
        available_evidence_sources=list(fixture.metadata.available_evidence),
        required_evidence_sources=list(fixture.answer_key.required_evidence_sources),
        started_at=datetime.now(UTC),
        wall_time_s=0.0,
    )
    return observation.canonical_report_payload


def test_canonical_payload_keys_are_stable() -> None:
    """Each scenario canonical payload contains the required contract keys."""
    fixtures = load_all_scenarios(SUITE_DIR)
    assert fixtures, "no scenarios found"
    for fixture in fixtures:
        payload = _build_empty_canonical_payload(fixture)
        for key in _CANONICAL_CONTRACT_KEYS:
            assert key in payload, (
                f"{fixture.scenario_id}: canonical payload missing top-level key {key!r}"
            )
        evidence = payload.get("evidence", {})
        for key in _CANONICAL_EVIDENCE_KEYS:
            assert key in evidence, (
                f"{fixture.scenario_id}: canonical payload['evidence'] missing key {key!r}"
            )
        trajectory = payload.get("trajectory", {})
        for key in _CANONICAL_TRAJECTORY_KEYS:
            assert key in trajectory, (
                f"{fixture.scenario_id}: canonical payload['trajectory'] missing key {key!r}"
            )


def test_write_observation_canonical_filename_is_deterministic(tmp_path: Path) -> None:
    """write_observation must produce the same filename for the same canonical payload."""
    import re

    fixtures = load_all_scenarios(SUITE_DIR)
    fixture = fixtures[0]

    # Build via build_observation so we get a full RunObservation
    final_state: dict[str, Any] = {
        "root_cause": "",
        "root_cause_category": "unknown",
        "validated_claims": [],
        "non_validated_claims": [],
        "causal_chain": [],
        "evidence": {},
        "executed_hypotheses": [],
        "investigation_loop_count": 0,
        "report": "",
    }
    score = score_result(fixture, final_state)
    golden_trajectory, max_loops, golden_cfg = _resolved_golden_trajectory(fixture)
    trajectory_metrics = compute_trajectory_metrics(
        executed_hypotheses=[],
        golden=golden_trajectory,
        loops_used=0,
        max_loops=max_loops,
    )
    trajectory_policy = (
        evaluate_trajectory_policy(
            metrics=trajectory_metrics,
            golden_actions=golden_trajectory,
            policy=_trajectory_policy_for_fixture(max_loops=max_loops, golden_cfg=golden_cfg),
        )
        if golden_cfg is not None
        else None
    )
    score = _apply_trajectory_policy_to_score(score, trajectory_policy)
    obs = build_observation(
        scenario_id=fixture.scenario_id,
        suite="axis1",
        backend="FixtureGrafanaBackend",
        score=asdict(score),
        reasoning=None,
        trajectory=trajectory_metrics,
        evaluated_golden_actions=golden_trajectory,
        trajectory_policy=trajectory_policy,
        final_state=final_state,
        available_evidence_sources=list(fixture.metadata.available_evidence),
        required_evidence_sources=list(fixture.answer_key.required_evidence_sources),
        started_at=datetime.now(UTC),
        wall_time_s=0.0,
    )

    dir_a = tmp_path / "a"
    dir_b = tmp_path / "b"
    path_a = write_observation(obs, dir_a)
    path_b = write_observation(obs, dir_b)

    assert path_a.name == path_b.name, (
        f"Non-deterministic filenames: {path_a.name!r} vs {path_b.name!r}"
    )
    assert re.fullmatch(r"[0-9a-f]{12}\.json", path_a.name), (
        f"Filename {path_a.name!r} does not match content-addressed pattern"
    )