open-metadata--openmetadata/ingestion/tests/unit/diagnostics/test_tripwire.py

#  Copyright 2025 Collate
#  Licensed under the Collate Community License, Version 1.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""Memory-pressure tripwires (PSI, cgroup events) + MemoryError context manager."""

import logging
import time

import pytest

from metadata.ingestion import diagnostics
from metadata.ingestion.diagnostics.collectors.operation_registry import OperationRegistry
from metadata.ingestion.diagnostics.config import DiagnosticsConfig
from metadata.ingestion.diagnostics.monitors.watchdog import Verdict, Watchdog
from metadata.ingestion.diagnostics.samplers.memory import MemorySample, MemoryTracker


class _FixedSampleTracker(MemoryTracker):
    """MemoryTracker whose `sample()` returns a scripted MemorySample sequence."""

    def __init__(self, samples: list[MemorySample]) -> None:
        super().__init__()
        self._scripted = list(samples)

    def sample(self) -> MemorySample:
        if not self._scripted:
            return MemorySample(
                ts=time.monotonic(),
                rss=1000,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
            )
        return self._scripted.pop(0)


def _watchdog_with(samples: list[MemorySample]) -> Watchdog:
    return Watchdog(OperationRegistry(), _FixedSampleTracker(samples), DiagnosticsConfig())


def _dump_verdicts(verdicts: list[Verdict]) -> list[Verdict]:
    return [v for v in verdicts if v.dump_reason is not None]


# ---- PSI tripwire ----


def test_psi_below_threshold_does_not_trip():
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                psi_some_avg10=DiagnosticsConfig().pressure_psi_avg10_threshold - 0.1,
            )
        ]
    )
    assert _dump_verdicts(wd.check(time.monotonic())) == []


def test_psi_above_threshold_emits_dump_verdict():
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                psi_some_avg10=42.5,
            )
        ]
    )
    verdicts = _dump_verdicts(wd.check(time.monotonic()))
    assert len(verdicts) == 1
    verdict = verdicts[0]
    assert "diag.warn.memory_pressure" in verdict.message
    assert "memory-pressure-psi:avg10=42.5" in verdict.message
    assert verdict.dump_reason == "memory-pressure-psi:avg10=42.5"


# ---- cgroup memory.events.high ----


def test_events_high_increment_emits_dump_verdict():
    """First tick: baseline (no fire). Second tick: counter increased → fire."""
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_high=5,
            ),
            MemorySample(
                ts=1,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_high=12,
            ),
        ]
    )
    assert _dump_verdicts(wd.check(time.monotonic())) == []
    second = _dump_verdicts(wd.check(time.monotonic()))
    assert len(second) == 1
    assert "memory-pressure-cgroup-high:delta=7" in second[0].message


def test_events_high_unchanged_does_not_trip():
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_high=5,
            ),
            MemorySample(
                ts=1,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_high=5,
            ),
        ]
    )
    assert _dump_verdicts(wd.check(time.monotonic())) == []
    assert _dump_verdicts(wd.check(time.monotonic())) == []


# ---- cgroup memory.events.oom ----


def test_events_oom_increment_emits_dump_verdict():
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_oom=0,
            ),
            MemorySample(
                ts=1,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                cgroup_events_oom=1,
            ),
        ]
    )
    wd.check(time.monotonic())  # baseline
    second = _dump_verdicts(wd.check(time.monotonic()))
    assert len(second) == 1
    assert "memory-pressure-cgroup-oom" in second[0].message


# ---- throttling ----


def test_psi_tripwire_is_throttled():
    wd = _watchdog_with(
        [
            MemorySample(
                ts=0,
                rss=1,
                cgroup_current=None,
                cgroup_max=None,
                oom_kill_count=None,
                psi_some_avg10=50.0,
            )
            for _ in range(5)
        ]
    )
    first = _dump_verdicts(wd.check(time.monotonic()))
    second = _dump_verdicts(wd.check(time.monotonic()))
    third = _dump_verdicts(wd.check(time.monotonic()))
    assert len(first) == 1
    assert second == []
    assert third == []


# ---- MemoryError context manager (facade-level integration) ----


@pytest.fixture()
def _diag_installed():
    class _Cfg:
        class loggerLevel:  # noqa: N801
            value = "DEBUG"

    class _W:
        workflow_config = _Cfg()

        def workflow_steps(self):
            return []

    diagnostics.shutdown()
    assert diagnostics.install(_W())
    yield
    diagnostics.shutdown()


def test_memory_error_triggers_dump_then_reraises(_diag_installed, caplog):
    """Python-side MemoryError should produce a dump and propagate."""
    with (
        caplog.at_level(logging.WARNING, logger="metadata.Diagnostics"),
        pytest.raises(MemoryError),
        diagnostics.dump_on_memory_error(),
    ):
        raise MemoryError("simulated")
    out = "\n".join(r.getMessage() for r in caplog.records)
    assert "memory-error:" in out
    assert "diag.dump.begin" in out


def test_dump_on_memory_error_passes_through_other_exceptions(_diag_installed):
    with pytest.raises(RuntimeError), diagnostics.dump_on_memory_error():
        raise RuntimeError("not a memory error")


def test_dump_on_memory_error_noop_when_diagnostics_off(caplog):
    diagnostics.shutdown()
    with pytest.raises(MemoryError), diagnostics.dump_on_memory_error():
        raise MemoryError("simulated")
    diag_records = [r for r in caplog.records if "diag" in r.getMessage()]
    assert diag_records == []


# ---- perf smoke + emergency reserve (unchanged behavior; lives here for proximity) ----


def test_sample_is_under_5ms():
    """Each tracker.sample() should be sub-millisecond on average.

    Five samples must complete in under 25 ms total — well under the 10s
    watchdog tick. This guards against accidental I/O regression in the
    consolidated readers.
    """
    tracker = MemoryTracker()
    started = time.monotonic()
    for _ in range(5):
        tracker.sample()
    elapsed_ms = (time.monotonic() - started) * 1000
    assert elapsed_ms < 25, f"5x sample took {elapsed_ms:.1f}ms"


def test_emergency_reserve_is_allocated_at_construction():
    tracker = MemoryTracker()
    assert tracker._emergency_reserve is not None
    assert len(tracker._emergency_reserve) == 10 * 1024 * 1024


def test_top_object_types_releases_then_restores_reserve():
    tracker = MemoryTracker()
    initial = tracker._emergency_reserve
    assert initial is not None
    tracker.top_object_types(limit=3)
    if tracker._emergency_reserve is not None:
        assert len(tracker._emergency_reserve) == 10 * 1024 * 1024