sgl-project--sglang/python/sglang/test/kv_canary/violation_assert_mixin.py

from __future__ import annotations

import time
from typing import Literal, Optional

from sglang.srt.kv_canary.perturb.config import TargetGroupKind
from sglang.test.kv_canary.violation_log_utils import (
    assert_no_violation_in_log,
    find_violation_in_log,
)

_Side = Optional[Literal["prefill", "decode"]]


class CanaryViolationAssertMixin:
    def _captured_log_text(self, side: _Side = None) -> str:
        raise NotImplementedError

    def assert_per_forward_violation_reported(
        self,
        *,
        fail_reason: str,
        target_group: Optional[TargetGroupKind] = None,
        side: _Side = None,
        flush_wait_seconds: float = 2.0,
    ) -> None:
        suffix = "" if target_group is None else f"_{target_group.name}"
        self.assert_violation_logged_any(
            launch_tag_patterns=(f"HEAD_*{suffix}", f"TAIL_*{suffix}"),
            fail_reason=fail_reason,
            side=side,
            flush_wait_seconds=flush_wait_seconds,
        )

    def assert_sweep_violation_reported(
        self,
        *,
        fail_reason: str,
        target_group: TargetGroupKind,
        side: _Side = None,
        flush_wait_seconds: float = 2.0,
        max_retries: int = 4,
    ) -> None:
        self.assert_violation_logged_any(
            launch_tag_patterns=(f"SWEEP_*_{target_group.name}",),
            fail_reason=fail_reason,
            side=side,
            flush_wait_seconds=flush_wait_seconds,
            max_retries=max_retries,
        )

    def assert_any_launch_tag_violation_reported(
        self,
        *,
        fail_reason: str,
        side: _Side = None,
        flush_wait_seconds: float = 3.0,
        max_retries: int = 10,
    ) -> None:
        self.assert_violation_logged_any(
            launch_tag_patterns=("*",),
            fail_reason=fail_reason,
            side=side,
            flush_wait_seconds=flush_wait_seconds,
            max_retries=max_retries,
        )

    def assert_any_launch_tag_violation_absent(
        self, *, fail_reason: str, side: _Side = None
    ) -> None:
        self.assert_no_violation_matching(
            launch_tag_patterns=("*",), fail_reason=fail_reason, side=side
        )

    def assert_violation_logged_any(
        self,
        *,
        launch_tag_patterns: tuple[str, ...],
        fail_reason: str,
        side: _Side = None,
        flush_wait_seconds: float = 2.0,
        max_retries: int = 1,
    ) -> None:
        log_text = ""
        for _ in range(max_retries):
            time.sleep(flush_wait_seconds)
            log_text = self._captured_log_text(side)
            if find_violation_in_log(
                log_text,
                launch_tag_patterns=launch_tag_patterns,
                fail_reason=fail_reason,
            ):
                return
        side_label = "" if side is None else f" on side={side}"
        other_side_diag = ""
        if side in ("prefill", "decode"):
            other = "decode" if side == "prefill" else "prefill"
            try:
                other_text = self._captured_log_text(other)
                other_match = find_violation_in_log(
                    other_text,
                    launch_tag_patterns=launch_tag_patterns,
                    fail_reason=fail_reason,
                )
                other_side_diag = (
                    f"\n[diag] other side ({other}) buf len={len(other_text)} "
                    f"contains_match={other_match}"
                )
            except (NotImplementedError, ValueError):
                pass
        raise AssertionError(
            f"No canary violation matching launch_tag_patterns={launch_tag_patterns!r} "
            f"fail_reason={fail_reason!r}{side_label} after max_retries={max_retries} "
            f"(wait={flush_wait_seconds}s each). "
            f"log_text len={len(log_text)}.{other_side_diag} Log tail:\n"
            f"{log_text[-2000:]}"
        )

    def assert_no_violation_matching(
        self,
        *,
        launch_tag_patterns: tuple[str, ...],
        fail_reason: str,
        side: _Side = None,
    ) -> None:
        log_text = self._captured_log_text(side)
        if find_violation_in_log(
            log_text,
            launch_tag_patterns=launch_tag_patterns,
            fail_reason=fail_reason,
        ):
            raise AssertionError(
                f"Unexpected canary violation matching "
                f"launch_tag_patterns={launch_tag_patterns!r} "
                f"fail_reason={fail_reason!r}. Log tail:\n{log_text[-2000:]}"
            )

    def assert_no_violation(
        self,
        *,
        side: _Side = None,
        wait_seconds: float = 2.0,
    ) -> None:
        time.sleep(wait_seconds)
        assert_no_violation_in_log(self._captured_log_text(side))