chore: import upstream snapshot with attribution

python/sglang/test/ci/__init__.py

@@ -0,0 +1 @@
+"""CI utilities for SGLang test infrastructure."""

python/sglang/test/ci/ci_register.py

@@ -0,0 +1,420 @@
+import ast
+import warnings
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import List, Optional, Tuple
+
+__all__ = [
+    "HWBackend",
+    "CIRegistry",
+    "collect_tests",
+    "auto_partition",
+    "register_cpu_ci",
+    "register_cuda_ci",
+    "register_amd_ci",
+    "register_musa_ci",
+    "register_npu_ci",
+    "register_xpu_ci",
+    "register_musa_ci",
+    "register_mlx_ci",
+    "ut_parse_one_file",
+]
+
+# `suite` stays in positional slot 2 for backward compat with existing
+# `register_cpu_ci(5, "base-a-test-cpu")` style positional calls. New fields
+# (`stage`, `runner_config`) are kwarg-only.
+_PARAM_ORDER = ("est_time", "suite", "nightly", "disabled")
+_KWARG_ONLY = ("stage", "runner_config")
+_ALL_PARAMS = _PARAM_ORDER + _KWARG_ONLY
+_UNSET = object()
+
+
+class HWBackend(Enum):
+    CPU = auto()
+    CUDA = auto()
+    AMD = auto()
+    NPU = auto()
+    XPU = auto()
+    MUSA = auto()
+    MLX = auto()
+
+
+@dataclass
+class CIRegistry:
+    backend: HWBackend
+    filename: str
+    est_time: float
+    stage: Optional[str] = None
+    runner_config: Optional[str] = None
+    # Legacy single-string suite; kept for nightly/stress/weekly + AMD/CPU/NPU
+    # suites whose names don't follow `{stage}-test-{runner_config}` shape.
+    suite: Optional[str] = None
+    nightly: bool = False
+    disabled: Optional[str] = None
+
+    @property
+    def effective_suite(self) -> Optional[str]:
+        if self.stage is not None and self.runner_config is not None:
+            return f"{self.stage}-test-{self.runner_config}"
+        return self.suite
+
+
+def register_cpu_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for CPU CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_cuda_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for CUDA CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_amd_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for AMD CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_musa_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for MUSA CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_npu_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for NPU CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_xpu_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for XPU CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_musa_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for MUSA CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+def register_mlx_ci(
+    est_time: float,
+    suite: Optional[str] = None,
+    nightly: bool = False,
+    disabled: Optional[str] = None,
+    *,
+    stage: Optional[str] = None,
+    runner_config: Optional[str] = None,
+):
+    """Marker for MLX CI registration (parsed via AST; runtime no-op)."""
+    return None
+
+
+REGISTER_MAPPING = {
+    "register_cpu_ci": HWBackend.CPU,
+    "register_cuda_ci": HWBackend.CUDA,
+    "register_amd_ci": HWBackend.AMD,
+    "register_musa_ci": HWBackend.MUSA,
+    "register_npu_ci": HWBackend.NPU,
+    "register_xpu_ci": HWBackend.XPU,
+    "register_musa_ci": HWBackend.MUSA,
+    "register_mlx_ci": HWBackend.MLX,
+}
+
+
+class RegistryVisitor(ast.NodeVisitor):
+    def __init__(self, filename: str):
+        self.filename = filename
+        self.registries: list[CIRegistry] = []
+        self.has_main_entry: bool = False
+
+    def _constant_value(self, node: ast.AST) -> object:
+        if isinstance(node, ast.Constant):
+            return node.value
+        return _UNSET
+
+    def _parse_call_args(self, func_call: ast.Call) -> dict:
+        args = {name: _UNSET for name in _ALL_PARAMS}
+        seen = set()
+
+        if any(isinstance(arg, ast.Starred) for arg in func_call.args):
+            raise ValueError(
+                f"{self.filename}: starred arguments are not supported in {func_call.func.id}()"
+            )
+        if len(func_call.args) > len(_PARAM_ORDER):
+            raise ValueError(
+                f"{self.filename}: too many positional arguments in {func_call.func.id}()"
+            )
+
+        for name, arg in zip(_PARAM_ORDER, func_call.args):
+            seen.add(name)
+            args[name] = self._constant_value(arg)
+
+        for kw in func_call.keywords:
+            if kw.arg is None:
+                raise ValueError(
+                    f"{self.filename}: **kwargs are not supported in {func_call.func.id}()"
+                )
+            if kw.arg not in args:
+                raise ValueError(
+                    f"{self.filename}: unknown argument '{kw.arg}' in {func_call.func.id}()"
+                )
+            if kw.arg in seen:
+                raise ValueError(
+                    f"{self.filename}: duplicated argument '{kw.arg}' in {func_call.func.id}()"
+                )
+            seen.add(kw.arg)
+            args[kw.arg] = self._constant_value(kw.value)
+
+        if args["est_time"] is _UNSET:
+            raise ValueError(
+                f"{self.filename}: est_time is a required constant in {func_call.func.id}()"
+            )
+
+        # The only valid (stage, runner_config, suite) shapes are:
+        #   (set,   set,   unset)  -> new-style pair
+        #   (unset, unset, set)    -> legacy single-string
+        # Any other combination is rejected with the actual triple in the error.
+        stage_set = args["stage"] is not _UNSET
+        runner_set = args["runner_config"] is not _UNSET
+        suite_set = args["suite"] is not _UNSET
+        valid_shape = (stage_set and runner_set and not suite_set) or (
+            not stage_set and not runner_set and suite_set
+        )
+        if not valid_shape:
+            raise ValueError(
+                f"{self.filename}: {func_call.func.id}() must specify exactly one of "
+                f"(stage, runner_config) pair or suite; got stage={stage_set}, "
+                f"runner_config={runner_set}, suite={suite_set}"
+            )
+
+        est_time = args["est_time"]
+        if not isinstance(est_time, (int, float)):
+            raise ValueError(
+                f"{self.filename}: est_time must be a number in {func_call.func.id}()"
+            )
+
+        suite = args["suite"] if suite_set else None
+        if suite is not None and not isinstance(suite, str):
+            raise ValueError(
+                f"{self.filename}: suite must be a string in {func_call.func.id}()"
+            )
+
+        stage = args["stage"] if stage_set else None
+        runner_config = args["runner_config"] if runner_set else None
+        for name, value in (("stage", stage), ("runner_config", runner_config)):
+            if value is not None and not isinstance(value, str):
+                raise ValueError(
+                    f"{self.filename}: {name} must be a string in {func_call.func.id}()"
+                )
+
+        nightly_value = args["nightly"]
+        if nightly_value is _UNSET:
+            nightly = False
+        elif isinstance(nightly_value, bool):
+            nightly = nightly_value
+        else:
+            raise ValueError(
+                f"{self.filename}: nightly must be a boolean in {func_call.func.id}()"
+            )
+
+        disabled = args["disabled"] if args["disabled"] is not _UNSET else None
+        if disabled is not None and not isinstance(disabled, str):
+            raise ValueError(
+                f"{self.filename}: disabled must be a string in {func_call.func.id}()"
+            )
+
+        return {
+            "est_time": float(est_time),
+            "stage": stage,
+            "runner_config": runner_config,
+            "suite": suite,
+            "nightly": nightly,
+            "disabled": disabled,
+        }
+
+    def _collect_ci_registry(self, func_call: ast.Call):
+        if not isinstance(func_call.func, ast.Name):
+            return None
+
+        backend = REGISTER_MAPPING.get(func_call.func.id)
+        if backend is None:
+            return None
+
+        parsed = self._parse_call_args(func_call)
+        return CIRegistry(
+            backend=backend,
+            filename=self.filename,
+            **parsed,
+        )
+
+    @staticmethod
+    def _is_main_block_with_call(stmt: ast.If) -> bool:
+        """True iff `stmt` is `if __name__ == "__main__":` with a body that
+        contains at least one call (i.e. actually runs something, not just
+        `pass`). This is what makes `python3 file.py` execute tests."""
+        test = stmt.test
+        if not isinstance(test, ast.Compare):
+            return False
+        if not (isinstance(test.left, ast.Name) and test.left.id == "__name__"):
+            return False
+        if len(test.ops) != 1 or not isinstance(test.ops[0], ast.Eq):
+            return False
+        if len(test.comparators) != 1:
+            return False
+        rhs = test.comparators[0]
+        if not (isinstance(rhs, ast.Constant) and rhs.value == "__main__"):
+            return False
+        for child in ast.walk(ast.Module(body=stmt.body, type_ignores=[])):
+            if isinstance(child, ast.Call):
+                return True
+        return False
+
+    def visit_Module(self, node):
+        for stmt in node.body:
+            if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
+                cr = self._collect_ci_registry(stmt.value)
+                if cr is not None:
+                    self.registries.append(cr)
+            elif isinstance(stmt, ast.If) and self._is_main_block_with_call(stmt):
+                self.has_main_entry = True
+
+        self.generic_visit(node)
+
+
+def ut_parse_one_file(filename: str) -> Tuple[List[CIRegistry], bool]:
+    """Parse a test file and return (registries, has_main_entry).
+
+    `has_main_entry` is True iff the file has `if __name__ == "__main__":`
+    with a call in its body -- required for `python3 file.py` to actually
+    run tests (the CI runner's invocation pattern).
+    """
+    with open(filename, "r") as f:
+        file_content = f.read()
+    tree = ast.parse(file_content, filename=filename)
+    visitor = RegistryVisitor(filename=filename)
+    visitor.visit(tree)
+    return visitor.registries, visitor.has_main_entry
+
+
+def auto_partition(
+    files: List[CIRegistry],
+    rank: int,
+    size: int,
+    live_est: Optional[dict] = None,
+) -> List[CIRegistry]:
+    """Partition files into `size` sublists with approximately equal sums of
+    estimated times using a greedy algorithm (LPT heuristic), and return the
+    partition for the specified rank.
+
+    `live_est`: optional `filename -> est seconds` overrides; missing
+    files fall back to in-source `est_time`.
+    """
+    if not files or size <= 0:
+        return []
+
+    def est_of(f: CIRegistry) -> float:
+        if live_est is not None and f.filename in live_est:
+            return live_est[f.filename]
+        return f.est_time
+
+    # Sort by estimated_time descending; filename as tie-breaker for
+    # deterministic partitioning regardless of glob ordering.
+    sorted_files = sorted(files, key=lambda f: (-est_of(f), f.filename))
+
+    partitions: List[List[CIRegistry]] = [[] for _ in range(size)]
+    partition_sums = [0.0] * size
+
+    # Greedily assign each file to the partition with the smallest current total time
+    for file in sorted_files:
+        min_sum_idx = min(range(size), key=partition_sums.__getitem__)
+        partitions[min_sum_idx].append(file)
+        partition_sums[min_sum_idx] += est_of(file)
+
+    if rank < size:
+        return partitions[rank]
+    return []
+
+
+def collect_tests(files: list[str], sanity_check: bool = True) -> List[CIRegistry]:
+    ci_tests = []
+    for file in files:
+        registries, has_main_entry = ut_parse_one_file(file)
+        if len(registries) == 0:
+            msg = f"No CI registry found in {file}"
+            if sanity_check:
+                raise ValueError(msg)
+            else:
+                warnings.warn(msg)
+                continue
+
+        # Every file with at least one enabled registry must have an
+        # executable `if __name__ == "__main__":` block; otherwise
+        # `python3 file.py -f` (how run_unittest_files invokes tests)
+        # silently exits and the file shows green without running.
+        has_enabled = any(r.disabled is None for r in registries)
+        if sanity_check and has_enabled and not has_main_entry:
+            raise ValueError(
+                f'{file}: missing `if __name__ == "__main__":` entry. '
+                f"Pytest-style tests in this file will silently skip under "
+                f"`python3 file.py -f`. Add `unittest.main()` (for "
+                f"unittest.TestCase) or `sys.exit(pytest.main([__file__, "
+                f'"-v"]))` (for pytest-style).'
+            )
+
+        ci_tests.extend(registries)
+
+    return ci_tests

python/sglang/test/ci/ci_stress_utils.py

@@ -0,0 +1,224 @@
+"""Utilities for running stress tests with bench_serving."""
+
+import subprocess
+from typing import List, Optional
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    is_in_ci,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+
+class StressTestRunner:
+    """Helper class for running stress tests with bench_serving.
+
+    This class encapsulates common patterns for stress testing, including:
+    - Server launch and cleanup
+    - Stress test command construction and execution
+    - Result collection and reporting
+    """
+
+    def __init__(
+        self,
+        test_name: str,
+        base_url: str,
+        num_prompts: int = 50000,
+        duration_minutes: int = 45,
+    ):
+        """Initialize the stress test runner.
+
+        Args:
+            test_name: Name of the test (used for reporting)
+            base_url: Base URL for the server
+            num_prompts: Number of prompts to send (default: 50000)
+            duration_minutes: Timeout in minutes (default: 45)
+        """
+        self.test_name = test_name
+        self.base_url = base_url
+        self.num_prompts = num_prompts
+        self.duration_minutes = duration_minutes
+        self.full_report = f"## {test_name}\n"
+
+    def build_stress_test_command(
+        self,
+        model_path: str,
+        random_input_len: int,
+        random_output_len: int,
+        output_file: str,
+        random_range_ratio: float = 0.2,
+        extra_args: Optional[List[str]] = None,
+    ) -> List[str]:
+        """Build the bench_serving stress test command.
+
+        Args:
+            model_path: Path to the model
+            random_input_len: Random input length
+            random_output_len: Random output length
+            output_file: Output JSONL file path
+            random_range_ratio: Random range ratio (default: 0.2)
+            extra_args: Optional extra arguments
+
+        Returns:
+            List of command arguments ready for subprocess.run()
+        """
+        command = [
+            "python3",
+            "-m",
+            "sglang.benchmark.serving",
+            "--backend",
+            "sglang-oai",
+            "--base-url",
+            self.base_url,
+            "--dataset-name",
+            "random",
+            "--random-input-len",
+            str(random_input_len),
+            "--random-output-len",
+            str(random_output_len),
+            "--random-range-ratio",
+            str(random_range_ratio),
+            "--num-prompts",
+            str(self.num_prompts),
+            "--output-file",
+            output_file,
+        ]
+
+        if extra_args:
+            command.extend(extra_args)
+
+        return command
+
+    def run_stress_test_command(
+        self, command: List[str], timeout_minutes: Optional[int] = None
+    ) -> subprocess.CompletedProcess:
+        """Execute the stress test command with timeout.
+
+        Args:
+            command: Command to execute
+            timeout_minutes: Timeout in minutes (uses class default if None)
+
+        Returns:
+            CompletedProcess result
+        """
+        timeout = (timeout_minutes or self.duration_minutes) * 60
+        print(f"Running stress test command (timeout: {timeout}s):")
+        print(f"  {' '.join(command)}")
+
+        result = subprocess.run(
+            command, capture_output=True, text=True, timeout=timeout
+        )
+
+        if result.returncode != 0:
+            print(f"Error running stress test:")
+            print(result.stderr)
+            raise RuntimeError(
+                f"Stress test failed with return code {result.returncode}"
+            )
+
+        return result
+
+    def run_stress_test_for_model(
+        self,
+        model_path: str,
+        random_input_len: int,
+        random_output_len: int,
+        output_file: str,
+        server_args: Optional[List[str]] = None,
+        extra_bench_args: Optional[List[str]] = None,
+        timeout_minutes: Optional[int] = None,
+    ) -> bool:
+        """Run a complete stress test for a single model with server management.
+
+        This method handles:
+        - Server launch and cleanup
+        - Stress test command construction and execution
+        - Error handling and reporting
+
+        Args:
+            model_path: Path to the model
+            random_input_len: Random input length
+            random_output_len: Random output length
+            output_file: Output JSONL file path
+            server_args: Arguments to pass to server launch
+            extra_bench_args: Extra arguments for bench_serving
+            timeout_minutes: Timeout in minutes (uses class default if None)
+
+        Returns:
+            True if successful, False otherwise
+        """
+        print(f"\n{'='*60}")
+        print(f"Starting stress test for: {model_path}")
+        print(f"Input length: {random_input_len}")
+        print(f"Output length: {random_output_len}")
+        print(f"Num prompts: {self.num_prompts}")
+        print(f"{'='*60}\n")
+
+        # Launch server
+        process = popen_launch_server(
+            model=model_path,
+            base_url=self.base_url,
+            other_args=server_args or [],
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        )
+
+        try:
+            # Build and run stress test command
+            command = self.build_stress_test_command(
+                model_path,
+                random_input_len,
+                random_output_len,
+                output_file,
+                extra_args=extra_bench_args,
+            )
+
+            self.run_stress_test_command(command, timeout_minutes)
+
+            print(f"\nStress test completed successfully for {model_path}")
+            self._add_success_to_report(model_path, random_input_len, random_output_len)
+            return True
+
+        except Exception as e:
+            print(f"\nStress test failed for {model_path}: {e}")
+            self._add_failure_to_report(model_path, str(e))
+            return False
+
+        finally:
+            # Always clean up server process
+            print(f"Cleaning up server process...")
+            kill_process_tree(process.pid)
+
+    def _add_success_to_report(
+        self, model_path: str, input_len: int, output_len: int
+    ) -> None:
+        """Add success entry to report."""
+        model_name = model_path.split("/")[-1]
+        self.full_report += f"### {model_name} - Success\n"
+        self.full_report += f"- Model: `{model_path}`\n"
+        self.full_report += f"- Input Length: {input_len}\n"
+        self.full_report += f"- Output Length: {output_len}\n"
+        self.full_report += f"- Num Prompts: {self.num_prompts}\n"
+        self.full_report += f"- Status: **PASSED**\n\n"
+
+    def _add_failure_to_report(self, model_path: str, error: str) -> None:
+        """Add failure entry to report."""
+        model_name = model_path.split("/")[-1]
+        self.full_report += f"### {model_name} - Failure\n"
+        self.full_report += f"- Model: `{model_path}`\n"
+        self.full_report += f"- Status: **FAILED**\n"
+        self.full_report += f"- Error: {error}\n\n"
+
+    def write_final_report(self) -> None:
+        """Write the final report to GitHub summary if in CI."""
+        if is_in_ci():
+            write_github_step_summary(self.full_report)
+
+    def get_full_report(self) -> str:
+        """Get the accumulated full report.
+
+        Returns:
+            The full markdown report as a string
+        """
+        return self.full_report

python/sglang/test/ci/ci_utils.py

@@ -0,0 +1,346 @@
+import json
+import logging
+import os
+import re
+import subprocess
+import threading
+import time
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Optional, Union
+
+from sglang.srt.debug_utils import cuda_coredump
+from sglang.srt.utils.common import kill_process_tree
+from sglang.test.ci.ci_register import CIRegistry
+
+# Configure logger to output to stdout
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TestFile:
+    name: str
+    estimated_time: float = 60
+
+
+# Patterns that indicate retriable accuracy/performance failures
+RETRIABLE_PATTERNS = [
+    r"AssertionError:.*not greater than",
+    r"AssertionError:.*not less than",
+    r"AssertionError:.*not equal to",
+    r"AssertionError:.*!=.*expected",
+    r"accuracy",
+    r"score",
+    r"latency",
+    r"throughput",
+    r"timeout",
+]
+
+# Patterns that indicate non-retriable failures (real code errors)
+NON_RETRIABLE_PATTERNS = [
+    r"SyntaxError",
+    r"ImportError",
+    r"ModuleNotFoundError",
+    r"NameError",
+    r"TypeError",
+    r"AttributeError",
+    r"RuntimeError",
+    r"CUDA out of memory",
+    r"OOM",
+    r"Segmentation fault",
+    r"core dumped",
+    r"ConnectionRefusedError",
+    r"FileNotFoundError",
+]
+
+
+def is_retriable_failure(output: str) -> tuple[bool, str]:
+    """
+    Determine if a test failure is retriable based on output patterns.
+
+    Returns:
+        tuple: (is_retriable, reason)
+    """
+    # Check for non-retriable patterns first
+    for pattern in NON_RETRIABLE_PATTERNS:
+        if re.search(pattern, output, re.IGNORECASE):
+            return False, f"non-retriable error: {pattern}"
+
+    # Check for retriable patterns
+    for pattern in RETRIABLE_PATTERNS:
+        if re.search(pattern, output, re.IGNORECASE):
+            return True, f"retriable pattern: {pattern}"
+
+    # If we have an AssertionError but didn't match non-retriable, assume retriable
+    if re.search(r"AssertionError", output):
+        return True, "AssertionError (assuming retriable)"
+
+    # Default: not retriable
+    return False, "unknown failure type"
+
+
+def run_with_timeout(
+    func: Callable,
+    args: tuple = (),
+    kwargs: Optional[dict] = None,
+    timeout: float = None,
+):
+    """Run a function with timeout."""
+    ret_value = []
+
+    def _target_func():
+        ret_value.append(func(*args, **(kwargs or {})))
+
+    t = threading.Thread(target=_target_func)
+    t.start()
+    t.join(timeout=timeout)
+    if t.is_alive():
+        raise TimeoutError()
+
+    if not ret_value:
+        raise RuntimeError()
+
+    return ret_value[0]
+
+
+def write_github_step_summary(content: str):
+    """Write content to GitHub Step Summary if available."""
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file:
+        with open(summary_file, "a") as f:
+            f.write(content)
+
+
+def _repo_relative_path(p: str) -> str:
+    """Return path stripped to repo-relative form (e.g. 'test/srt/foo.py').
+
+    Used in the machine-readable TIMINGS block so downstream scrapers
+    get a stable key regardless of CI runner checkout layout.
+    """
+    if not os.path.isabs(p):
+        p = os.path.join(os.getcwd(), p)
+    marker = "/sglang/"
+    idx = p.rfind(marker)
+    return p[idx + len(marker) :] if idx >= 0 else p
+
+
+def run_unittest_files(
+    files: Union[List[TestFile], List[CIRegistry]],
+    timeout_per_file: float,
+    continue_on_error: bool = False,
+    enable_retry: bool = False,
+    max_attempts: int = 2,
+    retry_wait_seconds: int = 60,
+):
+    """
+    Run a list of test files.
+
+    Args:
+        files: List of TestFile objects to run
+        timeout_per_file: Timeout in seconds for each test file
+        continue_on_error: If True, continue running remaining tests even if one fails.
+                          If False, stop at first failure (default behavior for PR tests).
+        enable_retry: If True, retry failed tests that appear to be accuracy/performance
+                     assertion failures (not code errors).
+        max_attempts: Maximum number of attempts per file including initial run (default: 2).
+        retry_wait_seconds: Seconds to wait between retries (default: 60).
+    """
+    coredump_enabled = cuda_coredump.is_enabled()
+    if coredump_enabled:
+        cuda_coredump.cleanup_dump_dir()
+
+    tic = time.perf_counter()
+    success = True
+    passed_tests = []
+    failed_tests = []
+    retried_tests = []  # Track which tests were retried
+    # Per-file elapsed seconds, latest attempt wins. Consumed by the
+    # TIMINGS block emitted at the end of this function.
+    file_elapsed: Dict[str, float] = {}
+
+    for i, file in enumerate(files):
+        if isinstance(file, CIRegistry):
+            filename, estimated_time = file.filename, file.est_time
+        else:
+            # FIXME: remove this branch after migrating all tests to use CIRegistry
+            filename, estimated_time = file.name, file.estimated_time
+
+        process = None
+        output_lines = []
+
+        def run_one_file(filename, capture_output=False):
+            nonlocal process, output_lines
+
+            full_path = os.path.join(os.getcwd(), filename)
+            logger.info(
+                f".\n.\nBegin ({i}/{len(files) - 1}):\npython3 {full_path}\n.\n.\n"
+            )
+            file_tic = time.perf_counter()
+
+            cmd = ["python3", full_path, "-f"]
+
+            if capture_output:
+                # Capture output for retry decision
+                process = subprocess.Popen(
+                    cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    errors="ignore",  # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
+                )
+                output_lines = []
+                for line in process.stdout:
+                    logger.info(line.rstrip())
+                    output_lines.append(line)
+                process.wait()
+            else:
+                process = subprocess.Popen(cmd, stdout=None, stderr=None)
+                process.wait()
+
+            elapsed = time.perf_counter() - file_tic
+            file_elapsed[filename] = elapsed
+
+            logger.info(
+                f".\n.\nEnd ({i}/{len(files) - 1}):\n{filename=}, {elapsed=:.0f}, {estimated_time=}\n.\n.\n"
+            )
+            return process.returncode
+
+        # Retry loop for each file
+        attempt = 1
+        file_passed = False
+        was_retried = False
+
+        while attempt <= (max_attempts if enable_retry else 1):
+            if attempt > 1:
+                logger.info(
+                    f"\n[CI Retry] Attempt {attempt}/{max_attempts} for {filename}\n"
+                )
+                was_retried = True
+
+            try:
+                ret_code = run_with_timeout(
+                    run_one_file,
+                    args=(filename,),
+                    kwargs={"capture_output": enable_retry},
+                    timeout=timeout_per_file,
+                )
+
+                if ret_code == 0:
+                    file_passed = True
+                    if was_retried:
+                        logger.info(
+                            f"\n✓ PASSED on retry (attempt {attempt}): {filename}\n"
+                        )
+                        retried_tests.append((filename, attempt, "passed"))
+                    passed_tests.append(filename)
+                    break
+                else:
+                    # Check if we should retry
+                    if enable_retry and attempt < max_attempts:
+                        output = "".join(output_lines)
+                        is_retriable, reason = is_retriable_failure(output)
+
+                        if is_retriable:
+                            logger.info(f"\n[CI Retry] {filename} failed with {reason}")
+                            logger.info(
+                                f"[CI Retry] Waiting {retry_wait_seconds}s before retry...\n"
+                            )
+                            time.sleep(retry_wait_seconds)
+                            attempt += 1
+                            continue
+                        else:
+                            logger.info(
+                                f"\n[CI Retry] {filename} failed with {reason} - not retrying\n"
+                            )
+
+                    # No retry or not retriable
+                    logger.info(
+                        f"\n✗ FAILED: {filename} returned exit code {ret_code}\n"
+                    )
+                    if was_retried:
+                        retried_tests.append((filename, attempt, "failed"))
+                    failed_tests.append((filename, f"exit code {ret_code}"))
+                    break
+
+            except TimeoutError:
+                kill_process_tree(process.pid)
+                time.sleep(5)
+                # TimeoutError aborts run_one_file before its elapsed write;
+                # record the timeout cap as an upper bound so the file still
+                # appears in the TIMINGS block below.
+                file_elapsed[filename] = float(timeout_per_file)
+                logger.info(
+                    f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n"
+                )
+                if was_retried:
+                    retried_tests.append((filename, attempt, "timeout"))
+                failed_tests.append((filename, f"timeout after {timeout_per_file}s"))
+                break
+
+        if not file_passed:
+            success = False
+            if not continue_on_error:
+                break
+
+    elapsed_total = time.perf_counter() - tic
+
+    if coredump_enabled and not success:
+        cuda_coredump.report()
+
+    if success:
+        logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
+    else:
+        logger.info(f"Fail. Time elapsed: {elapsed_total:.2f}s")
+
+    # Print summary
+    logger.info(f"\n{'='*60}")
+    logger.info(f"Test Summary: {len(passed_tests)}/{len(files)} passed")
+    if enable_retry and retried_tests:
+        logger.info(f"Retries: {len(retried_tests)} test(s) were retried")
+    logger.info(f"{'='*60}")
+    if passed_tests:
+        logger.info("✓ PASSED:")
+        for test in passed_tests:
+            logger.info(f"  {test}")
+    if failed_tests:
+        logger.info("\n✗ FAILED:")
+        for test, reason in failed_tests:
+            logger.info(f"  {test} ({reason})")
+    if retried_tests:
+        logger.info("\n↻ RETRIED:")
+        for test, attempts, result in retried_tests:
+            logger.info(f"  {test} ({attempts} attempts, {result})")
+    logger.info(f"{'='*60}\n")
+
+    # Machine-readable timings block for downstream scrapers/dashboards.
+    # One JSON object per executed file (post-retry: only the latest
+    # attempt's elapsed is recorded). Files skipped via fail-fast
+    # (continue_on_error=False) are omitted. Job wall-clock is read
+    # separately from the GitHub Actions API by consumers, so we don't
+    # emit any aggregate fields here.
+    passed_set = set(passed_tests)
+    logger.info("========== TIMINGS BEGIN ==========")
+    for fname, elapsed in file_elapsed.items():
+        logger.info(
+            json.dumps(
+                {
+                    "file": _repo_relative_path(fname),
+                    "passed": fname in passed_set,
+                    "elapsed": round(elapsed),
+                }
+            )
+        )
+    logger.info("========== TIMINGS END ==========")
+
+    # Write GitHub Step Summary only if retries occurred
+    if retried_tests:
+        passed_on_retry = [t for t, _, r in retried_tests if r == "passed"]
+        failed_after_retry = [t for t, _, r in retried_tests if r != "passed"]
+        summary = f"**↻ Retried {len(retried_tests)} test(s):**\n"
+        if passed_on_retry:
+            summary += f"- ✓ Passed on retry: {', '.join(passed_on_retry)}\n"
+        if failed_after_retry:
+            summary += f"- ✗ Still failed: {', '.join(failed_after_retry)}\n"
+        write_github_step_summary(summary)
+
+    return 0 if success else -1