ray-project--ray/python/ray/train/v2/tests/test_result.py

import stat
import uuid
from pathlib import Path
from urllib.parse import urlparse, urlunparse

import pytest

import ray
import ray.train.collective
from ray import train
from ray.train import Checkpoint, CheckpointConfig, RunConfig, ScalingConfig
from ray.train.tests.util import create_dict_checkpoint, load_dict_checkpoint
from ray.train.torch import TorchTrainer
from ray.train.v2._internal.constants import CHECKPOINT_MANAGER_SNAPSHOT_FILENAME
from ray.train.v2._internal.execution.storage import StorageContext
from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer
from ray.train.v2.api.exceptions import WorkerGroupError
from ray.train.v2.api.result import Result


def uri_join(base_uri: str, *paths: str) -> str:
    """
    Join a base URI (local or remote) with one or more subpaths.
    Preserves query parameters and scheme.
    """
    parsed = urlparse(base_uri)
    new_path = "/".join([p.strip("/") for p in [parsed.path, *paths] if p])

    # If it's a local path (no scheme), ensure we preserve the leading /
    if not parsed.scheme and not new_path.startswith("/"):
        new_path = "/" + new_path

    return urlunparse(
        (
            parsed.scheme,
            parsed.netloc,
            new_path,
            parsed.params,
            parsed.query,
            parsed.fragment,
        )
    )


def build_dummy_trainer(
    exp_name: str,
    storage_path: str,
    num_iterations: int,
    num_checkpoints: int,
    train_loop_config: dict,
):
    """Build a dummy TorchTrainer for testing purposes."""

    def worker_loop(_config):
        for i in range(num_iterations):
            # Do some random reports in between checkpoints.
            train.report({"metric_a": -100, "metric_b": -100})

            if ray.train.get_context().get_world_rank() == 0:
                with create_dict_checkpoint({"iter": i}) as checkpoint:
                    train.report(
                        metrics={"metric_a": i, "metric_b": -i},
                        checkpoint=checkpoint,
                    )
            else:
                train.report(metrics={"metric_a": i, "metric_b": -i})

        # Ensure that all checkpoints are saved before the RuntimeError
        ray.train.collective.barrier()
        raise RuntimeError()

    trainer = TorchTrainer(
        train_loop_per_worker=worker_loop,
        train_loop_config=train_loop_config,
        scaling_config=ScalingConfig(num_workers=2, use_gpu=False),
        run_config=RunConfig(
            name=exp_name,
            storage_path=storage_path,
            checkpoint_config=CheckpointConfig(
                num_to_keep=num_checkpoints,
                checkpoint_score_attribute="metric_a",
                checkpoint_score_order="max",
            ),
        ),
    )
    return trainer


def test_result_repr():
    """Test that the Result __repr__ function can return a string."""
    res = Result(
        metrics={"iter": 0, "metric": 1.0},
        checkpoint=Checkpoint("/bucket/path/ckpt0"),
        error=None,
        path="/bucket/path",
    )
    assert isinstance(repr(res), str)
    assert "Checkpoint(filesystem=local, path=/bucket/path/ckpt0)" in repr(res)
    assert "metrics={'iter': 0, 'metric': 1.0}" in repr(res)


def test_get_best_checkpoint():
    res = Result(
        metrics={},
        checkpoint=None,
        error=None,
        path="/bucket/path",
        best_checkpoints=[
            (Checkpoint("/bucket/path/ckpt0"), {"iter": 0, "metric": 1.0}),
            (Checkpoint("/bucket/path/ckpt1"), {"iter": 1, "metric": 2.0}),
            (Checkpoint("/bucket/path/ckpt2"), {"iter": 2, "metric": 3.0}),
            (Checkpoint("/bucket/path/ckpt3"), {"iter": 3, "metric": 4.0}),
        ],
    )
    assert (
        res.get_best_checkpoint(metric="metric", mode="max").path
        == "/bucket/path/ckpt3"
    )
    assert (
        res.get_best_checkpoint(metric="metric", mode="min").path
        == "/bucket/path/ckpt0"
    )


def test_get_best_checkpoint_nested_metrics():
    """Test that get_best_checkpoint works with nested metric dictionaries."""
    # Test with nested metric structure
    res = Result(
        metrics={},
        checkpoint=None,
        error=None,
        path="/bucket/path",
        best_checkpoints=[
            (
                Checkpoint("/bucket/path/ckpt0"),
                {
                    "iter": 0,
                    "env_runners": {"episode_return_mean": 100.0, "num_episodes": 10},
                },
            ),
            (
                Checkpoint("/bucket/path/ckpt1"),
                {
                    "iter": 1,
                    "env_runners": {"episode_return_mean": 200.0, "num_episodes": 10},
                },
            ),
            (
                Checkpoint("/bucket/path/ckpt2"),
                {
                    "iter": 2,
                    "env_runners": {"episode_return_mean": 300.0, "num_episodes": 10},
                },
            ),
            (
                Checkpoint("/bucket/path/ckpt3"),
                {
                    "iter": 3,
                    "env_runners": {"episode_return_mean": 400.0, "num_episodes": 10},
                },
            ),
        ],
    )

    # Test max mode with nested metric
    assert (
        res.get_best_checkpoint(
            metric="env_runners/episode_return_mean", mode="max"
        ).path
        == "/bucket/path/ckpt3"
    )

    # Test min mode with nested metric
    assert (
        res.get_best_checkpoint(
            metric="env_runners/episode_return_mean", mode="min"
        ).path
        == "/bucket/path/ckpt0"
    )

    # Test that flat keys still work (backwards compatibility)
    res_flat = Result(
        metrics={},
        checkpoint=None,
        error=None,
        path="/bucket/path",
        best_checkpoints=[
            (
                Checkpoint("/bucket/path/ckpt0"),
                {"iter": 0, "env_runners/episode_return_mean": 100.0},
            ),
            (
                Checkpoint("/bucket/path/ckpt1"),
                {"iter": 1, "env_runners/episode_return_mean": 200.0},
            ),
        ],
    )
    assert (
        res_flat.get_best_checkpoint(
            metric="env_runners/episode_return_mean", mode="max"
        ).path
        == "/bucket/path/ckpt1"
    )


@pytest.mark.parametrize("path_type", ["str", "PathLike"])
@pytest.mark.parametrize("pass_storage_filesystem", [True, False])
@pytest.mark.parametrize("trailing_slash", [False, True])
def test_result_restore(
    ray_start_4_cpus,
    tmp_path,
    path_type,
    pass_storage_filesystem,
    trailing_slash,
):
    """Test Result.from_path functionality similar to v1 test_result_restore."""

    num_iterations = 3
    num_checkpoints = 2

    storage_path = str(tmp_path)

    # Add UUID to ensure test isolation when sharing module-scoped S3 mock
    exp_name = f"test_result_restore_v2-{uuid.uuid4().hex[:8]}"

    trainer = build_dummy_trainer(
        exp_name,
        storage_path,
        num_iterations,
        num_checkpoints,
        train_loop_config={"a": 1, "b": 2},
    )
    with pytest.raises(WorkerGroupError):
        trainer.fit()

    if pass_storage_filesystem:
        storage_context = StorageContext(
            storage_path=storage_path,
            experiment_dir_name=exp_name,
        )

        trial_dir = storage_context.experiment_fs_path
        file_system = storage_context.storage_filesystem
    else:
        trial_dir = uri_join(storage_path, exp_name)
        file_system = None

    # Add trailing slash if parameterized to test that case
    if trailing_slash:
        trial_dir = trial_dir + "/"

    # For PathLike test, only use Path() for local paths, not URIs
    if path_type == "PathLike":
        trial_dir_arg = Path(trial_dir)
    else:
        trial_dir_arg = trial_dir

    result = Result.from_path(
        trial_dir_arg,
        storage_filesystem=file_system,
    )

    assert result.checkpoint
    assert len(result.best_checkpoints) == num_checkpoints

    """
    Top-2 checkpoints with metrics:

                        | iter   | metric_a    metric_b
    checkpoint_000002        2            2          -2
    checkpoint_000001        1            1          -1
    """
    # Check if the checkpoints bounded with correct metrics
    best_ckpt_a = result.get_best_checkpoint(metric="metric_a", mode="max")
    assert load_dict_checkpoint(best_ckpt_a)["iter"] == num_iterations - 1

    best_ckpt_b = result.get_best_checkpoint(metric="metric_b", mode="max")
    assert load_dict_checkpoint(best_ckpt_b)["iter"] == num_iterations - num_checkpoints

    with pytest.raises(RuntimeError, match="Invalid metric name.*"):
        result.get_best_checkpoint(metric="invalid_metric", mode="max")


def test_result_from_path_read_only_storage(
    ray_start_4_cpus,
    tmp_path,
):
    """Reproduces https://github.com/ray-project/ray/issues/64305.

    If checkpoints live on write-once / read-only storage, check that
    ``Result.from_path`` / ``get_best_checkpoint`` are read-only operations.
    """

    def train_fn():
        with create_dict_checkpoint({"iter": 1}) as ckpt:
            ray.train.report({"epoch": 1}, ckpt)
        with create_dict_checkpoint({"iter": 2}) as ckpt:
            ray.train.report({"epoch": 2}, ckpt)

    trainer = DataParallelTrainer(
        train_fn,
        run_config=RunConfig(
            name="test_read_only_storage",
            storage_path=str(tmp_path),
        ),
    )
    trainer.fit()

    # Strip write permissions from the experiment directory, so that any new write fails.
    no_write_mask = ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
    experiment_dir = Path(tmp_path) / "test_read_only_storage"
    assert experiment_dir.exists()
    original_modes = {
        path: stat.S_IMODE(path.stat().st_mode)
        for path in [experiment_dir, *experiment_dir.rglob("*")]
    }
    for path in original_modes:
        path.chmod(original_modes[path] & no_write_mask)

    try:
        # Reading the best checkpoint must not require write access.
        result = Result.from_path(str(experiment_dir))
        assert result.checkpoint
        assert len(result.best_checkpoints) == 2

        best_ckpt = result.get_best_checkpoint(metric="epoch", mode="max")
        assert load_dict_checkpoint(best_ckpt)["iter"] == 2
    finally:
        for path, mode in original_modes.items():
            path.chmod(mode)


def test_result_from_path_validation(
    ray_start_4_cpus,
    tmp_path,
):
    """Test that Result.from_path raises RuntimeError when folder or snapshot file doesn't exist."""

    nonexistent_folder = str(tmp_path / "nonexistent_experiment")
    existing_folder = str(tmp_path / "existing_experiment")

    # Test 1: Folder doesn't exist
    with pytest.raises(RuntimeError, match="Experiment folder .* doesn't exist."):
        Result.from_path(nonexistent_folder)

    # Test 2: Folder exists but snapshot file doesn't exist
    Path(existing_folder).mkdir(parents=True, exist_ok=True)

    with pytest.raises(
        RuntimeError,
        match=f"Failed to restore the Result object: {CHECKPOINT_MANAGER_SNAPSHOT_FILENAME} doesn't exist in the experiment folder. Make sure that this is an output directory created "
        "by a Ray Train run.",
    ):
        Result.from_path(existing_folder)


if __name__ == "__main__":
    import sys

    sys.exit(pytest.main(["-v", "-x", __file__]))