chore: import upstream snapshot with attribution

release/ray_release/tests/_test_catch_args.py

@@ -0,0 +1,14 @@
+import json
+import sys
+
+
+def main():
+    argv_file = sys.argv[1]
+    with open(argv_file, "wt") as fp:
+        json.dump(sys.argv, fp)
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

release/ray_release/tests/_test_run_release_test_sh.py

@@ -0,0 +1,43 @@
+import os
+import sys
+
+import click
+
+
+@click.command()
+@click.argument("state_file", type=str)
+@click.argument("exit_1", type=int)
+@click.argument("exit_2", type=int)
+@click.argument("exit_3", type=int)
+def main(
+    state_file: str,
+    exit_1: int,
+    exit_2: int,
+    exit_3: int,
+):
+    if not os.path.exists(state_file):
+        state = 0
+    else:
+        with open(state_file, "rt") as fp:
+            state = int(fp.read())
+
+    state += 1
+
+    with open(state_file, "wt") as fp:
+        fp.write(str(state))
+
+    if state == 1:
+        print(f"Exiting with status: {exit_1}")
+        sys.exit(exit_1)
+
+    if state == 2:
+        print(f"Exiting with status: {exit_2}")
+        sys.exit(exit_2)
+
+    if state == 3:
+        print(f"Exiting with status: {exit_3}")
+        sys.exit(exit_3)
+
+
+if __name__ == "__main__":
+    main()

release/ray_release/tests/sample_5_tests.yaml

@@ -0,0 +1,78 @@
+- name: hello_world
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod: {}
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws
+
+- name: hello_world_custom
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod:
+      type: gpu
+      post_build_script: byod_hello_world.sh
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws
+
+- name: hello_world_2
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod: {}
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws
+
+- name: hello_world_custom_2
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod:
+      type: gpu
+      post_build_script: byod_hello_world.sh
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws
+
+- name: hello_world_3
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod: {}
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws

release/ray_release/tests/sample_tests.yaml

@@ -0,0 +1,31 @@
+- name: hello_world
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod: {}
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws
+
+- name: hello_world_custom
+  team: reef
+  group: hello_world
+  frequency: nightly
+  working_dir: hello_world_tests
+  cluster:
+    anyscale_sdk_2026: true
+    byod:
+      type: gpu
+      post_build_script: byod_hello_world.sh
+    cluster_compute: hello_world_compute_config.yaml
+  run:
+    timeout: 1800
+    script: python hello_world.py
+  variations:
+    - __suffix__: aws

release/ray_release/tests/test_alerts.py

@@ -0,0 +1,50 @@
+import sys
+
+import pytest
+
+from ray_release.alerts import (
+    default,
+    # long_running_tests,
+    # rllib_tests,
+    # tune_tests,
+    # xgboost_tests,
+    handle,
+)
+from ray_release.exception import ReleaseTestConfigError, ResultsAlert
+from ray_release.result import (
+    Result,
+    ResultStatus,
+)
+from ray_release.test import Test
+
+
+def test_handle_alert():
+    # Unknown test suite
+    with pytest.raises(ReleaseTestConfigError):
+        handle.handle_result(
+            Test(name="unit_alert_test", alert="invalid"),
+            Result(status=ResultStatus.SUCCESS.value),
+        )
+
+    # Alert raised
+    with pytest.raises(ResultsAlert):
+        handle.handle_result(
+            Test(name="unit_alert_test", alert="default"),
+            Result(status="unsuccessful"),
+        )
+
+    # Everything fine
+    handle.handle_result(
+        Test(name="unit_alert_test", alert="default"),
+        Result(status=ResultStatus.SUCCESS.value),
+    )
+
+
+def test_default_alert():
+    test = Test(name="unit_alert_test", alert="default")
+    assert default.handle_result(test, Result(status="timeout"))
+    assert not default.handle_result(test, Result(status=ResultStatus.SUCCESS.value))
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_anyscale_job_manager.py

@@ -0,0 +1,76 @@
+import sys
+
+import pytest
+from anyscale.job.models import JobState
+
+from ray_release.anyscale_util import Anyscale
+from ray_release.cluster_manager.cluster_manager import ClusterManager
+from ray_release.job_manager.anyscale_job_manager import AnyscaleJobManager
+from ray_release.test import Test
+
+
+class FakeJobStatus:
+    def __init__(self, state: JobState):
+        self.state = state
+
+
+class FakeCreateJobResponse:
+    class Result:
+        id = "job_id"
+
+    result = Result()
+
+
+class FakeSDK(Anyscale):
+    def __init__(self):
+        super().__init__()
+        self.created_job = None
+
+    def project_name_by_id(self, project_id: str) -> str:
+        return "fake_project_name"
+
+    def create_job(self, job_request):
+        self.created_job = job_request
+        return FakeCreateJobResponse()
+
+
+def test_get_last_logs_long_running_job():
+    """Test calling get_last_logs() on long-running jobs.
+
+    When the job is running longer than 4 hours, get_last_logs() should skip
+    downloading the logs and return None.
+    """
+    fake_test = Test(name="fake_test")
+    fake_sdk = FakeSDK()
+    cluster_manager = ClusterManager(
+        test=fake_test, project_id="fake_project_id", sdk=fake_sdk
+    )
+    anyscale_job_manager = AnyscaleJobManager(cluster_manager=cluster_manager)
+    anyscale_job_manager._duration = 4 * 3_600 + 1
+    anyscale_job_manager._job_id = "foo"
+    anyscale_job_manager.save_last_job_status(FakeJobStatus(state=JobState.SUCCEEDED))
+    assert anyscale_job_manager.get_last_logs() is None
+
+
+def test_run_job_exports_cluster_compute_name():
+    fake_test = Test(name="fake_test")
+    fake_sdk = FakeSDK()
+    cluster_manager = ClusterManager(
+        test=fake_test, project_id="fake_project_id", sdk=fake_sdk
+    )
+    cluster_manager.cluster_env_name = "cluster_env"
+    cluster_manager.cluster_env_build_id = "build_id"
+    cluster_manager.cluster_compute_id = "compute_id"
+    cluster_manager.cluster_compute_name = "compute_name"
+
+    anyscale_job_manager = AnyscaleJobManager(cluster_manager=cluster_manager)
+    anyscale_job_manager._run_job("echo hi", {"USER_ENV": "1"})
+
+    env_vars = fake_sdk.created_job.config.runtime_env["env_vars"]
+    assert env_vars["USER_ENV"] == "1"
+    assert env_vars["ANYSCALE_JOB_CLUSTER_ENV_NAME"] == "cluster_env"
+    assert env_vars["ANYSCALE_JOB_CLUSTER_COMPUTE_NAME"] == "compute_name"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_anyscale_job_runner.py

@@ -0,0 +1,173 @@
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from ray_release.command_runner.anyscale_job_runner import (
+    TIMEOUT_RETURN_CODE,
+    AnyscaleJobRunner,
+)
+from ray_release.exception import (
+    JobBrokenError,
+    JobNoLogsError,
+    JobOutOfRetriesError,
+    PrepareCommandError,
+    PrepareCommandTimeout,
+    TestCommandError,
+    TestCommandTimeout,
+)
+from ray_release.job_manager.anyscale_job_manager import (
+    JOB_FAILED,
+    JOB_SOFT_INFRA_ERROR,
+    JOB_STATE_UNKNOWN,
+    JOB_SUCCEEDED,
+)
+
+
+def _make_output_json(
+    return_code=0,
+    workload_time_taken=10.0,
+    prepare_return_codes=None,
+    last_prepare_time_taken=5.0,
+):
+    return {
+        "return_code": return_code,
+        "workload_time_taken": workload_time_taken,
+        "prepare_return_codes": prepare_return_codes or [],
+        "last_prepare_time_taken": last_prepare_time_taken,
+        "uploaded_results": True,
+        "uploaded_metrics": True,
+        "uploaded_artifact": True,
+    }
+
+
+@pytest.fixture
+def runner():
+    with patch.object(AnyscaleJobRunner, "__init__", lambda self: None):
+        r = AnyscaleJobRunner()
+        r._results_uploaded = True
+        r._metrics_uploaded = True
+        r._artifact_uploaded = True
+        r.prepare_commands = ["echo prepare"]
+        return r
+
+
+class TestHandleCommandOutputJobReturnCodes:
+    def test_succeeded_with_output(self, runner):
+        output = _make_output_json(return_code=0)
+        runner.fetch_output = MagicMock(return_value=output)
+        runner._handle_command_output(JOB_SUCCEEDED)
+
+    def test_failed_with_output(self, runner):
+        output = _make_output_json(return_code=1)
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(TestCommandError, match="1"):
+            runner._handle_command_output(JOB_FAILED)
+
+    def test_failed_without_output(self, runner):
+        runner.fetch_output = MagicMock(side_effect=Exception("S3 error"))
+        runner.get_last_logs = MagicMock(return_value=None)
+        with pytest.raises(JobNoLogsError):
+            runner._handle_command_output(JOB_FAILED)
+
+    def test_soft_infra_error_raises(self, runner):
+        with pytest.raises(JobOutOfRetriesError, match="FAILED"):
+            runner._handle_command_output(JOB_SOFT_INFRA_ERROR)
+
+    def test_state_unknown_raises(self, runner):
+        with pytest.raises(JobBrokenError, match="UNKNOWN"):
+            runner._handle_command_output(JOB_STATE_UNKNOWN)
+
+
+class TestHandleCommandOutputFetchFailures:
+    def test_no_output_and_no_logs_raises(self, runner):
+        runner.fetch_output = MagicMock(side_effect=Exception("S3 error"))
+        runner.get_last_logs = MagicMock(return_value=None)
+        with pytest.raises(JobNoLogsError):
+            runner._handle_command_output(0)
+
+    def test_no_output_but_logs_parsed(self, runner):
+        output = _make_output_json(return_code=0)
+        log_line = f"### JSON |{json.dumps(output)}| ###"
+        runner.fetch_output = MagicMock(side_effect=Exception("S3 error"))
+        runner.get_last_logs = MagicMock(return_value=log_line)
+        # Should succeed without raising
+        runner._handle_command_output(0)
+
+    def test_no_output_logs_with_nonzero_workload_status(self, runner):
+        output = _make_output_json(return_code=1)
+        log_line = f"### JSON |{json.dumps(output)}| ###"
+        runner.fetch_output = MagicMock(side_effect=Exception("S3 error"))
+        runner.get_last_logs = MagicMock(return_value=log_line)
+        with pytest.raises(TestCommandError, match="1"):
+            runner._handle_command_output(0)
+
+
+class TestHandleCommandOutputPrepareCommands:
+    def test_prepare_timeout_raises(self, runner):
+        output = _make_output_json(
+            prepare_return_codes=[TIMEOUT_RETURN_CODE],
+            last_prepare_time_taken=60.0,
+        )
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(PrepareCommandTimeout, match="60"):
+            runner._handle_command_output(0)
+
+    def test_prepare_error_raises(self, runner):
+        output = _make_output_json(prepare_return_codes=[1])
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(PrepareCommandError, match="echo prepare"):
+            runner._handle_command_output(0)
+
+    def test_prepare_success_continues(self, runner):
+        output = _make_output_json(prepare_return_codes=[0])
+        runner.fetch_output = MagicMock(return_value=output)
+        # Should succeed without raising
+        runner._handle_command_output(0)
+
+
+class TestHandleCommandOutputWorkloadStatus:
+    def test_success(self, runner):
+        output = _make_output_json(return_code=0)
+        runner.fetch_output = MagicMock(return_value=output)
+        # Should return without raising
+        runner._handle_command_output(0)
+
+    def test_nonzero_raises(self, runner):
+        output = _make_output_json(return_code=42)
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(TestCommandError, match="42"):
+            runner._handle_command_output(0)
+
+    def test_none_return_code_raises(self, runner):
+        output = _make_output_json(return_code=None)
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(TestCommandError, match="None"):
+            runner._handle_command_output(0)
+
+    def test_timeout_raises_by_default(self, runner):
+        output = _make_output_json(
+            return_code=TIMEOUT_RETURN_CODE, workload_time_taken=300.0
+        )
+        runner.fetch_output = MagicMock(return_value=output)
+        with pytest.raises(TestCommandTimeout, match="300"):
+            runner._handle_command_output(0)
+
+    def test_timeout_suppressed_when_not_raising(self, runner):
+        output = _make_output_json(return_code=TIMEOUT_RETURN_CODE)
+        runner.fetch_output = MagicMock(return_value=output)
+        # Should return without raising
+        runner._handle_command_output(0, raise_on_timeout=False)
+
+
+class TestHandleCommandOutputSideEffects:
+    def test_upload_flags_set_from_output(self, runner):
+        output = _make_output_json()
+        output["uploaded_results"] = False
+        output["uploaded_metrics"] = False
+        output["uploaded_artifact"] = False
+        runner.fetch_output = MagicMock(return_value=output)
+        runner._handle_command_output(0)
+        assert runner._results_uploaded is False
+        assert runner._metrics_uploaded is False
+        assert runner._artifact_uploaded is False

release/ray_release/tests/test_anyscale_job_wrapper.py

@@ -0,0 +1,240 @@
+import json
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+
+from ray_release.command_runner._anyscale_job_wrapper import (
+    OUTPUT_JSON_FILENAME,
+    TIMEOUT_RETURN_CODE,
+    main,
+    run_bash_command,
+    run_spilling_check,
+)
+
+cloud_storage_kwargs = dict(
+    results_cloud_storage_uri=None,
+    metrics_cloud_storage_uri=None,
+    output_cloud_storage_uri=None,
+    upload_cloud_storage_uri=None,
+    artifact_path=None,
+)
+
+
+def test_run_bash_command_success():
+    assert run_bash_command("exit 0", 1000) == 0
+
+
+def test_run_bash_command_fail():
+    assert run_bash_command("exit 1", 1000) == 1
+
+
+def test_run_bash_command_timeout():
+    assert run_bash_command("sleep 10", 1) == TIMEOUT_RETURN_CODE
+
+
+def _check_output_json(expected_return_code, prepare_return_codes=None):
+    with open(OUTPUT_JSON_FILENAME, "r") as fp:
+        output = json.load(fp)
+    assert output["return_code"] == expected_return_code
+    assert output["prepare_return_codes"] == (prepare_return_codes or [])
+    assert output["uploaded_results"] is False
+    assert output["collected_metrics"] is False
+    assert output["uploaded_metrics"] is False
+
+
+def test_prepare_commands_validation(tmpdir):
+    with pytest.raises(ValueError):
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=["exit 0"],
+            prepare_commands_timeouts=[],
+            **cloud_storage_kwargs
+        )
+    with pytest.raises(ValueError):
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=[],
+            prepare_commands_timeouts=[1],
+            **cloud_storage_kwargs
+        )
+
+
+def test_end_to_end(tmpdir):
+    expected_return_code = 0
+    assert (
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=[],
+            prepare_commands_timeouts=[],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(expected_return_code)
+
+
+def test_end_to_end_prepare_commands(tmpdir):
+    expected_return_code = 0
+    assert (
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=["exit 0", "exit 0"],
+            prepare_commands_timeouts=[1, 1],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(expected_return_code, [0, 0])
+
+
+def test_end_to_end_long_running(tmpdir):
+    expected_return_code = 0
+    assert (
+        main(
+            test_workload="sleep 10",
+            test_workload_timeout=1,
+            test_no_raise_on_timeout=True,
+            prepare_commands=[],
+            prepare_commands_timeouts=[],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(TIMEOUT_RETURN_CODE)
+
+
+def test_end_to_end_timeout(tmpdir):
+    expected_return_code = TIMEOUT_RETURN_CODE
+    assert (
+        main(
+            test_workload="sleep 10",
+            test_workload_timeout=1,
+            test_no_raise_on_timeout=False,
+            prepare_commands=[],
+            prepare_commands_timeouts=[],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(expected_return_code)
+
+
+def test_end_to_end_prepare_timeout(tmpdir):
+    expected_return_code = 1
+    assert (
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=["exit 0", "sleep 10"],
+            prepare_commands_timeouts=[1, 1],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(None, [0, TIMEOUT_RETURN_CODE])
+
+
+@pytest.mark.parametrize("long_running", (True, False))
+def test_end_to_end_failure(tmpdir, long_running):
+    expected_return_code = 1
+    assert (
+        main(
+            test_workload="exit 1",
+            test_workload_timeout=1,
+            test_no_raise_on_timeout=long_running,
+            prepare_commands=[],
+            prepare_commands_timeouts=[],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(expected_return_code)
+
+
+def test_end_to_end_prepare_failure(tmpdir):
+    expected_return_code = 1
+    assert (
+        main(
+            test_workload="exit 0",
+            test_workload_timeout=10,
+            test_no_raise_on_timeout=False,
+            prepare_commands=["exit 0", "exit 1"],
+            prepare_commands_timeouts=[1, 1],
+            **cloud_storage_kwargs
+        )
+        == expected_return_code
+    )
+    _check_output_json(None, [0, 1])
+
+
+_PROM_SPILL_SAMPLE = [
+    {"metric": {}, "values": [[1700000000, "1073741824"]]},
+]
+
+
+@pytest.mark.parametrize(
+    "metrics_payload,expected_return_code",
+    [
+        # No spilling — empty list (Prometheus `> 0` filter dropped all points)
+        ({"spilled_bytes": []}, 0),
+        # Spilling occurred — non-empty list
+        ({"spilled_bytes": _PROM_SPILL_SAMPLE}, 1),
+        # Missing value (None) — fail with "could not retrieve" error
+        ({"spilled_bytes": None}, 1),
+        # Missing key entirely — fail with "could not retrieve" error
+        ({}, 1),
+    ],
+)
+def test_run_spilling_check_with_metrics_file(
+    tmpdir, metrics_payload, expected_return_code
+):
+    metrics_path = str(tmpdir / "metrics.json")
+    with open(metrics_path, "w") as f:
+        json.dump(metrics_payload, f)
+    with patch.dict(os.environ, {"METRICS_OUTPUT_JSON": metrics_path}):
+        assert run_spilling_check() == expected_return_code
+
+
+def test_run_spilling_check_missing_file(tmpdir):
+    metrics_path = str(tmpdir / "missing.json")
+    with patch.dict(os.environ, {"METRICS_OUTPUT_JSON": metrics_path}):
+        assert run_spilling_check() == 1
+
+
+def test_run_spilling_check_unset_metrics_env(tmpdir):
+    env = {k: v for k, v in os.environ.items() if k != "METRICS_OUTPUT_JSON"}
+    with patch.dict(os.environ, env, clear=True):
+        assert run_spilling_check() == 1
+
+
+def test_run_spilling_check_malformed_json(tmpdir):
+    metrics_path = str(tmpdir / "metrics.json")
+    with open(metrics_path, "w") as f:
+        f.write("{not valid json")
+    with patch.dict(os.environ, {"METRICS_OUTPUT_JSON": metrics_path}):
+        assert run_spilling_check() == 1
+
+
+@pytest.mark.parametrize("payload", [[1, 2, 3], "spilled_bytes", 42, None])
+def test_run_spilling_check_non_dict_json(tmpdir, payload):
+    # Valid JSON but not a dict, so `metrics.get(...)` raises AttributeError.
+    metrics_path = str(tmpdir / "metrics.json")
+    with open(metrics_path, "w") as f:
+        json.dump(payload, f)
+    with patch.dict(os.environ, {"METRICS_OUTPUT_JSON": metrics_path}):
+        assert run_spilling_check() == 1
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_bisect.py

@@ -0,0 +1,109 @@
+import sys
+from typing import Dict, List
+from unittest import mock
+
+import pytest
+
+from ray_release.scripts.ray_bisect import (
+    _bisect,
+    _get_test,
+    _obtain_test_result,
+    _sanity_check,
+)
+
+
+def test_sanity_check():
+    def _mock_run_test(*args, **kwawrgs) -> Dict[str, Dict[int, str]]:
+        return {
+            "passing_revision": {0: "passed", 1: "passed"},
+            "failing_revision": {0: "failed", 1: "failed"},
+            "flaky_revision": {0: "failed", 1: "passed"},
+        }
+
+    with mock.patch(
+        "ray_release.scripts.ray_bisect._run_test",
+        side_effect=_mock_run_test,
+    ):
+        assert _sanity_check({}, "passing_revision", "failing_revision", 2)
+        assert _sanity_check({}, "passing_revision", "flaky_revision", 2)
+        assert not _sanity_check({}, "failing_revision", "passing_revision", 2)
+        assert not _sanity_check({}, "passing_revision", "passing_revision", 2)
+        assert not _sanity_check({}, "failing_revision", "failing_revision", 2)
+        assert not _sanity_check({}, "flaky_revision", "failing_revision", 2)
+
+
+def test_obtain_test_result():
+    test_cases = [
+        {
+            "c0": {0: "passed"},
+        },
+        {
+            "c0": {0: "passed", 1: "passed"},
+            "c1": {0: "hard_failed", 1: "hard_failed"},
+        },
+    ]
+
+    def _mock_check_output(input: List[str]) -> str:
+        commit, run = tuple(input[-1].split("-"))
+        return bytes(test_case[commit][int(run)], "utf-8")
+
+    for test_case in test_cases:
+        with mock.patch(
+            "subprocess.check_output",
+            side_effect=_mock_check_output,
+        ):
+            commits = set(test_case.keys())
+            rerun_per_commit = len(test_case[list(commits)[0]])
+            _obtain_test_result(commits, rerun_per_commit) == test_case
+
+
+def test_get_test():
+    test = _get_test(
+        "test_name", ["release/ray_release/tests/test_collection_data.yaml"]
+    )
+    assert test.get_name() == "test_name"
+
+
+def test_bisect():
+    test_cases = {
+        "c3": {
+            "c0": {0: "passed"},
+            "c1": {0: "passed"},
+            "c3": {0: "hard_failed"},
+            "c4": {0: "soft_failed"},
+        },
+        "c1": {
+            "c0": {0: "passed"},
+            "c1": {0: "hard_failed"},
+            "c2": {0: "hard_failed"},
+        },
+        "cc1": {
+            "cc0": {0: "passed"},
+            "cc1": {0: "hard_failed"},
+        },
+        "c2": {
+            "c0": {0: "passed", 1: "passed"},
+            "c2": {0: "passed", 1: "hard_failed"},
+            "c3": {0: "hard_failed", 1: "passed"},
+            "c4": {0: "soft_failed", 1: "soft_failed"},
+        },
+    }
+
+    for output, input in test_cases.items():
+
+        def _side_effect(ret):
+            def _mock_run_test(*args, **kwawrgs) -> Dict[str, str]:
+                return ret
+
+            return _mock_run_test
+
+        with mock.patch(
+            "ray_release.scripts.ray_bisect._run_test",
+            side_effect=_side_effect(input),
+        ):
+            for concurreny in range(1, 4):
+                assert _bisect({}, list(input.keys()), concurreny, 1) == output
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_buildkite.py

@@ -0,0 +1,869 @@
+import os
+import sys
+import tempfile
+import unittest
+from typing import TYPE_CHECKING, Callable, Dict
+
+if TYPE_CHECKING:
+    from ray_release.github_client import GitHubRepo
+from unittest.mock import patch
+
+import yaml
+
+from ray_release.bazel import bazel_runfile
+from ray_release.buildkite.concurrency import (
+    get_concurrency_group,
+    get_test_resources_from_cluster_compute,
+)
+from ray_release.buildkite.filter import filter_tests, group_tests
+from ray_release.buildkite.settings import (
+    Frequency,
+    Priority,
+    get_default_settings,
+    get_test_filters,
+    split_ray_repo_str,
+    update_settings_from_buildkite,
+    update_settings_from_environment,
+)
+from ray_release.buildkite.step import (
+    RELEASE_QUEUE_CLIENT,
+    RELEASE_QUEUE_DEFAULT,
+    get_step,
+)
+from ray_release.configs.global_config import init_global_config
+from ray_release.exception import ReleaseTestConfigError
+from ray_release.test import Test
+from ray_release.wheels import (
+    DEFAULT_BRANCH,
+)
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+
+class MockBuildkiteAgent:
+    def __init__(self, return_dict: Dict):
+        self.return_dict = return_dict
+
+    def __call__(self, key: str):
+        return self.return_dict.get(key, None)
+
+
+class MockReturn:
+    return_dict = {}
+
+    def __getattribute__(self, item):
+        return_dict = object.__getattribute__(self, "return_dict")
+        if item in return_dict:
+            mocked = return_dict[item]
+            if isinstance(mocked, Callable):
+                return mocked()
+            else:
+                return lambda *a, **kw: mocked
+        return object.__getattribute__(self, item)
+
+
+class MockBuildkitePythonAPI(MockReturn):
+    def builds(self):
+        return self
+
+    def artifacts(self):
+        return self
+
+
+class MockTest(Test):
+    def update_from_s3(self) -> None:
+        self["update_from_s3"] = True
+
+    def is_jailed_with_open_issue(self, ray_github: "GitHubRepo") -> bool:
+        return False
+
+
+class BuildkiteSettingsTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.buildkite = {}
+        self.buildkite_mock = MockBuildkiteAgent(self.buildkite)
+
+    def testSplitRayRepoStr(self):
+        url, branch = split_ray_repo_str("https://github.com/ray-project/ray.git")
+        self.assertEqual(url, "https://github.com/ray-project/ray.git")
+        self.assertEqual(branch, DEFAULT_BRANCH)
+
+        url, branch = split_ray_repo_str(
+            "https://github.com/ray-project/ray/tree/branch/sub"
+        )
+        self.assertEqual(url, "https://github.com/ray-project/ray.git")
+        self.assertEqual(branch, "branch/sub")
+
+        url, branch = split_ray_repo_str("https://github.com/user/ray/tree/branch/sub")
+        self.assertEqual(url, "https://github.com/user/ray.git")
+        self.assertEqual(branch, "branch/sub")
+
+        url, branch = split_ray_repo_str("ray-project:branch/sub")
+        self.assertEqual(url, "https://github.com/ray-project/ray.git")
+        self.assertEqual(branch, "branch/sub")
+
+        url, branch = split_ray_repo_str("user:branch/sub")
+        self.assertEqual(url, "https://github.com/user/ray.git")
+        self.assertEqual(branch, "branch/sub")
+
+        url, branch = split_ray_repo_str("user")
+        self.assertEqual(url, "https://github.com/user/ray.git")
+        self.assertEqual(branch, DEFAULT_BRANCH)
+
+    def testGetTestAttrRegexFilters(self):
+        test_filters = get_test_filters("")
+        self.assertDictEqual(test_filters, {})
+
+        test_filters = get_test_filters("name:xxx")
+        self.assertDictEqual(test_filters, {"name": ["xxx"]})
+
+        test_filters = get_test_filters("name:xxx\n")
+        self.assertDictEqual(test_filters, {"name": ["xxx"]})
+
+        test_filters = get_test_filters("name:xxx\n\nteam:yyy")
+        self.assertDictEqual(test_filters, {"name": ["xxx"], "team": ["yyy"]})
+
+        test_filters = get_test_filters("name:xxx\n \nteam:yyy\n")
+        self.assertDictEqual(test_filters, {"name": ["xxx"], "team": ["yyy"]})
+
+        # Test multiple filters with same attribute (OR logic)
+        test_filters = get_test_filters("name:xxx\nname:yyy")
+        self.assertDictEqual(test_filters, {"name": ["xxx", "yyy"]})
+
+        with self.assertRaises(ReleaseTestConfigError):
+            get_test_filters("xxx")
+
+    def testSettingsOverrideEnv(self):
+        settings = get_default_settings()
+
+        # With no environment variables, default settings shouldn't be updated
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertDictEqual(settings, updated_settings)
+
+        environ = os.environ.copy()
+
+        # Invalid frequency
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["RELEASE_FREQUENCY"] = "invalid"
+        updated_settings = settings.copy()
+        with self.assertRaises(ReleaseTestConfigError):
+            update_settings_from_environment(updated_settings)
+
+        # Invalid priority
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["RELEASE_PRIORITY"] = "invalid"
+        updated_settings = settings.copy()
+        with self.assertRaises(ReleaseTestConfigError):
+            update_settings_from_environment(updated_settings)
+
+        # Invalid test attr regex filters
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["TEST_ATTR_REGEX_FILTERS"] = "xxxx"
+        updated_settings = settings.copy()
+        with self.assertRaises(ReleaseTestConfigError):
+            update_settings_from_environment(updated_settings)
+
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["TEST_ATTR_REGEX_FILTERS"] = "name:xxx\nteam:yyy\n"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+        print(updated_settings)
+        self.assertDictEqual(
+            updated_settings["test_filters"],
+            {
+                "name": ["xxx"],
+                "team": ["yyy"],
+            },
+        )
+
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["RELEASE_FREQUENCY"] = "nightly"
+        os.environ["RAY_TEST_REPO"] = "https://github.com/user/ray.git"
+        os.environ["RAY_TEST_BRANCH"] = "sub/branch"
+        os.environ["TEST_NAME"] = "name_filter"
+        os.environ["RELEASE_PRIORITY"] = "manual"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertDictEqual(
+            updated_settings,
+            {
+                "frequency": Frequency.NIGHTLY,
+                "prefer_smoke_tests": False,
+                "test_filters": {"name": ["name_filter"]},
+                "ray_test_repo": "https://github.com/user/ray.git",
+                "ray_test_branch": "sub/branch",
+                "priority": Priority.MANUAL,
+                "no_concurrency_limit": False,
+            },
+        )
+
+        os.environ["RELEASE_FREQUENCY"] = "any-smoke"
+        update_settings_from_environment(updated_settings)
+        self.assertDictEqual(
+            updated_settings,
+            {
+                "frequency": Frequency.ANY,
+                "prefer_smoke_tests": True,
+                "test_filters": {"name": ["name_filter"]},
+                "ray_test_repo": "https://github.com/user/ray.git",
+                "ray_test_branch": "sub/branch",
+                "priority": Priority.MANUAL,
+                "no_concurrency_limit": False,
+            },
+        )
+
+        ###
+        # Buildkite PR build settings
+
+        # Default PR
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
+        os.environ[
+            "BUILDKITE_PULL_REQUEST_REPO"
+        ] = "https://github.com/user/ray-fork.git"
+        os.environ["BUILDKITE_BRANCH"] = "user:some_branch"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertEqual(
+            updated_settings["ray_test_repo"], "https://github.com/user/ray-fork.git"
+        )
+        self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
+
+        # PR without prefix
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
+        os.environ["BUILDKITE_PULL_REQUEST_REPO"] = "git://github.com/user/ray-fork.git"
+        os.environ["BUILDKITE_BRANCH"] = "some_branch"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertEqual(
+            updated_settings["ray_test_repo"], "https://github.com/user/ray-fork.git"
+        )
+        self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
+
+        # Branch build but pointing to fork
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
+        os.environ["BUILDKITE_BRANCH"] = "user:some_branch"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertEqual(
+            updated_settings["ray_test_repo"], "https://github.com/user/ray.git"
+        )
+        self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
+
+        # Branch build but pointing to main repo branch
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
+        os.environ["BUILDKITE_BRANCH"] = "some_branch"
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertEqual(
+            updated_settings["ray_test_repo"], "https://github.com/ray-project/ray.git"
+        )
+        self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
+
+        # Empty BUILDKITE_PULL_REQUEST_REPO
+        os.environ.clear()
+        os.environ.update(environ)
+        os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
+        os.environ["BUILDKITE_BRANCH"] = "some_branch"
+        os.environ["BUILDKITE_PULL_REQUEST_REPO"] = ""
+        updated_settings = settings.copy()
+        update_settings_from_environment(updated_settings)
+
+        self.assertEqual(
+            updated_settings["ray_test_repo"], "https://github.com/ray-project/ray.git"
+        )
+        self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
+
+    def testSettingsOverrideBuildkite(self):
+        settings = get_default_settings()
+
+        with patch(
+            "ray_release.buildkite.settings.get_buildkite_prompt_value",
+            self.buildkite_mock,
+        ):
+            # With no buildkite variables, default settings shouldn't be updated
+            updated_settings = settings.copy()
+            update_settings_from_buildkite(updated_settings)
+
+            self.assertDictEqual(settings, updated_settings)
+
+            buildkite = self.buildkite.copy()
+
+            # Invalid frequency
+            self.buildkite.clear()
+            self.buildkite.update(buildkite)
+            self.buildkite["release-frequency"] = "invalid"
+            updated_settings = settings.copy()
+            with self.assertRaises(ReleaseTestConfigError):
+                update_settings_from_buildkite(updated_settings)
+
+            # Invalid priority
+            self.buildkite.clear()
+            self.buildkite.update(buildkite)
+            self.buildkite["release-priority"] = "invalid"
+            updated_settings = settings.copy()
+            with self.assertRaises(ReleaseTestConfigError):
+                update_settings_from_buildkite(updated_settings)
+
+            # Invalid test attr regex filters
+            self.buildkite.clear()
+            self.buildkite.update(buildkite)
+            self.buildkite["release-test-filters"] = "xxxx"
+            updated_settings = settings.copy()
+            with self.assertRaises(ReleaseTestConfigError):
+                update_settings_from_buildkite(updated_settings)
+
+            self.buildkite.clear()
+            self.buildkite.update(buildkite)
+            self.buildkite["release-test-filters"] = "name:xxx\ngroup:yyy"
+            updated_settings = settings.copy()
+            update_settings_from_buildkite(updated_settings)
+            self.assertDictEqual(
+                updated_settings["test_filters"],
+                {
+                    "name": ["xxx"],
+                    "group": ["yyy"],
+                },
+            )
+
+            self.buildkite.clear()
+            self.buildkite.update(buildkite)
+            self.buildkite["release-frequency"] = "nightly"
+            self.buildkite["release-ray-test-repo-branch"] = "user:sub/branch"
+            self.buildkite["release-test-name"] = "name_filter"
+            self.buildkite["release-priority"] = "manual"
+            updated_settings = settings.copy()
+            update_settings_from_buildkite(updated_settings)
+
+            self.assertDictEqual(
+                updated_settings,
+                {
+                    "frequency": Frequency.NIGHTLY,
+                    "prefer_smoke_tests": False,
+                    "test_filters": {"name": ["name_filter"]},
+                    "ray_test_repo": "https://github.com/user/ray.git",
+                    "ray_test_branch": "sub/branch",
+                    "priority": Priority.MANUAL,
+                    "no_concurrency_limit": False,
+                },
+            )
+
+            self.buildkite["release-frequency"] = "any-smoke"
+            update_settings_from_buildkite(updated_settings)
+
+            self.assertDictEqual(
+                updated_settings,
+                {
+                    "frequency": Frequency.ANY,
+                    "prefer_smoke_tests": True,
+                    "test_filters": {"name": ["name_filter"]},
+                    "ray_test_repo": "https://github.com/user/ray.git",
+                    "ray_test_branch": "sub/branch",
+                    "priority": Priority.MANUAL,
+                    "no_concurrency_limit": False,
+                },
+            )
+
+    def _filter_names(self, *args, **kwargs):
+        filtered = filter_tests(*args, **kwargs)
+        return [(t[0]["name"], t[1]) for t in filtered]
+
+    @patch(
+        "ray_release.test_automation.state_machine.TestStateMachine.get_ray_repo",
+        return_value=None,
+    )
+    def testFilterTests(self, *args):
+        test = MockTest(
+            {
+                "name": "test_1",
+                "frequency": "nightly",
+                "smoke_test": {"frequency": "nightly"},
+                "team": "team_1",
+                "run": {"type": "job"},
+            }
+        )
+        tests = [
+            test,
+            MockTest(
+                {
+                    "name": "test_2",
+                    "frequency": "weekly",
+                    "smoke_test": {"frequency": "nightly"},
+                    "team": "team_2",
+                    "run": {"type": "client"},
+                }
+            ),
+            MockTest({"name": "other_1", "frequency": "weekly", "team": "team_2"}),
+            MockTest(
+                {
+                    "name": "other_2",
+                    "frequency": "nightly",
+                    "smoke_test": {"frequency": "manual"},
+                    "team": "team_2",
+                    "run": {"type": "job"},
+                }
+            ),
+            MockTest({"name": "other_3", "frequency": "manual", "team": "team_2"}),
+            MockTest({"name": "test_3", "frequency": "nightly", "team": "team_2"}),
+            MockTest(
+                {
+                    "name": "test_4.kuberay",
+                    "frequency": "nightly",
+                    "env": "kuberay",
+                    "team": "team_2",
+                    "run": {"type": "job"},
+                }
+            ),
+        ]
+
+        # Test filter by prefix alone
+        filtered = self._filter_names(
+            tests, frequency=Frequency.ANY, test_filters={"prefix": ["test"]}
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", False),
+                ("test_2", False),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        # Test filter by prefix and regex together
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"prefix": ["test"], "name": ["other.*"]},
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [],
+        )
+
+        filtered = self._filter_names(tests, frequency=Frequency.ANY)
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", False),
+                ("test_2", False),
+                ("other_1", False),
+                ("other_2", False),
+                ("other_3", False),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+        assert not test.get("update_from_s3")
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.ANY,
+            prefer_smoke_tests=True,
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", True),
+                ("test_2", True),
+                ("other_1", False),
+                ("other_2", True),
+                ("other_3", False),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        filtered = self._filter_names(tests, frequency=Frequency.NIGHTLY)
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", False),
+                ("test_2", True),
+                ("other_2", False),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            prefer_smoke_tests=True,
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", True),
+                ("test_2", True),
+                ("other_2", True),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        filtered = self._filter_names(tests, frequency=Frequency.WEEKLY)
+        self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)])
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["other.*"]},
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("other_2", False),
+            ],
+        )
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["test.*"]},
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", False),
+                ("test_2", True),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        filtered = self._filter_names(
+            tests, frequency=Frequency.NIGHTLY, test_filters={"name": ["test"]}
+        )
+        self.assertSequenceEqual(
+            filtered,
+            [
+                ("test_1", False),
+                ("test_2", True),
+                ("test_3", False),
+                ("test_4.kuberay", False),
+            ],
+        )
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["test.*"], "team": ["team_1"]},
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False)])
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["test_1|test_2"]},
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False), ("test_2", True)])
+
+        # Test OR logic within same attribute
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={"name": ["^test_1$", "^test_3$"]},
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False), ("test_3", False)])
+
+        # Test OR logic with AND across attributes
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.NIGHTLY,
+            test_filters={
+                "name": ["^test_1$", "^test_3$"],
+                "team": ["team_1", "team_2"],
+            },
+        )
+        self.assertSequenceEqual(filtered, [("test_1", False), ("test_3", False)])
+
+        # Filter by nested properties
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.ANY,
+            test_filters={"run/type": ["job"]},
+        )
+        self.assertSequenceEqual(
+            filtered, [("test_1", False), ("other_2", False), ("test_4.kuberay", False)]
+        )
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.ANY,
+            test_filters={"run/type": ["client"]},
+        )
+        self.assertSequenceEqual(filtered, [("test_2", False)])
+
+        filtered = self._filter_names(
+            tests,
+            frequency=Frequency.ANY,
+            test_filters={"run/invalid": ["xxx"]},
+        )
+        self.assertSequenceEqual(filtered, [])
+
+    def testGroupTests(self):
+        tests = [
+            (Test(name="x1", group="x"), False),
+            (Test(name="x2", group="x"), False),
+            (Test(name="y1", group="y"), False),
+            (Test(name="ungrouped"), False),
+            (Test(name="x3", group="x"), False),
+        ]
+
+        grouped = group_tests(tests)
+        self.assertEqual(len(grouped), 3)  # Three groups
+        self.assertEqual(len(grouped["x"]), 3)
+        self.assertSequenceEqual(
+            [t["name"] for t, _ in grouped["x"]], ["x1", "x2", "x3"]
+        )
+        self.assertEqual(len(grouped["y"]), 1)
+
+    def testGetStep(self):
+        test = MockTest(
+            {
+                "name": "test",
+                "frequency": "nightly",
+                "run": {"script": "test_script.py"},
+                "smoke_test": {"frequency": "nightly"},
+                "cluster": {"byod": {"type": "cpu"}},
+            }
+        )
+
+        with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
+            step = get_step(test, smoke_test=False)
+            joined_commands = "\n".join(step["commands"])
+            self.assertNotIn("--smoke-test", joined_commands)
+
+            step = get_step(test, smoke_test=True)
+            joined_commands = "\n".join(step["commands"])
+            self.assertIn("--smoke-test", joined_commands)
+
+            step = get_step(test, priority_val=20)
+            self.assertEqual(step["priority"], 20)
+
+    def testInstanceResources(self):
+        # AWS instances
+        cpus, gpus = get_test_resources_from_cluster_compute(
+            {
+                "head_node_type": {"instance_type": "m5.4xlarge"},  # 16 CPUs, 0 GPUs
+                "worker_node_types": [
+                    {
+                        "instance_type": "m5.8xlarge",  # 32 CPUS, 0 GPUs
+                        "max_workers": 4,
+                    },
+                    {
+                        "instance_type": "g3.8xlarge",  # 32 CPUs, 2 GPUs
+                        "min_workers": 8,
+                    },
+                ],
+            }
+        )
+        self.assertEqual(cpus, 16 + 32 * 4 + 32 * 8)
+        self.assertEqual(gpus, 2 * 8)
+
+        cpus, gpus = get_test_resources_from_cluster_compute(
+            {
+                "head_node_type": {
+                    "instance_type": "n1-standard-16"  # 16 CPUs, 0 GPUs
+                },
+                "worker_node_types": [
+                    {
+                        "instance_type": "random-str-xxx-32",  # 32 CPUS, 0 GPUs
+                        "max_workers": 4,
+                    },
+                    {
+                        "instance_type": "a2-highgpu-2g",  # 24 CPUs, 2 GPUs
+                        "min_workers": 8,
+                    },
+                ],
+            }
+        )
+        self.assertEqual(cpus, 16 + 32 * 4 + 24 * 8)
+        self.assertEqual(gpus, 2 * 8)
+
+    def testInstanceResourcesNewSchema(self):
+        # New schema: head_node + worker_nodes
+        cpus, gpus = get_test_resources_from_cluster_compute(
+            {
+                "head_node": {"instance_type": "m5.4xlarge"},  # 16 CPUs, 0 GPUs
+                "worker_nodes": [
+                    {
+                        "instance_type": "m5.8xlarge",  # 32 CPUS, 0 GPUs
+                        "min_nodes": 1,
+                        "max_nodes": 4,
+                    },
+                    {
+                        "instance_type": "g3.8xlarge",  # 32 CPUs, 2 GPUs
+                        "min_nodes": 8,
+                        "max_nodes": 8,
+                    },
+                ],
+            },
+            is_new_schema=True,
+        )
+        self.assertEqual(cpus, 16 + 32 * 4 + 32 * 8)
+        self.assertEqual(gpus, 2 * 8)
+
+    def testInstanceResourcesNewSchemaEmpty(self):
+        # New schema: empty config returns 0 resources
+        cpus, gpus = get_test_resources_from_cluster_compute({}, is_new_schema=True)
+        self.assertEqual(cpus, 0)
+        self.assertEqual(gpus, 0)
+
+    def testInstanceResourcesNewSchemaMissingInstanceType(self):
+        # New schema: worker without instance_type is skipped
+        cpus, gpus = get_test_resources_from_cluster_compute(
+            {
+                "head_node": {"instance_type": "m5.4xlarge"},  # 16 CPUs
+                "worker_nodes": [
+                    {"min_nodes": 4},  # no instance_type, skipped
+                ],
+            },
+            is_new_schema=True,
+        )
+        self.assertEqual(cpus, 16)
+        self.assertEqual(gpus, 0)
+
+    def testConcurrencyGroups(self):
+        def _return(ret):
+            def _inner(*args, **kwargs):
+                return ret
+
+            return _inner
+
+        test = Test(
+            {
+                "name": "test_1",
+            }
+        )
+
+        def test_concurrency(cpu, gpu, group):
+            with patch(
+                "ray_release.buildkite.concurrency.get_test_resources",
+                _return((cpu, gpu)),
+            ):
+                group_name, _ = get_concurrency_group(test)
+                self.assertEqual(group_name, group)
+
+        test_concurrency(12800, 9, "large-gpu")
+        test_concurrency(12800, 8, "small-gpu")
+        test_concurrency(12800, 1, "small-gpu")
+        test_concurrency(12800, 0, "enormous")
+        test_concurrency(1025, 0, "enormous")
+        test_concurrency(1024, 0, "large")
+        test_concurrency(513, 0, "large")
+        test_concurrency(512, 0, "medium")
+        test_concurrency(129, 0, "medium")
+        test_concurrency(128, 0, "small")
+        test_concurrency(9, 0, "tiny")
+        test_concurrency(32, 0, "tiny")
+        test_concurrency(8, 0, "minuscule")
+        test_concurrency(1, 0, "minuscule")
+        test_concurrency(33, 0, "small")
+
+    def testConcurrencyGroupSmokeTest(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cluster_config_full = {
+                "head_node_type": {
+                    "instance_type": "n1-standard-16"  # 16 CPUs, 0 GPUs
+                },
+                "worker_node_types": [
+                    {
+                        "instance_type": "random-str-xxx-32",  # 32 CPUS, 0 GPUs
+                        "max_workers": 10,
+                    },
+                ],
+            }
+
+            cluster_config_smoke = {
+                "head_node_type": {
+                    "instance_type": "n1-standard-16"  # 16 CPUs, 0 GPUs
+                },
+                "worker_node_types": [
+                    {
+                        "instance_type": "random-str-xxx-32",  # 32 CPUS, 0 GPUs
+                        "max_workers": 1,
+                    },
+                ],
+            }
+
+            cluster_config_full_path = os.path.join(tmpdir, "full.yaml")
+            with open(cluster_config_full_path, "w") as fp:
+                yaml.safe_dump(cluster_config_full, fp)
+
+            cluster_config_smoke_path = os.path.join(tmpdir, "smoke.yaml")
+            with open(cluster_config_smoke_path, "w") as fp:
+                yaml.safe_dump(cluster_config_smoke, fp)
+
+            test = MockTest(
+                {
+                    "name": "test_1",
+                    "cluster": {
+                        "cluster_compute": cluster_config_full_path,
+                        "byod": {"type": "cpu"},
+                    },
+                    "smoke_test": {
+                        "cluster": {
+                            "cluster_compute": cluster_config_smoke_path,
+                            "byod": {"type": "cpu"},
+                        },
+                    },
+                }
+            )
+            with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
+                step = get_step(test, smoke_test=False)
+                self.assertEqual(step["concurrency_group"], "medium")
+
+                step = get_step(test, smoke_test=True)
+                self.assertEqual(step["concurrency_group"], "small")
+
+    def testStepQueueClient(self):
+        test_regular = MockTest(
+            {
+                "name": "test",
+                "frequency": "nightly",
+                "run": {"script": "test_script.py"},
+                "cluster": {"byod": {"type": "cpu"}},
+            }
+        )
+        test_client = MockTest(
+            {
+                "name": "test",
+                "frequency": "nightly",
+                "run": {"script": "test_script.py", "type": "client"},
+                "cluster": {"byod": {"type": "cpu"}},
+            }
+        )
+
+        with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
+            step = get_step(test_regular)
+            self.assertEqual(step["agents"]["queue"], str(RELEASE_QUEUE_DEFAULT))
+
+            step = get_step(test_client)
+            self.assertEqual(step["agents"]["queue"], str(RELEASE_QUEUE_CLIENT))
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_byod_build.py

@@ -0,0 +1,191 @@
+import os
+import sys
+import tempfile
+from typing import List
+from unittest.mock import patch
+
+import pytest
+
+from ray_release.bazel import bazel_runfile
+from ray_release.byod.build import (
+    _get_ray_commit,
+    build_anyscale_base_byod_images,
+    build_anyscale_custom_byod_image,
+)
+from ray_release.byod.build_context import BuildContext
+from ray_release.configs.global_config import get_global_config, init_global_config
+from ray_release.test import Test
+
+
+def test_get_ray_commit() -> None:
+    assert (
+        _get_ray_commit(
+            {
+                "RAY_WANT_COMMIT_IN_IMAGE": "abc123",
+                "COMMIT_TO_TEST": "def456",
+                "BUILDKITE_COMMIT": "987789",
+            }
+        )
+        == "abc123"
+    )
+
+    assert (
+        _get_ray_commit(
+            {
+                "COMMIT_TO_TEST": "def456",
+                "BUILDKITE_COMMIT": "987789",
+            }
+        )
+        == "def456"
+    )
+    assert _get_ray_commit({"BUILDKITE_COMMIT": "987789"}) == "987789"
+    assert _get_ray_commit({"PATH": "/usr/bin"}) == ""
+
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+
+def test_build_anyscale_custom_byod_image() -> None:
+    cmds = []
+
+    def _mock_check_call(
+        cmd: List[str],
+        *args,
+        **kwargs,
+    ) -> None:
+        cmds.append(cmd)
+
+    with patch("ray_release.byod.build._image_exist", return_value=False), patch.dict(
+        "os.environ",
+        {
+            "BUILDKITE_COMMIT": "abc123",
+            "RAYCI_BUILD_ID": "a1b2c3d4",
+        },
+    ), patch("subprocess.check_call", side_effect=_mock_check_call,), patch(
+        "subprocess.check_output",
+        return_value=b"abc123",
+    ):
+        test = Test(
+            name="name",
+            cluster={"byod": {"post_build_script": "foo.sh"}},
+        )
+        with tempfile.TemporaryDirectory() as byod_dir:
+            with open(os.path.join(byod_dir, "foo.sh"), "wt") as f:
+                f.write("echo foo")
+
+            build_context: BuildContext = {
+                "post_build_script": test.get_byod_post_build_script(),
+            }
+            build_anyscale_custom_byod_image(
+                test.get_anyscale_byod_image(),
+                test.get_anyscale_base_byod_image(),
+                build_context,
+                release_byod_dir=byod_dir,
+            )
+        assert (" ".join(cmds[0])).startswith(
+            "docker build --progress=plain . "
+            "--build-arg BASE_IMAGE=029272617770.dkr.ecr.us-west-2."
+            "amazonaws.com/anyscale/ray:a1b2c3d4-py310-cpu "
+            "-t 029272617770.dkr.ecr.us-west-2."
+            "amazonaws.com/anyscale/ray:a1b2c3d4-py310-cpu-"
+        )
+
+
+def test_build_anyscale_base_byod_images() -> None:
+    def _mock_image_exist(image: str) -> bool:
+        return True
+
+    with patch(
+        "os.environ",
+        {
+            "BUILDKITE_COMMIT": "abc123",
+            "RAYCI_BUILD_ID": "a1b2c3d4",
+        },
+    ), patch("subprocess.check_call", return_value=None), patch(
+        "ray_release.byod.build._image_exist", side_effect=_mock_image_exist
+    ):
+        tests = [
+            Test(name="aws", env="aws", cluster={"byod": {}}),
+            Test(name="aws", env="aws", cluster={"byod": {"type": "gpu"}}),
+            Test(
+                # This is a duplicate of the default.
+                name="aws",
+                env="aws",
+                python="3.10",
+                cluster={"byod": {"type": "cpu"}},
+            ),
+            Test(name="aws", env="aws", python="3.11", cluster={"byod": {}}),
+            Test(name="aws", env="aws", cluster={"byod": {"type": "cu121"}}),
+            Test(
+                name="aws",
+                env="aws",
+                python="3.10",
+                cluster={"byod": {"type": "cu116"}},
+            ),
+            Test(
+                name="aws",
+                env="aws",
+                python="3.11",
+                cluster={"byod": {"type": "cu118"}},
+            ),
+            Test(name="gce", env="gce", cluster={"byod": {}}),
+        ]
+        images = build_anyscale_base_byod_images(tests)
+        global_config = get_global_config()
+        aws_cr = global_config["byod_ecr"]
+        # Base images are always from AWS ECR (see get_anyscale_base_byod_image),
+        # so GCE and AWS CPU tests resolve to the same image (6 unique, not 7).
+        assert set(images) == {
+            f"{aws_cr}/anyscale/ray:a1b2c3d4-py310-cpu",
+            f"{aws_cr}/anyscale/ray:a1b2c3d4-py310-cu116",
+            f"{aws_cr}/anyscale/ray:a1b2c3d4-py310-cu121",
+            f"{aws_cr}/anyscale/ray:a1b2c3d4-py311-cu118",
+            f"{aws_cr}/anyscale/ray-ml:a1b2c3d4-py310-gpu",
+            f"{aws_cr}/anyscale/ray:a1b2c3d4-py311-cpu",
+        }
+
+
+def test_get_anyscale_base_byod_image_always_uses_aws_ecr() -> None:
+    """Base images should always come from AWS ECR, regardless of test env."""
+    global_config = get_global_config()
+    aws_cr = global_config["byod_ecr"]
+
+    with patch.dict(
+        "os.environ",
+        {
+            "BUILDKITE_COMMIT": "abc123",
+            "RAYCI_BUILD_ID": "a1b2c3d4",
+        },
+    ):
+        expected_cpu = f"{aws_cr}/anyscale/ray:a1b2c3d4-py310-cpu"
+        for env in ("aws", "gce", "azure", "kuberay"):
+            test = Test(name="t", env=env, cluster={"byod": {}})
+            assert test.get_anyscale_base_byod_image() == expected_cpu, env
+
+        gpu_test = Test(name="t", env="gce", cluster={"byod": {"type": "gpu"}})
+        assert gpu_test.get_anyscale_base_byod_image() == (
+            f"{aws_cr}/anyscale/ray-ml:a1b2c3d4-py310-gpu"
+        )
+
+        # When ray_version is set, base image uses anyscale/ray prefix
+        # instead of byod_ecr.
+        versioned_cpu_test = Test(
+            name="t",
+            env="gce",
+            cluster={"byod": {}, "ray_version": "2.50.0"},
+        )
+        versioned_gpu_test = Test(
+            name="t",
+            env="gce",
+            cluster={"byod": {"type": "gpu"}, "ray_version": "2.50.0"},
+        )
+        assert versioned_cpu_test.get_anyscale_base_byod_image() == (
+            "anyscale/ray:2.50.0-py310-cpu"
+        )
+        assert versioned_gpu_test.get_anyscale_base_byod_image() == (
+            "anyscale/ray:2.50.0-py310-cu121"
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-vv", __file__]))

release/ray_release/tests/test_byod_build_context.py

@@ -0,0 +1,240 @@
+import os
+import sys
+import tempfile
+
+import pytest
+
+from ray_release.byod.build_context import (
+    _INSTALL_PYTHON_DEPS_SCRIPT,
+    build_context_digest,
+    decode_build_context,
+    encode_build_context,
+    fill_build_context_dir,
+    make_build_context,
+)
+
+
+def test_make_build_context() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        script_path = os.path.join(tmpdir, "post_build.sh")
+        with open(script_path, "w") as f:
+            f.write("echo hello")
+
+        depset_path = os.path.join(tmpdir, "deps.lock")
+        with open(depset_path, "w") as f:
+            f.write("numpy==1.0.0")
+
+        ctx = make_build_context(
+            base_dir=tmpdir,
+            envs={"FOO": "bar"},
+            post_build_script="post_build.sh",
+            python_depset="deps.lock",
+        )
+
+        assert ctx == {
+            "envs": {"FOO": "bar"},
+            "post_build_script": "post_build.sh",
+            "post_build_script_digest": ctx["post_build_script_digest"],
+            "python_depset": "deps.lock",
+            "python_depset_digest": ctx["python_depset_digest"],
+            "install_python_deps_script_digest": ctx[
+                "install_python_deps_script_digest"
+            ],
+        }
+        assert ctx["post_build_script_digest"].startswith("sha256:")
+        assert ctx["python_depset_digest"].startswith("sha256:")
+        assert ctx["install_python_deps_script_digest"].startswith("sha256:")
+
+
+def test_make_build_context_partial() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        script_path = os.path.join(tmpdir, "post_build.sh")
+        with open(script_path, "w") as f:
+            f.write("echo hello")
+
+        ctx = make_build_context(
+            base_dir=tmpdir,
+            post_build_script="post_build.sh",
+        )
+
+        assert ctx == {
+            "post_build_script": "post_build.sh",
+            "post_build_script_digest": ctx["post_build_script_digest"],
+        }
+        assert ctx["post_build_script_digest"].startswith("sha256:")
+
+
+def test_make_build_context_empty() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        ctx = make_build_context(base_dir=tmpdir)
+        assert ctx == {}
+
+
+def test_encode_build_context() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with open(os.path.join(tmpdir, "build.sh"), "w") as f:
+            f.write("echo hello")
+
+        ctx = make_build_context(
+            base_dir=tmpdir,
+            envs={"ZZZ": "last", "AAA": "first"},
+            post_build_script="build.sh",
+        )
+        encoded = encode_build_context(ctx)
+
+        # Verify minified (no spaces)
+        assert " " not in encoded
+
+        # Verify deterministic via digest - same content in different order produces same digest
+        ctx_reordered = {
+            "post_build_script_digest": ctx["post_build_script_digest"],
+            "post_build_script": "build.sh",
+            "envs": {"AAA": "first", "ZZZ": "last"},
+        }
+        assert build_context_digest(ctx) == build_context_digest(ctx_reordered)
+
+
+def test_decode_build_context() -> None:
+    data = '{"envs":{"FOO":"bar"},"post_build_script":"build.sh"}'
+    ctx = decode_build_context(data)
+
+    assert ctx["envs"] == {"FOO": "bar"}
+    assert ctx["post_build_script"] == "build.sh"
+
+
+def test_encode_decode_roundtrip() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with open(os.path.join(tmpdir, "script.sh"), "w") as f:
+            f.write("echo hello")
+
+        with open(os.path.join(tmpdir, "deps.lock"), "w") as f:
+            f.write("numpy==1.0.0")
+
+        ctx = make_build_context(
+            base_dir=tmpdir,
+            envs={"KEY": "value"},
+            post_build_script="script.sh",
+            python_depset="deps.lock",
+        )
+
+        encoded = encode_build_context(ctx)
+        decoded = decode_build_context(encoded)
+
+        assert decoded == ctx
+
+
+def test_build_context_digest() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with open(os.path.join(tmpdir, "build.sh"), "w") as f:
+            f.write("echo hello")
+
+        with open(os.path.join(tmpdir, "build2.sh"), "w") as f:
+            f.write("echo world")
+
+        ctx1 = make_build_context(
+            base_dir=tmpdir,
+            post_build_script="build.sh",
+        )
+        ctx2 = make_build_context(
+            base_dir=tmpdir,
+            post_build_script="build2.sh",
+        )
+        ctx3 = make_build_context(
+            base_dir=tmpdir,
+            envs={"FOO": "bar"},
+            post_build_script="build.sh",
+        )
+
+        digest1 = build_context_digest(ctx1)
+        digest2 = build_context_digest(ctx2)
+        digest3 = build_context_digest(ctx3)
+
+        # Different file contents produce different digests
+        assert digest1 != digest2
+        # Different envs produce different digests
+        assert digest1 != digest3
+        # All three are different
+        assert len({digest1, digest2, digest3}) == 3
+
+
+def test_fill_build_context_dir_empty() -> None:
+    with (
+        tempfile.TemporaryDirectory() as source_dir,
+        tempfile.TemporaryDirectory() as build_dir,
+    ):
+        ctx = make_build_context(base_dir=source_dir)
+
+        fill_build_context_dir(ctx, source_dir, build_dir)
+
+        # Check Dockerfile
+        with open(os.path.join(build_dir, "Dockerfile")) as f:
+            dockerfile = f.read()
+        expected_dockerfile = "\n".join(
+            [
+                "# syntax=docker/dockerfile:1.3-labs",
+                "ARG BASE_IMAGE",
+                "FROM ${BASE_IMAGE}",
+                "",
+            ]
+        )
+        assert dockerfile == expected_dockerfile
+
+        # Check no other files were created
+        assert os.listdir(build_dir) == ["Dockerfile"]
+
+
+def test_fill_build_context_dir() -> None:
+    with (
+        tempfile.TemporaryDirectory() as source_dir,
+        tempfile.TemporaryDirectory() as build_dir,
+    ):
+        # Create input files
+        with open(os.path.join(source_dir, "post_build.sh"), "w") as f:
+            f.write("#!/bin/bash\necho hello")
+
+        with open(os.path.join(source_dir, "deps.lock"), "w") as f:
+            f.write("numpy==1.0.0\npandas==2.0.0")
+
+        ctx = make_build_context(
+            base_dir=source_dir,
+            envs={"FOO": "bar", "BAZ": "qux"},
+            post_build_script="post_build.sh",
+            python_depset="deps.lock",
+        )
+
+        fill_build_context_dir(ctx, source_dir, build_dir)
+
+        # Check Dockerfile
+        with open(os.path.join(build_dir, "Dockerfile")) as f:
+            dockerfile = f.read()
+        expected_dockerfile = "\n".join(
+            [
+                "# syntax=docker/dockerfile:1.3-labs",
+                "ARG BASE_IMAGE",
+                "FROM ${BASE_IMAGE}",
+                "ENV \\",
+                "  BAZ=qux \\",
+                "  FOO=bar",
+                "COPY install_python_deps.sh /tmp/install_python_deps.sh",
+                "COPY python_depset.lock python_depset.lock",
+                "RUN bash /tmp/install_python_deps.sh python_depset.lock",
+                "COPY post_build_script.sh /tmp/post_build_script.sh",
+                "RUN bash /tmp/post_build_script.sh",
+                "",
+            ]
+        )
+        assert dockerfile == expected_dockerfile
+
+        # Check copied files
+        with open(os.path.join(build_dir, "post_build_script.sh")) as f:
+            assert f.read() == "#!/bin/bash\necho hello"
+
+        with open(os.path.join(build_dir, "python_depset.lock")) as f:
+            assert f.read() == "numpy==1.0.0\npandas==2.0.0"
+
+        with open(os.path.join(build_dir, "install_python_deps.sh")) as f:
+            assert f.read() == _INSTALL_PYTHON_DEPS_SCRIPT
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-vv", __file__]))

release/ray_release/tests/test_cloud_util.py

@@ -0,0 +1,110 @@
+import os
+import sys
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+from ray_release.cloud_util import (
+    _parse_abfss_uri,
+    _upload_file_to_azure,
+    upload_working_dir_to_azure,
+)
+
+
+class FakeBlobServiceClient:
+    def __init__(self, account_url, credential):
+        self.account_url = account_url
+        self.credential = credential
+        self.blob_client = FakeBlobClient()
+
+    def get_blob_client(self, container, blob):
+        return self.blob_client
+
+
+class FakeBlobClient:
+    def __init__(self):
+        self.uploaded_data = None
+
+    def upload_blob(self, data, overwrite=True):
+        self.uploaded_data = data.read()
+
+
+def test_upload_file_to_azure():
+    with tempfile.TemporaryDirectory() as tmp_path:
+        local_file = os.path.join(tmp_path, "test.txt")
+        expected_content = "test content"
+        with open(local_file, "w") as f:
+            f.write(expected_content)
+        container = "test_container"
+        account = "test_account"
+        azure_path = f"abfss://{container}@{account}.dfs.core.windows.net/path/test.txt"
+        fake_blob_client = FakeBlobClient()
+        fake_blob_service_client = FakeBlobServiceClient(
+            f"https://{account}.blob.core.windows.net", "test-credential"
+        )
+        fake_blob_service_client.blob_client = fake_blob_client
+
+        _upload_file_to_azure(str(local_file), azure_path, fake_blob_service_client)
+
+        with open(local_file, "rb") as f:
+            expected_data = f.read()
+            assert fake_blob_client.uploaded_data == expected_data
+
+
+@patch("ray_release.cloud_util._upload_file_to_azure")
+def test_upload_working_dir_to_azure(mock_upload_file_to_azure):
+    with tempfile.TemporaryDirectory() as tmp_path:
+        working_dir = os.path.join(tmp_path, "working_dir")
+        os.makedirs(working_dir)
+        with open(os.path.join(working_dir, "test.txt"), "w") as f:
+            f.write("test content")
+        azure_directory_uri = (
+            "abfss://container@account.dfs.core.windows.net/path/working_dir"
+        )
+        upload_working_dir_to_azure(working_dir, azure_directory_uri)
+        args = mock_upload_file_to_azure.call_args.kwargs
+        assert args["local_file_path"].endswith(".zip")
+        assert args["azure_file_path"].startswith(f"{azure_directory_uri}/")
+        assert args["azure_file_path"].endswith(".zip")
+
+
+@pytest.mark.parametrize(
+    "uri, expected_account, expected_container, expected_path",
+    [
+        (
+            "abfss://container@account.dfs.core.windows.net/path/test.txt",
+            "account",
+            "container",
+            "path/test.txt",
+        ),
+        ("abfss://container@account.dfs.core.windows.net/", "account", "container", ""),
+        (
+            "abfss://container@account.dfs.core.windows.net/path/",
+            "account",
+            "container",
+            "path/",
+        ),
+        (
+            "abfss://container@account.dfs.core.windows.net/path/to/file.txt",
+            "account",
+            "container",
+            "path/to/file.txt",
+        ),
+        (
+            "abfss://container-name@account-123.dfs.core.windows.net/path",
+            "account-123",
+            "container-name",
+            "path",
+        ),
+    ],
+)
+def test_parse_abfss_uri(uri, expected_account, expected_container, expected_path):
+    account, container, path = _parse_abfss_uri(uri)
+    assert account == expected_account
+    assert container == expected_container
+    assert path == expected_path
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_collection_data.yaml

@@ -0,0 +1,12 @@
+- name: test_name
+  group: test_group
+  working_dir: cluster_tests
+  frequency: nightly
+  team: team
+  cluster:
+    byod: {}
+    cluster_compute: cpt_autoscaling_1-3_aws.yaml
+    anyscale_sdk_2026: true
+  run:
+    timeout: 3600
+    script: test_script.py

release/ray_release/tests/test_config.py

@@ -0,0 +1,588 @@
+import copy
+import sys
+
+import pytest
+import yaml
+
+from ray_release.config import (
+    CLOUD_ID_TO_NAME,
+    _substitute_variable,
+    get_test_cloud_name,
+    load_schema_file,
+    parse_test_definition,
+    read_and_validate_release_test_collection,
+    validate_cluster_compute,
+    validate_test,
+)
+from ray_release.exception import ReleaseTestConfigError
+from ray_release.test import Test
+
+_TEST_COLLECTION_FILES = [
+    "release/release_tests.yaml",
+    "release/release_data_tests.yaml",
+    "release/release_multimodal_inference_benchmarks_tests.yaml",
+    "release/ray_release/tests/test_collection_data.yaml",
+]
+
+VALID_TEST = {
+    "name": "validation_test",
+    "group": "validation_group",
+    "working_dir": "validation_dir",
+    "python": "3.10",
+    "frequency": "nightly",
+    "team": "release",
+    "cluster": {
+        "byod": {"type": "gpu"},
+        "cluster_compute": "tpl_cpu_small.yaml",
+        "autosuspend_mins": 10,
+    },
+    "run": {
+        "timeout": 100,
+        "script": "python validate.py",
+        "wait_for_nodes": {"num_nodes": 2, "timeout": 100},
+        "type": "client",
+    },
+    "smoke_test": {"run": {"timeout": 20}, "frequency": "nightly"},
+    "alert": "default",
+}
+
+
+def test_parse_test_definition():
+    """
+    Unit test for the ray_release.config.parse_test_definition function. In particular,
+    we check that the code correctly parse a test definition that have the 'variations'
+    field.
+    """
+    test_definitions = yaml.safe_load(
+        """
+        - name: sample_test
+          working_dir: sample_dir
+          frequency: nightly
+          team: sample
+          cluster:
+            byod:
+              type: gpu
+            cluster_compute: compute.yaml
+          run:
+            timeout: 100
+            script: python script.py
+          variations:
+            - __suffix__: aws
+            - __suffix__: gce
+              cluster:
+                cluster_compute: compute_gce.yaml
+    """
+    )
+    # Check that parsing returns two tests, one for each variation (aws and gce). Check
+    # that both tests are valid, and their fields are populated correctly
+    tests = parse_test_definition(test_definitions)
+    aws_test = tests[0]
+    gce_test = tests[1]
+    schema = load_schema_file()
+    assert not validate_test(aws_test, schema)
+    assert not validate_test(gce_test, schema)
+    assert aws_test["name"] == "sample_test.aws"
+    assert gce_test["cluster"]["cluster_compute"] == "compute_gce.yaml"
+    assert gce_test["cluster"]["byod"]["type"] == "gpu"
+    invalid_test_definition = test_definitions[0]
+    # Intentionally make the test definition invalid by create an empty 'variations'
+    # field. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = []
+    with pytest.raises(ReleaseTestConfigError):
+        parse_test_definition([invalid_test_definition])
+    # Intentionally make the test definition invalid by making one 'variation' entry
+    # missing the __suffix__ entry. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = [{"__suffix__": "aws"}, {}]
+    with pytest.raises(ReleaseTestConfigError):
+        parse_test_definition([invalid_test_definition])
+
+
+def test_parse_test_definition_with_python_version():
+    """
+    Unit test for the ray_release.config.parse_test_definition function. In particular,
+    we check that the code correctly parse a test definition that have the 'variations' & 'python'
+    field.
+    """
+    test_definitions = yaml.safe_load(
+        """
+        - name: sample_test
+          working_dir: sample_dir
+          frequency: nightly
+          team: sample
+          python: "3.10"
+          cluster:
+            byod:
+              type: gpu
+            cluster_compute: compute.yaml
+          run:
+            timeout: 100
+            script: python script.py
+          variations:
+            - __suffix__: aws
+            - __suffix__: gce
+              cluster:
+                cluster_compute: compute_gce.yaml
+    """
+    )
+    # Check that parsing returns two tests, one for each variation (aws and gce). Check
+    # that both tests are valid, and their fields are populated correctly
+    tests = parse_test_definition(test_definitions)
+    aws_test = tests[0]
+    gce_test = tests[1]
+    schema = load_schema_file()
+    assert not validate_test(aws_test, schema)
+    assert not validate_test(gce_test, schema)
+    assert aws_test["name"] == "sample_test.aws"
+    assert gce_test["cluster"]["cluster_compute"] == "compute_gce.yaml"
+    assert gce_test["cluster"]["byod"]["type"] == "gpu"
+    invalid_test_definition = test_definitions[0]
+    # Intentionally make the test definition invalid by create an empty 'variations'
+    # field. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = []
+    with pytest.raises(ReleaseTestConfigError):
+        parse_test_definition([invalid_test_definition])
+    # Intentionally make the test definition invalid by making one 'variation' entry
+    # missing the __suffix__ entry. Check that the parser throws exception at runtime
+    invalid_test_definition["variations"] = [{"__suffix__": "aws"}, {}]
+    with pytest.raises(ReleaseTestConfigError):
+        parse_test_definition([invalid_test_definition])
+
+
+def test_parse_test_definition_with_defaults():
+    test_definitions = yaml.safe_load(
+        """
+        - name: DEFAULTS
+          working_dir: default_working_dir
+        - name: sample_test_with_default_working_dir
+          frequency: nightly
+          team: sample
+          cluster:
+            byod:
+              type: gpu
+            cluster_compute: compute.yaml
+          run:
+            timeout: 100
+            script: python script.py
+        - name: sample_test_with_overridden_working_dir
+          working_dir: overridden_working_dir
+          frequency: nightly
+          team: sample
+          cluster:
+            byod:
+              type: gpu
+            cluster_compute: compute.yaml
+          run:
+            timeout: 100
+            script: python script.py
+    """
+    )
+    test_with_default, test_with_override = parse_test_definition(test_definitions)
+    schema = load_schema_file()
+    assert not validate_test(test_with_default, schema)
+    assert not validate_test(test_with_override, schema)
+    assert test_with_default["working_dir"] == "default_working_dir"
+    assert test_with_override["working_dir"] == "overridden_working_dir"
+
+
+def test_parse_test_definition_with_matrix_and_variations_raises():
+    # Matrix and variations are mutually exclusive.
+    test_definitions = yaml.safe_load(
+        """
+        - name: test
+          frequency: nightly
+          team: team
+          working_dir: sample_dir
+          cluster:
+            byod:
+              type: gpu
+            cluster_compute: "{{os}}.yaml"
+          matrix:
+            setup:
+              os: [windows, linux]
+          run:
+            timeout: 100
+            script: python script.py
+          variations:
+            - __suffix__: amd64
+            - __suffix__: arm64
+    """
+    )
+    with pytest.raises(ReleaseTestConfigError):
+        parse_test_definition(test_definitions)
+
+
+def test_parse_test_definition_with_matrix_and_adjustments():
+    test_definitions = yaml.safe_load(
+        """
+        - name: "test-{{compute}}-{{arg}}"
+          matrix:
+            setup:
+              compute: [fixed, autoscaling]
+              arg: [0, 1]
+            adjustments:
+                - with:
+                    # Only run arg 2 with fixed compute
+                    compute: fixed
+                    arg: 2
+          frequency: nightly
+          team: team
+          working_dir: sample_dir
+          cluster:
+            byod:
+              type: gpu
+              runtime_env:
+                - SCALING_MODE={{compute}}
+            cluster_compute: "{{compute}}.yaml"
+          run:
+            timeout: 100
+            script: python script.py --arg "{{arg}}"
+    """
+    )
+    tests = parse_test_definition(test_definitions)
+    schema = load_schema_file()
+
+    assert len(tests) == 5  # 4 from matrix, 1 from adjustments
+    assert not any(validate_test(test, schema) for test in tests)
+    for i, (compute, arg) in enumerate(
+        [
+            ("fixed", 0),
+            ("fixed", 1),
+            ("autoscaling", 0),
+            ("autoscaling", 1),
+            ("fixed", 2),
+        ]
+    ):
+        assert tests[i]["name"] == f"test-{compute}-{arg}"
+        assert tests[i]["cluster"]["cluster_compute"] == f"{compute}.yaml"
+        assert tests[i]["cluster"]["byod"]["runtime_env"] == [f"SCALING_MODE={compute}"]
+
+
+class TestSubstituteVariable:
+    def test_does_not_mutate_original(self):
+        test_definition = {"name": "test-{{arg}}"}
+
+        substituted = _substitute_variable(test_definition, "arg", "1")
+
+        assert substituted is not test_definition
+        assert test_definition == {"name": "test-{{arg}}"}
+
+    def test_substitute_variable_in_string(self):
+        test_definition = {"name": "test-{{arg}}"}
+
+        substituted = _substitute_variable(test_definition, "arg", "1")
+
+        assert substituted == {"name": "test-1"}
+
+    def test_substitute_variable_in_list(self):
+        test_definition = {"items": ["item-{{arg}}"]}
+
+        substituted = _substitute_variable(test_definition, "arg", "1")
+
+        assert substituted == {"items": ["item-1"]}
+
+    def test_substitute_variable_in_dict(self):
+        test_definition = {"outer": {"inner": "item-{{arg}}"}}
+
+        substituted = _substitute_variable(test_definition, "arg", "1")
+
+        assert substituted == {"outer": {"inner": "item-1"}}
+
+
+def test_schema_validation():
+    test = VALID_TEST.copy()
+
+    schema = load_schema_file()
+
+    assert not validate_test(Test(**test), schema)
+
+    # Remove some optional arguments
+    del test["alert"]
+    del test["python"]
+    del test["run"]["wait_for_nodes"]
+    del test["cluster"]["autosuspend_mins"]
+
+    assert not validate_test(Test(**test), schema)
+
+    # Add some faulty arguments
+
+    # Faulty frequency
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["frequency"] = "invalid"
+
+    assert validate_test(invalid_test, schema)
+
+    # Faulty job type
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["run"]["type"] = "invalid"
+
+    assert validate_test(invalid_test, schema)
+
+    # Faulty file manager type
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["run"]["file_manager"] = "invalid"
+
+    assert validate_test(invalid_test, schema)
+
+    # Faulty smoke test
+
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    del invalid_test["smoke_test"]["frequency"]
+
+    assert validate_test(invalid_test, schema)
+
+    # Faulty Python version
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["python"] = "invalid"
+
+    assert validate_test(invalid_test, schema)
+
+    # Faulty BYOD type
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["cluster"]["byod"]["type"] = "invalid"
+    assert validate_test(invalid_test, schema)
+
+    # Faulty BYOD and Python version match
+    invalid_test = Test(**copy.deepcopy(VALID_TEST))
+    invalid_test["cluster"]["byod"]["type"] = "gpu"
+    invalid_test["python"] = "3.11"
+    assert validate_test(invalid_test, schema)
+
+
+def test_compute_config_invalid_ebs():
+    compute_config = {
+        "aws": {
+            "BlockDeviceMappings": [
+                {
+                    "DeviceName": "/dev/sda1",
+                    "Ebs": {
+                        "VolumeSize": 1000,
+                    },
+                }
+            ]
+        }
+    }
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["aws"]["BlockDeviceMappings"][0]["Ebs"][
+        "DeleteOnTermination"
+    ] = False
+
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["aws"]["BlockDeviceMappings"][0]["Ebs"]["DeleteOnTermination"] = True
+
+    assert not validate_cluster_compute(compute_config)
+
+    compute_config["head_node_type"] = {}
+    compute_config["head_node_type"]["aws_advanced_configurations"] = {
+        "BlockDeviceMappings": [
+            {
+                "DeviceName": "/dev/sda1",
+                "Ebs": {
+                    "VolumeSize": 1000,
+                },
+            }
+        ]
+    }
+
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["head_node_type"]["aws_advanced_configurations"][
+        "BlockDeviceMappings"
+    ][0]["Ebs"]["DeleteOnTermination"] = False
+
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["head_node_type"]["aws_advanced_configurations"][
+        "BlockDeviceMappings"
+    ][0]["Ebs"]["DeleteOnTermination"] = True
+
+    assert not validate_cluster_compute(compute_config)
+
+    compute_config["worker_node_types"] = [{}]
+    compute_config["worker_node_types"][0]["aws_advanced_configurations"] = {
+        "BlockDeviceMappings": [
+            {
+                "DeviceName": "/dev/sda1",
+                "Ebs": {
+                    "VolumeSize": 1000,
+                },
+            }
+        ]
+    }
+
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["worker_node_types"][0]["aws_advanced_configurations"][
+        "BlockDeviceMappings"
+    ][0]["Ebs"]["DeleteOnTermination"] = False
+
+    assert validate_cluster_compute(compute_config)
+
+    compute_config["worker_node_types"][0]["aws_advanced_configurations"][
+        "BlockDeviceMappings"
+    ][0]["Ebs"]["DeleteOnTermination"] = True
+
+    assert not validate_cluster_compute(compute_config)
+
+
+def test_validate_cluster_compute_new_schema_valid():
+    """New schema: empty config is valid (all fields optional)."""
+    assert not validate_cluster_compute({}, is_new_schema=True)
+
+
+def test_validate_cluster_compute_new_schema_valid_with_fields():
+    """New schema: config with new-schema keys is valid."""
+    compute_config = {
+        "cloud": "my_cloud",
+        "head_node": {"instance_type": "m5.4xlarge"},
+        "worker_nodes": [{"instance_type": "m5.xlarge", "min_nodes": 1}],
+    }
+    assert not validate_cluster_compute(compute_config, is_new_schema=True)
+
+
+def test_validate_cluster_compute_new_schema_rejects_legacy_keys():
+    """New schema: config with legacy keys is rejected."""
+    compute_config = {
+        "cloud_id": "cld_123",
+        "head_node_type": {"instance_type": "m5.4xlarge"},
+    }
+    error = validate_cluster_compute(compute_config, is_new_schema=True)
+    assert error is not None
+    assert "legacy schema keys" in error
+    assert "anyscale_sdk_2026=true" in error
+
+
+def test_validate_cluster_compute_legacy_rejects_new_keys():
+    """Legacy schema: config with new-schema keys is rejected."""
+    compute_config = {
+        "cloud_id": "cld_123",
+        "head_node": {"instance_type": "m5.4xlarge"},
+    }
+    error = validate_cluster_compute(compute_config, is_new_schema=False)
+    assert error is not None
+    assert "new schema keys" in error
+    assert "anyscale_sdk_2026=false" in error
+
+
+def test_validate_cluster_compute_legacy_rejects_empty():
+    """Legacy schema: empty config is rejected (no legacy keys)."""
+    error = validate_cluster_compute({}, is_new_schema=False)
+    assert error is not None
+    assert "does not have legacy schema keys" in error
+
+
+def test_validate_cluster_compute_new_schema_ebs_top_level():
+    """New schema: EBS DeleteOnTermination is checked in top-level advanced_instance_config."""
+    compute_config = {
+        "advanced_instance_config": {
+            "BlockDeviceMappings": [
+                {
+                    "DeviceName": "/dev/sda1",
+                    "Ebs": {"VolumeSize": 1000},
+                }
+            ]
+        },
+    }
+    # Missing DeleteOnTermination should fail
+    assert validate_cluster_compute(compute_config, is_new_schema=True)
+
+    # Set DeleteOnTermination to True should pass
+    compute_config["advanced_instance_config"]["BlockDeviceMappings"][0]["Ebs"][
+        "DeleteOnTermination"
+    ] = True
+    assert not validate_cluster_compute(compute_config, is_new_schema=True)
+
+
+def test_validate_cluster_compute_new_schema_ebs_head_node():
+    """New schema: EBS DeleteOnTermination is checked in head_node.advanced_instance_config."""
+    compute_config = {
+        "head_node": {
+            "instance_type": "m5.4xlarge",
+            "advanced_instance_config": {
+                "BlockDeviceMappings": [
+                    {
+                        "DeviceName": "/dev/sda1",
+                        "Ebs": {"VolumeSize": 1000},
+                    }
+                ]
+            },
+        },
+    }
+    # Missing DeleteOnTermination should fail
+    assert validate_cluster_compute(compute_config, is_new_schema=True)
+
+    # Set DeleteOnTermination to True should pass
+    compute_config["head_node"]["advanced_instance_config"]["BlockDeviceMappings"][0][
+        "Ebs"
+    ]["DeleteOnTermination"] = True
+    assert not validate_cluster_compute(compute_config, is_new_schema=True)
+
+
+def test_validate_cluster_compute_new_schema_ebs_worker_nodes():
+    """New schema: EBS checks in worker_nodes[*].advanced_instance_config."""
+    compute_config = {
+        "worker_nodes": [
+            {
+                "instance_type": "m5.xlarge",
+                "advanced_instance_config": {
+                    "BlockDeviceMappings": [
+                        {
+                            "DeviceName": "/dev/sda1",
+                            "Ebs": {"VolumeSize": 500},
+                        }
+                    ]
+                },
+            }
+        ],
+    }
+    assert validate_cluster_compute(compute_config, is_new_schema=True)
+
+    compute_config["worker_nodes"][0]["advanced_instance_config"][
+        "BlockDeviceMappings"
+    ][0]["Ebs"]["DeleteOnTermination"] = True
+    assert not validate_cluster_compute(compute_config, is_new_schema=True)
+
+
+def test_get_test_cloud_name_from_cluster_cloud():
+    """get_test_cloud_name() returns cluster.cloud when set."""
+    test = Test(
+        {
+            "name": "test",
+            "cluster": {"cluster_compute": "tpl.yaml", "cloud": "my_cloud"},
+        }
+    )
+    assert get_test_cloud_name(test) == "my_cloud"
+
+
+def test_get_test_cloud_name_from_cloud_id_mapping():
+    """get_test_cloud_name() falls back to CLOUD_ID_TO_NAME mapping."""
+    for cloud_id, expected_name in CLOUD_ID_TO_NAME.items():
+        test = Test(
+            {
+                "name": "test",
+                "cluster": {"cluster_compute": "tpl.yaml", "cloud_id": cloud_id},
+            }
+        )
+        assert get_test_cloud_name(test) == expected_name
+
+
+def test_get_test_cloud_name_unknown_cloud_id():
+    """get_test_cloud_name() raises ReleaseTestConfigError for unknown cloud_id."""
+    test = Test(
+        {
+            "name": "test",
+            "cluster": {"cluster_compute": "tpl.yaml", "cloud_id": "cld_unknown"},
+        }
+    )
+    with pytest.raises(ReleaseTestConfigError):
+        get_test_cloud_name(test)
+
+
+def test_load_and_validate_test_collection_file():
+    tests = read_and_validate_release_test_collection(_TEST_COLLECTION_FILES)
+    assert [test for test in tests if test.get_name() == "test_name"]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_custom_byod_build.py

@@ -0,0 +1,149 @@
+import sys
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from ray_release.scripts.custom_byod_build import main
+
+
+@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image")
+def test_custom_byod_build(mock_build_anyscale_custom_byod_image):
+    mock_build_anyscale_custom_byod_image.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--base-image",
+            "test-base-image",
+            "--post-build-script",
+            "test_post_build_script.sh",
+            "--python-depset",
+            "python_depset.lock",
+        ],
+    )
+    assert result.exit_code == 0
+
+
+@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image")
+def test_custom_byod_build_without_lock_file(
+    mock_build_anyscale_custom_byod_image,
+):
+    mock_build_anyscale_custom_byod_image.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--base-image",
+            "test-base-image",
+            "--post-build-script",
+            "test_post_build_script.sh",
+        ],
+    )
+    assert result.exit_code == 0
+
+
+@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image")
+def test_custom_byod_build_missing_arg(mock_build_anyscale_custom_byod_image):
+    mock_build_anyscale_custom_byod_image.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--base-image",
+            "test-base-image",
+            "--post-build-script",
+            "test_post_build_script.sh",
+        ],
+    )
+    assert result.exit_code == 2
+    assert "Error: Missing option '--image-name'" in result.output
+
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--post-build-script",
+            "test_post_build_script.sh",
+        ],
+    )
+    assert result.exit_code == 2
+    assert "Error: Missing option '--base-image'" in result.output
+
+    result = runner.invoke(
+        main, ["--image-name", "test-image", "--base-image", "test-base-image"]
+    )
+    assert result.exit_code == 2
+    assert (
+        "At least one of post_build_script, python_depset, or env must be provided"
+        in result.output
+    )
+
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--base-image",
+            "test-base-image",
+            "--python-depset",
+            "python_depset.lock",
+        ],
+    )
+    assert result.exit_code == 0
+
+
+@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image")
+def test_custom_byod_build_with_env(mock_build_anyscale_custom_byod_image):
+    mock_build_anyscale_custom_byod_image.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--base-image",
+            "test-base-image",
+            "--env",
+            "FOO=bar",
+            "--env",
+            "BAZ=qux",
+        ],
+    )
+    assert result.exit_code == 0
+    build_context = mock_build_anyscale_custom_byod_image.call_args[0][2]
+    assert build_context["envs"] == {"FOO": "bar", "BAZ": "qux"}
+    assert "post_build_script" not in build_context
+    assert "python_depset" not in build_context
+
+
+@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image")
+def test_custom_byod_build_with_env_and_script(mock_build_anyscale_custom_byod_image):
+    mock_build_anyscale_custom_byod_image.return_value = None
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--image-name",
+            "test-image",
+            "--base-image",
+            "test-base-image",
+            "--post-build-script",
+            "test_post_build_script.sh",
+            "--env",
+            "KEY=value",
+        ],
+    )
+    assert result.exit_code == 0
+    build_context = mock_build_anyscale_custom_byod_image.call_args[0][2]
+    assert build_context["envs"] == {"KEY": "value"}
+    assert build_context["post_build_script"] == "test_post_build_script.sh"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_custom_byod_build_init_helper.py

@@ -0,0 +1,347 @@
+import os
+import sys
+import tempfile
+from pathlib import Path
+from unittest import mock
+
+import pytest
+import yaml
+
+from ray_release.bazel import bazel_runfile
+from ray_release.configs.global_config import get_global_config, init_global_config
+from ray_release.custom_byod_build_init_helper import (
+    _get_step_name,
+    _short_tag,
+    build_short_gpu_map,
+    create_custom_build_yaml,
+    generate_custom_build_step_key,
+    get_prerequisite_step,
+)
+from ray_release.test import Test
+
+# AZURE_REGISTRY_NAME import temporarily commented out: the Azure ACR login
+# step it was used to verify is disabled due to CI issues.
+# from ray_release.util import AZURE_REGISTRY_NAME
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+_GPU_MAP = build_short_gpu_map(bazel_runfile("ray-images.json"))
+
+
+@mock.patch.dict(os.environ, {"RAY_WANT_COMMIT_IN_IMAGE": "abc123"})
+@mock.patch("ray_release.custom_byod_build_init_helper.get_images_from_tests")
+def test_create_custom_build_yaml(mock_get_images_from_tests):
+    config = get_global_config()
+    custom_byod_images = [
+        (
+            "ray-project/ray-ml:abc123-custom-123456789abc123456789",
+            "ray-project/ray-ml:abc123-base",
+            "custom_script.sh",
+            None,
+            None,
+        ),
+        (
+            "ray-project/ray-ml:abc123-custom1",
+            "ray-project/ray-ml:abc123-base",
+            "",
+            None,
+            None,
+        ),
+        (
+            "ray-project/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc123456789",
+            "ray-project/ray-ml:abc123-py37-cpu-base",
+            "custom_script.sh",
+            None,
+            None,
+        ),  # longer than 40 chars
+        (
+            "ray-project/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc987654321",
+            "ray-project/ray-ml:abc123-py37-cpu-base",
+            "custom_script.sh",
+            "python_depset.lock",
+            None,
+        ),
+        (
+            "custom_ecr/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc987654321",
+            "anyscale/ray:2.50.0-py37-cpu",
+            "custom_script.sh",
+            "python_depset.lock",
+            None,
+        ),
+        (
+            "ray-project/ray-ml:abc123-envonly-abcdef123456789abc000000000",
+            "ray-project/ray-ml:abc123-base",
+            None,
+            None,
+            {"MY_ENV": "my_value", "OTHER_ENV": "other_value"},
+        ),
+    ]
+    custom_image_test_names_map = {
+        "ray-project/ray-ml:abc123-custom-123456789abc123456789": ["test_1"],
+        "ray-project/ray-ml:abc123-custom1": ["test_2"],
+        "ray-project/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc123456789": [
+            "test_1",
+            "test_2",
+        ],
+        "ray-project/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc987654321": [
+            "test_1",
+            "test_2",
+        ],
+        "custom_ecr/ray-ml:abc123-py37-cpu-custom-abcdef123456789abc987654321": [
+            "test_3",
+        ],
+        "ray-project/ray-ml:abc123-envonly-abcdef123456789abc000000000": [
+            "test_4",
+        ],
+    }
+    mock_get_images_from_tests.return_value = (
+        custom_byod_images,
+        custom_image_test_names_map,
+    )
+    step_keys = [
+        generate_custom_build_step_key(image)
+        for image, _, _, _, _ in custom_byod_images
+    ]
+    # List of dummy tests
+    tests = [
+        Test(
+            name="test_1",
+            frequency="manual",
+            group="test_group",
+            team="test_team",
+            working_dir="test_working_dir",
+        ),
+        Test(
+            name="test_2",
+            frequency="manual",
+            group="test_group",
+            team="test_team",
+            working_dir="test_working_dir",
+        ),
+        Test(
+            name="test_3",
+            frequency="manual",
+            group="test_group",
+            team="test_team",
+            working_dir="test_working_dir",
+            cluster={
+                "ray_version": "2.50.0",
+            },
+        ),
+        Test(
+            name="test_4",
+            frequency="manual",
+            group="test_group",
+            team="test_team",
+            working_dir="test_working_dir",
+        ),
+    ]
+    with tempfile.TemporaryDirectory() as tmpdir:
+        create_custom_build_yaml(
+            os.path.join(tmpdir, "custom_byod_build.rayci.yml"),
+            tests,
+            _GPU_MAP,
+        )
+        with open(os.path.join(tmpdir, "custom_byod_build.rayci.yml"), "r") as f:
+            content = yaml.safe_load(f)
+            assert content["group"] == "Custom images build"
+            assert len(content["steps"]) == 5
+            assert (
+                content["steps"][0]["label"]
+                == f":tapioca: build custom: ray-ml:custom ({step_keys[0]}) test_1"
+            )
+            assert (
+                content["steps"][1]["label"]
+                == f":tapioca: build custom: ray-ml:py37-cpu-custom ({step_keys[2]}) test_1 test_2"
+            )
+            assert (
+                content["steps"][2]["label"]
+                == f":tapioca: build custom: ray-ml:py37-cpu-custom ({step_keys[3]}) test_1 test_2"
+            )
+            assert (
+                "export RAY_WANT_COMMIT_IN_IMAGE=abc123"
+                in content["steps"][0]["commands"][0]
+            )
+            assert (
+                content["steps"][0]["commands"][1]
+                == f"bash release/gcloud_docker_login.sh {config['aws2gce_credentials']}"
+            )
+            # Azure ACR login step is replaced by a placeholder echo while
+            # Azure image push is disabled due to CI issues. Verify the
+            # placeholder is present at the expected index instead of an
+            # `az acr login --name <AZURE_REGISTRY_NAME>` command.
+            assert "Skipping Azure ACR login" in content["steps"][0]["commands"][3]
+            assert (
+                f"--region {config['byod_ecr_region']}"
+                in content["steps"][0]["commands"][4]
+            )
+            assert f"{config['byod_ecr']}" in content["steps"][0]["commands"][4]
+            assert (
+                f"--image-name {custom_byod_images[0][0]}"
+                in content["steps"][0]["commands"][5]
+            )
+            assert (
+                f"--image-name {custom_byod_images[2][0]}"
+                in content["steps"][1]["commands"][5]
+            )
+            assert (
+                f"--image-name {custom_byod_images[3][0]}"
+                in content["steps"][2]["commands"][5]
+            )
+            assert content["steps"][3]["depends_on"] == "forge"
+            # Verify env-only image step has --env flags. Command index
+            # decremented from 6 to 5 because the Azure ACR login command was
+            # collapsed into a single placeholder echo while Azure image push
+            # is disabled due to CI issues.
+            env_only_cmd = content["steps"][4]["commands"][5]
+            assert f"--image-name {custom_byod_images[5][0]}" in env_only_cmd
+            assert "--env MY_ENV=my_value" in env_only_cmd
+            assert "--env OTHER_ENV=other_value" in env_only_cmd
+            # Env-only step should not have --post-build-script or --python-depset
+            assert "--post-build-script" not in env_only_cmd
+            assert "--python-depset" not in env_only_cmd
+
+
+_ECR = get_global_config()["byod_ecr"]
+
+
+@pytest.mark.parametrize(
+    ("image", "base_image", "expected"),
+    [
+        # Anyscale-prefixed base images → "forge"
+        ("anyscale/ray:abc123-custom", "anyscale/ray:abc123-base", "forge"),
+        ("custom_ecr/ray-ml:abc123-custom", "anyscale/ray:2.50.0-py310-cpu", "forge"),
+        # ray-ml: python dimension only
+        (
+            "ray-project/ray-ml:a-c",
+            "ray-project/ray-ml:a-py310-gpu",
+            "anyscalemlbuild--python310",
+        ),
+        (
+            "ray-project/ray-ml:a-c",
+            "ray-project/ray-ml:a-py311-gpu",
+            "anyscalemlbuild--python311",
+        ),
+        # ray cpu → anyscalecpubuild (python only)
+        (
+            "ray-project/ray:a-c",
+            "ray-project/ray:a-py310-cpu",
+            "anyscalecpubuild--python310",
+        ),
+        (
+            "ray-project/ray:a-c",
+            "ray-project/ray:a-py313-cpu",
+            "anyscalecpubuild--python313",
+        ),
+        # ray cuda → anyscalecudabuild (platform + python)
+        (
+            "ray-project/ray:a-c",
+            "ray-project/ray:a-py311-cu123",
+            "anyscalecudabuild--gpucu1232cudnn9-python311",
+        ),
+        (
+            "ray-project/ray:a-c",
+            "ray-project/ray:a-py312-cu130",
+            "anyscalecudabuild--gpucu1300cudnn-python312",
+        ),
+        # ray-llm
+        (
+            "ray-project/ray-llm:a-c",
+            "ray-project/ray-llm:a-py311-cu128",
+            "anyscalellmbuild--gpucu1281cudnn-python311",
+        ),
+        (
+            "ray-project/ray-llm:a-c",
+            "ray-project/ray-llm:a-py312-cu130",
+            "anyscalellmbuild--gpucu1300cudnn-python312",
+        ),
+        # ECR-prefixed base_image (production format)
+        (
+            f"{_ECR}/anyscale/ray:bid-py310-cpu-hash",
+            f"{_ECR}/anyscale/ray:bid-py310-cpu",
+            "anyscalecpubuild--python310",
+        ),
+        # Build ID with dashes
+        (
+            "ray-project/ray:a-b-c-x",
+            "ray-project/ray:a-b-c-py312-cpu",
+            "anyscalecpubuild--python312",
+        ),
+        # Unknown platform falls back to raw value
+        (
+            "ray-project/ray:a-c",
+            "ray-project/ray:a-py310-cu999",
+            "anyscalecudabuild--gpucu999-python310",
+        ),
+        # No python version → bare config key (cuda is default for ray without suffix)
+        ("ray-project/ray:a-c", "ray-project/ray:a-base", "anyscalecudabuild"),
+        ("ray-project/ray-ml:a-c", "ray-project/ray-ml:a-base", "anyscalemlbuild"),
+    ],
+    ids=[
+        "forge-bare",
+        "forge-versioned",
+        "ray-ml-py310",
+        "ray-ml-py311",
+        "ray-cpu-py310",
+        "ray-cpu-py313",
+        "ray-cuda-cu123",
+        "ray-cuda-cu130",
+        "ray-llm-cu128",
+        "ray-llm-cu130",
+        "ecr-prefix",
+        "dashed-build-id",
+        "unknown-platform",
+        "no-python-ray",
+        "no-python-ray-ml",
+    ],
+)
+def test_get_prerequisite_step(image, base_image, expected):
+    assert get_prerequisite_step(image, base_image, _GPU_MAP) == expected
+
+
+@pytest.mark.parametrize(
+    "gpu, expected",
+    [
+        ("cpu", "cpu"),
+        ("tpu", "tpu"),
+        ("cu12.3.2-cudnn9", "cu123"),
+        ("cu11.8.0-cudnn8", "cu118"),
+        ("cu12-cudnn9", "cu12"),
+    ],
+)
+def test_short_tag(gpu, expected):
+    assert _short_tag(gpu) == expected
+
+
+def test_short_gpu_map_built_from_ray_images_json():
+    """The map built from ray-images.json has no short-tag collisions."""
+    ray_images_path = str(Path(bazel_runfile("ray-images.json")))
+    gpu_map = build_short_gpu_map(ray_images_path)
+    assert "cpu" in gpu_map
+    # Verify at least one CUDA gpu is present.
+    cuda_entries = {k: v for k, v in gpu_map.items() if k.startswith("cu")}
+    assert len(cuda_entries) > 0
+    # Verify round-trip: _short_tag(full) maps back to the same key.
+    for short, full in gpu_map.items():
+        assert (
+            _short_tag(full) == short
+        ), f"_short_tag({full!r}) = {_short_tag(full)!r}, expected {short!r}"
+
+
+def test_get_step_name():
+    test_names = [
+        "test_1",
+        "test_2",
+        "test_3",
+    ]
+    assert (
+        _get_step_name(
+            "ray-project/ray-ml:a1b2c3d4-py39-cpu-abcdef123456789abc123456789",
+            "abc123",
+            test_names,
+        )
+        == ":tapioca: build custom: ray-ml:py39-cpu (abc123) test_1 test_2"
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_custom_image_build_and_test_init.py

@@ -0,0 +1,489 @@
+import json
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ray_release.buildkite.filter import filter_tests
+from ray_release.buildkite.settings import get_frequency, get_test_filters
+from ray_release.config import read_and_validate_release_test_collection
+from ray_release.configs.global_config import init_global_config
+from ray_release.custom_byod_build_init_helper import (
+    build_short_gpu_map,
+    collect_rayci_select_keys,
+)
+from ray_release.scripts.custom_image_build_and_test_init import (
+    _split_into_batches,
+    main,
+)
+
+_bazel_workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "")
+
+
+def _expected_rayci_select_keys(sample_yaml: str) -> set:
+    """Mirror the script's filter + key computation so expectations can't drift."""
+    init_global_config(
+        os.path.join(
+            _bazel_workspace_dir,
+            "release/ray_release/configs/oss_config.yaml",
+        )
+    )
+    gpu_map = build_short_gpu_map(os.path.join(_bazel_workspace_dir, "ray-images.json"))
+    test_collection = read_and_validate_release_test_collection(
+        [os.path.join(_bazel_workspace_dir, sample_yaml)]
+    )
+    filtered = filter_tests(
+        test_collection,
+        frequency=get_frequency("nightly"),
+        test_filters=get_test_filters("prefix:hello_world"),
+        run_jailed_tests=True,
+        run_unstable_tests=True,
+    )
+    tests = [test for test, _ in filtered]
+    return collect_rayci_select_keys(tests, gpu_map)
+
+
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    runner = CliRunner()
+    custom_build_jobs_output_file = "custom_build_jobs.yaml"
+    test_jobs_output_file = "test_jobs.json"
+    rayci_select_output_file = "rayci_select.txt"
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            custom_build_jobs_output_file,
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+            "--rayci-select-output-file",
+            rayci_select_output_file,
+        ],
+        catch_exceptions=False,
+    )
+    with open(
+        os.path.join(_bazel_workspace_dir, custom_build_jobs_output_file), "r"
+    ) as f:
+        custom_build_jobs = yaml.safe_load(f)
+        assert len(custom_build_jobs["steps"]) == 1  # 1 custom build job
+    with open(
+        os.path.join(
+            _bazel_workspace_dir, f"{os.path.splitext(test_jobs_output_file)[0]}_0.json"
+        ),
+        "r",
+    ) as f:
+        test_jobs = json.load(f)
+        assert len(test_jobs) == 1  # 1 group
+        assert len(test_jobs[0]["steps"]) == 2  # 2 tests
+        assert test_jobs[0]["steps"][0]["label"].startswith("hello_world.aws")
+        assert test_jobs[0]["steps"][1]["label"].startswith("hello_world_custom.aws")
+
+    with open(os.path.join(_bazel_workspace_dir, rayci_select_output_file), "r") as f:
+        raw = f.read()
+        keys = [k for k in raw.split(",") if k]
+        expected = _expected_rayci_select_keys(
+            "release/ray_release/tests/sample_tests.yaml"
+        )
+        assert set(keys) == expected
+        assert len(keys) == len(set(keys))  # no duplicates
+        assert keys == sorted(keys)  # stable ordering
+        assert len(expected) == 3  # cpu base + gpu base + custom-BYOD
+
+    assert result.exit_code == 0
+
+
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init_with_block_step(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    num_tests_expected = 5
+    runner = CliRunner()
+    custom_build_jobs_output_file = "custom_build_jobs.yaml"
+    test_jobs_output_file = "test_jobs.json"
+    rayci_select_output_file = "rayci_select.txt"
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_5_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            custom_build_jobs_output_file,
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+            "--rayci-select-output-file",
+            rayci_select_output_file,
+            "--selection-block-threshold",
+            "5",
+        ],
+        catch_exceptions=False,
+    )
+    with open(
+        os.path.join(_bazel_workspace_dir, custom_build_jobs_output_file), "r"
+    ) as f:
+        custom_build_jobs = yaml.safe_load(f)
+        assert len(custom_build_jobs["steps"]) == 1  # 1 custom build job
+    with open(
+        os.path.join(
+            _bazel_workspace_dir, f"{os.path.splitext(test_jobs_output_file)[0]}_0.json"
+        ),
+        "r",
+    ) as f:
+        test_jobs = json.load(f)
+        print(test_jobs)
+        assert len(test_jobs) == 2  # 2 groups: block and hello_world
+        assert len(test_jobs[0]["steps"]) == 1  # 1 block step
+        assert test_jobs[0]["steps"][0]["block"] == "Run release tests"
+        assert test_jobs[0]["steps"][0]["key"] == "block_run_release_tests"
+        assert (
+            test_jobs[0]["steps"][0]["prompt"]
+            == f"You are triggering {num_tests_expected} tests. Do you want to proceed?"
+        )
+        assert len(test_jobs[1]["steps"]) == num_tests_expected  # 5 tests
+        assert test_jobs[1]["steps"][0]["label"].startswith("hello_world.aws")
+        assert test_jobs[1]["steps"][1]["label"].startswith("hello_world_custom.aws")
+
+    with open(os.path.join(_bazel_workspace_dir, rayci_select_output_file), "r") as f:
+        raw = f.read()
+        keys = [k for k in raw.split(",") if k]
+        expected = _expected_rayci_select_keys(
+            "release/ray_release/tests/sample_5_tests.yaml"
+        )
+        assert set(keys) == expected
+        assert len(keys) == len(set(keys))  # no duplicates
+        assert keys == sorted(keys)  # stable ordering
+        assert len(expected) == 3  # cpu base + gpu base + custom-BYOD (collapsed)
+
+    assert result.exit_code == 0
+
+
+@patch.dict("os.environ", {"AUTOMATIC": "1"})
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init_without_block_step_automatic(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    num_tests_expected = 5
+    runner = CliRunner()
+    custom_build_jobs_output_file = "custom_build_jobs.yaml"
+    test_jobs_output_file = "test_jobs.json"
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_5_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            custom_build_jobs_output_file,
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+        ],
+        catch_exceptions=False,
+    )
+    with open(
+        os.path.join(_bazel_workspace_dir, custom_build_jobs_output_file), "r"
+    ) as f:
+        custom_build_jobs = yaml.safe_load(f)
+        assert len(custom_build_jobs["steps"]) == 1  # 1 custom build job
+    with open(
+        os.path.join(
+            _bazel_workspace_dir, f"{os.path.splitext(test_jobs_output_file)[0]}_0.json"
+        ),
+        "r",
+    ) as f:
+        test_jobs = json.load(f)
+        print(test_jobs)
+        assert len(test_jobs) == 1  # 1 group: hello_world
+        assert len(test_jobs[0]["steps"]) == num_tests_expected  # 5 tests
+        assert test_jobs[0]["steps"][0]["label"].startswith("hello_world.aws")
+        assert test_jobs[0]["steps"][1]["label"].startswith("hello_world_custom.aws")
+
+    assert result.exit_code == 0
+
+
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_rayci_select_skipped_when_no_filter(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    """Unfiltered runs (e.g. full nightly) skip RAYCI_SELECT so rayci runs everything."""
+    runner = CliRunner()
+    custom_build_jobs_output_file = "custom_build_jobs.yaml"
+    test_jobs_output_file = "test_jobs.json"
+    rayci_select_output_file = "rayci_select_no_filter.txt"
+    abs_path = os.path.join(_bazel_workspace_dir, rayci_select_output_file)
+    if os.path.exists(abs_path):
+        os.remove(abs_path)
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--custom-build-jobs-output-file",
+            custom_build_jobs_output_file,
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+            "--rayci-select-output-file",
+            rayci_select_output_file,
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert not os.path.exists(abs_path)
+
+
+def test_split_into_batches_single_batch():
+    """All groups fit under the limit → 1 batch."""
+    steps = [
+        {"group": "a", "steps": [{"label": "1"}, {"label": "2"}]},
+        {"group": "b", "steps": [{"label": "3"}]},
+    ]
+    batches = _split_into_batches(steps, max_jobs=500)
+    assert len(batches) == 1
+    assert batches[0] == steps
+
+
+def test_split_into_batches_multi_batch():
+    """Groups that don't fit together are split across batches."""
+    # Each group counts as 1 + len(steps) jobs (Buildkite counts the group
+    # itself). group a = 1+3 = 4, group b = 1+3 = 4, group c = 1+2 = 3.
+    steps = [
+        {"group": "a", "steps": [{"x": 1}, {"x": 2}, {"x": 3}]},
+        {"group": "b", "steps": [{"x": 4}, {"x": 5}, {"x": 6}]},
+        {"group": "c", "steps": [{"x": 7}, {"x": 8}]},
+    ]
+    batches = _split_into_batches(steps, max_jobs=5)
+    # a alone = 4 jobs, + b (4) would be 8 > 5, so split
+    # batch 0: [a] (4 jobs)
+    # batch 1: [b] (4 jobs)
+    # batch 2: [c] (3 jobs)
+    assert [[g["group"] for g in batch] for batch in batches] == [
+        ["a"],
+        ["b"],
+        ["c"],
+    ]
+
+
+def test_split_into_batches_packs_small_groups():
+    """Small groups are packed into the same batch when they fit."""
+    steps = [
+        {"group": "a", "steps": [{"x": 1}]},  # 2 jobs
+        {"group": "b", "steps": [{"x": 2}]},  # 2 jobs
+        {"group": "c", "steps": [{"x": 3}]},  # 2 jobs
+    ]
+    batches = _split_into_batches(steps, max_jobs=5)
+    # a+b=4 jobs fits; +c=6 > 5, so c goes to batch 1
+    assert [[g["group"] for g in batch] for batch in batches] == [
+        ["a", "b"],
+        ["c"],
+    ]
+
+
+def test_split_into_batches_oversized_group_raises():
+    """A single group exceeding the limit is a user error; raise."""
+    steps = [
+        {"group": "huge", "steps": [{"x": i} for i in range(10)]},  # 11 jobs
+    ]
+    with pytest.raises(ValueError, match="exceeds the limit"):
+        _split_into_batches(steps, max_jobs=5)
+
+
+def test_split_into_batches_counts_parallelism():
+    """A step with parallelism=N counts as N jobs."""
+    steps = [
+        {
+            "group": "a",
+            "steps": [{"parallelism": 4}, {"label": "x"}],
+        },  # 1 + 4 + 1 = 6 jobs
+    ]
+    batches = _split_into_batches(steps, max_jobs=6)
+    assert len(batches) == 1
+    with pytest.raises(ValueError, match="exceeds the limit"):
+        _split_into_batches(steps, max_jobs=5)
+
+
+@patch.dict("os.environ", {"AUTOMATIC": "1"})
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init_writes_indexed_chunk_file(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    """Single-batch runs should still write an indexed chunk file (no un-suffixed path)."""
+    runner = CliRunner()
+    custom_build_jobs_output_file = "custom_build_jobs.yaml"
+    test_jobs_output_file = "test_jobs_indexed.json"
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_5_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            custom_build_jobs_output_file,
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    # Default max (450) is well above 5 tests, so expect exactly one chunk.
+    assert os.path.exists(
+        os.path.join(_bazel_workspace_dir, "test_jobs_indexed_0.json")
+    )
+    # The un-suffixed path must not exist — callers should glob *_N.json.
+    assert not os.path.exists(os.path.join(_bazel_workspace_dir, test_jobs_output_file))
+
+
+@patch.dict("os.environ", {"AUTOMATIC": "1"})
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init_cleans_stale_chunks(
+    mock_update_from_s3, mock_is_jailed_with_open_issue
+):
+    """Chunks from a previous run must be removed before writing new ones.
+
+    Persistent Buildkite checkouts keep the workspace between runs; a stale
+    release_tests_2.json from a prior invocation would otherwise get picked
+    up by the upload-loop and inject outdated steps into the pipeline.
+    """
+    stale_path = os.path.join(_bazel_workspace_dir, "test_jobs_stale_9.json")
+    with open(stale_path, "wt") as fp:
+        json.dump([{"group": "leftover", "steps": [{"command": "echo stale"}]}], fp)
+    assert os.path.exists(stale_path)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_5_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            "custom_build_jobs.yaml",
+            "--test-jobs-output-file",
+            "test_jobs_stale.json",
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    assert not os.path.exists(
+        stale_path
+    ), f"Stale chunk file {stale_path} was not cleaned up"
+
+
+@patch(
+    "ray_release.scripts.custom_image_build_and_test_init.subprocess.run",
+    return_value=None,
+)
+@patch.dict("os.environ", {"AUTOMATIC": "1"})
+@patch.dict("os.environ", {"BUILDKITE": "1"})
+@patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"})
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+@patch("ray_release.test.Test.is_jailed_with_open_issue", return_value=False)
+def test_custom_image_build_and_test_init_uploads_chunks(
+    mock_update_from_s3, mock_is_jailed_with_open_issue, mock_subprocess_run
+):
+    """--upload-to-buildkite shells out to buildkite-agent once per chunk."""
+    runner = CliRunner()
+    test_jobs_output_file = "test_jobs_upload.json"
+    result = runner.invoke(
+        main,
+        [
+            "--test-collection-file",
+            "release/ray_release/tests/sample_5_tests.yaml",
+            "--global-config",
+            "oss_config.yaml",
+            "--frequency",
+            "nightly",
+            "--run-jailed-tests",
+            "--run-unstable-tests",
+            "--test-filters",
+            "prefix:hello_world",
+            "--custom-build-jobs-output-file",
+            "custom_build_jobs.yaml",
+            "--test-jobs-output-file",
+            test_jobs_output_file,
+            "--upload-to-buildkite",
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    # Filter for pipeline uploads; settings loading also calls
+    # `buildkite-agent meta-data get ...` through the same patched subprocess.
+    upload_calls = [
+        c
+        for c in mock_subprocess_run.call_args_list
+        if c.args and c.args[0][:3] == ["buildkite-agent", "pipeline", "upload"]
+    ]
+    # 5 tests under default max (450) → exactly one chunk → one upload call.
+    assert len(upload_calls) == 1
+    cmd = upload_calls[0].args[0]
+    assert cmd[3].endswith("test_jobs_upload_0.json")
+    assert upload_calls[0].kwargs.get("check") is True
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_env.py

@@ -0,0 +1,45 @@
+import os
+
+import pytest
+
+from ray_release.config import DEFAULT_ANYSCALE_PROJECT
+from ray_release.env import load_environment, populate_os_env
+from ray_release.exception import ReleaseTestConfigError
+from ray_release.util import DeferredEnvVar
+
+TEST_ENV_VAR = DeferredEnvVar(
+    "TEST_ENV_VAR",
+    "value1",
+)
+
+
+def test_deferred_env_var():
+    assert str(TEST_ENV_VAR) == "value1"
+
+    os.environ["TEST_ENV_VAR"] = "other2"
+
+    assert str(TEST_ENV_VAR) == "other2"
+
+
+def test_load_env_invalid():
+    with pytest.raises(ReleaseTestConfigError):
+        load_environment("invalid")
+
+
+def test_load_env_changes():
+    old_val = str(DEFAULT_ANYSCALE_PROJECT)
+
+    env_dict = load_environment("aws")
+    populate_os_env(env_dict)
+
+    new_val = str(DEFAULT_ANYSCALE_PROJECT)
+
+    assert new_val
+    assert old_val != new_val
+    assert "prj_" in new_val
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_github_client.py

@@ -0,0 +1,449 @@
+"""
+Tests for the GitHub REST API client.
+
+Uses the `responses` library to intercept outbound HTTP requests without
+patching Python internals, so tests exercise the full client code path:
+URL construction, headers, JSON parsing, and dataclass population.
+"""
+import sys
+
+import pytest
+import responses
+
+from ray_release.github_client import (
+    GitHubClient,
+    GitHubException,
+    GitHubIssue,
+    GitHubLabel,
+    GitHubPull,
+    GitHubRepo,
+)
+
+REPO = "owner/repo"
+BASE = "https://api.github.com"
+
+
+def _client() -> GitHubClient:
+    return GitHubClient("test-token")
+
+
+def _repo(client: GitHubClient = None) -> GitHubRepo:
+    return (_client() if client is None else client).get_repo(REPO)
+
+
+def _issue_json(**overrides) -> dict:
+    base = {
+        "number": 42,
+        "state": "open",
+        "title": "Flaky test in CI",
+        "html_url": f"https://github.com/{REPO}/issues/42",
+        "labels": [{"name": "bug"}, {"name": "core"}],
+    }
+    base.update(overrides)
+    return base
+
+
+# ---------------------------------------------------------------------------
+# GitHubClient — auth headers
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_auth_header_is_sent():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/pulls/1",
+        json={"number": 1, "user": {"login": "octocat"}},
+    )
+    client = GitHubClient("my-secret-token")
+    client.get_repo(REPO).get_pull(1)
+
+    sent_headers = responses.calls[0].request.headers
+    assert sent_headers["Authorization"] == "Bearer my-secret-token"
+    assert sent_headers["Accept"] == "application/vnd.github+json"
+    assert "X-GitHub-Api-Version" in sent_headers
+
+
+@responses.activate
+def test_token_not_leaked_on_api_error():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/pulls/1",
+        json={"message": "Bad credentials"},
+        status=401,
+    )
+    secret = "super-secret-token-xyz"
+    with pytest.raises(GitHubException) as exc_info:
+        GitHubClient(secret).get_repo(REPO).get_pull(1)
+
+    exc = exc_info.value
+    assert secret not in str(exc)
+    assert secret not in str(exc.data)
+    assert secret not in str(dict(exc.headers))
+
+
+# ---------------------------------------------------------------------------
+# GitHubRepo.get_issue
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_get_issue_returns_populated_issue():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(),
+    )
+    issue = _repo().get_issue(42)
+
+    assert isinstance(issue, GitHubIssue)
+    assert issue.number == 42
+    assert issue.state == "open"
+    assert issue.title == "Flaky test in CI"
+    assert issue.html_url == f"https://github.com/{REPO}/issues/42"
+
+
+@responses.activate
+def test_get_issue_populates_labels():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(
+            labels=[{"name": "weekly-release-blocker"}, {"name": "serve"}]
+        ),
+    )
+    issue = _repo().get_issue(42)
+    labels = issue.get_labels()
+
+    assert len(labels) == 2
+    assert labels[0].name == "weekly-release-blocker"
+    assert labels[1].name == "serve"
+
+
+@responses.activate
+def test_get_issue_with_no_labels():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(labels=[]),
+    )
+    issue = _repo().get_issue(42)
+    assert issue.get_labels() == []
+
+
+@responses.activate
+def test_get_issue_raises_github_exception_on_404():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/9999",
+        json={"message": "Not Found"},
+        status=404,
+    )
+    with pytest.raises(GitHubException) as exc_info:
+        _repo().get_issue(9999)
+
+    assert exc_info.value.status == 404
+    assert exc_info.value.data == {"message": "Not Found"}
+
+
+# ---------------------------------------------------------------------------
+# GitHubRepo.get_label + get_issues
+# ---------------------------------------------------------------------------
+
+
+def test_get_label_returns_label_with_name():
+    label = _repo().get_label("weekly-release-blocker")
+    assert isinstance(label, GitHubLabel)
+    assert label.name == "weekly-release-blocker"
+
+
+@responses.activate
+def test_get_issues_passes_state_and_label_params():
+    responses.add(responses.GET, f"{BASE}/repos/{REPO}/issues", json=[])
+
+    repo = _repo()
+    label = repo.get_label("weekly-release-blocker")
+    repo.get_issues(state="open", labels=[label])
+
+    qs = responses.calls[0].request.url
+    assert "state=open" in qs
+    assert "labels=weekly-release-blocker" in qs
+    assert "per_page=100" in qs
+
+
+@responses.activate
+def test_get_issues_returns_list_of_issues():
+    page1 = [_issue_json(number=1, title="A"), _issue_json(number=2, title="B")]
+    responses.add(responses.GET, f"{BASE}/repos/{REPO}/issues", json=page1)
+
+    issues = _repo().get_issues(state="open")
+
+    assert len(issues) == 2
+    assert issues[0].number == 1
+    assert issues[1].number == 2
+
+
+@responses.activate
+def test_get_issues_paginates():
+    page1 = [_issue_json(number=i) for i in range(100)]
+    page2 = [_issue_json(number=i) for i in range(100, 115)]
+    next_url = f"{BASE}/repos/{REPO}/issues?page=2&per_page=100"
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues",
+        json=page1,
+        headers={"Link": f'<{next_url}>; rel="next"'},
+    )
+    responses.add(responses.GET, next_url, json=page2)
+
+    issues = _repo().get_issues()
+    assert len(issues) == 115
+
+
+@responses.activate
+def test_get_issues_raises_on_error():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues",
+        json={"message": "Forbidden"},
+        status=403,
+    )
+    with pytest.raises(GitHubException) as exc_info:
+        _repo().get_issues()
+    assert exc_info.value.status == 403
+
+
+# ---------------------------------------------------------------------------
+# GitHubRepo.get_pull
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_get_pull_returns_number_and_user_login():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/pulls/99",
+        json={"number": 99, "user": {"login": "octocat"}},
+    )
+    pull = _repo().get_pull(99)
+
+    assert isinstance(pull, GitHubPull)
+    assert pull.number == 99
+    assert pull.user.login == "octocat"
+
+
+@responses.activate
+def test_get_pull_raises_on_error():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/pulls/9999",
+        json={"message": "Not Found"},
+        status=404,
+    )
+    with pytest.raises(GitHubException) as exc_info:
+        _repo().get_pull(9999)
+    assert exc_info.value.status == 404
+
+
+# ---------------------------------------------------------------------------
+# GitHubIssue.create_comment
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_create_comment_posts_to_correct_url():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(),
+    )
+    responses.add(
+        responses.POST,
+        f"{BASE}/repos/{REPO}/issues/42/comments",
+        json={"id": 1},
+        status=201,
+    )
+
+    issue = _repo().get_issue(42)
+    issue.create_comment("Test has been failing. Jailing.")
+
+    post_body = responses.calls[1].request.body
+    assert b"Test has been failing. Jailing." in post_body
+
+
+@responses.activate
+def test_create_comment_raises_on_error():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(),
+    )
+    responses.add(
+        responses.POST,
+        f"{BASE}/repos/{REPO}/issues/42/comments",
+        json={"message": "Unprocessable"},
+        status=422,
+    )
+
+    issue = _repo().get_issue(42)
+    with pytest.raises(GitHubException) as exc_info:
+        issue.create_comment("hi")
+    assert exc_info.value.status == 422
+
+
+# ---------------------------------------------------------------------------
+# GitHubIssue.edit
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_edit_state_sends_patch_and_updates_local_state():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(state="open"),
+    )
+    responses.add(
+        responses.PATCH,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(state="closed"),
+        status=200,
+    )
+
+    issue = _repo().get_issue(42)
+    assert issue.state == "open"
+
+    issue.edit(state="closed")
+
+    assert issue.state == "closed"
+    patch_body = responses.calls[1].request.body
+    assert b'"closed"' in patch_body
+
+
+@responses.activate
+def test_edit_labels_sends_patch_without_state():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(),
+    )
+    responses.add(
+        responses.PATCH,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(labels=[{"name": "jailed-test"}, {"name": "core"}]),
+        status=200,
+    )
+
+    issue = _repo().get_issue(42)
+    issue.edit(labels=["jailed-test", "core"])
+
+    patch_body = responses.calls[1].request.body
+    assert b"jailed-test" in patch_body
+    assert b"state" not in patch_body
+
+    labels = issue.get_labels()
+    assert len(labels) == 2
+    assert labels[0].name == "jailed-test"
+    assert labels[1].name == "core"
+
+
+@responses.activate
+def test_edit_title_sends_patch_and_updates_local_title():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(title="Old title"),
+    )
+    responses.add(
+        responses.PATCH,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(title="New title", state="open"),
+        status=200,
+    )
+
+    issue = _repo().get_issue(42)
+    issue.edit(title="New title", state="open")
+
+    patch_body = responses.calls[1].request.body
+    assert b"New title" in patch_body
+    assert issue.title == "New title"
+
+
+@responses.activate
+def test_edit_raises_on_error():
+    responses.add(
+        responses.GET,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json=_issue_json(),
+    )
+    responses.add(
+        responses.PATCH,
+        f"{BASE}/repos/{REPO}/issues/42",
+        json={"message": "Not Found"},
+        status=404,
+    )
+
+    issue = _repo().get_issue(42)
+    with pytest.raises(GitHubException) as exc_info:
+        issue.edit(state="closed")
+    assert exc_info.value.status == 404
+
+
+# ---------------------------------------------------------------------------
+# GitHubRepo.create_issue
+# ---------------------------------------------------------------------------
+
+
+@responses.activate
+def test_create_issue_posts_and_returns_issue():
+    responses.add(
+        responses.POST,
+        f"{BASE}/repos/{REPO}/issues",
+        json=_issue_json(number=99, title="New bug", state="open"),
+        status=201,
+    )
+
+    issue = _repo().create_issue(title="New bug", body="It broke.", labels=["bug"])
+
+    assert issue.number == 99
+    assert issue.title == "New bug"
+    assert issue.state == "open"
+    post_body = responses.calls[0].request.body
+    assert b"New bug" in post_body
+    assert b"It broke." in post_body
+    assert b"bug" in post_body
+
+
+@responses.activate
+def test_create_issue_raises_on_error():
+    responses.add(
+        responses.POST,
+        f"{BASE}/repos/{REPO}/issues",
+        json={"message": "Forbidden"},
+        status=403,
+    )
+
+    with pytest.raises(GitHubException) as exc_info:
+        _repo().create_issue(title="New bug", body="body")
+    assert exc_info.value.status == 403
+
+
+# ---------------------------------------------------------------------------
+# GitHubException
+# ---------------------------------------------------------------------------
+
+
+def test_github_exception_stores_status_and_data():
+    exc = GitHubException(404, {"message": "Not Found"}, {"x-header": "val"})
+    assert exc.status == 404
+    assert exc.data == {"message": "Not Found"}
+    assert exc.headers == {"x-header": "val"}
+    assert "404" in str(exc)
+
+
+def test_github_exception_is_exception():
+    assert isinstance(GitHubException(500), Exception)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_global_config.py

@@ -0,0 +1,117 @@
+import os
+import sys
+from tempfile import TemporaryDirectory
+
+import pytest
+
+from ray_release.configs.global_config import (
+    get_global_config,
+    init_global_config,
+)
+
+_TEST_CONFIG = """
+byod:
+  ray_cr_repo: ray
+release_byod:
+  ray_ml_cr_repo: ray-ml
+  ray_llm_cr_repo: ray-llm
+  byod_ecr: 029272617770.dkr.ecr.us-west-2.amazonaws.com
+  byod_ecr_region: us-west-2
+  gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci
+  azure_cr: rayreleasetest.azurecr.io
+state_machine:
+  pr:
+    aws_bucket: ray-ci-pr-results
+  branch:
+    aws_bucket: ray-ci-results
+  disabled: 1
+  github_repo: anyscale/ray
+  bisect:
+    disabled: 1
+credentials:
+  aws2gce: release/aws2gce_iam.json
+ci_pipeline:
+  buildkite_org: ray-project
+  premerge:
+    - w00t
+  postmerge:
+    - hi
+    - three
+release_image_step:
+  ray_cpu: anyscalecpubuild
+  ray_cuda: anyscalecudabuild
+  ray_ml: anyscalemlbuild
+  ray_llm: anyscalellmbuild
+"""
+
+
+def test_init_global_config() -> None:
+    with TemporaryDirectory() as tmp:
+        config_file = os.path.join(tmp, "config")
+        with open(config_file, "w") as f:
+            f.write(_TEST_CONFIG)
+        init_global_config(os.path.join(tmp, "config"))
+        config = get_global_config()
+        assert config["aws2gce_credentials"] == "release/aws2gce_iam.json"
+        assert config["ci_pipeline_premerge"] == ["w00t"]
+        assert config["ci_pipeline_postmerge"] == ["hi", "three"]
+        assert (
+            os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
+            == "/workdir/release/aws2gce_iam.json"
+        )
+        assert config["state_machine_pr_aws_bucket"] == "ray-ci-pr-results"
+        assert config["state_machine_disabled"] is True
+        assert config["release_image_step_ray_cpu"] == "anyscalecpubuild"
+        assert config["release_image_step_ray_cuda"] == "anyscalecudabuild"
+        assert config["release_image_step_ray_ml"] == "anyscalemlbuild"
+        assert config["release_image_step_ray_llm"] == "anyscalellmbuild"
+        assert config["byod_ecr"] == "029272617770.dkr.ecr.us-west-2.amazonaws.com"
+        assert config["byod_ecr_region"] == "us-west-2"
+        assert config["byod_gcp_cr"] == "us-west1-docker.pkg.dev/anyscale-oss-ci"
+        assert config["byod_azure_cr"] == "rayreleasetest.azurecr.io"
+        assert config["state_machine_branch_aws_bucket"] == "ray-ci-results"
+        assert config["state_machine_github_repo"] == "anyscale/ray"
+        assert config["buildkite_org"] == "ray-project"
+        assert config["state_machine_bisect_disabled"] is True
+
+
+_TEST_CONFIG_NO_OPTIONAL = """
+release_byod:
+  byod_ecr: 029272617770.dkr.ecr.us-west-2.amazonaws.com
+  byod_ecr_region: us-west-2
+  gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci
+  aws2gce_credentials: release/aws2gce_iam.json
+state_machine:
+  branch:
+    aws_bucket: ray-ci-results
+ci_pipeline:
+  postmerge:
+    - hi
+"""
+
+
+def test_global_config_absent_targets_are_none() -> None:
+    """A config that omits the targeting keys yields None (no silent fallback to
+    ray's repo/org), so misconfiguration fails loudly instead of mis-targeting."""
+    import ray_release.configs.global_config as gc
+
+    with TemporaryDirectory() as tmp:
+        config_file = os.path.join(tmp, "config")
+        with open(config_file, "w") as f:
+            f.write(_TEST_CONFIG_NO_OPTIONAL)
+        # Load a fresh config, then restore the prior singleton so this test does
+        # not contaminate others sharing the process.
+        prev = gc.config
+        gc.config = None
+        try:
+            gc._init_global_config(config_file)
+            config = gc.get_global_config()
+            assert config["state_machine_github_repo"] is None
+            assert config["buildkite_org"] is None
+            assert config["state_machine_bisect_disabled"] is False
+        finally:
+            gc.config = prev
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_glue.py

@@ -0,0 +1,446 @@
+import os
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+from typing import Callable, Optional, Type
+
+import pytest
+
+from ray_release.alerts.handle import result_to_handle_map
+from ray_release.cluster_manager.cluster_manager import ClusterManager
+from ray_release.cluster_manager.minimal import MinimalClusterManager
+from ray_release.command_runner.anyscale_job_runner import AnyscaleJobRunner
+from ray_release.exception import (
+    CommandError,
+    CommandTimeout,
+    ExitCode,
+    FetchResultError,
+    LogsError,
+    PrepareCommandError,
+    PrepareCommandTimeout,
+    ReleaseTestConfigError,
+    ResultsAlert,
+    TestCommandError,
+    TestCommandTimeout,
+)
+from ray_release.file_manager.job_file_manager import JobFileManager
+from ray_release.glue import (
+    command_runner_to_cluster_manager,
+    run_release_test,
+    type_str_to_command_runner,
+)
+from ray_release.logger import logger
+from ray_release.reporter.reporter import Reporter
+from ray_release.result import Result
+from ray_release.test import Test
+from ray_release.tests.utils import APIDict, MockSDK
+
+
+def _fail_on_call(error_type: Type[Exception] = RuntimeError, message: str = "Fail"):
+    def _fail(*args, **kwargs):
+        raise error_type(message)
+
+    return _fail
+
+
+class MockReturn:
+    return_dict = {}
+
+    def __getattribute__(self, item):
+        return_dict = object.__getattribute__(self, "return_dict")
+        if item in return_dict:
+            mocked = return_dict[item]
+            if isinstance(mocked, Callable):
+                return mocked()
+            else:
+                return lambda *a, **kw: mocked
+        return object.__getattribute__(self, item)
+
+
+class MockTest(Test):
+    def get_anyscale_byod_image(self) -> str:
+        return ""
+
+
+class GlueTest(unittest.TestCase):
+    def writeClusterEnv(self, content: str):
+        with open(os.path.join(self.tempdir, "cluster_env.yaml"), "wt") as fp:
+            fp.write(content)
+
+    def writeClusterCompute(self, content: str):
+        with open(os.path.join(self.tempdir, "cluster_compute.yaml"), "wt") as fp:
+            fp.write(content)
+
+    def setUp(self) -> None:
+        self.tempdir = tempfile.mkdtemp()
+        self.sdk = MockSDK()
+
+        self.sdk.returns["get_project"] = APIDict(
+            result=APIDict(name="unit_test_project")
+        )
+        self.sdk.returns["get_cloud"] = APIDict(result=APIDict(provider="AWS"))
+
+        self.writeClusterEnv("{'env': true}")
+        self.writeClusterCompute(
+            "{'head_node_type': {'name': 'head_node', 'instance_type': 'm5a.4xlarge'}, 'worker_node_types': []}"
+        )
+
+        with open(os.path.join(self.tempdir, "driver_fail.sh"), "wt") as f:
+            f.write("exit 1\n")
+
+        with open(os.path.join(self.tempdir, "driver_succeed.sh"), "wt") as f:
+            f.write("exit 0\n")
+
+        this_sdk = self.sdk
+        this_tempdir = self.tempdir
+
+        self.instances = {}
+
+        self.cluster_manager_return = {}
+        self.command_runner_return = {}
+
+        this_instances = self.instances
+        this_cluster_manager_return = self.cluster_manager_return
+        this_command_runner_return = self.command_runner_return
+
+        class MockClusterManager(MockReturn, MinimalClusterManager):
+            def __init__(
+                self,
+                test_name: str,
+                project_id: str,
+                sdk=None,
+                smoke_test: bool = False,
+            ):
+                super(MockClusterManager, self).__init__(
+                    test_name,
+                    project_id,
+                    this_sdk,
+                    smoke_test=smoke_test,
+                )
+                self.return_dict = this_cluster_manager_return
+                this_instances["cluster_manager"] = self
+
+        class FakeFileManager(JobFileManager):
+            def __init__(self, cluster_manager: ClusterManager):
+                super(FakeFileManager, self).__init__(cluster_manager)
+
+            def download_from_cloud(
+                self, key: str, target: str, delete_after_download: bool = False
+            ):
+                with open(target, "wt") as f:
+                    f.write("fake download content")
+
+            def delete(self, key: str, recursive: bool = False):
+                pass
+
+        class MockCommandRunner(MockReturn, AnyscaleJobRunner):
+            return_dict = self.cluster_manager_return
+
+            def __init__(
+                self,
+                cluster_manager: ClusterManager,
+                file_manager: JobFileManager,
+                working_dir,
+                sdk=None,
+                artifact_path: Optional[str] = None,
+            ):
+                super(MockCommandRunner, self).__init__(
+                    cluster_manager,
+                    FakeFileManager(cluster_manager),
+                    this_tempdir,
+                    sdk=this_sdk,
+                    artifact_path=artifact_path,
+                )
+                self.return_dict = this_command_runner_return
+
+        self.mock_alert_return = None
+
+        def mock_alerter(test: Test, result: Result):
+            return self.mock_alert_return
+
+        result_to_handle_map["unit_test_alerter"] = (mock_alerter, False)
+
+        type_str_to_command_runner["unit_test"] = MockCommandRunner
+        command_runner_to_cluster_manager[MockCommandRunner] = MockClusterManager
+
+        self.test = MockTest(
+            name="unit_test_end_to_end",
+            run=dict(
+                type="unit_test",
+                prepare="prepare_cmd",
+                script="test_cmd",
+                wait_for_nodes=dict(num_nodes=4, timeout=40),
+            ),
+            working_dir=self.tempdir,
+            cluster=dict(
+                cluster_env="cluster_env.yaml",
+                cluster_compute="cluster_compute.yaml",
+                byod={},
+            ),
+            alert="unit_test_alerter",
+        )
+        self.kuberay_test = MockTest(
+            name="unit_test_end_to_end_kuberay",
+            run=dict(
+                type="unit_test",
+                prepare="prepare_cmd",
+                script="test_cmd",
+                wait_for_nodes=dict(num_nodes=4, timeout=40),
+            ),
+            working_dir=self.tempdir,
+            cluster=dict(
+                cluster_env="cluster_env.yaml",
+                cluster_compute="cluster_compute.yaml",
+                byod={},
+            ),
+            env="kuberay",
+            alert="unit_test_alerter",
+        )
+        self.anyscale_project = "prj_unit12345678"
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.tempdir)
+
+    def _succeed_until(self, until: str):
+        # These commands should succeed
+        self.cluster_manager_return["cluster_compute_id"] = "valid"
+        self.cluster_manager_return["create_cluster_compute"] = None
+
+        if until == "cluster_compute":
+            return
+
+        self.cluster_manager_return["cluster_env_id"] = "valid"
+        self.cluster_manager_return["create_cluster_env"] = None
+        self.cluster_manager_return["cluster_env_build_id"] = "valid"
+        self.cluster_manager_return["build_cluster_env"] = None
+
+        if until == "cluster_env":
+            return
+
+        if until == "cluster_start":
+            return
+
+        self.command_runner_return["prepare_remote_env"] = None
+
+        if until == "remote_env":
+            return
+
+        self.command_runner_return["wait_for_nodes"] = None
+
+        if until == "wait_for_nodes":
+            return
+
+        self.command_runner_return["run_prepare_command"] = None
+
+        self.command_runner_return["job_url"] = "http://mock-job-url"
+        self.command_runner_return["job_id"] = "mock-job-id"
+
+        if until == "prepare_command":
+            return
+
+        self.command_runner_return["run_command"] = None
+
+        if until == "test_command":
+            return
+
+        self.command_runner_return["fetch_results"] = {
+            "time_taken": 50,
+            "last_update": time.time() - 60,
+        }
+
+        if until == "fetch_results":
+            return
+
+        self.command_runner_return["get_last_logs_ex"] = "Lorem ipsum"
+
+        if until == "get_last_logs":
+            return
+
+        self.mock_alert_return = None
+
+    def _run(self, result: Result, kuberay: bool = False, **kwargs):
+        if kuberay:
+            run_release_test(test=self.kuberay_test, result=result, **kwargs)
+        else:
+            run_release_test(
+                test=self.test,
+                anyscale_project=self.anyscale_project,
+                result=result,
+                **kwargs
+            )
+
+    def testInvalidClusterCompute(self):
+        result = Result()
+
+        # Fails because file not found
+        os.unlink(os.path.join(self.tempdir, "cluster_compute.yaml"))
+        with self.assertRaisesRegex(ReleaseTestConfigError, "Path not found"):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+        with self.assertRaisesRegex(ReleaseTestConfigError, "Path not found"):
+            self._run(result, True)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+
+        # Fails because invalid jinja template
+        self.writeClusterCompute("{{ INVALID")
+        with self.assertRaisesRegex(ReleaseTestConfigError, "yaml template"):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+        with self.assertRaisesRegex(ReleaseTestConfigError, "yaml template"):
+            self._run(result, True)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+
+        # Fails because invalid json
+        self.writeClusterCompute("{'test': true, 'fail}")
+        with self.assertRaisesRegex(ReleaseTestConfigError, "quoted scalar"):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+        with self.assertRaisesRegex(ReleaseTestConfigError, "quoted scalar"):
+            self._run(result, True)
+        self.assertEqual(result.return_code, ExitCode.CONFIG_ERROR.value)
+
+    def testPrepareCommandFails(self):
+        result = Result()
+
+        self._succeed_until("wait_for_nodes")
+
+        # Prepare command fails
+        self.command_runner_return["run_prepare_command"] = _fail_on_call(CommandError)
+        with self.assertRaises(PrepareCommandError):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.PREPARE_ERROR.value)
+
+        # Prepare command times out
+        self.command_runner_return["run_prepare_command"] = _fail_on_call(
+            CommandTimeout
+        )
+        with self.assertRaises(PrepareCommandTimeout):
+            self._run(result)
+        # Special case: Prepare commands are usually waiting for nodes
+        # (this may change in the future!)
+        self.assertEqual(result.return_code, ExitCode.CLUSTER_WAIT_TIMEOUT.value)
+        self.assertIsNone(result.job_url)
+
+    def testTestCommandFails(self):
+        result = Result()
+
+        self._succeed_until("prepare_command")
+
+        # Test command fails
+        self.command_runner_return["run_command"] = _fail_on_call(CommandError)
+        with self.assertRaises(TestCommandError):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.COMMAND_ERROR.value)
+
+        # Test command times out
+        self.command_runner_return["run_command"] = _fail_on_call(CommandTimeout)
+        with self.assertRaises(TestCommandTimeout):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.COMMAND_TIMEOUT.value)
+        self.assertIsNotNone(result.job_url)
+
+    def testTestCommandTimeoutLongRunning(self):
+        result = Result()
+
+        self._succeed_until("fetch_results")
+
+        # Test command times out
+        self.command_runner_return["run_command"] = _fail_on_call(CommandTimeout)
+        with self.assertRaises(TestCommandTimeout):
+            self._run(result)
+        self.assertEqual(result.return_code, ExitCode.COMMAND_TIMEOUT.value)
+
+        # But now set test to long running
+        self.test["run"]["long_running"] = True
+        self._run(result)  # Will not fail this time
+
+        self.assertGreaterEqual(result.results["last_update_diff"], 60.0)
+
+    def testSmokeUnstableTest(self):
+        result = Result()
+
+        self._succeed_until("complete")
+
+        self.test["stable"] = False
+        self._run(result, smoke_test=True)
+
+        # Ensure stable and smoke_test are set correctly.
+        assert not result.stable
+        assert result.smoke_test
+
+    def testFetchResultFails(self):
+        result = Result()
+
+        self._succeed_until("test_command")
+
+        self.command_runner_return["fetch_results"] = _fail_on_call(FetchResultError)
+        with self.assertLogs(logger, "ERROR") as cm:
+            self._run(result)
+            self.assertTrue(any("Could not fetch results" in o for o in cm.output))
+        self.assertEqual(result.return_code, ExitCode.SUCCESS.value)
+        self.assertEqual(result.status, "success")
+
+    def testFetchResultFailsReqNonEmptyResult(self):
+        # set `require_result` bit.
+        new_handler = (result_to_handle_map["unit_test_alerter"], True)
+        result_to_handle_map["unit_test_alerter"] = new_handler
+
+        result = Result()
+
+        self._succeed_until("test_command")
+
+        self.command_runner_return["fetch_results"] = _fail_on_call(FetchResultError)
+        with self.assertRaisesRegex(FetchResultError, "Fail"):
+            with self.assertLogs(logger, "ERROR") as cm:
+                self._run(result)
+                self.assertTrue(any("Could not fetch results" in o for o in cm.output))
+        self.assertEqual(result.return_code, ExitCode.FETCH_RESULT_ERROR.value)
+        self.assertEqual(result.status, "infra_error")
+
+    def testLastLogsFails(self):
+        result = Result()
+
+        self._succeed_until("fetch_results")
+
+        self.command_runner_return["get_last_logs_ex"] = _fail_on_call(LogsError)
+
+        with self.assertLogs(logger, "ERROR") as cm:
+            self._run(result)
+            self.assertTrue(any("Error fetching logs" in o for o in cm.output))
+        self.assertEqual(result.return_code, ExitCode.SUCCESS.value)
+        self.assertEqual(result.status, "success")
+
+    def testAlertFails(self):
+        result = Result()
+
+        self._succeed_until("get_last_logs")
+
+        self.mock_alert_return = "Alert raised"
+
+        with self.assertRaises(ResultsAlert):
+            self._run(result)
+
+        self.assertEqual(result.return_code, ExitCode.COMMAND_ALERT.value)
+        self.assertEqual(result.status, "error")
+
+    def testReportFails(self):
+        result = Result()
+
+        self._succeed_until("complete")
+
+        class FailReporter(Reporter):
+            def report_result_ex(self, test: Test, result: Result):
+                raise RuntimeError
+
+        with self.assertLogs(logger, "ERROR") as cm:
+            self._run(result, reporters=[FailReporter()])
+            self.assertTrue(any("Error reporting results" in o for o in cm.output))
+
+        self.assertEqual(result.return_code, ExitCode.SUCCESS.value)
+        self.assertEqual(result.status, "success")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_kuberay_util.py

@@ -0,0 +1,70 @@
+import sys
+
+import pytest
+
+from ray_release.kuberay_util import convert_cluster_compute_to_kuberay_compute_config
+
+
+def test_convert_cluster_compute_to_kuberay_compute_config():
+    compute_config = {
+        "head_node_type": {
+            "resources": {
+                "limits": {
+                    "cpu": "16",
+                    "memory": "32Gi",
+                }
+            }
+        },
+        "worker_node_types": [
+            {
+                "name": "worker",
+                "resources": {
+                    "limits": {
+                        "cpu": "4",
+                        "memory": "8Gi",
+                    },
+                    "requests": {
+                        "cpu": "4",
+                        "memory": "8Gi",
+                    },
+                },
+                "min_workers": 0,
+                "max_workers": 2,
+                "use_spot": False,
+            }
+        ],
+    }
+    kuberay_compute_config = convert_cluster_compute_to_kuberay_compute_config(
+        compute_config
+    )
+    assert kuberay_compute_config == {
+        "head_node": {
+            "resources": {
+                "limits": {
+                    "cpu": "16",
+                    "memory": "32Gi",
+                }
+            }
+        },
+        "worker_nodes": [
+            {
+                "group_name": "worker",
+                "min_nodes": 0,
+                "max_nodes": 2,
+                "resources": {
+                    "limits": {
+                        "cpu": "4",
+                        "memory": "8Gi",
+                    },
+                    "requests": {
+                        "cpu": "4",
+                        "memory": "8Gi",
+                    },
+                },
+            }
+        ],
+    }
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_log_aggregator.py

@@ -0,0 +1,76 @@
+import sys
+
+import pytest
+
+from ray_release.log_aggregator import LogAggregator
+
+
+def test_compute_stack_pattern():
+    assert (
+        LogAggregator(
+            "\n".join(
+                [
+                    "haha",
+                    "Traceback (most recent call last):",
+                    '    File "/tmp/something", line 584',
+                    "Exception: yaya45",
+                    "hehe",
+                ]
+            )
+        ).compute_crash_pattern()
+        == "somethingline Exception: yaya"
+    )
+
+
+def test_compute_signature():
+    assert (
+        LogAggregator._compute_signature(
+            [
+                "Traceback (most recent call last):",
+                '   File "/tmp/something", line 584',
+                '   File "/tmp/another", deedeebeeaacfa-abc' "Exception: yaya45",
+            ]
+        )
+        == "somethingline another-abcException: yaya"
+    )
+
+
+def test_compute_stack_trace():
+    trace = [
+        "Traceback (most recent call last):",
+        '   File "/tmp/something", line 584, in run_release_test',
+        "       raise pipeline_exception",
+        "ray_release.exception.JobNoLogsError: Could not obtain logs for the job.",
+    ]
+    error_trace = [
+        "[2023-01-01] ERROR: something is wrong",
+        "Traceback (most recent call last):",
+        '   File "/tmp/something", line 584, in run_release_test',
+        "       raise pipeline_exception",
+        "ray_release.exception.JobStartupTimeout: Cluster did not start.",
+    ]
+    error_trace_short = [
+        "[2023-01-01] ERROR: something is wrong"
+        '   File "/tmp/something", line 584, in run_release_test',
+        "       raise pipeline_exception",
+        "ray_release.exception.JobStartupTimeout: Cluster did not start.",
+    ]
+    assert LogAggregator._compute_stack_trace(["haha"] + trace + ["hehe"]) == trace
+    assert (
+        LogAggregator._compute_stack_trace(["haha"] + error_trace + ["hehe"])
+        == error_trace
+    )
+    assert (
+        LogAggregator._compute_stack_trace(["haha"] + error_trace_short + ["hehe"])
+        == error_trace_short
+    )
+    assert (
+        LogAggregator._compute_stack_trace(
+            ["haha"] + trace + ["w00t"] + error_trace + ["hehe"]
+        )
+        == error_trace
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_new_sdk_compute_configs.py

@@ -0,0 +1,124 @@
+import os
+import sys
+
+import pytest
+import yaml
+
+from ray_release.bazel import bazel_runfile
+from ray_release.config import (
+    DEFAULT_CLOUD_ID,
+    DEFAULT_CLOUD_NAME,
+    RELEASE_TEST_CONFIG_FILES,
+    parse_test_definition,
+)
+from ray_release.template import (
+    _load_and_render_yaml_template,
+    load_test_cluster_compute,
+)
+
+_COMPUTE_CONFIG_FILE_ENV = "COMPUTE_CONFIG_FILE"
+_COMPUTE_CONFIG_FILE_FLAG = "--compute-config-file"
+
+
+def _extract_compute_config_file_arg(argv):
+    """Extract ``--compute-config-file[=VALUE]`` from argv.
+
+    Returns (override_path, remaining_argv). pytest's ``pytest_addoption``
+    hook is only collected from conftest.py or plugins — not test modules —
+    so we parse the flag ourselves and pass the value via an env var that
+    ``pytest_generate_tests`` reads.
+    """
+    override = None
+    remaining = []
+    i = 0
+    while i < len(argv):
+        arg = argv[i]
+        if arg.startswith(f"{_COMPUTE_CONFIG_FILE_FLAG}="):
+            override = arg.split("=", 1)[1]
+        elif (
+            arg == _COMPUTE_CONFIG_FILE_FLAG
+            and i + 1 < len(argv)
+            and not argv[i + 1].startswith("-")
+        ):
+            override = argv[i + 1]
+            i += 1
+        else:
+            remaining.append(arg)
+        i += 1
+    return override, remaining
+
+
+def _collect_new_sdk_tests():
+    """Flattened list of release-collection tests with anyscale_sdk_2026=true."""
+    tests = []
+    for config_file in RELEASE_TEST_CONFIG_FILES:
+        with open(bazel_runfile(config_file)) as fp:
+            tests += parse_test_definition(yaml.safe_load(fp))
+    return [t for t in tests if t.uses_anyscale_sdk_2026()]
+
+
+def pytest_generate_tests(metafunc):
+    if "target" not in metafunc.fixturenames:
+        return
+    path = os.environ.get(_COMPUTE_CONFIG_FILE_ENV)
+    if path:
+        metafunc.parametrize(
+            "target",
+            [("path", os.path.abspath(path))],
+            ids=[os.path.basename(path)],
+        )
+    else:
+        tests = _collect_new_sdk_tests()
+        if not tests:
+            raise RuntimeError(
+                "No tests with anyscale_sdk_2026=true found across "
+                f"{RELEASE_TEST_CONFIG_FILES}. Parametrizing on an empty "
+                "list would cause this validation gate to pass silently. "
+                "If the anyscale_sdk_2026 flag has been retired, update "
+                "this test to discover the right compute-config files via "
+                "the new mechanism — do not leave the gate empty."
+            )
+        metafunc.parametrize(
+            "target",
+            [("test", t) for t in tests],
+            ids=[t["name"] for t in tests],
+        )
+
+
+def _load_single_file(path):
+    env = {
+        "ANYSCALE_CLOUD_ID": str(DEFAULT_CLOUD_ID),
+        "ANYSCALE_CLOUD_NAME": str(DEFAULT_CLOUD_NAME),
+    }
+    return _load_and_render_yaml_template(path, env=env)
+
+
+def test_new_sdk_compute_config_is_valid(target):
+    from anyscale.compute_config import ComputeConfig
+
+    kind, payload = target
+    if kind == "test":
+        cluster_compute = load_test_cluster_compute(payload)
+        label = (
+            f"test {payload['name']!r} "
+            f"(cluster_compute={payload['cluster']['cluster_compute']!r})"
+        )
+    else:
+        cluster_compute = _load_single_file(payload)
+        label = f"file {payload!r}"
+
+    assert cluster_compute is not None, f"{label}: failed to load/render YAML"
+
+    try:
+        ComputeConfig.from_dict(cluster_compute)
+    except Exception as e:
+        pytest.fail(
+            f"ComputeConfig.from_dict failed for {label}: {type(e).__name__}: {e}"
+        )
+
+
+if __name__ == "__main__":
+    override, remaining = _extract_compute_config_file_arg(sys.argv[1:])
+    if override:
+        os.environ[_COMPUTE_CONFIG_FILE_ENV] = override
+    sys.exit(pytest.main(remaining + ["-v", __file__]))

release/ray_release/tests/test_result.py

@@ -0,0 +1,71 @@
+import os
+import sys
+from unittest import mock
+
+import pytest
+
+from ray_release.exception import (
+    ExitCode,
+    ReleaseTestConfigError,
+    ReleaseTestError,
+    ReleaseTestSetupError,
+)
+from ray_release.result import Result, ResultStatus, update_result_from_exception
+
+
+def test_update_result_from_exception():
+    # config error
+    result = Result()
+    update_result_from_exception(result, ReleaseTestConfigError())
+    assert result.return_code == ExitCode.CONFIG_ERROR.value
+    assert result.last_logs is None
+
+    # release test error
+    result = Result()
+    result.runtime = 10
+    try:
+        raise ReleaseTestError()
+    except ReleaseTestError as e:
+        update_result_from_exception(result, e, with_last_logs=True)
+    assert result.return_code == ExitCode.UNSPECIFIED.value
+    assert result.status == ResultStatus.RUNTIME_ERROR.value
+    assert result.runtime == 10
+    assert "ReleaseTestError" in result.last_logs
+    assert __file__ in result.last_logs
+
+    # unknown error
+    result = Result()
+    update_result_from_exception(result, Exception("generic"))
+    assert result.return_code == ExitCode.UNKNOWN.value
+    assert result.status == ResultStatus.UNKNOWN.value
+    assert result.runtime == 0
+    assert result.last_logs is None
+
+    # retriable
+    with mock.patch.dict(os.environ, {"BUILDKITE_TIME_LIMIT_FOR_RETRY": "100"}):
+        result = Result()
+        result.runtime = 10
+        update_result_from_exception(result, ReleaseTestSetupError())
+        assert result.return_code == ExitCode.SETUP_ERROR.value
+        assert result.status == ResultStatus.TRANSIENT_INFRA_ERROR.value
+        assert result.runtime == 10
+    # retry limit reached, not retriable
+    with mock.patch.dict(os.environ, {"BUILDKITE_RETRY_COUNT": "1"}):
+        result = Result()
+        result.runtime = 10
+        update_result_from_exception(result, ReleaseTestSetupError())
+        assert result.return_code == ExitCode.SETUP_ERROR.value
+        assert result.status == ResultStatus.INFRA_ERROR.value
+        assert result.runtime == 10
+    # too long to run, not retriable
+    with mock.patch.dict(os.environ, {"BUILDKITE_TIME_LIMIT_FOR_RETRY": "1"}):
+        result = Result()
+        result.runtime = 3600
+        update_result_from_exception(result, ReleaseTestSetupError())
+        assert result.return_code == ExitCode.SETUP_ERROR.value
+        assert result.status == ResultStatus.INFRA_ERROR.value
+        assert result.runtime == 3600
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_retry.py

@@ -0,0 +1,76 @@
+import sys
+
+import pytest
+
+from ray_release import retry
+
+
+def test_retry_with_no_error():
+    invocation_count = 0
+
+    # Function doesn't raise exception; use a dummy value to check invocation.
+    @retry.retry()
+    def no_error_func() -> int:
+        nonlocal invocation_count
+        invocation_count += 1
+        return 1
+
+    assert no_error_func() == 1
+    assert invocation_count == 1
+
+
+# Test senario: exception count is less than retry count.
+def test_retry_with_limited_error():
+    invocation_count = 0
+
+    # Function doesn't raise exception; use a dummy value to check invocation.
+    @retry.retry(init_delay_sec=1, jitter_sec=1)
+    def limited_error() -> int:
+        nonlocal invocation_count
+
+        invocation_count += 1
+
+        if invocation_count == 1:
+            raise Exception("Manual exception")
+        return 1
+
+    assert limited_error() == 1
+    assert invocation_count == 2
+
+
+# Test senario: exception count exceeds retry count.
+def test_retry_with_unlimited_error():
+    invocation_count = 0
+
+    @retry.retry(init_delay_sec=1, jitter_sec=1, backoff=1, max_retry_count=3)
+    def unlimited_error() -> int:
+        nonlocal invocation_count
+
+        invocation_count += 1
+        raise Exception("Manual exception")
+
+    with pytest.raises(Exception, match="Manual exception"):
+        unlimited_error()
+    assert invocation_count == 3
+
+
+def test_retry_on_certain_errors():
+    invocation_count = 0
+
+    # Function doesn't raise exception; use a dummy value to check invocation.
+    @retry.retry(init_delay_sec=1, jitter_sec=1, exceptions=(KeyError,))
+    def limited_error() -> int:
+        nonlocal invocation_count
+
+        invocation_count += 1
+
+        if invocation_count == 1:
+            raise KeyError("Manual exception")
+        return 1
+
+    assert limited_error() == 1
+    assert invocation_count == 2
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-sv", __file__]))

release/ray_release/tests/test_run_script.py

@@ -0,0 +1,139 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import pytest
+
+from ray_release.exception import ExitCode
+
+
+@pytest.fixture
+def setup(tmpdir):
+    state_file = os.path.join(tmpdir, "state.txt")
+    test_script = os.path.join(
+        os.path.dirname(__file__), "..", "..", "run_release_test.sh"
+    )
+
+    os.environ["NO_INSTALL"] = "1"
+    os.environ["NO_CLONE"] = "1"
+    os.environ["NO_ARTIFACTS"] = "1"
+    os.environ[
+        "RAY_TEST_SCRIPT"
+    ] = "python ray_release/tests/_test_run_release_test_sh.py"
+    os.environ["OVERRIDE_SLEEP_TIME"] = "0"
+    os.environ["MAX_RETRIES"] = "3"
+
+    yield state_file, test_script
+
+
+def _read_state(state_file):
+    with open(state_file, "rt") as f:
+        return int(f.read())
+
+
+def _run_script(test_script, state_file, *exits):
+    assert len(exits) == 3
+
+    if os.path.exists(state_file):
+        os.unlink(state_file)
+
+    try:
+        return subprocess.check_call(
+            f"{test_script} "
+            f"{state_file} "
+            f"{' '.join(str(e.value) for e in exits)}",
+            shell=True,
+        )
+    except subprocess.CalledProcessError as e:
+        return e.returncode
+
+
+def test_repeat(setup):
+    state_file, test_script = setup
+
+    assert (
+        _run_script(
+            test_script,
+            state_file,
+            ExitCode.SUCCESS,
+            ExitCode.SUCCESS,
+            ExitCode.SUCCESS,
+        )
+        == ExitCode.SUCCESS.value
+    )
+    assert _read_state(state_file) == 1
+
+    assert (
+        _run_script(
+            test_script,
+            state_file,
+            ExitCode.RAY_WHEELS_TIMEOUT,
+            ExitCode.SUCCESS,
+            ExitCode.SUCCESS,
+        )
+        == ExitCode.SUCCESS.value
+    )
+    assert _read_state(state_file) == 2
+
+    assert (
+        _run_script(
+            test_script,
+            state_file,
+            ExitCode.RAY_WHEELS_TIMEOUT,
+            ExitCode.CLUSTER_ENV_BUILD_TIMEOUT,
+            ExitCode.SUCCESS,
+        )
+        == ExitCode.SUCCESS.value
+    )
+    assert _read_state(state_file) == 3
+
+    assert (
+        _run_script(
+            test_script,
+            state_file,
+            ExitCode.CLUSTER_STARTUP_TIMEOUT,
+            ExitCode.CLUSTER_WAIT_TIMEOUT,
+            ExitCode.RAY_WHEELS_TIMEOUT,
+        )
+        == 79  # BUILDKITE_RETRY_CODE
+    )
+    assert _read_state(state_file) == 3
+
+    assert (
+        _run_script(
+            test_script,
+            state_file,
+            ExitCode.RAY_WHEELS_TIMEOUT,
+            ExitCode.COMMAND_ALERT,
+            ExitCode.SUCCESS,
+        )
+        == 79  # BUILDKITE_RETRY_CODE
+    )
+    assert _read_state(state_file) == 2
+
+
+def test_parameters(setup):
+    state_file, test_script = setup
+
+    os.environ["RAY_TEST_SCRIPT"] = "python ray_release/tests/_test_catch_args.py"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        argv_file = os.path.join(tmpdir, "argv.json")
+
+        subprocess.check_call(
+            f"{test_script} " f"{argv_file} " f"--smoke-test",
+            shell=True,
+        )
+
+        with open(argv_file, "rt") as fp:
+            data = json.load(fp)
+
+    assert "--smoke-test" in data
+
+
+if __name__ == "__main__":
+    import pytest
+
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_state_machine.py

@@ -0,0 +1,399 @@
+import sys
+from typing import List, Optional
+
+import pytest
+
+from ray_release.bazel import bazel_runfile
+from ray_release.configs.global_config import init_global_config
+from ray_release.result import (
+    Result,
+    ResultStatus,
+)
+from ray_release.test import (
+    Test,
+    TestResult,
+    TestState,
+)
+from ray_release.test_automation.ci_state_machine import (
+    CONTINUOUS_FAILURE_TO_FLAKY,
+    CONTINUOUS_PASSING_TO_PASSING,
+    FAILING_TO_FLAKY_MESSAGE,
+    JAILED_MESSAGE,
+    JAILED_TAG,
+    CITestStateMachine,
+)
+from ray_release.test_automation.release_state_machine import ReleaseTestStateMachine
+from ray_release.test_automation.state_machine import (
+    NO_TEAM,
+    WEEKLY_RELEASE_BLOCKER_TAG,
+    TestStateMachine,
+)
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+
+class MockLabel:
+    def __init__(self, name: str):
+        self.name = name
+
+
+class MockIssue:
+    def __init__(
+        self,
+        number: int,
+        title: str,
+        state: str = "open",
+        labels: Optional[List[MockLabel]] = None,
+    ):
+        self.number = number
+        self.title = title
+        self.state = state
+        self.labels = labels or []
+        self.comments = []
+
+    def edit(
+        self, state: str = None, labels: List[MockLabel] = None, title: str = None
+    ):
+        if state:
+            self.state = state
+        if labels:
+            self.labels = labels
+        if title:
+            self.title = title
+        if state:
+            self.state = state
+
+    def create_comment(self, comment: str):
+        self.comments.append(comment)
+
+    def get_labels(self):
+        return self.labels
+
+
+class MockIssueDB:
+    issue_id = 1
+    issue_db = {}
+
+
+class MockRepo:
+    def create_issue(self, labels: List[str], title: str, *args, **kwargs):
+        label_objs = [MockLabel(label) for label in labels]
+        issue = MockIssue(MockIssueDB.issue_id, title=title, labels=label_objs)
+        MockIssueDB.issue_db[MockIssueDB.issue_id] = issue
+        MockIssueDB.issue_id += 1
+        return issue
+
+    def get_issue(self, number: int):
+        return MockIssueDB.issue_db[number]
+
+    def get_issues(self, state: str, labels: List[MockLabel]) -> List[MockIssue]:
+        issues = []
+        for issue in MockIssueDB.issue_db.values():
+            if issue.state != state:
+                continue
+            issue_labels = [label.name for label in issue.labels]
+            if all(label.name in issue_labels for label in labels):
+                issues.append(issue)
+
+        return issues
+
+    def get_label(self, name: str):
+        return MockLabel(name)
+
+
+class MockBuildkiteBuild:
+    def create_build(self, *args, **kwargs):
+        return {
+            "number": 1,
+            "jobs": [{"id": "1"}],
+        }
+
+    def list_all_for_pipeline(self, *args, **kwargs):
+        return []
+
+
+class MockBuildkiteJob:
+    def unblock_job(self, *args, **kwargs):
+        return {}
+
+
+class MockBuildkite:
+    def builds(self):
+        return MockBuildkiteBuild()
+
+    def jobs(self):
+        return MockBuildkiteJob()
+
+
+TestStateMachine.ray_repo = MockRepo()
+TestStateMachine.ray_buildkite = MockBuildkite()
+
+
+def test_ci_empty_results():
+    test = Test(name="w00t", team="ci", state=TestState.FLAKY)
+    test.test_results = []
+    CITestStateMachine(test).move()
+    # do not change the state
+    assert test.get_state() == TestState.FLAKY
+
+
+def test_ci_move_from_passing_to_flaky():
+    """
+    Test the entire lifecycle of a CI test when it moves from passing to flaky.
+    """
+    test = Test(name="w00t", team="ci")
+    # start from passing
+    assert test.get_state() == TestState.PASSING
+
+    # passing to flaky
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ] * 10
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.FLAKY
+    issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
+    assert issue.state == "open"
+    assert issue.title == "CI test w00t is flaky"
+
+    # flaky to jail
+    issue.edit(labels=[MockLabel(JAILED_TAG)])
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.JAILED
+    assert issue.comments[-1] == JAILED_MESSAGE
+
+
+def test_ci_move_from_passing_to_failing_to_flaky():
+    """
+    Test the entire lifecycle of a CI test when it moves from passing to failing.
+    Check that the conditions are met for each state transition. Also check that
+    gihub issues are created and closed correctly.
+    """
+    test = Test(name="test", team="ci")
+    # start from passing
+    assert test.get_state() == TestState.PASSING
+
+    # passing to failing
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ]
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.FAILING
+
+    # failing to consistently failing
+    test.test_results.extend(
+        [
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+            TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        ]
+    )
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
+    assert issue.state == "open"
+    assert "ci-test" in [label.name for label in issue.labels]
+
+    # move from consistently failing to flaky
+    test.test_results.extend(
+        [TestResult.from_result(Result(status=ResultStatus.ERROR.value))]
+        * CONTINUOUS_FAILURE_TO_FLAKY
+    )
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.FLAKY
+    assert issue.comments[-1] == FAILING_TO_FLAKY_MESSAGE
+
+    # go back to passing
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    ] * CONTINUOUS_PASSING_TO_PASSING
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.PASSING
+    assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
+    assert issue.state == "closed"
+
+    # go back to failing and reuse the github issue
+    test.test_results = 3 * [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value))
+    ]
+    CITestStateMachine(test).move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    assert test.get(Test.KEY_GITHUB_ISSUE_NUMBER) == issue.number
+    assert issue.state == "open"
+
+
+def test_release_move_from_passing_to_failing():
+    test = Test(name="test", team="ci")
+    # Test original state
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    ]
+    assert test.get_state() == TestState.PASSING
+
+    # Test moving from passing to failing
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.FAILING
+    assert test[Test.KEY_BISECT_BUILD_NUMBER] == 1
+
+    # Test moving from failing to consistently failing
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
+
+
+def test_release_move_from_failing_to_consisently_failing():
+    test = Test(name="test", team="ci")
+    test[Test.KEY_BISECT_BUILD_NUMBER] = 1
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ]
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.FAILING
+    test[Test.KEY_BISECT_BLAMED_COMMIT] = "1234567890"
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    sm.comment_blamed_commit_on_github_issue()
+    issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    assert "Blamed commit: 1234567890" in issue.comments[0]
+    labels = [label.name for label in issue.get_labels()]
+    assert "ci" in labels
+
+
+def test_release_move_from_failing_to_passing():
+    test = Test(name="test", team="ci")
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ]
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    assert test[Test.KEY_GITHUB_ISSUE_NUMBER] == MockIssueDB.issue_id - 1
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.PASSING
+    assert test.get(Test.KEY_BISECT_BUILD_NUMBER) is None
+    assert test.get(Test.KEY_BISECT_BLAMED_COMMIT) is None
+
+
+def test_release_move_from_failing_to_jailed():
+    test = Test(name="test", team="ci")
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    ]
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.CONSITENTLY_FAILING
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.JAILED
+
+    # Test moving from jailed to jailed
+    issue = MockIssueDB.issue_db[test.get(Test.KEY_GITHUB_ISSUE_NUMBER)]
+    issue.edit(state="closed")
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.ERROR.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.JAILED
+    assert issue.state == "open"
+
+    # Test moving from jailed to passing
+    test.test_results.insert(
+        0,
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.PASSING
+    assert issue.state == "closed"
+
+
+def test_get_release_blockers() -> None:
+    MockIssueDB.issue_id = 1
+    MockIssueDB.issue_db = {}
+    TestStateMachine.ray_repo.create_issue(labels=["non-blocker"], title="non-blocker")
+    TestStateMachine.ray_repo.create_issue(
+        labels=[WEEKLY_RELEASE_BLOCKER_TAG], title="blocker"
+    )
+    issues = TestStateMachine.get_release_blockers()
+    assert len(issues) == 1
+    assert issues[0].title == "blocker"
+
+
+def test_get_issue_owner() -> None:
+    issue = TestStateMachine.ray_repo.create_issue(labels=["core"], title="hi")
+    assert TestStateMachine.get_issue_owner(issue) == "core"
+    issue = TestStateMachine.ray_repo.create_issue(labels=["w00t"], title="bye")
+    assert TestStateMachine.get_issue_owner(issue) == NO_TEAM
+
+
+def test_release_bisect_disabled(monkeypatch) -> None:
+    """When bisect is disabled in config, no bisect build is triggered."""
+    import ray_release.test_automation.state_machine as sm_mod
+
+    real_get_global_config = sm_mod.get_global_config
+
+    def fake_get_global_config():
+        cfg = dict(real_get_global_config())
+        cfg["state_machine_bisect_disabled"] = True
+        return cfg
+
+    monkeypatch.setattr(sm_mod, "get_global_config", fake_get_global_config)
+
+    test = Test(name="bisect-off", team="ci")
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    ]
+    assert test.get_state() == TestState.PASSING
+    test.test_results.insert(
+        0, TestResult.from_result(Result(status=ResultStatus.ERROR.value))
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.FAILING
+    # Bisect gated off -> no build number recorded.
+    assert test.get(Test.KEY_BISECT_BUILD_NUMBER) is None
+
+
+def test_release_bisect_enabled_triggers(monkeypatch) -> None:
+    """When bisect is enabled (default), a bisect build is triggered."""
+    test = Test(name="bisect-on", team="ci")
+    test.test_results = [
+        TestResult.from_result(Result(status=ResultStatus.SUCCESS.value)),
+    ]
+    test.test_results.insert(
+        0, TestResult.from_result(Result(status=ResultStatus.ERROR.value))
+    )
+    sm = ReleaseTestStateMachine(test)
+    sm.move()
+    assert test.get_state() == TestState.FAILING
+    assert test[Test.KEY_BISECT_BUILD_NUMBER] == 1
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_step.py

@@ -0,0 +1,70 @@
+import shlex
+import sys
+from unittest.mock import patch
+
+import pytest
+
+from ray_release.bazel import bazel_runfile
+from ray_release.buildkite.step import get_step, get_step_for_test_group
+from ray_release.configs.global_config import init_global_config
+from ray_release.test import Test
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+
+def _stub_test(val: dict) -> Test:
+    """
+    A helper function to create a test object with a given dictionary.
+    """
+    test = Test(
+        {
+            "name": "test with spaces",
+            "cluster": {
+                "byod": {},
+            },
+            "run": {
+                "script": "python test.py",
+                "timeout": 100,
+                "num_retries": 3,
+            },
+        }
+    )
+    test.update(val)
+    return test
+
+
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+def test_get_step(mock):
+    with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
+        step = get_step(_stub_test({}), run_id=2)
+    assert step["label"] == "test with spaces (None) (2)"
+    assert step["retry"]["automatic"][0]["limit"] == 3
+    assert "commands" in step
+    first_command = shlex.split(step["commands"][0])
+    assert first_command[0] == "./release/run_release_test.sh"
+    assert first_command[1] == "test with spaces"
+
+
+@patch("ray_release.test.Test.update_from_s3", return_value=None)
+def test_get_step_for_test_group(mock):
+    grouped_tests = {
+        "group1": [
+            (_stub_test({"name": "test1", "repeated_run": 3}), False),
+            (_stub_test({"name": "test2"}), False),
+        ],
+        "group2": [(_stub_test({"name": "test3"}), False)],
+    }
+    with patch.dict("os.environ", {"RAYCI_BUILD_ID": "a1b2c3d4"}):
+        steps = get_step_for_test_group(grouped_tests)
+    assert len(steps) == 2
+    assert steps[0]["group"] == "group1"
+    assert [step["label"] for step in steps[0]["steps"]] == [
+        "test1 (None) (0)",
+        "test1 (None) (1)",
+        "test1 (None) (2)",
+        "test2 (None) (0)",
+    ]
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_template.py

@@ -0,0 +1,56 @@
+import sys
+
+import pytest
+
+from ray_release.exception import ReleaseTestConfigError
+from ray_release.template import bazel_runfile, get_working_dir
+from ray_release.test import Test
+
+
+def test_get_working_dir_with_path_from_root():
+    test_with_path_from_root = Test(
+        {
+            "name": "test",
+            "working_dir": "//ray_testing/ray_release/tests",
+        }
+    )
+    assert (
+        get_working_dir(test_with_path_from_root, None, "/tmp/bazel_workspace")
+        == "/tmp/bazel_workspace/ray_testing/ray_release/tests"
+    )
+    assert get_working_dir(test_with_path_from_root, None, None) == bazel_runfile(
+        "ray_testing/ray_release/tests"
+    )
+
+
+def test_get_working_dir_with_relative_path():
+    test_with_relative_path = Test(
+        {
+            "name": "test",
+            "working_dir": "ray_release/tests",
+        }
+    )
+    assert (
+        get_working_dir(test_with_relative_path, None, "/tmp/bazel_workspace")
+        == "/tmp/bazel_workspace/release/ray_release/tests"
+    )
+    assert get_working_dir(test_with_relative_path, None, None) == bazel_runfile(
+        "release/ray_release/tests"
+    )
+
+
+def test_get_working_dir_fail():
+    test_with_path_from_root = Test(
+        {
+            "name": "test",
+            "working_dir": "//ray_testing/ray_release/tests",
+        }
+    )
+    with pytest.raises(ReleaseTestConfigError):
+        get_working_dir(
+            test_with_path_from_root, "/tmp/test_definition_root", "tmp/bazel_workspace"
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/test_test.py

@@ -0,0 +1,656 @@
+import asyncio
+import json
+import os
+import platform
+import sys
+from typing import List
+from unittest import mock
+from unittest.mock import AsyncMock, patch
+
+import aioboto3
+import boto3
+import pytest
+import responses
+
+from ray_release.bazel import bazel_runfile
+from ray_release.configs.global_config import (
+    get_global_config,
+    init_global_config,
+)
+from ray_release.github_client import GitHubClient
+from ray_release.test import (
+    DATAPLANE_ECR_ML_REPO,
+    DATAPLANE_ECR_REPO,
+    LINUX_TEST_PREFIX,
+    MACOS_BISECT_DAILY_RATE_LIMIT,
+    MACOS_TEST_PREFIX,
+    WINDOWS_TEST_PREFIX,
+    ResultStatus,
+    Test,
+    TestResult,
+    TestState,
+    TestType,
+    _convert_env_list_to_dict,
+)
+from ray_release.util import ANYSCALE_RAY_IMAGE_PREFIX, dict_hash
+
+init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml"))
+
+
+class MockTest(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_name(self) -> str:
+        return self.get("name", "")
+
+    def get_test_results(self, limit: int) -> List[TestResult]:
+        return self.get("test_results", [])
+
+    def is_high_impact(self) -> bool:
+        return self.get(Test.KEY_IS_HIGH_IMPACT, "false") == "true"
+
+
+def _stub_test(val: dict) -> Test:
+    test = Test(
+        {
+            "name": "test",
+            "cluster": {},
+        }
+    )
+    test.update(val)
+    return test
+
+
+def _stub_test_result(
+    status: ResultStatus = ResultStatus.SUCCESS, rayci_step_id="123", commit="456"
+) -> TestResult:
+    return TestResult(
+        status=status.value,
+        commit=commit,
+        branch="master",
+        url="url",
+        timestamp=0,
+        pull_request="1",
+        rayci_step_id=rayci_step_id,
+        duration_ms=5.0,
+    )
+
+
+def test_convert_env_list_to_dict():
+    with mock.patch.dict(os.environ, {"ENV": "env"}):
+        assert _convert_env_list_to_dict(["a=b", "c=d=e", "ENV"]) == {
+            "a": "b",
+            "c": "d=e",
+            "ENV": "env",
+        }
+
+
+def test_get_python_version():
+    assert _stub_test({}).get_python_version() == "3.10"
+    assert _stub_test({"python": "3.11"}).get_python_version() == "3.11"
+
+
+def test_get_byod_runtime_env():
+    test = _stub_test(
+        {
+            "python": "3.11",
+            "cluster": {
+                "byod": {
+                    "runtime_env": ["a=b"],
+                },
+            },
+        }
+    )
+    runtime_env = test.get_byod_runtime_env()
+    assert runtime_env.get("a") == "b"
+
+
+def test_get_anyscale_byod_image():
+    os.environ["RAYCI_BUILD_ID"] = "a1b2c3d4"
+    assert (
+        _stub_test({"python": "3.7", "cluster": {"byod": {}}}).get_anyscale_byod_image()
+        == f"{get_global_config()['byod_ecr']}/{DATAPLANE_ECR_REPO}:a1b2c3d4-py37-cpu"
+    )
+    assert _stub_test(
+        {
+            "python": "3.8",
+            "cluster": {
+                "byod": {
+                    "type": "gpu",
+                }
+            },
+        }
+    ).get_anyscale_byod_image() == (
+        f"{get_global_config()['byod_ecr']}/"
+        f"{DATAPLANE_ECR_ML_REPO}:a1b2c3d4-py38-gpu"
+    )
+    assert _stub_test(
+        {
+            "python": "3.8",
+            "cluster": {
+                "byod": {
+                    "type": "gpu",
+                    "post_build_script": "foo.sh",
+                }
+            },
+        }
+    ).get_anyscale_byod_image() == (
+        f"{get_global_config()['byod_ecr']}"
+        f"/{DATAPLANE_ECR_ML_REPO}:a1b2c3d4-py38-gpu-"
+        "5f311914c59730d72cee8e2a015c5d6eedf6523bfbf5abe2494e0cb85a5a7b70"
+    )
+
+
+def test_get_anyscale_byod_image_ray_version():
+    os.environ["RAYCI_BUILD_ID"] = "a1b2c3d4"
+    assert (
+        _stub_test({"python": "3.7", "cluster": {"byod": {}}}).get_anyscale_byod_image()
+        == f"{get_global_config()['byod_ecr']}/{DATAPLANE_ECR_REPO}:a1b2c3d4-py37-cpu"
+    )
+    assert _stub_test(
+        {
+            "python": "3.8",
+            "cluster": {
+                "ray_version": "2.50.0",
+                "byod": {
+                    "type": "gpu",
+                },
+            },
+        }
+    ).get_anyscale_byod_image() == (f"{ANYSCALE_RAY_IMAGE_PREFIX}:2.50.0-py38-cu121")
+    assert _stub_test(
+        {
+            "python": "3.8",
+            "cluster": {
+                "ray_version": "2.50.0",
+                "byod": {
+                    "type": "gpu",
+                    "post_build_script": "foo.sh",
+                },
+            },
+        }
+    ).get_anyscale_byod_image() == (
+        f"{get_global_config()['byod_ecr']}"
+        f"/{DATAPLANE_ECR_ML_REPO}:a1b2c3d4-py38-gpu-"
+        "5f311914c59730d72cee8e2a015c5d6eedf6523bfbf5abe2494e0cb85a5a7b70"
+        "-2.50.0"
+    )
+
+
+_ISSUE_URL = "https://api.github.com/repos/owner/repo/issues/1"
+_ISSUE_JSON = {"number": 1, "state": "open", "title": "", "html_url": "", "labels": []}
+
+
+def _repo():
+    return GitHubClient("token").get_repo("owner/repo")
+
+
+def test_is_jailed_with_open_issue_not_jailed() -> None:
+    assert not Test(state="passing").is_jailed_with_open_issue(_repo())
+
+
+def test_is_jailed_with_open_issue_no_issue_number() -> None:
+    assert not Test(state="jailed").is_jailed_with_open_issue(_repo())
+
+
+@responses.activate
+def test_is_jailed_with_open_issue_open() -> None:
+    responses.add(responses.GET, _ISSUE_URL, json={**_ISSUE_JSON, "state": "open"})
+    assert Test(state="jailed", github_issue_number="1").is_jailed_with_open_issue(
+        _repo()
+    )
+
+
+@responses.activate
+def test_is_jailed_with_open_issue_closed() -> None:
+    responses.add(responses.GET, _ISSUE_URL, json={**_ISSUE_JSON, "state": "closed"})
+    assert not Test(state="jailed", github_issue_number="1").is_jailed_with_open_issue(
+        _repo()
+    )
+
+
+@responses.activate
+def test_is_jailed_with_open_issue_github_exception() -> None:
+    responses.add(responses.GET, _ISSUE_URL, json={"message": "Not Found"}, status=404)
+    assert not Test(
+        name="test", state="jailed", github_issue_number="1"
+    ).is_jailed_with_open_issue(_repo())
+
+
+def test_is_stable() -> None:
+    assert Test().is_stable()
+    assert Test(stable=True).is_stable()
+    assert not Test(stable=False).is_stable()
+
+
+@patch.dict(
+    os.environ,
+    {
+        "BUILDKITE_BRANCH": "food",
+        "BUILDKITE_PULL_REQUEST": "1",
+        "RAYCI_STEP_ID": "g4_s5",
+    },
+)
+def test_result_from_bazel_event() -> None:
+    result = TestResult.from_bazel_event(
+        {
+            "testResult": {"status": "PASSED", "testAttemptDurationMillis": "5"},
+        }
+    )
+    assert result.is_passing()
+    assert result.branch == "food"
+    assert result.pull_request == "1"
+    assert result.rayci_step_id == "g4_s5"
+    assert result.duration_ms == 5
+    result = TestResult.from_bazel_event(
+        {
+            "testResult": {"status": "FAILED"},
+        }
+    )
+    assert result.is_failing()
+    assert result.duration_ms is None
+
+
+def test_from_bazel_event() -> None:
+    test = Test.from_bazel_event(
+        {
+            "id": {"testResult": {"label": "//ray/ci:test"}},
+        },
+        "ci",
+    )
+    assert test.get_name() == f"{platform.system().lower()}://ray/ci:test"
+    assert test.get_oncall() == "ci"
+
+
+@patch.object(boto3, "client")
+@patch.dict(
+    os.environ,
+    {"BUILDKITE_PIPELINE_ID": get_global_config()["ci_pipeline_postmerge"][0]},
+)
+def test_update_from_s3(mock_client) -> None:
+    mock_object = mock.Mock()
+    mock_object.return_value.get.return_value.read.return_value = json.dumps(
+        {
+            "state": "failing",
+            "team": "core",
+            "github_issue_number": "1234",
+        }
+    ).encode("utf-8")
+    mock_client.return_value.get_object = mock_object
+    test = _stub_test({"team": "ci"})
+    test.update_from_s3()
+    assert test.get_state() == TestState.FAILING
+    assert test.get_oncall() == "ci"
+    assert test["github_issue_number"] == "1234"
+
+
+@patch("ray_release.test.Test._get_s3_name")
+@patch("ray_release.test.Test.gen_from_s3")
+def test_gen_from_name(mock_gen_from_s3, _) -> None:
+    mock_gen_from_s3.return_value = [
+        _stub_test({"name": "a"}),
+        _stub_test({"name": "good"}),
+        _stub_test({"name": "test"}),
+    ]
+
+    assert Test.gen_from_name("good").get_name() == "good"
+
+
+def test_get_test_type() -> None:
+    assert (
+        _stub_test({"name": f"{LINUX_TEST_PREFIX}_test"}).get_test_type()
+        == TestType.LINUX_TEST
+    )
+    assert (
+        _stub_test({"name": f"{MACOS_TEST_PREFIX}_test"}).get_test_type()
+        == TestType.MACOS_TEST
+    )
+    assert (
+        _stub_test({"name": f"{WINDOWS_TEST_PREFIX}_test"}).get_test_type()
+        == TestType.WINDOWS_TEST
+    )
+    assert _stub_test({"name": "release_test"}).get_test_type() == TestType.RELEASE_TEST
+
+
+def test_get_bisect_daily_rate_limit() -> None:
+    assert (
+        _stub_test({"name": f"{MACOS_TEST_PREFIX}_test"}).get_bisect_daily_rate_limit()
+    ) == MACOS_BISECT_DAILY_RATE_LIMIT
+
+
+def test_get_s3_name() -> None:
+    assert Test._get_s3_name("linux://python/ray/test") == "linux:__python_ray_test"
+
+
+def test_is_high_impact() -> None:
+    assert _stub_test(
+        {"name": "test", Test.KEY_IS_HIGH_IMPACT: "true"}
+    ).is_high_impact()
+    assert not _stub_test(
+        {"name": "test", Test.KEY_IS_HIGH_IMPACT: "false"}
+    ).is_high_impact()
+    assert not _stub_test({"name": "test"}).is_high_impact()
+
+
+@patch("ray_release.test.Test._gen_test_result")
+def test_gen_test_results(mock_gen_test_result) -> None:
+    def _mock_gen_test_result(
+        client: aioboto3.Session.client,
+        bucket: str,
+        key: str,
+    ) -> TestResult:
+        return (
+            _stub_test_result(ResultStatus.SUCCESS)
+            if key == "good"
+            else _stub_test_result(ResultStatus.ERROR)
+        )
+
+    mock_gen_test_result.side_effect = AsyncMock(side_effect=_mock_gen_test_result)
+
+    results = asyncio.run(
+        _stub_test({})._gen_test_results(
+            bucket="bucket",
+            keys=["good", "bad", "bad", "good"],
+        )
+    )
+    assert [result.status for result in results] == [
+        ResultStatus.SUCCESS.value,
+        ResultStatus.ERROR.value,
+        ResultStatus.ERROR.value,
+        ResultStatus.SUCCESS.value,
+    ]
+
+
+@patch("ray_release.test.Test.gen_microcheck_test")
+@patch("ray_release.test.Test.gen_from_name")
+def gen_microcheck_step_ids(mock_gen_from_name, mock_gen_microcheck_test) -> None:
+    core_test = MockTest(
+        {
+            "name": "linux://core_test",
+            Test.KEY_IS_HIGH_IMPACT: "false",
+            "test_results": [
+                _stub_test_result(rayci_step_id="corebuild", commit="123"),
+            ],
+        }
+    )
+    data_test_01 = MockTest(
+        {
+            "name": "linux://data_test_01",
+            Test.KEY_IS_HIGH_IMPACT: "true",
+            "test_results": [
+                _stub_test_result(rayci_step_id="databuild", commit="123"),
+            ],
+        }
+    )
+    data_test_02 = MockTest(
+        {
+            "name": "linux://data_test_02",
+            Test.KEY_IS_HIGH_IMPACT: "true",
+            "test_results": [
+                _stub_test_result(rayci_step_id="data15build", commit="123"),
+                _stub_test_result(rayci_step_id="databuild", commit="123"),
+                _stub_test_result(rayci_step_id="databuild", commit="456"),
+            ],
+        }
+    )
+    all_tests = [core_test, data_test_01, data_test_02]
+    mock_gen_microcheck_test.return_value = [test.get_target() for test in all_tests]
+    mock_gen_from_name.side_effect = lambda x: [
+        test for test in all_tests if test.get_name() == x
+    ][0]
+
+    assert Test.gen_microcheck_step_ids("linux", "") == {"databuild"}
+
+
+def test_get_test_target():
+    input_to_output = {
+        "linux://test": "//test",
+        "darwin://test": "//test",
+        "windows://test": "//test",
+        "test": "test",
+    }
+    for input, output in input_to_output.items():
+        assert Test({"name": input}).get_target() == output
+
+
+@mock.patch.dict(
+    os.environ,
+    {"BUILDKITE_PULL_REQUEST_BASE_BRANCH": "base", "BUILDKITE_COMMIT": "commit"},
+)
+@mock.patch("subprocess.check_call")
+@mock.patch("subprocess.check_output")
+def test_get_changed_files(mock_check_output, mock_check_call) -> None:
+    mock_check_output.return_value = b"file1\nfile2\n"
+    assert Test._get_changed_files("") == {"file1", "file2"}
+
+
+@mock.patch("ray_release.test.Test._get_test_targets_per_file")
+@mock.patch("ray_release.test.Test._get_changed_files")
+def test_get_changed_tests(
+    mock_get_changed_files, mock_get_test_targets_per_file
+) -> None:
+    mock_get_changed_files.return_value = {"test_src", "build_src"}
+    mock_get_test_targets_per_file.side_effect = (
+        lambda x, _: {"//t1", "//t2"} if x == "test_src" else {}
+    )
+
+    assert Test._get_changed_tests("") == {"//t1", "//t2"}
+
+
+@mock.patch.dict(
+    os.environ,
+    {"BUILDKITE_PULL_REQUEST_BASE_BRANCH": "base", "BUILDKITE_COMMIT": "commit"},
+)
+@mock.patch("subprocess.check_call")
+@mock.patch("subprocess.check_output")
+def test_get_human_specified_tests(mock_check_output, mock_check_call) -> None:
+    mock_check_output.return_value = b"hi\n@microcheck //test01 //test02\nthere"
+    assert Test._get_human_specified_tests("") == {"//test01", "//test02"}
+
+
+def test_gen_microcheck_tests() -> None:
+    test_harness = [
+        {
+            "input": [],
+            "changed_tests": set(),
+            "human_tests": set(),
+            "output": set(),
+        },
+        {
+            "input": [
+                _stub_test(
+                    {
+                        "name": "linux://core_good",
+                        "team": "core",
+                        Test.KEY_IS_HIGH_IMPACT: "true",
+                    }
+                ),
+                _stub_test(
+                    {
+                        "name": "linux://serve_good",
+                        "team": "serve",
+                        Test.KEY_IS_HIGH_IMPACT: "true",
+                    }
+                ),
+            ],
+            "changed_tests": {"//core_new"},
+            "human_tests": {"//human_test"},
+            "output": {
+                "//core_good",
+                "//core_new",
+                "//human_test",
+            },
+        },
+    ]
+    for test in test_harness:
+        with mock.patch(
+            "ray_release.test.Test.gen_from_s3",
+            return_value=test["input"],
+        ), mock.patch(
+            "ray_release.test.Test._get_changed_tests",
+            return_value=test["changed_tests"],
+        ), mock.patch(
+            "ray_release.test.Test._get_human_specified_tests",
+            return_value=test["human_tests"],
+        ):
+            assert (
+                Test.gen_microcheck_tests(
+                    prefix="linux",
+                    bazel_workspace_dir="",
+                    team="core",
+                )
+                == test["output"]
+            )
+
+
+@patch("ray_release.test.Test.get_byod_base_image_tag")
+def test_get_byod_image_tag(mock_get_byod_base_image_tag):
+    test = _stub_test(
+        {
+            "name": "linux://test",
+            "cluster": {
+                "byod": {
+                    "post_build_script": "test_post_build_script.sh",
+                    "python_depset": "test_python_depset.lock",
+                },
+            },
+        }
+    )
+    mock_get_byod_base_image_tag.return_value = "test-image"
+    custom_info = {
+        "post_build_script": "test_post_build_script.sh",
+        "python_depset": "test_python_depset.lock",
+    }
+    hash_value = dict_hash(custom_info)
+    assert test.get_byod_image_tag() == f"test-image-{hash_value}"
+
+
+@patch("ray_release.test.Test.get_byod_base_image_tag")
+def test_get_byod_image_tag_ray_version(mock_get_byod_base_image_tag):
+    test = _stub_test(
+        {
+            "name": "linux://test",
+            "cluster": {
+                "ray_version": "2.50.0",
+                "byod": {
+                    "post_build_script": "test_post_build_script.sh",
+                    "python_depset": "test_python_depset.lock",
+                },
+            },
+        }
+    )
+    mock_get_byod_base_image_tag.return_value = "test-image"
+    custom_info = {
+        "post_build_script": "test_post_build_script.sh",
+        "python_depset": "test_python_depset.lock",
+    }
+    hash_value = dict_hash(custom_info)
+    assert test.get_byod_image_tag() == f"test-image-{hash_value}-2.50.0"
+
+
+def test_require_custom_byod_image():
+    # No custom build needed
+    assert not _stub_test({"cluster": {"byod": {}}}).require_custom_byod_image()
+    # post_build_script triggers custom build
+    assert _stub_test(
+        {"cluster": {"byod": {"post_build_script": "foo.sh"}}}
+    ).require_custom_byod_image()
+    # python_depset triggers custom build
+    assert _stub_test(
+        {"cluster": {"byod": {"python_depset": "deps.lock"}}}
+    ).require_custom_byod_image()
+    # runtime_env triggers custom build
+    assert _stub_test(
+        {"cluster": {"byod": {"runtime_env": ["FOO=bar"]}}}
+    ).require_custom_byod_image()
+    # empty runtime_env does not trigger custom build
+    assert not _stub_test(
+        {"cluster": {"byod": {"runtime_env": []}}}
+    ).require_custom_byod_image()
+
+
+@patch("ray_release.test.Test.get_byod_base_image_tag")
+def test_get_byod_image_tag_runtime_env_only(mock_get_byod_base_image_tag):
+    """Tests with only runtime_env get a custom image tag including env hash."""
+    test = _stub_test(
+        {
+            "name": "linux://test",
+            "cluster": {
+                "byod": {
+                    "runtime_env": ["MY_VAR=123"],
+                },
+            },
+        }
+    )
+    mock_get_byod_base_image_tag.return_value = "test-image"
+    custom_info = {
+        "post_build_script": None,
+        "python_depset": None,
+        "runtime_env": {"MY_VAR": "123"},
+    }
+    hash_value = dict_hash(custom_info)
+    assert test.get_byod_image_tag() == f"test-image-{hash_value}"
+
+    # A different test with the same runtime_env but a different run script
+    # should produce the same image tag (run script doesn't affect the image).
+    test2 = _stub_test(
+        {
+            "name": "linux://other_test",
+            "cluster": {
+                "byod": {
+                    "runtime_env": ["MY_VAR=123"],
+                },
+            },
+            "run": {
+                "script": "python different_script.py",
+            },
+        }
+    )
+    assert test2.get_byod_image_tag() == f"test-image-{hash_value}"
+
+
+@patch("ray_release.test.Test.get_byod_base_image_tag")
+def test_get_byod_image_tag_with_runtime_env_and_script(mock_get_byod_base_image_tag):
+    """Tests with runtime_env AND post_build_script include both in the hash."""
+    test = _stub_test(
+        {
+            "name": "linux://test",
+            "cluster": {
+                "byod": {
+                    "post_build_script": "test_script.sh",
+                    "runtime_env": ["KEY=val"],
+                },
+            },
+        }
+    )
+    mock_get_byod_base_image_tag.return_value = "test-image"
+    custom_info = {
+        "post_build_script": "test_script.sh",
+        "python_depset": None,
+        "runtime_env": {"KEY": "val"},
+    }
+    hash_value = dict_hash(custom_info)
+    assert test.get_byod_image_tag() == f"test-image-{hash_value}"
+
+    # Verify this is different from the hash without runtime_env
+    custom_info_no_env = {
+        "post_build_script": "test_script.sh",
+        "python_depset": None,
+    }
+    assert dict_hash(custom_info) != dict_hash(custom_info_no_env)
+
+
+def test_uses_anyscale_sdk_2026():
+    assert not Test({"name": "t", "cluster": {}}).uses_anyscale_sdk_2026()
+    assert not Test({"name": "t"}).uses_anyscale_sdk_2026()
+    assert Test(
+        {"name": "t", "cluster": {"anyscale_sdk_2026": True}}
+    ).uses_anyscale_sdk_2026()
+    assert not Test(
+        {"name": "t", "cluster": {"anyscale_sdk_2026": False}}
+    ).uses_anyscale_sdk_2026()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

release/ray_release/tests/utils.py

@@ -0,0 +1,52 @@
+from collections import Counter
+from typing import Any
+
+UNIT_TEST_PROJECT_ID = "prj_HqxHjwtn2uRtzR3DW6AmBYZh"
+
+UNIT_TEST_CLOUD_ID = "cld_4F7k8814aZzGG8TNUGPKnc"
+
+
+class UnitTestError(RuntimeError):
+    pass
+
+
+def fail_always(*a, **kw):
+    raise UnitTestError()
+
+
+def fail_once(result: Any):
+    class _Failer:
+        def __init__(self):
+            self.failed = False
+
+        def __call__(self, *args, **kwargs):
+            if not self.failed:
+                self.failed = True
+                raise UnitTestError()
+            return result
+
+    return _Failer()
+
+
+class APIDict(dict):
+    __slots__ = ()
+    __getattr__ = dict.__getitem__
+    __setattr__ = dict.__setattr__
+
+
+class MockSDK:
+    def __init__(self):
+        self.returns = {}
+        self.call_counter = Counter()
+
+    def reset(self):
+        self.returns = {}
+        self.call_counter = Counter()
+
+    def __getattr__(self, item):
+        self.call_counter[item] += 1
+        result = self.returns.get(item)
+        if callable(result):
+            return result
+        else:
+            return lambda *a, **kw: result