chore: import upstream snapshot with attribution

release/ray_release/.custom_build_and_test_init.stamp

@@ -0,0 +1,2 @@
+This stamp file, if exists, indicates that custom BYOD image builds for release have been decoupled from test_init, allowing each image to be built independently.
+The job generation process for building these custom images is now handled during the init step of the build.

release/ray_release/alerts/default.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+from ray_release.result import Result, ResultStatus
+from ray_release.test import Test
+
+
+def handle_result(
+    test: Test,
+    result: Result,
+) -> Optional[str]:
+
+    if result.status != ResultStatus.SUCCESS.value:
+        return f"Test script did not finish successfully ({result.status})."
+
+    return None

release/ray_release/alerts/handle.py

@@ -0,0 +1,50 @@
+from ray_release.alerts import (
+    default,
+    long_running_tests,
+    tune_tests,
+    xgboost_tests,
+)
+from ray_release.exception import ReleaseTestConfigError, ResultsAlert
+from ray_release.logger import logger
+from ray_release.result import Result
+from ray_release.test import Test
+
+# The second bit in the tuple indicates whether a result is required to pass the alert.
+# If true, the release test will throw a FetchResultError when result cannot be fetched
+# successfully.
+result_to_handle_map = {
+    "default": (default.handle_result, False),
+    "long_running_tests": (
+        long_running_tests.handle_result,
+        True,
+    ),
+    "tune_tests": (tune_tests.handle_result, True),
+    "xgboost_tests": (xgboost_tests.handle_result, True),
+}
+
+
+def require_result(test: Test) -> bool:
+    alert_suite = test.get("alert", "default")
+    if alert_suite not in result_to_handle_map:
+        raise ReleaseTestConfigError(f"Alert suite {alert_suite} not found.")
+    return result_to_handle_map[alert_suite][1]
+
+
+def handle_result(test: Test, result: Result):
+    alert_suite = test.get("alert", "default")
+
+    logger.info(
+        f"Checking results for test {test['name']} using alerting suite "
+        f"{alert_suite}"
+    )
+
+    if alert_suite not in result_to_handle_map:
+        raise ReleaseTestConfigError(f"Alert suite {alert_suite} not found.")
+
+    handler = result_to_handle_map[alert_suite][0]
+    error = handler(test, result)
+
+    if error:
+        raise ResultsAlert(error)
+
+    logger.info("No alerts have been raised - test passed successfully!")

release/ray_release/alerts/long_running_tests.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+from ray_release.result import Result
+from ray_release.test import Test
+
+
+def handle_result(
+    test: Test,
+    result: Result,
+) -> Optional[str]:
+    last_update_diff = result.results.get("last_update_diff", float("inf"))
+
+    test_name = test["name"]
+
+    if test_name in [
+        "long_running_actor_deaths",
+        "long_running_many_actor_tasks",
+        "long_running_many_drivers",
+        "long_running_many_tasks",
+        "long_running_many_tasks_serialized_ids",
+        "long_running_node_failures",
+    ]:
+        # Core tests
+        target_update_diff = 300
+    elif test_name in ["long_running_serve"]:
+        # Serve tests have workload logs every five minutes.
+        # Leave up to 180 seconds overhead.
+        target_update_diff = 480
+    elif test_name in ["long_running_serve_failure"]:
+        # TODO (shrekris-anyscale): set update_diff limit for serve failure
+        target_update_diff = float("inf")
+    else:
+        return None
+
+    if last_update_diff > target_update_diff:
+        return (
+            f"Last update to results json was too long ago "
+            f"({last_update_diff:.2f} > {target_update_diff})"
+        )
+
+    return None

release/ray_release/alerts/tune_tests.py

@@ -0,0 +1,71 @@
+from typing import Optional
+
+from ray_release.result import (
+    Result,
+    ResultStatus,
+)
+from ray_release.test import Test
+
+
+def handle_result(
+    test: Test,
+    result: Result,
+) -> Optional[str]:
+    test_name = test["name"]
+
+    msg = ""
+    success = result.status == ResultStatus.SUCCESS.value
+    time_taken = result.results.get("time_taken", float("inf"))
+    num_terminated = result.results.get("trial_states", {}).get("TERMINATED", 0)
+    was_smoke_test = result.results.get("smoke_test", False)
+
+    if not success:
+        if result.status == "timeout":
+            msg += "Test timed out."
+        else:
+            msg += "Test script failed. "
+
+    if test_name == "tune_scalability_long_running_large_checkpoints":
+        last_update_diff = result.results.get("last_update_diff", float("inf"))
+        target_update_diff = 360
+
+        if last_update_diff > target_update_diff:
+            return (
+                f"Last update to results json was too long ago "
+                f"({last_update_diff:.2f} > {target_update_diff})"
+            )
+        return None
+
+    elif test_name == "tune_scalability_bookkeeping_overhead":
+        target_terminated = 10000
+        target_time = 800
+    elif test_name == "tune_scalability_durable_trainable":
+        target_terminated = 16
+        target_time = 650
+    elif test_name == "tune_scalability_network_overhead":
+        target_terminated = 100 if not was_smoke_test else 20
+        target_time = 900 if not was_smoke_test else 400
+    elif test_name == "tune_scalability_result_throughput_cluster":
+        target_terminated = 1000
+        target_time = 130
+    elif test_name == "tune_scalability_result_throughput_single_node":
+        target_terminated = 96
+        target_time = 120
+    elif test_name == "tune_scalability_xgboost_sweep":
+        target_terminated = 31
+        target_time = 3600
+    else:
+        return None
+
+    if num_terminated < target_terminated:
+        msg += (
+            f"Some trials failed "
+            f"(num_terminated={num_terminated} < {target_terminated}). "
+        )
+    if time_taken > target_time:
+        msg += (
+            f"Took too long to complete "
+            f"(time_taken={time_taken:.2f} > {target_time}). "
+        )
+
+    return msg or None

release/ray_release/alerts/xgboost_tests.py

@@ -0,0 +1,61 @@
+from typing import Optional
+
+from ray_release.result import Result
+from ray_release.test import Test
+
+
+def handle_result(
+    test: Test,
+    result: Result,
+) -> Optional[str]:
+    test_name = test["name"]
+
+    time_taken = result.results.get("time_taken", float("inf"))
+    num_terminated = result.results.get("trial_states", {}).get("TERMINATED", 0)
+
+    if test_name.startswith("xgboost_tune_"):
+        msg = ""
+        if test_name == "xgboost_tune_small":
+            target_terminated = 4
+            target_time = 90
+        elif test_name == "xgboost_tune_4x32":
+            target_terminated = 4
+            target_time = 120
+        elif test_name == "xgboost_tune_32x4":
+            target_terminated = 32
+            target_time = 600
+        else:
+            return None
+
+        if num_terminated < target_terminated:
+            msg += (
+                f"Some trials failed "
+                f"(num_terminated={num_terminated} < {target_terminated}). "
+            )
+        if time_taken > target_time:
+            msg += (
+                f"Took too long to complete "
+                f"(time_taken={time_taken} > {target_time}). "
+            )
+
+        return msg or None
+    else:
+        # train scripts
+        if test_name == "xgboost_train_small":
+            # Leave a couple of seconds for ray connect setup
+            # (without connect it should finish in < 30)
+            target_time = 45
+        elif test_name == "xgboost_train_moderate":
+            target_time = 60
+        elif test_name == "xgboost_train_gpu":
+            target_time = 40
+        else:
+            return None
+
+        if time_taken > target_time:
+            return (
+                f"Took too long to complete "
+                f"(time_taken={time_taken:.2f} > {target_time}). "
+            )
+
+    return None

release/ray_release/anyscale_util.py

@@ -0,0 +1,127 @@
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+
+from ray_release.exception import ClusterEnvCreateError
+from ray_release.logger import logger
+from ray_release.util import get_anyscale_sdk
+
+if TYPE_CHECKING:
+    from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
+
+
+LAST_LOGS_LENGTH = 100
+
+
+class Anyscale:
+    """
+    A wrapper class for latest version of Anyscale SDK.
+
+    Methods of this class can be overwritten for testing.
+    """
+
+    def __init__(self):
+        # We need to late-import Anyscale SDK as merely importing it
+        # will trigger credential lookup on the system.
+        self._anyscale_pkg = None
+
+    def _anyscale(self) -> Any:
+        if self._anyscale_pkg is None:
+            import anyscale
+
+            self._anyscale_pkg = anyscale
+
+        return self._anyscale_pkg
+
+    @property
+    def compute_config(self) -> ModuleType:
+        return self._anyscale().compute_config
+
+    @property
+    def cloud(self) -> ModuleType:
+        return self._anyscale().cloud
+
+    def project_name_by_id(self, project_id: str) -> str:
+        return self._anyscale().project.get(project_id).name
+
+
+# Global singleton for the V2 Anyscale SDK.
+the_v2_sdk = Anyscale()
+
+
+def get_project_name(
+    project_id: str, sdk: Optional[Union[Anyscale, "AnyscaleSDK"]] = None
+) -> str:
+    sdk = sdk or the_v2_sdk
+
+    if not isinstance(sdk, Anyscale):
+        # Fallback to old SDK if provided.
+        # TODO(aslonnie): remove this once we fully migrate to new SDK.
+        return sdk.get_project(project_id).result.name
+
+    return sdk.project_name_by_id(project_id)
+
+
+def get_custom_cluster_env_name(image: str, test_name: str) -> str:
+    # TODO(aslonnie): remove this; new SDK does not need creating cluster envs anymore.
+    image_normalized = image.replace("/", "_").replace(":", "_").replace(".", "_")
+    return f"test_env_{image_normalized}_{test_name}"
+
+
+def create_cluster_env_from_image(
+    image: str,
+    test_name: str,
+    runtime_env: Dict[str, Any],
+    sdk: Optional["AnyscaleSDK"] = None,
+    cluster_env_id: Optional[str] = None,
+    cluster_env_name: Optional[str] = None,
+) -> str:
+    # TODO(aslonnie): remove this; new SDK does not need creating cluster envs anymore.
+    anyscale_sdk = sdk or get_anyscale_sdk()
+    if not cluster_env_name:
+        cluster_env_name = get_custom_cluster_env_name(image, test_name)
+
+    # Find whether there is identical cluster env
+    paging_token = None
+    while not cluster_env_id:
+        result = anyscale_sdk.search_cluster_environments(
+            dict(
+                name=dict(equals=cluster_env_name),
+                paging=dict(count=50, paging_token=paging_token),
+                project_id=None,
+            )
+        )
+        paging_token = result.metadata.next_paging_token
+
+        for res in result.results:
+            if res.name == cluster_env_name:
+                cluster_env_id = res.id
+                logger.info(f"Cluster env already exists with ID " f"{cluster_env_id}")
+                break
+
+        if not paging_token or cluster_env_id:
+            break
+
+    if not cluster_env_id:
+        logger.info("Cluster env not found. Creating new one.")
+        try:
+            result = anyscale_sdk.create_byod_cluster_environment(
+                dict(
+                    name=cluster_env_name,
+                    config_json=dict(
+                        docker_image=image,
+                        ray_version="nightly",
+                    ),
+                )
+            )
+            cluster_env_id = result.result.id
+        except Exception as e:
+            logger.warning(
+                f"Got exception when trying to create cluster "
+                f"env: {e}. Sleeping for 10 seconds with jitter and then "
+                f"try again..."
+            )
+            raise ClusterEnvCreateError("Could not create cluster env.") from e
+
+        logger.info(f"Cluster env created with ID {cluster_env_id}")
+
+    return cluster_env_id

release/ray_release/aws.py

@@ -0,0 +1,174 @@
+import io
+import os
+import sys
+import time
+from copy import deepcopy
+from typing import Optional
+
+import boto3
+import requests
+from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
+from botocore.exceptions import ClientError
+
+from ray_release.logger import logger
+from ray_release.util import DeferredEnvVar
+
+RELEASE_AWS_BUCKET = DeferredEnvVar(
+    "RELEASE_AWS_BUCKET", "ray-release-automation-results"
+)
+RELEASE_AWS_DB_NAME = DeferredEnvVar("RELEASE_AWS_DB_NAME", "ray_ci")
+RELEASE_AWS_DB_TABLE = DeferredEnvVar("RELEASE_AWS_DB_TABLE", "release_test_result")
+
+RELEASE_AWS_ANYSCALE_SECRET_ARN = DeferredEnvVar(
+    "RELEASE_AWS_ANYSCALE_SECRET_ARN",
+    "arn:aws:secretsmanager:us-west-2:029272617770:secret:"
+    "release-automation/"
+    "anyscale-token20210505220406333800000001-BcUuKB",
+)
+
+# If changed, update
+# test_cluster_manager::MinimalSessionManagerTest.testClusterComputeExtraTags
+RELEASE_AWS_RESOURCE_TYPES_TO_TRACK_FOR_BILLING = [
+    "instance",
+    "volume",
+]
+
+S3_PRESIGNED_CACHE = None
+S3_PRESIGNED_KEY = "rayci_result_bucket"
+
+
+def get_secret_token(secret_id: str) -> str:
+    return boto3.client("secretsmanager", region_name="us-west-2").get_secret_value(
+        SecretId=secret_id
+    )["SecretString"]
+
+
+def maybe_fetch_api_token():
+    from anyscale.authenticate import AuthenticationBlock
+
+    if not os.environ.get("ANYSCALE_CLI_TOKEN"):
+        try:
+            token, _ = AuthenticationBlock._load_credentials()
+            logger.info("Loaded anyscale credentials from local storage.")
+            os.environ["ANYSCALE_CLI_TOKEN"] = token
+            return
+        except Exception:
+            pass  # Ignore errors
+
+        logger.info("Missing ANYSCALE_CLI_TOKEN, retrieving from AWS secrets store")
+        # NOTE(simon) This should automatically retrieve
+        # release-automation@anyscale.com's anyscale token
+        cli_token = boto3.client(
+            "secretsmanager", region_name="us-west-2"
+        ).get_secret_value(SecretId=str(RELEASE_AWS_ANYSCALE_SECRET_ARN))[
+            "SecretString"
+        ]
+        os.environ["ANYSCALE_CLI_TOKEN"] = cli_token
+
+
+def add_tags_to_aws_config(aws_config: dict, tags_to_add: dict, resource_types: list):
+    aws_config = deepcopy(aws_config)
+    tag_specifications = aws_config.setdefault("TagSpecifications", [])
+
+    for resource in resource_types:
+        # Check if there is already a tag specification for the resource.
+        # If so, return first item.
+        resource_tags: dict = next(
+            (x for x in tag_specifications if x.get("ResourceType", "") == resource),
+            None,
+        )
+
+        # If no tag specification exists, add
+        if resource_tags is None:
+            resource_tags = {"ResourceType": resource, "Tags": []}
+            tag_specifications.append(resource_tags)
+
+        # Add our tags to the specification
+        tags = resource_tags["Tags"]
+        for key, value in tags_to_add.items():
+            tags.append({"Key": key, "Value": value})
+
+    return aws_config
+
+
+def upload_to_s3(src_path: str, bucket: str, key_path: str) -> Optional[str]:
+    """Upload a file to a S3 bucket
+
+    This assumes the bucket has public read access on the objects uploaded.
+
+    Args:
+        src_path: local file path.
+        bucket: S3 bucket name.
+        key_path: destination url of the uploaded object.
+
+    Return:
+        HTTP URL where the uploaded object could be downloaded if successful,
+        or None if fails.
+
+    Raises:
+        ClientError if upload fails
+    """
+    s3_client = boto3.client("s3")
+    try:
+        s3_client.upload_file(Filename=src_path, Bucket=bucket, Key=key_path)
+    except ClientError as e:
+        logger.warning(f"Failed to upload to s3:  {e}")
+        return None
+
+    return f"https://{bucket}.s3.us-west-2.amazonaws.com/{key_path}"
+
+
+def _retry(f):
+    def inner():
+        resp = None
+        for _ in range(5):
+            resp = f()
+            print("Getting Presigned URL, status_code", resp.status_code)
+            if resp.status_code >= 500:
+                print("errored, retrying...")
+                print(resp.text)
+                time.sleep(5)
+            else:
+                return resp
+        if resp is None or resp.status_code >= 500:
+            print("still errorred after many retries")
+            sys.exit(1)
+
+    return inner
+
+
+@_retry
+def _get_s3_rayci_test_data_presigned():
+    global S3_PRESIGNED_CACHE
+    if not S3_PRESIGNED_CACHE:
+        auth = BotoAWSRequestsAuth(
+            aws_host="vop4ss7n22.execute-api.us-west-2.amazonaws.com",
+            aws_region="us-west-2",
+            aws_service="execute-api",
+        )
+        S3_PRESIGNED_CACHE = requests.get(
+            "https://vop4ss7n22.execute-api.us-west-2.amazonaws.com/endpoint/",
+            auth=auth,
+            params={"job_id": os.environ["BUILDKITE_JOB_ID"]},
+        )
+    return S3_PRESIGNED_CACHE
+
+
+def s3_put_rayci_test_data(Bucket: str, Key: str, Body: str):
+    try:
+        boto3.client("s3").put_object(
+            Bucket=Bucket,
+            Key=Key,
+            Body=Body,
+        )
+    except ClientError:
+        # or use presigned URL
+        resp = _get_s3_rayci_test_data_presigned().json()[S3_PRESIGNED_KEY]
+        data = resp["fields"]
+        data.update(
+            {
+                "key": Key,
+                "file": io.StringIO(Body),
+            }
+        )
+        requests.post(resp["url"], files=data)

release/ray_release/bazel.py

@@ -0,0 +1,24 @@
+import os
+
+import runfiles
+
+REPO_NAME = "io_ray"
+_LEGACY_REPO_ROOT = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../.."),
+)
+
+the_runfiles = runfiles.Create()
+
+
+def _norm_path_join(*args):
+    return os.path.normpath(os.path.join(*args))
+
+
+def bazel_runfile(*args):
+    """Return the path to a runfile in the release directory."""
+    p = _norm_path_join(*args)
+    if the_runfiles:
+        f = the_runfiles.Rlocation(os.path.join(REPO_NAME, p))
+        if f:
+            return f
+    return os.path.join(_LEGACY_REPO_ROOT, p)

release/ray_release/buildkite/aws_instance_types.csv

@@ -0,0 +1,486 @@
+instance,cpus,gpus
+a1.2xlarge,8,0
+a1.4xlarge,16,0
+a1.large,2,0
+a1.medium,1,0
+a1.metal,16,0
+a1.xlarge,4,0
+c1.medium,2,0
+c1.xlarge,8,0
+c3.2xlarge,8,0
+c3.4xlarge,16,0
+c3.8xlarge,32,0
+c3.large,2,0
+c3.xlarge,4,0
+c4.2xlarge,8,0
+c4.4xlarge,16,0
+c4.8xlarge,36,0
+c4.large,2,0
+c4.xlarge,4,0
+c5.12xlarge,48,0
+c5.18xlarge,72,0
+c5.24xlarge,96,0
+c5.2xlarge,8,0
+c5.4xlarge,16,0
+c5.9xlarge,36,0
+c5.large,2,0
+c5.metal,96,0
+c5.xlarge,4,0
+c5a.12xlarge,48,0
+c5a.16xlarge,64,0
+c5a.24xlarge,96,0
+c5a.2xlarge,8,0
+c5a.4xlarge,16,0
+c5a.8xlarge,32,0
+c5a.large,2,0
+c5a.xlarge,4,0
+c5ad.12xlarge,48,0
+c5ad.16xlarge,64,0
+c5ad.24xlarge,96,0
+c5ad.2xlarge,8,0
+c5ad.4xlarge,16,0
+c5ad.8xlarge,32,0
+c5ad.large,2,0
+c5ad.xlarge,4,0
+c5d.12xlarge,48,0
+c5d.18xlarge,72,0
+c5d.24xlarge,96,0
+c5d.2xlarge,8,0
+c5d.4xlarge,16,0
+c5d.9xlarge,36,0
+c5d.large,2,0
+c5d.metal,96,0
+c5d.xlarge,4,0
+c5n.18xlarge,72,0
+c5n.2xlarge,8,0
+c5n.4xlarge,16,0
+c5n.9xlarge,36,0
+c5n.large,2,0
+c5n.metal,72,0
+c5n.xlarge,4,0
+c6a.12xlarge,48,0
+c6a.16xlarge,64,0
+c6a.24xlarge,96,0
+c6a.2xlarge,8,0
+c6a.32xlarge,128,0
+c6a.48xlarge,192,0
+c6a.4xlarge,16,0
+c6a.8xlarge,32,0
+c6a.large,2,0
+c6a.xlarge,4,0
+c6g.12xlarge,48,0
+c6g.16xlarge,64,0
+c6g.2xlarge,8,0
+c6g.4xlarge,16,0
+c6g.8xlarge,32,0
+c6g.large,2,0
+c6g.medium,1,0
+c6g.metal,64,0
+c6g.xlarge,4,0
+c6gd.12xlarge,48,0
+c6gd.16xlarge,64,0
+c6gd.2xlarge,8,0
+c6gd.4xlarge,16,0
+c6gd.8xlarge,32,0
+c6gd.large,2,0
+c6gd.medium,1,0
+c6gd.metal,64,0
+c6gd.xlarge,4,0
+c6gn.12xlarge,48,0
+c6gn.16xlarge,64,0
+c6gn.2xlarge,8,0
+c6gn.4xlarge,16,0
+c6gn.8xlarge,32,0
+c6gn.large,2,0
+c6gn.medium,1,0
+c6gn.xlarge,4,0
+c6i.12xlarge,48,0
+c6i.16xlarge,64,0
+c6i.24xlarge,96,0
+c6i.2xlarge,8,0
+c6i.32xlarge,128,0
+c6i.4xlarge,16,0
+c6i.8xlarge,32,0
+c6i.large,2,0
+c6i.metal,128,0
+c6i.xlarge,4,0
+cc2.8xlarge,32,0
+d2.2xlarge,8,0
+d2.4xlarge,16,0
+d2.8xlarge,36,0
+d2.xlarge,4,0
+d3.2xlarge,8,0
+d3.4xlarge,16,0
+d3.8xlarge,32,0
+d3.xlarge,4,0
+d3en.12xlarge,48,0
+d3en.2xlarge,8,0
+d3en.4xlarge,16,0
+d3en.6xlarge,24,0
+d3en.8xlarge,32,0
+d3en.xlarge,4,0
+dl1.24xlarge,96,8
+f1.16xlarge,64,0
+f1.2xlarge,8,0
+f1.4xlarge,16,0
+g2.2xlarge,8,1
+g2.8xlarge,32,4
+g3.16xlarge,64,4
+g3.4xlarge,16,1
+g3.8xlarge,32,2
+g3s.xlarge,4,1
+g4ad.16xlarge,64,4
+g4ad.2xlarge,8,1
+g4ad.4xlarge,16,1
+g4ad.8xlarge,32,2
+g4ad.xlarge,4,1
+g4dn.12xlarge,48,4
+g4dn.16xlarge,64,1
+g4dn.2xlarge,8,1
+g4dn.4xlarge,16,1
+g4dn.8xlarge,32,1
+g4dn.metal,96,8
+g4dn.xlarge,4,1
+g5.12xlarge,48,4
+g5.16xlarge,64,1
+g5.24xlarge,96,4
+g5.2xlarge,8,1
+g5.48xlarge,192,8
+g5.4xlarge,16,1
+g5.8xlarge,32,1
+g5.xlarge,4,1
+g5g.16xlarge,64,2
+g5g.2xlarge,8,1
+g5g.4xlarge,16,1
+g5g.8xlarge,32,1
+g5g.metal,64,2
+g5g.xlarge,4,1
+h1.16xlarge,64,0
+h1.2xlarge,8,0
+h1.4xlarge,16,0
+h1.8xlarge,32,0
+i2.2xlarge,8,0
+i2.4xlarge,16,0
+i2.8xlarge,32,0
+i2.xlarge,4,0
+i3.16xlarge,64,0
+i3.2xlarge,8,0
+i3.4xlarge,16,0
+i3.8xlarge,32,0
+i3.large,2,0
+i3.metal,72,0
+i3.xlarge,4,0
+i3en.12xlarge,48,0
+i3en.24xlarge,96,0
+i3en.2xlarge,8,0
+i3en.3xlarge,12,0
+i3en.6xlarge,24,0
+i3en.large,2,0
+i3en.metal,96,0
+i3en.xlarge,4,0
+im4gn.16xlarge,64,0
+im4gn.2xlarge,8,0
+im4gn.4xlarge,16,0
+im4gn.8xlarge,32,0
+im4gn.large,2,0
+im4gn.xlarge,4,0
+inf1.24xlarge,96,0
+inf1.2xlarge,8,0
+inf1.6xlarge,24,0
+inf1.xlarge,4,0
+is4gen.2xlarge,8,0
+is4gen.4xlarge,16,0
+is4gen.8xlarge,32,0
+is4gen.large,2,0
+is4gen.medium,1,0
+is4gen.xlarge,4,0
+m1.large,2,0
+m1.medium,1,0
+m1.small,1,0
+m1.xlarge,4,0
+m2.2xlarge,4,0
+m2.4xlarge,8,0
+m2.xlarge,2,0
+m3.2xlarge,8,0
+m3.large,2,0
+m3.medium,1,0
+m3.xlarge,4,0
+m4.10xlarge,40,0
+m4.16xlarge,64,0
+m4.2xlarge,8,0
+m4.4xlarge,16,0
+m4.large,2,0
+m4.xlarge,4,0
+m5.12xlarge,48,0
+m5.16xlarge,64,0
+m5.24xlarge,96,0
+m5.2xlarge,8,0
+m5.4xlarge,16,0
+m5.8xlarge,32,0
+m5.large,2,0
+m5.metal,96,0
+m5.xlarge,4,0
+m5a.12xlarge,48,0
+m5a.16xlarge,64,0
+m5a.24xlarge,96,0
+m5a.2xlarge,8,0
+m5a.4xlarge,16,0
+m5a.8xlarge,32,0
+m5a.large,2,0
+m5a.xlarge,4,0
+m5ad.12xlarge,48,0
+m5ad.16xlarge,64,0
+m5ad.24xlarge,96,0
+m5ad.2xlarge,8,0
+m5ad.4xlarge,16,0
+m5ad.8xlarge,32,0
+m5ad.large,2,0
+m5ad.xlarge,4,0
+m5d.12xlarge,48,0
+m5d.16xlarge,64,0
+m5d.24xlarge,96,0
+m5d.2xlarge,8,0
+m5d.4xlarge,16,0
+m5d.8xlarge,32,0
+m5d.large,2,0
+m5d.metal,96,0
+m5d.xlarge,4,0
+m5dn.12xlarge,48,0
+m5dn.16xlarge,64,0
+m5dn.24xlarge,96,0
+m5dn.2xlarge,8,0
+m5dn.4xlarge,16,0
+m5dn.8xlarge,32,0
+m5dn.large,2,0
+m5dn.metal,96,0
+m5dn.xlarge,4,0
+m5n.12xlarge,48,0
+m5n.16xlarge,64,0
+m5n.24xlarge,96,0
+m5n.2xlarge,8,0
+m5n.4xlarge,16,0
+m5n.8xlarge,32,0
+m5n.large,2,0
+m5n.metal,96,0
+m5n.xlarge,4,0
+m5zn.12xlarge,48,0
+m5zn.2xlarge,8,0
+m5zn.3xlarge,12,0
+m5zn.6xlarge,24,0
+m5zn.large,2,0
+m5zn.metal,48,0
+m5zn.xlarge,4,0
+m6a.12xlarge,48,0
+m6a.16xlarge,64,0
+m6a.24xlarge,96,0
+m6a.2xlarge,8,0
+m6a.32xlarge,128,0
+m6a.48xlarge,192,0
+m6a.4xlarge,16,0
+m6a.8xlarge,32,0
+m6a.large,2,0
+m6a.xlarge,4,0
+m6g.12xlarge,48,0
+m6g.16xlarge,64,0
+m6g.2xlarge,8,0
+m6g.4xlarge,16,0
+m6g.8xlarge,32,0
+m6g.large,2,0
+m6g.medium,1,0
+m6g.metal,64,0
+m6g.xlarge,4,0
+m6gd.12xlarge,48,0
+m6gd.16xlarge,64,0
+m6gd.2xlarge,8,0
+m6gd.4xlarge,16,0
+m6gd.8xlarge,32,0
+m6gd.large,2,0
+m6gd.medium,1,0
+m6gd.metal,64,0
+m6gd.xlarge,4,0
+m6i.12xlarge,48,0
+m6i.16xlarge,64,0
+m6i.24xlarge,96,0
+m6i.2xlarge,8,0
+m6i.32xlarge,128,0
+m6i.4xlarge,16,0
+m6i.8xlarge,32,0
+m6i.large,2,0
+m6i.metal,128,0
+m6i.xlarge,4,0
+mac1.metal,12,0
+p2.16xlarge,64,16
+p2.8xlarge,32,8
+p2.xlarge,4,1
+p3.16xlarge,64,8
+p3.2xlarge,8,1
+p3.8xlarge,32,4
+p3dn.24xlarge,96,8
+p4d.24xlarge,96,8
+r3.2xlarge,8,0
+r3.4xlarge,16,0
+r3.8xlarge,32,0
+r3.large,2,0
+r3.xlarge,4,0
+r4.16xlarge,64,0
+r4.2xlarge,8,0
+r4.4xlarge,16,0
+r4.8xlarge,32,0
+r4.large,2,0
+r4.xlarge,4,0
+r5.12xlarge,48,0
+r5.16xlarge,64,0
+r5.24xlarge,96,0
+r5.2xlarge,8,0
+r5.4xlarge,16,0
+r5.8xlarge,32,0
+r5.large,2,0
+r5.metal,96,0
+r5.xlarge,4,0
+r5a.12xlarge,48,0
+r5a.16xlarge,64,0
+r5a.24xlarge,96,0
+r5a.2xlarge,8,0
+r5a.4xlarge,16,0
+r5a.8xlarge,32,0
+r5a.large,2,0
+r5a.xlarge,4,0
+r5ad.12xlarge,48,0
+r5ad.16xlarge,64,0
+r5ad.24xlarge,96,0
+r5ad.2xlarge,8,0
+r5ad.4xlarge,16,0
+r5ad.8xlarge,32,0
+r5ad.large,2,0
+r5ad.xlarge,4,0
+r5b.12xlarge,48,0
+r5b.16xlarge,64,0
+r5b.24xlarge,96,0
+r5b.2xlarge,8,0
+r5b.4xlarge,16,0
+r5b.8xlarge,32,0
+r5b.large,2,0
+r5b.metal,96,0
+r5b.xlarge,4,0
+r5d.12xlarge,48,0
+r5d.16xlarge,64,0
+r5d.24xlarge,96,0
+r5d.2xlarge,8,0
+r5d.4xlarge,16,0
+r5d.8xlarge,32,0
+r5d.large,2,0
+r5d.metal,96,0
+r5d.xlarge,4,0
+r5dn.12xlarge,48,0
+r5dn.16xlarge,64,0
+r5dn.24xlarge,96,0
+r5dn.2xlarge,8,0
+r5dn.4xlarge,16,0
+r5dn.8xlarge,32,0
+r5dn.large,2,0
+r5dn.metal,96,0
+r5dn.xlarge,4,0
+r5n.12xlarge,48,0
+r5n.16xlarge,64,0
+r5n.24xlarge,96,0
+r5n.2xlarge,8,0
+r5n.4xlarge,16,0
+r5n.8xlarge,32,0
+r5n.large,2,0
+r5n.metal,96,0
+r5n.xlarge,4,0
+r6g.12xlarge,48,0
+r6g.16xlarge,64,0
+r6g.2xlarge,8,0
+r6g.4xlarge,16,0
+r6g.8xlarge,32,0
+r6g.large,2,0
+r6g.medium,1,0
+r6g.metal,64,0
+r6g.xlarge,4,0
+r6gd.12xlarge,48,0
+r6gd.16xlarge,64,0
+r6gd.2xlarge,8,0
+r6gd.4xlarge,16,0
+r6gd.8xlarge,32,0
+r6gd.large,2,0
+r6gd.medium,1,0
+r6gd.metal,64,0
+r6gd.xlarge,4,0
+r6i.12xlarge,48,0
+r6i.16xlarge,64,0
+r6i.24xlarge,96,0
+r6i.2xlarge,8,0
+r6i.32xlarge,128,0
+r6i.4xlarge,16,0
+r6i.8xlarge,32,0
+r6i.large,2,0
+r6i.metal,128,0
+r6i.xlarge,4,0
+t1.micro,1,0
+t2.2xlarge,8,0
+t2.large,2,0
+t2.medium,2,0
+t2.micro,1,0
+t2.nano,1,0
+t2.small,1,0
+t2.xlarge,4,0
+t3.2xlarge,8,0
+t3.large,2,0
+t3.medium,2,0
+t3.micro,2,0
+t3.nano,2,0
+t3.small,2,0
+t3.xlarge,4,0
+t3a.2xlarge,8,0
+t3a.large,2,0
+t3a.medium,2,0
+t3a.micro,2,0
+t3a.nano,2,0
+t3a.small,2,0
+t3a.xlarge,4,0
+t4g.2xlarge,8,0
+t4g.large,2,0
+t4g.medium,2,0
+t4g.micro,2,0
+t4g.nano,2,0
+t4g.small,2,0
+t4g.xlarge,4,0
+u-12tb1.112xlarge,448,0
+u-3tb1.56xlarge,224,0
+u-6tb1.112xlarge,448,0
+u-6tb1.56xlarge,224,0
+u-9tb1.112xlarge,448,0
+vt1.24xlarge,96,0
+vt1.3xlarge,12,0
+vt1.6xlarge,24,0
+x1.16xlarge,64,0
+x1.32xlarge,128,0
+x1e.16xlarge,64,0
+x1e.2xlarge,8,0
+x1e.32xlarge,128,0
+x1e.4xlarge,16,0
+x1e.8xlarge,32,0
+x1e.xlarge,4,0
+x2gd.12xlarge,48,0
+x2gd.16xlarge,64,0
+x2gd.2xlarge,8,0
+x2gd.4xlarge,16,0
+x2gd.8xlarge,32,0
+x2gd.large,2,0
+x2gd.medium,1,0
+x2gd.metal,64,0
+x2gd.xlarge,4,0
+x2iezn.12xlarge,48,0
+x2iezn.2xlarge,8,0
+x2iezn.4xlarge,16,0
+x2iezn.6xlarge,24,0
+x2iezn.8xlarge,32,0
+x2iezn.metal,48,0
+z1d.12xlarge,48,0
+z1d.2xlarge,8,0
+z1d.3xlarge,12,0
+z1d.6xlarge,24,0
+z1d.large,2,0
+z1d.metal,48,0
+z1d.xlarge,4,0

release/ray_release/buildkite/concurrency.py

@@ -0,0 +1,179 @@
+import csv
+from collections import namedtuple
+from typing import Dict, Optional, Tuple
+
+from ray_release.bazel import bazel_runfile
+from ray_release.logger import logger
+from ray_release.template import load_test_cluster_compute
+from ray_release.test import Test
+
+# Keep 10% for the buffer.
+limit = int(15784 * 0.9)
+
+
+Condition = namedtuple(
+    "Condition", ["min_gpu", "max_gpu", "min_cpu", "max_cpu", "group", "limit"]
+)
+
+aws_gpu_cpu_to_concurrency_groups = [
+    Condition(min_gpu=9, max_gpu=-1, min_cpu=0, max_cpu=-1, group="large-gpu", limit=4),
+    Condition(
+        min_gpu=1, max_gpu=9, min_cpu=0, max_cpu=-128, group="small-gpu", limit=8
+    ),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=1025, max_cpu=-1, group="enormous", limit=1
+    ),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=513, max_cpu=1024, group="large", limit=8),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=129, max_cpu=512, group="medium", limit=6),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=9, max_cpu=32, group="tiny", limit=32),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=8, group="minuscule", limit=128),
+    # Make sure "small" is the last in the list, because it is the fallback.
+    Condition(min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=128, group="small", limit=16),
+]
+
+gce_gpu_cpu_to_concurrent_groups = [
+    Condition(min_gpu=8, max_gpu=-1, min_cpu=0, max_cpu=-1, group="gpu-gce", limit=4),
+    Condition(min_gpu=4, max_gpu=-1, min_cpu=0, max_cpu=-1, group="gpu-gce", limit=8),
+    Condition(min_gpu=2, max_gpu=-1, min_cpu=0, max_cpu=-1, group="gpu-gce", limit=16),
+    Condition(min_gpu=1, max_gpu=-1, min_cpu=0, max_cpu=-1, group="gpu-gce", limit=32),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=1025, max_cpu=-1, group="enormous-gce", limit=1
+    ),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=513, max_cpu=1024, group="large-gce", limit=8
+    ),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=129, max_cpu=512, group="medium-gce", limit=6
+    ),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=9, max_cpu=32, group="tiny-gce", limit=32),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=8, group="minuscule-gce", limit=128
+    ),
+    Condition(
+        min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=128, group="small-gce", limit=16
+    ),
+]
+
+
+# Obtained from https://cloud.google.com/compute/docs/accelerator-optimized-machines
+gcp_gpu_instances = {
+    "a2-highgpu-1g": (12, 1),
+    "a2-highgpu-2g": (24, 2),
+    "a2-highgpu-4g": (48, 4),
+    "a2-highgpu-8g": (96, 8),
+    "a2-megagpu-16g": (96, 16),
+    "n1-standard-16-nvidia-tesla-t4-1": (16, 1),
+    "n1-standard-64-nvidia-tesla-t4-4": (64, 4),
+    "n1-standard-32-nvidia-tesla-t4-2": (32, 2),
+    "n1-highmem-64-nvidia-tesla-v100-8": (64, 8),
+    "n1-highmem-96-nvidia-tesla-v100-8": (96, 8),
+}
+
+
+def load_instance_types(path: Optional[str] = None) -> Dict[str, Tuple[int, int]]:
+    if not path:
+        path = bazel_runfile(
+            "release/ray_release/buildkite/aws_instance_types.csv",
+        )
+
+    instance_to_resources = {}
+    with open(path, "rt") as fp:
+        reader = csv.DictReader(fp)
+        for row in reader:
+            instance_to_resources[row["instance"]] = (
+                int(row["cpus"]),
+                int(row["gpus"]),
+            )
+
+    return instance_to_resources
+
+
+def parse_instance_resources(instance: str) -> Tuple[int, int]:
+    """Parse (GCP) instance strings to resources"""
+    # Assumes that GPU instances have already been parsed
+    num_cpus = int(instance.split("-")[-1])
+    num_gpus = 0
+    return num_cpus, num_gpus
+
+
+def parse_condition(cond: int, limit: float = float("inf")) -> float:
+    return cond if cond > -1 else limit
+
+
+def get_concurrency_group(test: Test) -> Tuple[str, int]:
+    if test.get("env", None) == "gce":
+        concurrent_group = gce_gpu_cpu_to_concurrent_groups
+    else:
+        concurrent_group = aws_gpu_cpu_to_concurrency_groups
+    default_concurrent = concurrent_group[-1]
+    try:
+        test_cpus, test_gpus = get_test_resources(test)
+    except Exception as e:
+        logger.warning(f"Couldn't get test resources for test {test['name']}: {e}")
+        return default_concurrent.group, default_concurrent.limit
+
+    for condition in concurrent_group:
+        min_gpu = parse_condition(condition.min_gpu, float("-inf"))
+        max_gpu = parse_condition(condition.max_gpu, float("inf"))
+        min_cpu = parse_condition(condition.min_cpu, float("-inf"))
+        max_cpu = parse_condition(condition.max_cpu, float("inf"))
+
+        if min_cpu <= test_cpus <= max_cpu and min_gpu <= test_gpus <= max_gpu:
+            return condition.group, condition.limit
+
+    # Return default
+    logger.warning(
+        f"Could not find concurrency group for test {test['name']} "
+        f"based on used resources."
+    )
+    return default_concurrent.group, default_concurrent.limit
+
+
+def get_test_resources(test: Test) -> Tuple[int, int]:
+    cluster_compute = load_test_cluster_compute(test)
+    return get_test_resources_from_cluster_compute(
+        cluster_compute, is_new_schema=test.uses_anyscale_sdk_2026()
+    )
+
+
+def get_test_resources_from_cluster_compute(
+    cluster_compute: Dict, is_new_schema: bool = False
+) -> Tuple[int, int]:
+    instances = []
+
+    if is_new_schema:
+        # New schema: head_node, worker_nodes, min_nodes/max_nodes
+        head_node = cluster_compute.get("head_node", {})
+        if head_node.get("instance_type"):
+            instances.append((head_node["instance_type"], 1))
+
+        for w in cluster_compute.get("worker_nodes", []):
+            if w.get("instance_type"):
+                instances.append(
+                    (w["instance_type"], w.get("max_nodes", w.get("min_nodes", 1)))
+                )
+    else:
+        # Legacy schema: head_node_type, worker_node_types, min_workers/max_workers
+        instances.append((cluster_compute["head_node_type"]["instance_type"], 1))
+
+        instances.extend(
+            (w["instance_type"], w.get("max_workers", w.get("min_workers", 1)))
+            for w in cluster_compute["worker_node_types"]
+        )
+
+    aws_instance_types = load_instance_types()
+    total_cpus = 0
+    total_gpus = 0
+
+    for instance, count in instances:
+        if instance in aws_instance_types:
+            instance_cpus, instance_gpus = aws_instance_types[instance]
+        elif instance in gcp_gpu_instances:
+            instance_cpus, instance_gpus = gcp_gpu_instances[instance]
+        else:
+            instance_cpus, instance_gpus = parse_instance_resources(instance)
+
+        total_cpus += instance_cpus * count
+        total_gpus += instance_gpus * count
+
+    return total_cpus, total_gpus

release/ray_release/buildkite/filter.py

@@ -0,0 +1,101 @@
+import copy
+import re
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
+
+from ray_release.buildkite.settings import Frequency, get_frequency
+from ray_release.configs.global_config import get_global_config
+from ray_release.test import Test
+from ray_release.test_automation.state_machine import TestStateMachine
+
+
+def _unflattened_lookup(lookup: Dict, flat_key: str, delimiter: str = "/") -> Any:
+    curr = lookup
+    for k in flat_key.split(delimiter):
+        try:
+            curr = curr.get(k, {})
+        except Exception:
+            return None
+    return curr
+
+
+def filter_tests(
+    test_collection: List[Test],
+    frequency: Frequency,
+    test_filters: Optional[Dict[str, list]] = None,
+    prefer_smoke_tests: bool = False,
+    run_jailed_tests: bool = False,
+    run_unstable_tests: bool = False,
+) -> List[Tuple[Test, bool]]:
+    if test_filters is None:
+        test_filters = {}
+
+    tests_to_run = []
+    for test in test_collection:
+        attr_mismatch = False
+        # Skip kuberay tests for now.
+        # TODO: (khluu) Remove this once we start running KubeRay release tests.
+        if test.is_kuberay() and get_global_config()["kuberay_disabled"]:
+            continue
+
+        # Check if test attributes match filters
+        # Logic: OR within same attribute, AND across different attributes
+        if test_filters:
+            for attr, values in test_filters.items():
+                # Check if at least one value matches for this attribute (OR logic)
+                attr_matched = False
+                for value in values:
+                    # Only prefix filter doesn't use regex
+                    if attr == "prefix":
+                        if test.get_name().startswith(value):
+                            attr_matched = True
+                            break
+                    else:  # Match filters using regex
+                        attr_value = _unflattened_lookup(test, attr) or ""
+                        if re.match(value, attr_value):
+                            attr_matched = True
+                            break
+
+                # If none of the values matched for this attribute, skip this test
+                if not attr_matched:
+                    attr_mismatch = True
+                    break
+        if attr_mismatch:
+            continue
+
+        if not run_jailed_tests:
+            clone_test = copy.deepcopy(test)
+            clone_test.update_from_s3()
+            if clone_test.is_jailed_with_open_issue(TestStateMachine.get_ray_repo()):
+                continue
+        if not run_unstable_tests:
+            if not test.get("stable", True):
+                continue
+
+        test_frequency = get_frequency(test["frequency"])
+
+        if frequency == Frequency.ANY or frequency == test_frequency:
+            if prefer_smoke_tests and "smoke_test" in test:
+                # If we prefer smoke tests and a smoke test is available for this test,
+                # then use the smoke test
+                smoke_test = True
+            else:
+                smoke_test = False
+            tests_to_run.append((test, smoke_test))
+            continue
+
+        elif "smoke_test" in test:
+            smoke_frequency = get_frequency(test["smoke_test"]["frequency"])
+            if smoke_frequency == frequency:
+                tests_to_run.append((test, True))
+    return tests_to_run
+
+
+def group_tests(
+    test_collection_filtered: List[Tuple[Test, bool]]
+) -> Dict[str, List[Tuple[Test, bool]]]:
+    groups = defaultdict(list)
+    for test, smoke in test_collection_filtered:
+        group = test.get("group", "Ungrouped release tests")
+        groups[group].append((test, smoke))
+    return groups

release/ray_release/buildkite/output.py

@@ -0,0 +1,22 @@
+import os
+from typing import Callable, Optional
+
+
+def buildkite_echo(message: str, print_fn: Callable[[str], None] = print):
+    if "BUILDKITE" in os.environ:
+        print_fn(message)
+
+
+def buildkite_group(
+    name: str, open: Optional[bool] = None, print_fn: Callable[[str], None] = print
+):
+    if open is True:
+        buildkite_echo(f"+++ {name}", print_fn=print_fn)
+    elif open is False:
+        buildkite_echo(f"~~~ {name}", print_fn=print_fn)
+    else:  # None
+        buildkite_echo(f"--- {name}", print_fn=print_fn)
+
+
+def buildkite_open_last(print_fn: Callable[[str], None] = print):
+    buildkite_echo("^^^ +++", print_fn=print_fn)

release/ray_release/buildkite/settings.py

@@ -0,0 +1,213 @@
+import enum
+import os
+import subprocess
+from typing import Dict, Optional, Tuple
+
+from ray_release.exception import ReleaseTestConfigError
+from ray_release.logger import logger
+from ray_release.wheels import DEFAULT_BRANCH, get_buildkite_repo_branch
+
+
+class Frequency(enum.Enum):
+    MANUAL = enum.auto()
+    ANY = enum.auto()
+    NIGHTLY = enum.auto()
+    NIGHTLY_3x = enum.auto()
+    WEEKLY = enum.auto()
+    MONTHTLY = enum.auto()
+
+
+frequency_str_to_enum = {
+    "manual": Frequency.MANUAL,
+    "any": Frequency.ANY,
+    "any-smoke": Frequency.ANY,
+    "nightly": Frequency.NIGHTLY,
+    "nightly-3x": Frequency.NIGHTLY_3x,
+    "weekly": Frequency.WEEKLY,
+    "monthly": Frequency.MONTHTLY,
+}
+
+
+class Priority(enum.Enum):
+    DEFAULT = 0
+    MANUAL = 10
+    HIGH = 50
+    HIGHEST = 100
+
+
+priority_str_to_enum = {
+    "default": Priority.DEFAULT,
+    "manual": Priority.MANUAL,
+    "high": Priority.HIGH,
+    "highest": Priority.HIGHEST,
+}
+
+
+def get_frequency(frequency_str: str) -> Frequency:
+    frequency_str = frequency_str.lower()
+    if frequency_str not in frequency_str_to_enum:
+        raise ReleaseTestConfigError(
+            f"Frequency not found: {frequency_str}. Must be one of "
+            f"{list(frequency_str_to_enum.keys())}."
+        )
+    return frequency_str_to_enum[frequency_str]
+
+
+def get_priority(priority_str: str) -> Priority:
+    priority_str = priority_str.lower()
+    if priority_str not in priority_str_to_enum:
+        raise ReleaseTestConfigError(
+            f"Priority not found: {priority_str}. Must be one of "
+            f"{list(priority_str_to_enum.keys())}."
+        )
+    return priority_str_to_enum[priority_str]
+
+
+def get_test_filters(filters_str: str) -> Dict[str, list]:
+    if not filters_str:
+        return {}
+
+    test_filters = {}
+    for line in filters_str.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parts = line.split(":", maxsplit=1)
+        if len(parts) != 2:
+            raise ReleaseTestConfigError(
+                f"Invalid test filter: {line}. " "Should be of the form attr:value"
+            )
+        # Support multiple values for the same attribute (OR logic)
+        if parts[0] not in test_filters:
+            test_filters[parts[0]] = []
+        test_filters[parts[0]].append(parts[1])
+    return test_filters
+
+
+def split_ray_repo_str(repo_str: str) -> Tuple[str, str]:
+    if "https://" in repo_str:
+        if "/tree/" in repo_str:
+            url, branch = repo_str.split("/tree/", maxsplit=2)
+            return f"{url}.git", branch.rstrip("/")
+        return repo_str, DEFAULT_BRANCH  # Default branch
+
+    if ":" in repo_str:
+        owner_or_url, commit_or_branch = repo_str.split(":")
+    else:
+        owner_or_url = repo_str
+        commit_or_branch = DEFAULT_BRANCH
+
+    # Else, construct URL
+    url = f"https://github.com/{owner_or_url}/ray.git"
+    return url, commit_or_branch
+
+
+def get_buildkite_prompt_value(key: str) -> Optional[str]:
+    try:
+        value = subprocess.check_output(
+            ["buildkite-agent", "meta-data", "get", key], text=True
+        )
+    except Exception as e:
+        logger.warning(f"Could not fetch metadata for {key}: {e}")
+        return None
+    logger.debug(f"Got Buildkite prompt value for {key}: {value}")
+    return value
+
+
+def get_pipeline_settings() -> Dict:
+    """Get pipeline settings.
+
+    Retrieves settings from the buildkite agent, environment variables,
+    and default values (in that order of preference)."""
+    settings = get_default_settings()
+    settings = update_settings_from_environment(settings)
+    settings = update_settings_from_buildkite(settings)
+    return settings
+
+
+def get_default_settings() -> Dict:
+    settings = {
+        "frequency": Frequency.ANY,
+        "prefer_smoke_tests": False,
+        "test_filters": None,
+        "ray_test_repo": None,
+        "ray_test_branch": None,
+        "priority": Priority.DEFAULT,
+        "no_concurrency_limit": False,
+    }
+    return settings
+
+
+def update_settings_from_environment(settings: Dict) -> Dict:
+    if "RELEASE_FREQUENCY" in os.environ:
+        settings["frequency"] = get_frequency(os.environ["RELEASE_FREQUENCY"])
+
+    if "RELEASE_PREFER_SMOKE_TESTS" in os.environ:
+        settings["prefer_smoke_tests"] = bool(
+            int(os.environ["RELEASE_PREFER_SMOKE_TESTS"])
+        )
+    elif os.environ.get("RELEASE_FREQUENCY", "").endswith("-smoke"):
+        settings["prefer_smoke_tests"] = True
+
+    if "RAY_TEST_REPO" in os.environ:
+        settings["ray_test_repo"] = os.environ["RAY_TEST_REPO"]
+        settings["ray_test_branch"] = os.environ.get("RAY_TEST_BRANCH", DEFAULT_BRANCH)
+    elif "BUILDKITE_BRANCH" in os.environ:
+        repo_url, branch = get_buildkite_repo_branch()
+
+        settings["ray_test_repo"] = repo_url
+        settings["ray_test_branch"] = branch
+
+    if "TEST_NAME" in os.environ:
+        # This is for backward compatibility.
+        settings["test_filters"] = get_test_filters("name:" + os.environ["TEST_NAME"])
+
+    if "TEST_FILTERS" in os.environ:
+        settings["test_filters"] = os.environ["TEST_FILTERS"]
+
+    if "TEST_ATTR_REGEX_FILTERS" in os.environ:
+        settings["test_filters"] = get_test_filters(
+            os.environ["TEST_ATTR_REGEX_FILTERS"]
+        )
+
+    if "RELEASE_PRIORITY" in os.environ:
+        settings["priority"] = get_priority(os.environ["RELEASE_PRIORITY"])
+
+    if "NO_CONCURRENCY_LIMIT" in os.environ:
+        settings["no_concurrency_limit"] = bool(int(os.environ["NO_CONCURRENCY_LIMIT"]))
+
+    return settings
+
+
+def update_settings_from_buildkite(settings: Dict):
+    release_frequency = get_buildkite_prompt_value("release-frequency")
+    if release_frequency:
+        settings["frequency"] = get_frequency(release_frequency)
+        if release_frequency.endswith("-smoke"):
+            settings["prefer_smoke_tests"] = True
+
+    ray_test_repo_branch = get_buildkite_prompt_value("release-ray-test-repo-branch")
+    if ray_test_repo_branch:
+        repo, branch = split_ray_repo_str(ray_test_repo_branch)
+        settings["ray_test_repo"] = repo
+        settings["ray_test_branch"] = branch
+
+    test_name_filter = get_buildkite_prompt_value("release-test-name")
+    if test_name_filter:
+        settings["test_filters"] = get_test_filters("name:" + test_name_filter)
+
+    test_filters = get_buildkite_prompt_value(
+        "release-test-filters"
+    ) or get_buildkite_prompt_value("release-test-attr-regex-filters")
+    if test_filters:
+        settings["test_filters"] = get_test_filters(test_filters)
+
+    test_priority = get_buildkite_prompt_value("release-priority")
+    if test_priority:
+        settings["priority"] = get_priority(test_priority)
+
+    no_concurrency_limit = get_buildkite_prompt_value("release-no-concurrency-limit")
+    if no_concurrency_limit == "yes":
+        settings["no_concurrency_limit"] = True
+
+    return settings

release/ray_release/buildkite/step.py

@@ -0,0 +1,224 @@
+import copy
+import os
+import shlex
+from typing import Any, Dict, List, Optional, Tuple
+
+from ray_release.aws import RELEASE_AWS_BUCKET
+from ray_release.buildkite.concurrency import get_concurrency_group
+from ray_release.config import (
+    DEFAULT_ANYSCALE_PROJECT,
+    DEFAULT_CLOUD_ID,
+    DEFAULT_CLOUD_NAME,
+    as_smoke_test,
+    get_test_project_id,
+)
+from ray_release.custom_byod_build_init_helper import (
+    generate_custom_build_step_key,
+    get_prerequisite_step,
+)
+from ray_release.env import DEFAULT_ENVIRONMENT, load_environment
+from ray_release.template import get_test_env_var
+from ray_release.test import Test, TestState
+from ray_release.util import DeferredEnvVar
+
+DEFAULT_ARTIFACTS_DIR_HOST = "/tmp/ray_release_test_artifacts"
+
+# TODO (can): unify release_queue_small and runner_queue_small_branch queues
+# having too many type of queues make them difficult to maintain
+RELEASE_QUEUE_DEFAULT = DeferredEnvVar("RELEASE_QUEUE_DEFAULT", "release_queue_small")
+RELEASE_QUEUE_CLIENT = DeferredEnvVar("RELEASE_QUEUE_CLIENT", "release_queue_small")
+
+DOCKER_PLUGIN_KEY = "docker#v5.8.0"
+
+_DEFAULT_STEP_TEMPLATE: Dict[str, Any] = {
+    "env": {
+        "ANYSCALE_CLOUD_ID": str(DEFAULT_CLOUD_ID),
+        "ANYSCALE_CLOUD_NAME": str(DEFAULT_CLOUD_NAME),
+        "ANYSCALE_PROJECT": str(DEFAULT_ANYSCALE_PROJECT),
+        "RELEASE_AWS_BUCKET": str(RELEASE_AWS_BUCKET),
+        "RELEASE_AWS_LOCATION": "dev",
+        "RELEASE_AWS_DB_NAME": "ray_ci",
+        "RELEASE_AWS_DB_TABLE": "release_test_result",
+        "AWS_REGION": "us-west-2",
+    },
+    "agents": {"queue": str(RELEASE_QUEUE_DEFAULT)},
+    "plugins": [
+        {
+            DOCKER_PLUGIN_KEY: {
+                "image": "python:3.10",
+                "shell": ["/bin/bash", "-elic"],
+                "propagate-environment": True,
+                "volumes": [
+                    "/var/lib/buildkite/builds:/var/lib/buildkite/builds",
+                    "/usr/local/bin/buildkite-agent:/usr/local/bin/buildkite-agent",
+                    f"{DEFAULT_ARTIFACTS_DIR_HOST}:{DEFAULT_ARTIFACTS_DIR_HOST}",
+                ],
+                "environment": ["BUILDKITE_BUILD_PATH=/var/lib/buildkite/builds"],
+            }
+        }
+    ],
+    "artifact_paths": [f"{DEFAULT_ARTIFACTS_DIR_HOST}/**/*"],
+    "priority": 0,
+    "retry": {
+        "automatic": [
+            {
+                "exit_status": os.environ.get("BUILDKITE_RETRY_CODE", 79),
+                "limit": os.environ.get("BUILDKITE_MAX_RETRIES", 1),
+            }
+        ]
+    },
+}
+
+
+def get_step_for_test_group(
+    grouped_tests: Dict[str, List[Tuple[Test, bool]]],
+    minimum_run_per_test: int = 1,
+    test_collection_file: List[str] = None,
+    env: Optional[Dict] = None,
+    priority: int = 0,
+    global_config: Optional[str] = None,
+    is_concurrency_limit: bool = True,
+    block_step_key: Optional[str] = None,
+    gpu_map: Optional[Dict[str, str]] = None,
+):
+    steps = []
+    for group in sorted(grouped_tests):
+        tests = grouped_tests[group]
+        group_steps = []
+        for test, smoke_test in tests:
+            for run_id in range(max(test.get("repeated_run", 1), minimum_run_per_test)):
+                step = get_step(
+                    test,
+                    test_collection_file,
+                    run_id=run_id,
+                    # Always report performance data to databrick. Since the data is
+                    # indexed by branch and commit hash, we can always filter data later
+                    report=True,
+                    smoke_test=smoke_test,
+                    env=env,
+                    priority_val=priority,
+                    global_config=global_config,
+                    block_step_key=block_step_key,
+                    gpu_map=gpu_map,
+                )
+
+                if not is_concurrency_limit:
+                    step.pop("concurrency", None)
+                    step.pop("concurrency_group", None)
+
+                group_steps.append(step)
+
+        group_step = {"group": group, "steps": group_steps}
+        steps.append(group_step)
+
+    return steps
+
+
+def get_step(
+    test: Test,
+    test_collection_file: List[str] = None,
+    run_id: int = 1,
+    report: bool = False,
+    smoke_test: bool = False,
+    env: Optional[Dict] = None,
+    priority_val: int = 0,
+    global_config: Optional[str] = None,
+    block_step_key: Optional[str] = None,
+    gpu_map: Optional[Dict[str, str]] = None,
+):
+    env = env or {}
+    step = copy.deepcopy(_DEFAULT_STEP_TEMPLATE)
+
+    cmd = [
+        "./release/run_release_test.sh",
+        shlex.quote(test["name"]),
+    ]
+
+    for file in test_collection_file or []:
+        cmd += ["--test-collection-file", shlex.quote(file)]
+
+    if global_config:
+        cmd += ["--global-config", shlex.quote(global_config)]
+
+    if report and not bool(int(os.environ.get("NO_REPORT_OVERRIDE", "0"))):
+        cmd += ["--report"]
+
+    if smoke_test:
+        cmd += ["--smoke-test"]
+
+    num_retries = test.get("run", {}).get("num_retries")
+    if num_retries:
+        step["retry"]["automatic"][0]["limit"] = num_retries
+
+    step["commands"] = [" ".join(cmd)]
+
+    env_to_use = test.get("env", DEFAULT_ENVIRONMENT)
+    env_dict = load_environment(env_to_use)
+    env_dict.update(env)
+
+    # Set the project id for the test, based on the follow priority:
+    #  1. Specified in the test, as "project_id" field.
+    #  2. Specified in the specific test environment, as RELEASE_DEFAULT_PROJECT env var
+    #  3. Specified in the global environment, as RELEASE_DEFAULT_PROJECT env var
+    default_project_id = env_dict.get("RELEASE_DEFAULT_PROJECT")
+    env_dict["ANYSCALE_PROJECT"] = get_test_project_id(test, default_project_id)
+
+    step["env"].update(env_dict)
+
+    commit = get_test_env_var("RAY_COMMIT")
+    branch = get_test_env_var("RAY_BRANCH")
+    label = commit[:7] if commit else branch
+
+    if smoke_test:
+        concurrency_test = as_smoke_test(test)
+    else:
+        concurrency_test = test
+    concurrency_group, concurrency_limit = get_concurrency_group(concurrency_test)
+
+    step["concurrency_group"] = concurrency_group
+    step["concurrency"] = concurrency_limit
+
+    step["priority"] = priority_val
+
+    # Set queue to QUEUE_CLIENT for client tests
+    # (otherwise keep default QUEUE_DEFAULT)
+    if test.get("run", {}).get("type") == "client":
+        step["agents"]["queue"] = str(RELEASE_QUEUE_CLIENT)
+
+    clone_test = copy.deepcopy(test)  # avoid modifying the original test
+    clone_test.update_from_s3()
+    jailed = clone_test.get_state() == TestState.JAILED
+    full_label = ""
+    if jailed:
+        full_label += "[jailed]"
+
+    full_label += test["name"]
+    if smoke_test:
+        full_label += " [smoke test] "
+    full_label += f" ({label}) ({run_id})"
+
+    step["label"] = full_label
+
+    image = test.get_anyscale_byod_image()
+    base_image = test.get_anyscale_base_byod_image()
+    if test.require_custom_byod_image():
+        step["depends_on"] = generate_custom_build_step_key(image)
+    else:
+        step["depends_on"] = get_prerequisite_step(image, base_image, gpu_map)
+
+    if block_step_key:
+        if not step["depends_on"]:
+            step["depends_on"] = block_step_key
+        else:
+            step["depends_on"] = [step["depends_on"], block_step_key]
+    return step
+
+
+def generate_block_step(num_tests: int):
+    step = {
+        "block": "Run release tests",
+        "depends_on": None,
+        "key": "block_run_release_tests",
+        "prompt": f"You are triggering {num_tests} tests. Do you want to proceed?",
+    }
+    return step

release/ray_release/byod/build.py

@@ -0,0 +1,204 @@
+import os
+import subprocess
+import sys
+import tempfile
+from typing import Dict, List, Optional
+
+from ray_release.byod.build_context import BuildContext, fill_build_context_dir
+from ray_release.config import RELEASE_PACKAGE_DIR
+from ray_release.logger import logger
+from ray_release.test import (
+    Test,
+)
+
+# AZURE_REGISTRY_NAME import temporarily removed below: Azure image push is
+# disabled due to CI issues. Re-add `AZURE_REGISTRY_NAME` to the import below
+# when re-enabling the Azure tag/push block in build_anyscale_custom_byod_image.
+from ray_release.util import ANYSCALE_RAY_IMAGE_PREFIX
+
+bazel_workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "")
+
+
+def build_anyscale_custom_byod_image(
+    image: str,
+    base_image: str,
+    build_context: BuildContext,
+    release_byod_dir: Optional[str] = None,
+) -> None:
+    if _image_exist(image):
+        logger.info(f"Image {image} already exists")
+        return
+
+    if not release_byod_dir:
+        if bazel_workspace_dir:
+            release_byod_dir = os.path.join(
+                bazel_workspace_dir, "release/ray_release/byod"
+            )
+        else:
+            release_byod_dir = os.path.join(RELEASE_PACKAGE_DIR, "ray_release/byod")
+
+    with tempfile.TemporaryDirectory() as build_dir:
+        fill_build_context_dir(build_context, release_byod_dir, build_dir)
+
+        docker_build_cmd = "docker build --progress=plain .".split()
+        docker_build_cmd += ["--build-arg", f"BASE_IMAGE={base_image}"]
+        docker_build_cmd += ["-t", image]
+
+        env = os.environ.copy()
+        env["DOCKER_BUILDKIT"] = "1"
+
+        subprocess.check_call(
+            docker_build_cmd,
+            stdout=sys.stderr,
+            cwd=build_dir,
+            env=env,
+        )
+
+    if not base_image.startswith(ANYSCALE_RAY_IMAGE_PREFIX):
+        _check_ray_commit_in_image(image)
+
+    _push_image(image)
+    if os.environ.get("BUILDKITE"):
+        subprocess.run(
+            [
+                "buildkite-agent",
+                "annotate",
+                "--style=info",
+                "--context=custom-images",
+                "--append",
+                f"{image}<br/>",
+            ],
+        )
+    # Azure tag-and-push for custom BYOD images temporarily disabled due to
+    # CI issues.
+    logger.info(
+        "Skipping Azure ACR tag/push for custom BYOD image %s: Azure image "
+        "push is temporarily disabled due to CI issues.",
+        image,
+    )
+    # tag_without_registry = image.split("/")[-1]
+    # azure_tag = f"{AZURE_REGISTRY_NAME}.azurecr.io/{tag_without_registry}"
+    # _tag_and_push(source=image, target=azure_tag)
+
+
+def build_anyscale_base_byod_images(tests: List[Test]) -> List[str]:
+    """
+    Builds the Anyscale BYOD images for the given tests.
+    """
+    images = set()
+    for test in tests:
+        images.add(test.get_anyscale_base_byod_image())
+
+    image_list = list(images)
+    image_list.sort()
+
+    for image in image_list:
+        if not _image_exist(image):
+            raise RuntimeError(f"Image {image} not found")
+
+    return image_list
+
+
+def _check_ray_commit_in_image(byod_image: str) -> None:
+    docker_ray_commit = (
+        subprocess.check_output(
+            [
+                "docker",
+                "run",
+                "-ti",
+                "--entrypoint",
+                "python",
+                byod_image,
+                "-c",
+                "import ray; print(ray.__commit__)",
+            ],
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    if os.environ.get("RAY_IMAGE_TAG"):
+        logger.info(f"Ray commit from image: {docker_ray_commit}")
+    else:
+        expected_ray_commit = _get_ray_commit()
+        assert (
+            docker_ray_commit == expected_ray_commit
+        ), f"Expected ray commit {expected_ray_commit}, found {docker_ray_commit}"
+
+
+def _push_image(byod_image: str) -> None:
+    logger.info(f"Pushing image to registry: {byod_image}")
+    subprocess.check_call(
+        ["docker", "push", byod_image],
+        stdout=sys.stderr,
+    )
+
+
+def _validate_and_push(byod_image: str) -> None:
+    """
+    Validates the given image and pushes it to ECR.
+    """
+    docker_ray_commit = (
+        subprocess.check_output(
+            [
+                "docker",
+                "run",
+                "-ti",
+                "--entrypoint",
+                "python",
+                byod_image,
+                "-c",
+                "import ray; print(ray.__commit__)",
+            ],
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    if os.environ.get("RAY_IMAGE_TAG"):
+        logger.info(f"Ray commit from image: {docker_ray_commit}")
+    else:
+        expected_ray_commit = _get_ray_commit()
+        assert (
+            docker_ray_commit == expected_ray_commit
+        ), f"Expected ray commit {expected_ray_commit}, found {docker_ray_commit}"
+    logger.info(f"Pushing image to registry: {byod_image}")
+    subprocess.check_call(
+        ["docker", "push", byod_image],
+        stdout=sys.stderr,
+    )
+
+
+def _get_ray_commit(envs: Optional[Dict[str, str]] = None) -> str:
+    if envs is None:
+        envs = os.environ
+    for key in [
+        "RAY_WANT_COMMIT_IN_IMAGE",
+        "COMMIT_TO_TEST",
+        "BUILDKITE_COMMIT",
+    ]:
+        commit = envs.get(key, "")
+        if commit:
+            return commit
+    return ""
+
+
+def _image_exist(image: str) -> bool:
+    """
+    Checks if the given image exists in Docker
+    """
+    p = subprocess.run(
+        ["docker", "manifest", "inspect", image],
+        stdout=sys.stderr,
+        stderr=sys.stderr,
+    )
+    return p.returncode == 0
+
+
+def _tag_and_push(source: str, target: str) -> None:
+    subprocess.check_call(
+        ["docker", "tag", source, target],
+        stdout=sys.stderr,
+    )
+    subprocess.check_call(
+        ["docker", "push", target],
+        stdout=sys.stderr,
+    )

release/ray_release/byod/build_context.py

@@ -0,0 +1,159 @@
+import hashlib
+import json
+import os
+import shutil
+from typing import Dict, List, Optional
+
+from typing_extensions import TypedDict
+
+_INSTALL_PYTHON_DEPS_SCRIPT = """\
+#!/bin/bash
+
+set -euo pipefail
+
+LOCK_FILE="${1:-python_depset.lock}"
+
+if [[ ! -f "${LOCK_FILE}" ]]; then
+    echo "Lock file ${LOCK_FILE} does not exist" >/dev/stderr
+    exit 1
+fi
+
+uv pip install --system --no-deps --index-strategy unsafe-best-match \\
+    -r "${LOCK_FILE}"
+"""
+
+
+class BuildContext(TypedDict, total=False):
+    """
+    Build context for custom BYOD image builds.
+
+    Attributes:
+        envs: Environment variables to set in the image.
+        post_build_script: Filename of the post-build script.
+        post_build_script_digest: SHA256 digest of the post-build script.
+        python_depset: Filename of the Python dependencies lock file.
+        python_depset_digest: SHA256 digest of the Python dependencies lock file.
+        install_python_deps_script_digest: SHA256 digest of the install script.
+    """
+
+    envs: Dict[str, str]
+
+    post_build_script: str
+    post_build_script_digest: str
+
+    python_depset: str
+    python_depset_digest: str
+    install_python_deps_script_digest: str
+
+
+def make_build_context(
+    base_dir: str,
+    envs: Optional[Dict[str, str]] = None,
+    post_build_script: Optional[str] = None,
+    python_depset: Optional[str] = None,
+) -> BuildContext:
+    """
+    Create a BuildContext with computed file digests.
+
+    Args:
+        base_dir: Directory containing the source files.
+        envs: Environment variables to set in the image.
+        post_build_script: Filename of the post-build script.
+        python_depset: Filename of the Python dependencies lock file.
+
+    Returns:
+        A BuildContext with filenames and their SHA256 digests.
+    """
+    ctx: BuildContext = {}
+
+    if envs:
+        ctx["envs"] = envs
+
+    if post_build_script:
+        ctx["post_build_script"] = post_build_script
+        path = os.path.join(base_dir, post_build_script)
+        ctx["post_build_script_digest"] = _sha256_file(path)
+
+    if python_depset:
+        ctx["python_depset"] = python_depset
+        path = os.path.join(base_dir, python_depset)
+        ctx["python_depset_digest"] = _sha256_file(path)
+        ctx["install_python_deps_script_digest"] = _sha256_str(
+            _INSTALL_PYTHON_DEPS_SCRIPT
+        )
+
+    return ctx
+
+
+def encode_build_context(ctx: BuildContext) -> str:
+    """Encode a BuildContext to deterministic minified JSON."""
+    return json.dumps(ctx, sort_keys=True, separators=(",", ":"))
+
+
+def decode_build_context(data: str) -> BuildContext:
+    """Decode a JSON string to a BuildContext."""
+    return json.loads(data)
+
+
+def build_context_digest(ctx: BuildContext) -> str:
+    """Compute SHA256 digest of the encoded BuildContext."""
+    encoded = encode_build_context(ctx)
+    digest = hashlib.sha256(encoded.encode()).hexdigest()
+    return f"sha256:{digest}"
+
+
+def fill_build_context_dir(
+    ctx: BuildContext,
+    source_dir: str,
+    context_dir: str,
+) -> None:
+    """
+    Generate Dockerfile and copy source files to the build directory.
+
+    Args:
+        ctx: The BuildContext specifying what to include.
+        source_dir: Source directory containing the original files.
+        context_dir: Target directory for the generated Dockerfile and copied files.
+    """
+    dockerfile: List[str] = ["# syntax=docker/dockerfile:1.3-labs"]
+    dockerfile.append("ARG BASE_IMAGE")
+    dockerfile.append("FROM ${BASE_IMAGE}")
+
+    if "envs" in ctx and ctx["envs"]:
+        dockerfile.append("ENV \\")
+        env_lines = [f"  {k}={v}" for k, v in sorted(ctx["envs"].items())]
+        dockerfile.append(" \\\n".join(env_lines))
+
+    if "python_depset" in ctx:
+        shutil.copy(
+            os.path.join(source_dir, ctx["python_depset"]),
+            os.path.join(context_dir, "python_depset.lock"),
+        )
+        with open(os.path.join(context_dir, "install_python_deps.sh"), "w") as f:
+            f.write(_INSTALL_PYTHON_DEPS_SCRIPT)
+        dockerfile.append("COPY install_python_deps.sh /tmp/install_python_deps.sh")
+        dockerfile.append("COPY python_depset.lock python_depset.lock")
+        dockerfile.append("RUN bash /tmp/install_python_deps.sh python_depset.lock")
+
+    if "post_build_script" in ctx:
+        shutil.copy(
+            os.path.join(source_dir, ctx["post_build_script"]),
+            os.path.join(context_dir, "post_build_script.sh"),
+        )
+        dockerfile.append("COPY post_build_script.sh /tmp/post_build_script.sh")
+        dockerfile.append("RUN bash /tmp/post_build_script.sh")
+
+    dockerfile_path = os.path.join(context_dir, "Dockerfile")
+    with open(dockerfile_path, "w") as f:
+        f.write("\n".join(dockerfile) + "\n")
+
+
+def _sha256_file(path: str) -> str:
+    with open(path, "rb") as f:
+        digest = hashlib.sha256(f.read()).hexdigest()
+    return f"sha256:{digest}"
+
+
+def _sha256_str(content: str) -> str:
+    digest = hashlib.sha256(content.encode()).hexdigest()
+    return f"sha256:{digest}"

release/ray_release/byod/byod.Dockerfile

@@ -0,0 +1,74 @@
+# syntax=docker/dockerfile:1.3-labs
+# shellcheck disable=SC2148
+
+ARG BASE_IMAGE
+FROM "$BASE_IMAGE"
+
+ARG PYTHON_VERSION=3.10
+ARG IMAGE_TYPE="ray"
+ARG PIP_REQUIREMENTS="python/deplocks/base_extra_testdeps/${IMAGE_TYPE}-base_extra_testdeps_py${PYTHON_VERSION}.lock"
+
+COPY "$PIP_REQUIREMENTS" extra-test-requirements.txt
+
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
+APT_PKGS=(
+    apt-transport-https
+    ca-certificates
+    htop
+    libaio1
+    libgl1-mesa-glx
+    libglfw3
+    libjemalloc-dev
+    libosmesa6-dev
+    lsb-release
+    patchelf
+)
+
+sudo apt-get update -y
+sudo apt-get install -y --no-install-recommends "${APT_PKGS[@]}"
+sudo apt-get autoclean
+sudo rm -rf /etc/apt/sources.list.d/*
+
+sudo mkdir -p /etc/apt/keyrings
+curl -sLS https://packages.microsoft.com/keys/microsoft.asc |
+  gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null
+sudo chmod go+r /etc/apt/keyrings/microsoft.gpg
+
+AZ_VER=2.72.0
+AZ_DIST="$(lsb_release -cs)"
+echo "Types: deb
+URIs: https://packages.microsoft.com/repos/azure-cli/
+Suites: ${AZ_DIST}
+Components: main
+Architectures: $(dpkg --print-architecture)
+Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources
+
+sudo apt-get update -y
+sudo apt-get install -y azure-cli="${AZ_VER}"-1~"${AZ_DIST}"
+
+git clone --branch=4.2.0 --depth=1 https://github.com/wg/wrk.git /tmp/wrk
+make -C /tmp/wrk -j
+sudo cp /tmp/wrk/wrk /usr/local/bin/wrk
+rm -rf /tmp/wrk
+
+"$HOME/anaconda3/bin/pip" install --no-cache-dir -r extra-test-requirements.txt
+
+EOF
+
+# RAY_BACKEND_LOG_JSON=1
+#   Uses JSON structured logging.
+#
+# RAY_DATA_LOG_INTERNAL_STACK_TRACE_TO_STDOUT=1
+#   Logs the full stack trace from Ray Data in case of exception,
+#   which is useful for debugging failures.
+#
+# RAY_DATA_AUTOLOAD_PYEXTENSIONTYPE=1
+#   To make ray data compatible across multiple pyarrow versions.
+ENV \
+  RAY_BACKEND_LOG_JSON=1 \
+  RAY_DATA_LOG_INTERNAL_STACK_TRACE_TO_STDOUT=1 \
+  RAY_DATA_AUTOLOAD_PYEXTENSIONTYPE=1

release/ray_release/byod/byod_agent_stress_test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the agent stress test.
+
+set -exo pipefail
+
+pip install https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl

release/ray_release/byod/byod_asynchronous_inference.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install dependencies
+pip3 install --no-cache-dir \
+    "ray[serve-async-inference]>=2.50.0" \
+    "requests>=2.31.0" \
+    "PyPDF2>=3.0.0" \
+    "celery[redis]>=5.3.0"

release/ray_release/byod/byod_azure_cluster_launcher.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# This script is used to build an extra layer to run release tests on Azure.
+
+set -exo pipefail
+
+pip3 install azure-cli-core==2.21.0 azure-core azure-identity azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-common msrest msrestazure

release/ray_release/byod/byod_data_pipelined_test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the horovod tests
+
+set -exo pipefail
+
+pip install -U git+https://github.com/ray-project/ray_shuffling_data_loader.git@add-embedding-model
+HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip3 install -U git+https://github.com/horovod/horovod.git

release/ray_release/byod/byod_dataset_shuffle.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the agent stress test.
+
+set -exo pipefail
+
+{
+  echo "yes N | sudo mkfs -t ext4 /dev/nvme1n1 || true"
+  echo "mkdir -p /tmp/data0"
+  echo "mkdir -p /tmp/data1"
+  echo "sudo chmod 0777 /tmp/data0"
+  echo "sudo chmod 0777 /tmp/data1"
+  echo "sudo mount /dev/nvme1n1 /tmp/data1 || true"
+} >> ~/.bashrc

release/ray_release/byod/byod_deployment_serve_llm.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+set -exo pipefail

release/ray_release/byod/byod_dolly_test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the agent stress test.
+
+set -exo pipefail
+
+pip3 install -c "$HOME/requirements_compiled.txt" myst-parser myst-nb
+
+pip3 uninstall -y pytorch-lightning
+pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+
+pip3 install lightning==2.0.3

release/ray_release/byod/byod_e2e_audio.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    accelerate==1.7.0 \
+    datasets[audio]==2.2.1 \
+    flashinfer-python==0.2.2.post1 \
+    huggingface-hub[hf_xet]==0.32.6 \
+    pydantic==2.9.2 \
+    transformers==4.52.4 \
+    xgrammar==0.1.19

release/ray_release/byod/byod_e2e_multimodal_ai_workloads.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    "matplotlib==3.10.0" \
+    "torch==2.7.1" \
+    "transformers==4.52.3" \
+    "scikit-learn==1.6.0" \
+    "mlflow==2.19.0" \
+    "ipywidgets==8.1.3"

release/ray_release/byod/byod_e2e_rag.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install system dependencies
+sudo apt-get update && \
+    sudo apt-get install -y libgl1-mesa-glx libmagic1 poppler-utils tesseract-ocr libreoffice && \
+    sudo rm -rf /var/lib/apt/lists/*
+
+# Install python dependencies
+pip3 install --no-cache-dir \
+    "unstructured[all-docs]==0.16.23" \
+    "sentence-transformers==3.4.1" \
+    "chromadb==0.6.3" \
+    "langchain_text_splitters==0.3.6" \
+    "pandas==2.2.3" \
+    "tiktoken==0.9.0"

release/ray_release/byod/byod_e2e_timeseries.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Set bash options for safer script execution:
+# -e: Exit immediately if any command fails
+# -x: Print each command before executing it (for debugging)
+# -o pipefail: Fail if any command in a pipeline fails (not just the last one)
+set -exo pipefail
+
+# Install Python dependencies.
+pip3 install --no-cache-dir \
+    aiohttp==3.11.16 \
+    nbformat==5.9.2 \
+    numpy==1.26.4 \
+    pandas==2.3.0 \
+    pyyaml==6.0.1 \
+    s3fs==2023.5.0 \
+    scikit-learn==1.3.2 \
+    torch==2.3.0

release/ray_release/byod/byod_e2e_xgboost.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -exo pipefail
+
+pip3 install --no-cache-dir mlflow==2.19.0 scikit-learn==1.6.0 xgboost>=3.0.0

release/ray_release/byod/byod_finetune_llvms.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the workspace_template_finetuning_llms_with_deepspeed_llama_2_7b test
+
+set -exo pipefail
+
+pip3 install -U \
+    torch==2.1.1 \
+    deepspeed==0.10.2 \
+    fairscale==0.4.13 \
+    datasets==2.14.4 \
+    accelerate==0.21.0 \
+    evaluate==0.4.0 \
+    wandb==0.23.1 \
+    pytorch-lightning==2.0.6 \
+    protobuf \
+    torchmetrics==1.0.3 \
+    sentencepiece==0.1.99 \
+    "urllib3<1.27" \
+    transformers==4.36.2 \
+    peft==0.7.0
+pip3 install -U flash-attn==2.4.2 --no-build-isolation

release/ray_release/byod/byod_gptj_test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -exo pipefail
+
+pip3 install -c "$HOME/requirements_compiled.txt" myst-parser myst-nb
+
+pip3 install "accelerate==0.33.0"
+pip3 uninstall -y peft

release/ray_release/byod/byod_hello_world.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install Python dependencies
+uv pip install -r python_depset.lock --system --no-deps --index-strategy unsafe-best-match

release/ray_release/byod/byod_horovod_master_test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the horovod tests
+
+set -exo pipefail
+
+pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \
+  torch-scatter==2.1.1+pt20cu118 torch-sparse==0.6.17+pt20cu118 \
+  torch-cluster==1.6.1+pt20cu118 torch-spline-conv==1.2.2+pt20cu118 \
+  --extra-index-url https://download.pytorch.org/whl/cu118 \
+  --find-links https://data.pyg.org/whl/torch-2.0.1+cu118.html
+
+HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip3 install -U git+https://github.com/horovod/horovod.git
+
+# Required because horovod is compiled from source on this image, which has
+# a higher version of libstdc++, not compatible with the conda environment.
+ln -sf /usr/lib/gcc/x86_64-linux-gnu/11/libstdc++.so /home/ray/anaconda3/lib/libstdc++.so
+ln -sf /usr/lib/gcc/x86_64-linux-gnu/11/libstdc++.so /home/ray/anaconda3/lib/libstdc++.so.6

release/ray_release/byod/byod_horovod_test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the horovod tests
+
+set -exo pipefail
+
+pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \
+  torch-scatter==2.1.1+pt20cu118 torch-sparse==0.6.17+pt20cu118 \
+  torch-cluster==1.6.1+pt20cu118 torch-spline-conv==1.2.2+pt20cu118 \
+  --extra-index-url https://download.pytorch.org/whl/cu118 \
+  --find-links https://data.pyg.org/whl/torch-2.0.1+cu118.html
+
+HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip3 install -U horovod
+
+# Required because horovod is compiled from source on this image, which has
+# a higher version of libstdc++, not compatible with the conda environment.
+ln -sf /usr/lib/gcc/x86_64-linux-gnu/11/libstdc++.so /home/ray/anaconda3/lib/libstdc++.so
+ln -sf /usr/lib/gcc/x86_64-linux-gnu/11/libstdc++.so /home/ray/anaconda3/lib/libstdc++.so.6

release/ray_release/byod/byod_huggingface_transformers_test.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the Hugging Face Transformers release test.
+
+set -exo pipefail
+
+# Update accelerate version
+pip3 install accelerate==0.32.0
+pip3 install peft==0.10.0

release/ray_release/byod/byod_huggingface_trl.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the Hugging Face TRL example release test.
+
+set -exo pipefail
+
+# Install TRL and math_verify
+pip3 install --no-cache-dir "trl[vllm]==1.0.0" math_verify
+pip3 install --no-cache-dir --force-reinstall numpy pandas
+# vllm requires numpy>=2.x; upgrade matplotlib to a version compatible with numpy 2.x.
+pip3 install --no-cache-dir --upgrade matplotlib
+# `transformers` auto-imports TF when present and TF's bundled grpc collides with Ray's grpc on the `grpc_experiments` absl flag
+pip3 uninstall -y wandb comet_ml tensorflow tf_keras

release/ray_release/byod/byod_install_mosaicml.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+pip3 install mosaicml-streaming==0.5.1

release/ray_release/byod/byod_install_multimodal_inference_benchmarks_transcription.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+uv pip install -r python_depset.lock --system --no-deps --index-strategy unsafe-best-match

release/ray_release/byod/byod_install_pybase64.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+pip3 install --no-cache-dir pybase64==1.4.2

release/ray_release/byod/byod_install_pyiceberg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+pip3 install --no-cache-dir "pyiceberg[glue,s3fs]==0.11.0"

release/ray_release/byod/byod_install_text_embedding.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+pip3 install --no-cache-dir --upgrade-strategy only-if-needed \
+  transformers==4.56.2 \
+  sentence-transformers==5.1.0 \
+  torch==2.8.0

release/ray_release/byod/byod_install_torchrec.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# shellcheck disable=SC2102
+
+set -exo pipefail
+
+pip3 install torch==2.6.0 \
+  torchrec==1.1.0 \
+  fbgemm-gpu==1.1.0 \
+  --extra-index-url https://download.pytorch.org/whl/cu121

release/ray_release/byod/byod_langchain_agent_ray_serve.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# Python dependencies
+pip3 install --no-cache-dir \
+    "fastapi==0.115.12" \
+    "langchain==1.0.5" \
+    "langchain-mcp-adapters==0.1.12" \
+    "langchain-openai==1.0.2" \
+    "langgraph==1.0.3" \
+    "openai==2.7.2" \
+    "uvicorn==0.38.0" \
+    "httpx==0.27.1" \
+    "mcp==1.22.0"

release/ray_release/byod/byod_llamafactory_llm_fine_tune.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Python dependencies
+pip3 install --no-cache-dir \
+    "llamafactory==0.9.3" \
+    "deepspeed==0.16.9" \
+    "wandb==0.23.1" \
+    "tensorboard==2.20.0" \
+    "mlflow==3.4.0" \
+    "bitsandbytes==0.47.0" \
+    "autoawq==0.2.9" \
+    "liger-kernel==0.6.2"

release/ray_release/byod/byod_llm_batch_inference_text.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+set -exo pipefail

release/ray_release/byod/byod_llm_batch_inference_vision.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Python dependencies
+pip3 install --no-cache-dir "datasets==4.4.2"

release/ray_release/byod/byod_llm_kv_router_test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base llm image
+# to run the KV aware router release tests.
+
+set -exo pipefail
+
+# TODO (jeffreywang): Move to official wheels once dynamo changes are upstreamed.
+cat > /tmp/ai-dynamo-runtime.txt <<'EOF'
+ai-dynamo-runtime @ https://air-example-data.s3.us-west-2.amazonaws.com/rayllm-ossci/dynamo/298f1e0/ai_dynamo_runtime-1.3.0-cp310-abi3-manylinux_2_35_x86_64.whl \
+    --hash=sha256:107d78a5714962f568cb0578890362f8eeaf389b41c02b119925ed3d9830f484
+EOF
+pip3 install --no-deps --require-hashes -r /tmp/ai-dynamo-runtime.txt

release/ray_release/byod/byod_llm_lmcache_test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base llm image
+# to run the llm sglang release tests
+
+set -exo pipefail
+
+pip3 install "lmcache==0.3.3"

release/ray_release/byod/byod_llm_ner.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Will use lockfile instead later
+# pip3 install --no-cache-dir -r https://raw.githubusercontent.com/anyscale/e2e-llm-workflows/refs/heads/main/lockfile.txt
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    "xgrammar==0.1.11" \
+    "pynvml==12.0.0" \
+    "hf_transfer==0.1.9" \
+    "tensorboard==2.19.0" \
+    "git+https://github.com/hiyouga/LLaMA-Factory.git@ac8c6fdd3ab7fb6372f231f238e6b8ba6a17eb16#egg=llamafactory"
+
+
+# Env vars
+export HF_HUB_ENABLE_HF_TRANSFER=1

release/ray_release/byod/byod_llm_pin_cupy_cuda12x.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# The LLM BYOD extra-testdeps lock can resolve a newer cupy-cuda12x than the
+# ray-llm base image (see python/deplocks/llm/rayllm_py312_cu130.lock).
+# That breaks Ray's NCCL collectives, which import CuPy. Re-pin to the same
+# version as the LLM base after BYOD pip install.
+pip3 install --no-cache-dir "cupy-cuda12x==13.6.0"

release/ray_release/byod/byod_llm_sglang_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base llm image
+# to run the llm sglang release tests
+
+set -exo pipefail
+pip3 uninstall -y vllm
+pip3 install "sglang[all,ray]==0.5.10rc0"
+# Reinstall opentelemetry-proto to regenerate _pb2.py files compatible with
+# the protobuf version that sglang pulls in (protobuf 4.x+ removed old-style
+# descriptor creation, causing Ray dashboard to crash on startup).
+pip3 install --upgrade opentelemetry-proto opentelemetry-sdk opentelemetry-exporter-prometheus

release/ray_release/byod/byod_llm_single_node_baseline_benchmark.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -exo pipefail
+
+uv pip install --system --no-deps --index-strategy unsafe-best-match -r python_depset.lock

release/ray_release/byod/byod_mcp-ray-serve.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Python dependencies
+pip3 install --no-cache-dir \
+    "mcp==1.11.0" \
+    "asyncio==3.4.3" \
+    "pydantic==2.9.2"
+
+# Podman (used in stdio examples)
+sudo apt-get update && sudo apt-get install -y podman

release/ray_release/byod/byod_model_composition_recsys.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+set -euxo pipefail

release/ray_release/byod/byod_model_multiplexing_forecast.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Post-build script for model multiplexing forecast example
+# Install dependencies needed for tests
+
+set -exo pipefail

release/ray_release/byod/byod_multi_agent_a2a.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+pip install --no-cache-dir \
+    "fastapi==0.115.12" \
+    "langchain==1.0.5" \
+    "langchain-mcp-adapters==0.1.12" \
+    "langchain-openai==1.0.2" \
+    "langgraph==1.0.3" \
+    "openai==2.7.2" \
+    "uvicorn==0.38.0" \
+    "a2a-sdk[http-server]==0.3.22" \
+    "httpx==0.28.1" \
+    "mcp==1.22.0" \
+    "protego==0.5.0" \
+    "readabilipy==0.3.0" \
+    "markdownify==0.14.1" \
+    "pytest==9.0.2"

release/ray_release/byod/byod_object_detection.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the object detection example notebooks.
+
+set -exo pipefail
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    boto3==1.26.76 \
+    imageio-ffmpeg==0.6.0 \
+    opencv-python-headless==4.11.0.86 \
+    pillow==11.1.0 \
+    pycocotools==2.0.8 \
+    requests==2.32.5 \
+    smart-open==6.2.0 \
+    torch==2.6.0 \
+    torchvision==0.21.0 \
+    xmltodict==0.14.2 \
+    torchmetrics==1.6.1 \
+    decord==0.6.0 \
+    jupytext==0.6.5

release/ray_release/byod/byod_pytorch_lightning_test.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the PyTorch Lightning release test.
+
+set -exo pipefail
+
+# Replace pytorch-lightning with lightning
+pip3 uninstall -y pytorch-lightning
+pip3 install lightning==2.4.0

release/ray_release/byod/byod_ray_lightning_master_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the agent stress test.
+
+set -exo pipefail
+
+pip3 install -U --force-reinstall pytorch-lightning lightning-bolts
+pip uninstall ray_lightning -y # Uninstall first so pip does a reinstall.
+pip3 install -U --no-cache-dir git+https://github.com/ray-project/ray_lightning#ray_lightning
+pip3 install --force-reinstall torch==1.11.0
+pip3 install --force-reinstall torchvision==0.12.0

release/ray_release/byod/byod_ray_lightning_test.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the agent stress test.
+
+set -exo pipefail
+
+pip3 install -U --force-reinstall ray-lightning pytorch-lightning lightning-bolts
+pip3 install --force-reinstall torch==1.11.0
+pip3 install --force-reinstall torchvision==0.12.0

release/ray_release/byod/byod_ray_train_workloads.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Will use lockfile instead later
+# pip3 install --no-cache-dir -r https://raw.githubusercontent.com/anyscale/e2e-llm-workflows/refs/heads/main/lockfile.txt
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    "torch==2.8.0" \
+    "torchvision==0.23.0" \
+    "matplotlib==3.10.6" \
+    "pyarrow==17.0.0" \
+    "datasets==2.19.2" \
+    "lightning==2.5.5" \
+    "scikit-learn==1.7.2" \
+    "xgboost-cpu==3.0.5" \
+    "seaborn==0.13.2" \
+    "statsmodels==0.14.5" \
+    "pycocotools==2.0.10" \
+    "transformers==4.56.2" \
+    "accelerate==1.10.1"
+
+# Env vars
+export RAY_TRAIN_V2_ENABLED=1
+# DO NOT hardcode HUGGING_FACE_HUB_TOKEN here — set it in Workspace Secrets instead

release/ray_release/byod/byod_rllib.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -exo pipefail
+
+pip3 install -U "gymnasium[mujoco]"==1.2.2 ale_py==0.10.1 imageio==2.34.2 opencv-python-headless==4.9.0.80
+pip3 install -U torch==2.7 torchvision==0.22 --index-url https://download.pytorch.org/whl/cu128
+pip3 install -U pettingzoo==1.24.3
+pip3 install -U pygame wandb

release/ray_release/byod/byod_serve_video_analysis.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# Install Python dependencies
+pip3 install --no-cache-dir \
+    "numpy>=1.24.0,<2.0" \
+    "torch>=2.0.0" \
+    "transformers>=4.35.0" \
+    "accelerate>=0.25.0" \
+    "sentencepiece>=0.1.99" \
+    "httpx>=0.25.0" \
+    "aioboto3>=12.0.0" \
+    "pillow>=10.0.0"
+
+
+sudo apt-get update && sudo apt-get install -y --no-install-recommends \
+    ffmpeg \
+    && sudo rm -rf /var/lib/apt/lists/*
+
+export S3_BUCKET=anyscale-example-video-analysis-test-bucket

release/ray_release/byod/byod_stable_diffusion.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run stable diffusion test.
+
+set -exo pipefail
+
+pip3 install transformers==4.31.0 diffusers==0.21.3

release/ray_release/byod/byod_tensor_parallel_autotp.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -exo pipefail
+
+pip3 install --no-cache-dir transformers==4.48.0 datasets==2.21.0
+pip3 install --no-cache-dir --no-build-isolation deepspeed==0.18.9

release/ray_release/byod/byod_train_persistence_test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the train_multinode_persistence test.
+
+set -exo pipefail
+
+pip3 install -U torch fsspec s3fs gcsfs pyarrow>=17.0.0 pytest

release/ray_release/byod/byod_tune_pytorch_asha.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -exo pipefail
+
+pip3 install --no-cache-dir \
+    "torch==2.8.0" \
+    "torchvision==0.23.0" \
+    "matplotlib==3.10.6" \
+    "pyarrow==14.0.2"

release/ray_release/byod/byod_unstructured_data_ingestion.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -exo pipefail
+
+sudo apt-get update -y
+sudo apt-get install --no-install-recommends -y libgl1-mesa-glx libmagic1 poppler-utils tesseract-ocr libreoffice
+sudo rm -f /etc/apt/sources.list.d/*
+
+# Install runtime deps
+pip install "unstructured[all-docs]==0.18.21"
+pip install --force-reinstall --no-cache-dir pandas==2.3.3

release/ray_release/byod/byod_vicuna_test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# This script is used to build an extra layer on top of the base anyscale/ray image
+# to run the air_example_vicuna_13b_lightning_deepspeed_finetuning test.
+
+set -exo pipefail
+
+cat >> "$HOME/.bashrc" <<EOF
+sudo lsblk -f
+yes N | sudo mkfs -t ext4 /dev/nvme1n1 || true
+mkdir -p /mnt/local_storage
+sudo chmod 0777 /mnt/local_storage
+sudo mount /dev/nvme1n1 /mnt/local_storage || true
+EOF
+
+pip3 install -c "$HOME/requirements_compiled.txt" myst-parser myst-nb
+
+pip3 uninstall -y pytorch-lightning
+pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+
+pip3 install lightning==2.0.3

release/ray_release/byod/emoji.lock

@@ -0,0 +1,6 @@
+--index-url https://pypi.org/simple
+--extra-index-url https://download.pytorch.org/whl/cpu
+
+emoji==2.10.0 \
+    --hash=sha256:7e68435eecd2c428c3b4aaa5f72d61a5b1a36c81a5138681cba13d19d94aa3a0 \
+    --hash=sha256:aed4332caa23553a7218f032c08b0a325ae53b010f7fb98ad272c0f7841bc1d3

release/ray_release/byod/requirements_byod_3.10.in

@@ -0,0 +1,42 @@
+# Python requirements to run release tests from anyscale byod (cpu type)
+
+ale-py
+boto3
+cmake
+crc32c
+cython
+fastapi
+gcsfs==2023.12.1
+gsutil
+gymnasium
+gymnasium[atari]
+httpx
+importlib-metadata
+jsonschema
+lightgbm
+locust==2.18.0
+memray
+openskill
+orjson
+petastorm
+protobuf
+pyarrow
+pydantic>=2.5.0
+pytest
+pyyaml
+requests>=2.31.0
+semidbm
+s3fs
+scikit-learn
+scipy
+tblib
+terminado
+tensorboardx==2.6.2.2
+tensorflow
+trueskill
+tqdm
+typer
+typing-extensions
+xarray
+xgboost
+zarr

release/ray_release/byod/requirements_byod_3.11.in

@@ -0,0 +1 @@
+# Python requirements to run release tests from anyscale byod (cpu type)

release/ray_release/byod/requirements_byod_3.11.txt

@@ -0,0 +1,6 @@
+#
+# This file is autogenerated by pip-compile with python 3.11
+# To update, run:
+#
+#    bazel run //release:requirements_byod_3.11.update
+#

release/ray_release/byod/requirements_byod_3.12.in

@@ -0,0 +1 @@
+# Python requirements to run release tests from anyscale byod (cpu type)

release/ray_release/byod/requirements_byod_3.12.txt

@@ -0,0 +1,6 @@
+#
+# This file is autogenerated by pip-compile with python 3.12
+# To update, run:
+#
+#    bazel run //release:requirements_byod_3.12.update
+#

release/ray_release/byod/requirements_byod_3.13.in

@@ -0,0 +1 @@
+# Python requirements to run release tests from anyscale byod (cpu type)

release/ray_release/byod/requirements_byod_3.14.in

@@ -0,0 +1 @@
+# Python requirements to run release tests from anyscale byod (cpu type)

release/ray_release/byod/requirements_byod_gpu_3.10.in

@@ -0,0 +1,42 @@
+# Python requirements to run release tests from anyscale byod (gpu type)
+
+ale-py
+boto3
+cmake
+crc32c
+cython
+fastapi
+gcsfs==2023.12.1
+gsutil
+gymnasium
+gymnasium[atari]
+httpx
+importlib-metadata
+jsonschema
+lightgbm
+locust==2.18.0
+memray
+openskill
+orjson
+petastorm
+protobuf
+pyarrow
+pydantic>=2.5.0
+pytest
+pyyaml
+requests>=2.32.5
+semidbm
+s3fs
+scikit-learn
+scipy
+tblib
+terminado
+tensorboardx==2.6.2.2
+tensorflow
+trueskill
+tqdm
+typer
+typing-extensions
+xarray
+xgboost
+zarr

release/ray_release/byod/requirements_byod_gpu_cu130.in

@@ -0,0 +1,6 @@
+# Extra cu130-only requirements layered onto the gpu-cu130 base image.
+# cupy-cuda12x is relaxed out of the cu130 ray_img chain (see rayimg.depsets.yaml)
+# and replaced with the matching CUDA 13.x build. Pinned to 13.6.0 to match the
+# CuPy 13.x line the published cu130 image swaps to (same API), and because
+# 13.6.0 is the floor with CUDA-13 wheels for py3.10-3.13.
+cupy-cuda13x==13.6.0

release/ray_release/byod/requirements_compiled.txt

@@ -0,0 +1 @@
+../../../python/requirements_compiled.txt

release/ray_release/byod/requirements_gpu_cu130.in

@@ -0,0 +1 @@
+torch==2.9.1

release/ray_release/byod/requirements_llm_byod_3.11.txt

@@ -0,0 +1,6 @@
+pytest-asyncio==0.21.2
+pytest-timeout==2.1.0
+locust==2.33.0
+orjson==3.10.15
+backoff==2.2.1
+langchain_text_splitters==0.3.9

release/ray_release/byod/requirements_llm_byod_3.12.txt

@@ -0,0 +1,6 @@
+pytest-asyncio==0.21.2
+pytest-timeout==2.1.0
+locust==2.33.0
+orjson==3.10.15
+backoff==2.2.1
+langchain_text_splitters==0.3.9

release/ray_release/byod/requirements_ml_byod_3.10.in

@@ -0,0 +1,60 @@
+# Python requirements to run release tests from anyscale byod (gpu type, python 3.9)
+
+-c requirements_compiled.txt
+accelerate
+bitsandbytes
+boto3
+cmake
+crc32c
+datasets
+decord
+deepspeed==0.17.2
+diffusers==0.12.1
+evaluate
+fairscale
+fastapi
+filelock
+gcsfs==2023.12.1
+gsutil
+ipywidgets
+jupytext
+lm_eval==0.4.0
+locust==2.18.0
+matplotlib
+memray
+modin
+# mosaicml-streaming
+numpy
+openai-whisper
+openskill
+orjson
+petastorm
+protobuf
+pyarrow
+pydantic>=2.5.0
+pytest
+pytorch-lightning
+scikit-learn
+semidbm
+sentencepiece
+statsforecast
+tblib
+tensorboardX
+tiktoken
+triton==3.3.0
+torch==2.7.0
+torchaudio
+torchmetrics
+torchtext
+tqdm
+transformers
+trueskill
+typepy>=1.3.2
+typer
+typing-extensions
+urllib3
+uvicorn
+validators
+wandb
+xgboost
+albumentations