chore: import upstream snapshot with attribution

release/benchmarks/README.md

@@ -0,0 +1,33 @@
+# Ray Scalability Envelope
+
+**NOTE**: the Ray scalability benchmarks are in the process of being refreshed. If you have questions about a specific workload or limit, please get in touch by filing a [GitHub issue](https://github.com/ray-project/ray/issues).
+
+## Distributed Benchmarks
+
+All distributed tests are run on 64 nodes with 64 cores/node. Maximum number of nodes is achieved by adding 4 core nodes.
+
+| Dimension                                       | Quantity |
+| ---------                                       | -------- |
+| # nodes in cluster (with trivial task workload) | 2k+      |
+| # actors in cluster (with trivial workload)     | 40k+     |
+| # simultaneously running tasks                  | 10k+     |
+| # simultaneously running placement groups       | 1k+      |
+
+## Object Store Benchmarks
+
+| Dimension                           | Quantity |
+| ---------                           | -------- |
+| 1 GiB object broadcast (# of nodes) | 50+      |
+
+
+## Single Node Benchmarks.
+
+All single node benchmarks are run on a single m4.16xlarge.
+
+| Dimension                                      | Quantity   |
+| ---------                                      | --------   |
+| # of object arguments to a single task         | 10000+     |
+| # of objects returned from a single task       | 3000+      |
+| # of plasma objects in a single `ray.get` call | 10000+     |
+| # of tasks queued on a single node             | 1,000,000+ |
+| Maximum `ray.get` numpy object size            | 100GiB+    |

release/benchmarks/distributed.yaml

@@ -0,0 +1,22 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: r5dn.16xlarge # Network optimized.
+    resources:
+      CPU: 0
+      node: 1
+      small: 1
+
+worker_nodes:
+    - instance_type: m5.16xlarge
+      min_nodes: 32
+      max_nodes: 32
+      market_type: ON_DEMAND
+      resources:
+        node: 1
+    - instance_type: r5.16xlarge
+      min_nodes: 32
+      max_nodes: 32
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/distributed/many_nodes_tests/actor_test.py

@@ -0,0 +1,145 @@
+import argparse
+import os
+import math
+from time import sleep, perf_counter
+import json
+import ray
+
+from dashboard_test import DashboardTestAtScale
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors):
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+    print("Start launch actors")
+    actors = [Actor.options(max_restarts=-1).remote() for _ in range(total_actors)]
+    return actors
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", nargs="+", type=int, required=True)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--no-wait", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def scale_cluster_up(num_cpus):
+    print(f"Start to scale up to {num_cpus} cpus")
+
+    def get_curr_cpus():
+        return int(sum([r.get("Resources", {}).get("CPU", 0) for r in ray.nodes()]))
+
+    step = 1000
+    curr_cpus = get_curr_cpus()
+    target_cpus = curr_cpus
+
+    while curr_cpus < num_cpus:
+        curr_cpus = get_curr_cpus()
+        new_target_cpus = min(curr_cpus + step, num_cpus)
+        if new_target_cpus != target_cpus:
+            target_cpus = new_target_cpus
+            ray.autoscaler.sdk.request_resources(num_cpus=target_cpus)
+        print(f"Waiting for cluster to be up: {curr_cpus}->{target_cpus}->{num_cpus}")
+        sleep(10)
+
+
+def get_head_node_cpus():
+    head_ip = ray.util.get_node_ip_address()
+    for node in ray.nodes():
+        if node["Alive"] and node["NodeManagerAddress"] == head_ip:
+            return int(node.get("Resources", {}).get("CPU", 0))
+    return 0
+
+
+def run_one(total_actors, cpus_per_actor, no_wait):
+    total_cpus = cpus_per_actor * total_actors + get_head_node_cpus()
+    total_cpus = int(math.ceil(total_cpus))
+    scale_cluster_up(total_cpus)
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(cpus_per_actor, total_actors)
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    actor_ready_start = perf_counter()
+    total_actors = len(actors)
+    objs = [actor.foo.remote() for actor in actors]
+
+    while len(objs) != 0:
+        timeout = None if no_wait else 30
+        objs_ready, objs = ray.wait(objs, num_returns=len(objs), timeout=timeout)
+        print(
+            f"Status: {total_actors - len(objs)}/{total_actors}, "
+            f"{perf_counter() - actor_ready_start}"
+        )
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    throughput = total_actors / (actor_ready_time + actor_launch_time)
+    print(f"Actor launch time: {actor_launch_time} ({total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({total_actors} actors)"
+    )
+    print(f"Through put: {throughput}")
+
+    return {
+        "actor_launch_time": actor_launch_time,
+        "actor_ready_time": actor_ready_time,
+        "total_time": actor_launch_time + actor_ready_time,
+        "num_actors": total_actors,
+        "throughput": throughput,
+    }
+
+
+def main():
+    args, unknown = parse_script_args()
+    args.total_actors.sort()
+
+    addr = ray.init(address="auto")
+    dashboard_test = DashboardTestAtScale(addr)
+
+    result = {}
+    for i in args.total_actors:
+        result[f"many_nodes_actor_tests_{i}"] = run_one(
+            i, args.cpus_per_actor, args.no_wait
+        )
+
+    # Print the results early so if failed in the future, we still
+    # can see it in the log.
+    print(f"Result: {json.dumps(result, indent=2)}")
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            perf = [
+                {
+                    "perf_metric_name": name,
+                    "perf_metric_value": r["throughput"],
+                    "perf_metric_type": "THROUGHPUT",
+                }
+                for (name, r) in result.items()
+            ]
+            result["perf_metrics"] = perf
+            dashboard_test.update_release_test_result(result)
+
+            print(f"Writing data into file: {os.environ['TEST_OUTPUT_JSON']}")
+            json.dump(result, out_file)
+
+    print("Test finished successfully!")
+    ray.shutdown()
+
+    # We need to make sure GCS cool down otherwise, testing infra
+    # might get timeout when fetching the result because when the driver
+    # got shutdown, many actors needs to be terminated which will
+    # overload GCS.
+    print("Sleep for 60s, waiting for the cluster to cool down.")
+    sleep(60)
+
+
+if __name__ == "__main__":
+    main()

release/benchmarks/distributed/many_nodes_tests/compute_config.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+# NFS needs to be disabled for this test, since the test spawns too many nodes
+# and may hit the limit on the # of clients.
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: as-feature-disable-nfs-mount
+          Value: "true"
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        DeleteOnTermination: true
+        VolumeSize: 30
+
+head_node:
+  instance_type: m5.16xlarge
+
+worker_nodes:
+  - instance_type: m6i.large
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/compute_config_gce.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+  - us-west1-c
+
+advanced_instance_config:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 30
+
+    # NFS needs to be disabled for this test, since the test spawns too many nodes
+    # and may hit the limit on the # of clients.
+    labels:
+      as-feature-disable-nfs-mount: "true"
+
+head_node:
+  instance_type: n2-standard-64
+
+worker_nodes:
+  - instance_type: n2-standard-2
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/dashboard_test.py

@@ -0,0 +1,185 @@
+import asyncio
+import time
+import urllib
+from typing import Dict, Optional, List
+from pprint import pprint
+
+import requests
+import ray
+import logging
+import os
+
+from collections import defaultdict
+from ray.util.state import list_nodes
+from ray._private.test_utils import get_system_metric_for_component
+from pydantic import BaseModel
+from ray.dashboard.utils import get_address_for_submission_client
+from ray.dashboard.modules.metrics.metrics_head import (
+    DEFAULT_PROMETHEUS_HOST,
+    PROMETHEUS_HOST_ENV_VAR,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def calc_p(latencies, percent):
+    if len(latencies) == 0:
+        return 0
+
+    return round(sorted(latencies)[int(len(latencies) / 100 * percent)] * 1000, 3)
+
+
+class Result(BaseModel):
+    success: bool
+    # endpoints -> list of latencies
+    result: Dict[str, List[float]]
+    # Dashboard memory usage in MB.
+    memory_mb: Optional[float]
+
+
+# Currently every endpoint is GET endpoints.
+endpoints = [
+    "/logical/actors",
+    "/nodes?view=summary",
+    "/",
+    "/api/cluster_status",
+    "/events",
+    "/api/jobs/",
+    "/api/v0/logs",
+    "/api/prometheus_health",
+]
+
+
+@ray.remote(num_cpus=0)
+class DashboardTester:
+    def __init__(self, interval_s: int = 1):
+        self.dashboard_url = get_address_for_submission_client(None)
+        # Ping interval for all endpoints.
+        self.interval_s = interval_s
+        # endpoint -> a list of latencies
+        self.result = defaultdict(list)
+
+    async def run(self):
+        await asyncio.gather(*[self.ping(endpoint) for endpoint in endpoints])
+
+    async def ping(self, endpoint):
+        """Synchronously call an endpoint."""
+        node_id = ray.get_runtime_context().get_node_id()
+        while True:
+            start = time.monotonic()
+            # for logs API, we should append node ID and glob.
+            if "/api/v0/logs" in endpoint:
+                glob_filter = "*"
+
+                options_dict = {"node_id": node_id, "glob": glob_filter}
+                url = (
+                    f"{self.dashboard_url}{endpoint}?"
+                    f"{urllib.parse.urlencode(options_dict)}"
+                )
+            else:
+                url = f"{self.dashboard_url}{endpoint}"
+
+            resp = requests.get(url, timeout=30)
+            elapsed = time.monotonic() - start
+
+            if resp.status_code == 200:
+                self.result[endpoint].append(time.monotonic() - start)
+            else:
+                try:
+                    resp.raise_for_status()
+                except Exception as e:
+                    logger.exception(e)
+            await asyncio.sleep(max(0, self.interval_s, elapsed))
+
+    def get_result(self):
+        return self.result
+
+
+class DashboardTestAtScale:
+    """This is piggybacked into existing scalability tests."""
+
+    def __init__(self, addr: ray._private.worker.RayContext):
+        self.addr = addr
+
+        # Schedule the actor on the current node (which is a head node).
+        current_node_ip = ray._private.worker.global_worker.node_ip_address
+        nodes = list_nodes(filters=[("node_ip", "=", current_node_ip)])
+        assert len(nodes) > 0, f"{current_node_ip} not found in the cluster"
+        node = nodes[0]
+        # Schedule on a head node.
+        self.tester = DashboardTester.options(
+            label_selector={ray._raylet.RAY_NODE_ID_KEY: node["node_id"]}
+        ).remote()
+
+        self.tester.run.remote()
+
+    def get_result(self):
+        """Get the result from the test.
+
+        Returns:
+            A tuple of success, and the result (Result object).
+        """
+        try:
+            result = ray.get(self.tester.get_result.remote(), timeout=60)
+        except ray.exceptions.GetTimeoutError:
+            return Result(success=False)
+
+        # Get the memory usage.
+        memories = get_system_metric_for_component(
+            "ray_component_uss_bytes",
+            "dashboard",
+            os.environ.get(PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST),
+        )
+
+        return Result(
+            success=True,
+            result=result,
+            memory_mb=max(memories) / 1.0e6 if memories else None,
+        )
+
+    def update_release_test_result(self, release_result: dict):
+        test_result = self.get_result()
+
+        def calc_endpoints_p(result, percent):
+            return {
+                # sort -> get PX -> convert second to ms -> round up.
+                endpoint: calc_p(latencies, percent)
+                for endpoint, latencies in result.items()
+            }
+
+        print("======Print per dashboard endpoint latencies======")
+        print("=====================P50==========================")
+        pprint(calc_endpoints_p(test_result.result, 50))
+        print("=====================P95==========================")
+        pprint(calc_endpoints_p(test_result.result, 95))
+        print("=====================P99==========================")
+        pprint(calc_endpoints_p(test_result.result, 99))
+
+        latencies = []
+        for per_endpoint_latencies in test_result.result.values():
+            latencies.extend(per_endpoint_latencies)
+        aggregated_metrics = {
+            "p50": calc_p(latencies, 50),
+            "p95": calc_p(latencies, 95),
+            "p99": calc_p(latencies, 99),
+        }
+
+        print("=====================Aggregated====================")
+        pprint(aggregated_metrics)
+
+        release_result["_dashboard_test_success"] = test_result.success
+        if test_result.success:
+            if "perf_metrics" not in release_result:
+                release_result["perf_metrics"] = []
+
+            release_result["perf_metrics"].extend(
+                [
+                    {
+                        "perf_metric_name": f"dashboard_{p}_latency_ms",
+                        "perf_metric_value": value,
+                        "perf_metric_type": "LATENCY",
+                    }
+                    for p, value in aggregated_metrics.items()
+                ]
+            )
+            release_result["_dashboard_memory_usage_mb"] = test_result.memory_mb

release/benchmarks/distributed/many_nodes_tests/multi_master_test.py

@@ -0,0 +1,90 @@
+import argparse
+import os
+from time import sleep, perf_counter
+import json
+import ray
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors, num_masters):
+    # By default, there are 50 groups, each group has 1 master and 99 slaves.
+    num_slaves_per_master = total_actors / num_masters - 1
+
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+        def create(self):
+            return [
+                Actor.options(max_restarts=-1).remote()
+                for _ in range(num_slaves_per_master)
+            ]
+
+    print("Start launch actors")
+    # The 50 masters are spreaded.
+    actors = [
+        Actor.options(max_restarts=-1, scheduling_strategy="SPREAD").remote()
+        for _ in range(num_masters)
+    ]
+    slaves_per_master = []
+    for master in actors:
+        slaves_per_master.append(master.create.remote())
+    for slaves in slaves_per_master:
+        actors.extend(ray.get(slaves))
+    return actors
+
+
+def test_actor_ready(actors):
+    remaining = [actor.foo.remote() for actor in actors]
+    ray.get(remaining)
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", type=int, default=5000)
+    parser.add_argument("--num-masters", type=int, default=50)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--fail", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def main():
+    args, unknown = parse_script_args()
+
+    ray.init(address="auto")
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(
+        args.cpus_per_actor, args.total_actors, args.num_masters
+    )
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    if args.fail:
+        sleep(10)
+        return
+    actor_ready_start = perf_counter()
+    test_actor_ready(actors)
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    print(f"Actor launch time: {actor_launch_time} ({args.total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({args.total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({args.total_actors} actors)"
+    )
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            results = {
+                "actor_launch_time": actor_launch_time,
+                "actor_ready_time": actor_ready_time,
+                "total_time": actor_launch_time + actor_ready_time,
+                "num_actors": args.total_actors,
+            }
+            json.dump(results, out_file)
+
+
+if __name__ == "__main__":
+    main()

release/benchmarks/distributed/test_many_actors.py

@@ -0,0 +1,90 @@
+import os
+import time
+
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray._private.state_api_test_utils import summarize_worker_startup_time
+
+is_smoke_test = True
+if "SMOKE_TEST" in os.environ:
+    MAX_ACTORS_IN_CLUSTER = 100
+else:
+    MAX_ACTORS_IN_CLUSTER = 10000
+    is_smoke_test = False
+
+
+def test_max_actors():
+    # TODO (Alex): Dynamically set this based on number of cores
+    cpus_per_actor = 0.25
+
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+    actors = [
+        Actor.remote()
+        for _ in tqdm.trange(MAX_ACTORS_IN_CLUSTER, desc="Launching actors")
+    ]
+
+    done = ray.get([actor.foo.remote() for actor in actors])
+    for result in done:
+        assert result is None
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+addr = ray.init(address="auto")
+
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+monitor_actor = test_utils.monitor_memory_usage()
+dashboard_test = DashboardTestAtScale(addr)
+
+start_time = time.time()
+test_max_actors()
+end_time = time.time()
+
+ray.get(monitor_actor.stop_run.remote())
+used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+print(f"Peak memory usage: {round(used_gb, 2)}GB")
+print(f"Peak memory usage per processes:\n {usage}")
+del monitor_actor
+
+# Get the dashboard result
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+rate = MAX_ACTORS_IN_CLUSTER / (end_time - start_time)
+try:
+    summarize_worker_startup_time()
+except Exception as e:
+    print("Failed to summarize worker startup time.")
+    print(e)
+
+print(
+    f"Success! Started {MAX_ACTORS_IN_CLUSTER} actors in "
+    f"{end_time - start_time}s. ({rate} actors/s)"
+)
+
+results = {
+    "actors_per_second": rate,
+    "num_actors": MAX_ACTORS_IN_CLUSTER,
+    "time": end_time - start_time,
+    "_peak_memory": round(used_gb, 2),
+    "_peak_process_memory": usage,
+}
+if not is_smoke_test:
+    results["perf_metrics"] = [
+        {
+            "perf_metric_name": "actors_per_second",
+            "perf_metric_value": rate,
+            "perf_metric_type": "THROUGHPUT",
+        }
+    ]
+dashboard_test.update_release_test_result(results)
+test_utils.safe_write_to_results_json(results)

release/benchmarks/distributed/test_many_pgs.py

@@ -0,0 +1,121 @@
+import os
+import time
+
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray.util.placement_group import placement_group, remove_placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+is_smoke_test = True
+if "SMOKE_TEST" in os.environ:
+    MAX_PLACEMENT_GROUPS = 20
+else:
+    MAX_PLACEMENT_GROUPS = 1000
+    is_smoke_test = False
+
+
+def test_many_placement_groups():
+    # @ray.remote(num_cpus=1, resources={"node": 0.02})
+    @ray.remote
+    class C1:
+        def ping(self):
+            return "pong"
+
+    # @ray.remote(num_cpus=1)
+    @ray.remote
+    class C2:
+        def ping(self):
+            return "pong"
+
+    # @ray.remote(resources={"node": 0.02})
+    @ray.remote
+    class C3:
+        def ping(self):
+            return "pong"
+
+    bundle1 = {"node": 0.02, "CPU": 1}
+    bundle2 = {"CPU": 1}
+    bundle3 = {"node": 0.02}
+
+    pgs = []
+    for _ in tqdm.trange(MAX_PLACEMENT_GROUPS, desc="Creating pgs"):
+        pg = placement_group(bundles=[bundle1, bundle2, bundle3])
+        pgs.append(pg)
+
+    for pg in tqdm.tqdm(pgs, desc="Waiting for pgs to be ready"):
+        ray.get(pg.ready())
+
+    actors = []
+    for pg in tqdm.tqdm(pgs, desc="Scheduling tasks"):
+        actors.append(
+            C1.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+        actors.append(
+            C2.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+        actors.append(
+            C3.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+
+    not_ready = [actor.ping.remote() for actor in actors]
+    for _ in tqdm.trange(len(actors)):
+        ready, not_ready = ray.wait(not_ready)
+        assert ray.get(*ready) == "pong"
+
+    for pg in tqdm.tqdm(pgs, desc="Cleaning up pgs"):
+        remove_placement_group(pg)
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+addr = ray.init(address="auto")
+
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+monitor_actor = test_utils.monitor_memory_usage()
+dashboard_test = DashboardTestAtScale(addr)
+
+start_time = time.time()
+test_many_placement_groups()
+end_time = time.time()
+ray.get(monitor_actor.stop_run.remote())
+used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+print(f"Peak memory usage: {round(used_gb, 2)}GB")
+print(f"Peak memory usage per processes:\n {usage}")
+del monitor_actor
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+rate = MAX_PLACEMENT_GROUPS / (end_time - start_time)
+print(
+    f"Success! Started {MAX_PLACEMENT_GROUPS} pgs in "
+    f"{end_time - start_time}s. ({rate} pgs/s)"
+)
+
+results = {
+    "pgs_per_second": rate,
+    "num_pgs": MAX_PLACEMENT_GROUPS,
+    "time": end_time - start_time,
+    "_peak_memory": round(used_gb, 2),
+    "_peak_process_memory": usage,
+}
+if not is_smoke_test:
+    results["perf_metrics"] = [
+        {
+            "perf_metric_name": "pgs_per_second",
+            "perf_metric_value": rate,
+            "perf_metric_type": "THROUGHPUT",
+        }
+    ]
+dashboard_test.update_release_test_result(results)
+test_utils.safe_write_to_results_json(results)

release/benchmarks/distributed/test_many_tasks.py

@@ -0,0 +1,138 @@
+import time
+
+import click
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray._private.state_api_test_utils import (
+    StateAPICallSpec,
+    periodic_invoke_state_apis_with_actor,
+    summarize_worker_startup_time,
+)
+from ray.util.state import summarize_tasks
+
+sleep_time = 300
+
+
+def test_max_running_tasks(num_tasks):
+    cpus_per_task = 0.25
+
+    @ray.remote(num_cpus=cpus_per_task)
+    def task():
+        time.sleep(sleep_time)
+
+    def time_up(start_time):
+        return time.time() - start_time >= sleep_time
+
+    refs = [task.remote() for _ in tqdm.trange(num_tasks, desc="Launching tasks")]
+
+    max_cpus = ray.cluster_resources()["CPU"]
+    min_cpus_available = max_cpus
+    start_time = time.time()
+    for _ in tqdm.trange(int(sleep_time / 0.1), desc="Waiting"):
+        try:
+            cur_cpus = ray.available_resources().get("CPU", 0)
+            min_cpus_available = min(min_cpus_available, cur_cpus)
+        except Exception:
+            # There are race conditions `.get` can fail if a new heartbeat
+            # comes at the same time.
+            pass
+        if time_up(start_time):
+            print(f"Time up for sleeping {sleep_time} seconds")
+            break
+        time.sleep(0.1)
+
+    # There are some relevant magic numbers in this check. 10k tasks each
+    # require 1/4 cpus. Therefore, ideally 2.5k cpus will be used.
+    used_cpus = max_cpus - min_cpus_available
+    err_str = f"Only {used_cpus}/{max_cpus} cpus used."
+    # 1500 tasks. Note that it is a pretty low threshold, and the
+    # performance should be tracked via perf dashboard.
+    threshold = num_tasks * cpus_per_task * 0.60
+    print(f"{used_cpus}/{max_cpus} used.")
+    assert used_cpus > threshold, err_str
+
+    for _ in tqdm.trange(num_tasks, desc="Ensuring all tasks have finished"):
+        done, refs = ray.wait(refs)
+        assert ray.get(done[0]) is None
+
+    return used_cpus
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+@click.command()
+@click.option("--num-tasks", required=True, type=int, help="Number of tasks to launch.")
+def test(num_tasks):
+    addr = ray.init(address="auto")
+
+    ray._common.test_utils.wait_for_condition(no_resource_leaks)
+    monitor_actor = test_utils.monitor_memory_usage()
+    dashboard_test = DashboardTestAtScale(addr)
+
+    def not_none(res):
+        return res is not None
+
+    api_caller = periodic_invoke_state_apis_with_actor(
+        apis=[StateAPICallSpec(summarize_tasks, not_none)],
+        call_interval_s=4,
+        print_result=True,
+    )
+
+    start_time = time.time()
+    used_cpus = test_max_running_tasks(num_tasks)
+    end_time = time.time()
+    ray.get(monitor_actor.stop_run.remote())
+    used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+    print(f"Peak memory usage: {round(used_gb, 2)}GB")
+    print(f"Peak memory usage per processes:\n {usage}")
+    ray.get(api_caller.stop.remote())
+
+    del api_caller
+    del monitor_actor
+    ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+    try:
+        summarize_worker_startup_time()
+    except Exception as e:
+        print("Failed to summarize worker startup time.")
+        print(e)
+
+    rate = num_tasks / (end_time - start_time - sleep_time)
+    print(
+        f"Success! Started {num_tasks} tasks in {end_time - start_time}s. "
+        f"({rate} tasks/s)"
+    )
+
+    results = {
+        "tasks_per_second": rate,
+        "num_tasks": num_tasks,
+        "time": end_time - start_time,
+        "used_cpus": used_cpus,
+        "_peak_memory": round(used_gb, 2),
+        "_peak_process_memory": usage,
+        "perf_metrics": [
+            {
+                "perf_metric_name": "tasks_per_second",
+                "perf_metric_value": rate,
+                "perf_metric_type": "THROUGHPUT",
+            },
+            {
+                "perf_metric_name": "used_cpus_by_deadline",
+                "perf_metric_value": used_cpus,
+                "perf_metric_type": "THROUGHPUT",
+            },
+        ],
+    }
+
+    dashboard_test.update_release_test_result(results)
+    test_utils.safe_write_to_results_json(results)
+
+
+if __name__ == "__main__":
+    test()

release/benchmarks/distributed/test_scheduling.py

@@ -0,0 +1,134 @@
+import argparse
+from math import floor
+from time import sleep, time
+
+import ray
+import ray._private.test_utils as test_utils
+from ray._private.test_utils import safe_write_to_results_json
+
+
+@ray.remote
+def simple_task(t):
+    sleep(t)
+
+
+@ray.remote
+class SimpleActor:
+    def __init__(self, job=None):
+        self._job = job
+
+    def ready(self):
+        return
+
+    def do_job(self):
+        if self._job is not None:
+            self._job()
+
+
+def start_tasks(num_task, num_cpu_per_task, task_duration):
+    ray.get(
+        [
+            simple_task.options(num_cpus=num_cpu_per_task).remote(task_duration)
+            for _ in range(num_task)
+        ]
+    )
+
+
+def measure(f):
+    start = time()
+    ret = f()
+    end = time()
+    return (end - start, ret)
+
+
+def start_actor(num_actors, num_actors_per_nodes, job):
+    resources = {"node": floor(1.0 / num_actors_per_nodes)}
+    submission_cost, actors = measure(
+        lambda: [
+            SimpleActor.options(resources=resources, num_cpus=0).remote(job)
+            for _ in range(num_actors)
+        ]
+    )
+    ready_cost, _ = measure(lambda: ray.get([actor.ready.remote() for actor in actors]))
+    actor_job_cost, _ = measure(
+        lambda: ray.get([actor.do_job.remote() for actor in actors])
+    )
+    return (submission_cost, ready_cost, actor_job_cost)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Test Scheduling")
+    # Task workloads
+    parser.add_argument(
+        "--total-num-task", type=int, help="Total number of tasks.", required=False
+    )
+    parser.add_argument(
+        "--num-cpu-per-task",
+        type=int,
+        help="Resources needed for tasks.",
+        required=False,
+    )
+    parser.add_argument(
+        "--task-duration-s",
+        type=int,
+        help="How long does each task execute.",
+        required=False,
+        default=1,
+    )
+
+    # Actor workloads
+    parser.add_argument(
+        "--total-num-actors", type=int, help="Total number of actors.", required=True
+    )
+    parser.add_argument(
+        "--num-actors-per-nodes",
+        type=int,
+        help="How many actors to allocate for each nodes.",
+        required=True,
+    )
+
+    ray.init(address="auto")
+    monitor_actor = test_utils.monitor_memory_usage()
+
+    total_cpus_per_node = [node["Resources"].get("CPU", 0) for node in ray.nodes()]
+    num_nodes = len(total_cpus_per_node)
+    total_cpus = sum(total_cpus_per_node)
+
+    args = parser.parse_args()
+    job = None
+    if args.total_num_task is not None:
+        if args.num_cpu_per_task is None:
+            args.num_cpu_per_task = floor(1.0 * total_cpus / args.total_num_task)
+        job = lambda: start_tasks(  # noqa: E731
+            args.total_num_task, args.num_cpu_per_task, args.task_duration_s
+        )
+
+    submission_cost, ready_cost, actor_job_cost = start_actor(
+        args.total_num_actors, args.num_actors_per_nodes, job
+    )
+
+    ray.get(monitor_actor.stop_run.remote())
+    used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+    print(f"Peak memory usage: {round(used_gb, 2)}GB")
+    print(f"Peak memory usage per processes:\n {usage}")
+    del monitor_actor
+
+    result = {
+        "total_num_task": args.total_num_task,
+        "num_cpu_per_task": args.num_cpu_per_task,
+        "task_duration_s": args.task_duration_s,
+        "total_num_actors": args.total_num_actors,
+        "num_actors_per_nodes": args.num_actors_per_nodes,
+        "num_nodes": num_nodes,
+        "total_cpus": total_cpus,
+        "submission_cost": submission_cost,
+        "ready_cost": ready_cost,
+        "actor_job_cost": actor_job_cost,
+        "_peak_memory": round(used_gb, 2),
+        "_peak_process_memory": usage,
+        "_runtime": submission_cost + ready_cost + actor_job_cost,
+    }
+
+    safe_write_to_results_json(result)
+
+    print(result)

release/benchmarks/distributed_gce.yaml

@@ -0,0 +1,26 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-64 # Network optimized.
+    resources:
+      CPU: 0
+      node: 1
+      small: 1
+
+worker_nodes:
+    - name: worker_node_m
+      instance_type: n2-standard-64
+      min_nodes: 32
+      max_nodes: 32
+      market_type: ON_DEMAND
+      resources:
+        node: 1
+    - name: worker_node_r
+      instance_type: n2-standard-64
+      min_nodes: 32
+      max_nodes: 32
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/distributed_smoke_test.yaml

@@ -0,0 +1,16 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: r5dn.16xlarge # Network optimized.
+    resources:
+      CPU: 0
+      node: 1
+      small: 1
+
+worker_nodes:
+    - instance_type: m5.16xlarge
+      min_nodes: 1
+      max_nodes: 1
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/many_nodes.yaml

@@ -0,0 +1,15 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: r6idn.16xlarge # Network optimized.
+    resources:
+      node: 1
+      small: 1
+
+worker_nodes:
+    - instance_type: m6i.2xlarge
+      min_nodes: 249
+      max_nodes: 249
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/many_nodes_gce.yaml

@@ -0,0 +1,17 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-64 # Network optimized.
+    resources:
+      node: 1
+      small: 1
+
+worker_nodes:
+    - instance_type: n2-standard-8
+      min_nodes: 249
+      max_nodes: 249
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/object_store.yaml

@@ -0,0 +1,14 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m6i.16xlarge
+    resources:
+      node: 1
+
+worker_nodes:
+    - instance_type: m6i.2xlarge
+      min_nodes: 49
+      max_nodes: 49
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/object_store/callback_throughput.yaml

@@ -0,0 +1,14 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m6i.2xlarge
+    resources:
+      node: 1
+
+worker_nodes:
+    - instance_type: m6i.2xlarge
+      min_nodes: 10
+      max_nodes: 10
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/object_store/large_objects.yaml

@@ -0,0 +1,14 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m6i.16xlarge
+    resources:
+      node: 1
+
+worker_nodes:
+    - instance_type: m6i.4xlarge
+      min_nodes: 9
+      max_nodes: 9
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/object_store/small_objects.yaml

@@ -0,0 +1,10 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m6i.16xlarge
+
+worker_nodes:
+    - instance_type: m6i.4xlarge
+      min_nodes: 4
+      max_nodes: 4
+      market_type: ON_DEMAND

release/benchmarks/object_store/test_callback_throughput.py

@@ -0,0 +1,108 @@
+import json
+import os
+import threading
+import time
+
+import numpy as np
+
+import ray
+import ray._private.worker
+
+NUM_WORKERS = 10
+OBJECT_SIZE = 1024 * 1024  # 1 MiB, above the 100 KB inlining threshold
+
+
+@ray.remote(num_cpus=1)
+def produce_block():
+    return np.zeros(OBJECT_SIZE, dtype=np.uint8)
+
+
+@ray.remote(num_cpus=1)
+def consume_block(block_ref):
+    return len(block_ref)
+
+
+def test_callback_pipeline(num_blocks, timeout_s=60):
+    core_worker = ray._private.worker.global_worker.core_worker
+
+    latencies = []
+    drop_times = {}
+    lock = threading.Lock()
+    done = threading.Event()
+
+    def on_freed(id_bytes):
+        with lock:
+            latencies.append(time.perf_counter() - drop_times[id_bytes])
+            if len(latencies) == num_blocks:
+                done.set()
+
+    refs = [
+        produce_block.options(scheduling_strategy="SPREAD").remote()
+        for _ in range(num_blocks)
+    ]
+    ray.wait(refs, num_returns=len(refs))
+
+    # live_refs keeps each block ref alive until its consumer completes.
+    live_refs = {}
+    for ref in refs:
+        assert core_worker.add_object_out_of_scope_callback(ref, on_freed)
+        consumer = consume_block.remote(ref)
+        live_refs[consumer] = ref
+    del refs
+
+    # Release each ref as its consumer completes.
+    pending = list(live_refs.keys())
+    while pending:
+        done_list, pending = ray.wait(pending, num_returns=1)
+        for consumer in done_list:
+            ref = live_refs.pop(consumer)
+            drop_times[ref.binary()] = time.perf_counter()
+            del ref
+
+    if not done.wait(timeout=timeout_s):
+        raise TimeoutError(
+            f"Only {len(latencies)}/{num_blocks} callbacks fired within {timeout_s}s"
+        )
+
+    latencies.sort()
+    p95 = latencies[int(len(latencies) * 0.95)]
+    print(f"  {num_blocks} blocks: p95={p95:.4f}s")
+    return p95
+
+
+ray.init(address="auto")
+
+# Warm up gRPC connections and worker pools.
+ray.get(
+    [
+        produce_block.options(scheduling_strategy="SPREAD").remote()
+        for _ in range(NUM_WORKERS)
+    ]
+)
+
+p95_100 = test_callback_pipeline(100)
+p95_1k = test_callback_pipeline(1000)
+
+print("\nSummary:")
+print(f"  100 blocks: p95={p95_100:.4f}s")
+print(f"  1k blocks: p95={p95_1k:.4f}s")
+
+if "TEST_OUTPUT_JSON" in os.environ:
+    with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+        results = {
+            "p95_100": p95_100,
+            "p95_1k": p95_1k,
+            "perf_metrics": [
+                {
+                    "perf_metric_name": "callback_p95_latency_100_blocks_s",
+                    "perf_metric_value": p95_100,
+                    "perf_metric_type": "LATENCY",
+                },
+                {
+                    "perf_metric_name": "callback_p95_latency_1k_blocks_s",
+                    "perf_metric_value": p95_1k,
+                    "perf_metric_type": "LATENCY",
+                },
+            ],
+        }
+        json.dump(results, out_file)

release/benchmarks/object_store/test_large_objects.py

@@ -0,0 +1,101 @@
+import json
+import os
+from time import perf_counter
+
+import numpy as np
+from tqdm import tqdm
+
+import ray
+
+NUM_NODES = 9
+OBJECT_SIZE = 2**32
+
+
+def test_object_many_to_one():
+    @ray.remote(num_cpus=1, resources={"node": 1})
+    class Actor:
+        def foo(self):
+            pass
+
+        def send_objects(self):
+            return np.ones(OBJECT_SIZE, dtype=np.uint8)
+
+    actors = [Actor.remote() for _ in range(NUM_NODES)]
+
+    for actor in tqdm(actors, desc="Ensure all actors have started."):
+        ray.get(actor.foo.remote())
+
+    start = perf_counter()
+    result_refs = []
+    for actor in tqdm(actors, desc="Tasks kickoff"):
+        result_refs.append(actor.send_objects.remote())
+
+    results = ray.get(result_refs)
+    end = perf_counter()
+
+    for result in results:
+        assert len(result) == OBJECT_SIZE
+
+    return end - start
+
+
+def test_object_one_to_many():
+    @ray.remote(num_cpus=1, resources={"node": 1})
+    class Actor:
+        def foo(self):
+            pass
+
+        def data_len(self, arr):
+            return len(arr)
+
+    actors = [Actor.remote() for _ in range(NUM_NODES)]
+
+    arr = np.ones(OBJECT_SIZE, dtype=np.uint8)
+    ref = ray.put(arr)
+
+    for actor in tqdm(actors, desc="Ensure all actors have started."):
+        ray.get(actor.foo.remote())
+
+    start = perf_counter()
+    result_refs = []
+    for actor in tqdm(actors, desc="Tasks kickoff"):
+        result_refs.append(actor.data_len.remote(ref))
+
+    results = ray.get(result_refs)
+    end = perf_counter()
+
+    for result in results:
+        assert result == OBJECT_SIZE
+
+    return end - start
+
+
+ray.init(address="auto")
+many_to_one_duration = test_object_many_to_one()
+print(f"many_to_one time: {many_to_one_duration} ({OBJECT_SIZE} B x {NUM_NODES} nodes)")
+one_to_many_duration = test_object_one_to_many()
+print(f"one_to_many time: {one_to_many_duration} ({OBJECT_SIZE} B x {NUM_NODES} nodes)")
+
+
+if "TEST_OUTPUT_JSON" in os.environ:
+    with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+        results = {
+            "many_to_one_time": many_to_one_duration,
+            "one_to_many_time": one_to_many_duration,
+            "object_size": OBJECT_SIZE,
+            "num_nodes": NUM_NODES,
+        }
+        results["perf_metrics"] = [
+            {
+                "perf_metric_name": f"time_many_to_one_{OBJECT_SIZE}_bytes_from_{NUM_NODES}_nodes",
+                "perf_metric_value": many_to_one_duration,
+                "perf_metric_type": "LATENCY",
+            },
+            {
+                "perf_metric_name": f"time_one_to_many_{OBJECT_SIZE}_bytes_to_{NUM_NODES}_nodes",
+                "perf_metric_value": one_to_many_duration,
+                "perf_metric_type": "LATENCY",
+            },
+        ]
+
+        json.dump(results, out_file)

release/benchmarks/object_store/test_object_store.py

@@ -0,0 +1,75 @@
+import json
+import os
+from time import perf_counter
+
+import numpy as np
+from tqdm import tqdm
+
+import ray
+import ray.autoscaler.sdk
+
+NUM_NODES = 50
+OBJECT_SIZE = 2**30
+
+
+def num_alive_nodes():
+    n = 0
+    for node in ray.nodes():
+        if node["Alive"]:
+            n += 1
+    return n
+
+
+def test_object_broadcast():
+    assert num_alive_nodes() == NUM_NODES
+
+    @ray.remote(num_cpus=1, resources={"node": 1})
+    class Actor:
+        def foo(self):
+            pass
+
+        def data_len(self, arr):
+            return len(arr)
+
+    actors = [Actor.remote() for _ in range(NUM_NODES)]
+
+    arr = np.ones(OBJECT_SIZE, dtype=np.uint8)
+    ref = ray.put(arr)
+
+    for actor in tqdm(actors, desc="Ensure all actors have started."):
+        ray.get(actor.foo.remote())
+
+    start = perf_counter()
+    result_refs = []
+    for actor in tqdm(actors, desc="Broadcasting objects"):
+        result_refs.append(actor.data_len.remote(ref))
+
+    results = ray.get(result_refs)
+    end = perf_counter()
+
+    for result in results:
+        assert result == OBJECT_SIZE
+
+    return end - start
+
+
+ray.init(address="auto")
+duration = test_object_broadcast()
+print(f"Broadcast time: {duration} ({OBJECT_SIZE} B x {NUM_NODES} nodes)")
+
+if "TEST_OUTPUT_JSON" in os.environ:
+    with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+        results = {
+            "broadcast_time": duration,
+            "object_size": OBJECT_SIZE,
+            "num_nodes": NUM_NODES,
+        }
+        perf_metric_name = f"time_to_broadcast_{OBJECT_SIZE}_bytes_to_{NUM_NODES}_nodes"
+        results["perf_metrics"] = [
+            {
+                "perf_metric_name": perf_metric_name,
+                "perf_metric_value": duration,
+                "perf_metric_type": "LATENCY",
+            }
+        ]
+        json.dump(results, out_file)

release/benchmarks/object_store/test_small_objects.py

@@ -0,0 +1,81 @@
+import json
+import os
+import time
+
+import numpy as np
+
+import ray
+
+
+def test_small_objects_many_to_one():
+    @ray.remote(num_cpus=1)
+    class Actor:
+        def send(self, _, actor_idx):
+            # this size is chosen because it's >100kb so big enough to be stored in plasma
+            numpy_arr = np.ones((20, 1024))
+            return (numpy_arr, actor_idx)
+
+    actors = [Actor.remote() for _ in range(64)]
+    not_ready = []
+    for index, actor in enumerate(actors):
+        not_ready.append(actor.send.remote(0, index))
+    num_messages = 0
+    start_time = time.time()
+    while time.time() - start_time < 60:
+        ready, not_ready = ray.wait(not_ready, num_returns=10)
+        for ready_ref in ready:
+            _, actor_idx = ray.get(ready_ref)
+            not_ready.append(actors[actor_idx].send.remote(0, actor_idx))
+        num_messages += 10
+    return num_messages / 60
+
+
+def test_small_objects_one_to_many():
+    @ray.remote(num_cpus=1)
+    class Actor:
+        def receive(self, numpy_arr, actor_idx):
+            return actor_idx
+
+    actors = [Actor.remote() for _ in range(64)]
+    numpy_arr_ref = ray.put(np.ones((20, 1024)))
+    not_ready = []
+
+    num_messages = 0
+    start_time = time.time()
+    for idx, actor in enumerate(actors):
+        not_ready.append(actor.receive.remote(numpy_arr_ref, idx))
+    while time.time() - start_time < 60:
+        ready, not_ready = ray.wait(not_ready, num_returns=10)
+        actor_idxs = ray.get(ready)
+        for actor_idx in actor_idxs:
+            not_ready.append(actors[actor_idx].receive.remote(numpy_arr_ref, actor_idx))
+        num_messages += 10
+    return num_messages / 60
+
+
+ray.init(address="auto")
+many_to_one_throughput = test_small_objects_many_to_one()
+print(f"Number of messages per second many_to_one: {many_to_one_throughput}")
+one_to_many_throughput = test_small_objects_one_to_many()
+print(f"Number of messages per second one_to_many: {one_to_many_throughput}")
+
+
+if "TEST_OUTPUT_JSON" in os.environ:
+    with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+        results = {
+            "num_messages_many_to_one": many_to_one_throughput,
+            "num_messages_one_to_many": one_to_many_throughput,
+        }
+        results["perf_metrics"] = [
+            {
+                "perf_metric_name": "num_small_objects_many_to_one",
+                "perf_metric_value": many_to_one_throughput,
+                "perf_metric_type": "THROUGHPUT",
+            },
+            {
+                "perf_metric_name": "num_small_objects_one_to_many_per_second",
+                "perf_metric_value": one_to_many_throughput,
+                "perf_metric_type": "THROUGHPUT",
+            },
+        ]
+        json.dump(results, out_file)

release/benchmarks/object_store_gce.yaml

@@ -0,0 +1,16 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-64
+    resources:
+      node: 1
+
+worker_nodes:
+    - instance_type: n2-standard-8
+      min_nodes: 49
+      max_nodes: 49
+      market_type: ON_DEMAND
+      resources:
+        node: 1

release/benchmarks/scheduling.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m5.4xlarge
+    resources:
+      # Assume the node has 64 CPU instead of 16.
+      # This should be fine since each task has little
+      # computation in scheduling tests.
+      CPU: 64
+      node: 1
+
+worker_nodes:
+    - instance_type: m5.4xlarge
+      min_nodes: 31
+      max_nodes: 31
+      market_type: ON_DEMAND
+      resources:
+        CPU: 64
+        node: 1

release/benchmarks/scheduling_gce.yaml

@@ -0,0 +1,21 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-16
+    resources:
+      # Assume the node has 64 CPU instead of 16.
+      # This should be fine since each task has little
+      # computation in scheduling tests.
+      CPU: 64
+      node: 1
+
+worker_nodes:
+    - instance_type: n2-standard-16
+      min_nodes: 31
+      max_nodes: 31
+      market_type: ON_DEMAND
+      resources:
+        CPU: 64
+        node: 1

release/benchmarks/single_node.yaml

@@ -0,0 +1,16 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+advanced_instance_config:
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+            DeleteOnTermination: true
+            VolumeSize: 500
+
+head_node:
+    instance_type: m4.16xlarge
+    resources:
+      # 128 GB
+      object_store_memory: 128000000000
+
+worker_nodes: []

release/benchmarks/single_node/test_single_node.py

@@ -0,0 +1,237 @@
+import json
+import os
+import time
+from time import perf_counter
+
+import numpy as np
+from tqdm import tqdm, trange
+
+import ray
+import ray.autoscaler.sdk
+from ray._common.test_utils import Semaphore
+
+MAX_ARGS = 10000
+MAX_RETURNS = 3000
+MAX_RAY_GET_ARGS = 10000
+MAX_QUEUED_TASKS = 1_000_000
+MAX_RAY_GET_SIZE = 100 * 2**30
+
+
+def assert_no_leaks():
+    total = ray.cluster_resources()
+    current = ray.available_resources()
+    total.pop("memory")
+    total.pop("object_store_memory")
+    current.pop("memory")
+    current.pop("object_store_memory")
+    assert total == current, (total, current)
+
+
+def test_many_args():
+    @ray.remote
+    def sum_args(*args):
+        return sum(sum(arg) for arg in args)
+
+    args = [[1 for _ in range(10000)] for _ in range(MAX_ARGS)]
+    result = ray.get(sum_args.remote(*args))
+    assert result == MAX_ARGS * 10000
+
+
+def test_many_returns():
+    @ray.remote(num_returns=MAX_RETURNS)
+    def f():
+        to_return = []
+        for _ in range(MAX_RETURNS):
+            obj = list(range(10000))
+            to_return.append(obj)
+
+        return tuple(to_return)
+
+    returned_refs = f.remote()
+    assert len(returned_refs) == MAX_RETURNS
+
+    for ref in returned_refs:
+        expected = list(range(10000))
+        obj = ray.get(ref)
+        assert obj == expected
+
+
+def test_ray_get_args():
+    def with_dese():
+        print("Putting test objects:")
+        refs = []
+        for _ in trange(MAX_RAY_GET_ARGS):
+            obj = list(range(10000))
+            refs.append(ray.put(obj))
+
+        print("Getting objects")
+        results = ray.get(refs)
+        assert len(results) == MAX_RAY_GET_ARGS
+
+        print("Asserting correctness")
+        for obj in tqdm(results):
+            expected = list(range(10000))
+            assert obj == expected
+
+    def with_zero_copy():
+        print("Putting test objects:")
+        refs = []
+        for _ in trange(MAX_RAY_GET_ARGS):
+            obj = np.arange(10000)
+            refs.append(ray.put(obj))
+
+        print("Getting objects")
+        results = ray.get(refs)
+        assert len(results) == MAX_RAY_GET_ARGS
+
+        print("Asserting correctness")
+        for obj in tqdm(results):
+            expected = np.arange(10000)
+            assert (obj == expected).all()
+
+    with_dese()
+    print("Done with dese")
+    with_zero_copy()
+    print("Done with zero copy")
+
+
+def test_many_queued_tasks():
+    sema = Semaphore.remote(0)
+
+    @ray.remote(num_cpus=1)
+    def block():
+        ray.get(sema.acquire.remote())
+
+    @ray.remote(num_cpus=1)
+    def f():
+        pass
+
+    num_cpus = int(ray.cluster_resources()["CPU"])
+    blocked_tasks = []
+    for _ in range(num_cpus):
+        blocked_tasks.append(block.remote())
+
+    print("Submitting many tasks")
+    pending_tasks = []
+    for _ in trange(MAX_QUEUED_TASKS):
+        pending_tasks.append(f.remote())
+
+    # Make sure all the tasks can actually run.
+    for _ in range(num_cpus):
+        sema.release.remote()
+
+    print("Unblocking tasks")
+    for ref in tqdm(pending_tasks):
+        assert ray.get(ref) is None
+
+
+def test_large_object():
+    print("Generating object")
+    obj = np.zeros(MAX_RAY_GET_SIZE, dtype=np.int8)
+    print("Putting object")
+    ref = ray.put(obj)
+    del obj
+    print("Getting object")
+    big_obj = ray.get(ref)
+
+    assert big_obj[0] == 0
+    assert big_obj[-1] == 0
+
+
+ray.init(address="auto")
+
+args_start = perf_counter()
+test_many_args()
+args_end = perf_counter()
+
+time.sleep(5)
+assert_no_leaks()
+print("Finished many args")
+
+returns_start = perf_counter()
+test_many_returns()
+returns_end = perf_counter()
+
+time.sleep(5)
+assert_no_leaks()
+print("Finished many returns")
+
+get_start = perf_counter()
+test_ray_get_args()
+get_end = perf_counter()
+
+time.sleep(5)
+assert_no_leaks()
+print("Finished ray.get on many objects")
+
+queued_start = perf_counter()
+test_many_queued_tasks()
+queued_end = perf_counter()
+
+time.sleep(5)
+assert_no_leaks()
+print("Finished queueing many tasks")
+
+large_object_start = perf_counter()
+test_large_object()
+large_object_end = perf_counter()
+
+time.sleep(5)
+assert_no_leaks()
+print("Done")
+
+args_time = args_end - args_start
+returns_time = returns_end - returns_start
+get_time = get_end - get_start
+queued_time = queued_end - queued_start
+large_object_time = large_object_end - large_object_start
+
+print(f"Many args time: {args_time} ({MAX_ARGS} args)")
+print(f"Many returns time: {returns_time} ({MAX_RETURNS} returns)")
+print(f"Ray.get time: {get_time} ({MAX_RAY_GET_ARGS} args)")
+print(f"Queued task time: {queued_time} ({MAX_QUEUED_TASKS} tasks)")
+print(f"Ray.get large object time: {large_object_time} " f"({MAX_RAY_GET_SIZE} bytes)")
+
+if "TEST_OUTPUT_JSON" in os.environ:
+    with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+        results = {
+            "args_time": args_time,
+            "num_args": MAX_ARGS,
+            "returns_time": returns_time,
+            "num_returns": MAX_RETURNS,
+            "get_time": get_time,
+            "num_get_args": MAX_RAY_GET_ARGS,
+            "queued_time": queued_time,
+            "num_queued": MAX_QUEUED_TASKS,
+            "large_object_time": large_object_time,
+            "large_object_size": MAX_RAY_GET_SIZE,
+            "success": "1",
+        }
+        results["perf_metrics"] = [
+            {
+                "perf_metric_name": f"{MAX_ARGS}_args_time",
+                "perf_metric_value": args_time,
+                "perf_metric_type": "LATENCY",
+            },
+            {
+                "perf_metric_name": f"{MAX_RETURNS}_returns_time",
+                "perf_metric_value": returns_time,
+                "perf_metric_type": "LATENCY",
+            },
+            {
+                "perf_metric_name": f"{MAX_RAY_GET_ARGS}_get_time",
+                "perf_metric_value": get_time,
+                "perf_metric_type": "LATENCY",
+            },
+            {
+                "perf_metric_name": f"{MAX_QUEUED_TASKS}_queued_time",
+                "perf_metric_value": queued_time,
+                "perf_metric_type": "LATENCY",
+            },
+            {
+                "perf_metric_name": f"{MAX_RAY_GET_SIZE}_large_object_time",
+                "perf_metric_value": large_object_time,
+                "perf_metric_type": "LATENCY",
+            },
+        ]
+        json.dump(results, out_file)

release/benchmarks/single_node_gce.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+advanced_instance_config:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 500
+
+head_node:
+    instance_type: n2-standard-64
+    resources:
+      # 128 GB
+      object_store_memory: 128000000000
+
+worker_nodes: []