chore: import upstream snapshot with attribution

release/benchmarks/distributed/many_nodes_tests/actor_test.py

@@ -0,0 +1,145 @@
+import argparse
+import os
+import math
+from time import sleep, perf_counter
+import json
+import ray
+
+from dashboard_test import DashboardTestAtScale
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors):
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+    print("Start launch actors")
+    actors = [Actor.options(max_restarts=-1).remote() for _ in range(total_actors)]
+    return actors
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", nargs="+", type=int, required=True)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--no-wait", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def scale_cluster_up(num_cpus):
+    print(f"Start to scale up to {num_cpus} cpus")
+
+    def get_curr_cpus():
+        return int(sum([r.get("Resources", {}).get("CPU", 0) for r in ray.nodes()]))
+
+    step = 1000
+    curr_cpus = get_curr_cpus()
+    target_cpus = curr_cpus
+
+    while curr_cpus < num_cpus:
+        curr_cpus = get_curr_cpus()
+        new_target_cpus = min(curr_cpus + step, num_cpus)
+        if new_target_cpus != target_cpus:
+            target_cpus = new_target_cpus
+            ray.autoscaler.sdk.request_resources(num_cpus=target_cpus)
+        print(f"Waiting for cluster to be up: {curr_cpus}->{target_cpus}->{num_cpus}")
+        sleep(10)
+
+
+def get_head_node_cpus():
+    head_ip = ray.util.get_node_ip_address()
+    for node in ray.nodes():
+        if node["Alive"] and node["NodeManagerAddress"] == head_ip:
+            return int(node.get("Resources", {}).get("CPU", 0))
+    return 0
+
+
+def run_one(total_actors, cpus_per_actor, no_wait):
+    total_cpus = cpus_per_actor * total_actors + get_head_node_cpus()
+    total_cpus = int(math.ceil(total_cpus))
+    scale_cluster_up(total_cpus)
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(cpus_per_actor, total_actors)
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    actor_ready_start = perf_counter()
+    total_actors = len(actors)
+    objs = [actor.foo.remote() for actor in actors]
+
+    while len(objs) != 0:
+        timeout = None if no_wait else 30
+        objs_ready, objs = ray.wait(objs, num_returns=len(objs), timeout=timeout)
+        print(
+            f"Status: {total_actors - len(objs)}/{total_actors}, "
+            f"{perf_counter() - actor_ready_start}"
+        )
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    throughput = total_actors / (actor_ready_time + actor_launch_time)
+    print(f"Actor launch time: {actor_launch_time} ({total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({total_actors} actors)"
+    )
+    print(f"Through put: {throughput}")
+
+    return {
+        "actor_launch_time": actor_launch_time,
+        "actor_ready_time": actor_ready_time,
+        "total_time": actor_launch_time + actor_ready_time,
+        "num_actors": total_actors,
+        "throughput": throughput,
+    }
+
+
+def main():
+    args, unknown = parse_script_args()
+    args.total_actors.sort()
+
+    addr = ray.init(address="auto")
+    dashboard_test = DashboardTestAtScale(addr)
+
+    result = {}
+    for i in args.total_actors:
+        result[f"many_nodes_actor_tests_{i}"] = run_one(
+            i, args.cpus_per_actor, args.no_wait
+        )
+
+    # Print the results early so if failed in the future, we still
+    # can see it in the log.
+    print(f"Result: {json.dumps(result, indent=2)}")
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            perf = [
+                {
+                    "perf_metric_name": name,
+                    "perf_metric_value": r["throughput"],
+                    "perf_metric_type": "THROUGHPUT",
+                }
+                for (name, r) in result.items()
+            ]
+            result["perf_metrics"] = perf
+            dashboard_test.update_release_test_result(result)
+
+            print(f"Writing data into file: {os.environ['TEST_OUTPUT_JSON']}")
+            json.dump(result, out_file)
+
+    print("Test finished successfully!")
+    ray.shutdown()
+
+    # We need to make sure GCS cool down otherwise, testing infra
+    # might get timeout when fetching the result because when the driver
+    # got shutdown, many actors needs to be terminated which will
+    # overload GCS.
+    print("Sleep for 60s, waiting for the cluster to cool down.")
+    sleep(60)
+
+
+if __name__ == "__main__":
+    main()

release/benchmarks/distributed/many_nodes_tests/compute_config.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+# NFS needs to be disabled for this test, since the test spawns too many nodes
+# and may hit the limit on the # of clients.
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: as-feature-disable-nfs-mount
+          Value: "true"
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        DeleteOnTermination: true
+        VolumeSize: 30
+
+head_node:
+  instance_type: m5.16xlarge
+
+worker_nodes:
+  - instance_type: m6i.large
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/compute_config_gce.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+  - us-west1-c
+
+advanced_instance_config:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 30
+
+    # NFS needs to be disabled for this test, since the test spawns too many nodes
+    # and may hit the limit on the # of clients.
+    labels:
+      as-feature-disable-nfs-mount: "true"
+
+head_node:
+  instance_type: n2-standard-64
+
+worker_nodes:
+  - instance_type: n2-standard-2
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/dashboard_test.py

@@ -0,0 +1,185 @@
+import asyncio
+import time
+import urllib
+from typing import Dict, Optional, List
+from pprint import pprint
+
+import requests
+import ray
+import logging
+import os
+
+from collections import defaultdict
+from ray.util.state import list_nodes
+from ray._private.test_utils import get_system_metric_for_component
+from pydantic import BaseModel
+from ray.dashboard.utils import get_address_for_submission_client
+from ray.dashboard.modules.metrics.metrics_head import (
+    DEFAULT_PROMETHEUS_HOST,
+    PROMETHEUS_HOST_ENV_VAR,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def calc_p(latencies, percent):
+    if len(latencies) == 0:
+        return 0
+
+    return round(sorted(latencies)[int(len(latencies) / 100 * percent)] * 1000, 3)
+
+
+class Result(BaseModel):
+    success: bool
+    # endpoints -> list of latencies
+    result: Dict[str, List[float]]
+    # Dashboard memory usage in MB.
+    memory_mb: Optional[float]
+
+
+# Currently every endpoint is GET endpoints.
+endpoints = [
+    "/logical/actors",
+    "/nodes?view=summary",
+    "/",
+    "/api/cluster_status",
+    "/events",
+    "/api/jobs/",
+    "/api/v0/logs",
+    "/api/prometheus_health",
+]
+
+
+@ray.remote(num_cpus=0)
+class DashboardTester:
+    def __init__(self, interval_s: int = 1):
+        self.dashboard_url = get_address_for_submission_client(None)
+        # Ping interval for all endpoints.
+        self.interval_s = interval_s
+        # endpoint -> a list of latencies
+        self.result = defaultdict(list)
+
+    async def run(self):
+        await asyncio.gather(*[self.ping(endpoint) for endpoint in endpoints])
+
+    async def ping(self, endpoint):
+        """Synchronously call an endpoint."""
+        node_id = ray.get_runtime_context().get_node_id()
+        while True:
+            start = time.monotonic()
+            # for logs API, we should append node ID and glob.
+            if "/api/v0/logs" in endpoint:
+                glob_filter = "*"
+
+                options_dict = {"node_id": node_id, "glob": glob_filter}
+                url = (
+                    f"{self.dashboard_url}{endpoint}?"
+                    f"{urllib.parse.urlencode(options_dict)}"
+                )
+            else:
+                url = f"{self.dashboard_url}{endpoint}"
+
+            resp = requests.get(url, timeout=30)
+            elapsed = time.monotonic() - start
+
+            if resp.status_code == 200:
+                self.result[endpoint].append(time.monotonic() - start)
+            else:
+                try:
+                    resp.raise_for_status()
+                except Exception as e:
+                    logger.exception(e)
+            await asyncio.sleep(max(0, self.interval_s, elapsed))
+
+    def get_result(self):
+        return self.result
+
+
+class DashboardTestAtScale:
+    """This is piggybacked into existing scalability tests."""
+
+    def __init__(self, addr: ray._private.worker.RayContext):
+        self.addr = addr
+
+        # Schedule the actor on the current node (which is a head node).
+        current_node_ip = ray._private.worker.global_worker.node_ip_address
+        nodes = list_nodes(filters=[("node_ip", "=", current_node_ip)])
+        assert len(nodes) > 0, f"{current_node_ip} not found in the cluster"
+        node = nodes[0]
+        # Schedule on a head node.
+        self.tester = DashboardTester.options(
+            label_selector={ray._raylet.RAY_NODE_ID_KEY: node["node_id"]}
+        ).remote()
+
+        self.tester.run.remote()
+
+    def get_result(self):
+        """Get the result from the test.
+
+        Returns:
+            A tuple of success, and the result (Result object).
+        """
+        try:
+            result = ray.get(self.tester.get_result.remote(), timeout=60)
+        except ray.exceptions.GetTimeoutError:
+            return Result(success=False)
+
+        # Get the memory usage.
+        memories = get_system_metric_for_component(
+            "ray_component_uss_bytes",
+            "dashboard",
+            os.environ.get(PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST),
+        )
+
+        return Result(
+            success=True,
+            result=result,
+            memory_mb=max(memories) / 1.0e6 if memories else None,
+        )
+
+    def update_release_test_result(self, release_result: dict):
+        test_result = self.get_result()
+
+        def calc_endpoints_p(result, percent):
+            return {
+                # sort -> get PX -> convert second to ms -> round up.
+                endpoint: calc_p(latencies, percent)
+                for endpoint, latencies in result.items()
+            }
+
+        print("======Print per dashboard endpoint latencies======")
+        print("=====================P50==========================")
+        pprint(calc_endpoints_p(test_result.result, 50))
+        print("=====================P95==========================")
+        pprint(calc_endpoints_p(test_result.result, 95))
+        print("=====================P99==========================")
+        pprint(calc_endpoints_p(test_result.result, 99))
+
+        latencies = []
+        for per_endpoint_latencies in test_result.result.values():
+            latencies.extend(per_endpoint_latencies)
+        aggregated_metrics = {
+            "p50": calc_p(latencies, 50),
+            "p95": calc_p(latencies, 95),
+            "p99": calc_p(latencies, 99),
+        }
+
+        print("=====================Aggregated====================")
+        pprint(aggregated_metrics)
+
+        release_result["_dashboard_test_success"] = test_result.success
+        if test_result.success:
+            if "perf_metrics" not in release_result:
+                release_result["perf_metrics"] = []
+
+            release_result["perf_metrics"].extend(
+                [
+                    {
+                        "perf_metric_name": f"dashboard_{p}_latency_ms",
+                        "perf_metric_value": value,
+                        "perf_metric_type": "LATENCY",
+                    }
+                    for p, value in aggregated_metrics.items()
+                ]
+            )
+            release_result["_dashboard_memory_usage_mb"] = test_result.memory_mb

release/benchmarks/distributed/many_nodes_tests/multi_master_test.py

@@ -0,0 +1,90 @@
+import argparse
+import os
+from time import sleep, perf_counter
+import json
+import ray
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors, num_masters):
+    # By default, there are 50 groups, each group has 1 master and 99 slaves.
+    num_slaves_per_master = total_actors / num_masters - 1
+
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+        def create(self):
+            return [
+                Actor.options(max_restarts=-1).remote()
+                for _ in range(num_slaves_per_master)
+            ]
+
+    print("Start launch actors")
+    # The 50 masters are spreaded.
+    actors = [
+        Actor.options(max_restarts=-1, scheduling_strategy="SPREAD").remote()
+        for _ in range(num_masters)
+    ]
+    slaves_per_master = []
+    for master in actors:
+        slaves_per_master.append(master.create.remote())
+    for slaves in slaves_per_master:
+        actors.extend(ray.get(slaves))
+    return actors
+
+
+def test_actor_ready(actors):
+    remaining = [actor.foo.remote() for actor in actors]
+    ray.get(remaining)
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", type=int, default=5000)
+    parser.add_argument("--num-masters", type=int, default=50)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--fail", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def main():
+    args, unknown = parse_script_args()
+
+    ray.init(address="auto")
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(
+        args.cpus_per_actor, args.total_actors, args.num_masters
+    )
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    if args.fail:
+        sleep(10)
+        return
+    actor_ready_start = perf_counter()
+    test_actor_ready(actors)
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    print(f"Actor launch time: {actor_launch_time} ({args.total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({args.total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({args.total_actors} actors)"
+    )
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            results = {
+                "actor_launch_time": actor_launch_time,
+                "actor_ready_time": actor_ready_time,
+                "total_time": actor_launch_time + actor_ready_time,
+                "num_actors": args.total_actors,
+            }
+            json.dump(results, out_file)
+
+
+if __name__ == "__main__":
+    main()

release/benchmarks/distributed/test_many_actors.py

@@ -0,0 +1,90 @@
+import os
+import time
+
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray._private.state_api_test_utils import summarize_worker_startup_time
+
+is_smoke_test = True
+if "SMOKE_TEST" in os.environ:
+    MAX_ACTORS_IN_CLUSTER = 100
+else:
+    MAX_ACTORS_IN_CLUSTER = 10000
+    is_smoke_test = False
+
+
+def test_max_actors():
+    # TODO (Alex): Dynamically set this based on number of cores
+    cpus_per_actor = 0.25
+
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+    actors = [
+        Actor.remote()
+        for _ in tqdm.trange(MAX_ACTORS_IN_CLUSTER, desc="Launching actors")
+    ]
+
+    done = ray.get([actor.foo.remote() for actor in actors])
+    for result in done:
+        assert result is None
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+addr = ray.init(address="auto")
+
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+monitor_actor = test_utils.monitor_memory_usage()
+dashboard_test = DashboardTestAtScale(addr)
+
+start_time = time.time()
+test_max_actors()
+end_time = time.time()
+
+ray.get(monitor_actor.stop_run.remote())
+used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+print(f"Peak memory usage: {round(used_gb, 2)}GB")
+print(f"Peak memory usage per processes:\n {usage}")
+del monitor_actor
+
+# Get the dashboard result
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+rate = MAX_ACTORS_IN_CLUSTER / (end_time - start_time)
+try:
+    summarize_worker_startup_time()
+except Exception as e:
+    print("Failed to summarize worker startup time.")
+    print(e)
+
+print(
+    f"Success! Started {MAX_ACTORS_IN_CLUSTER} actors in "
+    f"{end_time - start_time}s. ({rate} actors/s)"
+)
+
+results = {
+    "actors_per_second": rate,
+    "num_actors": MAX_ACTORS_IN_CLUSTER,
+    "time": end_time - start_time,
+    "_peak_memory": round(used_gb, 2),
+    "_peak_process_memory": usage,
+}
+if not is_smoke_test:
+    results["perf_metrics"] = [
+        {
+            "perf_metric_name": "actors_per_second",
+            "perf_metric_value": rate,
+            "perf_metric_type": "THROUGHPUT",
+        }
+    ]
+dashboard_test.update_release_test_result(results)
+test_utils.safe_write_to_results_json(results)

release/benchmarks/distributed/test_many_pgs.py

@@ -0,0 +1,121 @@
+import os
+import time
+
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray.util.placement_group import placement_group, remove_placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+is_smoke_test = True
+if "SMOKE_TEST" in os.environ:
+    MAX_PLACEMENT_GROUPS = 20
+else:
+    MAX_PLACEMENT_GROUPS = 1000
+    is_smoke_test = False
+
+
+def test_many_placement_groups():
+    # @ray.remote(num_cpus=1, resources={"node": 0.02})
+    @ray.remote
+    class C1:
+        def ping(self):
+            return "pong"
+
+    # @ray.remote(num_cpus=1)
+    @ray.remote
+    class C2:
+        def ping(self):
+            return "pong"
+
+    # @ray.remote(resources={"node": 0.02})
+    @ray.remote
+    class C3:
+        def ping(self):
+            return "pong"
+
+    bundle1 = {"node": 0.02, "CPU": 1}
+    bundle2 = {"CPU": 1}
+    bundle3 = {"node": 0.02}
+
+    pgs = []
+    for _ in tqdm.trange(MAX_PLACEMENT_GROUPS, desc="Creating pgs"):
+        pg = placement_group(bundles=[bundle1, bundle2, bundle3])
+        pgs.append(pg)
+
+    for pg in tqdm.tqdm(pgs, desc="Waiting for pgs to be ready"):
+        ray.get(pg.ready())
+
+    actors = []
+    for pg in tqdm.tqdm(pgs, desc="Scheduling tasks"):
+        actors.append(
+            C1.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+        actors.append(
+            C2.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+        actors.append(
+            C3.options(
+                scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+            ).remote()
+        )
+
+    not_ready = [actor.ping.remote() for actor in actors]
+    for _ in tqdm.trange(len(actors)):
+        ready, not_ready = ray.wait(not_ready)
+        assert ray.get(*ready) == "pong"
+
+    for pg in tqdm.tqdm(pgs, desc="Cleaning up pgs"):
+        remove_placement_group(pg)
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+addr = ray.init(address="auto")
+
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+monitor_actor = test_utils.monitor_memory_usage()
+dashboard_test = DashboardTestAtScale(addr)
+
+start_time = time.time()
+test_many_placement_groups()
+end_time = time.time()
+ray.get(monitor_actor.stop_run.remote())
+used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+print(f"Peak memory usage: {round(used_gb, 2)}GB")
+print(f"Peak memory usage per processes:\n {usage}")
+del monitor_actor
+ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+rate = MAX_PLACEMENT_GROUPS / (end_time - start_time)
+print(
+    f"Success! Started {MAX_PLACEMENT_GROUPS} pgs in "
+    f"{end_time - start_time}s. ({rate} pgs/s)"
+)
+
+results = {
+    "pgs_per_second": rate,
+    "num_pgs": MAX_PLACEMENT_GROUPS,
+    "time": end_time - start_time,
+    "_peak_memory": round(used_gb, 2),
+    "_peak_process_memory": usage,
+}
+if not is_smoke_test:
+    results["perf_metrics"] = [
+        {
+            "perf_metric_name": "pgs_per_second",
+            "perf_metric_value": rate,
+            "perf_metric_type": "THROUGHPUT",
+        }
+    ]
+dashboard_test.update_release_test_result(results)
+test_utils.safe_write_to_results_json(results)

release/benchmarks/distributed/test_many_tasks.py

@@ -0,0 +1,138 @@
+import time
+
+import click
+import tqdm
+from many_nodes_tests.dashboard_test import DashboardTestAtScale
+
+import ray
+import ray._common.test_utils
+import ray._private.test_utils as test_utils
+from ray._private.state_api_test_utils import (
+    StateAPICallSpec,
+    periodic_invoke_state_apis_with_actor,
+    summarize_worker_startup_time,
+)
+from ray.util.state import summarize_tasks
+
+sleep_time = 300
+
+
+def test_max_running_tasks(num_tasks):
+    cpus_per_task = 0.25
+
+    @ray.remote(num_cpus=cpus_per_task)
+    def task():
+        time.sleep(sleep_time)
+
+    def time_up(start_time):
+        return time.time() - start_time >= sleep_time
+
+    refs = [task.remote() for _ in tqdm.trange(num_tasks, desc="Launching tasks")]
+
+    max_cpus = ray.cluster_resources()["CPU"]
+    min_cpus_available = max_cpus
+    start_time = time.time()
+    for _ in tqdm.trange(int(sleep_time / 0.1), desc="Waiting"):
+        try:
+            cur_cpus = ray.available_resources().get("CPU", 0)
+            min_cpus_available = min(min_cpus_available, cur_cpus)
+        except Exception:
+            # There are race conditions `.get` can fail if a new heartbeat
+            # comes at the same time.
+            pass
+        if time_up(start_time):
+            print(f"Time up for sleeping {sleep_time} seconds")
+            break
+        time.sleep(0.1)
+
+    # There are some relevant magic numbers in this check. 10k tasks each
+    # require 1/4 cpus. Therefore, ideally 2.5k cpus will be used.
+    used_cpus = max_cpus - min_cpus_available
+    err_str = f"Only {used_cpus}/{max_cpus} cpus used."
+    # 1500 tasks. Note that it is a pretty low threshold, and the
+    # performance should be tracked via perf dashboard.
+    threshold = num_tasks * cpus_per_task * 0.60
+    print(f"{used_cpus}/{max_cpus} used.")
+    assert used_cpus > threshold, err_str
+
+    for _ in tqdm.trange(num_tasks, desc="Ensuring all tasks have finished"):
+        done, refs = ray.wait(refs)
+        assert ray.get(done[0]) is None
+
+    return used_cpus
+
+
+def no_resource_leaks():
+    return test_utils.no_resource_leaks_excluding_node_resources()
+
+
+@click.command()
+@click.option("--num-tasks", required=True, type=int, help="Number of tasks to launch.")
+def test(num_tasks):
+    addr = ray.init(address="auto")
+
+    ray._common.test_utils.wait_for_condition(no_resource_leaks)
+    monitor_actor = test_utils.monitor_memory_usage()
+    dashboard_test = DashboardTestAtScale(addr)
+
+    def not_none(res):
+        return res is not None
+
+    api_caller = periodic_invoke_state_apis_with_actor(
+        apis=[StateAPICallSpec(summarize_tasks, not_none)],
+        call_interval_s=4,
+        print_result=True,
+    )
+
+    start_time = time.time()
+    used_cpus = test_max_running_tasks(num_tasks)
+    end_time = time.time()
+    ray.get(monitor_actor.stop_run.remote())
+    used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+    print(f"Peak memory usage: {round(used_gb, 2)}GB")
+    print(f"Peak memory usage per processes:\n {usage}")
+    ray.get(api_caller.stop.remote())
+
+    del api_caller
+    del monitor_actor
+    ray._common.test_utils.wait_for_condition(no_resource_leaks)
+
+    try:
+        summarize_worker_startup_time()
+    except Exception as e:
+        print("Failed to summarize worker startup time.")
+        print(e)
+
+    rate = num_tasks / (end_time - start_time - sleep_time)
+    print(
+        f"Success! Started {num_tasks} tasks in {end_time - start_time}s. "
+        f"({rate} tasks/s)"
+    )
+
+    results = {
+        "tasks_per_second": rate,
+        "num_tasks": num_tasks,
+        "time": end_time - start_time,
+        "used_cpus": used_cpus,
+        "_peak_memory": round(used_gb, 2),
+        "_peak_process_memory": usage,
+        "perf_metrics": [
+            {
+                "perf_metric_name": "tasks_per_second",
+                "perf_metric_value": rate,
+                "perf_metric_type": "THROUGHPUT",
+            },
+            {
+                "perf_metric_name": "used_cpus_by_deadline",
+                "perf_metric_value": used_cpus,
+                "perf_metric_type": "THROUGHPUT",
+            },
+        ],
+    }
+
+    dashboard_test.update_release_test_result(results)
+    test_utils.safe_write_to_results_json(results)
+
+
+if __name__ == "__main__":
+    test()

release/benchmarks/distributed/test_scheduling.py

@@ -0,0 +1,134 @@
+import argparse
+from math import floor
+from time import sleep, time
+
+import ray
+import ray._private.test_utils as test_utils
+from ray._private.test_utils import safe_write_to_results_json
+
+
+@ray.remote
+def simple_task(t):
+    sleep(t)
+
+
+@ray.remote
+class SimpleActor:
+    def __init__(self, job=None):
+        self._job = job
+
+    def ready(self):
+        return
+
+    def do_job(self):
+        if self._job is not None:
+            self._job()
+
+
+def start_tasks(num_task, num_cpu_per_task, task_duration):
+    ray.get(
+        [
+            simple_task.options(num_cpus=num_cpu_per_task).remote(task_duration)
+            for _ in range(num_task)
+        ]
+    )
+
+
+def measure(f):
+    start = time()
+    ret = f()
+    end = time()
+    return (end - start, ret)
+
+
+def start_actor(num_actors, num_actors_per_nodes, job):
+    resources = {"node": floor(1.0 / num_actors_per_nodes)}
+    submission_cost, actors = measure(
+        lambda: [
+            SimpleActor.options(resources=resources, num_cpus=0).remote(job)
+            for _ in range(num_actors)
+        ]
+    )
+    ready_cost, _ = measure(lambda: ray.get([actor.ready.remote() for actor in actors]))
+    actor_job_cost, _ = measure(
+        lambda: ray.get([actor.do_job.remote() for actor in actors])
+    )
+    return (submission_cost, ready_cost, actor_job_cost)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Test Scheduling")
+    # Task workloads
+    parser.add_argument(
+        "--total-num-task", type=int, help="Total number of tasks.", required=False
+    )
+    parser.add_argument(
+        "--num-cpu-per-task",
+        type=int,
+        help="Resources needed for tasks.",
+        required=False,
+    )
+    parser.add_argument(
+        "--task-duration-s",
+        type=int,
+        help="How long does each task execute.",
+        required=False,
+        default=1,
+    )
+
+    # Actor workloads
+    parser.add_argument(
+        "--total-num-actors", type=int, help="Total number of actors.", required=True
+    )
+    parser.add_argument(
+        "--num-actors-per-nodes",
+        type=int,
+        help="How many actors to allocate for each nodes.",
+        required=True,
+    )
+
+    ray.init(address="auto")
+    monitor_actor = test_utils.monitor_memory_usage()
+
+    total_cpus_per_node = [node["Resources"].get("CPU", 0) for node in ray.nodes()]
+    num_nodes = len(total_cpus_per_node)
+    total_cpus = sum(total_cpus_per_node)
+
+    args = parser.parse_args()
+    job = None
+    if args.total_num_task is not None:
+        if args.num_cpu_per_task is None:
+            args.num_cpu_per_task = floor(1.0 * total_cpus / args.total_num_task)
+        job = lambda: start_tasks(  # noqa: E731
+            args.total_num_task, args.num_cpu_per_task, args.task_duration_s
+        )
+
+    submission_cost, ready_cost, actor_job_cost = start_actor(
+        args.total_num_actors, args.num_actors_per_nodes, job
+    )
+
+    ray.get(monitor_actor.stop_run.remote())
+    used_gb, usage = ray.get(monitor_actor.get_peak_memory_info.remote())
+    print(f"Peak memory usage: {round(used_gb, 2)}GB")
+    print(f"Peak memory usage per processes:\n {usage}")
+    del monitor_actor
+
+    result = {
+        "total_num_task": args.total_num_task,
+        "num_cpu_per_task": args.num_cpu_per_task,
+        "task_duration_s": args.task_duration_s,
+        "total_num_actors": args.total_num_actors,
+        "num_actors_per_nodes": args.num_actors_per_nodes,
+        "num_nodes": num_nodes,
+        "total_cpus": total_cpus,
+        "submission_cost": submission_cost,
+        "ready_cost": ready_cost,
+        "actor_job_cost": actor_job_cost,
+        "_peak_memory": round(used_gb, 2),
+        "_peak_process_memory": usage,
+        "_runtime": submission_cost + ready_cost + actor_job_cost,
+    }
+
+    safe_write_to_results_json(result)
+
+    print(result)