chore: import upstream snapshot with attribution

release/benchmarks/distributed/many_nodes_tests/actor_test.py

@@ -0,0 +1,145 @@
+import argparse
+import os
+import math
+from time import sleep, perf_counter
+import json
+import ray
+
+from dashboard_test import DashboardTestAtScale
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors):
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+    print("Start launch actors")
+    actors = [Actor.options(max_restarts=-1).remote() for _ in range(total_actors)]
+    return actors
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", nargs="+", type=int, required=True)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--no-wait", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def scale_cluster_up(num_cpus):
+    print(f"Start to scale up to {num_cpus} cpus")
+
+    def get_curr_cpus():
+        return int(sum([r.get("Resources", {}).get("CPU", 0) for r in ray.nodes()]))
+
+    step = 1000
+    curr_cpus = get_curr_cpus()
+    target_cpus = curr_cpus
+
+    while curr_cpus < num_cpus:
+        curr_cpus = get_curr_cpus()
+        new_target_cpus = min(curr_cpus + step, num_cpus)
+        if new_target_cpus != target_cpus:
+            target_cpus = new_target_cpus
+            ray.autoscaler.sdk.request_resources(num_cpus=target_cpus)
+        print(f"Waiting for cluster to be up: {curr_cpus}->{target_cpus}->{num_cpus}")
+        sleep(10)
+
+
+def get_head_node_cpus():
+    head_ip = ray.util.get_node_ip_address()
+    for node in ray.nodes():
+        if node["Alive"] and node["NodeManagerAddress"] == head_ip:
+            return int(node.get("Resources", {}).get("CPU", 0))
+    return 0
+
+
+def run_one(total_actors, cpus_per_actor, no_wait):
+    total_cpus = cpus_per_actor * total_actors + get_head_node_cpus()
+    total_cpus = int(math.ceil(total_cpus))
+    scale_cluster_up(total_cpus)
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(cpus_per_actor, total_actors)
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    actor_ready_start = perf_counter()
+    total_actors = len(actors)
+    objs = [actor.foo.remote() for actor in actors]
+
+    while len(objs) != 0:
+        timeout = None if no_wait else 30
+        objs_ready, objs = ray.wait(objs, num_returns=len(objs), timeout=timeout)
+        print(
+            f"Status: {total_actors - len(objs)}/{total_actors}, "
+            f"{perf_counter() - actor_ready_start}"
+        )
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    throughput = total_actors / (actor_ready_time + actor_launch_time)
+    print(f"Actor launch time: {actor_launch_time} ({total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({total_actors} actors)"
+    )
+    print(f"Through put: {throughput}")
+
+    return {
+        "actor_launch_time": actor_launch_time,
+        "actor_ready_time": actor_ready_time,
+        "total_time": actor_launch_time + actor_ready_time,
+        "num_actors": total_actors,
+        "throughput": throughput,
+    }
+
+
+def main():
+    args, unknown = parse_script_args()
+    args.total_actors.sort()
+
+    addr = ray.init(address="auto")
+    dashboard_test = DashboardTestAtScale(addr)
+
+    result = {}
+    for i in args.total_actors:
+        result[f"many_nodes_actor_tests_{i}"] = run_one(
+            i, args.cpus_per_actor, args.no_wait
+        )
+
+    # Print the results early so if failed in the future, we still
+    # can see it in the log.
+    print(f"Result: {json.dumps(result, indent=2)}")
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            perf = [
+                {
+                    "perf_metric_name": name,
+                    "perf_metric_value": r["throughput"],
+                    "perf_metric_type": "THROUGHPUT",
+                }
+                for (name, r) in result.items()
+            ]
+            result["perf_metrics"] = perf
+            dashboard_test.update_release_test_result(result)
+
+            print(f"Writing data into file: {os.environ['TEST_OUTPUT_JSON']}")
+            json.dump(result, out_file)
+
+    print("Test finished successfully!")
+    ray.shutdown()
+
+    # We need to make sure GCS cool down otherwise, testing infra
+    # might get timeout when fetching the result because when the driver
+    # got shutdown, many actors needs to be terminated which will
+    # overload GCS.
+    print("Sleep for 60s, waiting for the cluster to cool down.")
+    sleep(60)
+
+
+if __name__ == "__main__":
+    main()

release/benchmarks/distributed/many_nodes_tests/compute_config.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+# NFS needs to be disabled for this test, since the test spawns too many nodes
+# and may hit the limit on the # of clients.
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: as-feature-disable-nfs-mount
+          Value: "true"
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        DeleteOnTermination: true
+        VolumeSize: 30
+
+head_node:
+  instance_type: m5.16xlarge
+
+worker_nodes:
+  - instance_type: m6i.large
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/compute_config_gce.yaml

@@ -0,0 +1,25 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+  - us-west1-c
+
+advanced_instance_config:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 30
+
+    # NFS needs to be disabled for this test, since the test spawns too many nodes
+    # and may hit the limit on the # of clients.
+    labels:
+      as-feature-disable-nfs-mount: "true"
+
+head_node:
+  instance_type: n2-standard-64
+
+worker_nodes:
+  - instance_type: n2-standard-2
+    min_nodes: 500
+    max_nodes: 2000
+    market_type: ON_DEMAND

release/benchmarks/distributed/many_nodes_tests/dashboard_test.py

@@ -0,0 +1,185 @@
+import asyncio
+import time
+import urllib
+from typing import Dict, Optional, List
+from pprint import pprint
+
+import requests
+import ray
+import logging
+import os
+
+from collections import defaultdict
+from ray.util.state import list_nodes
+from ray._private.test_utils import get_system_metric_for_component
+from pydantic import BaseModel
+from ray.dashboard.utils import get_address_for_submission_client
+from ray.dashboard.modules.metrics.metrics_head import (
+    DEFAULT_PROMETHEUS_HOST,
+    PROMETHEUS_HOST_ENV_VAR,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def calc_p(latencies, percent):
+    if len(latencies) == 0:
+        return 0
+
+    return round(sorted(latencies)[int(len(latencies) / 100 * percent)] * 1000, 3)
+
+
+class Result(BaseModel):
+    success: bool
+    # endpoints -> list of latencies
+    result: Dict[str, List[float]]
+    # Dashboard memory usage in MB.
+    memory_mb: Optional[float]
+
+
+# Currently every endpoint is GET endpoints.
+endpoints = [
+    "/logical/actors",
+    "/nodes?view=summary",
+    "/",
+    "/api/cluster_status",
+    "/events",
+    "/api/jobs/",
+    "/api/v0/logs",
+    "/api/prometheus_health",
+]
+
+
+@ray.remote(num_cpus=0)
+class DashboardTester:
+    def __init__(self, interval_s: int = 1):
+        self.dashboard_url = get_address_for_submission_client(None)
+        # Ping interval for all endpoints.
+        self.interval_s = interval_s
+        # endpoint -> a list of latencies
+        self.result = defaultdict(list)
+
+    async def run(self):
+        await asyncio.gather(*[self.ping(endpoint) for endpoint in endpoints])
+
+    async def ping(self, endpoint):
+        """Synchronously call an endpoint."""
+        node_id = ray.get_runtime_context().get_node_id()
+        while True:
+            start = time.monotonic()
+            # for logs API, we should append node ID and glob.
+            if "/api/v0/logs" in endpoint:
+                glob_filter = "*"
+
+                options_dict = {"node_id": node_id, "glob": glob_filter}
+                url = (
+                    f"{self.dashboard_url}{endpoint}?"
+                    f"{urllib.parse.urlencode(options_dict)}"
+                )
+            else:
+                url = f"{self.dashboard_url}{endpoint}"
+
+            resp = requests.get(url, timeout=30)
+            elapsed = time.monotonic() - start
+
+            if resp.status_code == 200:
+                self.result[endpoint].append(time.monotonic() - start)
+            else:
+                try:
+                    resp.raise_for_status()
+                except Exception as e:
+                    logger.exception(e)
+            await asyncio.sleep(max(0, self.interval_s, elapsed))
+
+    def get_result(self):
+        return self.result
+
+
+class DashboardTestAtScale:
+    """This is piggybacked into existing scalability tests."""
+
+    def __init__(self, addr: ray._private.worker.RayContext):
+        self.addr = addr
+
+        # Schedule the actor on the current node (which is a head node).
+        current_node_ip = ray._private.worker.global_worker.node_ip_address
+        nodes = list_nodes(filters=[("node_ip", "=", current_node_ip)])
+        assert len(nodes) > 0, f"{current_node_ip} not found in the cluster"
+        node = nodes[0]
+        # Schedule on a head node.
+        self.tester = DashboardTester.options(
+            label_selector={ray._raylet.RAY_NODE_ID_KEY: node["node_id"]}
+        ).remote()
+
+        self.tester.run.remote()
+
+    def get_result(self):
+        """Get the result from the test.
+
+        Returns:
+            A tuple of success, and the result (Result object).
+        """
+        try:
+            result = ray.get(self.tester.get_result.remote(), timeout=60)
+        except ray.exceptions.GetTimeoutError:
+            return Result(success=False)
+
+        # Get the memory usage.
+        memories = get_system_metric_for_component(
+            "ray_component_uss_bytes",
+            "dashboard",
+            os.environ.get(PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST),
+        )
+
+        return Result(
+            success=True,
+            result=result,
+            memory_mb=max(memories) / 1.0e6 if memories else None,
+        )
+
+    def update_release_test_result(self, release_result: dict):
+        test_result = self.get_result()
+
+        def calc_endpoints_p(result, percent):
+            return {
+                # sort -> get PX -> convert second to ms -> round up.
+                endpoint: calc_p(latencies, percent)
+                for endpoint, latencies in result.items()
+            }
+
+        print("======Print per dashboard endpoint latencies======")
+        print("=====================P50==========================")
+        pprint(calc_endpoints_p(test_result.result, 50))
+        print("=====================P95==========================")
+        pprint(calc_endpoints_p(test_result.result, 95))
+        print("=====================P99==========================")
+        pprint(calc_endpoints_p(test_result.result, 99))
+
+        latencies = []
+        for per_endpoint_latencies in test_result.result.values():
+            latencies.extend(per_endpoint_latencies)
+        aggregated_metrics = {
+            "p50": calc_p(latencies, 50),
+            "p95": calc_p(latencies, 95),
+            "p99": calc_p(latencies, 99),
+        }
+
+        print("=====================Aggregated====================")
+        pprint(aggregated_metrics)
+
+        release_result["_dashboard_test_success"] = test_result.success
+        if test_result.success:
+            if "perf_metrics" not in release_result:
+                release_result["perf_metrics"] = []
+
+            release_result["perf_metrics"].extend(
+                [
+                    {
+                        "perf_metric_name": f"dashboard_{p}_latency_ms",
+                        "perf_metric_value": value,
+                        "perf_metric_type": "LATENCY",
+                    }
+                    for p, value in aggregated_metrics.items()
+                ]
+            )
+            release_result["_dashboard_memory_usage_mb"] = test_result.memory_mb

release/benchmarks/distributed/many_nodes_tests/multi_master_test.py

@@ -0,0 +1,90 @@
+import argparse
+import os
+from time import sleep, perf_counter
+import json
+import ray
+
+
+def test_max_actors_launch(cpus_per_actor, total_actors, num_masters):
+    # By default, there are 50 groups, each group has 1 master and 99 slaves.
+    num_slaves_per_master = total_actors / num_masters - 1
+
+    @ray.remote(num_cpus=cpus_per_actor)
+    class Actor:
+        def foo(self):
+            pass
+
+        def create(self):
+            return [
+                Actor.options(max_restarts=-1).remote()
+                for _ in range(num_slaves_per_master)
+            ]
+
+    print("Start launch actors")
+    # The 50 masters are spreaded.
+    actors = [
+        Actor.options(max_restarts=-1, scheduling_strategy="SPREAD").remote()
+        for _ in range(num_masters)
+    ]
+    slaves_per_master = []
+    for master in actors:
+        slaves_per_master.append(master.create.remote())
+    for slaves in slaves_per_master:
+        actors.extend(ray.get(slaves))
+    return actors
+
+
+def test_actor_ready(actors):
+    remaining = [actor.foo.remote() for actor in actors]
+    ray.get(remaining)
+
+
+def parse_script_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cpus-per-actor", type=float, default=0.2)
+    parser.add_argument("--total-actors", type=int, default=5000)
+    parser.add_argument("--num-masters", type=int, default=50)
+    parser.add_argument("--no-report", default=False, action="store_true")
+    parser.add_argument("--fail", default=False, action="store_true")
+    return parser.parse_known_args()
+
+
+def main():
+    args, unknown = parse_script_args()
+
+    ray.init(address="auto")
+
+    actor_launch_start = perf_counter()
+    actors = test_max_actors_launch(
+        args.cpus_per_actor, args.total_actors, args.num_masters
+    )
+    actor_launch_end = perf_counter()
+    actor_launch_time = actor_launch_end - actor_launch_start
+    if args.fail:
+        sleep(10)
+        return
+    actor_ready_start = perf_counter()
+    test_actor_ready(actors)
+    actor_ready_end = perf_counter()
+    actor_ready_time = actor_ready_end - actor_ready_start
+
+    print(f"Actor launch time: {actor_launch_time} ({args.total_actors} actors)")
+    print(f"Actor ready time: {actor_ready_time} ({args.total_actors} actors)")
+    print(
+        f"Total time: {actor_launch_time + actor_ready_time}"
+        f" ({args.total_actors} actors)"
+    )
+
+    if "TEST_OUTPUT_JSON" in os.environ and not args.no_report:
+        with open(os.environ["TEST_OUTPUT_JSON"], "w") as out_file:
+            results = {
+                "actor_launch_time": actor_launch_time,
+                "actor_ready_time": actor_ready_time,
+                "total_time": actor_launch_time + actor_ready_time,
+                "num_actors": args.total_actors,
+            }
+            json.dump(results, out_file)
+
+
+if __name__ == "__main__":
+    main()