chore: import upstream snapshot with attribution

release/long_running_tests/.gitignore

@@ -0,0 +1 @@
+config_temporary.yaml

release/long_running_tests/README.rst

@@ -0,0 +1,61 @@
+Long Running Tests
+==================
+
+This directory contains the long-running workloads which are intended to run
+forever until they fail. To set up the project you need to run
+
+.. code-block:: bash
+
+    $ pip install anyscale
+    $ anyscale init
+
+Note that all the long running test is running inside virtual environment, tensorflow_p36
+
+Running the Workloads
+---------------------
+The easiest approach to running these workloads is to use the
+`Releaser`_ tool to run them with the command
+``python cli.py suite:run long_running_tests``. By default, this
+will start a session to run each workload in the Anyscale product
+and kick them off.
+
+To run the tests manually, you can also use the `Anyscale UI <https://www.anyscale.dev/>`. First run ``anyscale snapshot create`` from the command line to create a project snapshot. Then from the UI, you can launch an individual session and execute the run command for each test.
+
+You can also start the workloads using the CLI with:
+
+.. code-block:: bash
+
+    $ anyscale start
+    $ anyscale run test_workload --workload=<WORKLOAD_NAME> --wheel=<RAY_WHEEL_LINK>
+
+
+Doing this for each workload will start one EC2 instance per workload and will start the workloads
+running (one per instance). A list of
+available workload options is available in the `ray_projects/project.yaml` file.
+
+
+Debugging
+---------
+The primary method to debug the test while it is running is to view the logs and the dashboard from the UI. After the test has failed, you can still view the stdout logs in the UI and also inspect
+the logs under ``/tmp/ray/session*/logs/`` and
+``/tmp/ray/session*/logs/debug_state.txt``.
+
+.. To check up on the workloads, run either
+.. ``anyscale session --name="*" execute check-load``, which
+.. will print the load on each machine, or
+.. ``anyscale session --name="*" execute show-output``, which
+.. will print the tail of the output for each workload.
+
+Shut Down the Workloads
+-----------------------
+
+The instances running the workloads can all be killed by running
+``anyscale stop <SESSION_NAME>``.
+
+Adding a Workload
+-----------------
+
+To create a new workload, simply add a new Python file under ``workloads/`` and
+add the workload in the run command in `ray-project/project.yaml`.
+
+.. _`Releaser`: https://github.com/ray-project/releaser

release/long_running_tests/app_config_np.yaml

@@ -0,0 +1,30 @@
+base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] }}
+env_vars: {}
+
+debian_packages:
+  - curl
+  - unzip
+
+python:
+  pip_packages:
+    - "gym[atari]>=0.21.0,<0.24.0"
+    - ale-py==0.7.5
+    - pygame
+    - pytest
+    - tensorflow
+    - torch
+    # AutoROM downloads ROMs via torrent when they are built. The torrent is unreliable,
+    # so we built it for py3 and use that instead. This wheel was tested for python 3.7, 3.8,
+    # and 3.9.
+    - https://ray-ci-deps-wheels.s3.us-west-2.amazonaws.com/AutoROM.accept_rom_license-0.5.4-py3-none-any.whl
+  conda_packages: []
+
+post_build_cmds:
+    - 'rm -r wrk || true && git clone https://github.com/wg/wrk.git /tmp/wrk && cd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin'
+    - pip3 install numpy==1.19 || true
+    - pip3 install pytest || true
+    - pip3 install ray[all]
+    # TODO (Alex): Ideally we would install all the dependencies from the new
+    # version too, but pip won't be able to find the new version of ray-cpp.
+    - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
+    - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}

release/long_running_tests/tpl_cpu_1.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m5.2xlarge
+
+worker_nodes: []
+
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '48'
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        VolumeSize: 300
+        DeleteOnTermination: true

release/long_running_tests/tpl_cpu_1_c5.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: c5.2xlarge
+
+worker_nodes: []
+
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '48'
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        VolumeSize: 300
+        DeleteOnTermination: true

release/long_running_tests/tpl_cpu_1_c5_gce.yaml

@@ -0,0 +1,21 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: c2-standard-8
+
+worker_nodes: []
+
+#advanced_instance_config:
+#  TagSpecifications:
+#    - ResourceType: "instance"
+#      Tags:
+#        - Key: ttl-hours
+#          Value: '48'
+#
+#  BlockDeviceMappings:
+#    - DeviceName: /dev/sda1
+#      Ebs:
+#        VolumeSize: 300
+#        DeleteOnTermination: true

release/long_running_tests/tpl_cpu_1_gce.yaml

@@ -0,0 +1,21 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-8
+
+worker_nodes: []
+
+#advanced_instance_config:
+#  TagSpecifications:
+#    - ResourceType: "instance"
+#      Tags:
+#        - Key: ttl-hours
+#          Value: '48'
+#
+#  BlockDeviceMappings:
+#    - DeviceName: /dev/sda1
+#      Ebs:
+#        VolumeSize: 300
+#        DeleteOnTermination: true

release/long_running_tests/tpl_cpu_1_large.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m5.4xlarge
+
+worker_nodes: []
+
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '48'
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        VolumeSize: 300
+        DeleteOnTermination: true

release/long_running_tests/tpl_cpu_2.yaml

@@ -0,0 +1,22 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 1
+
+head_node_type:
+    name: head_node
+    instance_type: m5.2xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: m5.2xlarge
+      min_workers: 1
+      max_workers: 1
+      use_spot: false
+
+advanced_configurations_json:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '48'

release/long_running_tests/tpl_cpu_4.yaml

@@ -0,0 +1,19 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+
+head_node:
+    instance_type: m5.2xlarge
+    resources:
+        CPU: 8
+
+worker_nodes:
+    - instance_type: m5.2xlarge
+      min_nodes: 3
+      max_nodes: 3
+      market_type: ON_DEMAND
+
+advanced_instance_config:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '48'

release/long_running_tests/tpl_cpu_4_gce.yaml

@@ -0,0 +1,21 @@
+cloud: {{env["ANYSCALE_CLOUD_NAME"]}}
+zones:
+    - us-west1-c
+
+head_node:
+    instance_type: n2-standard-8
+    resources:
+        CPU: 8
+
+worker_nodes:
+    - instance_type: n2-standard-8
+      min_nodes: 3
+      max_nodes: 3
+      market_type: ON_DEMAND
+
+#advanced_instance_config:
+#  TagSpecifications:
+#    - ResourceType: "instance"
+#      Tags:
+#        - Key: ttl-hours
+#          Value: '48'

release/long_running_tests/workloads/actor_deaths.py

@@ -0,0 +1,122 @@
+# This workload tests repeatedly killing actors and submitting tasks to them.
+import numpy as np
+import sys
+import time
+
+import ray
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import monitor_memory_usage, safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+object_store_memory = 10**8
+num_nodes = 2
+
+message = (
+    "Make sure there is enough memory on this machine to run this "
+    "workload. We divide the system memory by 2 to provide a buffer."
+)
+assert (
+    num_nodes * object_store_memory < ray._common.utils.get_system_memory() / 2
+), message
+
+# Simulate a cluster on one machine.
+
+cluster = Cluster()
+for i in range(num_nodes):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=8,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=object_store_memory,
+        dashboard_host="0.0.0.0",
+    )
+ray.init(address=cluster.address)
+monitor_actor = monitor_memory_usage()
+
+# Run the workload.
+
+num_parents = 5
+num_children = 5
+death_probability = 0.95
+
+
+@ray.remote
+class Child(object):
+    def __init__(self, death_probability):
+        self.death_probability = death_probability
+
+    def ping(self):
+        # Exit process with some probability.
+        exit_chance = np.random.rand()
+        if exit_chance > self.death_probability:
+            sys.exit(-1)
+
+
+@ray.remote
+class Parent(object):
+    def __init__(self, num_children, death_probability):
+        self.death_probability = death_probability
+        self.children = [Child.remote(death_probability) for _ in range(num_children)]
+
+    def ping(self, num_pings):
+        children_outputs = []
+        for _ in range(num_pings):
+            children_outputs += [child.ping.remote() for child in self.children]
+        try:
+            ray.get(children_outputs)
+        except Exception:
+            # Replace the children if one of them died.
+            self.__init__(len(self.children), self.death_probability)
+
+    def kill(self):
+        # Clean up children.
+        try:
+            ray.get([child.__ray_terminate__.remote() for child in self.children])
+        except ray.exceptions.RayActorError as e:
+            # Sleep for 30 more seconds so that drivers will get more
+            # information from GCS when actors are unexpectedly failed.
+            print("Failed to kill a children actor. Error: ", e)
+            time.sleep(30)
+            raise e
+
+
+parents = [Parent.remote(num_children, death_probability) for _ in range(num_parents)]
+
+iteration = 0
+start_time = time.time()
+previous_time = start_time
+while True:
+    ray.get([parent.ping.remote(10) for parent in parents])
+
+    # Kill a parent actor with some probability.
+    exit_chance = np.random.rand()
+    if exit_chance > death_probability:
+        parent_index = np.random.randint(len(parents))
+        parents[parent_index].kill.remote()
+        parents[parent_index] = Parent.remote(num_children, death_probability)
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )
+    previous_time = new_time
+    iteration += 1

release/long_running_tests/workloads/long_running_many_jobs.py

@@ -0,0 +1,135 @@
+"""Job submission long running test
+
+Submits many simple jobs on a long running cluster.
+
+Test owner: architkulkarni
+
+Acceptance criteria: Should run through and print "PASSED"
+"""
+
+import argparse
+import os
+import time
+import random
+from typing import List, Optional
+from ray.dashboard.modules.job.common import JobStatus
+from ray.dashboard.modules.job.pydantic_models import JobDetails
+import ray
+from ray.job_submission import JobSubmissionClient
+from ray._private.test_utils import safe_write_to_results_json
+
+NUM_CLIENTS = 4
+NUM_JOBS_PER_BATCH = 4
+
+SMOKE_TEST_TIMEOUT = 10 * 60  # 10 minutes
+FULL_TEST_TIMEOUT = 8 * 60 * 60  # 8 hours
+
+
+def wait_until_finish(
+    client: JobSubmissionClient,
+    job_id: str,
+    timeout_s: int = 10 * 60,
+    retry_interval_s: int = 10,
+) -> Optional[JobStatus]:
+    start_time_s = time.time()
+    while time.time() - start_time_s <= timeout_s:
+        # Test calling list_jobs
+        client.list_jobs()
+        status = client.get_job_status(job_id)
+        if status in {JobStatus.SUCCEEDED, JobStatus.STOPPED, JobStatus.FAILED}:
+            return status
+        time.sleep(retry_interval_s)
+    return None
+
+
+def submit_batch_jobs(
+    clients: List[JobSubmissionClient],
+    num_jobs: int,
+    timeout_s: int = 10 * 60,
+    retry_interval_s: int = 1,
+) -> bool:
+    job_ids = []
+    for i in range(num_jobs):
+        # Cycle through clients arbitrarily
+        client = clients[i % len(clients)]
+        job_id = client.submit_job(
+            entrypoint="echo hello",
+        )
+        job_ids.append(job_id)
+        print(f"submitted job: {job_id}")
+
+    for job_id in job_ids:
+        client = clients[job_ids.index(job_id) % len(clients)]
+        status = wait_until_finish(client, job_id, timeout_s, retry_interval_s)
+        if status != JobStatus.SUCCEEDED:
+            print(
+                f"Info for failed/timed-out job {job_id}: {client.get_job_info(job_id)}"
+            )
+            print(
+                f"Logs for failed/timed-out job {job_id}: {client.get_job_logs(job_id)}"
+            )
+            print(
+                f"Job {job_id} failed with status {status} (`None` indicates timeout)"
+            )
+            return False
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--smoke-test", action="store_true", help="Finish quickly for testing."
+    )
+
+    parser.add_argument("--num-clients", type=int, default=NUM_CLIENTS)
+    parser.add_argument("--num-jobs-per-batch", type=int, default=NUM_JOBS_PER_BATCH)
+
+    args = parser.parse_args()
+    if args.smoke_test:
+        print(f"Running smoke test with timeout {SMOKE_TEST_TIMEOUT} seconds")
+        timeout = SMOKE_TEST_TIMEOUT
+    else:
+        print(f"Running full test (timeout: {FULL_TEST_TIMEOUT}s)")
+        timeout = FULL_TEST_TIMEOUT
+
+    start = time.time()
+
+    ray.init()
+    address = os.environ.get("RAY_ADDRESS")
+    job_name = os.environ.get("RAY_JOB_NAME", "jobs_basic")
+
+    if address is None or not address.startswith("anyscale://"):
+        address = "http://127.0.0.1:8265"
+
+    clients = [JobSubmissionClient(address) for i in range(NUM_CLIENTS)]
+
+    batch_counter = 0
+    while time.time() - start < timeout:
+        batch_counter += 1
+        print(f"Submitting batch {batch_counter}...")
+        # Submit a batch of jobs
+        if not submit_batch_jobs(clients, NUM_JOBS_PER_BATCH):
+            print("FAILED")
+            exit(1)
+
+        # Test list jobs
+        jobs: List[JobDetails] = clients[0].list_jobs()
+        print(f"Total jobs submitted so far: {len(jobs)}")
+
+        # Get job logs from random submission job
+        is_submission_job = False
+        while not is_submission_job:
+            job_details = random.choice(jobs)
+            is_submission_job = job_details.type == "SUBMISSION"
+        job_id = job_details.submission_id
+        print(f"Getting logs for randomly chosen job {job_id}...")
+        logs = clients[0].get_job_logs(job_id)
+        print(logs)
+
+    time_taken = time.time() - start
+    result = {
+        "time_taken": time_taken,
+    }
+    safe_write_to_results_json(result, "/tmp/jobs_basic.json")
+
+    print("PASSED")

release/long_running_tests/workloads/many_actor_tasks.py

@@ -0,0 +1,86 @@
+# This workload tests submitting many actor methods.
+import time
+
+import numpy as np
+
+import ray
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+object_store_memory = 10**8
+num_nodes = 10
+
+message = (
+    "Make sure there is enough memory on this machine to run this "
+    "workload. We divide the system memory by 2 to provide a buffer."
+)
+assert (
+    num_nodes * object_store_memory < ray._common.utils.get_system_memory() / 2
+), message
+
+# Simulate a cluster on one machine.
+
+cluster = Cluster()
+for i in range(num_nodes):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=5,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=object_store_memory,
+        dashboard_host="0.0.0.0",
+    )
+ray.init(address=cluster.address)
+
+# Run the workload.
+
+
+@ray.remote
+class Actor(object):
+    def __init__(self):
+        self.value = 0
+
+    def method(self):
+        self.value += 1
+        return np.zeros(1024, dtype=np.uint8)
+
+
+actors = [
+    Actor._remote([], {}, num_cpus=0.1, resources={str(i % num_nodes): 0.1})
+    for i in range(num_nodes * 5)
+]
+
+iteration = 0
+start_time = time.time()
+previous_time = start_time
+while True:
+    for _ in range(100):
+        previous_ids = [a.method.remote() for a in actors]
+
+    ray.get(previous_ids)
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )
+    previous_time = new_time
+    iteration += 1

release/long_running_tests/workloads/many_drivers.py

@@ -0,0 +1,111 @@
+# This workload tests many drivers using the same cluster.
+import time
+import argparse
+
+import ray
+from ray._common.test_utils import run_string_as_driver
+from ray._private.test_utils import safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+ray.init()
+
+nodes = [node["NodeID"] for node in ray.nodes()]
+assert len(nodes) == 4
+
+# Run the workload.
+
+# Define a driver script that runs a few tasks and actors on each node in the
+# cluster.
+driver_script = f"""
+import ray
+
+ray.init()
+
+nodes = "{','.join(nodes)}".split(',')
+
+@ray.remote(num_cpus=1)
+def f():
+    return 1
+
+@ray.remote(num_cpus=1)
+class Actor(object):
+    def method(self):
+        return 1
+
+for _ in range(5):
+    for node in nodes:
+        assert ray.get(
+            f.options(label_selector={{ray._raylet.RAY_NODE_ID_KEY: node}}).remote()) == 1
+        actor = Actor.options(label_selector={{ray._raylet.RAY_NODE_ID_KEY: node}}).remote()
+        assert ray.get(actor.method.remote()) == 1
+
+print("success")
+"""
+
+
+@ray.remote(num_cpus=0)
+def run_driver():
+    output = run_string_as_driver(driver_script, encode="utf-8")
+    assert "success" in output
+
+
+iteration = 0
+running_ids = [
+    run_driver.options(label_selector={ray._raylet.RAY_NODE_ID_KEY: node}).remote()
+    for node in nodes
+]
+start_time = time.time()
+previous_time = start_time
+
+parser = argparse.ArgumentParser(prog="Many Drivers long running tests")
+parser.add_argument(
+    "--iteration-num", type=int, help="How many iterations to run", required=False
+)
+parser.add_argument(
+    "--smoke-test",
+    action="store_true",
+    help="Whether or not the test is smoke test.",
+    default=False,
+)
+args = parser.parse_args()
+
+iteration_num = args.iteration_num
+if args.smoke_test:
+    iteration_num = 400
+while True:
+    if iteration_num is not None and iteration_num < iteration:
+        break
+    # Wait for a driver to finish and start a new driver.
+    [ready_id], running_ids = ray.wait(running_ids, num_returns=1)
+    ray.get(ready_id)
+
+    running_ids.append(
+        run_driver.options(
+            label_selector={ray._raylet.RAY_NODE_ID_KEY: nodes[iteration % len(nodes)]}
+        ).remote()
+    )
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )
+    previous_time = new_time
+    iteration += 1

release/long_running_tests/workloads/many_tasks.py

@@ -0,0 +1,81 @@
+# This workload tests submitting and getting many tasks over and over.
+import time
+
+import numpy as np
+
+import ray
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+object_store_memory = 10**8
+num_nodes = 10
+
+message = (
+    "Make sure there is enough memory on this machine to run this "
+    "workload. We divide the system memory by 2 to provide a buffer."
+)
+assert (
+    num_nodes * object_store_memory < ray._common.utils.get_system_memory() / 2
+), message
+
+# Simulate a cluster on one machine.
+
+cluster = Cluster()
+for i in range(num_nodes):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=2,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=object_store_memory,
+        dashboard_host="0.0.0.0",
+    )
+ray.init(address=cluster.address)
+
+# Run the workload.
+
+
+@ray.remote
+def f(*xs):
+    return np.zeros(1024, dtype=np.uint8)
+
+
+iteration = 0
+ids = []
+start_time = time.time()
+previous_time = start_time
+while True:
+    for _ in range(50):
+        new_constrained_ids = [
+            f._remote(args=[*ids], resources={str(i % num_nodes): 1}) for i in range(25)
+        ]
+        new_unconstrained_ids = [f.remote(*ids) for _ in range(25)]
+        ids = new_constrained_ids + new_unconstrained_ids
+
+    ray.get(ids)
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )
+    previous_time = new_time
+    iteration += 1

release/long_running_tests/workloads/many_tasks_serialized_ids.py

@@ -0,0 +1,109 @@
+# This workload stresses distributed reference counting by passing and
+# returning serialized ObjectRefs.
+import time
+import random
+
+import numpy as np
+
+import ray
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+object_store_memory = 10**8
+num_nodes = 10
+
+message = (
+    "Make sure there is enough memory on this machine to run this "
+    "workload. We divide the system memory by 2 to provide a buffer."
+)
+assert (
+    num_nodes * object_store_memory < ray._common.utils.get_system_memory() / 2
+), message
+
+# Simulate a cluster on one machine.
+
+cluster = Cluster()
+for i in range(num_nodes):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=2,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=object_store_memory,
+        dashboard_host="0.0.0.0",
+    )
+ray.init(address=cluster.address)
+
+# Run the workload.
+
+
+@ray.remote(max_retries=0)
+def churn():
+    return ray.put(np.zeros(1024 * 1024, dtype=np.uint8))
+
+
+@ray.remote(max_retries=0)
+def child(*xs):
+    obj_ref = ray.put(np.zeros(1024 * 1024, dtype=np.uint8))
+    return obj_ref
+
+
+@ray.remote(max_retries=0)
+def f(*xs):
+    if xs:
+        return random.choice(xs)
+    else:
+        return child.remote(*xs)
+
+
+iteration = 0
+ids = []
+start_time = time.time()
+previous_time = start_time
+while True:
+    for _ in range(50):
+        new_constrained_ids = [
+            f._remote(args=ids, resources={str(i % num_nodes): 1}) for i in range(25)
+        ]
+        new_unconstrained_ids = [f.remote(*ids) for _ in range(25)]
+        ids = new_constrained_ids + new_unconstrained_ids
+
+    # Fill the object store while the tasks are running.
+    for i in range(num_nodes):
+        for _ in range(10):
+            [
+                churn._remote(args=[], resources={str(i % num_nodes): 1})
+                for _ in range(10)
+            ]
+
+    # Make sure that the objects are still available.
+    child_ids = ray.get(ids)
+    for child_id in child_ids:
+        ray.get(child_id)
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    previous_time = new_time
+    iteration += 1
+
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )

release/long_running_tests/workloads/node_failures.py

@@ -0,0 +1,83 @@
+# This workload tests repeatedly killing a node and adding a new node.
+import time
+
+import ray
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import get_other_nodes, safe_write_to_results_json
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+object_store_memory = 10**8
+num_nodes = 10
+
+message = (
+    "Make sure there is enough memory on this machine to run this "
+    "workload. We divide the system memory by 2 to provide a buffer."
+)
+assert (
+    num_nodes * object_store_memory < ray._common.utils.get_system_memory() / 2
+), message
+
+# Simulate a cluster on one machine.
+
+cluster = Cluster()
+for i in range(num_nodes):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=2,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=object_store_memory,
+        dashboard_host="0.0.0.0",
+    )
+ray.init(address=cluster.address)
+
+# Run the workload.
+
+
+@ray.remote
+def f(*xs):
+    return 1
+
+
+iteration = 0
+previous_ids = [1 for _ in range(100)]
+start_time = time.time()
+previous_time = start_time
+while True:
+    for _ in range(100):
+        previous_ids = [f.remote(previous_id) for previous_id in previous_ids]
+
+    ray.get(previous_ids)
+
+    for _ in range(100):
+        previous_ids = [f.remote(previous_id) for previous_id in previous_ids]
+    node_to_kill = get_other_nodes(cluster, exclude_head=True)[0]
+
+    # Remove the first non-head node.
+    cluster.remove_node(node_to_kill)
+    cluster.add_node()
+
+    new_time = time.time()
+    print(
+        "Iteration {}:\n"
+        "  - Iteration time: {}.\n"
+        "  - Absolute time: {}.\n"
+        "  - Total elapsed time: {}.".format(
+            iteration, new_time - previous_time, new_time, new_time - start_time
+        )
+    )
+    update_progress(
+        {
+            "iteration": iteration,
+            "iteration_time": new_time - previous_time,
+            "absolute_time": new_time,
+            "elapsed_time": new_time - start_time,
+        }
+    )
+    previous_time = new_time
+    iteration += 1

release/long_running_tests/workloads/serve.py

@@ -0,0 +1,136 @@
+import re
+import time
+import subprocess
+from subprocess import PIPE
+
+import requests
+
+import ray
+from ray import serve
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import safe_write_to_results_json
+
+# Global variables / constants appear only right after imports.
+# Ray serve deployment setup constants
+NUM_REPLICAS = 7
+MAX_BATCH_SIZE = 16
+
+# Cluster setup constants
+OBJECT_STORE_MEMORY = 10**8
+NUM_NODES = 4
+
+# wrk setup constants (might want to make these configurable ?)
+NUM_CONNECTIONS = int(NUM_REPLICAS * MAX_BATCH_SIZE * 0.75)
+NUM_THREADS = 2
+# Append and print every 5mins for quick status polling as well
+# as time series plotting
+TIME_PER_CYCLE = "5m"
+
+
+def update_progress(result):
+    """
+    Write test result json to /tmp/, which will be read from
+    anyscale product runs in each releaser test
+    """
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+cluster = Cluster()
+for i in range(NUM_NODES):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        dashboard_agent_listen_port=(52365 + i),
+        num_cpus=8,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=OBJECT_STORE_MEMORY,
+        dashboard_host="0.0.0.0",
+    )
+
+ray.init(address=cluster.address, log_to_driver=False, dashboard_host="0.0.0.0")
+serve.start()
+
+
+@serve.deployment(name="echo", num_replicas=NUM_REPLICAS)
+class Echo:
+    @serve.batch(max_batch_size=MAX_BATCH_SIZE)
+    async def handle_batch(self, requests):
+        time.sleep(0.01)
+        return ["hi" for _ in range(len(requests))]
+
+    async def __call__(self, request):
+        return await self.handle_batch(request)
+
+
+serve.run(Echo.bind(), route_prefix="/echo")
+
+print("Warming up")
+for _ in range(5):
+    resp = requests.get("http://127.0.0.1:8000/echo").text
+    print(resp)
+    time.sleep(0.5)
+
+print("Started load testing with the following config: ")
+print(f"num_connections: {NUM_CONNECTIONS}")
+print(f"num_threads: {NUM_THREADS}")
+print(f"time_per_cycle: {TIME_PER_CYCLE}")
+
+while True:
+    proc = subprocess.Popen(
+        [
+            "wrk",
+            "-c",
+            str(NUM_CONNECTIONS),
+            "-t",
+            str(NUM_THREADS),
+            "-d",
+            TIME_PER_CYCLE,
+            "http://127.0.0.1:8000/echo",
+        ],
+        stdout=PIPE,
+        stderr=PIPE,
+    )
+    proc.wait()
+    out, err = proc.communicate()
+    # Check if wrk command succeeded. If this happens repeatedly, the release test
+    # infrastructure will correctly fail the test with "Last update to results json
+    # was too long ago."
+    if proc.returncode != 0:
+        print("wrk failed with the following error: ")
+        print(err)
+        print("Will try again in 5 seconds")
+        time.sleep(5)
+        continue
+    # Sample wrk stdout:
+    #
+    # Running 10s test @ http://127.0.0.1:8000/echo
+    # 2 threads and 84 connections
+    # Thread Stats   Avg      Stdev     Max   +/- Stdev
+    #     Latency    59.33ms   13.51ms 113.83ms   64.20%
+    #     Req/Sec   709.16     61.73   848.00     78.50%
+    # 14133 requests in 10.02s, 2.08MB read
+    # Requests/sec:   1410.71
+    # Transfer/sec:    212.16KB
+    metrics_dict = {}
+    for line in out.decode().splitlines():
+        parsed = re.split(r"\s+", line.strip())
+        if parsed[0] == "Latency":
+            metrics_dict["latency_avg"] = parsed[1]
+            metrics_dict["latency_stdev"] = parsed[2]
+            metrics_dict["latency_max"] = parsed[3]
+            metrics_dict["latency_+/-_stdev"] = parsed[4]
+        elif parsed[0] == "Req/Sec":
+            metrics_dict["req/sec_avg"] = parsed[1]
+            metrics_dict["req/sec_stdev"] = parsed[2]
+            metrics_dict["req/sec_max"] = parsed[3]
+            metrics_dict["req/sec_+/-_stdev"] = parsed[4]
+        elif parsed[0] == "Requests/sec:":
+            metrics_dict["requests/sec"] = parsed[1]
+        elif parsed[0] == "Transfer/sec:":
+            metrics_dict["transfer/sec"] = parsed[1]
+
+    print(out.decode())
+    print(err.decode())
+
+    update_progress(metrics_dict)

release/long_running_tests/workloads/serve_failure.py

@@ -0,0 +1,201 @@
+import asyncio
+import logging
+import os
+import random
+import string
+import time
+from typing import List
+
+import requests
+
+import ray
+from ray import serve
+from ray.serve.context import _get_global_client
+from ray.cluster_utils import Cluster
+from ray._private.test_utils import safe_write_to_results_json
+
+# Global variables / constants appear only right after imports.
+# Ray serve deployment setup constants
+NUM_REPLICAS = 7
+MAX_BATCH_SIZE = 16
+
+# Cluster setup constants
+OBJECT_STORE_MEMORY = 10**8
+NUM_NODES = 4
+
+# RandomTest setup constants
+CPUS_PER_NODE = 10
+NUM_ITERATIONS = 350
+ACTIONS_PER_ITERATION = 20
+
+RAY_UNIT_TEST = "RAY_UNIT_TEST" in os.environ
+
+# RAY_OVERRIDE_RESOURCES will prevent "num_cpus" from working
+# when adding a node with custom resources to Cluster.
+if "RAY_OVERRIDE_RESOURCES" in os.environ:
+    del os.environ["RAY_OVERRIDE_RESOURCES"]
+
+
+def update_progress(result):
+    """
+    Write test result json to /tmp/, which will be read from
+    anyscale product runs in each releaser test
+    """
+    result["last_update"] = time.time()
+    safe_write_to_results_json(result)
+
+
+cluster = Cluster()
+for i in range(NUM_NODES):
+    cluster.add_node(
+        redis_port=6379 if i == 0 else None,
+        num_cpus=16,
+        num_gpus=0,
+        resources={str(i): 2},
+        object_store_memory=OBJECT_STORE_MEMORY,
+        dashboard_host="0.0.0.0",
+    )
+
+ray.init(
+    namespace="serve_failure_test",
+    address=cluster.address,
+    dashboard_host="0.0.0.0",
+    log_to_driver=True,
+)
+serve.start(proxy_location="HeadOnly")
+
+
+@ray.remote(max_restarts=-1, max_task_retries=-1)
+class RandomKiller:
+    def __init__(self, kill_period_s=1):
+        self.kill_period_s = kill_period_s
+        self.sanctuary = set()
+
+    async def run(self):
+        while True:
+            chosen = random.choice(self._get_serve_actors())
+            print(f"Killing {chosen}")
+            ray.kill(chosen, no_restart=False)
+            await asyncio.sleep(self.kill_period_s)
+
+    async def spare(self, app_name: str):
+        print(f'Sparing application "{app_name}" replicas.')
+        self.sanctuary.add(app_name)
+
+    async def stop_spare(self, app_name: str):
+        print(f'No longer sparing application "{app_name}" replicas.')
+        self.sanctuary.discard(app_name)
+
+    def _get_serve_actors(self):
+        controller = _get_global_client()._controller
+        routers = list(ray.get(controller.get_proxies.remote()).values())
+        all_handles = routers + [controller]
+        replica_dict = ray.get(controller._all_running_replicas.remote())
+        for deployment_id, replica_info_list in replica_dict.items():
+            if deployment_id.app not in self.sanctuary:
+                for replica_info in replica_info_list:
+                    all_handles.append(replica_info.actor_handle)
+
+        return all_handles
+
+
+class RandomTest:
+    def __init__(self, random_killer_handle, max_applications=1):
+        self.max_applications = max_applications
+        self.weighted_actions = [
+            (self.create_application, 1),
+            (self.verify_application, 4),
+        ]
+        self.applications = []
+
+        self.random_killer = random_killer_handle
+
+        # Deploy in parallel to avoid long test startup time.
+        self.wait_for_applications_running(
+            [self.create_application(blocking=False) for _ in range(max_applications)]
+        )
+
+        self.random_killer.run.remote()
+
+    def wait_for_applications_running(self, application_names: List[str]):
+        client = _get_global_client()
+        for name in application_names:
+            client._wait_for_application_running(name, timeout_s=60)
+
+    def create_application(self, blocking: bool = True) -> str:
+        if len(self.applications) == self.max_applications:
+            app_to_delete = self.applications.pop()
+            serve.delete(app_to_delete)
+
+        new_name = "".join([random.choice(string.ascii_letters) for _ in range(10)])
+
+        @serve.deployment(name=new_name)
+        def handler(self, *args):
+            logging.getLogger("ray.serve").setLevel(logging.ERROR)
+            return new_name
+
+        if blocking:
+            ray.get(self.random_killer.spare.remote(new_name))
+            serve._run(
+                handler.bind(),
+                name=new_name,
+                route_prefix=f"/{new_name}",
+                _blocking=True,
+            )
+            self.applications.append(new_name)
+            ray.get(self.random_killer.stop_spare.remote(new_name))
+        else:
+            serve._run(
+                handler.bind(),
+                name=new_name,
+                route_prefix=f"/{new_name}",
+                _blocking=False,
+            )
+            self.applications.append(new_name)
+
+        return new_name
+
+    def verify_application(self):
+        app = random.choice(self.applications)
+        for _ in range(100):
+            try:
+                r = requests.get("http://127.0.0.1:8000/" + app)
+                assert r.text == app
+            except Exception:
+                print("Request to {} failed.".format(app))
+                time.sleep(0.1)
+
+    def run(self):
+        start_time = time.time()
+        previous_time = start_time
+        for iteration in range(NUM_ITERATIONS):
+            for _ in range(ACTIONS_PER_ITERATION):
+                actions, weights = zip(*self.weighted_actions)
+                action_chosen = random.choices(actions, weights=weights)[0]
+                print(f"Executing {action_chosen}")
+                action_chosen()
+
+            new_time = time.time()
+            print(
+                f"Iteration {iteration}:\n"
+                f"  - Iteration time: {new_time - previous_time}.\n"
+                f"  - Absolute time: {new_time}.\n"
+                f"  - Total elapsed time: {new_time - start_time}."
+            )
+            update_progress(
+                {
+                    "iteration": iteration,
+                    "iteration_time": new_time - previous_time,
+                    "absolute_time": new_time,
+                    "elapsed_time": new_time - start_time,
+                }
+            )
+            previous_time = new_time
+
+            if RAY_UNIT_TEST:
+                break
+
+
+random_killer = RandomKiller.remote()
+tester = RandomTest(random_killer, max_applications=NUM_NODES * CPUS_PER_NODE)
+tester.run()