chore: import upstream snapshot with attribution

python/ray/tests/chaos/chaos_network_bandwidth.yaml

@@ -0,0 +1,17 @@
+# Injects a bandwidth limit to 1mbps to all traffic to the Ray nodes.
+apiVersion: chaos-mesh.org/v1alpha1
+kind: NetworkChaos
+metadata:
+  name: bandwidth
+spec:
+  action: bandwidth
+  mode: all
+  selector:
+    namespaces:
+      - default
+    labelSelectors:
+      'ray.io/cluster': 'raycluster-kuberay' # inject to all pods
+  bandwidth:
+    rate: '1mbps'
+    limit: 20971520
+    buffer: 10000

python/ray/tests/chaos/chaos_network_delay.yaml

@@ -0,0 +1,16 @@
+# Injects a 200ms delay to all traffic to the Ray nodes.
+apiVersion: chaos-mesh.org/v1alpha1
+kind: NetworkChaos
+metadata:
+  name: network-delay
+spec:
+  action: delay
+  mode: all # inject to all pods
+  selector:
+    namespaces:
+      - default
+    labelSelectors:
+      'ray.io/cluster': 'raycluster-kuberay' # inject to all pods
+  delay:
+    latency: '200ms'
+  duration: '12h'

python/ray/tests/chaos/potato_passer.py

@@ -0,0 +1,68 @@
+import argparse
+import asyncio
+
+import ray
+
+ray.init()
+
+"""
+Potato passer is a test script that lets multiple actors call each other's methods.
+Actors are wired in a round-trip fashion: actor 0 calls actor 1, which calls actor 2.
+The last actor calls actor 0. In each call, the actor sleeps for a time, occationally
+prints, and calls next actor.
+
+Note the number of tasks on-the-fly can go up to `pass-times` because the next call is
+made before exiting current call.
+"""
+
+
+@ray.remote
+class PotatoPasser:
+    def __init__(self, name, next_name, sleep_secs):
+        self.count = 0
+        self.name = name
+        self.next_name = next_name
+        self.sleep_secs = sleep_secs
+        self.print_every = 100
+
+    async def pass_potato(self, potato: int, target: int):
+        self.count += 1
+        if potato % self.print_every == 0:
+            print(
+                f"running, name {self.name}, count {self.count}, "
+                f"potato {potato}, target {target}"
+            )
+        if potato >= target:
+            print(f"target reached! name = {self.name}, count = {self.count}")
+            return target
+        next_actor = ray.get_actor(self.next_name)
+        await asyncio.sleep(self.sleep_secs)
+        return await next_actor.pass_potato.remote(potato + 1, target)
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--num-actors", type=int, help="Make this many actors")
+    parser.add_argument("--pass-times", type=int, help="Pass this many messages")
+    parser.add_argument(
+        "--sleep-secs",
+        type=float,
+        help="Sleep seconds before sending message to next actor",
+    )
+    args = parser.parse_args()
+
+    actors = []
+    for i in range(args.num_actors):
+        this_actor = "actor" + str(i)
+        next_actor = "actor" + str((i + 1) % args.num_actors)
+        actor = PotatoPasser.options(
+            name=this_actor, scheduling_strategy="SPREAD"
+        ).remote(this_actor, next_actor, args.sleep_secs)
+        actors.append(actor)
+
+    ret = await actors[0].pass_potato.remote(0, args.pass_times)
+    print(f"passed potato {ret} times! expected {args.pass_times} times.")
+    assert ret == args.pass_times
+
+
+asyncio.run(main())

python/ray/tests/chaos/prepare_env.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# Sets up environment for the Kubernetes chaos testing.
+# The environment consists of:
+# - a KubeRay cluster, port-forwarded to localhost:8265.
+# - a chaos-mesh operator ready to inject faults.
+
+set -euo pipefail
+
+echo "--- Preparing k8s environment."
+bash ci/k8s/prep-k8s-environment.sh
+
+kind load docker-image ray-ci:kuberay-test
+
+# Helm install KubeRay
+echo "--- Installing KubeRay operator from official Helm repo."
+helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+helm install kuberay-operator kuberay/kuberay-operator
+kubectl wait pod  -l app.kubernetes.io/name=kuberay-operator \
+    --for=condition=Ready=True  --timeout=2m
+
+echo "--- Installing KubeRay cluster and port forward."
+
+helm install raycluster kuberay/ray-cluster \
+    --set image.repository=ray-ci \
+    --set image.tag=kuberay-test \
+    --set worker.replicas=2 \
+    --set worker.resources.limits.cpu=500m \
+    --set worker.resources.requests.cpu=500m \
+    --set head.resources.limits.cpu=500m \
+    --set head.resources.requests.cpu=500m \
+    --set worker.resources.limits.memory=4Gi \
+    --set worker.resources.requests.memory=4Gi \
+    --set head.resources.limits.memory=4Gi \
+    --set head.resources.requests.memory=4Gi
+
+kubectl wait pod -l ray.io/cluster=raycluster-kuberay \
+    --for=condition=Ready=True --timeout=5m
+kubectl port-forward service/raycluster-kuberay-head-svc 8265:8265 &
+
+# Helm install chaos-mesh
+echo "--- Installing chaos-mesh operator and CR."
+helm repo add chaos-mesh https://charts.chaos-mesh.org
+kubectl create ns chaos-mesh
+helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh \
+    --set chaosDaemon.runtime=containerd \
+    --set chaosDaemon.socketPath=/run/containerd/containerd.sock \
+    --version 2.6.1
+
+echo "--- Waiting for chaos-mesh to be ready."
+kubectl wait pod --namespace chaos-mesh --timeout=300s \
+    -l app.kubernetes.io/instance=chaos-mesh --for=condition=Ready=True

python/ray/tests/chaos/runtime_env.yaml

@@ -0,0 +1,2 @@
+env_vars:
+  RAY_DEDUP_LOGS: "0"

python/ray/tests/chaos/streaming_llm.py

@@ -0,0 +1,98 @@
+import argparse
+import asyncio
+import logging
+
+import requests
+from fastapi import FastAPI
+from starlette.responses import StreamingResponse
+
+import ray
+from ray import serve
+
+logger = logging.getLogger("ray.serve")
+
+fastapi_app = FastAPI()
+
+
+# Input: a prompt of words
+# Output: each word reversed and produced N times.
+@serve.deployment(
+    num_replicas=6, ray_actor_options={"num_cpus": 0.01, "memory": 10 * 1024 * 1024}
+)
+class ReverseAndDupEachWord:
+    def __init__(self, dup_times: int):
+        self.dup_times = dup_times
+
+    async def __call__(self, prompt: str):
+        for word in prompt.split():
+            rev = word[::-1]
+            for _ in range(self.dup_times):
+                await asyncio.sleep(0.001)
+                # Ideally we want to do " ".join(words), but for the sake of
+                # simplicity we also have an extra trailing space.
+                yield rev + " "
+
+
+@serve.deployment(
+    num_replicas=6, ray_actor_options={"num_cpus": 0.01, "memory": 10 * 1024 * 1024}
+)
+@serve.ingress(fastapi_app)
+class Textbot:
+    def __init__(self, llm):
+        self.llm = llm.options(stream=True)
+
+    @fastapi_app.post("/")
+    async def handle_request(self, prompt: str) -> StreamingResponse:
+        logger.info(f'Got prompt with size "{len(prompt)}"')
+        return StreamingResponse(self.llm.remote(prompt), media_type="text/plain")
+
+
+@ray.remote(num_cpus=0.1, memory=10 * 1024 * 1024)
+def make_http_query(num_words, num_queries):
+    for _ in range(num_queries):
+        words = "Lorem ipsum dolor sit amet".split()
+        prompt_words = [words[i % len(words)] for i in range(num_words)]
+        prompt = " ".join(prompt_words)
+        expected_words = [word[::-1] for word in prompt_words for _ in range(2)]
+
+        response = requests.post(f"http://localhost:8000/?prompt={prompt}", stream=True)
+        response.raise_for_status()
+        content = response.content.decode()
+        assert content == " ".join(expected_words) + " ", content
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generates HTTP workloads with Ray.")
+
+    parser.add_argument("--num_tasks", type=int, required=True, help="Number of tasks.")
+    parser.add_argument(
+        "--num_queries_per_task",
+        type=int,
+        required=True,
+        help="Number of queries per task.",
+    )
+    parser.add_argument(
+        "--num_words_per_query",
+        type=int,
+        required=True,
+        help="Number of words per query",
+    )
+
+    args = parser.parse_args()
+
+    # Run the serve, run the client, then showdown serve.
+    llm = ReverseAndDupEachWord.bind(2)
+    app = Textbot.bind(llm)
+
+    serve.run(app)
+
+    objs = [
+        make_http_query.remote(args.num_words_per_query, args.num_queries_per_task)
+        for _ in range(args.num_tasks)
+    ]
+    ray.get(objs)
+
+    serve.shutdown()
+
+
+main()

python/ray/tests/chaos/streaming_llm.yaml

@@ -0,0 +1,2 @@
+env_vars:
+  RAY_DEDUP_LOGS: "0"

python/ray/tests/chaos/test_many_job_submissions.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Sets up environment for the Kubernetes chaos testing.
+# The environment consists of:
+# - a KubeRay cluster, port-forwarded to localhost:8265.
+# - a chaos-mesh operator ready to inject faults.
+
+set -xe
+
+for i in {1..50}; do
+    echo "submitting round ${i}"
+    ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=3 --sleep-secs=0.01
+done

python/ray/tests/chaos/test_potato_passer.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Sets up environment for the Kubernetes chaos testing.
+# The environment consists of:
+# - a KubeRay cluster, port-forwarded to localhost:8265.
+# - a chaos-mesh operator ready to inject faults.
+
+set -xe
+
+ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=1000 --sleep-secs=0.01

python/ray/tests/chaos/test_streaming_llm.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Sets up environment for the Kubernetes chaos testing.
+# The environment consists of:
+# - a KubeRay cluster, port-forwarded to localhost:8265.
+# - a chaos-mesh operator ready to inject faults.
+
+set -xe
+
+ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/streaming_llm.yaml --working-dir python/ray/tests/chaos -- python streaming_llm.py --num_queries_per_task=100 --num_tasks=2 --num_words_per_query=100