chore: import upstream snapshot with attribution

doc/source/ray-core/doc_code/actor-http-server.py

@@ -0,0 +1,32 @@
+import ray
+import asyncio
+import requests
+from aiohttp import web
+
+
+@ray.remote
+class Counter:
+    async def __init__(self):
+        self.counter = 0
+        asyncio.get_running_loop().create_task(self.run_http_server())
+
+    async def run_http_server(self):
+        app = web.Application()
+        app.add_routes([web.get("/", self.get)])
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "127.0.0.1", 25001)
+        await site.start()
+
+    async def get(self, request):
+        return web.Response(text=str(self.counter))
+
+    async def increment(self):
+        self.counter = self.counter + 1
+
+
+ray.init()
+counter = Counter.remote()
+[ray.get(counter.increment.remote()) for i in range(5)]
+r = requests.get("http://127.0.0.1:25001/")
+assert r.text == "5"

doc/source/ray-core/doc_code/actor-pool.py

@@ -0,0 +1,17 @@
+import ray
+from ray.util import ActorPool
+
+
+@ray.remote
+class Actor:
+    def double(self, n):
+        return n * 2
+
+
+a1, a2 = Actor.remote(), Actor.remote()
+pool = ActorPool([a1, a2])
+
+# pool.map(..) returns a Python generator object ActorPool.map
+gen = pool.map(lambda a, v: a.double.remote(v), [1, 2, 3, 4])
+print(list(gen))
+# [2, 4, 6, 8]

doc/source/ray-core/doc_code/actor-queue.py

@@ -0,0 +1,23 @@
+import ray
+from ray.util.queue import Queue, Empty
+
+ray.init()
+# You can pass this object around to different tasks/actors
+queue = Queue(maxsize=100)
+
+
+@ray.remote
+def consumer(id, queue):
+    try:
+        while True:
+            next_item = queue.get(block=True, timeout=1)
+            print(f"consumer {id} got work {next_item}")
+    except Empty:
+        pass
+
+
+[queue.put(i) for i in range(10)]
+print("Put work 1 - 10 to queue...")
+
+consumers = [consumer.remote(id, queue) for id in range(2)]
+ray.get(consumers)

doc/source/ray-core/doc_code/actor-repr.py

@@ -0,0 +1,19 @@
+import ray
+
+
+@ray.remote
+class MyActor:
+    def __init__(self, index):
+        self.index = index
+
+    def foo(self):
+        print("hello there")
+
+    def __repr__(self):
+        return f"MyActor(index={self.index})"
+
+
+a = MyActor.remote(1)
+b = MyActor.remote(2)
+ray.get(a.foo.remote())
+ray.get(b.foo.remote())

doc/source/ray-core/doc_code/actor-sync.py

@@ -0,0 +1,46 @@
+import asyncio
+
+import ray
+
+
+# We set num_cpus to zero because this actor will mostly just block on I/O.
+@ray.remote(num_cpus=0)
+class SignalActor:
+    def __init__(self):
+        self.ready_event = asyncio.Event()
+
+    def send(self, clear=False):
+        self.ready_event.set()
+        if clear:
+            self.ready_event.clear()
+
+    async def wait(self, should_wait=True):
+        if should_wait:
+            await self.ready_event.wait()
+
+
+@ray.remote
+def wait_and_go(signal):
+    ray.get(signal.wait.remote())
+
+    print("go!")
+
+
+signal = SignalActor.remote()
+tasks = [wait_and_go.remote(signal) for _ in range(4)]
+print("ready...")
+# Tasks will all be waiting for the signals.
+print("set..")
+ray.get(signal.send.remote())
+
+# Tasks are unblocked.
+ray.get(tasks)
+
+# Output is:
+# ready...
+# set..
+
+# (wait_and_go pid=77366) go!
+# (wait_and_go pid=77372) go!
+# (wait_and_go pid=77367) go!
+# (wait_and_go pid=77358) go!

doc/source/ray-core/doc_code/actor_checkpointing.py

@@ -0,0 +1,95 @@
+# __actor_checkpointing_manual_restart_begin__
+import os
+import sys
+import ray
+import json
+import tempfile
+import shutil
+
+
+@ray.remote(num_cpus=1)
+class Worker:
+    def __init__(self):
+        self.state = {"num_tasks_executed": 0}
+
+    def execute_task(self, crash=False):
+        if crash:
+            sys.exit(1)
+
+        # Execute the task
+        # ...
+
+        # Update the internal state
+        self.state["num_tasks_executed"] = self.state["num_tasks_executed"] + 1
+
+    def checkpoint(self):
+        return self.state
+
+    def restore(self, state):
+        self.state = state
+
+
+class Controller:
+    def __init__(self):
+        self.worker = Worker.remote()
+        self.worker_state = ray.get(self.worker.checkpoint.remote())
+
+    def execute_task_with_fault_tolerance(self):
+        i = 0
+        while True:
+            i = i + 1
+            try:
+                ray.get(self.worker.execute_task.remote(crash=(i % 2 == 1)))
+                # Checkpoint the latest worker state
+                self.worker_state = ray.get(self.worker.checkpoint.remote())
+                return
+            except ray.exceptions.RayActorError:
+                print("Actor crashes, restarting...")
+                # Restart the actor and restore the state
+                self.worker = Worker.remote()
+                ray.get(self.worker.restore.remote(self.worker_state))
+
+
+controller = Controller()
+controller.execute_task_with_fault_tolerance()
+controller.execute_task_with_fault_tolerance()
+assert ray.get(controller.worker.checkpoint.remote())["num_tasks_executed"] == 2
+# __actor_checkpointing_manual_restart_end__
+
+
+# __actor_checkpointing_auto_restart_begin__
+@ray.remote(max_restarts=-1, max_task_retries=-1)
+class ImmortalActor:
+    def __init__(self, checkpoint_file):
+        self.checkpoint_file = checkpoint_file
+
+        if os.path.exists(self.checkpoint_file):
+            # Restore from a checkpoint
+            with open(self.checkpoint_file, "r") as f:
+                self.state = json.load(f)
+        else:
+            self.state = {}
+
+    def update(self, key, value):
+        import random
+
+        if random.randrange(10) < 5:
+            sys.exit(1)
+
+        self.state[key] = value
+
+        # Checkpoint the latest state
+        with open(self.checkpoint_file, "w") as f:
+            json.dump(self.state, f)
+
+    def get(self, key):
+        return self.state[key]
+
+
+checkpoint_dir = tempfile.mkdtemp()
+actor = ImmortalActor.remote(os.path.join(checkpoint_dir, "checkpoint.json"))
+ray.get(actor.update.remote("1", 1))
+ray.get(actor.update.remote("2", 2))
+assert ray.get(actor.get.remote("1")) == 1
+shutil.rmtree(checkpoint_dir)
+# __actor_checkpointing_auto_restart_end__

doc/source/ray-core/doc_code/actor_creator_failure.py

@@ -0,0 +1,44 @@
+# flake8: noqa
+
+# fmt: off
+# __actor_creator_failure_begin__
+import ray
+import os
+import signal
+ray.init()
+
+@ray.remote(max_restarts=-1)
+class Actor:
+    def ping(self):
+        return "hello"
+
+@ray.remote
+class Parent:
+    def generate_actors(self):
+        self.child = Actor.remote()
+        self.detached_actor = Actor.options(name="actor", lifetime="detached").remote()
+        return self.child, self.detached_actor, os.getpid()
+
+parent = Parent.remote()
+actor, detached_actor, pid = ray.get(parent.generate_actors.remote())
+
+os.kill(pid, signal.SIGKILL)
+
+try:
+    print("actor.ping:", ray.get(actor.ping.remote()))
+except ray.exceptions.RayActorError as e:
+    print("Failed to submit actor call", e)
+# Failed to submit actor call The actor died unexpectedly before finishing this task.
+# 	class_name: Actor
+# 	actor_id: 56f541b178ff78470f79c3b601000000
+# 	namespace: ea8b3596-7426-4aa8-98cc-9f77161c4d5f
+# The actor is dead because because all references to the actor were removed.
+
+try:
+    print("detached_actor.ping:", ray.get(detached_actor.ping.remote()))
+except ray.exceptions.RayActorError as e:
+    print("Failed to submit detached actor call", e)
+# detached_actor.ping: hello
+
+# __actor_creator_failure_end__
+# fmt: on

doc/source/ray-core/doc_code/actor_restart.py

@@ -0,0 +1,44 @@
+# flake8: noqa
+
+# fmt: off
+# __actor_restart_begin__
+import os
+import ray
+
+ray.init()
+
+# This actor kills itself after executing 10 tasks.
+@ray.remote(max_restarts=4, max_task_retries=-1)
+class Actor:
+    def __init__(self):
+        self.counter = 0
+
+    def increment_and_possibly_fail(self):
+        # Exit after every 10 tasks.
+        if self.counter == 10:
+            os._exit(0)
+        self.counter += 1
+        return self.counter
+
+actor = Actor.remote()
+
+# The actor will be reconstructed up to 4 times, so we can execute up to 50
+# tasks successfully. The actor is reconstructed by rerunning its constructor.
+# Methods that were executing when the actor died will be retried and will not
+# raise a `RayActorError`. Retried methods may execute twice, once on the
+# failed actor and a second time on the restarted actor.
+for _ in range(50):
+    counter = ray.get(actor.increment_and_possibly_fail.remote())
+    print(counter)  # Prints the sequence 1-10 5 times.
+
+# After the actor has been restarted 4 times, all subsequent methods will
+# raise a `RayActorError`.
+for _ in range(10):
+    try:
+        counter = ray.get(actor.increment_and_possibly_fail.remote())
+        print(counter)  # Unreachable.
+    except ray.exceptions.RayActorError:
+        print("FAILURE")  # Prints 10 times.
+
+# __actor_restart_end__
+# fmt: on

doc/source/ray-core/doc_code/actors.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+
+# __cancel_start__
+import ray
+import asyncio
+import time
+
+
+@ray.remote
+class Actor:
+    async def f(self):
+        try:
+            await asyncio.sleep(5)
+        except asyncio.CancelledError:
+            print("Actor task canceled.")
+
+
+actor = Actor.remote()
+ref = actor.f.remote()
+
+# Wait until task is scheduled.
+time.sleep(1)
+ray.cancel(ref)
+
+try:
+    ray.get(ref)
+except ray.exceptions.RayTaskError:
+    print("Object reference was cancelled.")
+# __cancel_end__
+
+
+# __cancel_graceful_actor_start__
+import ray
+import time
+
+
+@ray.remote
+class SyncActor:
+    def __init__(self):
+        self.is_canceled = False
+
+    def long_running_method(self):
+        """A sync actor method that checks for cancellation periodically."""
+        for i in range(100):
+            # For sync actor tasks, is_canceled() can be checked in the task body
+            if ray.get_runtime_context().is_canceled():
+                self.is_canceled = True
+                print("Actor task canceled, cleaning up...")
+                return "canceled"
+            time.sleep(0.1)
+        return "completed"
+
+    def get_cancel_status(self):
+        return self.is_canceled
+
+
+# Sync actor task cancellation with periodic checking
+actor = SyncActor.remote()
+actor_task_ref = actor.long_running_method.remote()
+
+# Wait until task is scheduled.
+time.sleep(1)
+ray.cancel(actor_task_ref)
+
+# The TaskCancelledError will be raised when calling ray.get
+try:
+    result = ray.get(actor_task_ref)
+except ray.exceptions.TaskCancelledError:
+    print("Actor task was cancelled")
+
+# The get_cancel_status will return True after cancellation
+cancel_status = ray.get(actor.get_cancel_status.remote())
+print(f"Actor detected cancellation: {cancel_status}")
+# __cancel_graceful_actor_end__
+
+
+# __enable_task_events_start__
+@ray.remote
+class FooActor:
+
+    # Disable task events reporting for this method.
+    @ray.method(enable_task_events=False)
+    def foo(self):
+        pass
+
+
+foo_actor = FooActor.remote()
+ray.get(foo_actor.foo.remote())
+
+
+# __enable_task_events_end__

doc/source/ray-core/doc_code/anti_pattern_closure_capture_large_objects.py

@@ -0,0 +1,44 @@
+# __anti_pattern_start__
+import ray
+import numpy as np
+
+ray.init()
+
+large_object = np.zeros(10 * 1024 * 1024)
+
+
+@ray.remote
+def f1():
+    return len(large_object)  # large_object is serialized along with f1!
+
+
+ray.get(f1.remote())
+# __anti_pattern_end__
+
+# __better_approach_1_start__
+large_object_ref = ray.put(np.zeros(10 * 1024 * 1024))
+
+
+@ray.remote
+def f2(large_object):
+    return len(large_object)
+
+
+# Large object is passed through object store.
+ray.get(f2.remote(large_object_ref))
+# __better_approach_1_end__
+
+# __better_approach_2_start__
+large_object_creator = lambda: np.zeros(10 * 1024 * 1024)  # noqa E731
+
+
+@ray.remote
+def f3():
+    large_object = (
+        large_object_creator()
+    )  # Lambda is small compared with the large object.
+    return len(large_object)
+
+
+ray.get(f3.remote())
+# __better_approach_2_end__

doc/source/ray-core/doc_code/anti_pattern_fork_new_processes.py

@@ -0,0 +1,44 @@
+import os
+
+os.environ["RAY_DEDUP_LOGS"] = "0"
+
+import ray
+from concurrent.futures import ProcessPoolExecutor, as_completed
+import multiprocessing
+import numpy as np
+
+
+@ray.remote
+def generate_response(request):
+    print(request)
+    array = np.ones(100000)
+    return array
+
+
+def process_response(response, idx):
+    print(f"Processing response {idx}")
+    return response
+
+
+def main():
+    ray.init()
+    responses = ray.get([generate_response.remote(f"request {i}") for i in range(4)])
+
+    # Better approach: Set the start method to "spawn"
+    multiprocessing.set_start_method("spawn", force=True)
+
+    with ProcessPoolExecutor(max_workers=4) as executor:
+        future_to_task = {}
+        for idx, response in enumerate(responses):
+            future_to_task[executor.submit(process_response, response, idx)] = idx
+
+        for future in as_completed(future_to_task):
+            idx = future_to_task[future]
+            response_entry = future.result()
+            print(f"Response {idx} processed: {response_entry}")
+
+    ray.shutdown()
+
+
+if __name__ == "__main__":
+    main()

doc/source/ray-core/doc_code/anti_pattern_global_variables.py

@@ -0,0 +1,51 @@
+# __anti_pattern_start__
+import ray
+
+ray.init()
+
+global_var = 3
+
+
+@ray.remote
+class Actor:
+    def f(self):
+        return global_var + 3
+
+
+actor = Actor.remote()
+global_var = 4
+# This returns 6, not 7. It is because the value change of global_var
+# inside a driver is not reflected to the actor
+# because they are running in different processes.
+assert ray.get(actor.f.remote()) == 6
+# __anti_pattern_end__
+
+
+# __better_approach_start__
+@ray.remote
+class GlobalVarActor:
+    def __init__(self):
+        self.global_var = 3
+
+    def set_global_var(self, var):
+        self.global_var = var
+
+    def get_global_var(self):
+        return self.global_var
+
+
+@ray.remote
+class Actor:
+    def __init__(self, global_var_actor):
+        self.global_var_actor = global_var_actor
+
+    def f(self):
+        return ray.get(self.global_var_actor.get_global_var.remote()) + 3
+
+
+global_var_actor = GlobalVarActor.remote()
+actor = Actor.remote(global_var_actor)
+ray.get(global_var_actor.set_global_var.remote(4))
+# This returns 7 correctly.
+assert ray.get(actor.f.remote()) == 7
+# __better_approach_end__

doc/source/ray-core/doc_code/anti_pattern_nested_ray_get.py

@@ -0,0 +1,27 @@
+# __anti_pattern_start__
+import ray
+import time
+
+
+@ray.remote
+def f():
+    return 1
+
+
+@ray.remote
+def pass_via_nested_ref(refs):
+    print(sum(ray.get(refs)))
+
+
+@ray.remote
+def pass_via_direct_arg(*args):
+    print(sum(args))
+
+
+# Anti-pattern: Passing nested refs requires `ray.get` in a nested task.
+ray.get(pass_via_nested_ref.remote([f.remote() for _ in range(3)]))
+
+# Better approach: Pass refs as direct arguments. Use *args syntax to unpack
+# multiple arguments.
+ray.get(pass_via_direct_arg.remote(*[f.remote() for _ in range(3)]))
+# __anti_pattern_end__

doc/source/ray-core/doc_code/anti_pattern_out_of_band_object_ref_serialization.py

@@ -0,0 +1,64 @@
+# __anti_pattern_start__
+import ray
+import pickle
+from ray._private.internal_api import memory_summary
+import ray.exceptions
+
+ray.init()
+
+
+@ray.remote
+def out_of_band_serialization_pickle():
+    obj_ref = ray.put(1)
+    import pickle
+
+    # object_ref is serialized from user code using a regular pickle.
+    # Ray can't keep track of the reference, so the underlying object
+    # can be GC'ed unexpectedly, which can cause unexpected hangs.
+    return pickle.dumps(obj_ref)
+
+
+@ray.remote
+def out_of_band_serialization_ray_cloudpickle():
+    obj_ref = ray.put(1)
+    from ray import cloudpickle
+
+    # ray.cloudpickle can serialize only when
+    # RAY_allow_out_of_band_object_ref_serialization=1 env var is set.
+    # However, the object_ref is pinned for the lifetime of the worker,
+    # which can cause Ray object leaks that can cause spilling.
+    return cloudpickle.dumps(obj_ref)
+
+
+print("==== serialize object ref with pickle ====")
+result = ray.get(out_of_band_serialization_pickle.remote())
+try:
+    ray.get(pickle.loads(result), timeout=5)
+except ray.exceptions.GetTimeoutError:
+    print("Underlying object is unexpectedly GC'ed!\n\n")
+
+print("==== serialize object ref with ray.cloudpickle ====")
+# By default, it's allowed to serialize ray.ObjectRef using
+# ray.cloudpickle.
+ray.get(out_of_band_serialization_ray_cloudpickle.options().remote())
+# you can see objects are still pinned although it's GC'ed and not used anymore.
+print(memory_summary())
+
+print(
+    "==== serialize object ref with ray.cloudpickle with env var "
+    "RAY_allow_out_of_band_object_ref_serialization=0 for debugging ===="
+)
+try:
+    ray.get(
+        out_of_band_serialization_ray_cloudpickle.options(
+            runtime_env={
+                "env_vars": {
+                    "RAY_allow_out_of_band_object_ref_serialization": "0",
+                }
+            }
+        ).remote()
+    )
+except Exception as e:
+    print(f"Exception raised from out_of_band_serialization_ray_cloudpickle {e}\n\n")
+
+# __anti_pattern_end__

doc/source/ray-core/doc_code/anti_pattern_pass_large_arg_by_value.py

@@ -0,0 +1,23 @@
+# __anti_pattern_start__
+import ray
+import numpy as np
+
+ray.init()
+
+
+@ray.remote
+def func(large_arg, i):
+    return len(large_arg) + i
+
+
+large_arg = np.zeros(1024 * 1024)
+
+# 10 copies of large_arg are stored in the object store.
+outputs = ray.get([func.remote(large_arg, i) for i in range(10)])
+# __anti_pattern_end__
+
+# __better_approach_start__
+# 1 copy of large_arg is stored in the object store.
+large_arg_ref = ray.put(large_arg)
+outputs = ray.get([func.remote(large_arg_ref, i) for i in range(10)])
+# __better_approach_end__

doc/source/ray-core/doc_code/anti_pattern_ray_get_loop.py

@@ -0,0 +1,25 @@
+# __anti_pattern_start__
+import ray
+
+ray.init()
+
+
+@ray.remote
+def f(i):
+    return i
+
+
+# Anti-pattern: no parallelism due to calling ray.get inside of the loop.
+sequential_returns = []
+for i in range(100):
+    sequential_returns.append(ray.get(f.remote(i)))
+
+# Better approach: parallelism because the tasks are executed in parallel.
+refs = []
+for i in range(100):
+    refs.append(f.remote(i))
+
+parallel_returns = ray.get(refs)
+# __anti_pattern_end__
+
+assert sequential_returns == parallel_returns

doc/source/ray-core/doc_code/anti_pattern_ray_get_submission_order.py

@@ -0,0 +1,36 @@
+# __anti_pattern_start__
+import random
+import time
+import ray
+
+ray.init()
+
+
+@ray.remote
+def f(i):
+    time.sleep(random.random())
+    return i
+
+
+# Anti-pattern: process results in the submission order.
+sum_in_submission_order = 0
+refs = [f.remote(i) for i in range(100)]
+for ref in refs:
+    # Blocks until this ObjectRef is ready.
+    result = ray.get(ref)
+    # process result
+    sum_in_submission_order = sum_in_submission_order + result
+
+# Better approach: process results in the completion order.
+sum_in_completion_order = 0
+refs = [f.remote(i) for i in range(100)]
+unfinished = refs
+while unfinished:
+    # Returns the first ObjectRef that is ready.
+    finished, unfinished = ray.wait(unfinished, num_returns=1)
+    result = ray.get(finished[0])
+    # process result
+    sum_in_completion_order = sum_in_completion_order + result
+# __anti_pattern_end__
+
+assert sum_in_submission_order == sum_in_completion_order

doc/source/ray-core/doc_code/anti_pattern_ray_get_too_many_objects.py

@@ -0,0 +1,37 @@
+# __anti_pattern_start__
+import ray
+import numpy as np
+
+ray.init()
+
+
+def process_results(results):
+    # custom process logic
+    pass
+
+
+@ray.remote
+def return_big_object():
+    return np.zeros(1024 * 10)
+
+
+NUM_TASKS = 1000
+
+object_refs = [return_big_object.remote() for _ in range(NUM_TASKS)]
+# This will fail with heap out-of-memory
+# or object store out-of-space if NUM_TASKS is large enough.
+results = ray.get(object_refs)
+process_results(results)
+# __anti_pattern_end__
+
+# __better_approach_start__
+BATCH_SIZE = 100
+
+while object_refs:
+    # Process results in the finish order instead of the submission order.
+    ready_object_refs, object_refs = ray.wait(object_refs, num_returns=BATCH_SIZE)
+    # The node only needs enough space to store
+    # a batch of objects instead of all objects.
+    results = ray.get(ready_object_refs)
+    process_results(results)
+# __better_approach_end__

doc/source/ray-core/doc_code/anti_pattern_redefine_task_actor_loop.py

@@ -0,0 +1,34 @@
+# __anti_pattern_start__
+import ray
+
+ray.init()
+
+outputs = []
+for i in range(10):
+
+    @ray.remote
+    def double(i):
+        return i * 2
+
+    outputs.append(double.remote(i))
+outputs = ray.get(outputs)
+# The double remote function is pickled and uploaded 10 times.
+# __anti_pattern_end__
+
+assert outputs == [i * 2 for i in range(10)]
+
+
+# __better_approach_start__
+@ray.remote
+def double(i):
+    return i * 2
+
+
+outputs = []
+for i in range(10):
+    outputs.append(double.remote(i))
+outputs = ray.get(outputs)
+# The double remote function is pickled and uploaded 1 time.
+# __better_approach_end__
+
+assert outputs == [i * 2 for i in range(10)]

doc/source/ray-core/doc_code/anti_pattern_return_ray_put.py

@@ -0,0 +1,151 @@
+# __return_single_value_start__
+import ray
+import numpy as np
+
+
+@ray.remote
+def task_with_single_small_return_value_bad():
+    small_return_value = 1
+    # The value will be stored in the object store
+    # and the reference is returned to the caller.
+    small_return_value_ref = ray.put(small_return_value)
+    return small_return_value_ref
+
+
+@ray.remote
+def task_with_single_small_return_value_good():
+    small_return_value = 1
+    # Ray will return the value inline to the caller
+    # which is faster than the previous approach.
+    return small_return_value
+
+
+assert ray.get(ray.get(task_with_single_small_return_value_bad.remote())) == ray.get(
+    task_with_single_small_return_value_good.remote()
+)
+
+
+@ray.remote
+def task_with_single_large_return_value_bad():
+    large_return_value = np.zeros(10 * 1024 * 1024)
+    large_return_value_ref = ray.put(large_return_value)
+    return large_return_value_ref
+
+
+@ray.remote
+def task_with_single_large_return_value_good():
+    # Both approaches will store the large array to the object store
+    # but this is better since it's faster and more fault tolerant.
+    large_return_value = np.zeros(10 * 1024 * 1024)
+    return large_return_value
+
+
+assert np.array_equal(
+    ray.get(ray.get(task_with_single_large_return_value_bad.remote())),
+    ray.get(task_with_single_large_return_value_good.remote()),
+)
+
+
+# Same thing applies for actor tasks as well.
+@ray.remote
+class Actor:
+    def task_with_single_return_value_bad(self):
+        single_return_value = np.zeros(9 * 1024 * 1024)
+        return ray.put(single_return_value)
+
+    def task_with_single_return_value_good(self):
+        return np.zeros(9 * 1024 * 1024)
+
+
+actor = Actor.remote()
+assert np.array_equal(
+    ray.get(ray.get(actor.task_with_single_return_value_bad.remote())),
+    ray.get(actor.task_with_single_return_value_good.remote()),
+)
+# __return_single_value_end__
+
+
+# __return_static_multi_values_start__
+# This will return a single object
+# which is a tuple of two ObjectRefs to the actual values.
+@ray.remote(num_returns=1)
+def task_with_static_multiple_returns_bad1():
+    return_value_1_ref = ray.put(1)
+    return_value_2_ref = ray.put(2)
+    return (return_value_1_ref, return_value_2_ref)
+
+
+# This will return two objects each of which is an ObjectRef to the actual value.
+@ray.remote(num_returns=2)
+def task_with_static_multiple_returns_bad2():
+    return_value_1_ref = ray.put(1)
+    return_value_2_ref = ray.put(2)
+    return (return_value_1_ref, return_value_2_ref)
+
+
+# This will return two objects each of which is the actual value.
+@ray.remote(num_returns=2)
+def task_with_static_multiple_returns_good():
+    return_value_1 = 1
+    return_value_2 = 2
+    return (return_value_1, return_value_2)
+
+
+assert (
+    ray.get(ray.get(task_with_static_multiple_returns_bad1.remote())[0])
+    == ray.get(ray.get(task_with_static_multiple_returns_bad2.remote()[0]))
+    == ray.get(task_with_static_multiple_returns_good.remote()[0])
+)
+
+
+@ray.remote
+class Actor:
+    @ray.method(num_returns=1)
+    def task_with_static_multiple_returns_bad1(self):
+        return_value_1_ref = ray.put(1)
+        return_value_2_ref = ray.put(2)
+        return (return_value_1_ref, return_value_2_ref)
+
+    @ray.method(num_returns=2)
+    def task_with_static_multiple_returns_bad2(self):
+        return_value_1_ref = ray.put(1)
+        return_value_2_ref = ray.put(2)
+        return (return_value_1_ref, return_value_2_ref)
+
+    @ray.method(num_returns=2)
+    def task_with_static_multiple_returns_good(self):
+        # This is faster and more fault tolerant.
+        return_value_1 = 1
+        return_value_2 = 2
+        return (return_value_1, return_value_2)
+
+
+actor = Actor.remote()
+assert (
+    ray.get(ray.get(actor.task_with_static_multiple_returns_bad1.remote())[0])
+    == ray.get(ray.get(actor.task_with_static_multiple_returns_bad2.remote()[0]))
+    == ray.get(actor.task_with_static_multiple_returns_good.remote()[0])
+)
+# __return_static_multi_values_end__
+
+
+# __return_dynamic_multi_values_start__
+@ray.remote(num_returns=1)
+def task_with_dynamic_returns_bad(n):
+    return_value_refs = []
+    for i in range(n):
+        return_value_refs.append(ray.put(np.zeros(i * 1024 * 1024)))
+    return return_value_refs
+
+
+@ray.remote(num_returns="dynamic")
+def task_with_dynamic_returns_good(n):
+    for i in range(n):
+        yield np.zeros(i * 1024 * 1024)
+
+
+assert np.array_equal(
+    ray.get(ray.get(task_with_dynamic_returns_bad.remote(2))[0]),
+    ray.get(next(iter(ray.get(task_with_dynamic_returns_good.remote(2))))),
+)
+# __return_dynamic_multi_values_end__

doc/source/ray-core/doc_code/anti_pattern_too_fine_grained_tasks.py

@@ -0,0 +1,59 @@
+# __anti_pattern_start__
+import ray
+import time
+import itertools
+
+ray.init()
+
+numbers = list(range(10000))
+
+
+def double(number):
+    time.sleep(0.00001)
+    return number * 2
+
+
+start_time = time.time()
+serial_doubled_numbers = [double(number) for number in numbers]
+end_time = time.time()
+print(f"Ordinary function call takes {end_time - start_time} seconds")
+# Ordinary function call takes 0.16506004333496094 seconds
+
+
+@ray.remote
+def remote_double(number):
+    return double(number)
+
+
+start_time = time.time()
+doubled_number_refs = [remote_double.remote(number) for number in numbers]
+parallel_doubled_numbers = ray.get(doubled_number_refs)
+end_time = time.time()
+print(f"Parallelizing tasks takes {end_time - start_time} seconds")
+# Parallelizing tasks takes 1.6061789989471436 seconds
+# __anti_pattern_end__
+
+assert serial_doubled_numbers == parallel_doubled_numbers
+
+
+# __batching_start__
+@ray.remote
+def remote_double_batch(numbers):
+    return [double(number) for number in numbers]
+
+
+BATCH_SIZE = 1000
+start_time = time.time()
+doubled_batch_refs = []
+for i in range(0, len(numbers), BATCH_SIZE):
+    batch = numbers[i : i + BATCH_SIZE]
+    doubled_batch_refs.append(remote_double_batch.remote(batch))
+parallel_doubled_numbers_with_batching = list(
+    itertools.chain(*ray.get(doubled_batch_refs))
+)
+end_time = time.time()
+print(f"Parallelizing tasks with batching takes {end_time - start_time} seconds")
+# Parallelizing tasks with batching takes 0.030150890350341797 seconds
+# __batching_end__
+
+assert serial_doubled_numbers == parallel_doubled_numbers_with_batching

doc/source/ray-core/doc_code/anti_pattern_unnecessary_ray_get.py

@@ -0,0 +1,33 @@
+# __anti_pattern_start__
+import ray
+import numpy as np
+
+ray.init()
+
+
+@ray.remote
+def generate_rollout():
+    return np.ones((10000, 10000))
+
+
+@ray.remote
+def reduce(rollout):
+    return np.sum(rollout)
+
+
+# `ray.get()` downloads the result here.
+rollout = ray.get(generate_rollout.remote())
+# Now we have to reupload `rollout`
+reduced = ray.get(reduce.remote(rollout))
+# __anti_pattern_end__
+
+assert reduced == 100000000
+
+# __better_approach_start__
+# Don't need ray.get here.
+rollout_obj_ref = generate_rollout.remote()
+# Rollout object is passed by reference.
+reduced = ray.get(reduce.remote(rollout_obj_ref))
+# __better_approach_end__
+
+assert reduced == 100000000

doc/source/ray-core/doc_code/cgraph_nccl.py

@@ -0,0 +1,75 @@
+import os
+
+# CI environment is slow, set the timeout to 60 seconds
+os.environ["RAY_CGRAPH_get_timeout"] = "60"
+os.environ["RAY_CGRAPH_submit_timeout"] = "60"
+
+# __cgraph_cpu_to_gpu_actor_start__
+import torch
+import ray
+import ray.dag
+
+
+@ray.remote(num_gpus=1)
+class GPUActor:
+    def process(self, tensor: torch.Tensor):
+        assert tensor.device.type == "cuda"
+        return tensor.shape
+
+
+actor = GPUActor.remote()
+# __cgraph_cpu_to_gpu_actor_end__
+
+# __cgraph_cpu_to_gpu_start__
+with ray.dag.InputNode() as inp:
+    inp = inp.with_tensor_transport(device="cuda")
+    dag = actor.process.bind(inp)
+
+cdag = dag.experimental_compile()
+print(ray.get(cdag.execute(torch.zeros(10))))
+# __cgraph_cpu_to_gpu_end__
+
+# __cgraph_nccl_setup_start__
+import torch
+import ray
+import ray.dag
+from ray.experimental.channel.torch_tensor_type import TorchTensorType
+
+
+# Note that the following example requires at least 2 GPUs.
+assert (
+    ray.available_resources().get("GPU") >= 2
+), "At least 2 GPUs are required to run this example."
+
+
+@ray.remote(num_gpus=1)
+class GPUSender:
+    def send(self, shape):
+        return torch.zeros(shape, device="cuda")
+
+
+@ray.remote(num_gpus=1)
+class GPUReceiver:
+    def recv(self, tensor: torch.Tensor):
+        assert tensor.device.type == "cuda"
+        return tensor.shape
+
+
+sender = GPUSender.remote()
+receiver = GPUReceiver.remote()
+# __cgraph_nccl_setup_end__
+
+# __cgraph_nccl_exec_start__
+with ray.dag.InputNode() as inp:
+    dag = sender.send.bind(inp)
+    # Add a type hint that the return value of `send` should use NCCL.
+    dag = dag.with_tensor_transport("nccl")
+    # NOTE: With ray<2.42, use `with_type_hint()` instead.
+    # dag = dag.with_type_hint(TorchTensorType(transport="nccl"))
+    dag = receiver.recv.bind(dag)
+
+# Compile API prepares the NCCL communicator across all workers and schedule operations
+# accordingly.
+dag = dag.experimental_compile()
+assert ray.get(dag.execute((10,))) == (10,)
+# __cgraph_nccl_exec_end__

doc/source/ray-core/doc_code/cgraph_overlap.py

@@ -0,0 +1,70 @@
+# __cgraph_overlap_start__
+import ray
+import time
+import torch
+from ray.dag import InputNode, MultiOutputNode
+
+
+@ray.remote(num_cpus=0, num_gpus=1)
+class TorchTensorWorker:
+    def send(self, shape, dtype, value: int, send_tensor=True):
+        if not send_tensor:
+            return 1
+        return torch.ones(shape, dtype=dtype, device="cuda") * value
+
+    def recv_and_matmul(self, two_d_tensor):
+        """
+        Receive the tensor and do some expensive computation (matmul).
+
+        Args:
+            two_d_tensor: a 2D tensor that has the same size for its dimensions
+        """
+        # Check that tensor got loaded to the correct device.
+        assert two_d_tensor.dim() == 2
+        assert two_d_tensor.size(0) == two_d_tensor.size(1)
+        torch.matmul(two_d_tensor, two_d_tensor)
+        return (two_d_tensor[0][0].item(), two_d_tensor.shape, two_d_tensor.dtype)
+
+
+def test(overlap_gpu_communication):
+    num_senders = 3
+    senders = [TorchTensorWorker.remote() for _ in range(num_senders)]
+    receiver = TorchTensorWorker.remote()
+
+    shape = (10000, 10000)
+    dtype = torch.float16
+
+    with InputNode() as inp:
+        branches = [sender.send.bind(shape, dtype, inp) for sender in senders]
+        branches = [
+            branch.with_tensor_transport(
+                transport="nccl", _static_shape=True, _direct_return=True
+            )
+            # For a ray version before 2.42, use `with_type_hint()` instead.
+            # branch.with_type_hint(
+            #     TorchTensorType(
+            #         transport="nccl", _static_shape=True, _direct_return=True
+            #     )
+            # )
+            for branch in branches
+        ]
+        branches = [receiver.recv_and_matmul.bind(branch) for branch in branches]
+        dag = MultiOutputNode(branches)
+
+    compiled_dag = dag.experimental_compile(
+        _overlap_gpu_communication=overlap_gpu_communication
+    )
+
+    start = time.monotonic()
+    for i in range(5):
+        ref = compiled_dag.execute(i)
+        result = ray.get(ref)
+        assert result == [(i, shape, dtype)] * num_senders
+    duration = time.monotonic() - start
+    print(f"{overlap_gpu_communication=}, {duration=}")
+    compiled_dag.teardown(kill_actors=True)
+
+
+for overlap_gpu_communication in [False, True]:
+    test(overlap_gpu_communication)
+# __cgraph_overlap_end__

doc/source/ray-core/doc_code/cgraph_profiling.py

@@ -0,0 +1,63 @@
+import os
+import subprocess
+from pathlib import Path
+
+
+NSIGHT_FAKE_DIR = str(
+    Path(os.path.realpath(__file__)).parents[4]
+    / "python"
+    / "ray"
+    / "tests"
+    / "nsight_fake"
+)
+
+
+subprocess.check_call(
+    ["pip", "install", NSIGHT_FAKE_DIR],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+)
+
+
+# __profiling_setup_start__
+import ray
+import torch
+from ray.dag import InputNode
+
+
+@ray.remote(num_gpus=1, runtime_env={"nsight": "default"})
+class RayActor:
+    def send(self, shape, dtype, value: int):
+        return torch.ones(shape, dtype=dtype, device="cuda") * value
+
+    def recv(self, tensor):
+        return (tensor[0].item(), tensor.shape, tensor.dtype)
+
+
+sender = RayActor.remote()
+receiver = RayActor.remote()
+# __profiling_setup_end__
+
+# __profiling_execution_start__
+shape = (10,)
+dtype = torch.float16
+
+# Test normal execution.
+with InputNode() as inp:
+    dag = sender.send.bind(inp.shape, inp.dtype, inp[0])
+    dag = dag.with_tensor_transport(transport="nccl")
+    dag = receiver.recv.bind(dag)
+
+compiled_dag = dag.experimental_compile()
+
+for i in range(3):
+    shape = (10 * (i + 1),)
+    ref = compiled_dag.execute(i, shape=shape, dtype=dtype)
+    assert ray.get(ref) == (i, shape, dtype)
+# __profiling_execution_end__
+
+subprocess.check_call(
+    ["pip", "uninstall", "nsys", "--y"],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+)

doc/source/ray-core/doc_code/cgraph_quickstart.py

@@ -0,0 +1,174 @@
+# __simple_actor_start__
+import ray
+
+
+@ray.remote
+class SimpleActor:
+    def echo(self, msg):
+        return msg
+
+
+# __simple_actor_end__
+
+# __ray_core_usage_start__
+import time
+
+a = SimpleActor.remote()
+
+# warmup
+for _ in range(5):
+    msg_ref = a.echo.remote("hello")
+    ray.get(msg_ref)
+
+start = time.perf_counter()
+msg_ref = a.echo.remote("hello")
+ray.get(msg_ref)
+end = time.perf_counter()
+print(f"Execution takes {(end - start) * 1000 * 1000} us")
+# __ray_core_usage_end__
+
+# __dag_usage_start__
+import ray.dag
+
+with ray.dag.InputNode() as inp:
+    # Note that it uses `bind` instead of `remote`.
+    # This returns a ray.dag.DAGNode, instead of the usual ray.ObjectRef.
+    dag = a.echo.bind(inp)
+
+# warmup
+for _ in range(5):
+    msg_ref = dag.execute("hello")
+    ray.get(msg_ref)
+
+start = time.perf_counter()
+# `dag.execute` runs the DAG and returns an ObjectRef. You can use `ray.get` API.
+msg_ref = dag.execute("hello")
+ray.get(msg_ref)
+end = time.perf_counter()
+print(f"Execution takes {(end - start) * 1000 * 1000} us")
+# __dag_usage_end__
+
+# __cgraph_usage_start__
+dag = dag.experimental_compile()
+
+# warmup
+for _ in range(5):
+    msg_ref = dag.execute("hello")
+    ray.get(msg_ref)
+
+start = time.perf_counter()
+# `dag.execute` runs the DAG and returns CompiledDAGRef. Similar to
+# ObjectRefs, you can use the ray.get API.
+msg_ref = dag.execute("hello")
+ray.get(msg_ref)
+end = time.perf_counter()
+print(f"Execution takes {(end - start) * 1000 * 1000} us")
+# __cgraph_usage_end__
+
+# __teardown_start__
+dag.teardown()
+# __teardown_end__
+
+# __cgraph_bind_start__
+a = SimpleActor.remote()
+b = SimpleActor.remote()
+
+with ray.dag.InputNode() as inp:
+    # Note that it uses `bind` instead of `remote`.
+    # This returns a ray.dag.DAGNode, instead of the usual ray.ObjectRef.
+    dag = a.echo.bind(inp)
+    dag = b.echo.bind(dag)
+
+dag = dag.experimental_compile()
+print(ray.get(dag.execute("hello")))
+# __cgraph_bind_end__
+
+dag.teardown()
+
+# __cgraph_multi_output_start__
+import ray.dag
+
+a = SimpleActor.remote()
+b = SimpleActor.remote()
+
+with ray.dag.InputNode() as inp:
+    # Note that it uses `bind` instead of `remote`.
+    # This returns a ray.dag.DAGNode, instead of the usual ray.ObjectRef.
+    dag = ray.dag.MultiOutputNode([a.echo.bind(inp), b.echo.bind(inp)])
+
+dag = dag.experimental_compile()
+print(ray.get(dag.execute("hello")))
+# __cgraph_multi_output_end__
+
+dag.teardown()
+
+# __cgraph_async_compile_start__
+import ray
+
+
+@ray.remote
+class EchoActor:
+    def echo(self, msg):
+        return msg
+
+
+actor = EchoActor.remote()
+with ray.dag.InputNode() as inp:
+    dag = actor.echo.bind(inp)
+
+cdag = dag.experimental_compile(enable_asyncio=True)
+# __cgraph_async_compile_end__
+
+# __cgraph_async_execute_start__
+import asyncio
+
+
+async def async_method(i):
+    fut = await cdag.execute_async(i)
+    result = await fut
+    assert result == i
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(async_method(42))
+
+# __cgraph_async_execute_end__
+
+cdag.teardown()
+
+# __cgraph_actor_death_start__
+from ray.dag import InputNode, MultiOutputNode
+
+
+@ray.remote
+class EchoActor:
+    def echo(self, msg):
+        return msg
+
+
+actors = [EchoActor.remote() for _ in range(4)]
+with InputNode() as inp:
+    outputs = [actor.echo.bind(inp) for actor in actors]
+    dag = MultiOutputNode(outputs)
+
+compiled_dag = dag.experimental_compile()
+# Kill one of the actors to simulate unexpected actor death.
+ray.kill(actors[0])
+ref = compiled_dag.execute(1)
+
+live_actors = []
+try:
+    ray.get(ref)
+except ray.exceptions.ActorDiedError:
+    # At this point, the Compiled Graph is shutting down.
+    for actor in actors:
+        try:
+            # Check for live actors.
+            ray.get(actor.echo.remote("ping"))
+            live_actors.append(actor)
+        except ray.exceptions.RayActorError:
+            pass
+
+# Optionally, use the live actors to create a new Compiled Graph.
+assert live_actors == actors[1:]
+# __cgraph_actor_death_end__

doc/source/ray-core/doc_code/cgraph_troubleshooting.py

@@ -0,0 +1,43 @@
+# __numpy_troubleshooting_start__
+import ray
+import numpy as np
+from ray.dag import InputNode
+
+
+@ray.remote
+class NumPyActor:
+    def get_arr(self, _):
+        numpy_arr = np.ones((5, 1024))
+        return numpy_arr
+
+
+actor = NumPyActor.remote()
+with InputNode() as inp:
+    dag = actor.get_arr.bind(inp)
+cgraph = dag.experimental_compile()
+for _ in range(5):
+    ref = cgraph.execute(0)
+    result = ray.get(ref)
+    # Adding this explicit del would fix any issues
+    # del result
+# __numpy_troubleshooting_end__
+
+# __teardown_troubleshooting_start__
+@ray.remote
+class SendActor:
+    def send(self, x):
+        return x
+
+
+actor = SendActor.remote()
+with InputNode() as inp:
+    dag = actor.send.bind(inp)
+cgraph = dag.experimental_compile()
+
+# Not adding this explicit teardown before reusing `actor` could cause problems
+# cgraph.teardown()
+
+with InputNode() as inp:
+    dag = actor.send.bind(inp)
+cgraph = dag.experimental_compile()
+# __teardown_troubleshooting_end__

doc/source/ray-core/doc_code/cgraph_visualize.py

@@ -0,0 +1,31 @@
+# __cgraph_visualize_start__
+import ray
+from ray.dag import InputNode, MultiOutputNode
+
+
+@ray.remote
+class Worker:
+    def inc(self, x):
+        return x + 1
+
+    def double(self, x):
+        return x * 2
+
+    def echo(self, x):
+        return x
+
+
+sender1 = Worker.remote()
+sender2 = Worker.remote()
+receiver = Worker.remote()
+
+with InputNode() as inp:
+    w1 = sender1.inc.bind(inp)
+    w1 = receiver.echo.bind(w1)
+    w2 = sender2.double.bind(inp)
+    w2 = receiver.echo.bind(w2)
+    dag = MultiOutputNode([w1, w2])
+
+compiled_dag = dag.experimental_compile()
+compiled_dag.visualize()
+# __cgraph_visualize_end__

doc/source/ray-core/doc_code/cross_language.py

@@ -0,0 +1,102 @@
+# flake8: noqa
+
+# fmt: off
+# __crosslang_init_start__
+import ray
+
+ray.init(job_config=ray.job_config.JobConfig(code_search_path=["/path/to/code"]))
+# __crosslang_init_end__
+# fmt: on
+
+# fmt: off
+# __crosslang_multidir_start__
+import ray
+
+ray.init(job_config=ray.job_config.JobConfig(code_search_path="/path/to/jars:/path/to/pys"))
+# __crosslang_multidir_end__
+# fmt: on
+
+# fmt: off
+# __python_call_java_start__
+import ray
+
+with ray.init(job_config=ray.job_config.JobConfig(code_search_path=["/path/to/code"])):
+  # Define a Java class.
+  counter_class = ray.cross_language.java_actor_class(
+        "io.ray.demo.Counter")
+
+  # Create a Java actor and call actor method.
+  counter = counter_class.remote()
+  obj_ref1 = counter.increment.remote()
+  assert ray.get(obj_ref1) == 1
+  obj_ref2 = counter.increment.remote()
+  assert ray.get(obj_ref2) == 2
+
+  # Define a Java function.
+  add_function = ray.cross_language.java_function(
+        "io.ray.demo.Math", "add")
+
+  # Call the Java remote function.
+  obj_ref3 = add_function.remote(1, 2)
+  assert ray.get(obj_ref3) == 3
+# __python_call_java_end__
+# fmt: on
+
+# fmt: off
+# __python_module_start__
+# /path/to/the_dir/ray_demo.py
+
+import ray
+
+@ray.remote
+class Counter(object):
+  def __init__(self):
+      self.value = 0
+
+  def increment(self):
+      self.value += 1
+      return self.value
+
+@ray.remote
+def add(a, b):
+    return a + b
+# __python_module_end__
+# fmt: on
+
+# fmt: off
+# __serialization_start__
+# ray_serialization.py
+
+import ray
+
+@ray.remote
+def py_return_input(v):
+    return v
+# __serialization_end__
+# fmt: on
+
+# fmt: off
+# __raise_exception_start__
+# ray_exception.py
+
+import ray
+
+@ray.remote
+def raise_exception():
+    1 / 0
+# __raise_exception_end__
+# fmt: on
+
+# fmt: off
+# __raise_exception_demo_start__
+# ray_exception_demo.py
+
+import ray
+
+with ray.init(job_config=ray.job_config.JobConfig(code_search_path=["/path/to/ray_exception"])):
+  obj_ref = ray.cross_language.java_function(
+        "io.ray.demo.MyRayClass",
+        "raiseExceptionFromPython").remote()
+  ray.get(obj_ref)  # <-- raise exception from here.
+# __raise_exception_demo_end__
+# fmt: on

doc/source/ray-core/doc_code/deser.py

@@ -0,0 +1,18 @@
+import ray
+import numpy as np
+
+
+@ray.remote
+def f(arr):
+    # arr = arr.copy()  # Adding a copy will fix the error.
+    arr[0] = 1
+
+
+try:
+    ray.get(f.remote(np.zeros(100)))
+except ray.exceptions.RayTaskError as e:
+    print(e)
+# ray.exceptions.RayTaskError(ValueError): ray::f()
+#   File "test.py", line 6, in f
+#     arr[0] = 1
+# ValueError: assignment destination is read-only

doc/source/ray-core/doc_code/direct_transport_custom.py

@@ -0,0 +1,169 @@
+# flake8: noqa
+
+# __custom_metadata_start__
+import multiprocessing.shared_memory as shm
+import pickle
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import numpy
+
+import ray
+from ray.experimental import (
+    CommunicatorMetadata,
+    TensorTransportManager,
+    TensorTransportMetadata,
+    register_tensor_transport,
+)
+
+
+@dataclass
+class ShmTransportMetadata(TensorTransportMetadata):
+    """Custom metadata that stores the shared memory name and size."""
+
+    shm_name: Optional[str] = None
+    shm_size: Optional[int] = None
+
+
+@dataclass
+class ShmCommunicatorMetadata(CommunicatorMetadata):
+    """No extra communicator metadata needed for shared memory."""
+
+    pass
+
+
+# __custom_metadata_end__
+
+
+# __custom_properties_start__
+class SharedMemoryTransport(TensorTransportManager):
+    """A one-sided tensor transport that transfers numpy arrays through shared memory."""
+
+    def __init__(self):
+        self.shared_memory_objects: Dict[str, shm.SharedMemory] = {}
+
+    def tensor_transport_backend(self) -> str:
+        return "shared_memory"
+
+    @staticmethod
+    def is_one_sided() -> bool:
+        return True
+
+    @staticmethod
+    def can_abort_transport() -> bool:
+        return False
+
+    def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool:
+        return True
+
+    # __custom_properties_end__
+
+    # __custom_extract_start__
+    def extract_tensor_transport_metadata(
+        self,
+        obj_id: str,
+        rdt_object: List[numpy.ndarray],
+    ) -> TensorTransportMetadata:
+        # Record shapes and dtypes.
+        tensor_meta = []
+        if rdt_object:
+            for tensor in rdt_object:
+                tensor_meta.append((tensor.shape, tensor.dtype))
+
+        # Serialize the tensors and store them in shared memory.
+        serialized_rdt_object = pickle.dumps(rdt_object)
+        size = len(serialized_rdt_object)
+        name = obj_id[:20]
+        shm_obj = shm.SharedMemory(name=name, create=True, size=size)
+        shm_obj.buf[:size] = serialized_rdt_object
+        self.shared_memory_objects[obj_id] = shm_obj
+
+        return ShmTransportMetadata(
+            tensor_meta=tensor_meta, tensor_device="cpu", shm_name=name, shm_size=size
+        )
+
+    # __custom_extract_end__
+
+    # __custom_communicator_start__
+    def get_communicator_metadata(
+        self,
+        src_actor: "ray.actor.ActorHandle",
+        dst_actor: "ray.actor.ActorHandle",
+        backend: Optional[str] = None,
+    ) -> CommunicatorMetadata:
+        return ShmCommunicatorMetadata()
+
+    # __custom_communicator_end__
+
+    # __custom_send_recv_start__
+    def recv_multiple_tensors(
+        self,
+        obj_id: str,
+        tensor_transport_metadata: TensorTransportMetadata,
+        communicator_metadata: CommunicatorMetadata,
+        target_buffers: Optional[List[Any]] = None,
+    ):
+        # Open the shared memory block and deserialize.
+        shm_name = tensor_transport_metadata.shm_name
+        size = tensor_transport_metadata.shm_size
+        shm_block = shm.SharedMemory(name=shm_name)
+        recv_tensors = pickle.loads(shm_block.buf[:size])
+        shm_block.close()
+        return recv_tensors
+
+    def send_multiple_tensors(
+        self,
+        tensors: List[numpy.ndarray],
+        tensor_transport_metadata: TensorTransportMetadata,
+        communicator_metadata: CommunicatorMetadata,
+    ):
+        raise NotImplementedError("One-sided transport doesn't use send.")
+
+    # __custom_send_recv_end__
+
+    # __custom_cleanup_start__
+    def garbage_collect(
+        self,
+        obj_id: str,
+        tensor_transport_meta: TensorTransportMetadata,
+        tensors: List[numpy.ndarray],
+    ):
+        self.shared_memory_objects[obj_id].close()
+        self.shared_memory_objects[obj_id].unlink()
+        del self.shared_memory_objects[obj_id]
+
+    def abort_transport(
+        self,
+        obj_id: str,
+        communicator_metadata: CommunicatorMetadata,
+    ):
+        pass
+
+    # __custom_cleanup_end__
+
+
+# __custom_usage_start__
+register_tensor_transport(
+    "shared_memory",        # Transport name
+    ["cpu"],                # Supported device types
+    SharedMemoryTransport,  # TensorTransportManager class
+    numpy.ndarray,          # Data type for this transport
+)
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="shared_memory")
+    def echo(self, data):
+        return data
+
+    def sum(self, data):
+        return data.sum().item()
+
+
+actors = [MyActor.remote() for _ in range(2)]
+ref = actors[0].echo.remote(numpy.array([1, 2, 3]))
+result = actors[1].sum.remote(ref)
+print(ray.get(result))
+# 6
+# __custom_usage_end__

doc/source/ray-core/doc_code/direct_transport_gloo.py

@@ -0,0 +1,277 @@
+# flake8: noqa
+
+# __normal_example_start__
+import torch
+import ray
+
+
+@ray.remote
+class MyActor:
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+
+# __normal_example_end__
+
+# __gloo_example_start__
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+
+# __gloo_example_end__
+
+# __gloo_group_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+    def sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+# The tensor_transport specified here must match the one used in the @ray.method
+# decorator.
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+# __gloo_group_end__
+
+# __gloo_group_destroy_start__
+from ray.experimental.collective import destroy_collective_group
+
+destroy_collective_group(group)
+# __gloo_group_destroy_end__
+
+# __gloo_full_example_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+    def sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+# The tensor will be stored by the `sender` actor instead of in Ray's object
+# store.
+tensor = sender.random_tensor.remote()
+result = receiver.sum.remote(tensor)
+print(ray.get(result))
+# __gloo_full_example_end__
+
+# __gloo_multiple_tensors_example_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor_dict(self):
+        return {"tensor1": torch.randn(1000, 1000), "tensor2": torch.randn(1000, 1000)}
+
+    def sum(self, tensor_dict: dict):
+        return torch.sum(tensor_dict["tensor1"]) + torch.sum(tensor_dict["tensor2"])
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+# Both tensor values in the dictionary will be stored by the `sender` actor
+# instead of in Ray's object store.
+tensor_dict = sender.random_tensor_dict.remote()
+result = receiver.sum.remote(tensor_dict)
+print(ray.get(result))
+# __gloo_multiple_tensors_example_end__
+
+# __gloo_intra_actor_start__
+import torch
+import ray
+import pytest
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+    def sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+tensor = sender.random_tensor.remote()
+# Pass the ObjectRef back to the actor that produced it. The tensor will be
+# passed back to the same actor without copying.
+sum1 = sender.sum.remote(tensor)
+sum2 = receiver.sum.remote(tensor)
+assert torch.allclose(*ray.get([sum1, sum2]))
+# __gloo_intra_actor_end__
+
+# __gloo_get_start__
+
+# Wrong example of ray.get(). Since the tensor transport in the @ray.method decorator is Gloo,
+# ray.get() will try to use Gloo to fetch the tensor, which is not supported
+# because the caller is not part of the collective group.
+with pytest.raises(ValueError) as e:
+    ray.get(tensor)
+
+assert (
+    "ray.get is not allowed on RDT objects using the two-sided transport"
+    in str(e.value)
+)
+
+# Correct example of ray.get(), using the object store to fetch the RDT object because the caller
+# is not part of the collective group.
+print(ray.get(tensor, _use_object_store=True))
+# torch.Tensor(...)
+# __gloo_get_end__
+
+
+# __gloo_object_mutability_warning_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        return torch.randn(1000, 1000)
+
+    def increment_and_sum(self, tensor: torch.Tensor):
+        # In-place update.
+        tensor += 1
+        return torch.sum(tensor)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+tensor = sender.random_tensor.remote()
+tensor1 = sender.increment_and_sum.remote(tensor)
+tensor2 = receiver.increment_and_sum.remote(tensor)
+# A warning will be printed:
+# UserWarning: GPU ObjectRef(...) is being passed back to the actor that created it Actor(MyActor, ...). Note that GPU objects are mutable. If the tensor is modified, Ray's internal copy will also be updated, and subsequent passes to other actors will receive the updated version instead of the original.
+
+try:
+    # This assertion may fail because the tensor returned by sender.random_tensor
+    # is modified in-place by sender.increment_and_sum while being sent to
+    # receiver.increment_and_sum.
+    assert torch.allclose(ray.get(tensor1), ray.get(tensor2))
+except AssertionError:
+    print("AssertionError: sender and receiver returned different sums.")
+
+# __gloo_object_mutability_warning_end__
+
+# __gloo_wait_tensor_freed_bad_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        self.tensor = torch.randn(1000, 1000)
+        # After this function returns, Ray and this actor will both hold a
+        # reference to the same tensor.
+        return self.tensor
+
+    def increment_and_sum_stored_tensor(self):
+        # NOTE: In-place update, while Ray still holds a reference to the same tensor.
+        self.tensor += 1
+        return torch.sum(self.tensor)
+
+    def increment_and_sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor + 1)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+tensor = sender.random_tensor.remote()
+tensor1 = sender.increment_and_sum_stored_tensor.remote()
+# Wait for sender.increment_and_sum_stored_tensor task to finish.
+tensor1 = ray.get(tensor1)
+# Receiver will now receive the updated value instead of the original.
+tensor2 = receiver.increment_and_sum.remote(tensor)
+
+try:
+    # This assertion will fail because sender.increment_and_sum_stored_tensor
+    # modified the tensor in place before sending it to
+    # receiver.increment_and_sum.
+    assert torch.allclose(tensor1, ray.get(tensor2))
+except AssertionError:
+    print("AssertionError: sender and receiver returned different sums.")
+# __gloo_wait_tensor_freed_bad_end__
+
+# __gloo_wait_tensor_freed_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote
+class MyActor:
+    @ray.method(tensor_transport="gloo")
+    def random_tensor(self):
+        self.tensor = torch.randn(1000, 1000)
+        return self.tensor
+
+    def increment_and_sum_stored_tensor(self):
+        # 1. Sender actor waits for Ray to release all references to the tensor
+        # before modifying the tensor in place.
+        ray.experimental.wait_tensor_freed(self.tensor)
+        # NOTE: In-place update, but Ray guarantees that it has already released
+        # its references to this tensor.
+        self.tensor += 1
+        return torch.sum(self.tensor)
+
+    def increment_and_sum(self, tensor: torch.Tensor):
+        # Receiver task remains the same.
+        return torch.sum(tensor + 1)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="torch_gloo")
+
+tensor = sender.random_tensor.remote()
+tensor1 = sender.increment_and_sum_stored_tensor.remote()
+# 2. Skip `ray.get`` because `wait_tensor_freed`` will block until all
+# references to `tensor` are freed, so calling `ray.get` here would cause a
+# deadlock.
+# tensor1 = ray.get(tensor1)
+tensor2 = receiver.increment_and_sum.remote(tensor)
+
+# 3. Delete all references to `tensor`, to unblock wait_tensor_freed.
+del tensor
+
+# This assertion will now pass.
+assert torch.allclose(ray.get(tensor1), ray.get(tensor2))
+# __gloo_wait_tensor_freed_end__

doc/source/ray-core/doc_code/direct_transport_nccl.py

@@ -0,0 +1,27 @@
+# flake8: noqa
+
+# __nccl_full_example_start__
+import torch
+import ray
+from ray.experimental.collective import create_collective_group
+
+
+@ray.remote(num_gpus=1)
+class MyActor:
+    @ray.method(tensor_transport="nccl")
+    def random_tensor(self):
+        return torch.randn(1000, 1000).cuda()
+
+    def sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor)
+
+
+sender, receiver = MyActor.remote(), MyActor.remote()
+group = create_collective_group([sender, receiver], backend="nccl")
+
+# The tensor will be stored by the `sender` actor instead of in Ray's object
+# store.
+tensor = sender.random_tensor.remote()
+result = receiver.sum.remote(tensor)
+ray.get(result)
+# __nccl_full_example_end__

doc/source/ray-core/doc_code/direct_transport_nixl.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+
+# __nixl_full_example_start__
+import torch
+import ray
+
+
+@ray.remote(num_gpus=1)
+class MyActor:
+    @ray.method(tensor_transport="nixl")
+    def random_tensor(self):
+        return torch.randn(1000, 1000).cuda()
+
+    def sum(self, tensor: torch.Tensor):
+        return torch.sum(tensor)
+
+    def produce(self, tensors):
+        refs = []
+        for t in tensors:
+            refs.append(ray.put(t, _tensor_transport="nixl"))
+        return refs
+
+    def consume_with_nixl(self, refs):
+        # ray.get will also use NIXL to retrieve the
+        # result.
+        tensors = [ray.get(ref) for ref in refs]
+        sum = 0
+        for t in tensors:
+            assert t.device.type == "cuda"
+            sum += t.sum().item()
+        return sum
+
+
+# No collective group is needed. The two actors just need to have NIXL
+# installed.
+sender, receiver = MyActor.remote(), MyActor.remote()
+
+# The tensor will be stored by the `sender` actor instead of in Ray's object
+# store.
+tensor = sender.random_tensor.remote()
+result = receiver.sum.remote(tensor)
+ray.get(result)
+# __nixl_full_example_end__
+
+# __nixl_get_start__
+# ray.get will also use NIXL to retrieve the
+# result.
+print(ray.get(tensor))
+# torch.Tensor(...)
+# __nixl_get_end__
+
+# __nixl_put__and_get_start__
+tensor1 = torch.randn(1000, 1000).cuda()
+tensor2 = torch.randn(1000, 1000).cuda()
+refs = sender.produce.remote([tensor1, tensor2])
+ref1 = receiver.consume_with_nixl.remote(refs)
+print(ray.get(ref1))
+# __nixl_put__and_get_end__
+
+
+# __nixl_limitations_start__
+@ray.remote(num_gpus=1)
+class Actor:
+    def __init__(self):
+        self.tensor1 = torch.tensor([1, 2, 3])
+        self.tensor2 = torch.tensor([4, 5, 6])
+        self.tensor3 = torch.tensor([7, 8, 9])
+
+    @ray.method(tensor_transport="nixl")
+    def send_dict1(self):
+        return {"round1-1": self.tensor1, "round1-2": self.tensor2}
+
+    @ray.method(tensor_transport="nixl")
+    def send_dict2(self):
+        return {"round2-1": self.tensor1, "round2-3": self.tensor3}
+
+    def sum_dict(self, dict):
+        return sum(v.sum().item() for v in dict.values())
+
+
+sender, receiver = Actor.remote(), Actor.remote()
+ref1 = sender.send_dict1.remote()
+result1 = receiver.sum_dict.remote(ref1)
+print(ray.get(result1))
+ref2 = sender.send_dict2.remote()
+result2 = receiver.sum_dict.remote(ref2)
+try:
+    print(ray.get(result2))
+except ValueError as e:
+    print("Error caught:", e)
+# __nixl_limitations_end__

doc/source/ray-core/doc_code/fault_tolerance_tips.py

@@ -0,0 +1,88 @@
+# __return_ray_put_start__
+import ray
+
+
+# Non-fault tolerant version:
+@ray.remote
+def a():
+    x_ref = ray.put(1)
+    return x_ref
+
+
+x_ref = ray.get(a.remote())
+# Object x outlives its owner task A.
+try:
+    # If owner of x (i.e. the worker process running task A) dies,
+    # the application can no longer get value of x.
+    print(ray.get(x_ref))
+except ray.exceptions.OwnerDiedError:
+    pass
+# __return_ray_put_end__
+
+
+# __return_directly_start__
+# Fault tolerant version:
+@ray.remote
+def a():
+    # Here we return the value directly instead of calling ray.put() first.
+    return 1
+
+
+# The owner of x is the driver
+# so x is accessible and can be auto recovered
+# during the entire lifetime of the driver.
+x_ref = a.remote()
+print(ray.get(x_ref))
+# __return_directly_end__
+
+
+# __node_ip_resource_start__
+@ray.remote
+def b():
+    return 1
+
+
+# If the node with ip 127.0.0.3 fails while task b is running,
+# Ray cannot retry the task on other nodes.
+b.options(resources={"node:127.0.0.3": 1}).remote()
+# __node_ip_resource_end__
+
+# __node_affinity_scheduling_strategy_start__
+# Prefer running on the particular node specified by node id
+# but can also run on other nodes if the target node fails.
+b.options(
+    scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+        node_id=ray.get_runtime_context().get_node_id(), soft=True
+    )
+).remote()
+# __node_affinity_scheduling_strategy_end__
+
+
+# __manual_retry_start__
+@ray.remote
+class Actor:
+    def read_only(self):
+        import sys
+        import random
+
+        rand = random.random()
+        if rand < 0.2:
+            return 2 / 0
+        elif rand < 0.3:
+            sys.exit(1)
+
+        return 2
+
+
+actor = Actor.remote()
+# Manually retry the actor task.
+while True:
+    try:
+        print(ray.get(actor.read_only.remote()))
+        break
+    except ZeroDivisionError:
+        pass
+    except ray.exceptions.RayActorError:
+        # Manually restart the actor
+        actor = Actor.remote()
+# __manual_retry_end__

doc/source/ray-core/doc_code/generator.py

@@ -0,0 +1,120 @@
+# __program_start__
+import ray
+from ray import DynamicObjectRefGenerator
+
+# fmt: off
+# __dynamic_generator_start__
+import numpy as np
+
+
+@ray.remote(num_returns="dynamic")
+def split(array, chunk_size):
+    while len(array) > 0:
+        yield array[:chunk_size]
+        array = array[chunk_size:]
+
+
+array_ref = ray.put(np.zeros(np.random.randint(1000_000)))
+block_size = 1000
+
+# Returns an ObjectRef[DynamicObjectRefGenerator].
+dynamic_ref = split.remote(array_ref, block_size)
+print(dynamic_ref)
+# ObjectRef(c8ef45ccd0112571ffffffffffffffffffffffff0100000001000000)
+
+i = -1
+ref_generator = ray.get(dynamic_ref)
+print(ref_generator)
+# <ray._raylet.DynamicObjectRefGenerator object at 0x7f7e2116b290>
+for i, ref in enumerate(ref_generator):
+    # Each DynamicObjectRefGenerator iteration returns an ObjectRef.
+    assert len(ray.get(ref)) <= block_size
+num_blocks_generated = i + 1
+array_size = len(ray.get(array_ref))
+assert array_size <= num_blocks_generated * block_size
+print(f"Split array of size {array_size} into {num_blocks_generated} blocks of "
+      f"size {block_size} each.")
+# Split array of size 63153 into 64 blocks of size 1000 each.
+
+# NOTE: The dynamic_ref points to the generated ObjectRefs. Make sure that this
+# ObjectRef goes out of scope so that Ray can garbage-collect the internal
+# ObjectRefs.
+del dynamic_ref
+# __dynamic_generator_end__
+# fmt: on
+
+
+# fmt: off
+# __dynamic_generator_pass_start__
+@ray.remote
+def get_size(ref_generator : DynamicObjectRefGenerator):
+    print(ref_generator)
+    num_elements = 0
+    for ref in ref_generator:
+        array = ray.get(ref)
+        assert len(array) <= block_size
+        num_elements += len(array)
+    return num_elements
+
+
+# Returns an ObjectRef[DynamicObjectRefGenerator].
+dynamic_ref = split.remote(array_ref, block_size)
+assert array_size == ray.get(get_size.remote(dynamic_ref))
+# (get_size pid=1504184)
+# <ray._raylet.DynamicObjectRefGenerator object at 0x7f81c4250ad0>
+
+# This also works, but should be avoided because you have to call an additional
+# `ray.get`, which blocks the driver.
+ref_generator = ray.get(dynamic_ref)
+assert array_size == ray.get(get_size.remote(ref_generator))
+# (get_size pid=1504184)
+# <ray._raylet.DynamicObjectRefGenerator object at 0x7f81c4251b50>
+# __dynamic_generator_pass_end__
+# fmt: on
+
+
+# fmt: off
+# __generator_errors_start__
+@ray.remote
+def generator():
+    for i in range(2):
+        yield i
+    raise Exception("error")
+
+
+ref1, ref2, ref3, ref4 = generator.options(num_returns=4).remote()
+assert ray.get([ref1, ref2]) == [0, 1]
+# All remaining ObjectRefs will contain the error.
+try:
+    ray.get([ref3, ref4])
+except Exception as error:
+    print(error)
+
+dynamic_ref = generator.options(num_returns="dynamic").remote()
+ref_generator = ray.get(dynamic_ref)
+ref1, ref2, ref3 = ref_generator
+assert ray.get([ref1, ref2]) == [0, 1]
+# Generators with num_returns="dynamic" will store the exception in the final
+# ObjectRef.
+try:
+    ray.get(ref3)
+except Exception as error:
+    print(error)
+# __generator_errors_end__
+# fmt: on
+
+# fmt: off
+# __generator_errors_unsupported_start__
+# Generators that yield more values than expected currently do not throw an
+# exception (the error is only logged).
+# See https://github.com/ray-project/ray/issues/28689.
+ref1, ref2 = generator.options(num_returns=2).remote()
+assert ray.get([ref1, ref2]) == [0, 1]
+"""
+(generator pid=2375938) 2022-09-28 11:08:51,386 ERROR worker.py:755 --
+    Unhandled error: Task threw exception, but all return values already
+    created.  This should only occur when using generator tasks.
+...
+"""
+# __generator_errors_unsupported_end__
+# fmt: on

doc/source/ray-core/doc_code/get_or_create.py

@@ -0,0 +1,19 @@
+import ray
+
+
+@ray.remote
+class Greeter:
+    def __init__(self, value):
+        self.value = value
+
+    def say_hello(self):
+        return self.value
+
+
+# Actor `g1` doesn't yet exist, so it is created with the given args.
+a = Greeter.options(name="g1", get_if_exists=True).remote("Old Greeting")
+assert ray.get(a.say_hello.remote()) == "Old Greeting"
+
+# Actor `g1` already exists, so it is returned (new args are ignored).
+b = Greeter.options(name="g1", get_if_exists=True).remote("New Greeting")
+assert ray.get(b.say_hello.remote()) == "Old Greeting"

doc/source/ray-core/doc_code/getting_started.py

@@ -0,0 +1,75 @@
+# flake8: noqa
+# __starting_ray_start__
+import ray
+
+ray.init()
+# __starting_ray_end__
+
+
+# fmt: off
+# __running_task_start__
+# Define the square task.
+@ray.remote
+def square(x):
+    return x * x
+
+# Launch four parallel square tasks.
+futures = [square.remote(i) for i in range(4)]
+
+# Retrieve results.
+print(ray.get(futures))
+# -> [0, 1, 4, 9]
+# __running_task_end__
+# fmt: on
+
+
+# fmt: off
+# __calling_actor_start__
+# Define the Counter actor.
+@ray.remote
+class Counter:
+    def __init__(self):
+        self.i = 0
+
+    def get(self):
+        return self.i
+
+    def incr(self, value):
+        self.i += value
+
+# Create a Counter actor.
+c = Counter.remote()
+
+# Submit calls to the actor. These calls run asynchronously but in
+# submission order on the remote actor process.
+for _ in range(10):
+    c.incr.remote(1)
+
+# Retrieve final actor state.
+print(ray.get(c.get.remote()))
+# -> 10
+# __calling_actor_end__
+# fmt: on
+
+
+# fmt: off
+# __passing_object_start__
+import numpy as np
+
+# Define a task that sums the values in a matrix.
+@ray.remote
+def sum_matrix(matrix):
+    return np.sum(matrix)
+
+# Call the task with a literal argument value.
+print(ray.get(sum_matrix.remote(np.ones((100, 100)))))
+# -> 10000.0
+
+# Put a large array into the object store.
+matrix_ref = ray.put(np.ones((1000, 1000)))
+
+# Call the task with the object reference as an argument.
+print(ray.get(sum_matrix.remote(matrix_ref)))
+# -> 1000000.0
+# __passing_object_end__
+# fmt: on

doc/source/ray-core/doc_code/limit_pending_tasks.py

@@ -0,0 +1,38 @@
+# __without_backpressure_start__
+import ray
+
+ray.init()
+
+
+@ray.remote
+class Actor:
+    async def heavy_compute(self):
+        # taking a long time...
+        # await asyncio.sleep(5)
+        return
+
+
+actor = Actor.remote()
+
+NUM_TASKS = 1000
+result_refs = []
+# When NUM_TASKS is large enough, this will eventually OOM.
+for _ in range(NUM_TASKS):
+    result_refs.append(actor.heavy_compute.remote())
+ray.get(result_refs)
+# __without_backpressure_end__
+
+# __with_backpressure_start__
+MAX_NUM_PENDING_TASKS = 100
+result_refs = []
+for _ in range(NUM_TASKS):
+    if len(result_refs) > MAX_NUM_PENDING_TASKS:
+        # update result_refs to only
+        # track the remaining tasks.
+        ready_refs, result_refs = ray.wait(result_refs, num_returns=1)
+        ray.get(ready_refs)
+
+    result_refs.append(actor.heavy_compute.remote())
+
+ray.get(result_refs)
+# __with_backpressure_end__

doc/source/ray-core/doc_code/limit_running_tasks.py

@@ -0,0 +1,35 @@
+# __without_limit_start__
+import ray
+
+# Assume this Ray node has 16 CPUs and 16G memory.
+ray.init()
+
+
+@ray.remote
+def process(file):
+    # Actual work is reading the file and process the data.
+    # Assume it needs to use 2G memory.
+    pass
+
+
+NUM_FILES = 1000
+result_refs = []
+for i in range(NUM_FILES):
+    # By default, process task will use 1 CPU resource and no other resources.
+    # This means 16 tasks can run concurrently
+    # and will OOM since 32G memory is needed while the node only has 16G.
+    result_refs.append(process.remote(f"{i}.csv"))
+ray.get(result_refs)
+# __without_limit_end__
+
+# __with_limit_start__
+result_refs = []
+for i in range(NUM_FILES):
+    # Now each task will use 2G memory resource
+    # and the number of concurrently running tasks is limited to 8.
+    # In this case, setting num_cpus to 2 has the same effect.
+    result_refs.append(
+        process.options(memory=2 * 1024 * 1024 * 1024).remote(f"{i}.csv")
+    )
+ray.get(result_refs)
+# __with_limit_end__

doc/source/ray-core/doc_code/monte_carlo_pi.py

@@ -0,0 +1,90 @@
+# __starting_ray_start__
+import ray
+import math
+import time
+import random
+
+ray.init()
+# __starting_ray_end__
+
+
+# fmt: off
+# __defining_actor_start__
+@ray.remote
+class ProgressActor:
+    def __init__(self, total_num_samples: int):
+        self.total_num_samples = total_num_samples
+        self.num_samples_completed_per_task = {}
+
+    def report_progress(self, task_id: int, num_samples_completed: int) -> None:
+        self.num_samples_completed_per_task[task_id] = num_samples_completed
+
+    def get_progress(self) -> float:
+        return (
+            sum(self.num_samples_completed_per_task.values()) / self.total_num_samples
+        )
+# __defining_actor_end__
+# fmt: on
+
+
+# fmt: off
+# __defining_task_start__
+@ray.remote
+def sampling_task(num_samples: int, task_id: int,
+                  progress_actor: ray.actor.ActorHandle) -> int:
+    num_inside = 0
+    for i in range(num_samples):
+        x, y = random.uniform(-1, 1), random.uniform(-1, 1)
+        if math.hypot(x, y) <= 1:
+            num_inside += 1
+
+        # Report progress every 1 million samples.
+        if (i + 1) % 1_000_000 == 0:
+            # This is async.
+            progress_actor.report_progress.remote(task_id, i + 1)
+
+    # Report the final progress.
+    progress_actor.report_progress.remote(task_id, num_samples)
+    return num_inside
+# __defining_task_end__
+# fmt: on
+
+
+# __creating_actor_start__
+# Change this to match your cluster scale.
+NUM_SAMPLING_TASKS = 10
+NUM_SAMPLES_PER_TASK = 10_000_000
+TOTAL_NUM_SAMPLES = NUM_SAMPLING_TASKS * NUM_SAMPLES_PER_TASK
+
+# Create the progress actor.
+progress_actor = ProgressActor.remote(TOTAL_NUM_SAMPLES)
+# __creating_actor_end__
+
+# __executing_task_start__
+# Create and execute all sampling tasks in parallel.
+results = [
+    sampling_task.remote(NUM_SAMPLES_PER_TASK, i, progress_actor)
+    for i in range(NUM_SAMPLING_TASKS)
+]
+# __executing_task_end__
+
+# __calling_actor_start__
+# Query progress periodically.
+while True:
+    progress = ray.get(progress_actor.get_progress.remote())
+    print(f"Progress: {int(progress * 100)}%")
+
+    if progress == 1:
+        break
+
+    time.sleep(1)
+# __calling_actor_end__
+
+# __calculating_pi_start__
+# Get all the sampling tasks results.
+total_num_inside = sum(ray.get(results))
+pi = (total_num_inside * 4) / TOTAL_NUM_SAMPLES
+print(f"Estimated value of π is: {pi}")
+# __calculating_pi_end__
+
+assert str(pi).startswith("3.14")

doc/source/ray-core/doc_code/namespaces.py

@@ -0,0 +1,127 @@
+# flake8: noqa
+# fmt: off
+# __init_namespace_start__
+import ray
+
+ray.init(namespace="hello")
+# __init_namespace_end__
+# fmt: on
+
+ray.shutdown()
+
+# fmt: off
+# __actor_namespace_start__
+import subprocess
+import ray
+
+try:
+    subprocess.check_output(["ray", "start", "--head"])
+
+    @ray.remote
+    class Actor:
+      pass
+
+    # Job 1 creates two actors, "orange" and "purple" in the "colors" namespace.
+    with ray.init("ray://localhost:10001", namespace="colors"):
+      Actor.options(name="orange", lifetime="detached").remote()
+      Actor.options(name="purple", lifetime="detached").remote()
+
+    # Job 2 is now connecting to a different namespace.
+    with ray.init("ray://localhost:10001", namespace="fruits"):
+      # This fails because "orange" was defined in the "colors" namespace.
+      try:
+        ray.get_actor("orange")
+      except ValueError:
+        pass
+
+      # This succeeds because the name "orange" is unused in this namespace.
+      Actor.options(name="orange", lifetime="detached").remote()
+      Actor.options(name="watermelon", lifetime="detached").remote()
+
+    # Job 3 connects to the original "colors" namespace
+    context = ray.init("ray://localhost:10001", namespace="colors")
+
+    # This fails because "watermelon" was in the fruits namespace.
+    try:
+      ray.get_actor("watermelon")
+    except ValueError:
+      pass
+
+    # This returns the "orange" actor we created in the first job, not the second.
+    ray.get_actor("orange")
+
+    # We are manually managing the scope of the connection in this example.
+    context.disconnect()
+finally:
+    subprocess.check_output(["ray", "stop", "--force"])
+# __actor_namespace_end__
+# fmt: on
+
+# fmt: off
+# __specify_actor_namespace_start__
+import subprocess
+import ray
+
+try:
+    subprocess.check_output(["ray", "start", "--head"])
+
+    @ray.remote
+    class Actor:
+        pass
+
+    ctx = ray.init("ray://localhost:10001")
+
+    # Create an actor with specified namespace.
+    Actor.options(name="my_actor", namespace="actor_namespace", lifetime="detached").remote()
+
+    # It is accessible in its namespace.
+    ray.get_actor("my_actor", namespace="actor_namespace")
+    ctx.disconnect()
+finally:
+    subprocess.check_output(["ray", "stop", "--force"])
+# __specify_actor_namespace_end__
+# fmt: on
+
+# fmt: off
+# __anonymous_namespace_start__
+import subprocess
+import ray
+
+try:
+  subprocess.check_output(["ray", "start", "--head"])
+
+  @ray.remote
+  class Actor:
+      pass
+
+  # Job 1 connects to an anonymous namespace by default
+  with ray.init("ray://localhost:10001"):
+    Actor.options(name="my_actor", lifetime="detached").remote()
+
+  # Job 2 connects to a _different_ anonymous namespace by default
+  with ray.init("ray://localhost:10001"):
+    # This succeeds because the second job is in its own namespace.
+    Actor.options(name="my_actor", lifetime="detached").remote()
+
+finally:
+    subprocess.check_output(["ray", "stop", "--force"])
+# __anonymous_namespace_end__
+# fmt: on
+
+# fmt: off
+# __get_namespace_start__
+import subprocess
+import ray
+
+try:
+  subprocess.check_output(["ray", "start", "--head"])
+
+  ray.init(address="auto", namespace="colors")
+
+  # Will print namespace name "colors".
+  print(ray.get_runtime_context().namespace)
+
+finally:
+    subprocess.check_output(["ray", "stop", "--force"])
+# __get_namespace_end__
+# fmt: on

doc/source/ray-core/doc_code/nested-tasks.py

@@ -0,0 +1,42 @@
+# flake8: noqa
+
+# __nested_start__
+import ray
+
+
+@ray.remote
+def f():
+    return 1
+
+
+@ray.remote
+def g():
+    # Call f 4 times and return the resulting object refs.
+    return [f.remote() for _ in range(4)]
+
+
+@ray.remote
+def h():
+    # Call f 4 times, block until those 4 tasks finish,
+    # retrieve the results, and return the values.
+    return ray.get([f.remote() for _ in range(4)])
+
+
+# __nested_end__
+
+ray.init(num_cpus=4, num_gpus=1)
+
+obj_refs = ray.get(g.remote())
+assert len(obj_refs) == 4
+assert all(isinstance(o, ray.ObjectRef) for o in obj_refs)
+
+
+# __yield_start__
+@ray.remote(num_cpus=1, num_gpus=1)
+def g():
+    return ray.get(f.remote())
+
+
+# __yield_end__
+
+assert ray.get(g.remote()) == 1

doc/source/ray-core/doc_code/obj_capture.py

@@ -0,0 +1,16 @@
+import ray
+
+# Put the values (1, 2, 3) into Ray's object store.
+a, b, c = ray.put(1), ray.put(2), ray.put(3)
+
+
+@ray.remote
+def print_via_capture():
+    """This function prints the values of (a, b, c) to stdout."""
+    print(ray.get([a, b, c]))
+
+
+# Passing object references via closure-capture. Inside the `print_via_capture`
+# function, the global object refs (a, b, c) can be retrieved and printed.
+print_via_capture.remote()
+# -> prints [1, 2, 3]

doc/source/ray-core/doc_code/obj_ref.py

@@ -0,0 +1,18 @@
+import ray
+
+
+@ray.remote
+def echo_and_get(x_list):  # List[ObjectRef]
+    """This function prints its input values to stdout."""
+    print("args:", x_list)
+    print("values:", ray.get(x_list))
+
+
+# Put the values (1, 2, 3) into Ray's object store.
+a, b, c = ray.put(1), ray.put(2), ray.put(3)
+
+# Passing an object as a nested argument to `echo_and_get`. Ray does not
+# de-reference nested args, so `echo_and_get` sees the references.
+echo_and_get.remote([a, b, c])
+# -> prints args: [ObjectRef(...), ObjectRef(...), ObjectRef(...)]
+#           values: [1, 2, 3]

doc/source/ray-core/doc_code/obj_val.py

@@ -0,0 +1,20 @@
+import ray
+
+
+@ray.remote
+def echo(a: int, b: int, c: int):
+    """This function prints its input values to stdout."""
+    print(a, b, c)
+
+
+# Passing the literal values (1, 2, 3) to `echo`.
+echo.remote(1, 2, 3)
+# -> prints "1 2 3"
+
+# Put the values (1, 2, 3) into Ray's object store.
+a, b, c = ray.put(1), ray.put(2), ray.put(3)
+
+# Passing an object as a top-level argument to `echo`. Ray will de-reference top-level
+# arguments, so `echo` will see the literal values (1, 2, 3) in this case as well.
+echo.remote(a, b, c)
+# -> prints "1 2 3"

doc/source/ray-core/doc_code/object_ref_serialization.py

@@ -0,0 +1,25 @@
+import ray
+from ray import cloudpickle
+
+FILE = "external_store.pickle"
+
+ray.init()
+
+my_dict = {"hello": "world"}
+
+obj_ref = ray.put(my_dict)
+with open(FILE, "wb+") as f:
+    cloudpickle.dump(obj_ref, f)
+
+# ObjectRef remains pinned in memory because
+# it was serialized with ray.cloudpickle.
+del obj_ref
+
+with open(FILE, "rb") as f:
+    new_obj_ref = cloudpickle.load(f)
+
+# The deserialized ObjectRef works as expected.
+assert ray.get(new_obj_ref) == my_dict
+
+# Explicitly free the object.
+ray._private.internal_api.free(new_obj_ref)

doc/source/ray-core/doc_code/original_resource_unavailable_example.py

@@ -0,0 +1,28 @@
+import ray
+from ray.util.placement_group import (
+    placement_group,
+)
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+# Two "CPU"s are available.
+ray.init(num_cpus=2)
+
+# Create a placement group.
+pg = placement_group([{"CPU": 2}])
+ray.get(pg.ready())
+
+
+# Now, 2 CPUs are not available anymore because
+# they are pre-reserved by the placement group.
+@ray.remote(num_cpus=2)
+def f():
+    return True
+
+
+# Won't be scheduled because there are no 2 cpus.
+f.remote()
+
+# Will be scheduled because 2 cpus are reserved by the placement group.
+f.options(
+    scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg)
+).remote()

doc/source/ray-core/doc_code/owners.py

@@ -0,0 +1,19 @@
+# flake8: noqa
+# __owners_begin__
+import ray
+import numpy as np
+
+
+@ray.remote
+def large_array():
+    return np.zeros(int(1e5))
+
+
+x = ray.put(1)  # The driver owns x and also creates the value of x.
+
+y = large_array.remote()
+# The driver is the owner of y, even though the value may be stored somewhere else.
+# If the node that stores the value of y dies, Ray will automatically recover
+# it by re-executing the large_array task.
+# If the driver dies, anyone still using y will receive an OwnerDiedError.
+# __owners_end__

doc/source/ray-core/doc_code/pattern_async_actor.py

@@ -0,0 +1,70 @@
+# __sync_actor_start__
+import ray
+
+
+@ray.remote
+class TaskStore:
+    def get_next_task(self):
+        return "task"
+
+
+@ray.remote
+class TaskExecutor:
+    def __init__(self, task_store):
+        self.task_store = task_store
+        self.num_executed_tasks = 0
+
+    def run(self):
+        while True:
+            task = ray.get(self.task_store.get_next_task.remote())
+            self._execute_task(task)
+
+    def _execute_task(self, task):
+        # Executing the task
+        self.num_executed_tasks = self.num_executed_tasks + 1
+
+    def get_num_executed_tasks(self):
+        return self.num_executed_tasks
+
+
+task_store = TaskStore.remote()
+task_executor = TaskExecutor.remote(task_store)
+task_executor.run.remote()
+try:
+    # This will timeout since task_executor.run occupies the entire actor thread
+    # and get_num_executed_tasks cannot run.
+    ray.get(task_executor.get_num_executed_tasks.remote(), timeout=5)
+except ray.exceptions.GetTimeoutError:
+    print("get_num_executed_tasks didn't finish in 5 seconds")
+# __sync_actor_end__
+
+
+# __async_actor_start__
+@ray.remote
+class AsyncTaskExecutor:
+    def __init__(self, task_store):
+        self.task_store = task_store
+        self.num_executed_tasks = 0
+
+    async def run(self):
+        while True:
+            # Here we use await instead of ray.get() to
+            # wait for the next task and it will yield
+            # the control while waiting.
+            task = await self.task_store.get_next_task.remote()
+            self._execute_task(task)
+
+    def _execute_task(self, task):
+        # Executing the task
+        self.num_executed_tasks = self.num_executed_tasks + 1
+
+    def get_num_executed_tasks(self):
+        return self.num_executed_tasks
+
+
+async_task_executor = AsyncTaskExecutor.remote(task_store)
+async_task_executor.run.remote()
+# We are able to run get_num_executed_tasks while run method is running.
+num_executed_tasks = ray.get(async_task_executor.get_num_executed_tasks.remote())
+print(f"num of executed tasks so far: {num_executed_tasks}")
+# __async_actor_end__

doc/source/ray-core/doc_code/pattern_generators.py

@@ -0,0 +1,61 @@
+import sys
+
+if len(sys.argv) == 1:
+    # A small number for CI test.
+    sys.argv.append("5")
+
+# __program_start__
+import sys
+import ray
+
+# fmt: off
+# __large_values_start__
+import numpy as np
+
+
+@ray.remote
+def large_values(num_returns):
+    return [
+        np.random.randint(np.iinfo(np.int8).max, size=(100_000_000, 1), dtype=np.int8)
+        for _ in range(num_returns)
+    ]
+# __large_values_end__
+# fmt: on
+
+
+# fmt: off
+# __large_values_generator_start__
+@ray.remote
+def large_values_generator(num_returns):
+    for i in range(num_returns):
+        yield np.random.randint(
+            np.iinfo(np.int8).max, size=(100_000_000, 1), dtype=np.int8
+        )
+        print(f"yielded return value {i}")
+# __large_values_generator_end__
+# fmt: on
+
+
+# A large enough value (e.g. 100).
+num_returns = int(sys.argv[1])
+# Worker will likely OOM using normal returns.
+print("Using normal functions...")
+try:
+    ray.get(
+        large_values.options(num_returns=num_returns, max_retries=0).remote(
+            num_returns
+        )[0]
+    )
+except ray.exceptions.WorkerCrashedError:
+    print("Worker failed with normal function")
+
+# Using a generator will allow the worker to finish.
+# Note that this will block until the full task is complete, i.e. the
+# last yield finishes.
+print("Using generators...")
+ray.get(
+    large_values_generator.options(num_returns=num_returns, max_retries=0).remote(
+        num_returns
+    )[0]
+)
+print("Success!")

doc/source/ray-core/doc_code/pattern_nested_tasks.py

@@ -0,0 +1,72 @@
+# __pattern_start__
+import ray
+import time
+from numpy import random
+
+
+def partition(collection):
+    # Use the last element as the pivot
+    pivot = collection.pop()
+    greater, lesser = [], []
+    for element in collection:
+        if element > pivot:
+            greater.append(element)
+        else:
+            lesser.append(element)
+    return lesser, pivot, greater
+
+
+def quick_sort(collection):
+    if len(collection) <= 200000:  # magic number
+        return sorted(collection)
+    else:
+        lesser, pivot, greater = partition(collection)
+        lesser = quick_sort(lesser)
+        greater = quick_sort(greater)
+    return lesser + [pivot] + greater
+
+
+@ray.remote
+def quick_sort_distributed(collection):
+    # Tiny tasks are an antipattern.
+    # Thus, in our example we have a "magic number" to
+    # toggle when distributed recursion should be used vs
+    # when the sorting should be done in place. The rule
+    # of thumb is that the duration of an individual task
+    # should be at least 1 second.
+    if len(collection) <= 200000:  # magic number
+        return sorted(collection)
+    else:
+        lesser, pivot, greater = partition(collection)
+        lesser = quick_sort_distributed.remote(lesser)
+        greater = quick_sort_distributed.remote(greater)
+        return ray.get(lesser) + [pivot] + ray.get(greater)
+
+
+for size in [200000, 4000000, 8000000]:
+    print(f"Array size: {size}")
+    unsorted = random.randint(1000000, size=(size)).tolist()
+    s = time.time()
+    quick_sort(unsorted)
+    print(f"Sequential execution: {(time.time() - s):.3f}")
+    s = time.time()
+    ray.get(quick_sort_distributed.remote(unsorted))
+    print(f"Distributed execution: {(time.time() - s):.3f}")
+    print("--" * 10)
+
+# Outputs:
+
+# Array size: 200000
+# Sequential execution: 0.040
+# Distributed execution: 0.152
+# --------------------
+# Array size: 4000000
+# Sequential execution: 6.161
+# Distributed execution: 5.779
+# --------------------
+# Array size: 8000000
+# Sequential execution: 15.459
+# Distributed execution: 11.282
+# --------------------
+
+# __pattern_end__

doc/source/ray-core/doc_code/pattern_pipelining.py

@@ -0,0 +1,66 @@
+import ray
+
+
+@ray.remote
+class WorkQueue:
+    def __init__(self):
+        self.queue = list(range(10))
+
+    def get_work_item(self):
+        if self.queue:
+            return self.queue.pop(0)
+        else:
+            return None
+
+
+@ray.remote
+class WorkerWithoutPipelining:
+    def __init__(self, work_queue):
+        self.work_queue = work_queue
+
+    def process(self, work_item):
+        print(work_item)
+
+    def run(self):
+        while True:
+            # Get work from the remote queue.
+            work_item = ray.get(self.work_queue.get_work_item.remote())
+
+            if work_item is None:
+                break
+
+            # Do work.
+            self.process(work_item)
+
+
+@ray.remote
+class WorkerWithPipelining:
+    def __init__(self, work_queue):
+        self.work_queue = work_queue
+
+    def process(self, work_item):
+        print(work_item)
+
+    def run(self):
+        self.work_item_ref = self.work_queue.get_work_item.remote()
+
+        while True:
+            # Get work from the remote queue.
+            work_item = ray.get(self.work_item_ref)
+
+            if work_item is None:
+                break
+
+            self.work_item_ref = self.work_queue.get_work_item.remote()
+
+            # Do work while we are fetching the next work item.
+            self.process(work_item)
+
+
+work_queue = WorkQueue.remote()
+worker_without_pipelining = WorkerWithoutPipelining.remote(work_queue)
+ray.get(worker_without_pipelining.run.remote())
+
+work_queue = WorkQueue.remote()
+worker_with_pipelining = WorkerWithPipelining.remote(work_queue)
+ray.get(worker_with_pipelining.run.remote())

doc/source/ray-core/doc_code/pattern_tree_of_actors.py

@@ -0,0 +1,32 @@
+import ray
+
+
+@ray.remote(num_cpus=1)
+class Trainer:
+    def __init__(self, hyperparameter, data):
+        self.hyperparameter = hyperparameter
+        self.data = data
+
+    # Train the model on the given training data shard.
+    def fit(self):
+        return self.data * self.hyperparameter
+
+
+@ray.remote(num_cpus=1)
+class Supervisor:
+    def __init__(self, hyperparameter, data):
+        self.trainers = [Trainer.remote(hyperparameter, d) for d in data]
+
+    def fit(self):
+        # Train with different data shard in parallel.
+        return ray.get([trainer.fit.remote() for trainer in self.trainers])
+
+
+data = [1, 2, 3]
+supervisor1 = Supervisor.remote(1, data)
+supervisor2 = Supervisor.remote(2, data)
+# Train with different hyperparameters in parallel.
+model1 = supervisor1.fit.remote()
+model2 = supervisor2.fit.remote()
+assert ray.get(model1) == [1, 2, 3]
+assert ray.get(model2) == [2, 4, 6]

doc/source/ray-core/doc_code/placement_group_capture_child_tasks_example.py

@@ -0,0 +1,68 @@
+# __child_capture_pg_start__
+import ray
+from ray.util.placement_group import placement_group
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+ray.init(num_cpus=2)
+
+# Create a placement group.
+pg = placement_group([{"CPU": 2}])
+ray.get(pg.ready())
+
+
+@ray.remote(num_cpus=1)
+def child():
+    import time
+
+    time.sleep(5)
+
+
+@ray.remote(num_cpus=1)
+def parent():
+    # The child task is scheduled to the same placement group as its parent,
+    # although it didn't specify the PlacementGroupSchedulingStrategy.
+    ray.get(child.remote())
+
+
+# Since the child and parent use 1 CPU each, the placement group
+# bundle {"CPU": 2} is fully occupied.
+ray.get(
+    parent.options(
+        scheduling_strategy=PlacementGroupSchedulingStrategy(
+            placement_group=pg, placement_group_capture_child_tasks=True
+        )
+    ).remote()
+)
+# __child_capture_pg_end__
+
+
+# __child_capture_disable_pg_start__
+@ray.remote
+def parent():
+    # In this case, the child task isn't
+    # scheduled with the parent's placement group.
+    ray.get(
+        child.options(
+            scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=None)
+        ).remote()
+    )
+
+
+# This times out because we cannot schedule the child task.
+# The cluster has {"CPU": 2}, and both of them are reserved by
+# the placement group with a bundle {"CPU": 2}. Since the child shouldn't
+# be scheduled within this placement group, it cannot be scheduled because
+# there's no available CPU resources.
+try:
+    ray.get(
+        parent.options(
+            scheduling_strategy=PlacementGroupSchedulingStrategy(
+                placement_group=pg, placement_group_capture_child_tasks=True
+            )
+        ).remote(),
+        timeout=5,
+    )
+except Exception as e:
+    print("Couldn't create a child task!")
+    print(e)
+# __child_capture_disable_pg_end__

doc/source/ray-core/doc_code/placement_group_example.py

@@ -0,0 +1,141 @@
+# __create_pg_start__
+from pprint import pprint
+import time
+
+# Import placement group APIs.
+from ray.util.placement_group import (
+    placement_group,
+    placement_group_table,
+    remove_placement_group,
+)
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+# Initialize Ray.
+import ray
+
+# Create a single node Ray cluster with 2 CPUs and 2 GPUs.
+ray.init(num_cpus=2, num_gpus=2)
+
+# Reserve a placement group of 1 bundle that reserves 1 CPU and 1 GPU.
+pg = placement_group([{"CPU": 1, "GPU": 1}])
+# __create_pg_end__
+
+# __ready_pg_start__
+# Wait until placement group is created.
+ray.get(pg.ready(), timeout=10)
+
+# You can also use ray.wait.
+ready, unready = ray.wait([pg.ready()], timeout=10)
+
+# You can look at placement group states using this API.
+print(placement_group_table(pg))
+# __ready_pg_end__
+
+# __create_pg_failed_start__
+# Cannot create this placement group because we
+# cannot create a {"GPU": 2} bundle.
+pending_pg = placement_group([{"CPU": 1}, {"GPU": 2}])
+# This raises the timeout exception!
+try:
+    ray.get(pending_pg.ready(), timeout=5)
+except Exception as e:
+    print(
+        "Cannot create a placement group because "
+        "{'GPU': 2} bundle cannot be created."
+    )
+    print(e)
+# __create_pg_failed_end__
+
+
+# __schedule_pg_start__
+@ray.remote(num_cpus=1)
+class Actor:
+    def __init__(self):
+        pass
+
+    def ready(self):
+        pass
+
+
+# Create an actor to a placement group.
+actor = Actor.options(
+    scheduling_strategy=PlacementGroupSchedulingStrategy(
+        placement_group=pg,
+    )
+).remote()
+
+# Verify the actor is scheduled.
+ray.get(actor.ready.remote(), timeout=10)
+# __schedule_pg_end__
+
+
+# __schedule_pg_3_start__
+@ray.remote(num_cpus=0, num_gpus=1)
+class Actor:
+    def __init__(self):
+        pass
+
+    def ready(self):
+        pass
+
+
+# Create a GPU actor on the first bundle of index 0.
+actor2 = Actor.options(
+    scheduling_strategy=PlacementGroupSchedulingStrategy(
+        placement_group=pg,
+        placement_group_bundle_index=0,
+    )
+).remote()
+
+# Verify that the GPU actor is scheduled.
+ray.get(actor2.ready.remote(), timeout=10)
+# __schedule_pg_3_end__
+
+# __remove_pg_start__
+# This API is asynchronous.
+remove_placement_group(pg)
+
+# Wait until placement group is killed.
+time.sleep(1)
+# Check that the placement group has died.
+pprint(placement_group_table(pg))
+
+"""
+{'bundles': {0: {'GPU': 1.0}, 1: {'CPU': 1.0}},
+'name': 'unnamed_group',
+'placement_group_id': '40816b6ad474a6942b0edb45809b39c3',
+'state': 'REMOVED',
+'strategy': 'PACK'}
+"""
+# __remove_pg_end__
+
+# __strategy_pg_start__
+# Reserve a placement group of 2 bundles
+# that have to be packed on the same node.
+pg = placement_group([{"CPU": 1}, {"GPU": 1}], strategy="PACK")
+# __strategy_pg_end__
+
+remove_placement_group(pg)
+
+# __detached_pg_start__
+# driver_1.py
+# Create a detached placement group that survives even after
+# the job terminates.
+pg = placement_group([{"CPU": 1}], lifetime="detached", name="global_name")
+ray.get(pg.ready())
+# __detached_pg_end__
+remove_placement_group(pg)
+
+
+# __get_pg_start__
+# first_driver.py
+# Create a placement group with a unique name within a namespace.
+# Start Ray or connect to a Ray cluster using: ray.init(namespace="pg_namespace")
+pg = placement_group([{"CPU": 1}], name="pg_name")
+ray.get(pg.ready())
+
+# second_driver.py
+# Retrieve a placement group with a unique name within a namespace.
+# Start Ray or connect to a Ray cluster using: ray.init(namespace="pg_namespace")
+pg = ray.util.get_placement_group("pg_name")
+# __get_pg_end__

doc/source/ray-core/doc_code/ray-dag.py

@@ -0,0 +1,192 @@
+# flake8: noqa
+
+# fmt: off
+# __dag_tasks_begin__
+import ray
+
+ray.init()
+
+@ray.remote
+def func(src, inc=1):
+    return src + inc
+
+a_ref = func.bind(1, inc=2)
+assert ray.get(a_ref.execute()) == 3 # 1 + 2 = 3
+b_ref = func.bind(a_ref, inc=3)
+assert ray.get(b_ref.execute()) == 6 # (1 + 2) + 3 = 6
+c_ref = func.bind(b_ref, inc=a_ref)
+assert ray.get(c_ref.execute()) == 9 # ((1 + 2) + 3) + (1 + 2) = 9
+# __dag_tasks_end__
+# fmt: on
+
+ray.shutdown()
+
+# fmt: off
+# __dag_actors_begin__
+import ray
+
+ray.init()
+
+@ray.remote
+class Actor:
+    def __init__(self, init_value):
+        self.i = init_value
+
+    def inc(self, x):
+        self.i += x
+
+    def get(self):
+        return self.i
+
+a1 = Actor.bind(10)  # Instantiate Actor with init_value 10.
+val = a1.get.bind()  # ClassMethod that returns value from get() from
+                     # the actor created.
+assert ray.get(val.execute()) == 10
+
+@ray.remote
+def combine(x, y):
+    return x + y
+
+a2 = Actor.bind(10) # Instantiate another Actor with init_value 10.
+a1.inc.bind(2)  # Call inc() on the actor created with increment of 2.
+a1.inc.bind(4)  # Call inc() on the actor created with increment of 4.
+a2.inc.bind(6)  # Call inc() on the actor created with increment of 6.
+
+# Combine outputs from a1.get() and a2.get()
+dag = combine.bind(a1.get.bind(), a2.get.bind())
+
+# a1 +  a2 + inc(2) + inc(4) + inc(6)
+# 10 + (10 + ( 2   +    4    +   6)) = 32
+assert ray.get(dag.execute()) == 32
+# __dag_actors_end__
+# fmt: on
+
+ray.shutdown()
+
+# fmt: off
+# __dag_input_node_begin__
+import ray
+
+ray.init()
+
+from ray.dag.input_node import InputNode
+
+@ray.remote
+def a(user_input):
+    return user_input * 2
+
+@ray.remote
+def b(user_input):
+    return user_input + 1
+
+@ray.remote
+def c(x, y):
+    return x + y
+
+with InputNode() as dag_input:
+    a_ref = a.bind(dag_input)
+    b_ref = b.bind(dag_input)
+    dag = c.bind(a_ref, b_ref)
+
+#   a(2)  +   b(2)  = c
+# (2 * 2) + (2 + 1)
+assert ray.get(dag.execute(2)) == 7
+
+#   a(3)  +   b(3)  = c
+# (3 * 2) + (3 + 1)
+assert ray.get(dag.execute(3)) == 10
+# __dag_input_node_end__
+# fmt: on
+
+# fmt: off
+# __dag_multi_output_node_begin__
+import ray
+
+from ray.dag.input_node import InputNode
+from ray.dag.output_node import MultiOutputNode
+
+@ray.remote
+def f(input):
+    return input + 1
+
+with InputNode() as input_data:
+    dag = MultiOutputNode([f.bind(input_data["x"]), f.bind(input_data["y"])])
+
+refs = dag.execute({"x": 1, "y": 2})
+assert ray.get(refs) == [2, 3]
+# __dag_multi_output_node_end__
+# fmt: on
+
+# fmt: off
+# __dag_multi_output_node_begin__
+import ray
+
+from ray.dag.input_node import InputNode
+from ray.dag.output_node import MultiOutputNode
+
+@ray.remote
+def f(input):
+    return input + 1
+
+with InputNode() as input_data:
+    dag = MultiOutputNode([f.bind(input_data["x"]), f.bind(input_data["y"])])
+
+refs = dag.execute({"x": 1, "y": 2})
+assert ray.get(refs) == [2, 3]
+# __dag_multi_output_node_end__
+# fmt: on
+
+# fmt: off
+# __dag_multi_output_node_begin__
+import ray
+
+from ray.dag.input_node import InputNode
+from ray.dag.output_node import MultiOutputNode
+
+@ray.remote
+def f(input):
+    return input + 1
+
+with InputNode() as input_data:
+    dag = MultiOutputNode([f.bind(input_data["x"]), f.bind(input_data["y"])])
+
+refs = dag.execute({"x": 1, "y": 2})
+assert ray.get(refs) == [2, 3]
+# __dag_multi_output_node_end__
+# fmt: on
+
+# fmt: off
+# __dag_actor_reuse_begin__
+import ray
+from ray.dag.input_node import InputNode
+from ray.dag.output_node import MultiOutputNode
+
+@ray.remote
+class Worker:
+    def __init__(self):
+        self.forwarded = 0
+
+    def forward(self, input_data: int):
+        self.forwarded += 1
+        return input_data + 1
+
+    def num_forwarded(self):
+        return self.forwarded
+
+# Create an actor via ``remote`` API not ``bind`` API to avoid
+# killing actors when a DAG is finished.
+worker = Worker.remote()
+
+with InputNode() as input_data:
+    dag = MultiOutputNode([worker.forward.bind(input_data)])
+
+# Actors are reused. The DAG definition doesn't include
+# actor creation.
+assert ray.get(dag.execute(1)) == [2]
+assert ray.get(dag.execute(2)) == [3]
+assert ray.get(dag.execute(3)) == [4]
+
+# You can still use other actor methods via `remote` API.
+assert ray.get(worker.num_forwarded.remote()) == 3
+# __dag_actor_reuse_end__
+# fmt: on

doc/source/ray-core/doc_code/ray_oom_prevention.py

@@ -0,0 +1,87 @@
+# flake8: noqa
+import ray
+
+ray.init(
+    _system_config={
+        "memory_usage_threshold": 0.4,
+    },
+)
+# fmt: off
+# __last_task_start__
+import ray
+
+@ray.remote(max_retries=0)
+def leaks_memory():
+    chunks = []
+    bits_to_allocate = 8 * 100 * 1024 * 1024  # ~100 MiB
+    while True:
+        chunks.append([0] * bits_to_allocate)
+
+
+try:
+    ray.get(leaks_memory.remote())
+except ray.exceptions.OutOfMemoryError as ex:
+    print("task failed with OutOfMemoryError, which is expected")
+# __last_task_end__
+# fmt: on
+
+
+# fmt: off
+# __two_actors_start__
+from math import ceil
+import ray
+from ray._private.utils import (
+    get_system_memory,
+)  # do not use outside of this example as these are private methods.
+from ray._private.utils import (
+    get_used_memory,
+)  # do not use outside of this example as these are private methods.
+
+
+# estimates the number of bytes to allocate to reach the desired memory usage percentage.
+def get_additional_bytes_to_reach_memory_usage_pct(pct: float) -> int:
+    used = get_used_memory()
+    total = get_system_memory()
+    bytes_needed = int(total * pct) - used
+    assert (
+        bytes_needed > 0
+    ), "memory usage is already above the target. Increase the target percentage."
+    return bytes_needed
+
+
+@ray.remote
+class MemoryHogger:
+    def __init__(self):
+        self.allocations = []
+
+    def allocate(self, bytes_to_allocate: float) -> None:
+        # divide by 8 as each element in the array occupies 8 bytes
+        new_list = [0] * ceil(bytes_to_allocate / 8)
+        self.allocations.append(new_list)
+
+
+first_actor = MemoryHogger.options(
+    max_restarts=1, max_task_retries=1, name="first_actor"
+).remote()
+second_actor = MemoryHogger.options(
+    max_restarts=0, max_task_retries=0, name="second_actor"
+).remote()
+
+# We want total bytes to exceed 40% threshold to trigger the memory monitor.
+allocate_bytes = get_additional_bytes_to_reach_memory_usage_pct(0.5)
+
+first_actor_task = first_actor.allocate.remote(0.9 * allocate_bytes)
+second_actor_task = second_actor.allocate.remote(0.1* allocate_bytes)
+
+error_thrown = False
+try:
+    ray.get(first_actor_task)
+except ray.exceptions.OutOfMemoryError as ex:
+    error_thrown = True
+    print("First started actor, which is retriable, was killed by the memory monitor.")
+assert error_thrown
+
+ray.get(second_actor_task)
+print("Second started actor, which is not-retriable, finished.")
+# __two_actors_end__
+# fmt: on

doc/source/ray-core/doc_code/resources.py

@@ -0,0 +1,59 @@
+import ray
+
+# __specifying_node_resources_start__
+# This will start a Ray node with 3 logical cpus, 4 logical gpus,
+# 1 special_hardware resource and 1 custom_label resource.
+ray.init(num_cpus=3, num_gpus=4, resources={"special_hardware": 1, "custom_label": 1})
+# __specifying_node_resources_end__
+
+
+# __specifying_resource_requirements_start__
+# Specify the default resource requirements for this remote function.
+@ray.remote(num_cpus=2, num_gpus=2, resources={"special_hardware": 1})
+def func():
+    return 1
+
+
+# You can override the default resource requirements.
+func.options(num_cpus=3, num_gpus=1, resources={"special_hardware": 0}).remote()
+
+
+@ray.remote(num_cpus=0, num_gpus=1)
+class Actor:
+    pass
+
+
+# You can override the default resource requirements for actors as well.
+actor = Actor.options(num_cpus=1, num_gpus=0).remote()
+# __specifying_resource_requirements_end__
+
+
+# __specifying_fractional_resource_requirements_start__
+@ray.remote(num_cpus=0.5)
+def io_bound_task():
+    import time
+
+    time.sleep(1)
+    return 2
+
+
+io_bound_task.remote()
+
+
+@ray.remote(num_gpus=0.5)
+class IOActor:
+    def ping(self):
+        import os
+
+        print(f"CUDA_VISIBLE_DEVICES: {os.environ['CUDA_VISIBLE_DEVICES']}")
+
+
+# Two actors can share the same GPU.
+io_actor1 = IOActor.remote()
+io_actor2 = IOActor.remote()
+ray.get(io_actor1.ping.remote())
+ray.get(io_actor2.ping.remote())
+# Output:
+# (IOActor pid=96328) CUDA_VISIBLE_DEVICES: 1
+# (IOActor pid=96329) CUDA_VISIBLE_DEVICES: 1
+# __specifying_fractional_resource_requirements_end__

doc/source/ray-core/doc_code/runtime_env_example.py

@@ -0,0 +1,81 @@
+# flake8: noqa
+"""
+This file holds code for the runtime envs documentation.
+
+FIXME: We switched our code formatter from YAPF to Black. Check if we can enable code
+formatting on this module and update the paragraph below. See issue #21318.
+
+It ignores yapf because yapf doesn't allow comments right after code blocks,
+but we put comments right after code blocks to prevent large white spaces
+in the documentation.
+"""
+import ray
+
+# fmt: off
+
+# __runtime_env_pip_def_start__
+runtime_env = {
+    "pip": ["emoji"],
+    "env_vars": {"TF_WARNINGS": "none"}
+}
+# __runtime_env_pip_def_end__
+
+# __strong_typed_api_runtime_env_pip_def_start__
+from ray.runtime_env import RuntimeEnv
+runtime_env = RuntimeEnv(
+    pip=["emoji"],
+    env_vars={"TF_WARNINGS": "none"}
+)
+# __strong_typed_api_runtime_env_pip_def_end__
+
+# __ray_init_start__
+# Option 1: Starting a single-node local Ray cluster or connecting to existing local cluster
+ray.init(runtime_env=runtime_env)
+# __ray_init_end__
+
+@ray.remote
+def f_job():
+    pass
+
+@ray.remote
+class Actor_job:
+    def g(self):
+        pass
+
+ray.get(f_job.remote())
+a = Actor_job.remote()
+ray.get(a.g.remote())
+
+ray.shutdown()
+
+ray.init()
+
+@ray.remote
+def f():
+    pass
+
+@ray.remote
+class SomeClass:
+    pass
+
+# __per_task_per_actor_start__
+# Invoke a remote task that will run in a specified runtime environment.
+f.options(runtime_env=runtime_env).remote()
+
+# Instantiate an actor that will run in a specified runtime environment.
+actor = SomeClass.options(runtime_env=runtime_env).remote()
+
+# Specify a runtime environment in the task definition.  Future invocations via
+# `g.remote()` will use this runtime environment unless overridden by using
+# `.options()` as above.
+@ray.remote(runtime_env=runtime_env)
+def g():
+    pass
+
+# Specify a runtime environment in the actor definition.  Future instantiations
+# via `MyClass.remote()` will use this runtime environment unless overridden by
+# using `.options()` as above.
+@ray.remote(runtime_env=runtime_env)
+class MyClass:
+    pass
+# __per_task_per_actor_end__

doc/source/ray-core/doc_code/scheduling.py

@@ -0,0 +1,122 @@
+import ray
+
+ray.init(num_cpus=64)
+
+
+# __default_scheduling_strategy_start__
+@ray.remote
+def func():
+    return 1
+
+
+@ray.remote(num_cpus=1)
+class Actor:
+    pass
+
+
+# If unspecified, "DEFAULT" scheduling strategy is used.
+func.remote()
+actor = Actor.remote()
+# Explicitly set scheduling strategy to "DEFAULT".
+func.options(scheduling_strategy="DEFAULT").remote()
+actor = Actor.options(scheduling_strategy="DEFAULT").remote()
+
+# Zero-CPU (and no other resources) actors are randomly assigned to nodes.
+actor = Actor.options(num_cpus=0).remote()
+# __default_scheduling_strategy_end__
+
+
+# __spread_scheduling_strategy_start__
+@ray.remote(scheduling_strategy="SPREAD")
+def spread_func():
+    return 2
+
+
+@ray.remote(num_cpus=1)
+class SpreadActor:
+    pass
+
+
+# Spread tasks across the cluster.
+[spread_func.remote() for _ in range(10)]
+# Spread actors across the cluster.
+actors = [SpreadActor.options(scheduling_strategy="SPREAD").remote() for _ in range(10)]
+# __spread_scheduling_strategy_end__
+
+
+# __node_affinity_scheduling_strategy_start__
+@ray.remote
+def node_affinity_func():
+    return ray.get_runtime_context().get_node_id()
+
+
+@ray.remote(num_cpus=1)
+class NodeAffinityActor:
+    pass
+
+
+# Only run the task on the local node.
+node_affinity_func.options(
+    scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+        node_id=ray.get_runtime_context().get_node_id(),
+        soft=False,
+    )
+).remote()
+
+# Run the two node_affinity_func tasks on the same node if possible.
+node_affinity_func.options(
+    scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+        node_id=ray.get(node_affinity_func.remote()),
+        soft=True,
+    )
+).remote()
+
+# Only run the actor on the local node.
+actor = NodeAffinityActor.options(
+    scheduling_strategy=ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy(
+        node_id=ray.get_runtime_context().get_node_id(),
+        soft=False,
+    )
+).remote()
+# __node_affinity_scheduling_strategy_end__
+
+
+# __locality_aware_scheduling_start__
+@ray.remote
+def large_object_func():
+    # Large object is stored in the local object store
+    # and available in the distributed memory,
+    # instead of returning inline directly to the caller.
+    return [1] * (1024 * 1024)
+
+
+@ray.remote
+def small_object_func():
+    # Small object is returned inline directly to the caller,
+    # instead of storing in the distributed memory.
+    return [1]
+
+
+@ray.remote
+def consume_func(data):
+    return len(data)
+
+
+large_object = large_object_func.remote()
+small_object = small_object_func.remote()
+
+# Ray will try to run consume_func on the same node
+# where large_object_func runs.
+consume_func.remote(large_object)
+
+# Ray will try to spread consume_func across the entire cluster
+# instead of only running on the node where large_object_func runs.
+[
+    consume_func.options(scheduling_strategy="SPREAD").remote(large_object)
+    for i in range(10)
+]
+
+# Ray won't consider locality for scheduling consume_func
+# since the argument is small and will be sent to the worker node inline directly.
+consume_func.remote(small_object)
+# __locality_aware_scheduling_end__

doc/source/ray-core/doc_code/streaming_generator.py

@@ -0,0 +1,209 @@
+# flake8: noqa
+
+# fmt: off
+
+# __streaming_generator_define_start__
+import ray
+import time
+
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(5)
+        yield i
+
+# __streaming_generator_define_end__
+
+# __streaming_generator_execute_start__
+gen = task.remote()
+# Blocks for 5 seconds.
+ref = next(gen)
+# return 0
+ray.get(ref)
+# Blocks for 5 seconds.
+ref = next(gen)
+# Return 1
+ray.get(ref)
+
+# Returns 2~4 every 5 seconds.
+for ref in gen:
+    print(ray.get(ref))
+
+# __streaming_generator_execute_end__
+
+# __streaming_generator_exception_start__
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(1)
+        if i == 1:
+            raise ValueError
+        yield i
+
+gen = task.remote()
+# it's okay.
+ray.get(next(gen))
+
+# Raises an exception
+try:
+    ray.get(next(gen))
+except ValueError as e:
+    print(f"Exception is raised when i == 1 as expected {e}")
+
+# __streaming_generator_exception_end__
+
+# __streaming_generator_actor_model_start__
+@ray.remote
+class Actor:
+    def f(self):
+        for i in range(5):
+            yield i
+
+@ray.remote
+class AsyncActor:
+    async def f(self):
+        for i in range(5):
+            yield i
+
+@ray.remote(max_concurrency=5)
+class ThreadedActor:
+    def f(self):
+        for i in range(5):
+            yield i
+
+actor = Actor.remote()
+for ref in actor.f.remote():
+    print(ray.get(ref))
+
+actor = AsyncActor.remote()
+for ref in actor.f.remote():
+    print(ray.get(ref))
+
+actor = ThreadedActor.remote()
+for ref in actor.f.remote():
+    print(ray.get(ref))
+
+# __streaming_generator_actor_model_end__
+
+# __streaming_generator_asyncio_start__
+import asyncio
+
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(1)
+        yield i
+
+
+async def main():
+    async for ref in task.remote():
+        print(await ref)
+
+asyncio.run(main())
+
+# __streaming_generator_asyncio_end__
+
+# __streaming_generator_gc_start__
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(1)
+        yield i
+
+gen = task.remote()
+ref1 = next(gen)
+del gen
+
+# __streaming_generator_gc_end__
+
+# __streaming_generator_concurrency_asyncio_start__
+import asyncio
+
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(1)
+        yield i
+
+
+async def async_task():
+    async for ref in task.remote():
+        print(await ref)
+
+async def main():
+    t1 = async_task()
+    t2 = async_task()
+    await asyncio.gather(t1, t2)
+
+asyncio.run(main())
+# __streaming_generator_concurrency_asyncio_end__
+
+# __streaming_generator_wait_simple_start__
+@ray.remote
+def task():
+    for i in range(5):
+        time.sleep(5)
+        yield i
+
+gen = task.remote()
+
+# Because it takes 5 seconds to make the first yield,
+# with 0 timeout, the generator is unready.
+ready, unready = ray.wait([gen], timeout=0)
+print("timeout 0, nothing is ready.")
+print(ready)
+assert len(ready) == 0
+assert len(unready) == 1
+
+# Without a timeout argument, ray.wait waits until the given argument
+# is ready. When a next item is ready, it returns.
+ready, unready = ray.wait([gen])
+print("Wait for 5 seconds. The next item is ready.")
+assert len(ready) == 1
+assert len(unready) == 0
+next(gen)
+
+# Because the second yield hasn't happened yet,
+ready, unready = ray.wait([gen], timeout=0)
+print("Wait for 0 seconds. The next item is not ready.")
+print(ready, unready)
+assert len(ready) == 0
+assert len(unready) == 1
+
+# __streaming_generator_wait_simple_end__
+
+# __streaming_generator_wait_complex_start__
+from ray._raylet import ObjectRefGenerator
+
+@ray.remote
+def generator_task():
+    for i in range(5):
+        time.sleep(5)
+        yield i
+
+@ray.remote
+def regular_task():
+    for i in range(5):
+        time.sleep(5)
+    return
+
+gen = [generator_task.remote()]
+ref = [regular_task.remote()]
+ready, unready = [], [*gen, *ref]
+result = []
+
+while unready:
+    ready, unready = ray.wait(unready)
+    for r in ready:
+        if isinstance(r, ObjectRefGenerator):
+            try:
+                ref = next(r)
+                result.append(ray.get(ref))
+            except StopIteration:
+                pass
+            else:
+                unready.append(r)
+        else:
+            result.append(ray.get(r))
+
+# __streaming_generator_wait_complex_end__

doc/source/ray-core/doc_code/task_exceptions.py

@@ -0,0 +1,105 @@
+# flake8: noqa
+
+# fmt: off
+# __task_exceptions_begin__
+
+import ray
+
+@ray.remote
+def f():
+    raise Exception("the real error")
+
+@ray.remote
+def g(x):
+    return
+
+
+try:
+    ray.get(f.remote())
+except ray.exceptions.RayTaskError as e:
+    print(e)
+    # ray::f() (pid=71867, ip=XXX.XX.XXX.XX)
+    #   File "errors.py", line 5, in f
+    #     raise Exception("the real error")
+    # Exception: the real error
+
+try:
+    ray.get(g.remote(f.remote()))
+except ray.exceptions.RayTaskError as e:
+    print(e)
+    # ray::g() (pid=73085, ip=128.32.132.47)
+    #   At least one of the input arguments for this task could not be computed:
+    # ray.exceptions.RayTaskError: ray::f() (pid=73085, ip=XXX.XX.XXX.XX)
+    #   File "errors.py", line 5, in f
+    #     raise Exception("the real error")
+    # Exception: the real error
+
+# __task_exceptions_end__
+# __unserializable_exceptions_begin__
+
+import threading
+
+class UnserializableException(Exception):
+    def __init__(self):
+        self.lock = threading.Lock()
+
+@ray.remote
+def raise_unserializable_error():
+    raise UnserializableException
+
+try:
+    ray.get(raise_unserializable_error.remote())
+except ray.exceptions.RayTaskError as e:
+    print(e)
+    # ray::raise_unserializable_error() (pid=328577, ip=172.31.5.154)
+    #   File "/home/ubuntu/ray/tmp~/main.py", line 25, in raise_unserializable_error
+    #     raise UnserializableException
+    # UnserializableException
+    print(type(e.cause))
+    # <class 'ray.exceptions.RayError'>
+    print(e.cause)
+    # The original cause of the RayTaskError (<class '__main__.UnserializableException'>) isn't serializable: cannot pickle '_thread.lock' object. Overwriting the cause to a RayError.
+
+# __unserializable_exceptions_end__
+# __catch_user_exceptions_begin__
+
+class MyException(Exception):
+    ...
+
+@ray.remote
+def raises_my_exc():
+    raise MyException("a user exception")
+try:
+    ray.get(raises_my_exc.remote())
+except MyException as e:
+    print(e)
+    # ray::raises_my_exc() (pid=15329, ip=127.0.0.1)
+    #   File "<$PWD>/task_exceptions.py", line 45, in raises_my_exc
+    #     raise MyException("a user exception")
+    # MyException: a user exception
+
+# __catch_user_exceptions_end__
+# __catch_user_final_exceptions_begin__
+class MyFinalException(Exception):
+    def __init_subclass__(cls, /, *args, **kwargs):
+        raise TypeError("Cannot subclass this little exception class.")
+
+@ray.remote
+def raises_my_final_exc():
+    raise MyFinalException("a *final* user exception")
+try:
+    ray.get(raises_my_final_exc.remote())
+except ray.exceptions.RayTaskError as e:
+    assert isinstance(e.cause, MyFinalException)
+    print(e)
+    # 2024-04-08 21:11:47,417 WARNING exceptions.py:177 -- User exception type <class '__main__.MyFinalException'> in RayTaskError can not be subclassed! This exception will be raised as RayTaskError only. You can use `ray_task_error.cause` to access the user exception. Failure in subclassing: Cannot subclass this little exception class.
+    # ray::raises_my_final_exc() (pid=88226, ip=127.0.0.1)
+    # File "<$PWD>/task_exceptions.py", line 66, in raises_my_final_exc
+    #     raise MyFinalException("a *final* user exception")
+    # MyFinalException: a *final* user exception
+    print(type(e.cause))
+    # <class '__main__.MyFinalException'>
+    print(e.cause)
+    # a *final* user exception
+# __catch_user_final_exceptions_end__
+# fmt: on

doc/source/ray-core/doc_code/tasks.py

@@ -0,0 +1,137 @@
+# flake8: noqa
+
+# __tasks_start__
+import ray
+import time
+
+
+# A regular Python function.
+def normal_function():
+    return 1
+
+
+# By adding the `@ray.remote` decorator, a regular Python function
+# becomes a Ray remote function.
+@ray.remote
+def my_function():
+    return 1
+
+
+# To invoke this remote function, use the `remote` method.
+# This will immediately return an object ref (a future) and then create
+# a task that will be executed on a worker process.
+obj_ref = my_function.remote()
+
+# The result can be retrieved with ``ray.get``.
+assert ray.get(obj_ref) == 1
+
+
+@ray.remote
+def slow_function():
+    time.sleep(10)
+    return 1
+
+
+# Ray tasks are executed in parallel.
+# All computation is performed in the background, driven by Ray's internal event loop.
+for _ in range(4):
+    # This doesn't block.
+    slow_function.remote()
+# __tasks_end__
+
+# __pass_by_ref_start__
+@ray.remote
+def function_with_an_argument(value):
+    return value + 1
+
+
+obj_ref1 = my_function.remote()
+assert ray.get(obj_ref1) == 1
+
+# You can pass an object ref as an argument to another Ray task.
+obj_ref2 = function_with_an_argument.remote(obj_ref1)
+assert ray.get(obj_ref2) == 2
+# __pass_by_ref_end__
+
+# __wait_start__
+object_refs = [slow_function.remote() for _ in range(2)]
+# Return as soon as one of the tasks finished execution.
+ready_refs, remaining_refs = ray.wait(object_refs, num_returns=1, timeout=None)
+# __wait_end__
+
+# __multiple_returns_start__
+# By default, a Ray task only returns a single Object Ref.
+@ray.remote
+def return_single():
+    return 0, 1, 2
+
+
+object_ref = return_single.remote()
+assert ray.get(object_ref) == (0, 1, 2)
+
+
+# However, you can configure Ray tasks to return multiple Object Refs.
+@ray.remote(num_returns=3)
+def return_multiple():
+    return 0, 1, 2
+
+
+object_ref0, object_ref1, object_ref2 = return_multiple.remote()
+assert ray.get(object_ref0) == 0
+assert ray.get(object_ref1) == 1
+assert ray.get(object_ref2) == 2
+# __multiple_returns_end__
+
+# __generator_start__
+@ray.remote(num_returns=3)
+def return_multiple_as_generator():
+    for i in range(3):
+        yield i
+
+
+# NOTE: Similar to normal functions, these objects will not be available
+# until the full task is complete and all returns have been generated.
+a, b, c = return_multiple_as_generator.remote()
+# __generator_end__
+
+# __cancel_start__
+@ray.remote
+def blocking_operation():
+    time.sleep(10e6)
+
+
+obj_ref = blocking_operation.remote()
+ray.cancel(obj_ref)
+
+try:
+    ray.get(obj_ref)
+except ray.exceptions.TaskCancelledError:
+    print("Object reference was cancelled.")
+# __cancel_end__
+
+# __resource_start__
+# Specify required resources.
+@ray.remote(num_cpus=4, num_gpus=2)
+def my_function():
+    return 1
+
+
+# Override the default resource requirements.
+my_function.options(num_cpus=3).remote()
+# __resource_end__
+
+
+# __fraction_resource_start__
+# Ray also supports fractional resource requirements.
+@ray.remote(num_gpus=0.5)
+def h():
+    return 1
+
+
+# Ray support custom resources too.
+@ray.remote(resources={"Custom": 1})
+def f():
+    return 1
+
+
+# __fraction_resource_end__

doc/source/ray-core/doc_code/tasks_fault_tolerance.py

@@ -0,0 +1,91 @@
+# flake8: noqa
+
+# fmt: off
+# __tasks_fault_tolerance_retries_begin__
+import numpy as np
+import os
+import ray
+import time
+
+ray.init(ignore_reinit_error=True)
+
+@ray.remote(max_retries=1)
+def potentially_fail(failure_probability):
+    time.sleep(0.2)
+    if np.random.random() < failure_probability:
+        os._exit(0)
+    return 0
+
+for _ in range(3):
+    try:
+        # If this task crashes, Ray will retry it up to one additional
+        # time. If either of the attempts succeeds, the call to ray.get
+        # below will return normally. Otherwise, it will raise an
+        # exception.
+        ray.get(potentially_fail.remote(0.5))
+        print('SUCCESS')
+    except ray.exceptions.WorkerCrashedError:
+        print('FAILURE')
+# __tasks_fault_tolerance_retries_end__
+# fmt: on
+
+# fmt: off
+# __tasks_fault_tolerance_retries_exception_begin__
+import numpy as np
+import os
+import ray
+import time
+
+ray.init(ignore_reinit_error=True)
+
+class RandomError(Exception):
+    pass
+
+@ray.remote(max_retries=1, retry_exceptions=True)
+def potentially_fail(failure_probability):
+    if failure_probability < 0 or failure_probability > 1:
+        raise ValueError(
+            "failure_probability must be between 0 and 1, but got: "
+            f"{failure_probability}"
+        )
+    time.sleep(0.2)
+    if np.random.random() < failure_probability:
+        raise RandomError("Failed!")
+    return 0
+
+for _ in range(3):
+    try:
+        # If this task crashes, Ray will retry it up to one additional
+        # time. If either of the attempts succeeds, the call to ray.get
+        # below will return normally. Otherwise, it will raise an
+        # exception.
+        ray.get(potentially_fail.remote(0.5))
+        print('SUCCESS')
+    except RandomError:
+        print('FAILURE')
+
+# Provide the exceptions that we want to retry as an allowlist.
+retry_on_exception = potentially_fail.options(retry_exceptions=[RandomError])
+try:
+    # This will fail since we're passing in -1 for the failure_probability,
+    # which will raise a ValueError in the task and does not match the RandomError
+    # exception that we provided.
+    ray.get(retry_on_exception.remote(-1))
+except ValueError:
+    print("FAILED AS EXPECTED")
+else:
+    raise RuntimeError("An exception should be raised so this shouldn't be reached.")
+
+# These will retry on the RandomError exception.
+for _ in range(3):
+    try:
+        # If this task crashes, Ray will retry it up to one additional
+        # time. If either of the attempts succeeds, the call to ray.get
+        # below will return normally. Otherwise, it will raise an
+        # exception.
+        ray.get(retry_on_exception.remote(0.5))
+        print('SUCCESS')
+    except RandomError:
+        print('FAILURE AFTER RETRIES')
+# __tasks_fault_tolerance_retries_exception_end__
+# fmt: on

doc/source/ray-core/doc_code/tqdm.py

@@ -0,0 +1,14 @@
+import time
+import ray
+
+# Instead of "from tqdm import tqdm", use:
+from ray.experimental.tqdm_ray import tqdm
+
+
+@ray.remote
+def f(name):
+    for x in tqdm(range(100), desc=name):
+        time.sleep(0.1)
+
+
+ray.get([f.remote("task 1"), f.remote("task 2")])