chore: import upstream snapshot with attribution

doc/source/ray-observability/doc_code/app_logging.py

@@ -0,0 +1,22 @@
+import ray
+
+# Initiate a driver.
+ray.init()
+
+
+@ray.remote
+def task():
+    print("task")
+
+
+ray.get(task.remote())
+
+
+@ray.remote
+class Actor:
+    def ready(self):
+        print("actor")
+
+
+actor = Actor.remote()
+ray.get(actor.ready.remote())

doc/source/ray-observability/doc_code/gotchas.py

@@ -0,0 +1,34 @@
+# flake8: noqa
+# __env_var_start__
+import ray
+import os
+
+ray.init()
+
+
+@ray.remote
+def myfunc():
+    myenv = os.environ.get("FOO")
+    print(f"myenv is {myenv}")
+    return 1
+
+
+ray.get(myfunc.remote())
+# this prints: "myenv is None"
+# __env_var_end__
+ray.shutdown()
+
+# __env_var_fix_start__
+ray.init(runtime_env={"env_vars": {"FOO": "bar"}})
+
+
+@ray.remote
+def myfunc():
+    myenv = os.environ.get("FOO")
+    print(f"myenv is {myenv}")
+    return 1
+
+
+ray.get(myfunc.remote())
+# this prints: "myenv is bar"
+# __env_var_fix_end__

doc/source/ray-observability/doc_code/memray_profiling.py

@@ -0,0 +1,40 @@
+# __memray_profiling_start__
+import memray
+import ray
+
+
+@ray.remote
+class Actor:
+    def __init__(self):
+        # Every memory allocation after `__enter__` method will be tracked.
+        memray.Tracker(
+            "/tmp/ray/session_latest/logs/"
+            f"{ray.get_runtime_context().get_actor_id()}_mem_profile.bin"
+        ).__enter__()
+        self.arr = [bytearray(b"1" * 1000000)]
+
+    def append(self):
+        self.arr.append(bytearray(b"1" * 1000000))
+
+
+a = Actor.remote()
+ray.get(a.append.remote())
+# __memray_profiling_end__
+
+
+# __memray_profiling_task_start__
+import memray  # noqa
+import ray  # noqa
+
+
+@ray.remote
+def task():
+    with memray.Tracker(
+        "/tmp/ray/session_latest/logs/"
+        f"{ray.get_runtime_context().get_task_id()}_mem_profile.bin"
+    ):
+        arr = bytearray(b"1" * 1000000)  # noqa
+
+
+ray.get(task.remote())
+# __memray_profiling_task_end__

doc/source/ray-observability/doc_code/metrics_example.py

@@ -0,0 +1,61 @@
+import time
+
+import ray
+from ray.util.metrics import Counter, Gauge, Histogram
+
+ray.init(_metrics_export_port=8080)
+
+
+@ray.remote
+class MyActor:
+    def __init__(self, name):
+        self._curr_count = 0
+
+        self.counter = Counter(
+            "num_requests",
+            description="Number of requests processed by the actor.",
+            tag_keys=("actor_name",),
+        )
+        self.counter.set_default_tags({"actor_name": name})
+
+        self.gauge = Gauge(
+            "curr_count",
+            description="Current count held by the actor. Goes up and down.",
+            tag_keys=("actor_name",),
+        )
+        self.gauge.set_default_tags({"actor_name": name})
+
+        self.histogram = Histogram(
+            "request_latency",
+            description="Latencies of requests in ms.",
+            boundaries=[0.1, 1],
+            tag_keys=("actor_name",),
+        )
+        self.histogram.set_default_tags({"actor_name": name})
+
+    def process_request(self, num):
+        start = time.time()
+        self._curr_count += num
+
+        # Increment the total request count.
+        self.counter.inc()
+        # Update the gauge to the new value.
+        self.gauge.set(self._curr_count)
+        # Record the latency for this request in ms.
+        self.histogram.observe(1000 * (time.time() - start))
+
+        return self._curr_count
+
+
+print("Starting actor.")
+my_actor = MyActor.remote("my_actor")
+print("Calling actor.")
+my_actor.process_request.remote(-10)
+print("Calling actor.")
+my_actor.process_request.remote(5)
+print("Metrics should be exported.")
+print("See http://localhost:8080 (this may take a few seconds to load).")
+
+# Sleep so we can look at the metrics before exiting.
+time.sleep(30)
+print("Exiting!")

doc/source/ray-observability/doc_code/ray-distributed-debugger.py

@@ -0,0 +1,30 @@
+import ray
+import sys
+
+# Add the RAY_DEBUG_POST_MORTEM=1 environment variable
+# if you want to activate post-mortem debugging
+ray.init(
+    runtime_env={
+        "env_vars": {"RAY_DEBUG_POST_MORTEM": "1"},
+    }
+)
+
+
+@ray.remote
+def my_task(x):
+    y = x * x
+    breakpoint()  # Add a breakpoint in the Ray task.
+    return y
+
+
+@ray.remote
+def post_mortem(x):
+    x += 1
+    raise Exception("An exception is raised.")
+    return x
+
+
+if len(sys.argv) == 1:
+    ray.get(my_task.remote(10))
+else:
+    ray.get(post_mortem.remote(10))

doc/source/ray-observability/getting-started.rst

@@ -0,0 +1,432 @@
+.. _observability-getting-started:
+
+Ray Dashboard
+=============
+
+Ray provides a web-based dashboard for monitoring and debugging Ray applications.
+The visual representation of the system state, allows users to track the performance
+of applications and troubleshoot issues.
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/i33b1DYjYRQ" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+
+Set up Dashboard
+------------------
+
+To access the dashboard, use `ray[default]` or :ref:`other installation commands <installation>` that include the Ray Dashboard component. For example:
+
+.. code-block:: bash
+
+  pip install -U "ray[default]"
+
+When you start a single-node Ray Cluster on your laptop, access the dashboard with the URL that Ray prints when it initializes (the default URL is **http://localhost:8265**) or with the context object returned by `ray.init`.
+
+.. testcode::
+  :hide:
+
+  import ray
+  ray.shutdown()
+
+.. testcode::
+
+    import ray
+
+    context = ray.init()
+    print(context.dashboard_url)
+
+..
+    This test output is flaky. If Ray isn't completely shutdown, the port can be
+    "8266" instead of "8265".
+
+.. testoutput::
+    :options: +MOCK
+
+   127.0.0.1:8265
+
+.. code-block:: text
+
+  INFO worker.py:1487 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8265.
+
+.. note::
+
+    If you start Ray in a docker container, ``--dashboard-host`` is a required parameter. For example, ``ray start --head --dashboard-host=0.0.0.0``.
+
+
+
+When you start a remote Ray Cluster with the :ref:`VM Cluster Launcher <vm-cluster-quick-start>`, :ref:`KubeRay operator <kuberay-quickstart>`, or manual configuration, Ray Dashboard launches on the head node but the dashboard port may not be publicly exposed. View :ref:`configuring the dashboard <dashboard-in-browser>` for how to view Dashboard from outside the Head Node.
+
+.. note::
+
+  When using the Ray Dashboard, it is highly recommended to also set up Prometheus and Grafana.
+  They are necessary for critical features such as :ref:`Metrics View <dash-metrics-view>`.
+  See :ref:`Configuring and Managing the Dashboard <observability-visualization-setup>` for how to integrate Prometheus and Grafana with Ray Dashboard.
+
+
+Navigate the views
+------------------
+
+The Dashboard has multiple tabs called views. Depending on your goal, you may use one or a combination of views:
+
+- Analyze, monitor, or visualize status and resource utilization metrics for logical or physical components: :ref:`Metrics view <dash-metrics-view>`, :ref:`Cluster view <dash-node-view>`
+- Monitor Job and Task progress and status: :ref:`Jobs view <dash-jobs-view>`
+- Locate logs and error messages for failed Tasks and Actors: :ref:`Jobs view <dash-jobs-view>`, :ref:`Logs view <dash-logs-view>`
+- Analyze CPU and memory usage of Tasks and Actors: :ref:`Metrics view <dash-metrics-view>`,  :ref:`Cluster view <dash-node-view>`
+- Monitor a Serve application: :ref:`Serve view <dash-serve-view>`
+
+.. _dash-jobs-view:
+
+Jobs view
+---------
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/CrpXSSs0uaw" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+The Jobs view lets you monitor the different Jobs that ran on your Ray Cluster.
+A :ref:`Ray Job <jobs-overview>` is a Ray workload that uses Ray APIs (e.g., ``ray.init``). It is recommended to submit your Job to Clusters via :ref:`Ray Job API <jobs-quickstart>`. You can also interactively run Ray jobs (e.g., by executing a Python script within a Head Node).
+
+The Job view displays a list of active, finished, and failed Jobs, and clicking on an ID allows users to view detailed information about that Job.
+For more information on Ray Jobs, see the :ref:`Ray Job Overview section <jobs-overview>`.
+
+Custom names for jobs
+~~~~~~~~~~~~~~~~~~~~~
+
+The **Name** column in the Jobs view displays the value of the ``job_name`` key from the Job's metadata.
+You can set it when submitting a Job via the :ref:`Ray Job API <jobs-quickstart>`:
+
+.. code-block:: python
+
+    from ray.job_submission import JobSubmissionClient
+
+    client = JobSubmissionClient("http://127.0.0.1:8265")
+    client.submit_job(
+        entrypoint="echo hello",
+        metadata={"job_name": "my-training-job"},
+    )
+
+If ``job_name`` is not provided in the metadata, it defaults to the job ID. Job names do not need to be unique.
+
+Job Profiling
+~~~~~~~~~~~~~
+
+You can profile Ray Jobs by clicking on the “Stack Trace” or “CPU Flame Graph” actions. See :ref:`Profiling <profiling-concept>` for more details.
+
+.. _dash-workflow-job-progress:
+
+Task and Actor breakdown
+~~~~~~~~~~~~~~~~~~~~~~~~
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/advanced-progress.png
+    :align: center
+
+The Jobs view breaks down Tasks and Actors by their states.
+Tasks and Actors are grouped and nested by default. You can see the nested entries by clicking the expand button.
+
+Tasks and Actors are grouped and nested using the following criteria:
+
+- All Tasks and Actors are grouped together. View individual entries by expanding the corresponding row.
+- Tasks are grouped by their ``name`` attribute (e.g., ``task.options(name="<name_here>").remote()``).
+- Child Tasks (nested Tasks) are nested under their parent Task's row.
+- Actors are grouped by their class name.
+- Child Actors (Actors created within an Actor) are nested under their parent Actor's row.
+- Actor Tasks (remote methods within an Actor) are nested under the Actor for the corresponding Actor method.
+
+.. note::
+
+  Job detail page can only display or retrieve up to 10K Tasks per Job. For Jobs with more than 10K Tasks, the portion of Tasks that exceed the 10K limit are unaccounted. The number of unaccounted Tasks is available from the Task breakdown.
+
+
+.. _dashboard-timeline:
+
+Task Timeline
+~~~~~~~~~~~~~
+
+First, download the chrome tracing file by clicking the download button. Alternatively, you can :ref:`use CLI or SDK to export the tracing file <ray-core-timeline>`.
+
+Second, use tools like ``chrome://tracing`` or the `Perfetto UI <https://ui.perfetto.dev/>`_ and drop the downloaded chrome tracing file. We will use Perfetto as it is the recommended way to visualize chrome tracing files.
+
+In the timeline visualization of Ray Tasks and Actors, there are Node rows (hardware) and Worker rows (processes).
+Each Worker rows display a list of Task events (e.g., Task scheduled, Task running, input/output deserialization, etc.) happening from that Worker over time.
+
+
+
+Ray Status
+~~~~~~~~~~
+
+The Jobs view displays the status of the Ray Cluster. This information is the output of the ``ray status`` CLI command.
+
+The left panel shows the autoscaling status, including pending, active, and failed nodes.
+The right panel displays the resource demands, which are resources that cannot be scheduled to the Cluster at the moment. This page is useful for debugging resource deadlocks or slow scheduling.
+
+.. note::
+
+  The output shows the aggregated information across the Cluster (not by Job). If you run more than one Job, some of the demands may come from other Jobs.
+
+.. _dash-workflow-state-apis:
+
+Task, Actor, and Placement Group tables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Dashboard displays a table of the status of the Job's Tasks, Actors, and Placement Groups.
+This information is the output of the :ref:`Ray State APIs <state-api-overview-ref>`.
+
+You can expand the table to see a list of each Task, Actor, and Placement Group.
+
+.. _dash-serve-view:
+
+Serve view
+----------
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/eqXfwM641a4" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+See your general Serve configurations, a list of the Serve applications, and, if you configured :ref:`Grafana and Prometheus <observability-visualization-setup>`, high-level
+metrics of your Serve applications. Click the name of a Serve application to go to the Serve Application Detail page.
+
+Serve Application Detail page
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See the Serve application's configurations and metadata and the list of :ref:`Serve deployments and replicas <serve-key-concepts-deployment>`.
+Click the expand button of a deployment to see the replicas.
+
+Each deployment has two available actions. You can view the Deployment config and, if you configured :ref:`Grafana and Prometheus <observability-configure-manage-dashboard>`, you can open
+a Grafana dashboard with detailed metrics about that deployment.
+
+For each replica, there are two available actions. You can see the logs of that replica and, if you configured :ref:`Grafana and Prometheus <observability-visualization-setup>`, you can open
+a Grafana dashboard with detailed metrics about that replica. Click on the replica name to go to the Serve Replica Detail page.
+
+
+Serve Replica Detail page
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This page shows metadata about the Serve replica, high-level metrics about the replica if you configured :ref:`Grafana and Prometheus <observability-visualization-setup>`, and
+a history of completed :ref:`Tasks <core-key-concepts>` of that replica.
+
+
+Serve metrics
+~~~~~~~~~~~~~
+
+Ray Serve exports various time-series metrics to help you understand the status of your Serve application over time. Find more details about these metrics :ref:`here <serve-production-monitoring-metrics>`.
+To store and visualize these metrics, set up Prometheus and Grafana by following the instructions :ref:`here <observability-visualization-setup>`.
+
+These metrics are available in the Ray Dashboard in the Serve page and the Serve Replica Detail page. They are also accessible as Grafana dashboards.
+Within the Grafana dashboard, use the dropdown filters on the top to filter metrics by route, deployment, or replica. Exact descriptions
+of each graph are available by hovering over the "info" icon on the top left of each graph.
+
+
+.. _dash-node-view:
+
+Cluster view
+------------
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/K2jLoIhlsnY" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+The Cluster view is a visualization of the hierarchical relationship of
+machines (nodes) and Workers (processes). Each host machine consists of many Workers, that
+you can see by clicking the + button. See also the assignment of GPU resources to specific Actors or Tasks.
+
+Click the node ID to see the node detail page.
+
+In addition, the machine view lets you see **logs** for a node or a Worker.
+
+.. _dash-actors-view:
+
+Actors view
+-----------
+
+Use the Actors view to see the logs for an Actor and which Job created the Actor.
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/MChn6O1ecEQ" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+The information for up to 100000 dead Actors is stored.
+Override this value with the `RAY_maximum_gcs_destroyed_actor_cached_count` environment variable
+when starting Ray.
+
+Actor profiling
+~~~~~~~~~~~~~~~
+
+Run the profiler on a running Actor. See :ref:`Dashboard Profiling <dashboard-profiling>` for more details.
+
+Actor Detail page
+~~~~~~~~~~~~~~~~~
+
+Click the ID, to see the detail view of the Actor.
+
+On the Actor Detail page, see the metadata, state, and all of the Actor's Tasks that have run.
+
+.. _dash-metrics-view:
+
+Metrics view
+------------
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/yn5Q65iHAR8" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+Ray exports default metrics which are available from the :ref:`Metrics view <dash-metrics-view>`. Here are some available example metrics.
+
+- Tasks, Actors, and Placement Groups broken down by states
+- :ref:`Logical resource usage <logical-resources>` across nodes
+- Hardware resource usage across nodes
+- Autoscaler status
+
+See :ref:`System Metrics Page <system-metrics>` for available metrics.
+
+.. note::
+
+  The Metrics view requires the Prometheus and Grafana setup. See :ref:`Configuring and managing the Dashboard <observability-visualization-setup>` to learn how to set up Prometheus and Grafana.
+
+The Metrics view provides visualizations of the time series metrics emitted by Ray.
+
+You can select the time range of the metrics in the top right corner. The graphs refresh automatically every 15 seconds.
+
+There is also a convenient button to open the Grafana UI from the dashboard. The Grafana UI provides additional customizability of the charts.
+
+.. _dash-workflow-cpu-memory-analysis:
+
+Analyze the CPU and memory usage of Tasks and Actors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`Metrics view <dash-metrics-view>` in the Dashboard provides a "per-component CPU/memory usage graph" that displays CPU and memory usage over time for each Task and Actor in the application (as well as system components).
+You can identify Tasks and Actors that may be consuming more resources than expected and optimize the performance of the application.
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/node_cpu_by_comp.png
+    :align: center
+
+
+Per component CPU graph. 0.379 cores mean that it uses 40% of a single CPU core. Ray process names start with ``ray::``. ``raylet``, ``agent``, ``dashboard``, or ``gcs`` are system components.
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/node_memory_by_comp.png
+    :align: center
+
+Per component memory graph. Ray process names start with ``ray::``. ``raylet``, ``agent``, ``dashboard``, or ``gcs`` are system components.
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/cluster_page.png
+    :align: center
+
+Additionally, users can see a snapshot of hardware utilization from the :ref:`Cluster view <dash-node-view>`, which provides an overview of resource usage across the entire Ray Cluster.
+
+.. _dash-workflow-resource-utilization:
+
+View the resource utilization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ray requires users to specify the number of :ref:`resources <logical-resources>` their Tasks and Actors use through arguments such as ``num_cpus``, ``num_gpus``, ``memory``, and ``resource``.
+These values are used for scheduling, but may not always match the actual resource utilization (physical resource utilization).
+
+- See the logical and physical resource utilization over time from the :ref:`Metrics view <dash-metrics-view>`.
+- The snapshot of physical resource utilization (CPU, GPU, memory, disk, network) is also available from the :ref:`Cluster view <dash-node-view>`.
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/logical_resource.png
+    :align: center
+
+The :ref:`logical resources <logical-resources>` usage.
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/physical_resource.png
+    :align: center
+
+The physical resources (hardware) usage. Ray provides CPU, GPU, Memory, GRAM, disk, and network usage for each machine in a Cluster.
+
+.. _dash-logs-view:
+
+Logs view
+---------
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/8V187F2DsN0" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+The Logs view lists the Ray logs in your Cluster. It is organized by node and log file name. Many log links in the other pages link to this view and filter the list so the relevant logs appear.
+
+To understand the logging structure of Ray, see :ref:`logging directory and file structure <logging-directory-structure>`.
+
+
+The Logs view provides search functionality to help you find specific log messages.
+
+
+**Driver logs**
+
+If the Ray Job is submitted by the :ref:`Job API <jobs-quickstart>`, the Job logs are available from the Dashboard. The log file follows the following format: ``job-driver-<job_submission_id>.log``.
+
+.. note::
+
+  If you execute the Driver directly on the Head Node of the Ray Cluster (without using the Job API) or run with :ref:`Ray Client <ray-client-ref>`, the Driver logs are not accessible from the Dashboard. In this case, see the terminal or Jupyter Notebook output to view the Driver logs.
+
+**Task and Actor Logs (Worker logs)**
+
+Task and Actor logs are accessible from the :ref:`Task and Actor table view <dash-workflow-state-apis>`. Click the "Log" button.
+You can see the ``stdout`` and ``stderr`` logs that contain the output emitted from Tasks and Actors.
+For Actors, you can also see the system logs for the corresponding Worker process.
+
+.. note::
+
+    Logs of asynchronous Actor Tasks or threaded Actor Tasks (concurrency>1) are only available as part of the Actor logs. Follow the instruction in the Dashboard to view the Actor logs.
+
+**Task and Actor errors**
+
+You can easily identify failed Tasks or Actors by looking at the Job progress bar.
+
+The Task and Actor tables display the name of the failed Tasks or Actors, respectively. They also provide access to their corresponding log or error messages.
+
+.. _dash-overview:
+
+Overview view
+-------------
+
+.. image:: ./images/dashboard-overview.png
+    :align: center
+
+The Overview view provides a high-level status of the Ray Cluster.
+
+**Overview metrics**
+
+The Overview Metrics page provides the Cluster-level hardware utilization and autoscaling status (number of pending, active, and failed nodes).
+
+**Recent Jobs**
+
+The Recent Jobs pane provides a list of recently submitted Ray Jobs.
+
+**Serve applications**
+
+The Serve Applications pane provides a list of recently deployed Serve applications
+
+.. _dash-event:
+
+**Events view**
+
+.. image:: https://raw.githubusercontent.com/ray-project/Images/master/docs/new-dashboard-v2/dashboard-pics/event-page.png
+    :align: center
+
+The Events view displays a list of events associated with a specific type (e.g., Autoscaler or Job) in chronological order. The same information is accessible with the ``ray list cluster-events`` :ref:`(Ray state APIs)<state-api-overview-ref>` CLI commands.
+
+Two types of events are available:
+
+- Job: Events related to :ref:`Ray Jobs API <jobs-quickstart>`.
+- Autoscaler: Events related to the :ref:`Ray autoscaler <cluster-autoscaler>`.
+
+Resources
+---------
+- `Ray Summit observability talk <https://www.youtube.com/watch?v=v_JzurOkdVQ>`_
+- `Ray metrics blog <https://www.anyscale.com/blog/monitoring-and-debugging-ray-workloads-ray-metrics>`_
+- `Ray Dashboard roadmap <https://github.com/ray-project/ray/issues/30097#issuecomment-1445756658>`_
+- `Observability Training Module <https://github.com/ray-project/ray-educational-materials/blob/main/Observability/Ray_observability_part_1.ipynb>`_

doc/source/ray-observability/index.md

@@ -0,0 +1,40 @@
+(observability)=
+
+# Monitoring and Debugging
+
+```{toctree}
+:hidden:
+
+getting-started
+ray-distributed-debugger
+key-concepts
+User Guides <user-guides/index>
+Reference <reference/index>
+```
+
+This section covers how to **monitor and debug Ray applications and clusters** with Ray's Observability features.
+
+
+## What is observability
+In general, observability is a measure of how well the internal states of a system can be inferred from knowledge of its external outputs.
+
+In Ray's context, observability refers to the ability for users to observe and infer Ray applications' and Ray clusters' internal states with various external outputs, such as logs, metrics, events, etc.
+
+![what is ray's observability](./images/what-is-ray-observability.png)
+
+
+## Importance of observability
+Debugging a distributed system can be challenging due to the large scale and complexity. Good observability is important for Ray users to be able to easily monitor and debug their Ray applications and clusters.
+
+![Importance of observability](./images/importance-of-observability.png)
+
+
+## Monitoring and debugging workflow and tools
+
+Monitoring and debugging Ray applications consist of 4 major steps:
+1. Monitor the clusters and applications.
+2. Identify the surfaced problems or errors.
+3. Debug with various tools and data.
+4. Form a hypothesis, implement a fix, and validate it.
+
+The remainder of this section covers the observability tools that Ray provides to accelerate your monitoring and debugging workflow.

doc/source/ray-observability/key-concepts.rst

@@ -0,0 +1,161 @@
+.. _observability-key-concepts:
+
+Key Concepts
+============
+
+This section covers key concepts for monitoring and debugging tools and features in Ray.
+
+Dashboard (Web UI)
+------------------
+Ray provides a web-based dashboard to help users monitor and debug Ray applications and Clusters.
+
+See :ref:`Getting Started <observability-getting-started>` for more details about the Dashboard.
+
+
+Ray States
+----------
+Ray States refer to the state of various Ray entities (e.g., Actor, Task, Object, etc.). Ray 2.0 and later versions support :ref:`querying the states of entities with the CLI and Python APIs <observability-programmatic>`
+
+The following command lists all the Actors from the Cluster:
+
+.. code-block:: bash
+
+    ray list actors
+
+.. code-block:: text
+
+    ======== List: 2022-07-23 21:29:39.323925 ========
+    Stats:
+    ------------------------------
+    Total: 2
+
+    Table:
+    ------------------------------
+        ACTOR_ID                          CLASS_NAME    NAME      PID  STATE
+    0  31405554844820381c2f0f8501000000  Actor                 96956  ALIVE
+    1  f36758a9f8871a9ca993b1d201000000  Actor                 96955  ALIVE
+
+View :ref:`Monitoring with the CLI or SDK <state-api-overview-ref>` for more details.
+
+Metrics
+-------
+Ray collects and exposes the physical stats (e.g., CPU, memory, GRAM, disk, and network usage of each node),
+internal stats (e.g., number of Actors in the cluster, number of Worker failures in the Cluster),
+and custom application metrics (e.g., metrics defined by users). All stats can be exported as time series data (to Prometheus by default) and used
+to monitor the Cluster over time.
+
+View :ref:`Metrics View <dash-metrics-view>` for where to view the metrics in Ray Dashboard. View :ref:`collecting metrics <collect-metrics>` for how to collect metrics from Ray Clusters.
+
+Exceptions
+----------
+Creating a new Task or submitting an Actor Task generates an object reference. When ``ray.get`` is called on the Object Reference,
+the API raises an exception if anything goes wrong with a related Task, Actor or Object. For example,
+
+- :class:`RayTaskError <ray.exceptions.RayTaskError>` is raised when an error from user code throws an exception.
+- :class:`RayActorError <ray.exceptions.RayActorError>` is raised when an Actor is dead (by a system failure, such as a node failure, or a user-level failure, such as an exception from ``__init__`` method).
+- :class:`RuntimeEnvSetupError <ray.exceptions.RuntimeEnvSetupError>` is raised when the Actor or Task can't be started because :ref:`a runtime environment <runtime-environments>` failed to be created.
+
+See :ref:`Exceptions Reference <ray-core-exceptions>` for more details.
+
+Debugger
+--------
+Ray has a built-in debugger for debugging your distributed applications.
+Set breakpoints in Ray Tasks and Actors, and when hitting the breakpoint,
+drop into a PDB session to:
+
+- Inspect variables in that context
+- Step within a Task or Actor
+- Move up or down the stack
+
+View :ref:`Ray Debugger <ray-debugger>` for more details.
+
+.. _profiling-concept:
+
+Profiling
+---------
+Profiling is a way of analyzing the performance of an application by sampling the resource usage of it. Ray supports various profiling tools:
+
+- CPU profiling for Driver and Worker processes, including integration with :ref:`py-spy <profiling-pyspy>` and :ref:`cProfile <profiling-cprofile>`
+- Memory profiling for Driver and Worker processes with :ref:`memray <profiling-memray>`
+- GPU profiling with :ref:`Pytorch Profiler <profiling-pytorch-profiler>` and :ref:`Nsight System <profiling-nsight-profiler>`
+- Built in Task and Actor profiling tool called :ref:`Ray Timeline <profiling-timeline>`
+
+View :ref:`Profiling <profiling>` for more details. Note that this list isn't comprehensive and feel free to contribute to it if you find other useful tools.
+
+
+Tracing
+-------
+To help debug and monitor Ray applications, Ray supports distributed tracing (integration with OpenTelemetry) across Tasks and Actors.
+
+See :ref:`Ray Tracing <ray-tracing>` for more details.
+
+Application logs
+----------------
+Logs are important for general monitoring and debugging. For distributed Ray applications, logs are even more important but more complicated at the same time. A Ray application runs both on Driver and Worker processes (or even across multiple machines) and the logs of these processes are the main sources of application logs.
+
+.. image:: ./images/application-logging.png
+    :alt: Application logging
+
+Driver logs
+~~~~~~~~~~~
+An entry point of Ray applications that calls ``ray.init()`` is called a **Driver**.
+All the Driver logs are handled in the same way as normal Python programs.
+
+.. _ray-worker-logs:
+
+Worker logs (stdout and stderr)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Ray executes Tasks or Actors remotely within Ray's Worker processes. Task and Actor logs are captured in the Worker stdout and stderr.
+
+Ray has special support to improve the visibility of stdout and stderr produced by Worker processes so that the Ray program appears like a non-distributed program, also known as "Worker log redirection to driver".
+
+- Ray directs stdout and stderr from all Tasks and Actors to the Worker log files, including any log messages generated by the Worker. See :ref:`logging directory and file structure <logging-directory-structure>` to understand the Ray logging structure.
+- The Driver reads the Worker log files (where the stdout and stderr of all Tasks and Actors sit) and sends the log records to its own stdout and stderr (also known as "Worker logs being redirected to Driver output").
+
+For the following code:
+
+.. testcode::
+
+    import ray
+    # Initiate a driver.
+    ray.init()
+
+    @ray.remote
+    def task_foo():
+        print("task!")
+
+    ray.get(task_foo.remote())
+
+.. testoutput::
+    :options: +MOCK
+
+    (task_foo pid=12854) task!
+
+#. Ray Task ``task_foo`` runs on a Ray Worker process. String ``task!`` is saved into the corresponding Worker ``stdout`` log file.
+#. The Driver reads the Worker log file and sends it to its ``stdout`` (terminal) where you should be able to see the string ``task!``.
+
+When logs are printed, the process id (pid) and an IP address of the node that executes Tasks or Actors are printed together. Here is the output:
+
+.. code-block:: bash
+
+    (pid=45601) task!
+
+Actor log messages look like the following by default:
+
+.. code-block:: bash
+
+    (MyActor pid=480956) actor log message
+
+
+By default, all stdout and stderr of Tasks and Actors are redirected to the Driver output. View :ref:`Configuring Logging <log-redirection-to-driver>` for how to disable this feature.
+
+
+
+Job logs
+~~~~~~~~
+Ray applications are usually run as Ray Jobs. Worker logs of Ray Jobs are always captured in the :ref:`Ray logging directory <logging-directory-structure>` while Driver logs are not.
+
+Driver logs are captured only for Ray Jobs submitted via :ref:`Jobs API <jobs-quickstart>`. Find the captured Driver logs with the Dashboard UI, CLI (using the ``ray job logs`` :ref:`CLI command <ray-job-logs-doc>`), or the :ref:`Python SDK <ray-job-submission-sdk-ref>` (``JobSubmissionClient.get_logs()`` or ``JobSubmissionClient.tail_job_logs()``).
+
+.. note::
+   View the Driver logs in your terminal or Jupyter Notebooks if you run Ray Jobs by executing the Ray Driver on the Head node directly or connecting via Ray Client.

doc/source/ray-observability/ray-distributed-debugger.rst

@@ -0,0 +1,222 @@
+.. _ray-distributed-debugger:
+
+Ray Distributed Debugger
+========================
+
+The Ray Distributed Debugger includes a debugger backend and a `VS Code extension <https://www.anyscale.com/blog/ray-distributed-debugger?utm_source=ray_docs&utm_medium=docs&utm_campaign=promotion#download-for-free>`_ frontend that streamline the debugging process with an interactive debugging experience. The Ray Debugger enables you to:
+
+- **Break into remote tasks**: Set a breakpoint in any remote task. A breakpoint pauses execution and allows you to connect with VS Code for debugging.
+- **Post-mortem debugging**: When Ray tasks fail with unhandled exceptions, Ray automatically freezes the failing task and waits for the Ray Debugger to attach, allowing you to inspect the state of the program at the time of the error.
+
+Ray Distributed Debugger abstracts the complexities of debugging distributed systems for you to debug Ray applications more efficiently, saving time and effort in the development workflow.
+
+.. note::
+
+    The Ray Distributed Debugger frontend is only available in VS Code and other VS Code-compatible IDEs like Cursor. If you need support for other IDEs, file a feature request on `GitHub <https://github.com/ray-project/ray/issues>`_.
+
+
+.. raw:: html
+
+    <div style="position: relative; height: 0; overflow: hidden; max-width: 100%; height: auto;">
+        <iframe width="560" height="315" src="https://www.youtube.com/embed/EiGHHUXL0oI" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+    </div>
+
+
+Set up the environment
+~~~~~~~~~~~~~~~~~~~~~~
+
+Create a new virtual environment and install dependencies.
+
+.. testcode::
+    :skipif: True
+
+    conda create -n myenv python=3.10
+    conda activate myenv
+    pip install "ray[default]" debugpy
+
+
+Start a Ray cluster
+~~~~~~~~~~~~~~~~~~~
+
+.. tab-set::
+
+  .. tab-item:: Local
+
+    Run `ray start --head` to start a local Ray cluster.
+
+  .. tab-item:: KubeRay (SSH)
+
+    Follow the instructions in :doc:`the RayCluster quickstart <../cluster/kubernetes/getting-started/raycluster-quick-start>` to set up a cluster.
+    You need to connect VS Code to the cluster. For example, add the following to the `ray-head` container and make sure `sshd` is running in the `ray-head` container.
+
+    .. code-block:: yaml
+
+        ports:
+        - containerPort: 22
+          name: ssd
+
+
+
+    .. note::
+
+        How to run `sshd` in the `ray-head` container depends on your setup. For example you can use `supervisord`.
+        A simple way to run `sshd` interactively for testing is by logging into the head node pod and running:
+
+        .. code-block:: bash
+
+            sudo apt-get update && sudo apt-get install -y openssh-server
+            sudo mkdir -p /run/sshd
+            sudo /usr/sbin/sshd -D
+
+    You can then connect to the cluster via SSH by running:
+
+    .. code-block:: bash
+
+        kubectl port-forward service/raycluster-sample-head-svc 2222:22
+
+    After checking that `ssh -p 2222 ray@localhost` works, set up VS Code as described in the
+    `VS Code SSH documentation <https://code.visualstudio.com/docs/remote/ssh>`_.
+
+  .. tab-item:: KubeRay (Code Server, Community Maintained)
+
+    Follow the instructions in :doc:`the RayCluster quickstart <../cluster/kubernetes/getting-started/raycluster-quick-start>` to set up a cluster.
+    
+    A simpler approach is to run a browser-based VS Code (Code Server) as a sidecar container in the Ray head pod. This eliminates network connectivity issues by placing VS Code inside the Kubernetes cluster.
+
+    Add a sidecar container to the Ray head pod and configure a shared volume. Modify your Ray head pod template with the following additions:
+    
+    .. code-block:: yaml
+
+        # In your RayCluster YAML, under spec.headGroupSpec.template.spec
+        containers:
+          - name: ray-head
+            # ... your existing ray-head configuration ...
+            # Add this volumeMount:
+            volumeMounts:
+              - mountPath: /tmp/ray
+                name: shared-ray-volume
+          # Add this sidecar container:
+          - name: vscode-debugger
+            image: docker.io/onesizefitsquorum/code-server-with-ray-distributed-debugger:4.101.2
+            ports:
+              - containerPort: 8443
+            volumeMounts:
+              - mountPath: /tmp/ray
+                name: shared-ray-volume
+            env:
+              # Specifies the default directory that opens when VSCode Web starts, pointing to the workspace containing the Ray runtime resources.
+              - name: DEFAULT_WORKSPACE
+                value: "/tmp/ray/session_latest/runtime_resources"
+        # Add this volume at the same level as `containers`:
+        volumes:
+          - name: shared-ray-volume
+            emptyDir: {}
+
+    After the Ray cluster is running, forward the Code Server port:
+
+    .. code-block:: bash
+
+        kubectl port-forward pod/<ray-head-pod-name> 8443:8443
+
+    Access VS Code in your browser at http://127.0.0.1:8443 and use the Ray Distributed Debugger extension to connect to http://127.0.0.1:8265.
+
+    For more details, see the `Code Server with Ray Distributed Debugger <https://github.com/OneSizeFitsQuorum/Code-Server-With-Ray-Distributed-Debugger/blob/main/README.en.md>`_ project.
+
+
+Register the cluster
+~~~~~~~~~~~~~~~~~~~~
+
+Find and click the Ray extension in the VS Code left side nav. Add the Ray cluster `IP:PORT` to the cluster list. The default `IP:PORT` is `127.0.0.1:8265`. You can change it when you start the cluster. Make sure your current machine can access the IP and port.
+
+.. image:: ./images/register-cluster.gif
+    :align: center
+
+
+Create a Ray task
+~~~~~~~~~~~~~~~~~
+
+Create a file `job.py` with the following snippet. Add `breakpoint()` in the Ray task. If you want to use the post-mortem debugging below, also add the `RAY_DEBUG_POST_MORTEM=1` environment variable.
+
+.. literalinclude:: ./doc_code/ray-distributed-debugger.py
+    :language: python
+
+Run your Ray app
+~~~~~~~~~~~~~~~~
+
+Start running your Ray app.
+
+.. code-block:: bash
+
+    python job.py
+
+
+Attach to the paused task
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the debugger hits a breakpoint:
+
+- The task enters a paused state.
+- The terminal clearly indicates when the debugger pauses a task and waits for the debugger to attach.
+- The paused task is listed in the Ray Debugger extension.
+- Click the play icon next to the name of the paused task to attach the VS Code debugger.
+
+.. image:: ./images/attach-paused-task.gif
+    :align: center
+
+
+Start and stop debugging
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Debug your Ray app as you would when developing locally. After you're done debugging this particular
+breakpoint, click the **Disconnect** button in the debugging toolbar so you can join another task
+in the **Paused Tasks** list.
+
+.. figure:: ./images/debugger-disconnect.gif
+
+
+Post-mortem debugging
+=====================
+
+Use post-mortem debugging when Ray tasks encounter unhandled exceptions. In such cases, Ray automatically freezes the failing task, awaiting attachment by the Ray Debugger. This feature allows you to thoroughly investigate and inspect the program's state at the time of the error.
+
+Run a Ray task raised exception
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run the same `job.py` file with an additional argument to raise an exception.
+
+.. code-block:: bash
+
+    python job.py raise-exception
+
+
+Attach to the paused task
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the app throws an exception:
+
+- The debugger freezes the task.
+- The terminal clearly indicates when the debugger pauses a task and waits for the debugger to attach.
+- The paused task is listed in the Ray Debugger extension.
+- Click the play icon next to the name of the paused task to attach the debugger and start debugging.
+
+.. image:: ./images/post-mortem.gif
+    :align: center
+
+
+Start debugging
+~~~~~~~~~~~~~~~
+
+Debug your Ray app as you would when developing locally.
+
+
+Share feedback
+==============
+
+Join the `#ray-debugger <https://ray-distributed.slack.com/archives/C073MPGLAC9>`_ channel on the Ray Slack channel to get help.
+
+
+Next steps
+==========
+
+- For guidance on debugging distributed apps in Ray, see :doc:`General debugging <./user-guides/debug-apps/general-debugging>`.
+- For tips on using the Ray debugger, see :doc:`Ray debugging <./user-guides/debug-apps/ray-debugging>`.

doc/source/ray-observability/reference/api.rst

@@ -0,0 +1,103 @@
+.. _state-api-ref:
+
+State API
+=========
+
+.. note::
+
+    APIs are :ref:`alpha <api-stability-alpha>`. This feature requires a full installation of Ray using ``pip install "ray[default]"``.
+
+For an overview with examples see :ref:`Monitoring Ray States <state-api-overview-ref>`.
+
+For the CLI reference see :ref:`Ray State CLI Reference <state-api-cli-ref>` or :ref:`Ray Log CLI Reference <ray-logs-api-cli-ref>`.
+
+State Python SDK
+-----------------
+
+State APIs are also exported as functions.
+
+Summary APIs
+~~~~~~~~~~~~
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   ray.util.state.summarize_actors
+   ray.util.state.summarize_objects
+   ray.util.state.summarize_tasks
+
+List APIs
+~~~~~~~~~~
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+    ray.util.state.list_actors
+    ray.util.state.list_placement_groups
+    ray.util.state.list_nodes
+    ray.util.state.list_jobs
+    ray.util.state.list_workers
+    ray.util.state.list_tasks
+    ray.util.state.list_objects
+    ray.util.state.list_runtime_envs
+
+Get APIs
+~~~~~~~~~
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+    ray.util.state.get_actor
+    ray.util.state.get_placement_group
+    ray.util.state.get_node
+    ray.util.state.get_worker
+    ray.util.state.get_task
+    ray.util.state.get_objects
+
+Log APIs
+~~~~~~~~
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+    ray.util.state.list_logs
+    ray.util.state.get_log
+
+.. _state-api-schema:
+
+State APIs Schema
+-----------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+    ray.util.state.common.ActorState
+    ray.util.state.common.TaskState
+    ray.util.state.common.NodeState
+    ray.util.state.common.PlacementGroupState
+    ray.util.state.common.WorkerState
+    ray.util.state.common.ObjectState
+    ray.util.state.common.RuntimeEnvState
+    ray.util.state.common.JobState
+    ray.util.state.common.StateSummary
+    ray.util.state.common.TaskSummaries
+    ray.util.state.common.TaskSummaryPerFuncOrClassName
+    ray.util.state.common.ActorSummaries
+    ray.util.state.common.ActorSummaryPerClass
+    ray.util.state.common.ObjectSummaries
+    ray.util.state.common.ObjectSummaryPerKey
+
+State APIs Exceptions
+---------------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+    ray.util.state.exception.RayStateApiException

doc/source/ray-observability/reference/cli.rst

@@ -0,0 +1,45 @@
+.. _state-api-cli-ref:
+
+State CLI
+=========
+
+State
+-----
+This section contains commands to access the :ref:`live state of Ray resources (actor, task, object, etc.) <state-api-overview-ref>`.
+
+.. note::
+
+    APIs are :ref:`alpha <api-stability-alpha>`. This feature requires a full installation of Ray using ``pip install "ray[default]"``. This feature also requires the dashboard component to be available. The dashboard component needs to be included when starting the ray cluster, which is the default behavior for ``ray start`` and ``ray.init()``. For more in-depth debugging, you could check the dashboard log at ``<RAY_LOG_DIR>/dashboard.log``, which is usually ``/tmp/ray/session_latest/logs/dashboard.log``.
+
+State CLI allows users to access the state of various resources (e.g., actor, task, object).
+
+.. click:: ray.util.state.state_cli:task_summary
+   :prog: ray summary tasks
+
+.. click:: ray.util.state.state_cli:actor_summary
+   :prog: ray summary actors
+
+.. click:: ray.util.state.state_cli:object_summary
+   :prog: ray summary objects
+
+.. click:: ray.util.state.state_cli:ray_list
+   :prog: ray list
+
+.. click:: ray.util.state.state_cli:ray_get
+   :prog: ray get
+
+.. _ray-logs-api-cli-ref:
+
+Log
+---
+This section contains commands to :ref:`access logs <state-api-log-doc>` from Ray clusters.
+
+.. note::
+
+    APIs are :ref:`alpha <api-stability-alpha>`. This feature requires a full installation of Ray using ``pip install "ray[default]"``.
+
+Log CLI allows users to access the log from the cluster.
+Note that only the logs from alive nodes are available through this API.
+
+.. click:: ray.util.state.state_cli:logs_state_cli_group
+   :prog: ray logs

doc/source/ray-observability/reference/index.md

@@ -0,0 +1,18 @@
+(observability-reference)=
+
+# Reference
+
+```{toctree}
+:hidden:
+
+api
+cli
+system-metrics
+```
+
+Monitor and debug your Ray applications and clusters using the API and CLI documented in these references.
+
+The guides include:
+* {ref}`state-api-ref`
+* {ref}`state-api-cli-ref`
+* {ref}`system-metrics`

doc/source/ray-observability/reference/system-metrics.rst

@@ -0,0 +1,148 @@
+.. _system-metrics:
+
+System Metrics
+--------------
+Ray exports a number of system metrics, which provide introspection into the state of Ray workloads, as well as hardware utilization statistics. The following table describes the officially supported metrics:
+
+.. note::
+
+  Certain labels are common across all metrics, such as `SessionName` (uniquely identifies a Ray cluster instance), `instance` (per-node label applied by Prometheus), and `JobId` (Ray job ID, as applicable).
+
+  Starting with Ray 2.53+, the `WorkerId` label is no longer exported by default due to its high cardinality. 
+  The Ray team doesn't expect this to be a breaking change, as none of Ray’s built-in components rely on this label. 
+  However, if you have custom tooling that depends on `WorkerId` label, take note of this change.
+
+  You can restore or adjust label behavior using the environment variable `RAY_metric_cardinality_level`:
+
+  - `legacy`: Preserve all labels. (This was the default behavior before Ray 2.53.)
+  - `recommended`: Drop high-cardinality labels. Ray internally determines specific labels; currently this includes only `WorkerId`. (This is the default behavior since Ray 2.53.)
+  - `low`: Same as `recommended`, but also drops the Name label for tasks and actors.
+
+.. list-table:: Ray System Metrics
+   :header-rows: 1
+
+   * - Prometheus Metric
+     - Labels
+     - Description
+   * - `ray_tasks`
+     - `Name`, `State`, `IsRetry`
+     - Current number of tasks (both remote functions and actor calls) by state. The State label (e.g., RUNNING, FINISHED, FAILED) describes the state of the task. See `rpc::TaskState <https://github.com/ray-project/ray/blob/e85355b9b593742b4f5cb72cab92051980fa73d3/src/ray/protobuf/common.proto#L583>`_ for more information. The function/method name is available as the Name label. If the task was retried due to failure or reconstruction, the IsRetry label will be set to "1", otherwise "0".
+   * - `ray_actors`
+     - `Name`, `State`
+     - Current number of actors in each state described in `rpc::ActorTableData::ActorState <https://github.com/ray-project/ray/blob/b3799a53dcabd8d1a4d20f22faa98e781b0059c7/src/ray/protobuf/gcs.proto#L79>`. ALIVE has two sub-states: ALIVE_IDLE, and ALIVE_RUNNING_TASKS. An actor is considered ALIVE_IDLE if it is not running any tasks.
+   * - `ray_resources`
+     - `Name`, `State`, `instance`
+     - Logical resource usage for each node of the cluster. Each resource has some quantity that's either in the USED or AVAILABLE state. The Name label defines the resource name (e.g., CPU, GPU).
+   * - `ray_object_store_memory`
+     - `Location`, `ObjectState`, `instance`
+     - Object store memory usage in bytes, broken down by logical Location (SPILLED, MMAP_DISK, MMAP_SHM, and WORKER_HEAP). Definitions are as follows. SPILLED--Objects that have spilled to disk or a remote Storage solution (for example, AWS S3). The default is the disk. MMAP_DISK--Objects stored on a memory-mapped page on disk. This mode very slow and only happens under severe memory pressure. MMAP_SHM--Objects store on a memory-mapped page in Shared Memory. This mode is the default, in the absence of memory pressure. WORKER_HEAP--Objects, usually smaller, stored in the memory of the Ray Worker process itself. Small objects are stored in the worker heap.
+   * - `ray_placement_groups`
+     - `State`
+     - Current number of placement groups by state. The State label (e.g., PENDING, CREATED, REMOVED) describes the state of the placement group. See `rpc::PlacementGroupTable <https://github.com/ray-project/ray/blob/e85355b9b593742b4f5cb72cab92051980fa73d3/src/ray/protobuf/gcs.proto#L517>`_ for more information.
+   * - `ray_memory_manager_worker_eviction_total`
+     - `Type`, `Name`
+     - The number of tasks and actors killed by the Ray Out of Memory killer (https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html) broken down by types (whether it is tasks or actors) and names (name of tasks and actors).
+   * - `ray_node_cpu_utilization`
+     - `instance`
+     - The CPU utilization per node as a percentage quantity (0..100). This should be scaled by the number of cores per node to convert the units into cores.
+   * - `ray_node_cpu_count`
+     - `instance`
+     - The number of CPU cores per node.
+   * - `ray_node_gpus_utilization`
+     - `instance`, `GpuDeviceName`, `GpuIndex`
+     - The GPU utilization per GPU as a percentage quantity (0..NGPU*100). `GpuDeviceName` is a name of a GPU device (e.g., NVIDIA A10G) and `GpuIndex` is the index of the GPU.
+   * - `ray_node_disk_usage`
+     - `instance`
+     - The amount of disk space used per node, in bytes.
+   * - `ray_node_disk_free`
+     - `instance`
+     - The amount of disk space available per node, in bytes.
+   * - `ray_node_disk_write_iops`
+     - `instance`, `node_type`
+     - The disk write operations per second per node.
+   * - `ray_node_disk_io_write_speed`
+     - `instance`
+     - The disk write throughput per node, in bytes per second.
+   * - `ray_node_disk_read_iops`
+     - `instance`, `node_type`
+     - The disk read operations per second per node.
+   * - `ray_node_disk_io_read_speed`
+     - `instance`
+     - The disk read throughput per node, in bytes per second.
+   * - `ray_node_mem_available`
+     - `instance`, `node_type`
+     - The amount of physical memory available per node, in bytes.
+   * - `ray_node_mem_shared_bytes`
+     - `instance`, `node_type`
+     - The amount of shared memory per node, in bytes.
+   * - `ray_node_mem_used`
+     - `instance`
+     - The amount of physical memory used per node, in bytes.
+   * - `ray_node_mem_total`
+     - `instance`
+     - The amount of physical memory available per node, in bytes.
+   * - `ray_node_mem_used_host`
+     - `instance`
+     - The host (OS-level) physical memory used per node, in bytes.
+   * - `ray_node_mem_total_host`
+     - `instance`
+     - The host (OS-level) total physical memory per node, in bytes.
+   * - `ray_node_cgroup_mem_used`
+     - `instance`
+     - The container memory usage per node (from cgroup), in bytes. Only emitted when cgroup memory limits are present.
+   * - `ray_node_cgroup_mem_total`
+     - `instance`
+     - The container memory limit per node (from cgroup), in bytes. Only emitted when cgroup memory limits are present.
+   
+   * - `ray_component_rss_mb`
+     - `Component`, `instance`
+     - Note: This metric will be deprecated in the future, please use `ray_component_rss_bytes` instead. The measured resident set size in megabytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_component_rss_bytes`
+     - `Component`, `instance`
+     - The measured resident set size in bytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_component_shared_bytes`
+     - `Component`, `instance`
+     - The measured shared memory in bytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_component_uss_mb`
+     - `Component`, `instance`
+     - Note: This metric will be deprecated in the future, please use `ray_component_uss_bytes` instead. The measured unique set size in megabytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_component_uss_bytes`
+     - `Component`, `instance`
+     - The measured unique set size in bytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_component_cpu_percentage`
+     - `Component`, `instance`
+     - The measured CPU percentage, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.
+   * - `ray_node_gram_available`
+     - `instance`, `node_type`, `GpuIndex`, `GpuDeviceName`
+     - The amount of GPU memory available per GPU, in megabytes.
+   * - `ray_node_gram_used`
+     - `instance`, `GpuDeviceName`, `GpuIndex`
+     - The amount of GPU memory used per GPU, in bytes.
+   * - `ray_node_network_received`
+     - `instance`, `node_type`
+     - The total network traffic received per node, in bytes.
+   * - `ray_node_network_sent`
+     - `instance`, `node_type`
+     - The total network traffic sent per node, in bytes.
+   * - `ray_node_network_receive_speed`
+     - `instance`
+     - The network receive throughput per node, in bytes per second.
+   * - `ray_node_network_send_speed`
+     - `instance`
+     - The network send throughput per node, in bytes per second.
+   * - `ray_cluster_active_nodes`
+     - `node_type`
+     - The number of healthy nodes in the cluster, broken down by autoscaler node type.
+   * - `ray_cluster_failed_nodes`
+     - `node_type`
+     - The number of failed nodes reported by the autoscaler, broken down by node type.
+   * - `ray_cluster_pending_nodes`
+     - `node_type`
+     - The number of pending nodes reported by the autoscaler, broken down by node type.
+
+Metrics Semantics and Consistency
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ray guarantees all its internal state metrics are *eventually* consistent even in the presence of failures--- should any worker fail, eventually the right state will be reflected in the Prometheus time-series output. However, any particular metrics query is not guaranteed to reflect an exact snapshot of the cluster state.
+
+For the `ray_tasks` and `ray_actors` metrics, you should use sum queries to plot their outputs (e.g., ``sum(ray_tasks) by (Name, State)``). The reason for this is that Ray's task metrics are emitted from multiple distributed components. Hence, there are multiple metric points, including negative metric points, emitted from different processes that must be summed to produce the correct logical view of the distributed system. For example, for a single task submitted and executed, Ray may emit  ``(submitter) SUBMITTED_TO_WORKER: 1, (executor) SUBMITTED_TO_WORKER: -1, (executor) RUNNING: 1``, which reduces to ``SUBMITTED_TO_WORKER: 0, RUNNING: 1`` after summation.

doc/source/ray-observability/user-guides/add-app-metrics.rst

@@ -0,0 +1,33 @@
+.. _application-level-metrics:
+
+Adding Application-Level Metrics
+--------------------------------
+
+Ray provides a convenient API in :ref:`ray.util.metrics <custom-metric-api-ref>` for defining and exporting custom metrics for visibility into your applications.
+Three metrics are supported: Counter, Gauge, and Histogram.
+These metrics correspond to the same `Prometheus metric types <https://prometheus.io/docs/concepts/metric_types/>`_.
+Below is a simple example of an Actor that exports metrics using these APIs:
+
+.. literalinclude:: ../doc_code/metrics_example.py
+   :language: python
+
+While the script is running, the metrics are exported to ``localhost:8080`` (this is the endpoint that Prometheus would be configured to scrape).
+Open this in the browser. You should see the following output:
+
+.. code-block:: none
+
+  # HELP ray_request_latency Latencies of requests in ms.
+  # TYPE ray_request_latency histogram
+  ray_request_latency_bucket{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor",le="0.1"} 2.0
+  ray_request_latency_bucket{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor",le="1.0"} 2.0
+  ray_request_latency_bucket{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor",le="+Inf"} 2.0
+  ray_request_latency_count{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor"} 2.0
+  ray_request_latency_sum{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor"} 0.11992454528808594
+  # HELP ray_curr_count Current count held by the actor. Goes up and down.
+  # TYPE ray_curr_count gauge
+  ray_curr_count{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor"} -15.0
+  # HELP ray_num_requests_total Number of requests processed by the actor.
+  # TYPE ray_num_requests_total counter
+  ray_num_requests_total{Component="core_worker",Version="3.0.0.dev0",actor_name="my_actor"} 2.0
+
+Please see :ref:`ray.util.metrics <custom-metric-api-ref>` for more details.

doc/source/ray-observability/user-guides/cli-sdk.rst

@@ -0,0 +1,780 @@
+.. _observability-programmatic:
+
+Monitoring with the CLI or SDK
+===============================
+
+Monitoring and debugging capabilities in Ray are available through a CLI or SDK.
+
+
+CLI command ``ray status``
+----------------------------
+You can monitor node status and resource usage by running the CLI command, ``ray status``, on the head node. It displays
+
+- **Node Status**: Nodes that are running and autoscaling up or down. Addresses of running nodes. Information about pending nodes and failed nodes.
+- **Resource Usage**: The Ray resource usage of the cluster. For example, requested CPUs from all Ray Tasks and Actors. Number of GPUs that are used.
+
+Following is an example output:
+
+.. code-block:: shell
+
+   $ ray status
+   ======== Autoscaler status: 2021-10-12 13:10:21.035674 ========
+   Node status
+   ---------------------------------------------------------------
+   Healthy:
+    1 ray.head.default
+    2 ray.worker.cpu
+   Pending:
+    (no pending nodes)
+   Recent failures:
+    (no failures)
+
+   Resources
+   ---------------------------------------------------------------
+   Usage:
+    0.0/10.0 CPU
+    0.00/70.437 GiB memory
+    0.00/10.306 GiB object_store_memory
+
+   Demands:
+    (no resource demands)
+
+When you need more verbose info about each node, run ``ray status -v``. This is helpful when you need to investigate why particular nodes don't autoscale down.
+
+
+.. _state-api-overview-ref:
+
+Ray State CLI and SDK
+----------------------------
+
+.. tip:: Provide feedback on using Ray state APIs - `feedback form <https://forms.gle/gh77mwjEskjhN8G46>`_!
+
+Use Ray State APIs to access the current state (snapshot) of Ray through the CLI or Python SDK (developer APIs).
+
+.. note::
+
+    This feature requires a full installation of Ray using ``pip install "ray[default]"``. This feature also requires that the dashboard component is available. The dashboard component needs to be included when starting the Ray Cluster, which is the default behavior for ``ray start`` and ``ray.init()``.
+
+.. note::
+
+    State API CLI commands are :ref:`stable <api-stability-stable>`, while Python SDKs are :ref:`DeveloperAPI <developer-api-def>`. CLI usage is recommended over Python SDKs.
+
+
+Get started
+~~~~~~~~~~~
+
+This example uses the following script that runs two Tasks and creates two Actors.
+
+.. testcode::
+    :hide:
+
+    import ray
+
+    ray.shutdown()
+
+.. testcode::
+
+    import ray
+    import time
+
+    ray.init(num_cpus=4)
+
+    @ray.remote
+    def task_running_300_seconds():
+        time.sleep(300)
+
+    @ray.remote
+    class Actor:
+        def __init__(self):
+            pass
+
+    # Create 2 tasks
+    tasks = [task_running_300_seconds.remote() for _ in range(2)]
+
+    # Create 2 actors
+    actors = [Actor.remote() for _ in range(2)]
+
+.. testcode::
+    :hide:
+
+    # Wait for the tasks to be submitted.
+    time.sleep(2)
+
+See the summarized states of tasks. If it doesn't return the output immediately, retry the command.
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray summary tasks
+
+        .. code-block:: text
+
+            ======== Tasks Summary: 2022-07-22 08:54:38.332537 ========
+            Stats:
+            ------------------------------------
+            total_actor_scheduled: 2
+            total_actor_tasks: 0
+            total_tasks: 2
+
+
+            Table (group by func_name):
+            ------------------------------------
+                FUNC_OR_CLASS_NAME        STATE_COUNTS    TYPE
+            0   task_running_300_seconds  RUNNING: 2      NORMAL_TASK
+            1   Actor.__init__            FINISHED: 2     ACTOR_CREATION_TASK
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import summarize_tasks
+            print(summarize_tasks())
+
+        .. testoutput::
+
+            {'cluster': {'summary': {'task_running_300_seconds': {'func_or_class_name': 'task_running_300_seconds', 'type': 'NORMAL_TASK', 'state_counts': {'RUNNING': 2}}, 'Actor.__init__': {'func_or_class_name': 'Actor.__init__', 'type': 'ACTOR_CREATION_TASK', 'state_counts': {'FINISHED': 2}}}, 'total_tasks': 2, 'total_actor_tasks': 0, 'total_actor_scheduled': 2, 'summary_by': 'func_name'}}
+
+List all Actors.
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list actors
+
+        .. code-block:: text
+
+            ======== List: 2022-07-23 21:29:39.323925 ========
+            Stats:
+            ------------------------------
+            Total: 2
+
+            Table:
+            ------------------------------
+                ACTOR_ID                          CLASS_NAME    NAME      PID  STATE
+            0  31405554844820381c2f0f8501000000  Actor                 96956  ALIVE
+            1  f36758a9f8871a9ca993b1d201000000  Actor                 96955  ALIVE
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_actors
+            print(list_actors())
+
+        .. testoutput::
+
+            [ActorState(actor_id='...', class_name='Actor', state='ALIVE', job_id='01000000', name='', node_id='...', pid=..., ray_namespace='...', serialized_runtime_env=None, required_resources=None, death_cause=None, is_detached=None, placement_group_id=None, repr_name=None), ActorState(actor_id='...', class_name='Actor', state='ALIVE', job_id='01000000', name='', node_id='...', pid=..., ray_namespace='...', serialized_runtime_env=None, required_resources=None, death_cause=None, is_detached=None, placement_group_id=None, repr_name=None)]
+
+
+Get the state of a single Task using the get API.
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            # In this case, 31405554844820381c2f0f8501000000
+            ray get actors <ACTOR_ID>
+
+        .. code-block:: text
+
+            ---
+            actor_id: 31405554844820381c2f0f8501000000
+            class_name: Actor
+            death_cause: null
+            is_detached: false
+            name: ''
+            pid: 96956
+            resource_mapping: []
+            serialized_runtime_env: '{}'
+            state: ALIVE
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_actor
+            # In this case, 31405554844820381c2f0f8501000000
+            print(get_actor(id=<ACTOR_ID>))
+
+Access logs through the ``ray logs`` API.
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list actors
+            # In this case, ACTOR_ID is 31405554844820381c2f0f8501000000
+            ray logs actor --id <ACTOR_ID>
+
+        .. code-block:: text
+
+            --- Log has been truncated to last 1000 lines. Use `--tail` flag to toggle. ---
+
+            :actor_name:Actor
+            Actor created
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_log
+
+            # In this case, ACTOR_ID is 31405554844820381c2f0f8501000000
+            for line in get_log(actor_id=<ACTOR_ID>):
+                print(line)
+
+Key Concepts
+~~~~~~~~~~~~~
+Ray State APIs allow you to access **states** of **resources** through **summary**, **list**, and **get** APIs. It also supports **logs** API to access logs.
+
+- **states**: The state of the cluster of corresponding resources. States consist of immutable metadata (e.g., Actor's name) and mutable states (e.g., Actor's scheduling state or pid).
+- **resources**: Resources created by Ray. E.g., actors, tasks, objects, placement groups, and etc.
+- **summary**: API to return the summarized view of resources.
+- **list**: API to return every individual entity of resources.
+- **get**: API to return a single entity of resources in detail.
+- **logs**: API to access the log of Actors, Tasks, Workers, or system log files.
+
+
+
+User guides
+~~~~~~~~~~~~~
+
+Getting a summary of states of entities by type
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Return the summarized information of the given Ray entity (Objects, Actors, Tasks).
+It is recommended to start monitoring states through summary APIs first. When you find anomalies
+(e.g., Actors running for a long time, Tasks that are not scheduled for a long time),
+you can use ``list`` or ``get`` APIs to get more details for an individual abnormal entity.
+
+**Summarize all actors**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray summary actors
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import summarize_actors
+            print(summarize_actors())
+
+        .. testoutput::
+
+            {'cluster': {'summary': {'Actor': {'class_name': 'Actor', 'state_counts': {'ALIVE': 2}}}, 'total_actors': 2, 'summary_by': 'class'}}
+
+**Summarize all tasks**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray summary tasks
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import summarize_tasks
+            print(summarize_tasks())
+
+        .. testoutput::
+
+            {'cluster': {'summary': {'task_running_300_seconds': {'func_or_class_name': 'task_running_300_seconds', 'type': 'NORMAL_TASK', 'state_counts': {'RUNNING': 2}}, 'Actor.__init__': {'func_or_class_name': 'Actor.__init__', 'type': 'ACTOR_CREATION_TASK', 'state_counts': {'FINISHED': 2}}}, 'total_tasks': 2, 'total_actor_tasks': 0, 'total_actor_scheduled': 2, 'summary_by': 'func_name'}}
+
+**Summarize all objects**
+
+.. note::
+
+    By default, objects are summarized by callsite. However, callsite is not recorded by Ray by default.
+    To get callsite info, set env variable `RAY_record_ref_creation_sites=1` when starting the Ray cluster:
+
+    .. code-block:: bash
+
+        RAY_record_ref_creation_sites=1 ray start --head
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray summary objects
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import summarize_objects
+            print(summarize_objects())
+
+        .. testoutput::
+
+            {'cluster': {'summary': {'disabled': {'total_objects': 6, 'total_size_mb': 0.0, 'total_num_workers': 3, 'total_num_nodes': 1, 'task_state_counts': {'SUBMITTED_TO_WORKER': 2, 'FINISHED': 2, 'NIL': 2}, 'ref_type_counts': {'LOCAL_REFERENCE': 2, 'ACTOR_HANDLE': 4}}}, 'total_objects': 6, 'total_size_mb': 0.0, 'callsite_enabled': False, 'summary_by': 'callsite'}}
+
+See :ref:`state CLI reference <state-api-cli-ref>` for more details about ``ray summary`` command.
+
+
+List the states of all entities of certain type
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Get a list of resources. Possible resources include:
+
+- :ref:`Actors <actor-guide>`, e.g., Actor ID, State, PID, death_cause (:class:`output schema <ray.util.state.common.ActorState>`)
+- :ref:`Tasks <ray-remote-functions>`, e.g., name, scheduling state, type, runtime env info (:class:`output schema <ray.util.state.common.TaskState>`)
+- :ref:`Objects <objects-in-ray>`, e.g., object ID, callsites, reference types (:class:`output schema <ray.util.state.common.ObjectState>`)
+- :ref:`Jobs <jobs-overview>`, e.g., start/end time, entrypoint, status (:class:`output schema <ray.util.state.common.JobState>`)
+- :ref:`Placement Groups <ray-placement-group-doc-ref>`, e.g., name, bundles, stats (:class:`output schema <ray.util.state.common.PlacementGroupState>`)
+- Nodes (Ray worker nodes), e.g., node ID, node IP, node state (:class:`output schema <ray.util.state.common.NodeState>`)
+- Workers (Ray worker processes), e.g., worker ID, type, exit type and details (:class:`output schema <ray.util.state.common.WorkerState>`)
+- :ref:`Runtime environments <runtime-environments>`, e.g., runtime envs, creation time, nodes (:class:`output schema <ray.util.state.common.RuntimeEnvState>`)
+
+**List all nodes**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list nodes
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_nodes
+            list_nodes()
+
+**List all placement groups**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list placement-groups
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_placement_groups
+            list_placement_groups()
+
+
+**List local referenced objects created by a process**
+
+.. tip:: You can list resources with one or multiple filters: using `--filter` or `-f`
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list objects -f pid=<PID> -f reference_type=LOCAL_REFERENCE
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_objects
+            list_objects(filters=[("pid", "=", 1234), ("reference_type", "=", "LOCAL_REFERENCE")])
+
+**List alive actors**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list actors -f state=ALIVE
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_actors
+            list_actors(filters=[("state", "=", "ALIVE")])
+
+**List running tasks**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list tasks -f state=RUNNING
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_tasks
+            list_tasks(filters=[("state", "=", "RUNNING")])
+
+**List non-running tasks**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list tasks -f state!=RUNNING
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_tasks
+            list_tasks(filters=[("state", "!=", "RUNNING")])
+
+**List running tasks that have a name func**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list tasks -f state=RUNNING -f name="task_running_300_seconds()"
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_tasks
+            list_tasks(filters=[("state", "=", "RUNNING"), ("name", "=", "task_running_300_seconds()")])
+
+**List tasks with more details**
+
+.. tip:: When ``--detail`` is specified, the API can query more data sources to obtain state information in details.
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray list tasks --detail
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+
+            from ray.util.state import list_tasks
+            list_tasks(detail=True)
+
+See :ref:`state CLI reference <state-api-cli-ref>` for more details about ``ray list`` command.
+
+
+Get the states of a particular entity (task, actor, etc.)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Get a task's states**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray get tasks <TASK_ID>
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_task
+            get_task(id=<TASK_ID>)
+
+**Get a node's states**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray get nodes <NODE_ID>
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_node
+            get_node(id=<NODE_ID>)
+
+See :ref:`state CLI reference <state-api-cli-ref>` for more details about ``ray get`` command.
+
+
+Fetch the logs of a particular entity (task, actor, etc.)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. _state-api-log-doc:
+
+State API also allows you to access Ray logs. Note that you cannot access the logs from a dead node.
+By default, the API prints logs from a head node.
+
+**Get all retrievable log file names from a head node in a cluster**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray logs cluster
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            # You could get the node ID / node IP from `ray list nodes`
+            from ray.util.state import list_logs
+            # `ray logs` by default print logs from a head node.
+            # To list the same logs, you should provide the head node ID.
+            # Get the node ID / node IP from `ray list nodes`
+            list_logs(node_id=<HEAD_NODE_ID>)
+
+**Get a particular log file from a node**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            # Get the node ID / node IP from `ray list nodes`
+            ray logs cluster gcs_server.out --node-id <NODE_ID>
+            # `ray logs cluster` is alias to `ray logs` when querying with globs.
+            ray logs gcs_server.out --node-id <NODE_ID>
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_log
+
+            # Node IP can be retrieved from list_nodes() or ray.nodes()
+            for line in get_log(filename="gcs_server.out", node_id=<NODE_ID>):
+                print(line)
+
+**Stream a log file from a node**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            # Get the node ID / node IP from `ray list nodes`
+            ray logs raylet.out --node-ip <NODE_IP> --follow
+            # Or,
+            ray logs cluster raylet.out --node-ip <NODE_IP> --follow
+
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_log
+
+            # Retrieve the Node IP from list_nodes() or ray.nodes()
+            # The loop blocks with `follow=True`
+            for line in get_log(filename="raylet.out", node_ip=<NODE_IP>, follow=True):
+                print(line)
+
+**Stream log from an actor with actor id**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray logs actor --id=<ACTOR_ID> --follow
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_log
+
+            # Get the Actor's ID from the output of `ray list actors`.
+            # The loop blocks with `follow=True`
+            for line in get_log(actor_id=<ACTOR_ID>, follow=True):
+                print(line)
+
+**Stream log from a pid**
+
+.. tab-set::
+
+    .. tab-item:: CLI (Recommended)
+        :sync: CLI (Recommended)
+
+        .. code-block:: bash
+
+            ray logs worker --pid=<PID> --follow
+
+    .. tab-item:: Python SDK (Internal Developer API)
+        :sync: Python SDK (Internal Developer API)
+
+        .. testcode::
+            :skipif: True
+
+            from ray.util.state import get_log
+
+            # Retrieve the node IP from list_nodes() or ray.nodes()
+            # get the PID of the worker running the Actor easily when output
+            # of worker is directed to the driver (default)
+            # The loop blocks with `follow=True`
+            for line in get_log(pid=<PID>, node_ip=<NODE_IP>, follow=True):
+                print(line)
+
+See :ref:`state CLI reference<state-api-cli-ref>` for more details about ``ray logs`` command.
+
+
+Failure Semantics
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The State APIs don't guarantee to return a consistent or complete snapshot of the cluster all the time. By default,
+all Python SDKs raise an exception when output is missing from the API. The CLI returns a partial result
+and provides warning messages. Here are cases where there can be missing output from the API.
+
+**Query Failures**
+
+State APIs query "data sources" (e.g., GCS, raylets, etc.) to obtain and build the snapshot of the Cluster.
+However, data sources are sometimes unavailable (e.g., the source is down or overloaded). In this case, APIs
+return a partial (incomplete) snapshot of the Cluster, and users are informed that the output is incomplete through a warning message.
+All warnings are printed through Python's ``warnings`` library, and they can be suppressed.
+
+**Data Truncation**
+
+When the returned number of entities (number of rows) is too large (> 100K), state APIs truncate the output data to ensure system stability
+(when this happens, there's no way to choose truncated data). When truncation happens it is informed through Python's
+``warnings`` module.
+
+**Garbage Collected Resources**
+
+Depending on the lifecycle of the resources, some "finished" resources are not accessible
+through the APIs because they are already garbage collected.
+
+.. note::
+
+    Do not to rely on this API to obtain correct information on finished resources.
+    For example, Ray periodically garbage collects DEAD state Actor data to reduce memory usage.
+    Or it cleans up the FINISHED state of Tasks when its lineage goes out of scope.
+
+API Reference
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- For the CLI Reference, see :ref:`State CLI Reference <state-api-cli-ref>`.
+- For the SDK Reference, see :ref:`State API Reference <state-api-ref>`.
+- For the Log CLI Reference, see :ref:`Log CLI Reference <ray-logs-api-cli-ref>`.
+
+
+
+
+Using Ray CLI tools from outside the cluster
+--------------------------------------------------------
+These CLI commands have to be run on a node in the Ray Cluster. Examples for
+executing these commands from a machine outside the Ray Cluster are provided
+below.
+
+.. tab-set::
+
+    .. tab-item:: VM Cluster Launcher
+
+        Execute a command on the cluster using ``ray exec``:
+
+        .. code-block:: shell
+
+            $ ray exec <cluster config file> "ray status"
+
+    .. tab-item:: KubeRay
+
+        Execute a command on the cluster using ``kubectl exec`` and the configured
+        RayCluster name. Ray uses the Service targeting the Ray head pod to
+        execute a CLI command on the cluster.
+
+        .. code-block:: shell
+
+            # First, find the name of the Ray head service.
+            $ kubectl get pod | grep <RayCluster name>-head
+            # NAME                                             READY   STATUS    RESTARTS   AGE
+            # <RayCluster name>-head-xxxxx                     2/2     Running   0          XXs
+
+            # Then, use the name of the Ray head service to run `ray status`.
+            $ kubectl exec <RayCluster name>-head-xxxxx -- ray status

doc/source/ray-observability/user-guides/configure-logging.md

@@ -0,0 +1,600 @@
+(configure-logging)=
+
+# Configuring Logging
+
+This guide helps you understand and modify the configuration of Ray's logging system.
+
+(logging-directory)=
+## Logging directory
+By default, Ray stores the log files in a `/tmp/ray/session_*/logs` directory. View the {ref}`log files in logging directory <logging-directory-structure>` below to understand how Ray organizes the log files within the logs folder.
+
+:::{note}
+For Linux and macOS, Ray uses ``/tmp/ray`` as the default temp directory. To change the temp and the logging directory, specify it when you call ``ray start`` or ``ray.init()``.
+:::
+
+A new Ray session creates a new folder to the temp directory. Ray symlinks the latest session folder to `/tmp/ray/session_latest`. Here is an example temp directory:
+
+```
+├── tmp/ray
+│   ├── session_latest
+│   │   ├── logs
+│   │   ├── ...
+│   ├── session_2023-05-14_21-19-58_128000_45083
+│   │   ├── logs
+│   │   ├── ...
+│   ├── session_2023-05-15_21-54-19_361265_24281
+│   ├── ...
+```
+
+Usually, Ray clears up the temp directories whenever the machines reboot. As a result, log files may get lost whenever your cluster or some of the nodes are stopped.
+
+If you need to inspect logs after the clusters stop, you need to store and persist the logs. See the instructions for how to process and export logs for {ref}`Log persistence <vm-logging>` and {ref}`KubeRay Clusters <persist-kuberay-custom-resource-logs>`.
+
+(logging-directory-structure)=
+## Log files in logging directory
+
+Below are the log files in the logging directory. Broadly speaking, two types of log files exist: system log files and application log files. Note that ``.out`` logs are from stdout/stderr and ``.err`` logs are from stderr. Ray doesn't guarantee the backward compatibility of log directories.
+
+:::{note}
+System logs may include information about your applications. For example, ``runtime_env_setup-[job_id].log`` may include information about your application's environment and dependency.
+:::
+
+### Application logs
+- ``job-driver-[submission_id].log``: The stdout of a job submitted with the {ref}`Ray Jobs API <jobs-overview>`.
+- ``worker-[worker_id]-[job_id]-[pid].[out|err]``: Python or Java part of Ray drivers and workers. Ray streams all stdout and stderr from Tasks or Actors to these files. Note that job_id is the ID of the driver.
+
+### System/component logs
+- ``dashboard.[log|out|err]``: A log file of a Ray Dashboard. ``.log`` files contain logs generated from the dashboard's logger. ``.out`` and ``.err`` files contain stdout and stderr printed from the dashboard respectively. They're usually empty except when the dashboard crashes unexpectedly.
+- ``dashboard_agent.[log|out|err]``: Every Ray node has one dashboard agent. ``.log`` files contain logs generated from the dashboard agent's logger. ``.out`` and ``.err`` files contain stdout and stderr printed from the dashboard agent respectively. They're usually empty except when the dashboard agent crashes unexpectedly.
+- ``dashboard_[module_name].[log|out|err]``: The log files for the Ray Dashboard child processes, one per each module. ``.log`` files contain logs generated from the module's logger. ``.out`` and ``.err`` files contain stdout and stderr printed from the module respectively. They're usually empty except when the module crashes unexpectedly.
+- ``gcs_server.[out|err]``: The GCS server is a stateless server that manages Ray cluster metadata. It exists only in the head node.
+- ``io-worker-[worker_id]-[pid].[out|err]``: Ray creates IO workers to spill/restore objects to external storage by default from Ray 1.3+. This is a log file of IO workers.
+- ``log_monitor.[log|out|err]``: The log monitor is in charge of streaming logs to the driver. ``.log`` files contain logs generated from the log monitor's logger. ``.out`` and ``.err`` files contain the stdout and stderr printed from the log monitor respectively. They're usually empty except when the log monitor crashes unexpectedly.
+- ``monitor.[log|out|err]``: Log files of the Autoscaler. ``.log`` files contain logs generated from the autoscaler's logger. ``.out`` and ``.err`` files contain stdout and stderr printed from the autoscaler respectively. They're usually empty except when the autoscaler crashes unexpectedly.
+- ``python-core-driver-[worker_id]_[pid].log``: Ray drivers consist of C++ core and a Python or Java frontend. C++ code generates this log file.
+- ``python-core-worker-[worker_id]_[pid].log``: Ray workers consist of C++ core and a Python or Java frontend. C++ code generates this log file.
+- ``raylet.[out|err]``: A log file of raylets.
+- ``runtime_env_agent.[log|out|err]``: Every Ray node has one agent that manages {ref}`Runtime Environment <runtime-environments>` creation, deletion, and caching. ``.log`` files contain logs generated from the runtime env agent's logger. ``.out`` and ``.err`` files contain stdout and stderr printed from the runtime env agent respectively. They're usually empty except when the runtime env agent crashes unexpectedly. The logs of the actual installations for ``pip install`` logs are in the following ``runtime_env_setup-[job_id].log`` file.
+- ``runtime_env_setup-ray_client_server_[port].log``: Logs from installing {ref}`Runtime Environments <runtime-environments>` for a job when connecting with {ref}`Ray Client <ray-client-ref>`.
+- ``runtime_env_setup-[job_id].log``: Logs from installing {ref}`runtime environments <runtime-environments>` for a Task, Actor, or Job. This file is only present if you install a runtime environment.
+
+
+(log-redirection-to-driver)=
+## Redirecting Worker logs to the Driver
+By default, Worker stdout and stderr for Tasks and Actors stream to the Ray Driver (the entrypoint script that calls ``ray.init``). It helps users aggregate the logs for the distributed Ray application in a single place.
+
+```{literalinclude} ../doc_code/app_logging.py
+```
+
+Ray prints all stdout emitted from the ``print`` method to the driver with a ``(Task or Actor repr, process ID, IP address)`` prefix.
+
+``` bash
+(pid=45601) task
+(Actor pid=480956) actor
+```
+
+### Customizing prefixes for Actor logs
+
+It's often useful to distinguish between log messages from different Actors. For example, if you have a large number of worker Actors, you may want to easily see the index of the Actor that logged a particular message. Define the `__repr__ <https://docs.python.org/3/library/functions.html#repr>`__ method for the Actor class to replace the Actor name with the Actor repr. For example:
+
+```{literalinclude} /ray-core/doc_code/actor-repr.py
+```
+
+The resulting output follows:
+
+```bash
+(MyActor(index=2) pid=482120) hello there
+(MyActor(index=1) pid=482119) hello there
+```
+
+### Coloring Actor log prefixes
+By default, Ray prints Actor log prefixes in light blue. Turn color logging off by setting the environment variable ``RAY_COLOR_PREFIX=0``
+- for example, when outputting logs to a file or other location that doesn't support ANSI codes. Or activate multi-color prefixes by setting the environment variable ``RAY_COLOR_PREFIX=1``; this indexes into an array of colors modulo the PID of each process.
+
+![coloring-actor-log-prefixes](../images/coloring-actor-log-prefixes.png)
+
+### Disable logging to the driver
+In large scale runs, you may not want to route all worker logs to the driver. Disable this feature by setting ``log_to_driver=False`` in `ray.init`:
+
+```python
+import ray
+
+# Task and Actor logs are not copied to the driver stdout.
+ray.init(log_to_driver=False)
+```
+
+
+## Log deduplication
+
+By default, Ray deduplicates logs that appear redundantly across multiple processes. The first instance of each log message is always immediately printed. However, Ray buffers subsequent log messages of the same pattern for up to five seconds and prints them in batch. Note that Ray also ignores words with numeric components. For example, for the following code snippet:
+
+```python
+import ray
+import random
+
+@ray.remote
+def task():
+    print("Hello there, I am a task", random.random())
+
+ray.get([task.remote() for _ in range(100)])
+```
+
+The output is as follows:
+
+```bash
+2023-03-27 15:08:34,195	INFO worker.py:1603 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8265
+(task pid=534172) Hello there, I am a task 0.20583517821231412
+(task pid=534174) Hello there, I am a task 0.17536720316370757 [repeated 99x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication)
+```
+
+This feature is useful when importing libraries such as `tensorflow` or `numpy`, which may emit many verbose warning messages when you import them.
+
+Configure the following environment variables on the driver process **before importing Ray** to customize log deduplication:
+
+* Set ``RAY_DEDUP_LOGS=0`` to turn off this feature entirely.
+* Set ``RAY_DEDUP_LOGS_AGG_WINDOW_S=<int>`` to change the aggregation window.
+* Set ``RAY_DEDUP_LOGS_ALLOW_REGEX=<string>`` to specify log messages to never deduplicate.
+    * Example:
+        ```python
+        import os
+        os.environ["RAY_DEDUP_LOGS_ALLOW_REGEX"] = "ABC"
+
+        import ray
+
+        @ray.remote
+        def f():
+            print("ABC")
+            print("DEF")
+
+        ray.init()
+        ray.get([f.remote() for _ in range(5)])
+
+        # 2024-10-10 17:54:19,095 INFO worker.py:1614 -- Connecting to existing Ray cluster at address: 172.31.13.10:6379...
+        # 2024-10-10 17:54:19,102 INFO worker.py:1790 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8265
+        # (f pid=1574323) ABC
+        # (f pid=1574323) DEF
+        # (f pid=1574321) ABC
+        # (f pid=1574318) ABC
+        # (f pid=1574320) ABC
+        # (f pid=1574322) ABC
+        # (f pid=1574322) DEF [repeated 4x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
+        ```
+* Set ``RAY_DEDUP_LOGS_SKIP_REGEX=<string>`` to specify log messages to skip printing.
+    * Example:
+        ```python
+        import os
+        os.environ["RAY_DEDUP_LOGS_SKIP_REGEX"] = "ABC"
+
+        import ray
+
+        @ray.remote
+        def f():
+            print("ABC")
+            print("DEF")
+
+        ray.init()
+        ray.get([f.remote() for _ in range(5)])
+        # 2024-10-10 17:55:05,308 INFO worker.py:1614 -- Connecting to existing Ray cluster at address: 172.31.13.10:6379...
+        # 2024-10-10 17:55:05,314 INFO worker.py:1790 -- Connected to Ray cluster. View the dashboard at 127.0.0.1:8265
+        # (f pid=1574317) DEF
+        # (f pid=1575229) DEF [repeated 4x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
+        ```
+
+
+## Distributed progress bars with tqdm
+
+When using [tqdm](https://tqdm.github.io) in Ray remote Tasks or Actors, you may notice that the progress bar output is corrupted. To avoid this problem, use the Ray distributed tqdm implementation at ``ray.experimental.tqdm_ray``:
+
+```{literalinclude} /ray-core/doc_code/tqdm.py
+```
+
+This tqdm implementation works as follows:
+
+1. The ``tqdm_ray`` module translates tqdm calls into special JSON log messages written to the worker stdout.
+2. The Ray log monitor routes these log messages to a tqdm singleton, instead of copying them directly to the driver stdout.
+3. The tqdm singleton determines the positions of progress bars from various Ray Tasks or Actors, ensuring they don't collide or conflict with each other.
+
+Limitations:
+
+- Ray only supports a subset of tqdm features. Refer to the ray_tqdm [implementation](https://github.com/ray-project/ray/blob/master/python/ray/experimental/tqdm_ray.py) for more details.
+- Performance may be poor if there are more than a couple thousand updates per second because Ray doesn't batch updates.
+
+By default, the built-in print is also patched to use `ray.experimental.tqdm_ray.safe_print` when you use `tqdm_ray`. This avoids progress bar corruption on driver print statements. To turn off this, set `RAY_TQDM_PATCH_PRINT=0`.
+
+
+## Using Ray's logger
+When Ray executes ``import ray``, Ray initializes Ray's logger, generating a default configuration given in ``python/ray/_private/log.py``. The default logging level is ``logging.INFO``.
+
+All Ray loggers are automatically configured in ``ray._private.ray_logging``. To modify the Ray logger:
+
+```python
+import logging
+
+logger = logging.getLogger("ray")
+logger.setLevel(logging.WARNING) # Modify the Ray logging config
+```
+Similarly, to modify the logging configuration for Ray libraries, specify the appropriate logger name:
+
+```python
+import logging
+
+# First, get the handle for the logger you want to modify
+ray_data_logger = logging.getLogger("ray.data")
+ray_tune_logger = logging.getLogger("ray.tune")
+ray_rllib_logger = logging.getLogger("ray.rllib")
+ray_train_logger = logging.getLogger("ray.train")
+ray_serve_logger = logging.getLogger("ray.serve")
+
+# Modify the ray.data logging level
+ray_data_logger.setLevel(logging.WARNING)
+
+# Other loggers can be modified similarly.
+# Here's how to add an additional file handler for Ray Tune:
+ray_tune_logger.addHandler(logging.FileHandler("extra_ray_tune_log.log"))
+```
+
+### Using Ray logger for application logs
+A Ray app includes both driver and worker processes. For Python apps, use Python loggers to format your logs. As a result, you need to set up Python loggers for both driver and worker processes.
+
+::::{tab-set}
+
+:::{tab-item} Ray Core
+
+```{admonition} Caution
+:class: caution
+This is an experimental feature. It doesn't support [Ray Client](ray-client-ref) yet.
+```
+
+Set up the Python logger for driver and worker processes separately:
+1. Set up the logger for the driver process after importing `ray`.
+2. Use `worker_process_setup_hook` to configure the Python logger for all worker processes.
+
+![Set up python loggers](../images/setup-logger-application.png)
+
+If you want to control the logger for particular actors or tasks, view the following [customizing logger for individual worker process](#customizing-worker-process-loggers).
+
+:::
+
+:::{tab-item} Ray libraries
+If you are using any of the Ray libraries, follow the instructions provided in the documentation for the library.
+:::
+
+::::
+
+### Customizing worker process loggers
+
+Ray executes Tasks and Actors remotely in Ray's worker processes. To provide your own logging configuration for the worker processes, customize the worker loggers with the instructions below:
+::::{tab-set}
+
+:::{tab-item} Ray Core: individual worker process
+Customize the logger configuration when you define the Tasks or Actors.
+```python
+import ray
+import logging
+# Initiate a driver.
+ray.init()
+
+@ray.remote
+class Actor:
+    def __init__(self):
+        # Basic config automatically configures logs to
+        # stream to stdout and stderr.
+        # Set the severity to INFO so that info logs are printed to stdout.
+        logging.basicConfig(level=logging.INFO)
+
+    def log(self, msg):
+        logger = logging.getLogger(__name__)
+        logger.info(msg)
+
+actor = Actor.remote()
+ray.get(actor.log.remote("A log message for an actor."))
+
+@ray.remote
+def f(msg):
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+    logger.info(msg)
+
+ray.get(f.remote("A log message for a task."))
+```
+
+```bash
+(Actor pid=179641) INFO:__main__:A log message for an actor.
+(f pid=177572) INFO:__main__:A log message for a task.
+```
+:::
+
+:::{tab-item} Ray Core: all worker processes of a job
+
+```{admonition} Caution
+:class: caution
+This is an experimental feature. The semantic of the API is subject to change.
+It doesn't support [Ray Client](ray-client-ref) yet.
+```
+
+Use `worker_process_setup_hook` to apply the new logging configuration to all worker processes within a job.
+
+```python
+# driver.py
+def logging_setup_func():
+    logger = logging.getLogger("ray")
+    logger.setLevel(logging.DEBUG)
+    warnings.simplefilter("always")
+
+ray.init(runtime_env={"worker_process_setup_hook": logging_setup_func})
+
+logging_setup_func()
+```
+:::
+
+:::{tab-item} Ray libraries
+If you use any of the Ray libraries, follow the instructions provided in the documentation for the library.
+:::
+
+::::
+
+(structured-logging)=
+## Structured logging
+Implement structured logging to enable downstream users and applications to consume the logs efficiently.
+
+### Application logs
+
+Ray enables users to configure the Python logging library to output logs in a structured format. This setup standardizes log entries, making them easier to handle.
+
+#### Configure structured logging for Ray Core
+
+```{admonition} Ray libraries
+If you are using any of the Ray libraries, follow the instructions provided in the documentation for the library.
+```
+
+The following methods are ways to configure Ray Core's structure logging format:
+
+##### Method 1: Configure structured logging with `ray.init`
+```python
+ray.init(
+    log_to_driver=False,
+    logging_config=ray.LoggingConfig(encoding="JSON", log_level="INFO")
+)
+```
+
+You can configure the following parameters:
+
+* `encoding`: The encoding format for the logs. The default is `TEXT` for plain text logs. The other option is `JSON` for structured logs. In both `TEXT` and `JSON` encoding formats, the logs include Ray-specific fields such as `job_id`, `worker_id`, `node_id`, `actor_id`, `actor_name`, `task_id`, `task_name` and `task_function_name`, if available.
+
+* `log_level`: The log level for the driver process. The default is `INFO`. Available log levels are defined in the [Python logging library](https://docs.python.org/3/library/logging.html#logging-levels).
+
+* `additional_log_standard_attrs`: Since Ray version 2.43. A list of additional standard Python logger attributes to include in the log record. The default is an empty list. The list of already included standard attributes are: `asctime`, `levelname`, `message`, `filename`, `lineno`, `exc_text`. The list of all valid attributes are specified in the [Python logging library](http://docs.python.org/library/logging.html#logrecord-attributes).
+
+When you set up `logging_config` in `ray.init`, it configures the root loggers for the driver process, Ray actors, and Ray tasks.
+
+```{admonition} note
+The `log_to_driver` parameter is set to `False` to disable logging to the driver
+process as the redirected logs to the driver will include prefixes that made the logs
+not JSON parsable.
+```
+
+##### Method 2: Configure structured logging with an environment variable
+
+You can set the `RAY_LOGGING_CONFIG_ENCODING` environment variable to `TEXT` or `JSON` to set the encoding format for the logs. Note that you need to set the environment variables before `import ray`.
+
+```python
+import os
+os.environ["RAY_LOGGING_CONFIG_ENCODING"] = "JSON"
+
+import ray
+import logging
+
+ray.init(log_to_driver=False)
+# Use the root logger to print log messages.
+```
+#### Example
+
+The following example configures the `LoggingConfig` to output logs in a structured JSON format and sets the log level to `INFO`. It then logs messages with the root loggers in the driver process, Ray tasks, and Ray actors. The logs include Ray-specific fields such as `job_id`, `worker_id`, `node_id`, `actor_id`, `actor_name`, `task_id`, `task_name` and `task_function_name` when applicable.
+
+```python
+import ray
+import logging
+
+ray.init(
+    logging_config=ray.LoggingConfig(encoding="JSON", log_level="INFO", additional_log_standard_attrs=['name'])
+)
+
+def init_logger():
+    """Get the root logger"""
+    return logging.getLogger()
+
+logger = logging.getLogger()
+logger.info("Driver process")
+
+@ray.remote
+def f():
+    logger = init_logger()
+    logger.info("A Ray task")
+
+@ray.remote
+class actor:
+    def print_message(self):
+        logger = init_logger()
+        logger.info("A Ray actor")
+
+task_obj_ref = f.remote()
+ray.get(task_obj_ref)
+
+actor_instance = actor.remote()
+ray.get(actor_instance.print_message.remote())
+
+"""
+{"asctime": "2025-02-25 22:06:00,967", "levelname": "INFO", "message": "Driver process", "filename": "test-log-config-doc.py", "lineno": 13, "name": "root", "job_id": "01000000", "worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "node_id": "543c939946ec1321c9c1a10899bfb72f59aa6eab7655719f2611da04", "timestamp_ns": 1740549960968002000}
+{"asctime": "2025-02-25 22:06:00,974", "levelname": "INFO", "message": "A Ray task", "filename": "test-log-config-doc.py", "lineno": 18, "name": "root", "job_id": "01000000", "worker_id": "162f2bd846e84685b4c07eb75f2c1881b9df1cdbf58ffbbcccbf2c82", "node_id": "543c939946ec1321c9c1a10899bfb72f59aa6eab7655719f2611da04", "task_id": "c8ef45ccd0112571ffffffffffffffffffffffff01000000", "task_name": "f", "task_func_name": "test-log-config-doc.f", "timestamp_ns": 1740549960974027000}
+{"asctime": "2025-02-25 22:06:01,314", "levelname": "INFO", "message": "A Ray actor", "filename": "test-log-config-doc.py", "lineno": 24, "name": "root", "job_id": "01000000", "worker_id": "b7fd965bb12b1046ddfa3d73ead5ed54eb7678d97e743d98dfab852b", "node_id": "543c939946ec1321c9c1a10899bfb72f59aa6eab7655719f2611da04", "actor_id": "43b5d1828ad0a003ca6ebcfc01000000", "task_id": "c2668a65bda616c143b5d1828ad0a003ca6ebcfc01000000", "task_name": "actor.print_message", "task_func_name": "test-log-config-doc.actor.print_message", "actor_name": "", "timestamp_ns": 1740549961314391000}
+"""
+```
+
+#### Add metadata to structured logs
+
+Add extra fields to the log entries by using the `extra` parameter in the `logger.info` method.
+
+```python
+import ray
+import logging
+
+ray.init(
+    log_to_driver=False,
+    logging_config=ray.LoggingConfig(encoding="JSON", log_level="INFO")
+)
+
+logger = logging.getLogger()
+logger.info("Driver process with extra fields", extra={"username": "anyscale"})
+
+# The log entry includes the extra field "username" with the value "anyscale".
+
+# {"asctime": "2024-07-17 21:57:50,891", "levelname": "INFO", "message": "Driver process with extra fields", "filename": "test.py", "lineno": 9, "username": "anyscale", "job_id": "04000000", "worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", "node_id": "76cdbaa32b3938587dcfa278201b8cef2d20377c80ec2e92430737ae"}
+```
+
+If needed, you can fetch the metadata of Jobs, Tasks, or Actors with Ray’s {py:obj}`ray.runtime_context.get_runtime_context` API.
+::::{tab-set}
+
+:::{tab-item} Ray Job
+Get the job ID.
+
+```python
+import ray
+# Initiate a driver.
+ray.init()
+
+job_id = ray.get_runtime_context().get_job_id
+```
+
+```{admonition} Note
+:class: note
+The job submission ID is not supported yet. This [GitHub issue](https://github.com/ray-project/ray/issues/28089#issuecomment-1557891407) tracks the work to support it.
+```
+:::
+
+:::{tab-item} Ray Actor
+Get the actor ID.
+
+```python
+import ray
+# Initiate a driver.
+ray.init()
+@ray.remote
+class actor():
+    actor_id = ray.get_runtime_context().get_actor_id
+```
+
+
+:::
+
+:::{tab-item} Ray Task
+Get the task ID.
+
+```python
+import ray
+# Initiate a driver.
+ray.init()
+@ray.remote
+def task():
+    task_id = ray.get_runtime_context().get_task_id
+```
+:::
+
+:::{tab-item} Node
+Get the node ID.
+
+```python
+import ray
+# Initiate a driver.
+ray.init()
+
+# Get the ID of the node where the driver process is running
+driver_process_node_id = ray.get_runtime_context().get_node_id
+
+@ray.remote
+def task():
+    # Get the ID of the node where the worker process is running
+    worker_process_node_id = ray.get_runtime_context().get_node_id
+```
+
+```{admonition} Tip
+:class: tip
+If you need node IP, use {py:obj}`ray.nodes` API to fetch all nodes and map the node ID to the corresponding IP.
+```
+
+
+:::
+
+
+::::
+
+### System logs
+Ray structures most system or component logs by default. <br />
+
+Logging format for Python logs <br />
+```bash
+%(asctime)s\t%(levelname)s %(filename)s:%(lineno)s -- %(message)s
+```
+
+Example: <br />
+```
+2023-06-01 09:15:34,601	INFO job_manager.py:408 -- Submitting job with RAY_ADDRESS = 10.0.24.73:6379
+```
+
+Logging format for C++ logs <br />
+
+```bash
+[year-month-day, time, pid, thread_id] (component) [file]:[line] [message]
+```
+
+Example: <br />
+
+```bash
+[2023-06-01 08:47:47,457 I 31009 225171] (gcs_server) gcs_node_manager.cc:42: Registering node info, node id = 8cc65840f0a332f4f2d59c9814416db9c36f04ac1a29ac816ad8ca1e, address = 127.0.0.1, node name = 127.0.0.1
+```
+
+#### Enabling JSON format for system logs
+
+To output Ray's backend system logs in JSON format, set the ``RAY_BACKEND_LOG_JSON`` environment variable to ``1`` before starting Ray:
+
+```bash
+RAY_BACKEND_LOG_JSON=1 ray start --head
+```
+
+When enabled, the Job Supervisor Python logs output in the following JSON format:
+
+```json
+{"asctime": "2025-12-15 20:10:32,189", "levelname": "INFO", "message": "Submitting job with RAY_ADDRESS = 10.36.3.9:6379", "filename": "job_supervisor.py", "lineno": 365, "process": 1097, "job_id": "01000000", "worker_id": "f139ea85d60ab346eeb3d6fd4d80b4a39c877b44d035317ad36e558a", "node_id": "dcc7396b4fbd2c10740cbe9fe2c36acb9e560c45d770b2611b365629", "actor_id": "0aeeebe01a4be2a1ede54cb401000000", "task_id": "16310a0f0a45af5c0aeeebe01a4be2a1ede54cb401000000", "task_name": "JobSupervisor.run", "task_func_name": "ray.dashboard.modules.job.job_supervisor.JobSupervisor.run", "actor_name": "_ray_internal_job_actor_raysubmit_RKxecNZCNp4SET5g", "timestamp_ns": 1765858232189259719}
+```
+
+:::{note}
+Currently, only the Job Supervisor supports JSON format for Python system logs when ``RAY_BACKEND_LOG_JSON=1`` is set. Other Python system components such as the Dashboard, Dashboard Agent, Log Monitor, and Autoscaler Monitor do not yet support JSON format and continue to use the standard text format.
+:::
+
+C++ system logs (such as Raylet, GCS) output in the following JSON format:
+
+```json
+{"asctime":"2025-12-15 20:10:32,778","levelname":"I","message":"Initializing worker at address: 10.36.3.9:10006","filename":"core_worker_process.cc","lineno":261,"worker_id":"02000000ffffffffffffffffffffffffffffffffffffffffffffffff","node_id":"dcc7396b4fbd2c10740cbe9fe2c36acb9e560c45d770b2611b365629"}
+```
+
+:::{note}
+Some system component logs aren't structured as suggested preceding as of 2.5. The migration of system logs to structured logs is ongoing.
+:::
+
+(log-rotation)=
+## Log rotation
+
+Ray supports log rotation of log files. Note that not all components support log rotation. (Raylet, Python, and Java worker logs don't rotate).
+
+By default, logs rotate when they reach 512 MB (maxBytes), and have a maximum of five backup files (backupCount). Ray appends indexes to all backup files - for example, `raylet.out.1`. To change the log rotation configuration, specify environment variables. For example,
+
+```bash
+RAY_ROTATION_MAX_BYTES=1024; ray start --head # Start a ray instance with maxBytes 1KB.
+RAY_ROTATION_BACKUP_COUNT=1; ray start --head # Start a ray instance with backupCount 1.
+```
+
+The max size of a log file, including its backup, is `RAY_ROTATION_MAX_BYTES * RAY_ROTATION_BACKUP_COUNT + RAY_ROTATION_MAX_BYTES`
+
+
+## Log persistence
+
+To process and export logs to external storage or management systems, see {ref}`log persistence on Kubernetes <persist-kuberay-custom-resource-logs>` and {ref}`log persistence on VMs <vm-logging>` for more details.

doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst

@@ -0,0 +1,122 @@
+.. _observability-debug-failures:
+
+Debugging Failures
+==================
+
+What Kind of Failures Exist in Ray?
+-----------------------------------
+
+Ray consists of two major APIs. ``.remote()`` to create a Task or Actor, and :func:`ray.get <ray.get>` to get the result.
+Debugging Ray means identifying and fixing failures from remote processes that run functions and classes (Tasks and Actors) created by the ``.remote`` API.
+
+Ray APIs are future APIs (indeed, it is :ref:`possible to convert Ray object references to standard Python future APIs <async-ref-to-futures>`),
+and the error handling model is the same. When any remote Tasks or Actors fail, the returned object ref contains an exception.
+When you call ``get`` API to the object ref, it raises an exception.
+
+.. testcode::
+
+  import ray
+  @ray.remote
+  def f():
+      raise ValueError("it's an application error")
+
+  # Raises a ValueError.
+  try:
+    ray.get(f.remote())
+  except ValueError as e:
+    print(e)
+
+.. testoutput::
+
+  ...
+  ValueError: it's an application error
+
+In Ray, there are three types of failures. See exception APIs for more details.
+
+- **Application failures**: This means the remote task/actor fails by the user code. In this case, ``get`` API will raise the :func:`RayTaskError <ray.exceptions.RayTaskError>` which includes the exception raised from the remote process.
+- **Intentional system failures**: This means Ray is failed, but the failure is intended. For example, when you call cancellation APIs like ``ray.cancel`` (for task) or ``ray.kill`` (for actors), the system fails remote tasks and actors, but it is intentional.
+- **Unintended system failures**: This means the remote tasks and actors failed due to unexpected system failures such as processes crashing (for example, by out-of-memory error) or nodes failing.
+
+  1. `Linux Out of Memory killer <https://www.kernel.org/doc/gorman/html/understand/understand016.html>`_ or :ref:`Ray Memory Monitor <ray-oom-monitor>` kills processes with high memory usages to avoid out-of-memory.
+  2. The machine shuts down (e.g., spot instance termination) or a :term:`raylet <raylet>` crashed (e.g., by an unexpected failure).
+  3. System is highly overloaded or stressed (either machine or system components like Raylet or :term:`GCS <GCS / Global Control Service>`), which makes the system unstable and fail.
+
+Debugging Application Failures
+------------------------------
+
+Ray distributes users' code to multiple processes across many machines. Application failures mean bugs in users' code.
+Ray provides a debugging experience that's similar to debugging a single-process Python program.
+
+print
+~~~~~
+
+``print`` debugging is one of the most common ways to debug Python programs.
+:ref:`Ray's Task and Actor logs are printed to the Ray Driver <ray-worker-logs>` by default,
+which allows you to simply use the ``print`` function to debug the application failures.
+
+Debugger
+~~~~~~~~
+
+Many Python developers use a debugger to debug Python programs, and `Python pdb <https://docs.python.org/3/library/pdb.html>`_) is one of the popular choices.
+Ray has native integration to ``pdb``. You can simply add ``breakpoint()`` to Actors and Tasks code to enable ``pdb``. View :ref:`Ray Debugger <ray-debugger>` for more details.
+
+
+Running out of file descriptors (``Too many open files``)
+---------------------------------------------------------
+
+In a Ray cluster, arbitrary two system components can communicate with each other and make 1 or more connections.
+For example, some workers may need to communicate with GCS to schedule Actors (worker <-> GCS connection).
+Your Driver can invoke Actor methods (worker <-> worker connection).
+
+Ray can support 1000s of raylets and 10000s of worker processes. When a Ray cluster gets larger,
+each component can have an increasing number of network connections, which requires file descriptors.
+
+Linux typically limits the default file descriptors per process to 1024. When there are
+more than 1024 connections to the component, it can raise error messages below.
+
+.. code-block:: bash
+
+  Too many open files
+
+It is especially common for the head node GCS process because it is a centralized
+component that many other components in Ray communicate with. When you see this error message,
+we recommend you adjust the max file descriptors limit per process via the ``ulimit`` command.
+
+We recommend you apply ``ulimit -n 65536`` to your host configuration. However, you can also selectively apply it for
+Ray components (view below example). Normally, each worker has 2~3 connections to GCS. Each raylet has 1~2 connections to GCS.
+65536 file descriptors can handle 10000~15000 of workers and 1000~2000 of nodes.
+If you have more workers, you should consider using a higher number than 65536.
+
+.. code-block:: bash
+
+  # Start head node components with higher ulimit.
+  ulimit -n 65536 ray start --head
+
+  # Start worker node components with higher ulimit.
+  ulimit -n 65536 ray start --address <head_node>
+
+  # Start a Ray driver with higher ulimit.
+  ulimit -n 65536 <python script>
+
+If that fails, double-check that the hard limit is sufficiently large by running ``ulimit -Hn``.
+If it is too small, you can increase the hard limit as follows (these instructions work on EC2).
+
+* Increase the hard ulimit for open file descriptors system-wide by running
+  the following.
+
+  .. code-block:: bash
+
+    sudo bash -c "echo $USER hard nofile 65536 >> /etc/security/limits.conf"
+
+* Logout and log back in.
+
+
+Failures due to memory issues
+--------------------------------
+View :ref:`debugging memory issues <ray-core-mem-profiling>` for more details.
+
+
+This document discusses some common problems that people run into when using Ray
+as well as some known problems. If you encounter other problems, `let us know`_.
+
+.. _`let us know`: https://github.com/ray-project/ray/issues

doc/source/ray-observability/user-guides/debug-apps/debug-hangs.rst

@@ -0,0 +1,45 @@
+.. _observability-debug-hangs:
+
+Debugging Hangs
+===============
+View stack traces in Ray Dashboard
+-----------------------------------
+The :ref:`Ray dashboard <observability-getting-started>`  lets you profile Ray Driver or Worker processes, by clicking on the "CPU profiling" or "Stack Trace" actions for active Worker processes, Tasks, Actors, and Job's driver process.
+
+.. image:: /images/profile.png
+   :align: center
+   :width: 80%
+
+Clicking "Stack Trace" will return the current stack trace sample using ``py-spy``. By default, only the Python stack
+trace is shown. To show native code frames, set the URL parameter ``native=1`` (only supported on Linux).
+
+.. image:: /images/stack.png
+   :align: center
+   :width: 60%
+
+.. note::
+   You may run into permission errors when using py-spy in the docker containers. To fix the issue:
+   
+   * if you start Ray manually in a Docker container, follow the `py-spy documentation`_ to resolve it. 
+   * if you are a KubeRay user, follow the :ref:`guide to configure KubeRay <kuberay-pyspy-integration>` and resolve it.
+   
+.. note::
+   The following errors are conditional and not signals of failures for your Python programs:
+
+   * If you see "No such file or direction", check if your worker process has exited.
+   * If you see "No stack counts found", check if your worker process was sleeping and not active in the last 5s.
+
+.. _`py-spy documentation`: https://github.com/benfred/py-spy#how-do-i-run-py-spy-in-docker
+
+
+Use ``ray stack`` CLI command
+------------------------------
+
+Once ``py-spy`` is installed (it is automatically installed if "Ray Dashboard" component is included when :ref:`installing Ray <installation>`), you can run ``ray stack`` to dump the stack traces of all Ray Worker processes on
+the current node.
+
+This document discusses some common problems that people run into when using Ray
+as well as some known problems. If you encounter other problems, please
+`let us know`_.
+
+.. _`let us know`: https://github.com/ray-project/ray/issues

doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst

@@ -0,0 +1,323 @@
+.. _ray-core-mem-profiling:
+
+Debugging Memory Issues
+=======================
+
+
+.. _troubleshooting-out-of-memory:
+
+Debugging Out of Memory
+-----------------------
+
+Before reading this section, familiarize yourself with the Ray :ref:`Memory Management <memory>` model.
+
+- If your cluster has out-of-memory problems, view :ref:`How to Detect Out-of-Memory Errors <troubleshooting-out-of-memory-how-to-detect>`.
+- To locate the source of the memory problems, view :ref:`Find per Task and Actor Memory Usage <troubleshooting-out-of-memory-task-actor-mem-usage>`.
+- Once you've identified the source, address it at :ref:`Eliminating worker out-of-memory errors <troubleshooting-out-of-memory-eliminate-worker-oom>`.
+- If your head node has high memory usage, view :ref:`Head Node Out-of-Memory Error <troubleshooting-out-of-memory-head>`.
+- If your memory usage is high due to high parallelism, view :ref:`Reduce Parallelism <troubleshooting-out-of-memory-reduce-parallelism>`.
+- If you want to profile per Task and Actor memory usage, view :ref:`Profile Task and Actor Memory Usage <troubleshooting-out-of-memory-profile>`.
+
+What's the Out-of-Memory Error?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Memory is a limited resource. When a process requests memory and the OS fails to allocate it, the OS executes a routine to free up memory
+by killing a process that has high memory usage (via SIGKILL) to avoid the OS becoming unstable. This routine is called the `Linux Out of Memory killer <https://www.kernel.org/doc/gorman/html/understand/understand016.html>`_.
+
+For Ray, the Linux out-of-memory (OOM) killer kills Ray processes without the control plane noticing. This causes the following problems:
+
+1. The Linux OOM killer indiscriminately kills processes based on memory footprint.
+   For Ray, this can result in significant loss of progress and, in some scenarios, in the death of critical
+   Ray components, which leads to node deaths.
+2. The Linux OOM killer uses SIGKILL to kill processes. Because processes can't handle SIGKILL,
+   Ray has difficulty raising a proper error message and taking proper actions for fault tolerance.
+
+To solve this problem, Ray has (from Ray 2.2) an application-level :ref:`memory monitor <ray-oom-monitor>`,
+which continually monitors the memory usage of the host and kills the Ray Workers before the Linux out-of-memory killer executes. 
+
+.. _troubleshooting-out-of-memory-how-to-detect:
+
+Detecting Out-of-Memory errors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can monitor out-of-memory errors on the Ray Dashboard's :ref:`metrics page <dash-metrics-view>` via the Ray OOM Kills panel and the Unexpected System Level Worker Failures panel.
+The Ray OOM Kills panel shows the number of workers killed by the Ray OOM killer.
+The Unexpected System Level Worker Failures panel shows the number of workers that failed unexpectedly. These failures are typically
+caused by the Linux out-of-memory killer; correlate with memory usage metrics to confirm.
+
+.. image:: ../../images/ray-oom-kills.png
+    :align: center
+
+.. image:: ../../images/unexpected-system-level-worker-failures.png
+    :align: center
+
+
+If the Linux out-of-memory killer terminates Tasks or Actors, Ray Worker processes are unable to catch and display an exact root cause
+because SIGKILL cannot be handled by processes. If you call ``ray.get`` into the Tasks and Actors that were executed from the dead worker,
+it raises an exception with one of the following error messages (which indicates the worker is killed unexpectedly).
+
+.. code-block:: bash
+
+  Worker exit type: UNEXPECTED_SYSTEM_EXIT Worker exit detail: Worker unexpectedly exits with a connection error code 2. End of file. There are some potential root causes. (1) The process is killed by SIGKILL by OOM killer due to high memory usage. (2) ray stop --force is called. (3) The worker is crashed unexpectedly due to SIGSEGV or other unexpected errors.
+
+.. code-block:: bash
+
+  Worker exit type: SYSTEM_ERROR Worker exit detail: The leased worker has unrecoverable failure. Worker is requested to be destroyed when it is returned.
+
+You can also use the `dmesg <https://phoenixnap.com/kb/dmesg-linux#:~:text=The%20dmesg%20command%20is%20a,take%20place%20during%20system%20startup.>`_ CLI command to verify the processes are killed by the Linux out-of-memory killer.
+
+.. image:: ../../images/dmsg.png
+    :align: center
+
+
+As mentioned above, having the Linux OOM killer trigger before the Ray OOM killer is undesirable.
+In Ray 2.56 and above, enable resource isolation mode by passing ``--enable-resource-isolation`` when starting Ray
+to ensure that the Ray OOM killer triggers before the Linux OOM killer.
+
+It's still possible for Linux OOM kills to occur if the system overhead consumes more memory than
+what's reserved for it (the default is 10%, with a minimum of 500 MB and a maximum of 10 GB).
+Ray logs something similar to the following example if it detects this scenario.
+
+.. code-block:: bash
+
+  System slice memory usage 10869600256 bytes has exceeded the reserved system memory of 10737418240 bytes. This can prevent Ray from being able to provide the proper protection to critical system processes and can lead to node deaths and significant loss of progress. Please consider passing a system reserved memory value that is higher than the current system slice memory usage via the --system-reserved-memory flag when starting the raylet.
+
+When you see a kernel OOM or this log message with resource isolation enabled, try increasing the memory reserved for system processes
+by setting a higher value than the reported system slice memory usage for the ``--system-reserved-memory`` flag when starting Ray. 
+Try to allocate at least a GiB (depending on host size) of buffer space between the reported/expected system slice memory usage and the system reserved memory.
+
+.. note::
+
+  To enable resource isolation, complete the prerequisite steps in :ref:`How to Enable Cgroup v2 for Resource Isolation <enable-cgroupv2>`.
+
+If Ray's memory monitor kills the worker, Ray retries it automatically (see the :ref:`retry policy <ray-oom-retry-policy>` for details).
+Ray's memory monitor also logs the details of the out-of-memory kill to the ``raylet.out`` log file.
+An example log is shown below.
+
+.. code-block:: bash
+
+  Task hungry_hippo failed due to oom. There are infinite oom retries remaining, so the task will be retried. Error: 2 worker(s) were killed due to the node running low on memory. Memory on the node (IP: <ip address>, ID: 92edc4e97e4dac3cee61126133ee7ab6d0a2ee73803623d24a02979d) was 110.69GB / 124.35GB (0.890161)
+  OOM kill reason: user cgroup memory upper bound was met or exceeded
+  Object store memory usage: [- objects spillable: 0
+  - bytes spillable: 0
+  - objects unsealed: 0
+  - bytes unsealed: 0
+  - objects in use: 0
+  - bytes in use: 0
+  - objects evictable: 0
+  - bytes evictable: 0
+  
+  - objects created by worker: 0
+  - bytes created by worker: 0
+  - objects restored: 0
+  - bytes restored: 0
+  - objects received: 0
+  - bytes received: 0
+  - objects errored: 0
+  - bytes errored: 0
+  
+  Eviction Stats:
+  (global lru) capacity: 35098657996
+  (global lru) used: 0%
+  (global lru) num objects: 0
+  (global lru) num evictions: 0
+  (global lru) bytes evicted: 0]
+  Ray killed 2 worker(s) based on the killing policy
+  Considered workers: [
+  Selected to kill: (Task: job ID=01000000, lease ID=0600000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310152, actual memory used=0.67GB, worker ID=3e3d8f80b70d48b643d79ed2292b5d4f779820a964e55ad65413687d)
+  Selected to kill: (Task: job ID=01000000, lease ID=0400000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310153, actual memory used=21.64GB, worker ID=0e5649d39c15609c0db6a5cf95de94befded2ee7da2facbf64b52e6f)
+  (Task: job ID=01000000, lease ID=0500000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310151, actual memory used=21.95GB, worker ID=34241048bfb59ac29bd5e32d706c9bd41eafc6972c9bcbada99464e7)
+  (Task: job ID=01000000, lease ID=0000000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310149, actual memory used=21.87GB, worker ID=2cc6dbeef4ebc06789de65fb43e04fbe1feebf1e699902ece89a8328)
+  (Task: job ID=01000000, lease ID=0200000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310155, actual memory used=21.85GB, worker ID=14d4cc84e2f21ba3edbc9948b780d67013afbffda99dd33e829d56d3)
+  (Task: job ID=01000000, lease ID=0100000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, task name=hungry_hippo, required resources={CPU: 1}, pid=3310147, actual memory used=21.53GB, worker ID=c90db9af23d78530d1f848c1301bc4b877925fe9122bc70948ad9489)]
+  Total non-selected idle workers: 25
+  Total non-selected idle workers USS bytes: 1.00GB
+  To see more information about memory usage on this node, use `ray logs raylet.out -ip <ip address>`
+  Top 10 memory users: PID  MEM(GB) COMMAND
+  3310151   21.95   ray::hungry_hippo
+  3310149   21.87   ray::hungry_hippo
+  3310155   21.85   ray::hungry_hippo
+  3310153   21.64   ray::hungry_hippo
+  3310147   21.53   ray::hungry_hippo
+  3108574   1.95    bazel
+  3180337   1.61    ray::foo_actor
+  3310152   0.67    ray::hungry_hippo
+  2924839   0.53    ray::idle_worker
+  3149737   0.47    ray::idle_worker
+  Refer to the documentation on how to address the out of memory issue: https://docs.ray.io/en/latest/ray-core/scheduling/ray-oom-prevention.html. Consider provisioning more memory on this node or reducing task parallelism by requesting more CPUs per task. To adjust the kill threshold, set the environment variable `RAY_memory_usage_threshold` when starting Ray. To disable worker killing, set the environment variable `RAY_memory_monitor_refresh_ms` to zero. Since 2.56, Ray updated the oom killing policy to enabling killing multiple workers and selecting workers based on the time since the task start executing. To revert to the legacy policy of determining worker to oom kill based on owner group size or only selecting a single worker to kill at a time, set the environment variable `RAY_worker_killing_policy_by_group` to true before starting Ray. If the idle workers have a non-trivial memory footprint at the time of OOM (check OOM log for non-selected idle workers), consider setting the environment variable `RAY_idle_worker_killing_memory_threshold_bytes` to a lower value to consider idle workers with lower memory footprint for killing.
+
+If Tasks or Actors can't be retried, they raise an exception with
+a similar error message when you call ``ray.get`` on them.
+
+Ray memory monitor also periodically prints the aggregated out-of-memory killer summary to Ray drivers.
+
+.. code-block:: bash
+
+  (raylet) [2023-04-09 07:23:59,445 E 395 395] (raylet) node_manager.cc:3049: 10 Workers (tasks / actors) killed due to memory pressure (OOM), 0 Workers crashed due to other reasons at node (ID: e5d953ef03e55e26f13973ea1b5a0fd0ecc729cd820bc89e4aa50451, IP: 10.0.62.231) over the last time period. To see more information about the Workers killed on this node, use `ray logs raylet.out -ip 10.0.62.231`
+  (raylet) 
+  (raylet) Refer to the documentation on how to address the out of memory issue: https://docs.ray.io/en/latest/ray-core/scheduling/ray-oom-prevention.html. Consider provisioning more memory on this node or reducing task parallelism by requesting more CPUs per task. To adjust the kill threshold, set the environment variable `RAY_memory_usage_threshold` when starting Ray. To disable worker killing, set the environment variable `RAY_memory_monitor_refresh_ms` to zero.
+
+Ray Dashboard's :ref:`event page <dash-event>` also provides the out-of-memory killer-specific events and metrics.
+
+.. image:: ../../images/oom-events.png
+    :align: center
+
+.. _troubleshooting-out-of-memory-task-actor-mem-usage:
+
+Find per Task and Actor Memory Usage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If Tasks or Actors fail because of out-of-memory errors, they are retried based on :ref:`retry policies <ray-oom-retry-policy>`. 
+However, it is often preferred to find the root causes of memory issues and fix them instead of relying on fault tolerance mechanisms.
+This section explains how to debug out-of-memory errors in Ray.
+
+To view the memory usage of tasks and actors at the time of OOM, see the considered workers in the OOM log above. This information
+helps identify the tasks and actors responsible for the OOM. The OOM log also includes the memory footprint of all idle workers.
+See :ref:`ray-oom-worker-killing-policy` for how Ray considers idle workers for killing.
+
+To view the memory usage of Tasks and Actors based on type over time, view the :ref:`per Task and Actor memory usage graph <dash-workflow-cpu-memory-analysis>` for more details.
+The memory usage from the per component graph uses RSS - SHR. See below for reasoning.
+
+Alternatively, you can also use the CLI command `htop <https://htop.dev/>`_.
+
+.. image:: ../../images/htop.png
+    :align: center
+
+See the ``allocate_memory`` row. See two columns, RSS and SHR. 
+
+SHR usage is typically the memory usage from the Ray object store. The Ray object store allocates 30% of host memory to the shared memory (``/dev/shm``, unless you specify ``--object-store-memory``).
+If Ray workers access the object inside the object store using ``ray.get``, SHR usage increases. Since the Ray object store supports the :ref:`zero-copy <serialization-guide>`
+deserialization, several workers can access the same object without copying them to in-process memory. For example, if
+8 workers access the same object inside the Ray object store, each process' ``SHR`` usage increases. However, they are not using 8 * SHR memory (there's only 1 copy in the shared memory). 
+Also note that Ray object store triggers :ref:`object spilling <object-spilling>` when the object usage goes beyond the limit, which means the memory usage from the shared memory won't exceed 30%
+of the host memory.
+
+Out-of-memory issues from a host, are due to RSS usage from each worker. Calculate per
+process memory usage by RSS - SHR because SHR is for Ray object store as explained above. The total memory usage is typically
+``SHR (object store memory usage, 30% of memory) + sum(RSS - SHR from each ray proc) + sum(RSS - SHR from system components. e.g., raylet, GCS. Usually small)``.
+
+.. _troubleshooting-out-of-memory-eliminate-worker-oom:
+
+Eliminating worker Out-Of-Memory errors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Most out-of-memory errors come from oversubscribing memory on a node.
+By default, tasks and actors have no memory requirements, so the scheduler is unaware
+of their memory footprint and may schedule too many memory-hungry tasks or actors onto a single node.
+
+To prevent this oversubscription and eliminate OOM issues, pass a ``memory`` resource request to tasks or actors
+to reserve memory for them. See :ref:`resource requirements <resource-requirements>` for more details.
+This request doesn't impose any limit on memory usage; it's used for scheduling only.
+As shown in the example OOM log above, the log includes the resource request for each active worker. If the worker memory usage
+exceeds the requested memory at the time of OOM, adjust the resource request accordingly.
+
+.. _troubleshooting-out-of-memory-head:
+
+Head node out-of-Memory error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+First, check the head node memory usage from the metrics page. Find the head node address from the cluster page.
+
+.. image:: ../../images/head-node-addr.png
+    :align: center
+
+Then check the memory usage from the head node from the node memory usage view inside the Dashboard :ref:`metrics view <dash-metrics-view>`.
+
+.. image:: ../../images/metrics-node-view.png
+    :align: center
+
+The Ray head node has more memory-demanding system components such as GCS or the dashboard. 
+Also, the driver runs from a head node by default. If the head node has the same memory capacity as worker nodes
+and if you execute the same number of Tasks and Actors from a head node, it can easily have out-of-memory problems.
+In this case, do not run any Tasks and Actors on the head node by specifying ``--num-cpus=0`` when starting a head node by ``ray start --head``.
+If you use KubeRay, view :ref:`here <kuberay-num-cpus>`.
+
+.. _troubleshooting-out-of-memory-reduce-parallelism:
+
+Reduce Parallelism
+~~~~~~~~~~~~~~~~~~
+
+High parallelism can trigger out-of-memory errors. For example, if
+you have 8 training workers that perform the data preprocessing -> training. 
+If you load too much data into each worker, the total memory usage (``training worker mem usage * 8``) can exceed the
+memory capacity. 
+
+Verify the memory usage by looking at the :ref:`per Task and Actor memory usage graph <dash-workflow-cpu-memory-analysis>` and the Task metrics.
+
+First, see the memory usage of an ``allocate_memory`` task. The total is 18GB.
+At the same time, verify the 15 concurrent tasks that are running.
+
+.. image:: ../../images/component-memory.png
+    :align: center
+
+.. image:: ../../images/tasks-graph.png
+    :align: center
+
+Each task uses about 18GB / 15 == 1.2 GB. To reduce the parallelism:
+
+- `Limit the max number of running tasks <https://docs.ray.io/en/latest/ray-core/patterns/limit-running-tasks.html>`_. 
+- Increase the ``num_cpus`` options for :func:`ray.remote`. Modern hardware typically has 4GB of memory per CPU, so you can choose the CPU requirements accordingly. This example specifies 1 CPU per ``allocate_memory`` Task. Doubling the CPU requirements, runs only half(7) of the Tasks at the same time, and memory usage doesn't exceed 9GB.
+
+.. _troubleshooting-out-of-memory-profile:
+
+Profiling Task and Actor memory usage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is also possible tasks and actors use more memory than you expect. For example, actors or tasks can have a memory leak or have unnecessary copies.
+
+View the instructions below to learn how to memory profile individual actors and tasks.
+
+.. _memray-profiling:
+
+Memory Profiling Ray tasks and actors
+--------------------------------------
+
+To memory profile Ray tasks or actors, use `memray <https://bloomberg.github.io/memray/>`_.
+Note that you can also use other memory profiling tools if it supports a similar API.
+
+First, install ``memray``.
+
+.. code-block:: bash
+
+  pip install memray
+
+``memray`` supports a Python context manager to enable memory profiling. You can write the ``memray`` profiling file wherever you want.
+But in this example, we will write them to `/tmp/ray/session_latest/logs` because Ray dashboard allows you to download files inside the log folder.
+This will allow you to download profiling files from other nodes.
+
+.. tab-set::
+
+    .. tab-item:: Actors
+
+      .. literalinclude:: ../../doc_code/memray_profiling.py
+          :language: python
+          :start-after: __memray_profiling_start__
+          :end-before: __memray_profiling_end__
+
+    .. tab-item:: Tasks
+
+      Note that tasks have a shorter lifetime, so there could be lots of memory profiling files.
+
+      .. literalinclude:: ../../doc_code/memray_profiling.py
+          :language: python
+          :start-after: __memray_profiling_task_start__
+          :end-before: __memray_profiling_task_end__
+
+Once the task or actor runs, go to the :ref:`Logs view <dash-logs-view>` of the dashboard. Find and click the log file name.
+
+.. image:: ../../images/memory-profiling-files.png
+    :align: center
+
+Click the download button. 
+
+.. image:: ../../images/download-memory-profiling-files.png
+    :align: center
+
+Now, you have the memory profiling file. Running
+
+.. code-block:: bash
+
+  memray flamegraph <memory profiling bin file>
+
+And you can see the result of the memory profiling!
+

doc/source/ray-observability/user-guides/debug-apps/general-debugging.rst

@@ -0,0 +1,210 @@
+.. _observability-general-debugging:
+
+Common Issues
+=======================
+
+Distributed applications offer great power but also increased complexity. 
+Some of Ray's behaviors may initially surprise users, but these design choices serve important purposes in distributed computing environments.
+
+This document outlines common issues encountered when running Ray in a cluster, highlighting key differences compared to running Ray locally.
+
+Environment variables aren't passed from the Driver process to Worker processes
+---------------------------------------------------------------------------------
+
+**Issue:** When you set an environment variable on your Driver, it isn't propagated to the Worker processes.
+
+**Example:** Suppose you have a file ``baz.py`` in the directory where you run Ray, and you execute the following command:
+
+.. literalinclude:: /ray-observability/doc_code/gotchas.py
+   :language: python
+   :start-after: __env_var_start__
+   :end-before: __env_var_end__
+
+
+**Expected behavior:** Users may expect that setting environment variables on the Driver sends them to all Worker processes as if running on a single machine, but it doesn't.
+
+**Fix:** Enable Runtime Environments to explicitly pass environment variables. When you call ``ray.init(runtime_env=...)``, it sends the specified environment variables to the Workers.
+Alternatively, you can set the environment variables as part of your cluster setup configuration.
+
+.. literalinclude:: /ray-observability/doc_code/gotchas.py
+   :language: python
+   :start-after: __env_var_fix_start__
+   :end-before: __env_var_fix_end__
+
+
+Filenames work sometimes and not at other times
+-----------------------------------------------
+
+**Issue:** Referencing a file by its name in a Task or Actor may sometimes succeed and sometimes fail. 
+This inconsistency arises because the Task or Actor finds the file when running on the Head Node, but the file might not exist on other machines.
+
+**Example:** Consider the following scenario:
+
+.. code-block:: bash
+
+   % touch /tmp/foo.txt
+
+And this code:
+
+.. testcode::
+
+  import os
+  import ray
+
+  @ray.remote
+  def check_file():
+      foo_exists = os.path.exists("/tmp/foo.txt")
+      return foo_exists
+
+  futures = []
+  for _ in range(1000):
+      futures.append(check_file.remote())
+
+  print(ray.get(futures))
+
+In this case, you might receive a mixture of True and False. If ``check_file()`` runs on the Head Node or locally, it finds the file; however, on a Worker Node, it doesn't.
+
+**Expected behavior:** Users generally expect file references to either work consistently or to reliably fail, rather than behaving inconsistently.
+
+**Fix:**
+
+— Use only shared file paths for such applications. For example, a network file system or S3 storage can provide the required consistency.
+— Avoid relying on local files to be consistent across machines.
+
+
+Placement Groups aren't composable
+-----------------------------------
+
+**Issue:** If you schedule a new task from the tasks or actors running within a Placement Group, the system might fail to allocate resources properly, causing the operation to hang.
+
+**Example:** Imagine you are using Ray Tune (which creates Placement Groups) and want to apply it to an objective function that in turn uses Ray Tasks. For example:
+
+.. testcode::
+
+  import ray
+  from ray import tune
+  from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+  def create_task_that_uses_resources():
+      @ray.remote(num_cpus=10)
+      def sample_task():
+          print("Hello")
+          return
+
+      return ray.get([sample_task.remote() for i in range(10)])
+
+  def objective(config):
+      create_task_that_uses_resources()
+
+  tuner = tune.Tuner(objective, param_space={"a": 1})
+  tuner.fit()
+
+This code errors with the message:
+
+.. code-block::
+
+    ValueError: Cannot schedule create_task_that_uses_resources.<locals>.sample_task with the placement group
+    because the resource request {'CPU': 10} cannot fit into any bundles for the placement group, [{'CPU': 1.0}].
+
+**Expected behavior:** The code executes successfully without resource allocation issues.
+
+**Fix:** Ensure that in the ``@ray.remote`` declaration of tasks called within ``create_task_that_uses_resources()``, you include the parameter
+``scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=None)``.
+
+.. code-block:: diff
+
+  def create_task_that_uses_resources():
+  +     @ray.remote(num_cpus=10, scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=None))
+  -     @ray.remote(num_cpus=10)
+
+
+Outdated Function Definitions
+-----------------------------
+
+Because of Python's subtleties, redefining a remote function may not always update Ray to use the latest version. 
+For example, suppose you define a remote function ``f`` and then redefine it; Ray should use the new definition:
+
+.. testcode::
+
+  import ray
+
+  @ray.remote
+  def f():
+      return 1
+
+  @ray.remote
+  def f():
+      return 2
+
+  print(ray.get(f.remote()))  # This should print 2.
+
+.. testoutput::
+
+  2
+
+However, there are cases where modifying a remote function doesn't take effect without restarting the cluster:
+
+— **Imported function issue:** If ``f`` is defined in an external file (e.g., ``file.py``), and you modify its definition, re-importing the file may be ignored because Python treats the subsequent import as a no-op. A solution is to use ``from importlib import reload; reload(file)`` instead of a second import.
+
+— **Helper function dependency:** If ``f`` depends on a helper function ``h`` defined in an external file, changes to ``h`` may not propagate. The easiest solution is to restart the Ray cluster. Alternatively, you can redefine ``f`` to reload ``file.py`` before invoking ``h``:
+
+.. testcode::
+
+  @ray.remote
+  def f():
+      from importlib import reload
+      reload(file)
+      return file.h()
+
+This forces the external module to reload on the Workers. Note that in Python 3, you must use ``from importlib import reload``.
+
+
+Capture task and actor call sites
+---------------------------------
+
+Ray captures and displays a stack trace when you invoke a task, create an actor, or call an actor method.
+
+To enable call site capture, set the environment variable ``RAY_record_task_actor_creation_sites=true``. When enabled:
+
+— Ray captures a stack trace when creating tasks, actors, or invoking actor methods.
+— The captured stack trace is available in the Ray Dashboard (under task and actor details), output of the state CLI command ``ray list task --detail``, and state API responses.
+
+Note that Ray turns off stack trace capture by default due to potential performance impacts. Enable it only when you need it for debugging.
+
+Example:
+
+.. NOTE(edoakes): test is skipped because it reinitializes Ray.
+.. testcode::
+    :skipif: True
+
+    import ray
+
+    # Enable stack trace capture
+    ray.init(runtime_env={"env_vars": {"RAY_record_task_actor_creation_sites": "true"}})
+
+    @ray.remote
+    def my_task():
+        return 42
+
+    # Capture the stack trace upon task invocation.
+    future = my_task.remote()
+    result = ray.get(future)
+
+    @ray.remote
+    class Counter:
+        def __init__(self):
+            self.value = 0
+
+        def increment(self):
+            self.value += 1
+            return self.value
+
+    # Capture the stack trace upon actor creation.
+    counter = Counter.remote()
+
+    # Capture the stack trace upon method invocation.
+    counter.increment.remote()
+
+This document outlines common problems encountered when using Ray along with potential solutions. If you encounter additional issues, please report them.
+
+.. _`let us know`: https://github.com/ray-project/ray/issues

doc/source/ray-observability/user-guides/debug-apps/index.md

@@ -0,0 +1,24 @@
+(observability-debug-apps)=
+
+# Debugging Applications
+
+```{toctree}
+:hidden:
+
+general-debugging
+debug-memory
+debug-hangs
+debug-failures
+optimize-performance
+../../ray-distributed-debugger
+ray-debugging
+```
+
+These guides help you perform common debugging or optimization tasks for your distributed application on Ray:
+* {ref}`observability-general-debugging`
+* {ref}`ray-core-mem-profiling`
+* {ref}`observability-debug-hangs`
+* {ref}`observability-debug-failures`
+* {ref}`observability-optimize-performance`
+* {ref}`ray-distributed-debugger`
+* {ref}`ray-debugger` (deprecated)

doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst

@@ -0,0 +1,360 @@
+.. _observability-optimize-performance:
+
+Optimizing Performance
+======================
+
+No speedup
+----------
+
+You just ran an application using Ray, but it wasn't as fast as you expected it
+to be. Or worse, perhaps it was slower than the serial version of the
+application! The most common reasons are the following.
+
+- **Number of cores:** How many cores is Ray using? When you start Ray, it will
+  determine the number of CPUs on each machine with ``psutil.cpu_count()``. Ray
+  usually will not schedule more tasks in parallel than the number of CPUs. So
+  if the number of CPUs is 4, the most you should expect is a 4x speedup.
+
+- **Physical versus logical CPUs:** Do the machines you're running on have fewer
+  **physical** cores than **logical** cores? You can check the number of logical
+  cores with ``psutil.cpu_count()`` and the number of physical cores with
+  ``psutil.cpu_count(logical=False)``. This is common on a lot of machines and
+  especially on EC2. For many workloads (especially numerical workloads), you
+  often cannot expect a greater speedup than the number of physical CPUs.
+
+- **Small tasks:** Are your tasks very small? Ray introduces some overhead for
+  each task (the amount of overhead depends on the arguments that are passed
+  in). You will be unlikely to see speedups if your tasks take less than ten
+  milliseconds. For many workloads, you can easily increase the sizes of your
+  tasks by batching them together.
+
+- **Variable durations:** Do your tasks have variable duration? If you run 10
+  tasks with variable duration in parallel, you shouldn't expect an N-fold
+  speedup (because you'll end up waiting for the slowest task). In this case,
+  consider using ``ray.wait`` to begin processing tasks that finish first.
+
+- **Multi-threaded libraries:** Are all of your tasks attempting to use all of
+  the cores on the machine? If so, they are likely to experience contention and
+  prevent your application from achieving a speedup.
+  This is common with some versions of ``numpy``. To avoid contention, set an
+  environment variable like ``MKL_NUM_THREADS`` (or the equivalent depending on
+  your installation) to ``1``.
+
+  For many - but not all - libraries, you can diagnose this by opening ``top``
+  while your application is running. If one process is using most of the CPUs,
+  and the others are using a small amount, this may be the problem. The most
+  common exception is PyTorch, which will appear to be using all the cores
+  despite needing ``torch.set_num_threads(1)`` to be called to avoid contention.
+
+If you are still experiencing a slowdown, but none of the above problems apply,
+we'd really like to know! Create a `GitHub issue`_ and Submit a minimal code example that demonstrates the problem.
+
+.. _`Github issue`: https://github.com/ray-project/ray/issues
+
+This document discusses some common problems that people run into when using Ray
+as well as some known problems. If you encounter other problems, `let us know`_.
+
+.. _`let us know`: https://github.com/ray-project/ray/issues
+
+.. _ray-core-timeline:
+
+Visualizing Tasks with Ray Timeline
+-------------------------------------
+View :ref:`how to use Ray Timeline in the Dashboard <dashboard-timeline>` for more details.
+
+Instead of using Dashboard UI to download the tracing file, you can also export the tracing file as a JSON file by running ``ray timeline`` from the command line or ``ray.timeline`` from the Python API.
+
+.. testcode::
+
+  import ray
+
+  ray.init()
+
+  ray.timeline(filename="timeline.json")
+
+
+.. _dashboard-profiling:
+
+Python CPU profiling in the Dashboard
+-------------------------------------
+
+The :ref:`Ray dashboard <observability-getting-started>`  lets you profile Ray worker processes by clicking on the "Stack Trace" or "CPU Flame Graph"
+actions for active workers, actors, and jobs.
+
+.. image:: /images/profile.png
+   :align: center
+   :width: 80%
+
+Clicking "Stack Trace" returns the current stack trace sample using ``py-spy``. By default, only the Python stack
+trace is shown. To show native code frames, set the URL parameter ``native=1`` (only supported on Linux). To also
+dump stack traces for child processes of the target (for example, data loader or multiprocess inference workers),
+set the URL parameter ``subprocesses=1``.
+
+.. image:: /images/stack.png
+   :align: center
+   :width: 60%
+
+Clicking "CPU Flame Graph" takes a number of stack trace samples and combine them into a flame graph visualization.
+This flame graph can be useful for understanding the CPU activity of the particular process. To adjust the duration
+of the flame graph, you can change the ``duration`` parameter in the URL. Similarly, you can change the ``native``
+parameter to enable native profiling. To also include off-CPU (sleeping) threads, such as threads blocked on locks,
+I/O, or CUDA syncs, set the URL parameter ``idle=1``. To also profile child processes of the target (for example,
+data loader or multiprocess inference workers), set the URL parameter ``subprocesses=1``.
+
+.. image:: /images/flamegraph.png
+   :align: center
+   :width: 80%
+
+The profiling feature requires ``py-spy`` to be installed. If it is not installed, or if the ``py-spy`` binary does
+not have root permissions, the Dashboard prompts with instructions on how to setup ``py-spy`` correctly:
+
+.. code-block::
+
+    This command requires `py-spy` to be installed with root permissions. You
+    can install `py-spy` and give it root permissions as follows:
+      $ pip install py-spy
+      $ sudo chown root:root `which py-spy`
+      $ sudo chmod u+s `which py-spy`
+
+    Alternatively, you can start Ray with passwordless sudo / root permissions.
+
+.. note::
+   You may run into permission errors when using py-spy in the docker containers. To fix the issue:
+   
+   * If you start Ray manually in a Docker container, follow the `py-spy documentation`_ to resolve it. 
+   * if you are a KubeRay user, follow the :ref:`guide to configure KubeRay <kuberay-pyspy-integration>` and resolve it.
+   
+.. _`py-spy documentation`: https://github.com/benfred/py-spy#how-do-i-run-py-spy-in-docker
+
+
+
+.. _dashboard-cprofile:
+
+Profiling using Python's cProfile
+---------------------------------
+
+You can use Python's native cProfile `profiling module`_ to profile the performance of your Ray application. Rather than tracking
+line-by-line of your application code, cProfile can give the total runtime
+of each loop function, as well as list the number of calls made and
+execution time of all function calls made within the profiled code.
+
+.. _`profiling module`: https://docs.python.org/3/library/profile.html#module-cProfile
+
+Unlike ``line_profiler`` above, this detailed list of profiled function calls
+**includes** internal function calls and function calls made within Ray.
+
+However, similar to ``line_profiler``, cProfile can be enabled with minimal
+changes to your application code (given that each section of the code you want
+to profile is defined as its own function). To use cProfile, add an import
+statement, then replace calls to the loop functions as follows:
+
+.. testcode::
+  :skipif: True
+
+  import cProfile  # Added import statement
+
+  def ex1():
+      list1 = []
+      for i in range(5):
+          list1.append(ray.get(func.remote()))
+
+  def main():
+      ray.init()
+      cProfile.run('ex1()')  # Modified call to ex1
+      cProfile.run('ex2()')
+      cProfile.run('ex3()')
+
+  if __name__ == "__main__":
+      main()
+
+Now, when you execute your Python script, a cProfile list of profiled function
+calls are printed on the terminal for each call made to ``cProfile.run()``.
+At the very top of cProfile's output gives the total execution time for
+``'ex1()'``:
+
+.. code-block:: bash
+
+  601 function calls (595 primitive calls) in 2.509 seconds
+
+Following is a snippet of profiled function calls for ``'ex1()'``. Most of
+these calls are quick and take around 0.000 seconds, so the functions of
+interest are the ones with non-zero execution times:
+
+.. code-block:: bash
+
+  ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+  ...
+      1    0.000    0.000    2.509    2.509 your_script_here.py:31(ex1)
+      5    0.000    0.000    0.001    0.000 remote_function.py:103(remote)
+      5    0.000    0.000    0.001    0.000 remote_function.py:107(_remote)
+  ...
+     10    0.000    0.000    0.000    0.000 worker.py:2459(__init__)
+      5    0.000    0.000    2.508    0.502 worker.py:2535(get)
+      5    0.000    0.000    0.000    0.000 worker.py:2695(get_global_worker)
+     10    0.000    0.000    2.507    0.251 worker.py:374(retrieve_and_deserialize)
+      5    0.000    0.000    2.508    0.502 worker.py:424(get_object)
+      5    0.000    0.000    0.000    0.000 worker.py:514(submit_task)
+  ...
+
+The 5 separate calls to Ray's ``get``, taking the full 0.502 seconds each call,
+can be noticed at ``worker.py:2535(get)``. Meanwhile, the act of calling the
+remote function itself at ``remote_function.py:103(remote)`` only takes 0.001
+seconds over 5 calls, and thus is not the source of the slow performance of
+``ex1()``.
+
+
+Profiling Ray Actors with cProfile
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Considering that the detailed output of cProfile can be quite different depending
+on what Ray functionalities we use, let us see what cProfile's output might look
+like if our example involved Actors (for an introduction to Ray actors, see our
+:ref:`Actor documentation <actor-guide>`).
+
+Now, instead of looping over five calls to a remote function like in ``ex1``,
+let's create a new example and loop over five calls to a remote function
+**inside an actor**. Our actor's remote function again just sleeps for 0.5
+seconds:
+
+.. testcode::
+
+  # Our actor
+  @ray.remote
+  class Sleeper:
+      def __init__(self):
+          self.sleepValue = 0.5
+
+      # Equivalent to func(), but defined within an actor
+      def actor_func(self):
+          time.sleep(self.sleepValue)
+
+Recalling the suboptimality of ``ex1``, let's first see what happens if we
+attempt to perform all five ``actor_func()`` calls within a single actor:
+
+.. testcode::
+
+  def ex4():
+      # This is suboptimal in Ray, and should only be used for the sake of this example
+      actor_example = Sleeper.remote()
+
+      five_results = []
+      for i in range(5):
+          five_results.append(actor_example.actor_func.remote())
+
+      # Wait until the end to call ray.get()
+      ray.get(five_results)
+
+We enable cProfile on this example as follows:
+
+.. testcode::
+  :skipif: True
+
+  def main():
+      ray.init()
+      cProfile.run('ex4()')
+
+  if __name__ == "__main__":
+      main()
+
+Running our new Actor example, cProfile's abbreviated output is as follows:
+
+.. code-block:: bash
+
+  12519 function calls (11956 primitive calls) in 2.525 seconds
+
+  ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+  ...
+  1    0.000    0.000    0.015    0.015 actor.py:546(remote)
+  1    0.000    0.000    0.015    0.015 actor.py:560(_remote)
+  1    0.000    0.000    0.000    0.000 actor.py:697(__init__)
+  ...
+  1    0.000    0.000    2.525    2.525 your_script_here.py:63(ex4)
+  ...
+  9    0.000    0.000    0.000    0.000 worker.py:2459(__init__)
+  1    0.000    0.000    2.509    2.509 worker.py:2535(get)
+  9    0.000    0.000    0.000    0.000 worker.py:2695(get_global_worker)
+  4    0.000    0.000    2.508    0.627 worker.py:374(retrieve_and_deserialize)
+  1    0.000    0.000    2.509    2.509 worker.py:424(get_object)
+  8    0.000    0.000    0.001    0.000 worker.py:514(submit_task)
+  ...
+
+It turns out that the entire example still took 2.5 seconds to execute, or the
+time for five calls to ``actor_func()`` to run in serial. If you recall ``ex1``,
+this behavior was because we did not wait until after submitting all five
+remote function tasks to call ``ray.get()``, but we can verify on cProfile's
+output line ``worker.py:2535(get)`` that ``ray.get()`` was only called once at
+the end, for 2.509 seconds. What happened?
+
+It turns out Ray cannot parallelize this example, because we have only
+initialized a single ``Sleeper`` actor. Because each actor is a single,
+stateful worker, our entire code is submitted and ran on a single worker the
+whole time.
+
+To better parallelize the actors in ``ex4``, we can take advantage
+that each call to ``actor_func()`` is independent, and instead
+create five ``Sleeper`` actors. That way, we are creating five workers
+that can run in parallel, instead of creating a single worker that
+can only handle one call to ``actor_func()`` at a time.
+
+.. testcode::
+
+  def ex4():
+      # Modified to create five separate Sleepers
+      five_actors = [Sleeper.remote() for i in range(5)]
+
+      # Each call to actor_func now goes to a different Sleeper
+      five_results = []
+      for actor_example in five_actors:
+          five_results.append(actor_example.actor_func.remote())
+
+      ray.get(five_results)
+
+Our example in total now takes only 1.5 seconds to run:
+
+.. code-block:: bash
+
+  1378 function calls (1363 primitive calls) in 1.567 seconds
+
+  ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+  ...
+  5    0.000    0.000    0.002    0.000 actor.py:546(remote)
+  5    0.000    0.000    0.002    0.000 actor.py:560(_remote)
+  5    0.000    0.000    0.000    0.000 actor.py:697(__init__)
+  ...
+  1    0.000    0.000    1.566    1.566 your_script_here.py:71(ex4)
+  ...
+  21    0.000    0.000    0.000    0.000 worker.py:2459(__init__)
+  1    0.000    0.000    1.564    1.564 worker.py:2535(get)
+  25    0.000    0.000    0.000    0.000 worker.py:2695(get_global_worker)
+  3    0.000    0.000    1.564    0.521 worker.py:374(retrieve_and_deserialize)
+  1    0.000    0.000    1.564    1.564 worker.py:424(get_object)
+  20    0.001    0.000    0.001    0.000 worker.py:514(submit_task)
+  ...
+
+
+.. _performance-debugging-gpu-profiling:
+
+GPU Profiling with PyTorch Profiler
+-----------------------------------
+Here are the steps to use PyTorch Profiler during training with Ray Train or batch inference with Ray Data:
+
+* Follow the `PyTorch Profiler documentation <https://pytorch.org/tutorials/intermediate/tensorboard_profiler_tutorial.html>`_ to record events in your PyTorch code.
+
+* Convert your PyTorch script to a :ref:`Ray Train training script <train-pytorch>` or a :ref:`Ray Data batch inference script <batch_inference_home>`. (no change to your profiler-related code)
+
+* Run your training or batch inference script.
+
+* Collect the profiling results from all the nodes (compared to 1 node in a non-distributed setting).
+
+  * You may want to upload results on each Node to NFS or object storage like S3 so that you don't have to fetch results from each Node respectively.
+
+* Visualize the results with tools like Tensorboard.
+
+GPU Profiling with Nsight System Profiler
+------------------------------------------
+GPU profiling is critical for ML training and inference. Ray allows users to run Nsight System Profiler with Ray actors and tasks. :ref:`See for details <profiling-nsight-profiler>`.
+
+Profiling for developers
+------------------------
+If you are developing Ray Core or debugging some system level failures, profiling the Ray Core could help. In this case, see :ref:`Profiling for Ray developers <ray-core-internal-profiling>`.
+

doc/source/ray-observability/user-guides/debug-apps/ray-debugging.rst

@@ -0,0 +1,280 @@
+.. _ray-debugger:
+
+Using the Ray Debugger
+======================
+
+Ray has a built in debugger that allows you to debug your distributed applications. It allows
+to set breakpoints in your Ray tasks and actors and when hitting the breakpoint you can
+drop into a PDB session that you can then use to:
+
+- Inspect variables in that context
+- Step within that task or actor
+- Move up or down the stack
+
+.. warning::
+
+    The Ray Debugger is deprecated. Use the :doc:`Ray Distributed Debugger <../../ray-distributed-debugger>` instead.
+    Starting with Ray 2.39, the new debugger is the default and you need to set the environment variable `RAY_DEBUG=legacy` to
+    use the old debugger (e.g. by using a runtime environment).
+
+Getting Started
+---------------
+
+Take the following example:
+
+.. testcode::
+    :skipif: True
+
+    import ray
+
+    ray.init(runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
+
+    @ray.remote
+    def f(x):
+        breakpoint()
+        return x * x
+
+    futures = [f.remote(i) for i in range(2)]
+    print(ray.get(futures))
+
+Put the program into a file named ``debugging.py`` and execute it using:
+
+.. code-block:: bash
+
+    python debugging.py
+
+
+Each of the 2 executed tasks will drop into a breakpoint when the line
+``breakpoint()`` is executed. You can attach to the debugger by running
+the following command on the head node of the cluster:
+
+.. code-block:: bash
+
+    ray debug
+
+The ``ray debug`` command will print an output like this:
+
+.. code-block:: text
+
+    2021-07-13 16:30:40,112	INFO scripts.py:216 -- Connecting to Ray instance at 192.168.2.61:6379.
+    2021-07-13 16:30:40,112	INFO worker.py:740 -- Connecting to existing Ray cluster at address: 192.168.2.61:6379
+    Active breakpoints:
+    index | timestamp           | Ray task | filename:lineno
+    0     | 2021-07-13 23:30:37 | ray::f() | debugging.py:6
+    1     | 2021-07-13 23:30:37 | ray::f() | debugging.py:6
+    Enter breakpoint index or press enter to refresh:
+
+
+You can now enter ``0`` and hit Enter to jump to the first breakpoint. You will be dropped into PDB
+at the break point and can use the ``help`` to see the available actions. Run ``bt`` to see a backtrace
+of the execution:
+
+.. code-block:: text
+
+    (Pdb) bt
+      /home/ubuntu/ray/python/ray/workers/default_worker.py(170)<module>()
+    -> ray.worker.global_worker.main_loop()
+      /home/ubuntu/ray/python/ray/worker.py(385)main_loop()
+    -> self.core_worker.run_task_loop()
+    > /home/ubuntu/tmp/debugging.py(7)f()
+    -> return x * x
+
+You can inspect the value of ``x`` with ``print(x)``. You can see the current source code with ``ll``
+and change stack frames with ``up`` and ``down``. For now let us continue the execution with ``c``.
+
+After the execution is continued, hit ``Control + D`` to get back to the list of break points. Select
+the other break point and hit ``c`` again to continue the execution.
+
+The Ray program ``debugging.py`` now finished and should have printed ``[0, 1]``. Congratulations, you
+have finished your first Ray debugging session!
+
+Running on a Cluster
+--------------------
+
+The Ray debugger supports setting breakpoints inside of tasks and actors that are running across your
+Ray cluster. In order to attach to these from the head node of the cluster using ``ray debug``, you'll
+need to make sure to pass in the ``--ray-debugger-external`` flag to ``ray start`` when starting the
+cluster (likely in your ``cluster.yaml`` file or k8s Ray cluster spec).
+
+Note that this flag will cause the workers to listen for PDB commands on an external-facing IP address,
+so this should *only* be used if your cluster is behind a firewall.
+
+Debugger Commands
+-----------------
+
+The Ray debugger supports the
+`same commands as PDB
+<https://docs.python.org/3/library/pdb.html#debugger-commands>`_.
+
+Stepping between Ray tasks
+--------------------------
+
+You can use the debugger to step between Ray tasks. Let's take the
+following recursive function as an example:
+
+.. testcode::
+    :skipif: True
+
+    import ray
+
+    ray.init(runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
+
+    @ray.remote
+    def fact(n):
+        if n == 1:
+            return n
+        else:
+            n_ref = fact.remote(n - 1)
+            return n * ray.get(n_ref)
+
+    @ray.remote
+    def compute():
+        breakpoint()
+        result_ref = fact.remote(5)
+        result = ray.get(result_ref)
+
+    ray.get(compute.remote())
+
+
+After running the program by executing the Python file and calling
+``ray debug``, you can select the breakpoint by pressing ``0`` and
+enter. This will result in the following output:
+
+.. code-block:: shell
+
+    Enter breakpoint index or press enter to refresh: 0
+    > /home/ubuntu/tmp/stepping.py(16)<module>()
+    -> result_ref = fact.remote(5)
+    (Pdb)
+
+You can jump into the call with the ``remote`` command in Ray's debugger.
+Inside the function, print the value of `n` with ``p(n)``, resulting in
+the following output:
+
+.. code-block:: shell
+
+    -> result_ref = fact.remote(5)
+    (Pdb) remote
+    *** Connection closed by remote host ***
+    Continuing pdb session in different process...
+    --Call--
+    > /home/ubuntu/tmp/stepping.py(5)fact()
+    -> @ray.remote
+    (Pdb) ll
+      5  ->	@ray.remote
+      6  	def fact(n):
+      7  	    if n == 1:
+      8  	        return n
+      9  	    else:
+     10  	        n_ref = fact.remote(n - 1)
+     11  	        return n * ray.get(n_ref)
+    (Pdb) p(n)
+    5
+    (Pdb)
+
+Now step into the next remote call again with
+``remote`` and print `n`. You an now either continue recursing into
+the function by calling ``remote`` a few more times, or you can jump
+to the location where ``ray.get`` is called on the result by using the
+``get`` debugger command. Use ``get`` again to jump back to the original
+call site and use ``p(result)`` to print the result:
+
+.. code-block:: shell
+
+    Enter breakpoint index or press enter to refresh: 0
+    > /home/ubuntu/tmp/stepping.py(14)<module>()
+    -> result_ref = fact.remote(5)
+    (Pdb) remote
+    *** Connection closed by remote host ***
+    Continuing pdb session in different process...
+    --Call--
+    > /home/ubuntu/tmp/stepping.py(5)fact()
+    -> @ray.remote
+    (Pdb) p(n)
+    5
+    (Pdb) remote
+    *** Connection closed by remote host ***
+    Continuing pdb session in different process...
+    --Call--
+    > /home/ubuntu/tmp/stepping.py(5)fact()
+    -> @ray.remote
+    (Pdb) p(n)
+    4
+    (Pdb) get
+    *** Connection closed by remote host ***
+    Continuing pdb session in different process...
+    --Return--
+    > /home/ubuntu/tmp/stepping.py(5)fact()->120
+    -> @ray.remote
+    (Pdb) get
+    *** Connection closed by remote host ***
+    Continuing pdb session in different process...
+    --Return--
+    > /home/ubuntu/tmp/stepping.py(14)<module>()->None
+    -> result_ref = fact.remote(5)
+    (Pdb) p(result)
+    120
+    (Pdb)
+
+
+Post Mortem Debugging
+---------------------
+
+Often we do not know in advance where an error happens, so we cannot set a breakpoint. In these cases,
+we can automatically drop into the debugger when an error occurs or an exception is thrown. This is called *post-mortem debugging*.
+
+Copy the following code into a file called ``post_mortem_debugging.py``. The flag ``RAY_DEBUG_POST_MORTEM=1`` will have the effect
+that if an exception happens, Ray will drop into the debugger instead of propagating it further.
+
+.. testcode::
+    :skipif: True
+
+    import ray
+
+    ray.init(runtime_env={"env_vars": {"RAY_DEBUG": "legacy", "RAY_DEBUG_POST_MORTEM": "1"}})
+
+    @ray.remote
+    def post_mortem(x):
+        x += 1
+        raise Exception("An exception is raised.")
+        return x
+
+    ray.get(post_mortem.remote(10))
+
+Let's start the program:
+
+.. code-block:: bash
+
+    python post_mortem_debugging.py
+
+Now run ``ray debug``. After we do that, we see an output like the following:
+
+.. code-block:: text
+
+    Active breakpoints:
+    index | timestamp           | Ray task                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | filename:lineno
+    0     | 2024-11-01 20:14:00 | /Users/pcmoritz/ray/python/ray/_private/workers/default_worker.py --node-ip-address=127.0.0.1 --node-manager-port=49606 --object-store-name=/tmp/ray/session_2024-11-01_13-13-51_279910_8596/sockets/plasma_store --raylet-name=/tmp/ray/session_2024-11-01_13-13-51_279910_8596/sockets/raylet --redis-address=None --metrics-agent-port=58655 --runtime-env-agent-port=56999 --logging-rotate-bytes=536870912 --logging-rotate-backup-count=5 --runtime-env-agent-port=56999 --gcs-address=127.0.0.1:6379 --session-name=session_2024-11-01_13-13-51_279910_8596 --temp-dir=/tmp/ray --webui=127.0.0.1:8265 --cluster-id=6d341469ae0f85b6c3819168dde27cceda12e95c8efdfc256e0fd8ce --startup-token=12 --worker-launch-time-ms=1730492039955 --node-id=0d43573a606286125da39767a52ce45ad101324c8af02cc25a9fbac7 --runtime-env-hash=-1746935720 | /Users/pcmoritz/ray/python/ray/_private/worker.py:920
+    Traceback (most recent call last):
+
+    File "python/ray/_raylet.pyx", line 1856, in ray._raylet.execute_task
+
+    File "python/ray/_raylet.pyx", line 1957, in ray._raylet.execute_task
+
+    File "python/ray/_raylet.pyx", line 1862, in ray._raylet.execute_task
+
+    File "/Users/pcmoritz/ray-debugger-test/post_mortem_debugging.py", line 8, in post_mortem
+        raise Exception("An exception is raised.")
+
+    Exception: An exception is raised.
+
+    Enter breakpoint index or press enter to refresh:
+
+We now press ``0`` and then Enter to enter the debugger. With ``ll`` we can see the context and with
+``print(x)`` we an print the value of ``x``.
+
+In a similar manner as above, you can also debug Ray actors. Happy debugging!
+
+Debugging APIs
+--------------
+
+See :ref:`package-ref-debugging-apis`.

doc/source/ray-observability/user-guides/index.md

@@ -0,0 +1,25 @@
+(observability-user-guides)=
+
+# User Guides
+
+```{toctree}
+:hidden:
+
+Debugging Applications <debug-apps/index>
+cli-sdk
+configure-logging
+profiling
+add-app-metrics
+ray-tracing
+ray-event-export
+```
+
+These guides help you monitor and debug your Ray applications and clusters.
+
+The guides include:
+* {ref}`observability-debug-apps`
+* {ref}`observability-programmatic`
+* {ref}`configure-logging`
+* {ref}`application-level-metrics`
+* {ref}`ray-tracing`
+* {ref}`ray-event-export`

doc/source/ray-observability/user-guides/profiling.md

@@ -0,0 +1,190 @@
+(profiling)=
+# Profiling
+Profiling is one of the most important debugging tools to diagnose performance, out of memory, hanging, or other application issues. Here is a list of common profiling tools you may use when debugging Ray applications.
+- CPU profiling
+    - py-spy
+- Memory profiling
+    - memray
+- GPU profiling
+    - PyTorch Profiler
+    - Nsight System
+- Ray Task / Actor timeline
+
+If Ray doesn't work with certain profiling tools, try running them without Ray to debug the issues.
+
+(profiling-enabling)=
+## Enabling dashboard profiling
+
+The Ray Dashboard's built-in profiling features (CPU flame graphs, stack traces, and memory profiling) are disabled by default for security reasons. These endpoints trigger profiling work on Ray workers on demand and return the results. On deployments where the dashboard is exposed without authentication, a malicious web page could exploit DNS rebinding to reach these endpoints from a browser.
+
+To enable dashboard profiling, set the following environment variable on the Ray head node before starting Ray:
+
+```bash
+export RAY_DASHBOARD_ENABLE_PROFILING=1
+```
+
+:::{warning}
+If your dashboard is accessible over a network without authentication, enabling profiling exposes side-effecting endpoints to potential abuse. Enable {ref}`token authentication <token-auth>` when using profiling on an exposed dashboard.
+:::
+
+(profiling-cpu)=
+## CPU profiling
+Profile the CPU usage for Driver and Worker processes. This helps you understand the CPU usage by different processes and debug unexpectedly high or low usage.
+
+(profiling-pyspy)=
+### py-spy
+[py-spy](https://github.com/benfred/py-spy/tree/master) is a sampling profiler for Python programs. Ray Dashboard has native integration with pyspy:
+
+- It lets you visualize what your Python program is spending time on without restarting the program or modifying the code in any way.
+- It dumps the stacktrace of the running process so that you can see what the process is doing at a certain time. It is useful when programs hangs.
+
+:::{note}
+You may run into permission errors when using py-spy in the docker containers. To fix the issue:
+
+- if you start Ray manually in a Docker container, follow the `py-spy documentation`_ to resolve it. 
+- if you are a KubeRay user, follow the {ref}`guide to configure KubeRay <kuberay-pyspy-integration>` and resolve it.
+:::
+
+Here are the {ref}`steps to use py-spy with Ray and Ray Dashboard <observability-debug-hangs>`.
+
+(profiling-cprofile)=
+### cProfile
+cProfile is Python’s native profiling module to profile the performance of your Ray application.
+
+Here are the {ref}`steps to use cProfile <dashboard-cprofile>`.
+
+(profiling-memory)=
+## Memory profiling
+Profile the memory usage for Driver and Worker processes. This helps you analyze memory allocations in applications, trace memory leaks, and debug high/low memory or out of memory issues.
+
+(profiling-memray)=
+### memray
+memray is a memory profiler for Python. It can track memory allocations in Python code, in native extension modules, and in the Python interpreter itself.
+
+Here are the {ref}`steps to profile the memory usage of Ray Tasks and Actors <memray-profiling>`.
+
+#### Ray Dashboard View
+You can now do memory profiling for Ray Driver or Worker processes in the Ray Dashboard, by clicking on the "Memory profiling” actions for active Worker processes, Tasks, Actors, and a Job’s driver process.
+
+![memory profiling action](../images/memory-profiling-dashboard-view.png)
+
+Additionally, you can specify the following profiling Memray parameters from the dashboard view:
+- **Format:** Format of the profiling result. The value is either "flamegraph" or "table"
+- **Duration:** Duration to track for (in seconds)
+- **Leaks:** Enables the Memory Leaks View, which displays memory that Ray didn't deallocate, instead of peak memory usage
+- **Natives:** Track native (C/C++) stack frames (only supported in Linux)
+- **Python Allocator Tracing:** Record allocations made by the pymalloc allocator
+
+
+(profiling-gpu)=
+## GPU profiling
+GPU and GRAM profiling for your GPU workloads like distributed training. This helps you analyze performance and debug memory issues. 
+- PyTorch profiler is supported out of box when used with Ray Train
+- NVIDIA Nsight System is natively supported on Ray.
+
+(profiling-pytorch-profiler)=
+### PyTorch Profiler
+PyTorch Profiler is a tool that allows the collection of performance metrics (especially GPU metrics) during training and inference.
+
+Here are the {ref}`steps to use PyTorch Profiler with Ray Train or Ray Data <performance-debugging-gpu-profiling>`.
+
+(profiling-nsight-profiler)=
+### Nsight System Profiler
+
+#### Installation
+
+First, install the Nsight System CLI by following the [Nsight User Guide](https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html). 
+
+Confirm that you installed Nsight correctly:
+
+```bash
+$ nsys --version
+
+# NVIDIA Nsight Systems version 2022.4.1.21-0db2c85
+```
+
+(run-nsight-on-ray)=
+#### Run Nsight on Ray
+
+To enable GPU profiling, specify the config in the `runtime_env` as follows:
+
+```python
+import torch
+import ray
+
+ray.init()
+
+@ray.remote(num_gpus=1, runtime_env={ "nsight": "default"})
+class RayActor:
+    def run(self):
+        a = torch.tensor([1.0, 2.0, 3.0]).cuda()
+        b = torch.tensor([4.0, 5.0, 6.0]).cuda()
+        c = a * b
+
+        print("Result on GPU:", c)
+
+ray_actor = RayActor.remote()
+# The Actor or Task process runs with : "nsys profile [default options] ..."
+ray.get(ray_actor.run.remote())
+```
+
+You can find the `"default"` config in [nsight.py](https://github.com/ray-project/ray/blob/master/python/ray/_private/runtime_env/nsight.py#L20).
+
+#### Custom options
+
+You can also add [custom options](https://docs.nvidia.com/nsight-systems/UserGuide/index.html#cli-profile-command-switch-options) for Nsight System Profiler by specifying a dictionary of option values, which overwrites the `default` config, however, Ray preserves the `--output` option of the default config.
+
+
+```python
+import torch
+import ray
+
+ray.init()
+
+@ray.remote(
+num_gpus=1, 
+runtime_env={ "nsight": {
+    "t": "cuda,cudnn,cublas",
+    "cuda-memory-usage": "true",
+    "cuda-graph-trace": "graph",
+}})
+class RayActor:
+    def run(self):
+        a = torch.tensor([1.0, 2.0, 3.0]).cuda()
+        b = torch.tensor([4.0, 5.0, 6.0]).cuda()
+        c = a * b
+
+        print("Result on GPU:", c)
+
+ray_actor = RayActor.remote()
+
+# The Actor or Task process runs with :
+# "nsys profile -t cuda,cudnn,cublas --cuda-memory-usage=True --cuda-graph-trace=graph ..."
+ray.get(ray_actor.run.remote())
+```
+
+**Note:**: The default report filename (`-o, --output`) is `worker_process_{pid}.nsys-rep` in the logs dir.
+
+(profiling-result)=
+#### Profiling result
+
+Find profiling results under the `/tmp/ray/session_*/logs/{profiler_name}` directory. This specific directory location may change in the future. You can download the profiling reports from the {ref}`Ray Dashboard <dash-logs-view>`.
+
+![Nsight System Profiler folder](../images/nsight-profiler-folder.png)
+
+To visualize the results, install the [Nsight System GUI](https://developer.nvidia.com/nsight-systems/get-started#latest-Platforms) on your laptop, which becomes the host. Transfer the .nsys-rep file to your host and open it using the GUI. You can now view the visual profiling info.
+
+**Note**: The Nsight System Profiler output (-o, --output) option allows you to set the path to a filename. Ray uses the logs directory as the base and appends the output option to it. For example: 
+```
+--output job_name/ray_worker -> /tmp/ray/session_*/logs/nsight/job_name/ray_worker
+
+--output /Users/Desktop/job_name/ray_worker -> /Users/Desktop/job_name/ray_worker
+```
+The best practice is to only specify the filename in output option.
+
+
+(profiling-timeline)=
+## Ray Task or Actor timeline
+Ray Timeline profiles the execution time of Ray Tasks and Actors. This helps you analyze performance, identify the stragglers, and understand the distribution of workloads.
+
+Open your Ray Job in Ray Dashboard and follow the {ref}`instructions to download and visualize the trace files <dashboard-timeline>` generated by Ray Timeline.

doc/source/ray-observability/user-guides/ray-event-export.rst

@@ -0,0 +1,294 @@
+.. _ray-event-export:
+
+Ray Event Export
+================
+
+Starting from 2.49, Ray supports exporting structured events to a configured HTTP 
+endpoint. Each node sends events to the endpoint through an HTTP POST request.
+
+Previously, Ray's :ref:`task events <task-events>` were only used internally by the Ray Dashboard 
+and :ref:`State API <state-api-overview-ref>` for monitoring and debugging. With the new event 
+export feature, you can now send these raw events to external systems for custom analytics, 
+monitoring, and integration with third-party tools.
+
+.. note:: 
+    Ray Event Export is still in alpha. The way to configure event 
+    reporting and the format of the events is subject to change.
+
+Enable event reporting
+----------------------
+To enable event reporting, you need to set the ``RAY_enable_core_worker_ray_event_to_aggregator`` environment 
+variable to ``1`` when starting each Ray worker node.
+
+To set the target HTTP endpoint, set the ``RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR`` 
+environment variable to a valid HTTP URL with the ``http://`` URL scheme.
+
+Event format
+------------
+
+Events are JSON objects in the POST request body.
+
+All events contain the same base fields and different event specific fields. 
+See `src/ray/protobuf/public/events_base_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_base_event.proto>`_ for the base fields.
+
+Task events
+^^^^^^^^^^^
+
+For each task, Ray exports two types of events: Task Definition Event and Task Execution Event.
+
+* Each task attempt generates one Task Definition Event which contains the metadata of the task. 
+  See `src/ray/protobuf/public/events_task_definition_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_task_definition_event.proto>`_ 
+  and `src/ray/protobuf/public/events_actor_task_definition_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_actor_task_definition_event.proto>`_ for the event formats for normal tasks 
+  and actor tasks respectively.
+* Task Execution Events contain task state transition information and metadata 
+  generated during task execution. 
+  See `src/ray/protobuf/public/events_task_lifecycle_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_task_lifecycle_event.proto>`_ for the event format.
+
+An example of a Task Definition Event and a Task Execution Event:
+
+.. code-block:: json
+
+    // task definition event
+    {
+        "eventId":"N5n229xkwyjlZRFJDF2G1sh6ZNYlqChwJ4WPEQ==",
+        "sourceType":"CORE_WORKER",
+        "eventType":"TASK_DEFINITION_EVENT",
+        "timestamp":"2025-09-03T18:52:14.467290Z",
+        "severity":"INFO",
+        "sessionName":"session_2025-09-03_11-52-12_635210_85618",
+        "taskDefinitionEvent":{
+            "taskId":"yO9FzNARJXH///////////////8BAAAA",
+            "taskFunc":{
+                "pythonFunctionDescriptor":{
+                    "moduleName":"test-tasks",
+                    "functionName":"test_task",
+                    "functionHash":"37ddb110c0514b049bd4db5ab934627b",
+                    "className":""
+                }
+            },
+            "taskName":"test_task",
+            "requiredResources":{
+                "CPU":1.0
+            },
+            "serialized_runtime_env": "{}",
+            "jobId":"AQAAAA==",
+            "parentTaskId":"//////////////////////////8BAAAA",
+            "placementGroupId":"////////////////////////",
+            "taskAttempt":0,
+            "taskType":"NORMAL_TASK",
+            "language":"PYTHON",
+            "refIds":{
+                
+            }
+        },
+        "nodeId":"ZvxTI6x9dlMFqMlIHErJpg5UEGK1INsKhW2zyg==",
+        "message":""
+    }
+
+    // task lifecycle event
+    {
+        "eventId":"vkIaAHlQC5KoppGosqs2kBq5k2WzsAAbawDDbQ==",
+        "sourceType":"CORE_WORKER",
+        "eventType":"TASK_LIFECYCLE_EVENT",
+        "timestamp":"2025-09-03T18:52:14.469074Z",
+        "severity":"INFO",
+        "sessionName":"session_2025-09-03_11-52-12_635210_85618",
+        "taskLifecycleEvent":{
+            "taskId":"yO9FzNARJXH///////////////8BAAAA",
+            "stateTransitions": [
+                {
+                    "state":"PENDING_NODE_ASSIGNMENT",
+                    "timestamp":"2025-09-03T18:52:14.467402Z"
+                },
+                {
+                    "state":"PENDING_ARGS_AVAIL",
+                    "timestamp":"2025-09-03T18:52:14.467290Z"
+                },
+                {
+                    "state":"SUBMITTED_TO_WORKER",
+                    "timestamp":"2025-09-03T18:52:14.469074Z"
+                }
+            ],
+            "nodeId":"ZvxTI6x9dlMFqMlIHErJpg5UEGK1INsKhW2zyg==",
+            "workerId":"hMybCNYIFi+/yInYYhdc+qH8yMF65j/8+uCTmw==",
+            "jobId":"AQAAAA==",
+            "taskAttempt":0,
+            "workerPid":0
+        },
+        "nodeId":"ZvxTI6x9dlMFqMlIHErJpg5UEGK1INsKhW2zyg==",
+        "message":""
+    }
+
+Actor events
+^^^^^^^^^^^^
+
+For each actor, Ray exports two types of events: Actor Definition Events and Actor Lifecycle Events.
+
+* An Actor Definition Event contains the metadata of the actor when it is defined. See `src/ray/protobuf/public/events_actor_definition_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_actor_definition_event.proto>`_ for the event format.
+* An Actor Lifecycle Event contains the actor state transition information and metadata associated with each transition. See `src/ray/protobuf/public/events_actor_lifecycle_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_actor_lifecycle_event.proto>`_ for the event format.
+
+.. code-block:: json
+
+    // actor definition event
+    {
+        "eventId": "gsRtAfaWn5TZsjUPFm8nOXd/cKGz82FXdr3Lqg==",
+        "sourceType": "GCS",
+        "eventType": "ACTOR_DEFINITION_EVENT",
+        "timestamp": "2025-10-24T21:12:10.742651Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-12-05_804800_55420",
+        "actorDefinitionEvent": {
+            "actorId": "0AFtngcXtEoxwqmJAQAAAA==", 
+            "jobId": "AQAAAA==", 
+            "name": "actor-test",
+            "rayNamespace": "bd2ad7f8-650b-495c-b709-55d4c8a7d09f",
+            "serializedRuntimeEnv": "{}",
+            "className": "test_ray_actor_events.<locals>.A",
+            "isDetached": false,
+            "requiredResources": {},
+            "placementGroupId": "",
+            "labelSelector": {}
+        },
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+    // actor lifecycle event
+    {
+        "eventId": "mOdfn5SRx3X0B05OvEDV0rcIOzqf/SGBJmrD/Q==",
+        "sourceType": "GCS",
+        "eventType": "ACTOR_LIFECYCLE_EVENT",
+        "timestamp": "2025-10-24T21:12:10.742654Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-12-05_804800_55420",
+        "actorLifecycleEvent": {
+            "actorId": "0AFtngcXtEoxwqmJAQAAAA==",
+            "stateTransitions": [
+                {
+                    "timestamp": "2025-10-24T21:12:10.742654Z",
+                    "state": "ALIVE",
+                    "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+                    "workerId": "nrBehSG3HXu0PvHZBkPl2kovmjzAaoCuVj2KHA=="
+                }
+            ]
+        },
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+Driver job events
+^^^^^^^^^^^^^^^^^^
+
+For each driver job, Ray exports two types of events: Driver Job Definition Events and Driver Job Lifecycle Events.
+
+* A Driver Job Definition Event contains the metadata of the driver job when it is defined. See `src/ray/protobuf/public/events_driver_job_definition_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_driver_job_definition_event.proto>`_ for the event format.
+* A Driver Job Lifecycle Event contains the driver job state transition information and metadata associated with each transition. See `src/ray/protobuf/public/events_driver_job_lifecycle_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_driver_job_lifecycle_event.proto>`_ for the event format.
+
+.. code-block:: json
+
+    // driver job definition event
+    {
+        "eventId": "7YnwZPJr0KUC28T7KnzsvGyceEIrjNDTHuQfrg==",
+        "sourceType": "GCS",
+        "eventType": "DRIVER_JOB_DEFINITION_EVENT",
+        "timestamp": "2025-10-24T21:17:07.316482Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-17-05_575968_59360",
+        "driverJobDefinitionEvent": {
+            "jobId": "AQAAAA==", 
+            "driverPid": "59360", 
+            "driverNodeId": "9eHWUIruJWnMjQuPas0W+TRNUyjY5PwFpWUfjA==", 
+            "entrypoint": "...", 
+            "config": {
+                "serializedRuntimeEnv": "{}", 
+                "metadata": {}
+            }
+        },
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+    // driver job lifecycle event
+    {
+        "eventId": "0cmbCI/RQghYe4ZQiJ+HrnK1RiZH+cg8ltBx2w==",
+        "sourceType": "GCS",
+        "eventType": "DRIVER_JOB_LIFECYCLE_EVENT",
+        "timestamp": "2025-10-24T21:17:07.316483Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-17-05_575968_59360",
+        "driverJobLifecycleEvent": {
+            "jobId": "AQAAAA==", 
+            "stateTransitions": [
+                {
+                    "state": "CREATED", 
+                    "timestamp": "2025-10-24T21:17:07.316483Z"
+                }
+            ]
+        },
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+Node events
+^^^^^^^^^^^
+
+For each node, Ray exports two types of events: Node Definition Events and Node Lifecycle Events.
+
+* A Node Definition Event contains the metadata of the node when it is defined. See `src/ray/protobuf/public/events_node_definition_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_node_definition_event.proto>`_ for the event format.
+* A Node Lifecycle Event contains the node state transition information and metadata associated with each transition. See `src/ray/protobuf/public/events_node_lifecycle_event.proto <https://github.com/ray-project/ray/blob/master/src/ray/protobuf/public/events_node_lifecycle_event.proto>`_ for the event format.
+
+.. code-block:: json
+
+    // node definition event
+    {
+        "eventId": "l7r4gwq4UPhmZGFJYEym6mUkcxqafra60LB6/Q==",
+        "sourceType": "GCS",
+        "eventType": "NODE_DEFINITION_EVENT",
+        "timestamp": "2025-10-24T21:19:14.063953Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-19-12_675240_61141",
+        "nodeDefinitionEvent": {
+            "nodeId": "0yfRX1ex+VtcC+TFXjXcgesdpnEwM76+pEATrQ==", 
+            "nodeIpAddress": "127.0.0.1", 
+            "labels": {
+                "ray.io/node-id": "d327d15f57b1f95b5c0be4c55e35dc81eb1da6713033bebea44013ad"
+            }, 
+            "startTimestamp": "2025-10-24T21:19:14.063Z"
+        }, 
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+    // node lifecycle event
+    {
+        "eventId": "u3KTG8615MIKBH5PLcii0BMfGFWcvLuSOXM6zg==",
+        "sourceType": "GCS",
+        "eventType": "NODE_LIFECYCLE_EVENT",
+        "timestamp": "2025-10-24T21:19:14.063955Z",
+        "severity": "INFO",
+        "sessionName": "session_2025-10-24_14-19-12_675240_61141",
+        "nodeLifecycleEvent": {
+            "nodeId": "0yfRX1ex+VtcC+TFXjXcgesdpnEwM76+pEATrQ==", 
+            "stateTransitions": [
+                {
+                    "timestamp": "2025-10-24T21:19:14.063955Z", 
+                    "resources": {"node:__internal_head__": 1.0, "CPU": 1.0, "object_store_memory": 157286400.0, "node:127.0.0.1": 1.0, "memory": 42964287488.0}, 
+                    "state": "ALIVE", 
+                    "aliveSubState": "UNSPECIFIED"
+                }
+            ]
+        },
+        "nodeId": "zpLG7coqThVMl8df9RYHnhK6thhJqrgPodtfjg==",
+        "message": ""
+    }
+
+High-level Architecture
+-----------------------
+
+The following diagram shows the high-level architecture of Ray Event Export.
+
+.. image:: ../images/ray-event-export.png
+
+All Ray components send events to an aggregator agent through gRPC. There is an aggregator
+agent on each node. The aggregator agent collects all events on that node and sends the
+events to the configured HTTP endpoint. 

doc/source/ray-observability/user-guides/ray-tracing.rst

@@ -0,0 +1,121 @@
+.. _ray-tracing:
+
+Tracing
+=======
+To help debug and monitor Ray applications, Ray integrates with OpenTelemetry to facilitate exporting traces to external tracing stacks such as Jaeger.
+
+
+.. note::
+
+    Tracing is an Alpha feature and no longer under active development/being maintained. APIs are subject to change.
+
+Installation
+------------
+First, install OpenTelemetry:
+
+.. code-block:: shell
+
+    pip install opentelemetry-api==1.34.1
+    pip install opentelemetry-sdk==1.34.1
+    pip install opentelemetry-exporter-otlp==1.34.1
+
+Tracing startup hook
+--------------------
+To enable tracing, you must provide a tracing startup hook with a function that sets up the :ref:`Tracer Provider <tracer-provider>`, :ref:`Remote Span Processors <remote-span-processors>`, and :ref:`Additional Instruments <additional-instruments>`. The tracing startup hook is expected to be a function that is called with no args or kwargs. This hook needs to be available in the Python environment of all the worker processes.
+
+Below is an example tracing startup hook that sets up the default tracing provider, exports spans to files in ``/tmp/spans``, and does not have any additional instruments.
+
+.. testcode::
+
+  import ray
+  import os
+  from opentelemetry import trace
+  from opentelemetry.sdk.trace import TracerProvider
+  from opentelemetry.sdk.trace.export import (
+      ConsoleSpanExporter,
+      SimpleSpanProcessor,
+  )
+
+
+  def setup_tracing() -> None:
+      # Creates /tmp/spans folder
+      os.makedirs("/tmp/spans", exist_ok=True)
+      # Sets the tracer_provider. This is only allowed once per execution
+      # context and will log a warning if attempted multiple times.
+      trace.set_tracer_provider(TracerProvider())
+      trace.get_tracer_provider().add_span_processor(
+          SimpleSpanProcessor(
+              ConsoleSpanExporter(
+                  out=open(f"/tmp/spans/{os.getpid()}.json", "a")
+                  )
+          )
+      )
+
+
+For open-source users who want to experiment with tracing, Ray has a default tracing startup hook that exports spans to the folder ``/tmp/spans``. To run using this default hook, run the following code sample to set up tracing and trace a simple Ray Task.
+
+.. tab-set::
+
+    .. tab-item:: ray start
+
+        .. code-block:: shell
+
+            $ ray start --head --tracing-startup-hook=ray.util.tracing.setup_local_tmp_tracing:setup_tracing
+            $ python
+                ray.init()
+                @ray.remote
+                def my_function():
+                    return 1
+
+                obj_ref = my_function.remote()
+
+    .. tab-item:: ray.init()
+
+        .. testcode::
+
+            ray.init(_tracing_startup_hook="ray.util.tracing.setup_local_tmp_tracing:setup_tracing")
+
+            @ray.remote
+            def my_function():
+                return 1
+
+            obj_ref = my_function.remote()
+
+If you want to provide your own custom tracing startup hook, provide it in the format of ``module:attribute`` where the attribute is the ``setup_tracing`` function to be run.
+
+.. _tracer-provider:
+
+Tracer provider
+~~~~~~~~~~~~~~~
+This configures how to collect traces. View the TracerProvider API `here <https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.html#opentelemetry.sdk.trace.TracerProvider>`__.
+
+.. _remote-span-processors:
+
+Remote span processors
+~~~~~~~~~~~~~~~~~~~~~~
+This configures where to export traces to. View the SpanProcessor API `here <https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.html#opentelemetry.sdk.trace.SpanProcessor>`__.
+
+Users who want to experiment with tracing can configure their remote span processors to export spans to a local JSON file. Serious users developing locally can push their traces to Jaeger containers via the `Jaeger exporter <https://opentelemetry.io/docs/languages/js/exporters/#jaeger>`_.
+
+.. _additional-instruments:
+
+Additional instruments
+~~~~~~~~~~~~~~~~~~~~~~
+If you are using a library that has built-in tracing support, the ``setup_tracing`` function you provide should also patch those libraries. You can find more documentation for the instrumentation of these libraries `here <https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation>`_.
+
+Custom traces
+*************
+Add custom tracing in your programs. Within your program, get the tracer object with ``trace.get_tracer(__name__)`` and start a new span with ``tracer.start_as_current_span(...)``.
+
+See below for a simple example of adding custom tracing.
+
+.. testcode::
+
+  from opentelemetry import trace
+
+  @ray.remote
+  def my_func():
+      tracer = trace.get_tracer(__name__)
+
+      with tracer.start_as_current_span("foo"):
+          print("Hello world from OpenTelemetry Python!")