wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/dashboard/modules/metrics/dashboards/__init__.py
View File
python/ray/dashboard/modules/metrics/dashboards/common.py
+615
View File
@@ -0,0 +1,615 @@
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional
from ray.util.annotations import DeveloperAPI
@DeveloperAPI
@dataclass
class GridPos:
x: int
y: int
w: int
h: int
GRAPH_TARGET_TEMPLATE = {
"exemplar": True,
"expr": "0",
"interval": "",
"legendFormat": "",
"queryType": "randomWalk",
"refId": "A",
}
HEATMAP_TARGET_TEMPLATE = {
"format": "heatmap",
"fullMetaSearch": False,
"includeNullMetadata": True,
"instant": False,
"range": True,
"useBackend": False,
}
HISTOGRAM_BAR_CHART_TARGET_TEMPLATE = {
"exemplar": True,
"format": "heatmap",
"fullMetaSearch": False,
"includeNullMetadata": True,
"instant": True,
"range": False,
"useBackend": False,
}
@DeveloperAPI
class TargetTemplate(Enum):
GRAPH = GRAPH_TARGET_TEMPLATE
HEATMAP = HEATMAP_TARGET_TEMPLATE
HISTOGRAM_BAR_CHART = HISTOGRAM_BAR_CHART_TARGET_TEMPLATE
@DeveloperAPI
@dataclass
class Target:
"""Defines a Grafana target (time-series query) within a panel.
A panel will have one or more targets. By default, all targets are rendered as
stacked area charts, with the exception of legend="MAX", which is rendered as
a blue dotted line. Any legend="FINISHED|FAILED|DEAD|REMOVED" series will also be
rendered hidden by default.
Attributes:
expr: The prometheus query to evaluate.
legend: The legend string to format for each time-series.
"""
expr: str
legend: str
template: Optional[TargetTemplate] = TargetTemplate.GRAPH
HEATMAP_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
"id": 12,
"options": {
"calculate": False,
"cellGap": 1,
"cellValues": {"unit": "none"},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"min": 0,
"mode": "scheme",
"reverse": False,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64,
},
"exemplars": {"color": "rgba(255,0,255,0.7)"},
"filterValues": {"le": 1e-9},
"legend": {"show": True},
"rowsFrame": {"layout": "auto", "value": "Value"},
"tooltip": {"mode": "single", "showColorScale": False, "yHistogram": True},
"yAxis": {"axisPlacement": "left", "reverse": False, "unit": "none"},
},
"pluginVersion": "11.2.0",
"targets": [],
"title": "<Title>",
"type": "heatmap",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
GRAPH_PANEL_TEMPLATE = {
"aliasColors": {},
"bars": False,
"dashLength": 10,
"dashes": False,
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
# Setting height and width is important here to ensure the default panel has some size to it.
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"fill": 10,
"fillGradient": 0,
"hiddenSeries": False,
"id": 26,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"hideEmpty": False,
"hideZero": True,
"max": True,
"min": False,
"rightSide": False,
"show": True,
"sort": "current",
"sortDesc": True,
"total": False,
"values": True,
},
"lines": True,
"linewidth": 1,
"nullPointMode": None,
"options": {"alertThreshold": True},
"percentage": False,
"pluginVersion": "7.5.17",
"pointradius": 2,
"points": False,
"renderer": "flot",
# These series overrides are necessary to make the "MAX" and "MAX + PENDING" dotted lines
# instead of stacked filled areas.
"seriesOverrides": [
{
"$$hashKey": "object:2987",
"alias": "MAX",
"dashes": True,
"color": "#1F60C4",
"fill": 0,
"stack": False,
},
{
"$$hashKey": "object:78",
"alias": "/FINISHED|FAILED|DEAD|REMOVED|Failed Nodes:/",
"hiddenSeries": True,
},
{
"$$hashKey": "object:2987",
"alias": "MAX + PENDING",
"dashes": True,
"color": "#777777",
"fill": 0,
"stack": False,
},
{
"alias": "/Container/",
"hiddenSeries": True,
},
{
"alias": "Container MAX",
"dashes": True,
"color": "#73BF69",
"fill": 0,
"stack": False,
"hiddenSeries": True,
},
],
"spaceLength": 10,
"stack": True,
"steppedLine": False,
"targets": [],
"thresholds": [],
"timeFrom": None,
"timeRegions": [],
"timeShift": None,
"title": "<Title>",
"tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
"type": "graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": [],
},
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
"yaxis": {"align": False, "alignLevel": None},
}
STAT_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "percentage",
"steps": [
{"color": "super-light-yellow", "value": None},
{"color": "super-light-green", "value": 50},
{"color": "green", "value": 100},
],
},
"unit": "short",
},
"overrides": [],
},
"id": 78,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"text": {},
"textMode": "auto",
},
"pluginVersion": "7.5.17",
"targets": [],
"timeFrom": None,
"timeShift": None,
"title": "<Title>",
"type": "stat",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "Tokens",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
GAUGE_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"fieldConfig": {
"defaults": {
"color": {"mode": "continuous-YlBl"},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [{"color": "rgb(230, 230, 230)", "value": None}],
},
"unit": "short",
},
"overrides": [],
},
"id": 10,
"options": {
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"showThresholdLabels": False,
"showThresholdMarkers": False,
"text": {"titleSize": 12},
},
"pluginVersion": "7.5.17",
"targets": [],
"title": "<Title>",
"type": "gauge",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "Tokens",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
PIE_CHART_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
"id": 26,
"options": {
"displayLabels": [],
"legend": {
"displayMode": "table",
"placement": "right",
"values": ["percent", "value"],
},
"pieType": "pie",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"text": {},
},
"pluginVersion": "7.5.17",
"targets": [],
"timeFrom": None,
"timeShift": None,
"title": "<Title>",
"type": "piechart",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
BAR_CHART_PANEL_TEMPLATE = {
"aliasColors": {},
"dashLength": 10,
"dashes": False,
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
# Setting height and width is important here to ensure the default panel has some size to it.
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"hiddenSeries": False,
"id": 26,
"legend": {
"alignAsTable": True,
"avg": False,
"current": True,
"hideEmpty": False,
"hideZero": True,
"max": False,
"min": False,
"rightSide": False,
"show": False,
"sort": "current",
"sortDesc": True,
"total": False,
"values": True,
},
"lines": False,
"linewidth": 1,
"bars": True,
"nullPointMode": None,
"options": {
"alertThreshold": True,
"legend": {
"showLegend": False,
"displayMode": "table",
"placement": "bottom",
},
},
"percentage": False,
"pluginVersion": "7.5.17",
"pointradius": 2,
"points": False,
"renderer": "flot",
"spaceLength": 10,
"stack": True,
"steppedLine": False,
"targets": [],
"thresholds": [],
"timeFrom": None,
"timeRegions": [],
"timeShift": None,
"title": "<Title>",
"tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
"type": "graph",
"xaxis": {
"buckets": None,
"mode": "series",
"name": None,
"show": True,
"values": [
"total",
],
},
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
"yaxis": {"align": False, "alignLevel": None},
}
TABLE_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
},
"mappings": [],
},
"overrides": [],
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"id": 26,
"options": {
"showHeader": True,
"footer": {
"show": False,
"reducer": ["sum"],
"fields": "",
},
},
"pluginVersion": "7.5.17",
"targets": [],
"title": "<Title>",
"type": "table",
"transformations": [{"id": "organize", "options": {}}],
}
@DeveloperAPI
class PanelTemplate(Enum):
GRAPH = GRAPH_PANEL_TEMPLATE
HEATMAP = HEATMAP_TEMPLATE
PIE_CHART = PIE_CHART_TEMPLATE
STAT = STAT_PANEL_TEMPLATE
GAUGE = GAUGE_PANEL_TEMPLATE
BAR_CHART = BAR_CHART_PANEL_TEMPLATE
TABLE = TABLE_PANEL_TEMPLATE
@DeveloperAPI
@dataclass
class Panel:
"""Defines a Grafana panel (graph) for the Ray dashboard page.
A panel contains one or more targets (time-series queries).
Attributes:
title: Short name of the graph. Note: please keep this in sync with the title
definitions in Metrics.tsx.
description: Long form description of the graph.
id: Integer id used to reference the graph from Metrics.tsx.
unit: The unit to display on the y-axis of the graph.
targets: List of query targets.
fill: Whether or not the graph will be filled by a color.
stack: Whether or not the lines in the graph will be stacked.
linewidth: Width of the lines in the graph.
grid_pos: Grid position of the panel.
template: The panel template to use.
hideXAxis: Whether to hide the x-axis.
thresholds: Custom threshold configuration for stat/gauge panels.
Example: [
{"color": "green", "value": None},
{"color": "yellow", "value": 70},
{"color": "red", "value": 90}
]
value_mappings: Value mappings for displaying text instead of numbers.
Used for status panels.
color_mode: Color mode for stat panels ("value", "background", "none").
legend_mode: Legend display mode ("list", "table", "hidden").
min_val: Minimum value for gauge/graph y-axis.
max_val: Maximum value for gauge/graph y-axis.
reduce_calc: Reduce calculation method for stat panels (default: "lastNotNull").
heatmap_color_scheme: Color scheme for heatmap panels (e.g., "Spectral", "RdYlGn").
heatmap_color_reverse: Whether to reverse the heatmap color scheme.
heatmap_yaxis_label: Y-axis label for heatmap panels.
"""
title: str
description: str
id: int
unit: str
targets: List[Target]
fill: int = 10
stack: bool = True
linewidth: int = 1
grid_pos: Optional[GridPos] = None
template: Optional[PanelTemplate] = PanelTemplate.GRAPH
hideXAxis: bool = False
thresholds: Optional[List[Dict[str, Any]]] = None
value_mappings: Optional[List[Dict[str, Any]]] = None
color_mode: Optional[str] = None
legend_mode: Optional[str] = None
min_val: Optional[float] = None
max_val: Optional[float] = None
reduce_calc: Optional[str] = None
heatmap_color_scheme: Optional[str] = None
heatmap_color_reverse: Optional[bool] = None
heatmap_yaxis_label: Optional[str] = None
@DeveloperAPI
@dataclass
class Row:
"""Defines a Grafana row that can contain multiple panels.
Attributes:
title: The title of the row
panels: List of panels contained in this row
collapsed: Whether the row should be collapsed by default
"""
title: str
id: int
panels: List[Panel]
collapsed: bool = False
@DeveloperAPI
@dataclass
class DashboardConfig:
# This dashboard name is an internal key used to determine which env vars
# to check for customization
name: str
# The uid of the dashboard json if not overridden by a user
default_uid: str
# The global filters applied to all graphs in this dashboard. Users can
# add additional global_filters on top of this.
standard_global_filters: List[str]
base_json_file_name: str
# Panels can be specified in `panels`, or nested within `rows`.
# If both are specified, panels will be rendered before rows.
panels: List[Panel] = field(default_factory=list)
rows: List[Row] = field(default_factory=list)
def __post_init__(self):
if not self.panels and not self.rows:
raise ValueError("At least one of panels or rows must be specified")
python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py
+1588
View File
File diff suppressed because it is too large Load Diff
python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json
+183
View File
@@ -0,0 +1,183 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".+",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
"description": "Filter queries to specific ray sessions.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "SessionName",
"options": [],
"query": {
"query": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "{SessionName=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "${datasource}",
"definition": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "DatasetID",
"options": [],
"query": {
"query": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"refId": "Prometheus-Dataset-Variable-Query"
},
"refresh": 2,
"regex": "{dataset=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "${datasource}",
"definition": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Operator",
"options": [],
"query": {
"query": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"refId": "Prometheus-Dataset-Variable-Query"
},
"refresh": 2,
"regex": "{operator=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Data Dashboard",
"uid": "rayDataDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/data_llm_dashboard_panels.py
+325
View File
@@ -0,0 +1,325 @@
# ruff: noqa: E501
"""Ray Data LLM Dashboard panels for vLLM metrics visualization.
This dashboard provides visibility into vLLM engine metrics when using Ray Data LLM,
including latency metrics (TTFT, TPOT, E2E), throughput, cache utilization,
prefix cache hit rate, and scheduler state.
"""
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
DATA_LLM_GRAFANA_PANELS = [
Panel(
id=1,
title="vLLM: Token Throughput",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr='sum by (model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Prompt Tokens/Sec - {{model_name}} - {{WorkerId}}",
),
Target(
expr='sum by (model_name, WorkerId) (rate(ray_vllm_generation_tokens_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Generation Tokens/Sec - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 0, 12, 8),
),
Panel(
id=2,
title="vLLM: Time Per Output Token Latency",
description="P50, P90, P95, P99, and Mean TPOT latency",
unit="s",
targets=[
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Mean - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 0, 12, 8),
),
Panel(
id=3,
title="vLLM: Cache Utilization",
description="Percentage of used KV cache blocks by vLLM.",
unit="percentunit",
targets=[
Target(
expr='sum by (WorkerId) (ray_vllm_kv_cache_usage_perc{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}})',
legend="GPU Cache Usage - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 8, 12, 8),
),
Panel(
id=4,
title="vLLM: Prefix Cache Hit Rate",
description="Percentage of prefix cache hits. Higher is better for repeated prefixes.",
unit="percent",
targets=[
Target(
expr='max(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
legend="Max Hit Rate",
),
Target(
expr='min(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
legend="Min Hit Rate",
),
Target(
expr='100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Hit Rate: worker {{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 8, 12, 8),
),
Panel(
id=5,
title="vLLM: Time To First Token Latency",
description="P50, P90, P95, P99, and Mean TTFT latency",
unit="s",
targets=[
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Average - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 16, 12, 8),
),
Panel(
id=6,
title="vLLM: E2E Request Latency",
description="End-to-end request latency from arrival to completion.",
unit="s",
targets=[
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Average - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 16, 12, 8),
),
Panel(
id=7,
title="vLLM: Scheduler State",
description="Number of requests in RUNNING, WAITING, and SWAPPED state",
unit="Requests",
targets=[
Target(
expr='ray_vllm_num_requests_running{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Running - {{model_name}} - {{WorkerId}}",
),
Target(
expr='ray_vllm_num_requests_swapped{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Swapped - {{model_name}} - {{WorkerId}}",
),
Target(
expr='ray_vllm_num_requests_waiting{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Waiting - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 24, 12, 8),
),
Panel(
id=8,
title="vLLM: Queue Time",
description="P50, P90, P95, P99, and Mean time requests spend waiting in the queue.",
unit="s",
targets=[
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Mean - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 24, 12, 8),
),
Panel(
id=9,
title="vLLM: Prompt Length",
description="Distribution of prompt token lengths.",
unit="short",
targets=[
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50-{{model_name}}-{{WorkerId}}",
),
Target(
expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90-{{model_name}}-{{WorkerId}}",
),
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Average-{{model_name}}-{{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 32, 12, 8),
),
Panel(
id=10,
title="vLLM: Generation Length",
description="Distribution of generated token lengths.",
unit="short",
targets=[
Target(
expr='histogram_quantile(0.50, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50-{{model_name}}-{{WorkerId}}",
),
Target(
expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90-{{model_name}}-{{WorkerId}}",
),
Target(
expr=(
'(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
"\n/\n"
'(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
),
legend="Average-{{model_name}}-{{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 32, 12, 8),
),
Panel(
id=11,
title="vLLM: Finish Reason",
description="Number of finished requests by their finish reason: EOS token or max length reached.",
unit="Requests",
targets=[
Target(
expr='sum by(finished_reason, model_name, WorkerId) (increase(ray_vllm_request_success_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="{{finished_reason}} - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 40, 12, 8),
),
Panel(
id=12,
title="vLLM: Prefill and Decode Time",
description="Time spent in prefill vs decode phases.",
unit="s",
targets=[
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_prefill_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Prefill - {{model_name}} - {{WorkerId}}",
),
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_decode_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Decode - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 40, 12, 8),
),
]
data_llm_dashboard_config = DashboardConfig(
name="DATA_LLM",
default_uid="rayDataLlmDashboard",
panels=DATA_LLM_GRAFANA_PANELS,
standard_global_filters=[],
base_json_file_name="data_llm_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/data_llm_grafana_dashboard_base.json
+144
View File
@@ -0,0 +1,144 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"name": "vllm_model_name",
"label": "vLLM Model Name",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "workerid",
"label": "Worker ID",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "interval",
"label": "Interval",
"type": "custom",
"hide": 0,
"includeAll": false,
"multi": false,
"options": [
{
"selected": true,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
}
],
"current": {
"selected": true,
"text": "5m",
"value": "5m"
}
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Data LLM Dashboard",
"uid": "rayDataLlmDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py
+797
View File
@@ -0,0 +1,797 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
Panel,
Row,
Target,
)
"""
Queries for autoscaler resources.
"""
# Note: MAX & USED resources are reported from raylet to provide the most up to date information.
# But MAX + PENDING data is coming from the autoscaler. That said, MAX + PENDING can be
# more outdated. it is harmless because the actual MAX will catch up with MAX + PENDING
# eventually.
MAX_CPUS = 'sum(autoscaler_cluster_resources{{resource="CPU",{global_filters}}})'
PENDING_CPUS = 'sum(autoscaler_pending_resources{{resource="CPU",{global_filters}}})'
MAX_GPUS = 'sum(autoscaler_cluster_resources{{resource="GPU",{global_filters}}})'
PENDING_GPUS = 'sum(autoscaler_pending_resources{{resource="GPU",{global_filters}}})'
MAX_MEMORY = 'sum(autoscaler_cluster_resources{{resource="memory",{global_filters}}})'
PENDING_MEMORY = (
'sum(autoscaler_pending_resources{{resource="memory",{global_filters}}})'
)
def max_plus_pending(max_resource, pending_resource):
return f"({max_resource} or vector(0)) + ({pending_resource} or vector(0))"
MAX_PLUS_PENDING_CPUS = max_plus_pending(MAX_CPUS, PENDING_CPUS)
MAX_PLUS_PENDING_GPUS = max_plus_pending(MAX_GPUS, PENDING_GPUS)
MAX_PLUS_PENDING_MEMORY = max_plus_pending(MAX_MEMORY, PENDING_MEMORY)
MAX_PERCENTAGE_EXPRESSION = (
"100" # To help draw the max limit line on percentage panels
)
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# IMPORTANT: Please keep this in sync with Metrics.tsx and ray-metrics.rst
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
OVERVIEW_AND_HEALTH_PANELS = [
Panel(
id=24,
title="Node Count",
description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.',
unit="nodes",
targets=[
Target(
expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
legend="Active Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
legend="Failed Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
legend="Pending Nodes: {{NodeType}}",
),
],
),
Panel(
id=41,
title="Cluster Utilization",
description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.",
unit="%",
targets=[
# CPU
Target(
expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})',
legend="CPU (physical)",
),
# GPU
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}}) or vector(0))',
legend="GPU (physical)",
),
# Memory
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}})) * 100',
legend="Memory (RAM)",
),
# GRAM
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100',
legend="GRAM",
),
# Object Store
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100',
legend="Object Store Memory",
),
# Disk
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100',
legend="Disk",
),
],
fill=0,
stack=False,
),
Panel(
id=44,
title="Ray OOM Kills (Tasks and Actors)",
description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html. Note: The RayNodeType filter does not work on this graph.",
unit="failures",
targets=[
Target(
expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", {global_filters}}}) by (Name, instance)',
legend="OOM Killed: {{Name}}, {{instance}}",
),
],
),
Panel(
id=65,
title="Unexpected System Level Worker Failures",
description="The number of workers (potentially tasks or actors) that disconnected from the raylet unexpectedly. "
"This typically indicates the worker process unexpectedly failed due to "
"a Ray system error or a kernel kill (e.g. OOM, SIGKILL, Bad exit code). "
"Note that this metric only includes OOM kills from the kernel and does not "
"include OOM kills from Ray's memory monitor. "
"If errors of this type is encountered when the node is under memory pressure, "
"The failures are likely OOM kills.",
unit="failures",
targets=[
Target(
expr='sum(ray_node_manager_unexpected_worker_failure_total{{instance=~"$Instance", {global_filters}}}) by (Type, Name, instance)',
legend="Unexpected worker failure: {{Name}}, {{Type}}, {{instance}}",
),
],
),
]
RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS = [
Panel(
id=26,
title="All Tasks by State",
description="Current count of tasks, grouped by scheduler state (e.g., pending, running, finished).\n\nState: the task state, as described by rpc::TaskStatus proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='sum(max_over_time(ray_tasks{{IsRetry="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
legend="{{State}}",
),
Target(
expr='sum(max_over_time(ray_tasks{{IsRetry!="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
legend="{{State}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=35,
title="Active Tasks by Name",
description="Current count of active tasks (i.e. pending or running; not finished), grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}}",
),
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=38,
title="Running Tasks by Name",
description="Current count of tasks that are currently executing, grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}}",
),
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=64,
title="Running Tasks by Node",
description="Current count of tasks that are currently executing, grouped by node.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{State=~"RUNNING*",Source="executor",instance=~"$Instance",{global_filters}}}) by (instance), 0)',
legend="{{instance}}",
),
],
fill=0,
stack=False,
),
Panel(
id=33,
title="All Actors by State",
description='Note: not impacted by "Instance" variable.\n\nCurrent count of actors, grouped by lifecycle state (e.g., alive, restarting, dead/terminated).\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.',
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{Source="gcs",{global_filters}}}) by (State)',
legend="{{State}}",
)
],
),
Panel(
id=42,
title="Alive Actors by State",
description="Current count of alive actors (i.e. not dead/terminated), grouped by state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.",
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (State)',
legend="{{State}}",
)
],
),
Panel(
id=36,
title="Alive Actors by Name",
description="Current count of alive actors, grouped by actor name.",
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{State!="DEAD",Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (Name)',
legend="{{Name}}",
)
],
),
Panel(
id=40,
title="All Placement Groups by State",
description='Note: not impacted by "Instance" variable.\n\nCurrent count of placement groups, grouped by state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTableData proto in gcs.proto.',
unit="placement groups",
targets=[
Target(
expr="sum(ray_placement_groups{{{global_filters}}}) by (State)",
legend="{{State}}",
)
],
),
Panel(
id=29,
title="Object Store Memory by Location",
description="Object store memory usage by location. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)',
legend="{{Location}}",
),
Target(
expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
]
RAY_RESOURCES_PANELS = [
Panel(
id=27,
title="Logical CPUs Usage",
description="Logical CPU usage of Ray. The dotted line indicates the total number of CPUs. The logical CPU is allocated by `num_cpus` arguments from tasks and actors. PENDING means the number of CPUs that will be available when new nodes are up after the autoscaler scales up.\n\nNOTE: Ray's logical CPU is different from physical CPU usage. Ray's logical CPU is allocated by `num_cpus` arguments.",
unit="cores",
targets=[
Target(
expr='sum(ray_resources{{Name="CPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="CPU Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="CPU",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
# If max + pending > max, we display this value.
# (A and predicate) means to return A when the predicate satisfies in PromSql.
Target(
expr=f"({MAX_PLUS_PENDING_CPUS} and {MAX_PLUS_PENDING_CPUS} > ({MAX_CPUS} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=28,
title="Logical GPUs Usage",
description="Logical GPU usage of Ray. The dotted line indicates the total number of GPUs. The logical GPU is allocated by `num_gpus` arguments from tasks and actors. PENDING means the number of GPUs that will be available when new nodes are up after the autoscaler scales up.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_resources{{Name="GPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="GPU Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="GPU",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
# If max + pending > max, we display this value.
# (A and predicate) means to return A when the predicate satisfies in PromSql.
Target(
expr=f"({MAX_PLUS_PENDING_GPUS} and {MAX_PLUS_PENDING_GPUS} > ({MAX_GPUS} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=61,
title="Logical Memory Usage",
description="Logical memory usage of Ray by node. The dotted line indicates the total amount of memory available. Logical memory refers to Ray's view of memory resources allocated to tasks and actors. PENDING means the amount of memory that will be available when new nodes are up after the autoscaler scales up.",
unit="bytes",
targets=[
Target(
expr='sum(ray_resources{{Name="memory",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="Memory Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
Target(
expr=f"({MAX_PLUS_PENDING_MEMORY} and {MAX_PLUS_PENDING_MEMORY} > ({MAX_MEMORY} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=58,
title="Object Store Memory Usage",
description="Object store memory usage by instance, including memory that has been spilled to disk. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
Target(
expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=59,
title="Object Store Memory Usage %",
description="Object store memory usage % by instance, including memory that has been spilled to disk. This metric can go over 100% in case of spillage to disk.",
unit="%",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance) * 100 / sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
Target(
expr=MAX_PERCENTAGE_EXPRESSION, # To show the memory limit visually
legend="MAX",
),
],
fill=0,
stack=False,
),
Panel(
id=60,
title="Object Store Memory Spilled to Disk",
description="Object store memory that has been spilled to disk, by instance.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",Location="SPILLED",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
],
fill=0,
stack=False,
),
]
NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS = [
Panel(
id=37,
title="Node CPU Usage by Component",
description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
unit="cores",
targets=[
Target(
# ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU.
expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100',
legend="{{Component}}",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=34,
title="Node Memory Usage by Component",
description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
unit="bytes",
targets=[
Target(
expr='(sum(ray_component_rss_bytes{{instance=~"$Instance",{global_filters}}}) by (Component)) - (sum(ray_component_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))',
legend="{{Component}}",
),
Target(
expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})',
legend="shared_memory",
),
Target(
expr='min(label_replace(sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "host", "", "") or label_replace(sum(ray_node_cgroup_mem_total{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "cgroup", "", ""))',
legend="MAX",
),
],
),
Panel(
id=45,
title="Node GPU Usage by Component",
description="The physical (hardware) GPU usage across the cluster, broken down by component. This reports the summed GPU usage per Ray component.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_component_gpu_percentage{{instance=~"$Instance",{global_filters}}} / 100) by (Component)',
legend="{{Component}}",
),
],
),
Panel(
id=46,
title="Node GPU Memory Usage by Component",
description="The physical (hardware) GPU memory usage across the cluster, broken down by component. This reports the summed GPU memory usage per Ray component.",
unit="bytes",
targets=[
Target(
expr='sum(ray_component_gpu_memory_mb{{instance=~"$Instance",{global_filters}}} * 1024 * 1024) by (Component)',
legend="{{Component}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
),
]
NODE_HARDWARE_UTILIZATION_PANELS = [
Panel(
id=2,
title="Node CPU Usage",
description="The physical (hardware) CPU usage for each node.",
unit="cores",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=54,
title="Node CPU Usage %",
description="The percentage of physical (hardware) CPU usage for each node.",
unit="%",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=4,
title="Node Memory Usage (heap + object store)",
description="The physical (hardware) memory usage for each node. The dotted line means the total amount of memory from the cluster. "
"Node memory is a sum of object store memory (shared memory) and heap memory.\n\n"
"Host targets reflect memory as reported by the host machine. "
"Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
Target(
expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="Container MAX",
),
],
),
Panel(
id=48,
title="Node Memory Usage % (heap + object store)",
description="The percentage of physical (hardware) memory usage for each node.\n\n"
"Host targets reflect memory as reported by the host machine. "
"Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
unit="%",
targets=[
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=6,
title="Node Disk Usage",
description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=57,
title="Node Disk Usage %",
description="Node's physical (hardware) disk usage. \n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
unit="%",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / (sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType))',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=8,
title="Node GPU Usage",
description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ",
unit="GPUs",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=55,
title="Node GPU Usage %",
description="Node's physical (hardware) GPU usage.",
unit="%",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=18,
title="Node GPU Memory Usage (GRAM)",
description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
),
Panel(
id=56,
title="Node GPU Memory Usage (GRAM) %",
description="The percentage of physical (hardware) GPU memory usage for each node.",
unit="%",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) * 100 / (sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName))',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=62,
title="Node GPU Power",
description="Current GPU power draw per node. Reported in milliwatts; displayed in watts. Supported on NVIDIA and AMD GPUs.",
unit="mwatt",
targets=[
Target(
expr='sum(ray_node_gpu_power_milliwatts{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Power: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=63,
title="Node GPU Temperature",
description="Current GPU temperature per node in Celsius. Supported on NVIDIA GPUs.",
unit="celsius",
targets=[
Target(
expr='sum(ray_node_gpu_temperature_celsius{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Temperature: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=32,
title="Node Disk IO Speed",
description="Disk IO per node.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read: {{instance}} ({{RayNodeType}})",
),
],
),
Panel(
id=20,
title="Node Network",
description="Network speed per node",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Recv: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Send: {{instance}} ({{RayNodeType}})",
),
],
),
]
NODE_TPU_UTILIZATION_PANELS = [
Panel(
id=50,
title="Node TPU Tensorcore Utilization %",
description="Percentage of tensorcore utilization for the TPUs on this node. Computed by dividing the number of tensorcore operations by the maximum supported number of operations during the sample period.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_tensorcore_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=51,
title="Node TPU High Bandwidth Memory Utilization %",
description="Percentage of bandwidth memory utilization for the TPUs on this node. Computed by dividing the memory bandwidth used by the maximum supported memory bandwidth limit during the sample period.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_memory_bandwidth_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=52,
title="Node TPU Duty Cycle %",
description="Percentage of time over the sample period during which the TPU is actively processing.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_duty_cycle{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=53,
title="Node TPU Memory Used",
description="Total memory used/allocated for the TPUs on this node.",
unit="bytes",
targets=[
Target(
expr='sum(ray_tpu_memory_used{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="Memory Used: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
Target(
expr='sum(ray_tpu_memory_total{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="Memory Total: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
]
DEFAULT_GRAFANA_ROWS = [
Row(
title="Overview and Health",
id=1001,
panels=OVERVIEW_AND_HEALTH_PANELS,
collapsed=False,
),
Row(
title="Hardware Utilization by Node",
id=1005,
panels=NODE_HARDWARE_UTILIZATION_PANELS,
collapsed=False,
),
Row(
title="Hardware Utilization by Ray Component",
id=1004,
panels=NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS,
collapsed=False,
),
Row(
title="Ray Resources by Node",
id=1003,
panels=RAY_RESOURCES_PANELS,
collapsed=False,
),
Row(
title="Ray Tasks, Actors and Placement Groups",
id=1002,
panels=RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS,
collapsed=False,
),
Row(
title="TPU Utilization by Node",
id=1006,
panels=NODE_TPU_UTILIZATION_PANELS,
collapsed=True,
),
]
ids = []
for row in DEFAULT_GRAFANA_ROWS:
ids.append(row.id)
ids.extend(panel.id for panel in row.panels)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
default_dashboard_config = DashboardConfig(
name="DEFAULT",
default_uid="rayDefaultDashboard",
rows=DEFAULT_GRAFANA_ROWS,
standard_global_filters=[
'SessionName=~"$SessionName"',
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="default_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json
+177
View File
@@ -0,0 +1,177 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".+",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
"description": "Filter queries to specific Ray sessions.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "SessionName",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
"description": "Filter queries to specific Ray nodes by their IP address.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Instance",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple Ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
},
"description": "Filter queries to specific Ray node types (head or worker).",
"includeAll": true,
"multi": true,
"name": "RayNodeType",
"options": [
{
"selected": false,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "Head Node",
"value": "head"
},
{
"selected": false,
"text": "Worker Node",
"value": "worker"
}
],
"query": "head, worker",
"type": "custom"
}
]
},
"rayMeta": ["supportsFullGrafanaView"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Default Dashboard",
"uid": "rayDefaultDashboard",
"version": 4
}
python/ray/dashboard/modules/metrics/dashboards/serve_dashboard_panels.py
+423
View File
@@ -0,0 +1,423 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
SERVE_GRAFANA_PANELS = [
Panel(
id=5,
title="Cluster Utilization",
description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster. Ignores application variable.",
unit="%",
targets=[
# CPU
Target(
expr="avg(ray_node_cpu_utilization{{{global_filters}}})",
legend="CPU (physical)",
),
# GPU
Target(
expr="sum(ray_node_gpus_utilization{{{global_filters}}}) / on() (sum(autoscaler_cluster_resources{{resource='GPU',{global_filters}}}) or vector(0))",
legend="GPU (physical)",
),
# Memory
Target(
expr="sum(ray_node_mem_used{{{global_filters}}}) / on() (sum(ray_node_mem_total{{{global_filters}}})) * 100",
legend="Memory (RAM)",
),
# GRAM
Target(
expr="sum(ray_node_gram_used{{{global_filters}}}) / on() (sum(ray_node_gram_available{{{global_filters}}}) + sum(ray_node_gram_used{{{global_filters}}})) * 100",
legend="GRAM",
),
# Object Store
Target(
expr='sum(ray_object_store_memory{{{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",{global_filters}}}) * 100',
legend="Object Store Memory",
),
# Disk
Target(
expr="sum(ray_node_disk_usage{{{global_filters}}}) / on() (sum(ray_node_disk_free{{{global_filters}}}) + sum(ray_node_disk_usage{{{global_filters}}})) * 100",
legend="Disk",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 0, 12, 8),
),
Panel(
id=7,
title="QPS per application",
description="QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
legend="{{application, route}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
legend="{{application, method}}",
),
],
grid_pos=GridPos(12, 0, 12, 8),
),
Panel(
id=8,
title="Error QPS per application",
description="Error QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
legend="{{application, route}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
legend="{{application, method}}",
),
],
grid_pos=GridPos(0, 1, 12, 8),
),
Panel(
id=17,
title="Error QPS per application per error code",
description="Error QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, error_code)',
legend="{{application, route, error_code}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, error_code)',
legend="{{application, method, error_code}}",
),
],
grid_pos=GridPos(12, 1, 12, 8),
),
Panel(
id=12,
title="P50 latency per application",
description="P50 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 2, 8, 8),
),
Panel(
id=15,
title="P90 latency per application",
description="P90 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 2, 8, 8),
),
Panel(
id=16,
title="P99 latency per application",
description="P99 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 2, 8, 8),
),
Panel(
id=2,
title="Replicas per deployment",
description='Number of replicas per deployment. Ignores "Application" variable.',
unit="replicas",
targets=[
Target(
expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(0, 3, 8, 8),
),
Panel(
id=13,
title="QPS per deployment",
description="QPS for each deployment.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_request_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(8, 3, 8, 8),
),
Panel(
id=14,
title="Error QPS per deployment",
description="Error QPS for each deplyoment.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_error_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(16, 3, 8, 8),
),
Panel(
id=9,
title="P50 latency per deployment",
description="P50 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 4, 8, 8),
),
Panel(
id=10,
title="P90 latency per deployment",
description="P90 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 4, 8, 8),
),
Panel(
id=11,
title="P99 latency per deployment",
description="P99 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 4, 8, 8),
),
Panel(
id=3,
title="Queue size per deployment",
description='Number of requests queued per deployment. Ignores "Application" variable.',
unit="requests",
targets=[
Target(
expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
legend="{{application, deployment}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 5, 8, 8),
),
Panel(
id=4,
title="Node count",
description='Number of nodes in this cluster. Ignores "Application" variable.',
unit="nodes",
targets=[
# TODO(aguo): Update this to use autoscaler metrics instead
Target(
expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
legend="Active Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
legend="Failed Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
legend="Pending Nodes: {{NodeType}}",
),
],
grid_pos=GridPos(8, 5, 8, 8),
),
Panel(
id=6,
title="Node network",
description='Network speed per node. Ignores "Application" variable.',
unit="Bps",
targets=[
Target(
expr="sum(ray_node_network_receive_speed{{{global_filters}}}) by (instance)",
legend="Recv: {{instance}}",
),
Target(
expr="sum(ray_node_network_send_speed{{{global_filters}}}) by (instance)",
legend="Send: {{instance}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(16, 5, 8, 8),
),
Panel(
id=20,
title="Ongoing HTTP Requests",
description="The number of ongoing requests in the HTTP Proxy.",
unit="requests",
targets=[
Target(
expr="ray_serve_num_ongoing_http_requests{{{global_filters}}}",
legend="Ongoing HTTP Requests",
),
],
grid_pos=GridPos(0, 6, 8, 8),
),
Panel(
id=21,
title="Ongoing gRPC Requests",
description="The number of ongoing requests in the gRPC Proxy.",
unit="requests",
targets=[
Target(
expr="ray_serve_num_ongoing_grpc_requests{{{global_filters}}}",
legend="Ongoing gRPC Requests",
),
],
grid_pos=GridPos(8, 6, 8, 8),
),
Panel(
id=22,
title="Scheduling Tasks",
description="The number of request scheduling tasks in the router.",
unit="tasks",
targets=[
Target(
expr="ray_serve_num_scheduling_tasks{{{global_filters}}}",
legend="Scheduling Tasks",
),
],
grid_pos=GridPos(16, 6, 8, 8),
),
Panel(
id=23,
title="Scheduling Tasks in Backoff",
description="The number of request scheduling tasks in the router that are undergoing backoff.",
unit="tasks",
targets=[
Target(
expr="ray_serve_num_scheduling_tasks_in_backoff{{{global_filters}}}",
legend="Scheduling Tasks in Backoff",
),
],
grid_pos=GridPos(0, 7, 8, 8),
),
Panel(
id=24,
title="Controller Control Loop Duration",
description="The duration of the last control loop.",
unit="seconds",
targets=[
Target(
expr="ray_serve_controller_control_loop_duration_s{{{global_filters}}}",
legend="Control Loop Duration",
),
],
grid_pos=GridPos(8, 7, 8, 8),
),
Panel(
id=25,
title="Number of Control Loops",
description="The number of control loops performed by the controller. Increases monotonically over the controller's lifetime.",
unit="loops",
targets=[
Target(
expr="ray_serve_controller_num_control_loops{{{global_filters}}}",
legend="Control Loops",
),
],
grid_pos=GridPos(16, 7, 8, 8),
),
]
ids = []
for panel in SERVE_GRAFANA_PANELS:
ids.append(panel.id)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
serve_dashboard_config = DashboardConfig(
name="SERVE",
default_uid="rayServeDashboard",
panels=SERVE_GRAFANA_PANELS,
standard_global_filters=[
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/serve_deployment_dashboard_panels.py
+262
View File
@@ -0,0 +1,262 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
SERVE_DEPLOYMENT_GRAFANA_PANELS = [
Panel(
id=1,
title="Replicas per deployment",
description='Number of replicas per deployment. Ignores "Route" variable.',
unit="replicas",
targets=[
Target(
expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(0, 0, 8, 8),
),
Panel(
id=2,
title="QPS per replica",
description="QPS for each replica.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_request_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(8, 0, 8, 8),
),
Panel(
id=3,
title="Error QPS per replica",
description="Error QPS for each replica.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_error_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(16, 0, 8, 8),
),
Panel(
id=4,
title="P50 latency per replica",
description="P50 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 1, 8, 8),
),
Panel(
id=5,
title="P90 latency per replica",
description="P90 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 1, 8, 8),
),
Panel(
id=6,
title="P99 latency per replica",
description="P99 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 1, 8, 8),
),
Panel(
id=7,
title="Queue size per deployment",
description='Number of requests queued per deployment. Ignores "Replica" and "Route" variable.',
unit="requests",
targets=[
Target(
expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 2, 12, 8),
),
Panel(
id=8,
title="Running requests per replica",
description="Current running requests for each replica.",
unit="requests",
targets=[
Target(
expr="sum(ray_serve_replica_processing_queries{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(12, 2, 12, 8),
),
Panel(
id=9,
title="Multiplexed models per replica",
description="The number of multiplexed models for each replica.",
unit="models",
targets=[
Target(
expr="sum(ray_serve_num_multiplexed_models{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 3, 8, 8),
),
Panel(
id=10,
title="Multiplexed model loads per replica",
description="The number of times of multiplexed models loaded for each replica.",
unit="times",
targets=[
Target(
expr="sum(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 3, 8, 8),
),
Panel(
id=11,
title="Multiplexed model unloads per replica",
description="The number of times of multiplexed models unloaded for each replica.",
unit="times",
targets=[
Target(
expr="sum(ray_serve_multiplexed_models_unload_counter_total{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 3, 8, 8),
),
Panel(
id=12,
title="P99 latency of multiplexed model loads per replica",
description="P99 latency of multiplexed model load per replica.",
unit="ms",
targets=[
Target(
expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_load_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 4, 8, 8),
),
Panel(
id=13,
title="P99 latency of multiplexed model unloads per replica",
description="P99 latency of multiplexed model unload per replica.",
unit="ms",
targets=[
Target(
expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_unload_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 4, 8, 8),
),
Panel(
id=14,
title="Multiplexed model ids per replica",
description="The ids of multiplexed models for each replica.",
unit="model",
targets=[
Target(
expr="ray_serve_registered_multiplexed_model_id{{{global_filters}}}",
legend="{{replica}}:{{model_id}}",
),
],
grid_pos=GridPos(16, 4, 8, 8),
stack=False,
),
Panel(
id=15,
title="Multiplexed model cache hit rate",
description="The cache hit rate of multiplexed models for the deployment.",
unit="%",
targets=[
Target(
expr="(1 - sum(rate(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}[5m]))/sum(rate(ray_serve_multiplexed_get_model_requests_counter_total{{{global_filters}}}[5m])))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(0, 5, 8, 8),
),
]
ids = []
for panel in SERVE_DEPLOYMENT_GRAFANA_PANELS:
ids.append(panel.id)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
serve_deployment_dashboard_config = DashboardConfig(
name="SERVE_DEPLOYMENT",
default_uid="rayServeDeploymentDashboard",
panels=SERVE_DEPLOYMENT_GRAFANA_PANELS,
standard_global_filters=[
'application=~"$Application"',
'deployment=~"$Deployment"',
'replica=~"$Replica"',
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_deployment_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/serve_deployment_grafana_dashboard_base.json
+208
View File
@@ -0,0 +1,208 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries to specific prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Application",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Deployment",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Replica",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Route",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve Deployment Dashboard",
"uid": "rayServeDeploymentDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/serve_grafana_dashboard_base.json
+177
View File
@@ -0,0 +1,177 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Application",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "HTTP Route",
"multi": true,
"name": "HTTP_Route",
"options": [],
"query": {
"query": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "gRPC Service Method",
"multi": true,
"name": "gRPC_Method",
"options": [],
"query": {
"query": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve Dashboard",
"uid": "rayServeDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py
+810
View File
@@ -0,0 +1,810 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
PanelTemplate,
Row,
Target,
)
# ---------------------------------------------------------------------------
# Reusable PromQL fragments
# ---------------------------------------------------------------------------
# WorkerId join: attaches deployment + replica labels to vLLM-only metrics.
# The `* 0 + 1` trick turns the counter into a constant-1 lookup table keyed
# by WorkerId so the join resolves deployment + replica labels without
# affecting the numeric value of the left-hand side.
# Keep `{global_filters}` trailing so an empty substitution leaves a tolerated
# trailing comma instead of a leading/double comma that Prometheus rejects.
_WORKER_JOIN = (
"\n* on(WorkerId) group_left(deployment, replica)"
"\nmax by(WorkerId, deployment, replica) ("
'ray_serve_deployment_request_counter_total{{deployment=~"$deployment", {global_filters}}} * 0 + 1)'
)
# Standard vLLM metric filter
_VLLM_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", {global_filters}'
# vLLM filter scoped to a specific deployment (used for ray_serve_* metrics
# that also carry model_name / WorkerId labels).
_VLLM_DEPLOYMENT_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", deployment=~"$deployment", {global_filters}'
# Legend used by most per-worker panels
_DEP_REPLICA = "{{deployment}}: {{replica}}"
def _mean_with_join(metric_base: str) -> str:
"""Mean = sum(_sum) / sum(_count) with NaN guard + WorkerId join."""
return (
"(\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_sum{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" /\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
def _percentile_with_join(metric_base: str, quantile: float) -> str:
"""histogram_quantile with NaN guard + WorkerId join."""
return (
"(\n"
" histogram_quantile(\n"
f" {quantile},\n"
f" sum by (le, WorkerId) (rate({metric_base}_bucket{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
def _gauge_with_join(metric: str) -> str:
"""Simple gauge metric with WorkerId join (no rate, no guard)."""
return f"sum by(WorkerId) ({metric}{{{{{_VLLM_FILTER}}}}})" + _WORKER_JOIN
def _rate_with_join(metric: str, agg_fn: str = "rate") -> str:
"""rate() or increase() of a metric summed by WorkerId, with join."""
return (
f"sum by(WorkerId) ({agg_fn}({metric}{{{{{_VLLM_FILTER}}}}}[$interval]))"
+ _WORKER_JOIN
)
def _ratio_with_join_and_guard(
numerator_metric: str,
denominator_metric: str,
*,
scale: str = "",
guard_metric: str | None = None,
) -> str:
"""Ratio of two rate metrics with NaN guard + WorkerId join.
Optionally applies a scale factor (e.g. '* 1000' or '/ 1024 / 1024 / 1024').
"""
guard = guard_metric or denominator_metric
return (
"(\n"
" (\n"
f" sum by(WorkerId) (rate({numerator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" /\n"
f" sum by(WorkerId) (rate({denominator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+ (f" {scale}\n" if scale else "")
+ " )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({guard}{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
# ---------------------------------------------------------------------------
# Histogram helper: generates Mean / P50 / P90 panels for a given metric
# ---------------------------------------------------------------------------
def _histogram_panels(
metric_base: str,
label: str,
ids: tuple,
y: int,
unit: str = "s",
linewidth: int = 2,
description: str = "",
) -> list:
"""Return [Mean, P50, P90] panels for a histogram metric."""
return [
Panel(
id=ids[0],
title=f"{label} -- Mean",
description=description,
unit=unit,
targets=[Target(expr=_mean_with_join(metric_base), legend=_DEP_REPLICA)],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(0, y, 8, 8),
),
Panel(
id=ids[1],
title=f"{label} -- P50",
description=description,
unit=unit,
targets=[
Target(
expr=_percentile_with_join(metric_base, 0.5), legend=_DEP_REPLICA
)
],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(8, y, 8, 8),
),
Panel(
id=ids[2],
title=f"{label} -- P90",
description=description,
unit=unit,
targets=[
Target(
expr=_percentile_with_join(metric_base, 0.9), legend=_DEP_REPLICA
)
],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(16, y, 8, 8),
),
]
# ===================================================================
# Row 1: Throughput
# ===================================================================
_throughput_panels = [
Panel(
id=2,
title="Requests / s",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (deployment, replica) (rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
legend="Total QPS",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 1, 8, 8),
),
Panel(
id=3,
title="Prompt Tokens/s",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_prompt_tokens_sum"),
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_vllm_request_prompt_tokens_sum{{{{{_VLLM_FILTER}}}}}[$interval]))",
legend="Total",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(8, 1, 8, 8),
),
Panel(
id=4,
title="Generation Tokens/s",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_generation_tokens_total"),
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_vllm_generation_tokens_total{{{{{_VLLM_FILTER}}}}}[$interval]))",
legend="Total",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(16, 1, 8, 8),
),
]
# ===================================================================
# Row 2: Latency (3x3 grid)
# ===================================================================
_latency_panels_list = [
*_histogram_panels(
"ray_vllm_request_time_per_output_token_seconds", "TPOT", (6, 7, 8), 10
),
*_histogram_panels("ray_vllm_time_to_first_token_seconds", "TTFT", (9, 10, 11), 18),
*_histogram_panels(
"ray_vllm_e2e_request_latency_seconds",
"Request Latency",
(12, 13, 14),
26,
description="Latency from request start to first token returned (in seconds).",
),
]
# ===================================================================
# Row 3: Cache
# ===================================================================
_cache_panels = [
Panel(
id=16,
title="Cache Utilization",
description="Percentage of used cache blocks by vLLM.",
unit="percentunit",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_kv_cache_usage_perc"),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 35, 12, 8),
),
Panel(
id=17,
title="GPU KV Cache Hit Rate",
description="",
unit="percent",
targets=[
Target(
expr=(
f"100 * ("
f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{{{{_VLLM_FILTER}}}}}[$interval])) "
f"/ sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])))"
f" and on(WorkerId) "
f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])) > 0))"
+ _WORKER_JOIN
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 35, 12, 8),
),
]
# ===================================================================
# Row 4: Request Length
# ===================================================================
_request_length_panels = [
*_histogram_panels(
"ray_vllm_request_prompt_tokens",
"Prompt Length",
(19, 20, 21),
44,
unit="short",
linewidth=1,
),
*_histogram_panels(
"ray_vllm_request_generation_tokens",
"Generation Length",
(22, 23, 24),
52,
unit="short",
linewidth=1,
),
]
# ===================================================================
# Row 5: Scheduler
# ===================================================================
_scheduler_panels = [
Panel(
id=26,
title="Scheduler: Running",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_running"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 61, 8, 8),
),
Panel(
id=27,
title="Scheduler: Swapped",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_swapped"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 61, 8, 8),
),
Panel(
id=28,
title="Scheduler: Waiting",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_waiting"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 61, 8, 8),
),
Panel(
id=29,
title="Finish Reason",
description="Number of finished requests by their finish reason.",
unit="short",
targets=[
Target(
expr=(
f"sum by(finished_reason, WorkerId) (increase(ray_vllm_request_success_total{{{{{_VLLM_FILTER}}}}}[$interval]))"
+ _WORKER_JOIN
),
legend="{{finished_reason}} \u2014 {{deployment}}: {{replica}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 69, 12, 8),
),
Panel(
id=30,
title="Queue Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_queue_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 69, 12, 8),
),
Panel(
id=31,
title="Prefill Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_prefill_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 77, 12, 8),
),
Panel(
id=32,
title="Decode Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_decode_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 77, 12, 8),
),
]
# ===================================================================
# Row 6: NIXL
# ===================================================================
_nixl_panels = [
Panel(
id=34,
title="NIXL: Transfer Latency",
description="Average NIXL KV cache transfer latency in milliseconds.",
unit="ms",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_xfer_time_seconds_sum",
"ray_vllm_nixl_xfer_time_seconds_count",
scale="* 1000",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 86, 8, 8),
),
Panel(
id=35,
title="NIXL: Transfer Throughput",
description="NIXL KV cache transfer throughput in GB/s.",
unit="GBs",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_bytes_transferred_sum",
"ray_vllm_nixl_xfer_time_seconds_sum",
scale="/ 1024 / 1024 / 1024",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 86, 8, 8),
),
Panel(
id=36,
title="NIXL: Transfer Rate",
description="Number of NIXL KV cache transfers per second.",
unit="ops",
targets=[
Target(
expr=_rate_with_join("ray_vllm_nixl_xfer_time_seconds_count"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 86, 8, 8),
),
Panel(
id=37,
title="NIXL: Avg Post Time",
description="Average time to post/initiate a NIXL transfer in milliseconds.",
unit="ms",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_post_time_seconds_sum",
"ray_vllm_nixl_post_time_seconds_count",
scale="* 1000",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 94, 8, 8),
),
Panel(
id=38,
title="NIXL: KV Transfer Failures",
description="Number of failed NIXL KV cache transfers. Any non-zero value is concerning and indicates RDMA transfer errors.",
unit="short",
targets=[
Target(
expr=_rate_with_join(
"ray_vllm_nixl_num_failed_transfers", agg_fn="increase"
),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 94, 8, 8),
),
Panel(
id=39,
title="NIXL: KV Expired Requests",
description="Number of requests whose KV blocks expired before decode consumed them. Spikes indicate prefill is outrunning decode or the timeout is too short.",
unit="short",
targets=[
Target(
expr=_rate_with_join(
"ray_vllm_nixl_num_kv_expired_reqs_total", agg_fn="increase"
),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 94, 8, 8),
),
]
# ===================================================================
# Row 7: Token Distribution (collapsed)
# ===================================================================
_WORKERID_FILTER = 'WorkerId=~"$workerid", {global_filters}'
_token_distribution_panels = [
Panel(
id=41,
title="Tokens Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
legend="Input: {{model_name}}",
),
Target(
expr=f"(sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
legend="Generated: {{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 103, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=42,
title="Tokens Last Hour",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
legend="Input: {{model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
legend="Generated: {{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 103, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=43,
title="Ratio Input:Generated Tokens Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) / sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 111, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=44,
title="Distribution of Requests Per Model Last 24 Hours",
description="",
unit="Requests",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 111, 12, 8),
template=PanelTemplate.PIE_CHART,
),
Panel(
id=45,
title="Peak Tokens Per Second Per Model Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"max_over_time(sum by (model_name) (rate(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[2m]))[24h:1m])",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 119, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=46,
title="Tokens Per Model Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) + sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 119, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=47,
title="Avg Total Tokens Per Request Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=(
f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) +\n"
f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
f" / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
),
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 127, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=48,
title="Requests Per Model Last Week",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 127, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=49,
title="Tokens Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="In: {{ model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="Out: {{ model_name }}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 135, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=50,
title="Avg Total Tokens Per Request Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=(
f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) "
f"+ sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
f"/ sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
),
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 135, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=51,
title="Tokens Per Request Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="In: {{ model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="Out: {{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 143, 12, 8),
template=PanelTemplate.GAUGE,
),
]
# ===================================================================
# Assemble rows and config
# ===================================================================
_ALL_ROWS = [
Row(title="Throughput", id=501, panels=_throughput_panels),
Row(title="Latency", id=502, panels=_latency_panels_list),
Row(title="Cache", id=503, panels=_cache_panels),
Row(title="Request Length", id=504, panels=_request_length_panels),
Row(title="Scheduler", id=505, panels=_scheduler_panels),
Row(title="NIXL", id=506, panels=_nixl_panels),
Row(
title="Token Distribution",
id=507,
collapsed=True,
panels=_token_distribution_panels,
),
]
# Validate uniqueness of panel IDs across all rows
_all_ids = sorted(panel.id for row in _ALL_ROWS for panel in row.panels)
assert len(_all_ids) == len(
set(_all_ids)
), f"Duplicated id found. Use unique id for each panel. {_all_ids}"
serve_llm_dashboard_config = DashboardConfig(
name="SERVE_LLM",
default_uid="rayServeLlmDashboard",
standard_global_filters=[
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_llm_grafana_dashboard_base.json",
rows=_ALL_ROWS,
)
python/ray/dashboard/modules/metrics/dashboards/serve_llm_grafana_dashboard_base.json
+199
View File
@@ -0,0 +1,199 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"name": "vllm_model_name",
"label": "vLLM Model Name",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "workerid",
"label": "Worker ID",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "deployment",
"label": "Deployment",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
"query": {
"query": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"name": "interval",
"label": "Interval",
"type": "custom",
"hide": 0,
"includeAll": false,
"multi": false,
"options": [
{
"selected": true,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
}
],
"current": {
"selected": true,
"text": "5m",
"value": "5m"
}
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve LLM Dashboard",
"uid": "rayServeLlmDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py
+346
View File
@@ -0,0 +1,346 @@
# flake8: noqa E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
Panel,
Row,
Target,
)
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL = Panel(
id=1,
title="Controller State",
description="Current state of the Ray Train controller.",
unit="",
targets=[
Target(
expr='sum(ray_train_controller_state{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name, ray_train_controller_state)',
legend="Run Name: {{ray_train_run_name}}, Controller State: {{ray_train_controller_state}}",
),
],
)
CONTROLLER_OPERATION_TIME_PANEL = Panel(
id=2,
title="Cumulative Worker Group Start/Shutdown Time",
description="Cumulative time the controller spends starting and shutting down worker groups (re-created on worker failures and resizes).",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_worker_group_start_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
legend="Run Name: {{ray_train_run_name}}, Worker Group Start Time",
),
Target(
expr='sum(ray_train_worker_group_shutdown_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
legend="Run Name: {{ray_train_run_name}}, Worker Group Shutdown Time",
),
],
fill=0,
stack=False,
)
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL = Panel(
id=3,
title="Cumulative Time in ray.train.report",
description="Cumulative time workers spend blocked inside `ray.train.report()`. This includes the cross-rank checkpoint directory sync barrier, the checkpoint file transfer to storage, and the time waiting for the report queue ordering. See the Checkpoint Sync and Checkpoint Transfer panels for a breakdown.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_report_total_blocked_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
WORKER_CHECKPOINT_SYNC_TIME_PANEL = Panel(
id=16,
title="Cumulative Checkpoint Sync Time",
description="Cumulative time spent in the cross-rank barrier that synchronizes the checkpoint directory name across all workers. High values indicate workers are spending significant time waiting for each other to reach the synchronization point.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_checkpoint_sync_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL = Panel(
id=17,
title="Cumulative Checkpoint Transfer Time",
description="Cumulative time spent transferring checkpoint files to storage. High values indicate slow storage throughput or large checkpoint sizes.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_checkpoint_transfer_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
# Core System Resources
CPU_UTILIZATION_PANEL = Panel(
id=4,
title="CPU Usage",
description="CPU core utilization across all workers.",
unit="cores",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
MEMORY_UTILIZATION_PANEL = Panel(
id=5,
title="Total Memory Usage",
description="Total physical memory used vs total available memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
MEMORY_DETAILED_PANEL = Panel(
id=6,
title="Memory Allocation Details",
description="Memory allocation details including available and shared memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Available Memory: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Shared Memory: {{instance}} ({{RayNodeType}})",
),
],
)
# GPU Resources
# TODO: Add GPU Device/Index as a filter.
GPU_UTILIZATION_PANEL = Panel(
id=7,
title="GPU Usage",
description="GPU utilization across all workers.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})',
legend="MAX",
),
],
)
GPU_MEMORY_UTILIZATION_PANEL = Panel(
id=8,
title="GPU Memory Usage",
description="GPU memory usage across all workers.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
)
# Storage Resources
DISK_UTILIZATION_PANEL = Panel(
id=9,
title="Disk Space Usage",
description="Disk space usage across all workers.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
DISK_THROUGHPUT_PANEL = Panel(
id=10,
title="Disk Throughput",
description="Current disk read/write throughput.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read Speed: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write Speed: {{instance}} ({{RayNodeType}})",
),
],
)
DISK_OPERATIONS_PANEL = Panel(
id=11,
title="Disk Operations",
description="Current disk read/write operations per second.",
unit="ops/s",
targets=[
Target(
expr='sum(ray_node_disk_read_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read IOPS: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_write_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write IOPS: {{instance}} ({{RayNodeType}})",
),
],
)
# Network Resources
NETWORK_THROUGHPUT_PANEL = Panel(
id=12,
title="Network Throughput",
description="Current network send/receive throughput.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Receive Speed: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Send Speed: {{instance}} ({{RayNodeType}})",
),
],
)
NETWORK_TOTAL_PANEL = Panel(
id=13,
title="Network Total Traffic",
description="Total network traffic sent/received.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_network_sent{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Total Sent: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_received{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Total Received: {{instance}} ({{RayNodeType}})",
),
],
)
TRAIN_GRAFANA_PANELS = []
TRAIN_GRAFANA_ROWS = [
# Train Metrics Row
Row(
title="Train Metrics",
id=14,
panels=[
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL,
CONTROLLER_OPERATION_TIME_PANEL,
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
WORKER_CHECKPOINT_SYNC_TIME_PANEL,
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
],
collapsed=False,
),
# System Resources Row
Row(
title="Resource Utilization",
id=15,
panels=[
CPU_UTILIZATION_PANEL,
MEMORY_UTILIZATION_PANEL,
MEMORY_DETAILED_PANEL,
# GPU Resources
GPU_UTILIZATION_PANEL,
GPU_MEMORY_UTILIZATION_PANEL,
# Storage Resources
DISK_UTILIZATION_PANEL,
DISK_THROUGHPUT_PANEL,
DISK_OPERATIONS_PANEL,
# Network Resources
NETWORK_THROUGHPUT_PANEL,
NETWORK_TOTAL_PANEL,
],
collapsed=True,
),
]
TRAIN_RUN_PANELS = [
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL,
CONTROLLER_OPERATION_TIME_PANEL,
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
]
TRAIN_WORKER_PANELS = [
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
WORKER_CHECKPOINT_SYNC_TIME_PANEL,
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
# Core System Resources
CPU_UTILIZATION_PANEL,
MEMORY_UTILIZATION_PANEL,
# GPU Resources
GPU_UTILIZATION_PANEL,
GPU_MEMORY_UTILIZATION_PANEL,
# Storage Resources
DISK_UTILIZATION_PANEL,
# Network Resources
NETWORK_THROUGHPUT_PANEL,
]
# Get all panel IDs from both top-level panels and panels within rows
all_panel_ids = [panel.id for panel in TRAIN_GRAFANA_PANELS]
for row in TRAIN_GRAFANA_ROWS:
all_panel_ids.append(row.id)
all_panel_ids.extend(panel.id for panel in row.panels)
all_panel_ids.sort()
assert len(all_panel_ids) == len(
set(all_panel_ids)
), f"Duplicated id found. Use unique id for each panel. {all_panel_ids}"
train_dashboard_config = DashboardConfig(
name="TRAIN",
default_uid="rayTrainDashboard",
rows=TRAIN_GRAFANA_ROWS,
standard_global_filters=['SessionName=~"$SessionName"'],
base_json_file_name="train_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json
+267
View File
@@ -0,0 +1,267 @@
{
"title": "Train Dashboard",
"uid": "rayTrainDashboard",
"version": 1,
"schemaVersion": 27,
"style": "dark",
"editable": true,
"graphTooltip": 1,
"refresh": false,
"panels": [],
"time": {
"from": "now-30m",
"to": "now"
},
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"description": "Filter queries of a specific Prometheus type.",
"datasource": null,
"query": "prometheus",
"refresh": 1,
"hide": 2,
"includeAll": false,
"multi": false,
"current": {
"selected": false
}
},
{
"name": "SessionName",
"type": "query",
"description": "Filter queries to specific ray sessions.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainRunName",
"type": "query",
"description": "Filter queries to specific Ray Train run names.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainRunId",
"type": "query",
"description": "Filter queries to specific Ray Train run ids.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainWorkerWorldRank",
"type": "query",
"description": "Filter queries to specific Ray Train worker world ranks.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
"query": {
"query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainWorkerActorId",
"type": "query",
"description": "Filter queries to specific Ray Train worker actor ids.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
"query": {
"query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "Instance",
"type": "query",
"description": "Filter queries to specific node instances.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "GpuIndex",
"type": "query",
"description": "Filter queries to specific GPU indices.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
"query": {
"query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "GpuDeviceName",
"type": "query",
"description": "Filter queries to specific GPU device names.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
"query": {
"query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
},
"description": "Filter queries to specific Ray node types (head or worker).",
"includeAll": true,
"multi": true,
"name": "RayNodeType",
"options": [
{
"selected": false,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "Head Node",
"value": "head"
},
{
"selected": false,
"text": "Worker Node",
"value": "worker"
}
],
"query": "head, worker",
"type": "custom"
}
]
}
}