wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/dashboard/modules/metrics/__init__.py
View File
python/ray/dashboard/modules/metrics/dashboards/__init__.py
View File
python/ray/dashboard/modules/metrics/dashboards/common.py
+615
View File
@@ -0,0 +1,615 @@
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional
from ray.util.annotations import DeveloperAPI
@DeveloperAPI
@dataclass
class GridPos:
x: int
y: int
w: int
h: int
GRAPH_TARGET_TEMPLATE = {
"exemplar": True,
"expr": "0",
"interval": "",
"legendFormat": "",
"queryType": "randomWalk",
"refId": "A",
}
HEATMAP_TARGET_TEMPLATE = {
"format": "heatmap",
"fullMetaSearch": False,
"includeNullMetadata": True,
"instant": False,
"range": True,
"useBackend": False,
}
HISTOGRAM_BAR_CHART_TARGET_TEMPLATE = {
"exemplar": True,
"format": "heatmap",
"fullMetaSearch": False,
"includeNullMetadata": True,
"instant": True,
"range": False,
"useBackend": False,
}
@DeveloperAPI
class TargetTemplate(Enum):
GRAPH = GRAPH_TARGET_TEMPLATE
HEATMAP = HEATMAP_TARGET_TEMPLATE
HISTOGRAM_BAR_CHART = HISTOGRAM_BAR_CHART_TARGET_TEMPLATE
@DeveloperAPI
@dataclass
class Target:
"""Defines a Grafana target (time-series query) within a panel.
A panel will have one or more targets. By default, all targets are rendered as
stacked area charts, with the exception of legend="MAX", which is rendered as
a blue dotted line. Any legend="FINISHED|FAILED|DEAD|REMOVED" series will also be
rendered hidden by default.
Attributes:
expr: The prometheus query to evaluate.
legend: The legend string to format for each time-series.
"""
expr: str
legend: str
template: Optional[TargetTemplate] = TargetTemplate.GRAPH
HEATMAP_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
"id": 12,
"options": {
"calculate": False,
"cellGap": 1,
"cellValues": {"unit": "none"},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"min": 0,
"mode": "scheme",
"reverse": False,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64,
},
"exemplars": {"color": "rgba(255,0,255,0.7)"},
"filterValues": {"le": 1e-9},
"legend": {"show": True},
"rowsFrame": {"layout": "auto", "value": "Value"},
"tooltip": {"mode": "single", "showColorScale": False, "yHistogram": True},
"yAxis": {"axisPlacement": "left", "reverse": False, "unit": "none"},
},
"pluginVersion": "11.2.0",
"targets": [],
"title": "<Title>",
"type": "heatmap",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
GRAPH_PANEL_TEMPLATE = {
"aliasColors": {},
"bars": False,
"dashLength": 10,
"dashes": False,
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
# Setting height and width is important here to ensure the default panel has some size to it.
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"fill": 10,
"fillGradient": 0,
"hiddenSeries": False,
"id": 26,
"legend": {
"alignAsTable": True,
"avg": True,
"current": True,
"hideEmpty": False,
"hideZero": True,
"max": True,
"min": False,
"rightSide": False,
"show": True,
"sort": "current",
"sortDesc": True,
"total": False,
"values": True,
},
"lines": True,
"linewidth": 1,
"nullPointMode": None,
"options": {"alertThreshold": True},
"percentage": False,
"pluginVersion": "7.5.17",
"pointradius": 2,
"points": False,
"renderer": "flot",
# These series overrides are necessary to make the "MAX" and "MAX + PENDING" dotted lines
# instead of stacked filled areas.
"seriesOverrides": [
{
"$$hashKey": "object:2987",
"alias": "MAX",
"dashes": True,
"color": "#1F60C4",
"fill": 0,
"stack": False,
},
{
"$$hashKey": "object:78",
"alias": "/FINISHED|FAILED|DEAD|REMOVED|Failed Nodes:/",
"hiddenSeries": True,
},
{
"$$hashKey": "object:2987",
"alias": "MAX + PENDING",
"dashes": True,
"color": "#777777",
"fill": 0,
"stack": False,
},
{
"alias": "/Container/",
"hiddenSeries": True,
},
{
"alias": "Container MAX",
"dashes": True,
"color": "#73BF69",
"fill": 0,
"stack": False,
"hiddenSeries": True,
},
],
"spaceLength": 10,
"stack": True,
"steppedLine": False,
"targets": [],
"thresholds": [],
"timeFrom": None,
"timeRegions": [],
"timeShift": None,
"title": "<Title>",
"tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
"type": "graph",
"xaxis": {
"buckets": None,
"mode": "time",
"name": None,
"show": True,
"values": [],
},
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
"yaxis": {"align": False, "alignLevel": None},
}
STAT_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "percentage",
"steps": [
{"color": "super-light-yellow", "value": None},
{"color": "super-light-green", "value": 50},
{"color": "green", "value": 100},
],
},
"unit": "short",
},
"overrides": [],
},
"id": 78,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"text": {},
"textMode": "auto",
},
"pluginVersion": "7.5.17",
"targets": [],
"timeFrom": None,
"timeShift": None,
"title": "<Title>",
"type": "stat",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "Tokens",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
GAUGE_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"fieldConfig": {
"defaults": {
"color": {"mode": "continuous-YlBl"},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [{"color": "rgb(230, 230, 230)", "value": None}],
},
"unit": "short",
},
"overrides": [],
},
"id": 10,
"options": {
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"showThresholdLabels": False,
"showThresholdMarkers": False,
"text": {"titleSize": 12},
},
"pluginVersion": "7.5.17",
"targets": [],
"title": "<Title>",
"type": "gauge",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "Tokens",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
PIE_CHART_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
"id": 26,
"options": {
"displayLabels": [],
"legend": {
"displayMode": "table",
"placement": "right",
"values": ["percent", "value"],
},
"pieType": "pie",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"text": {},
},
"pluginVersion": "7.5.17",
"targets": [],
"timeFrom": None,
"timeShift": None,
"title": "<Title>",
"type": "piechart",
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
}
BAR_CHART_PANEL_TEMPLATE = {
"aliasColors": {},
"dashLength": 10,
"dashes": False,
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {"defaults": {}, "overrides": []},
# Setting height and width is important here to ensure the default panel has some size to it.
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"hiddenSeries": False,
"id": 26,
"legend": {
"alignAsTable": True,
"avg": False,
"current": True,
"hideEmpty": False,
"hideZero": True,
"max": False,
"min": False,
"rightSide": False,
"show": False,
"sort": "current",
"sortDesc": True,
"total": False,
"values": True,
},
"lines": False,
"linewidth": 1,
"bars": True,
"nullPointMode": None,
"options": {
"alertThreshold": True,
"legend": {
"showLegend": False,
"displayMode": "table",
"placement": "bottom",
},
},
"percentage": False,
"pluginVersion": "7.5.17",
"pointradius": 2,
"points": False,
"renderer": "flot",
"spaceLength": 10,
"stack": True,
"steppedLine": False,
"targets": [],
"thresholds": [],
"timeFrom": None,
"timeRegions": [],
"timeShift": None,
"title": "<Title>",
"tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
"type": "graph",
"xaxis": {
"buckets": None,
"mode": "series",
"name": None,
"show": True,
"values": [
"total",
],
},
"yaxes": [
{
"$$hashKey": "object:628",
"format": "units",
"label": "",
"logBase": 1,
"max": None,
"min": "0",
"show": True,
},
{
"$$hashKey": "object:629",
"format": "short",
"label": None,
"logBase": 1,
"max": None,
"min": None,
"show": True,
},
],
"yaxis": {"align": False, "alignLevel": None},
}
TABLE_PANEL_TEMPLATE = {
"datasource": r"${datasource}",
"description": "<Description>",
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"displayMode": "auto",
},
"mappings": [],
},
"overrides": [],
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"id": 26,
"options": {
"showHeader": True,
"footer": {
"show": False,
"reducer": ["sum"],
"fields": "",
},
},
"pluginVersion": "7.5.17",
"targets": [],
"title": "<Title>",
"type": "table",
"transformations": [{"id": "organize", "options": {}}],
}
@DeveloperAPI
class PanelTemplate(Enum):
GRAPH = GRAPH_PANEL_TEMPLATE
HEATMAP = HEATMAP_TEMPLATE
PIE_CHART = PIE_CHART_TEMPLATE
STAT = STAT_PANEL_TEMPLATE
GAUGE = GAUGE_PANEL_TEMPLATE
BAR_CHART = BAR_CHART_PANEL_TEMPLATE
TABLE = TABLE_PANEL_TEMPLATE
@DeveloperAPI
@dataclass
class Panel:
"""Defines a Grafana panel (graph) for the Ray dashboard page.
A panel contains one or more targets (time-series queries).
Attributes:
title: Short name of the graph. Note: please keep this in sync with the title
definitions in Metrics.tsx.
description: Long form description of the graph.
id: Integer id used to reference the graph from Metrics.tsx.
unit: The unit to display on the y-axis of the graph.
targets: List of query targets.
fill: Whether or not the graph will be filled by a color.
stack: Whether or not the lines in the graph will be stacked.
linewidth: Width of the lines in the graph.
grid_pos: Grid position of the panel.
template: The panel template to use.
hideXAxis: Whether to hide the x-axis.
thresholds: Custom threshold configuration for stat/gauge panels.
Example: [
{"color": "green", "value": None},
{"color": "yellow", "value": 70},
{"color": "red", "value": 90}
]
value_mappings: Value mappings for displaying text instead of numbers.
Used for status panels.
color_mode: Color mode for stat panels ("value", "background", "none").
legend_mode: Legend display mode ("list", "table", "hidden").
min_val: Minimum value for gauge/graph y-axis.
max_val: Maximum value for gauge/graph y-axis.
reduce_calc: Reduce calculation method for stat panels (default: "lastNotNull").
heatmap_color_scheme: Color scheme for heatmap panels (e.g., "Spectral", "RdYlGn").
heatmap_color_reverse: Whether to reverse the heatmap color scheme.
heatmap_yaxis_label: Y-axis label for heatmap panels.
"""
title: str
description: str
id: int
unit: str
targets: List[Target]
fill: int = 10
stack: bool = True
linewidth: int = 1
grid_pos: Optional[GridPos] = None
template: Optional[PanelTemplate] = PanelTemplate.GRAPH
hideXAxis: bool = False
thresholds: Optional[List[Dict[str, Any]]] = None
value_mappings: Optional[List[Dict[str, Any]]] = None
color_mode: Optional[str] = None
legend_mode: Optional[str] = None
min_val: Optional[float] = None
max_val: Optional[float] = None
reduce_calc: Optional[str] = None
heatmap_color_scheme: Optional[str] = None
heatmap_color_reverse: Optional[bool] = None
heatmap_yaxis_label: Optional[str] = None
@DeveloperAPI
@dataclass
class Row:
"""Defines a Grafana row that can contain multiple panels.
Attributes:
title: The title of the row
panels: List of panels contained in this row
collapsed: Whether the row should be collapsed by default
"""
title: str
id: int
panels: List[Panel]
collapsed: bool = False
@DeveloperAPI
@dataclass
class DashboardConfig:
# This dashboard name is an internal key used to determine which env vars
# to check for customization
name: str
# The uid of the dashboard json if not overridden by a user
default_uid: str
# The global filters applied to all graphs in this dashboard. Users can
# add additional global_filters on top of this.
standard_global_filters: List[str]
base_json_file_name: str
# Panels can be specified in `panels`, or nested within `rows`.
# If both are specified, panels will be rendered before rows.
panels: List[Panel] = field(default_factory=list)
rows: List[Row] = field(default_factory=list)
def __post_init__(self):
if not self.panels and not self.rows:
raise ValueError("At least one of panels or rows must be specified")
python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py
+1588
View File
File diff suppressed because it is too large Load Diff
python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json
+183
View File
@@ -0,0 +1,183 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".+",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
"description": "Filter queries to specific ray sessions.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "SessionName",
"options": [],
"query": {
"query": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "{SessionName=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "${datasource}",
"definition": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "DatasetID",
"options": [],
"query": {
"query": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"refId": "Prometheus-Dataset-Variable-Query"
},
"refresh": 2,
"regex": "{dataset=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "${datasource}",
"definition": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Operator",
"options": [],
"query": {
"query": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
"refId": "Prometheus-Dataset-Variable-Query"
},
"refresh": 2,
"regex": "{operator=\"(?<value>.*)\".*",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Data Dashboard",
"uid": "rayDataDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/data_llm_dashboard_panels.py
+325
View File
@@ -0,0 +1,325 @@
# ruff: noqa: E501
"""Ray Data LLM Dashboard panels for vLLM metrics visualization.
This dashboard provides visibility into vLLM engine metrics when using Ray Data LLM,
including latency metrics (TTFT, TPOT, E2E), throughput, cache utilization,
prefix cache hit rate, and scheduler state.
"""
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
DATA_LLM_GRAFANA_PANELS = [
Panel(
id=1,
title="vLLM: Token Throughput",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr='sum by (model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Prompt Tokens/Sec - {{model_name}} - {{WorkerId}}",
),
Target(
expr='sum by (model_name, WorkerId) (rate(ray_vllm_generation_tokens_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Generation Tokens/Sec - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 0, 12, 8),
),
Panel(
id=2,
title="vLLM: Time Per Output Token Latency",
description="P50, P90, P95, P99, and Mean TPOT latency",
unit="s",
targets=[
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Mean - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 0, 12, 8),
),
Panel(
id=3,
title="vLLM: Cache Utilization",
description="Percentage of used KV cache blocks by vLLM.",
unit="percentunit",
targets=[
Target(
expr='sum by (WorkerId) (ray_vllm_kv_cache_usage_perc{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}})',
legend="GPU Cache Usage - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 8, 12, 8),
),
Panel(
id=4,
title="vLLM: Prefix Cache Hit Rate",
description="Percentage of prefix cache hits. Higher is better for repeated prefixes.",
unit="percent",
targets=[
Target(
expr='max(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
legend="Max Hit Rate",
),
Target(
expr='min(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
legend="Min Hit Rate",
),
Target(
expr='100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Hit Rate: worker {{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 8, 12, 8),
),
Panel(
id=5,
title="vLLM: Time To First Token Latency",
description="P50, P90, P95, P99, and Mean TTFT latency",
unit="s",
targets=[
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Average - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 16, 12, 8),
),
Panel(
id=6,
title="vLLM: E2E Request Latency",
description="End-to-end request latency from arrival to completion.",
unit="s",
targets=[
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Average - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 16, 12, 8),
),
Panel(
id=7,
title="vLLM: Scheduler State",
description="Number of requests in RUNNING, WAITING, and SWAPPED state",
unit="Requests",
targets=[
Target(
expr='ray_vllm_num_requests_running{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Running - {{model_name}} - {{WorkerId}}",
),
Target(
expr='ray_vllm_num_requests_swapped{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Swapped - {{model_name}} - {{WorkerId}}",
),
Target(
expr='ray_vllm_num_requests_waiting{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
legend="Num Waiting - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 24, 12, 8),
),
Panel(
id=8,
title="vLLM: Queue Time",
description="P50, P90, P95, P99, and Mean time requests spend waiting in the queue.",
unit="s",
targets=[
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Mean - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P95 - {{model_name}} - {{WorkerId}}",
),
Target(
expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P99 - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 24, 12, 8),
),
Panel(
id=9,
title="vLLM: Prompt Length",
description="Distribution of prompt token lengths.",
unit="short",
targets=[
Target(
expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50-{{model_name}}-{{WorkerId}}",
),
Target(
expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90-{{model_name}}-{{WorkerId}}",
),
Target(
expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="Average-{{model_name}}-{{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 32, 12, 8),
),
Panel(
id=10,
title="vLLM: Generation Length",
description="Distribution of generated token lengths.",
unit="short",
targets=[
Target(
expr='histogram_quantile(0.50, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P50-{{model_name}}-{{WorkerId}}",
),
Target(
expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
legend="P90-{{model_name}}-{{WorkerId}}",
),
Target(
expr=(
'(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
"\n/\n"
'(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
),
legend="Average-{{model_name}}-{{WorkerId}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 32, 12, 8),
),
Panel(
id=11,
title="vLLM: Finish Reason",
description="Number of finished requests by their finish reason: EOS token or max length reached.",
unit="Requests",
targets=[
Target(
expr='sum by(finished_reason, model_name, WorkerId) (increase(ray_vllm_request_success_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="{{finished_reason}} - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 40, 12, 8),
),
Panel(
id=12,
title="vLLM: Prefill and Decode Time",
description="Time spent in prefill vs decode phases.",
unit="s",
targets=[
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_prefill_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Prefill - {{model_name}} - {{WorkerId}}",
),
Target(
expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_decode_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
legend="Decode - {{model_name}} - {{WorkerId}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 40, 12, 8),
),
]
data_llm_dashboard_config = DashboardConfig(
name="DATA_LLM",
default_uid="rayDataLlmDashboard",
panels=DATA_LLM_GRAFANA_PANELS,
standard_global_filters=[],
base_json_file_name="data_llm_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/data_llm_grafana_dashboard_base.json
+144
View File
@@ -0,0 +1,144 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"name": "vllm_model_name",
"label": "vLLM Model Name",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "workerid",
"label": "Worker ID",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "interval",
"label": "Interval",
"type": "custom",
"hide": 0,
"includeAll": false,
"multi": false,
"options": [
{
"selected": true,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
}
],
"current": {
"selected": true,
"text": "5m",
"value": "5m"
}
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Data LLM Dashboard",
"uid": "rayDataLlmDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py
+797
View File
@@ -0,0 +1,797 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
Panel,
Row,
Target,
)
"""
Queries for autoscaler resources.
"""
# Note: MAX & USED resources are reported from raylet to provide the most up to date information.
# But MAX + PENDING data is coming from the autoscaler. That said, MAX + PENDING can be
# more outdated. it is harmless because the actual MAX will catch up with MAX + PENDING
# eventually.
MAX_CPUS = 'sum(autoscaler_cluster_resources{{resource="CPU",{global_filters}}})'
PENDING_CPUS = 'sum(autoscaler_pending_resources{{resource="CPU",{global_filters}}})'
MAX_GPUS = 'sum(autoscaler_cluster_resources{{resource="GPU",{global_filters}}})'
PENDING_GPUS = 'sum(autoscaler_pending_resources{{resource="GPU",{global_filters}}})'
MAX_MEMORY = 'sum(autoscaler_cluster_resources{{resource="memory",{global_filters}}})'
PENDING_MEMORY = (
'sum(autoscaler_pending_resources{{resource="memory",{global_filters}}})'
)
def max_plus_pending(max_resource, pending_resource):
return f"({max_resource} or vector(0)) + ({pending_resource} or vector(0))"
MAX_PLUS_PENDING_CPUS = max_plus_pending(MAX_CPUS, PENDING_CPUS)
MAX_PLUS_PENDING_GPUS = max_plus_pending(MAX_GPUS, PENDING_GPUS)
MAX_PLUS_PENDING_MEMORY = max_plus_pending(MAX_MEMORY, PENDING_MEMORY)
MAX_PERCENTAGE_EXPRESSION = (
"100" # To help draw the max limit line on percentage panels
)
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# IMPORTANT: Please keep this in sync with Metrics.tsx and ray-metrics.rst
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
OVERVIEW_AND_HEALTH_PANELS = [
Panel(
id=24,
title="Node Count",
description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.',
unit="nodes",
targets=[
Target(
expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
legend="Active Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
legend="Failed Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
legend="Pending Nodes: {{NodeType}}",
),
],
),
Panel(
id=41,
title="Cluster Utilization",
description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.",
unit="%",
targets=[
# CPU
Target(
expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})',
legend="CPU (physical)",
),
# GPU
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}}) or vector(0))',
legend="GPU (physical)",
),
# Memory
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}})) * 100',
legend="Memory (RAM)",
),
# GRAM
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100',
legend="GRAM",
),
# Object Store
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100',
legend="Object Store Memory",
),
# Disk
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100',
legend="Disk",
),
],
fill=0,
stack=False,
),
Panel(
id=44,
title="Ray OOM Kills (Tasks and Actors)",
description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html. Note: The RayNodeType filter does not work on this graph.",
unit="failures",
targets=[
Target(
expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", {global_filters}}}) by (Name, instance)',
legend="OOM Killed: {{Name}}, {{instance}}",
),
],
),
Panel(
id=65,
title="Unexpected System Level Worker Failures",
description="The number of workers (potentially tasks or actors) that disconnected from the raylet unexpectedly. "
"This typically indicates the worker process unexpectedly failed due to "
"a Ray system error or a kernel kill (e.g. OOM, SIGKILL, Bad exit code). "
"Note that this metric only includes OOM kills from the kernel and does not "
"include OOM kills from Ray's memory monitor. "
"If errors of this type is encountered when the node is under memory pressure, "
"The failures are likely OOM kills.",
unit="failures",
targets=[
Target(
expr='sum(ray_node_manager_unexpected_worker_failure_total{{instance=~"$Instance", {global_filters}}}) by (Type, Name, instance)',
legend="Unexpected worker failure: {{Name}}, {{Type}}, {{instance}}",
),
],
),
]
RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS = [
Panel(
id=26,
title="All Tasks by State",
description="Current count of tasks, grouped by scheduler state (e.g., pending, running, finished).\n\nState: the task state, as described by rpc::TaskStatus proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='sum(max_over_time(ray_tasks{{IsRetry="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
legend="{{State}}",
),
Target(
expr='sum(max_over_time(ray_tasks{{IsRetry!="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
legend="{{State}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=35,
title="Active Tasks by Name",
description="Current count of active tasks (i.e. pending or running; not finished), grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}}",
),
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=38,
title="Running Tasks by Name",
description="Current count of tasks that are currently executing, grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}}",
),
Target(
expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
legend="{{Name}} (retry)",
),
],
fill=0,
stack=False,
),
Panel(
id=64,
title="Running Tasks by Node",
description="Current count of tasks that are currently executing, grouped by node.",
unit="tasks",
targets=[
Target(
expr='clamp_min(sum(ray_tasks{{State=~"RUNNING*",Source="executor",instance=~"$Instance",{global_filters}}}) by (instance), 0)',
legend="{{instance}}",
),
],
fill=0,
stack=False,
),
Panel(
id=33,
title="All Actors by State",
description='Note: not impacted by "Instance" variable.\n\nCurrent count of actors, grouped by lifecycle state (e.g., alive, restarting, dead/terminated).\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.',
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{Source="gcs",{global_filters}}}) by (State)',
legend="{{State}}",
)
],
),
Panel(
id=42,
title="Alive Actors by State",
description="Current count of alive actors (i.e. not dead/terminated), grouped by state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.",
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (State)',
legend="{{State}}",
)
],
),
Panel(
id=36,
title="Alive Actors by Name",
description="Current count of alive actors, grouped by actor name.",
unit="actors",
targets=[
Target(
expr='sum(ray_actors{{State!="DEAD",Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (Name)',
legend="{{Name}}",
)
],
),
Panel(
id=40,
title="All Placement Groups by State",
description='Note: not impacted by "Instance" variable.\n\nCurrent count of placement groups, grouped by state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTableData proto in gcs.proto.',
unit="placement groups",
targets=[
Target(
expr="sum(ray_placement_groups{{{global_filters}}}) by (State)",
legend="{{State}}",
)
],
),
Panel(
id=29,
title="Object Store Memory by Location",
description="Object store memory usage by location. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)',
legend="{{Location}}",
),
Target(
expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
]
RAY_RESOURCES_PANELS = [
Panel(
id=27,
title="Logical CPUs Usage",
description="Logical CPU usage of Ray. The dotted line indicates the total number of CPUs. The logical CPU is allocated by `num_cpus` arguments from tasks and actors. PENDING means the number of CPUs that will be available when new nodes are up after the autoscaler scales up.\n\nNOTE: Ray's logical CPU is different from physical CPU usage. Ray's logical CPU is allocated by `num_cpus` arguments.",
unit="cores",
targets=[
Target(
expr='sum(ray_resources{{Name="CPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="CPU Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="CPU",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
# If max + pending > max, we display this value.
# (A and predicate) means to return A when the predicate satisfies in PromSql.
Target(
expr=f"({MAX_PLUS_PENDING_CPUS} and {MAX_PLUS_PENDING_CPUS} > ({MAX_CPUS} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=28,
title="Logical GPUs Usage",
description="Logical GPU usage of Ray. The dotted line indicates the total number of GPUs. The logical GPU is allocated by `num_gpus` arguments from tasks and actors. PENDING means the number of GPUs that will be available when new nodes are up after the autoscaler scales up.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_resources{{Name="GPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="GPU Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="GPU",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
# If max + pending > max, we display this value.
# (A and predicate) means to return A when the predicate satisfies in PromSql.
Target(
expr=f"({MAX_PLUS_PENDING_GPUS} and {MAX_PLUS_PENDING_GPUS} > ({MAX_GPUS} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=61,
title="Logical Memory Usage",
description="Logical memory usage of Ray by node. The dotted line indicates the total amount of memory available. Logical memory refers to Ray's view of memory resources allocated to tasks and actors. PENDING means the amount of memory that will be available when new nodes are up after the autoscaler scales up.",
unit="bytes",
targets=[
Target(
expr='sum(ray_resources{{Name="memory",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="Memory Usage: {{instance}}",
),
Target(
expr='sum(ray_resources{{Name="memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
Target(
expr=f"({MAX_PLUS_PENDING_MEMORY} and {MAX_PLUS_PENDING_MEMORY} > ({MAX_MEMORY} or vector(0)))",
legend="MAX + PENDING",
),
],
),
Panel(
id=58,
title="Object Store Memory Usage",
description="Object store memory usage by instance, including memory that has been spilled to disk. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
Target(
expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=59,
title="Object Store Memory Usage %",
description="Object store memory usage % by instance, including memory that has been spilled to disk. This metric can go over 100% in case of spillage to disk.",
unit="%",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance) * 100 / sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
Target(
expr=MAX_PERCENTAGE_EXPRESSION, # To show the memory limit visually
legend="MAX",
),
],
fill=0,
stack=False,
),
Panel(
id=60,
title="Object Store Memory Spilled to Disk",
description="Object store memory that has been spilled to disk, by instance.",
unit="bytes",
targets=[
Target(
expr='sum(ray_object_store_memory{{instance=~"$Instance",Location="SPILLED",{global_filters}}}) by (instance)',
legend="{{instance}}",
),
],
fill=0,
stack=False,
),
]
NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS = [
Panel(
id=37,
title="Node CPU Usage by Component",
description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
unit="cores",
targets=[
Target(
# ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU.
expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100',
legend="{{Component}}",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=34,
title="Node Memory Usage by Component",
description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
unit="bytes",
targets=[
Target(
expr='(sum(ray_component_rss_bytes{{instance=~"$Instance",{global_filters}}}) by (Component)) - (sum(ray_component_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))',
legend="{{Component}}",
),
Target(
expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})',
legend="shared_memory",
),
Target(
expr='min(label_replace(sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "host", "", "") or label_replace(sum(ray_node_cgroup_mem_total{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "cgroup", "", ""))',
legend="MAX",
),
],
),
Panel(
id=45,
title="Node GPU Usage by Component",
description="The physical (hardware) GPU usage across the cluster, broken down by component. This reports the summed GPU usage per Ray component.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_component_gpu_percentage{{instance=~"$Instance",{global_filters}}} / 100) by (Component)',
legend="{{Component}}",
),
],
),
Panel(
id=46,
title="Node GPU Memory Usage by Component",
description="The physical (hardware) GPU memory usage across the cluster, broken down by component. This reports the summed GPU memory usage per Ray component.",
unit="bytes",
targets=[
Target(
expr='sum(ray_component_gpu_memory_mb{{instance=~"$Instance",{global_filters}}} * 1024 * 1024) by (Component)',
legend="{{Component}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
),
]
NODE_HARDWARE_UTILIZATION_PANELS = [
Panel(
id=2,
title="Node CPU Usage",
description="The physical (hardware) CPU usage for each node.",
unit="cores",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=54,
title="Node CPU Usage %",
description="The percentage of physical (hardware) CPU usage for each node.",
unit="%",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=4,
title="Node Memory Usage (heap + object store)",
description="The physical (hardware) memory usage for each node. The dotted line means the total amount of memory from the cluster. "
"Node memory is a sum of object store memory (shared memory) and heap memory.\n\n"
"Host targets reflect memory as reported by the host machine. "
"Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
Target(
expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="Container MAX",
),
],
),
Panel(
id=48,
title="Node Memory Usage % (heap + object store)",
description="The percentage of physical (hardware) memory usage for each node.\n\n"
"Host targets reflect memory as reported by the host machine. "
"Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
unit="%",
targets=[
Target(
expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=6,
title="Node Disk Usage",
description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=57,
title="Node Disk Usage %",
description="Node's physical (hardware) disk usage. \n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
unit="%",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / (sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType))',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
],
fill=0,
stack=False,
),
Panel(
id=8,
title="Node GPU Usage",
description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ",
unit="GPUs",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
),
Panel(
id=55,
title="Node GPU Usage %",
description="Node's physical (hardware) GPU usage.",
unit="%",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=18,
title="Node GPU Memory Usage (GRAM)",
description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
),
Panel(
id=56,
title="Node GPU Memory Usage (GRAM) %",
description="The percentage of physical (hardware) GPU memory usage for each node.",
unit="%",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) * 100 / (sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName))',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=62,
title="Node GPU Power",
description="Current GPU power draw per node. Reported in milliwatts; displayed in watts. Supported on NVIDIA and AMD GPUs.",
unit="mwatt",
targets=[
Target(
expr='sum(ray_node_gpu_power_milliwatts{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Power: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=63,
title="Node GPU Temperature",
description="Current GPU temperature per node in Celsius. Supported on NVIDIA GPUs.",
unit="celsius",
targets=[
Target(
expr='sum(ray_node_gpu_temperature_celsius{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Temperature: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
],
fill=0,
stack=False,
),
Panel(
id=32,
title="Node Disk IO Speed",
description="Disk IO per node.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read: {{instance}} ({{RayNodeType}})",
),
],
),
Panel(
id=20,
title="Node Network",
description="Network speed per node",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Recv: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Send: {{instance}} ({{RayNodeType}})",
),
],
),
]
NODE_TPU_UTILIZATION_PANELS = [
Panel(
id=50,
title="Node TPU Tensorcore Utilization %",
description="Percentage of tensorcore utilization for the TPUs on this node. Computed by dividing the number of tensorcore operations by the maximum supported number of operations during the sample period.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_tensorcore_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=51,
title="Node TPU High Bandwidth Memory Utilization %",
description="Percentage of bandwidth memory utilization for the TPUs on this node. Computed by dividing the memory bandwidth used by the maximum supported memory bandwidth limit during the sample period.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_memory_bandwidth_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=52,
title="Node TPU Duty Cycle %",
description="Percentage of time over the sample period during which the TPU is actively processing.",
unit="%",
targets=[
Target(
expr='sum(ray_tpu_duty_cycle{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
Panel(
id=53,
title="Node TPU Memory Used",
description="Total memory used/allocated for the TPUs on this node.",
unit="bytes",
targets=[
Target(
expr='sum(ray_tpu_memory_used{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="Memory Used: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
Target(
expr='sum(ray_tpu_memory_total{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
legend="Memory Total: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
),
],
),
]
DEFAULT_GRAFANA_ROWS = [
Row(
title="Overview and Health",
id=1001,
panels=OVERVIEW_AND_HEALTH_PANELS,
collapsed=False,
),
Row(
title="Hardware Utilization by Node",
id=1005,
panels=NODE_HARDWARE_UTILIZATION_PANELS,
collapsed=False,
),
Row(
title="Hardware Utilization by Ray Component",
id=1004,
panels=NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS,
collapsed=False,
),
Row(
title="Ray Resources by Node",
id=1003,
panels=RAY_RESOURCES_PANELS,
collapsed=False,
),
Row(
title="Ray Tasks, Actors and Placement Groups",
id=1002,
panels=RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS,
collapsed=False,
),
Row(
title="TPU Utilization by Node",
id=1006,
panels=NODE_TPU_UTILIZATION_PANELS,
collapsed=True,
),
]
ids = []
for row in DEFAULT_GRAFANA_ROWS:
ids.append(row.id)
ids.extend(panel.id for panel in row.panels)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
default_dashboard_config = DashboardConfig(
name="DEFAULT",
default_uid="rayDefaultDashboard",
rows=DEFAULT_GRAFANA_ROWS,
standard_global_filters=[
'SessionName=~"$SessionName"',
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="default_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json
+177
View File
@@ -0,0 +1,177 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".+",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
"description": "Filter queries to specific Ray sessions.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "SessionName",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".+",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
"description": "Filter queries to specific Ray nodes by their IP address.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Instance",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple Ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
},
"description": "Filter queries to specific Ray node types (head or worker).",
"includeAll": true,
"multi": true,
"name": "RayNodeType",
"options": [
{
"selected": false,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "Head Node",
"value": "head"
},
{
"selected": false,
"text": "Worker Node",
"value": "worker"
}
],
"query": "head, worker",
"type": "custom"
}
]
},
"rayMeta": ["supportsFullGrafanaView"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Default Dashboard",
"uid": "rayDefaultDashboard",
"version": 4
}
python/ray/dashboard/modules/metrics/dashboards/serve_dashboard_panels.py
+423
View File
@@ -0,0 +1,423 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
SERVE_GRAFANA_PANELS = [
Panel(
id=5,
title="Cluster Utilization",
description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster. Ignores application variable.",
unit="%",
targets=[
# CPU
Target(
expr="avg(ray_node_cpu_utilization{{{global_filters}}})",
legend="CPU (physical)",
),
# GPU
Target(
expr="sum(ray_node_gpus_utilization{{{global_filters}}}) / on() (sum(autoscaler_cluster_resources{{resource='GPU',{global_filters}}}) or vector(0))",
legend="GPU (physical)",
),
# Memory
Target(
expr="sum(ray_node_mem_used{{{global_filters}}}) / on() (sum(ray_node_mem_total{{{global_filters}}})) * 100",
legend="Memory (RAM)",
),
# GRAM
Target(
expr="sum(ray_node_gram_used{{{global_filters}}}) / on() (sum(ray_node_gram_available{{{global_filters}}}) + sum(ray_node_gram_used{{{global_filters}}})) * 100",
legend="GRAM",
),
# Object Store
Target(
expr='sum(ray_object_store_memory{{{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",{global_filters}}}) * 100',
legend="Object Store Memory",
),
# Disk
Target(
expr="sum(ray_node_disk_usage{{{global_filters}}}) / on() (sum(ray_node_disk_free{{{global_filters}}}) + sum(ray_node_disk_usage{{{global_filters}}})) * 100",
legend="Disk",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 0, 12, 8),
),
Panel(
id=7,
title="QPS per application",
description="QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
legend="{{application, route}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
legend="{{application, method}}",
),
],
grid_pos=GridPos(12, 0, 12, 8),
),
Panel(
id=8,
title="Error QPS per application",
description="Error QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
legend="{{application, route}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
legend="{{application, method}}",
),
],
grid_pos=GridPos(0, 1, 12, 8),
),
Panel(
id=17,
title="Error QPS per application per error code",
description="Error QPS for each selected application.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, error_code)',
legend="{{application, route, error_code}}",
),
Target(
expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, error_code)',
legend="{{application, method, error_code}}",
),
],
grid_pos=GridPos(12, 1, 12, 8),
),
Panel(
id=12,
title="P50 latency per application",
description="P50 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 2, 8, 8),
),
Panel(
id=15,
title="P90 latency per application",
description="P90 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 2, 8, 8),
),
Panel(
id=16,
title="P99 latency per application",
description="P99 latency for selected applications.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
legend="{{application, route}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
legend="{{application, method}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 2, 8, 8),
),
Panel(
id=2,
title="Replicas per deployment",
description='Number of replicas per deployment. Ignores "Application" variable.',
unit="replicas",
targets=[
Target(
expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(0, 3, 8, 8),
),
Panel(
id=13,
title="QPS per deployment",
description="QPS for each deployment.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_request_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(8, 3, 8, 8),
),
Panel(
id=14,
title="Error QPS per deployment",
description="Error QPS for each deplyoment.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_error_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
legend="{{application, deployment}}",
),
],
grid_pos=GridPos(16, 3, 8, 8),
),
Panel(
id=9,
title="P50 latency per deployment",
description="P50 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 4, 8, 8),
),
Panel(
id=10,
title="P90 latency per deployment",
description="P90 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 4, 8, 8),
),
Panel(
id=11,
title="P99 latency per deployment",
description="P99 latency per deployment.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
legend="{{application, deployment}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 4, 8, 8),
),
Panel(
id=3,
title="Queue size per deployment",
description='Number of requests queued per deployment. Ignores "Application" variable.',
unit="requests",
targets=[
Target(
expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
legend="{{application, deployment}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 5, 8, 8),
),
Panel(
id=4,
title="Node count",
description='Number of nodes in this cluster. Ignores "Application" variable.',
unit="nodes",
targets=[
# TODO(aguo): Update this to use autoscaler metrics instead
Target(
expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
legend="Active Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
legend="Failed Nodes: {{NodeType}}",
),
Target(
expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
legend="Pending Nodes: {{NodeType}}",
),
],
grid_pos=GridPos(8, 5, 8, 8),
),
Panel(
id=6,
title="Node network",
description='Network speed per node. Ignores "Application" variable.',
unit="Bps",
targets=[
Target(
expr="sum(ray_node_network_receive_speed{{{global_filters}}}) by (instance)",
legend="Recv: {{instance}}",
),
Target(
expr="sum(ray_node_network_send_speed{{{global_filters}}}) by (instance)",
legend="Send: {{instance}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(16, 5, 8, 8),
),
Panel(
id=20,
title="Ongoing HTTP Requests",
description="The number of ongoing requests in the HTTP Proxy.",
unit="requests",
targets=[
Target(
expr="ray_serve_num_ongoing_http_requests{{{global_filters}}}",
legend="Ongoing HTTP Requests",
),
],
grid_pos=GridPos(0, 6, 8, 8),
),
Panel(
id=21,
title="Ongoing gRPC Requests",
description="The number of ongoing requests in the gRPC Proxy.",
unit="requests",
targets=[
Target(
expr="ray_serve_num_ongoing_grpc_requests{{{global_filters}}}",
legend="Ongoing gRPC Requests",
),
],
grid_pos=GridPos(8, 6, 8, 8),
),
Panel(
id=22,
title="Scheduling Tasks",
description="The number of request scheduling tasks in the router.",
unit="tasks",
targets=[
Target(
expr="ray_serve_num_scheduling_tasks{{{global_filters}}}",
legend="Scheduling Tasks",
),
],
grid_pos=GridPos(16, 6, 8, 8),
),
Panel(
id=23,
title="Scheduling Tasks in Backoff",
description="The number of request scheduling tasks in the router that are undergoing backoff.",
unit="tasks",
targets=[
Target(
expr="ray_serve_num_scheduling_tasks_in_backoff{{{global_filters}}}",
legend="Scheduling Tasks in Backoff",
),
],
grid_pos=GridPos(0, 7, 8, 8),
),
Panel(
id=24,
title="Controller Control Loop Duration",
description="The duration of the last control loop.",
unit="seconds",
targets=[
Target(
expr="ray_serve_controller_control_loop_duration_s{{{global_filters}}}",
legend="Control Loop Duration",
),
],
grid_pos=GridPos(8, 7, 8, 8),
),
Panel(
id=25,
title="Number of Control Loops",
description="The number of control loops performed by the controller. Increases monotonically over the controller's lifetime.",
unit="loops",
targets=[
Target(
expr="ray_serve_controller_num_control_loops{{{global_filters}}}",
legend="Control Loops",
),
],
grid_pos=GridPos(16, 7, 8, 8),
),
]
ids = []
for panel in SERVE_GRAFANA_PANELS:
ids.append(panel.id)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
serve_dashboard_config = DashboardConfig(
name="SERVE",
default_uid="rayServeDashboard",
panels=SERVE_GRAFANA_PANELS,
standard_global_filters=[
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/serve_deployment_dashboard_panels.py
+262
View File
@@ -0,0 +1,262 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
Target,
)
SERVE_DEPLOYMENT_GRAFANA_PANELS = [
Panel(
id=1,
title="Replicas per deployment",
description='Number of replicas per deployment. Ignores "Route" variable.',
unit="replicas",
targets=[
Target(
expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(0, 0, 8, 8),
),
Panel(
id=2,
title="QPS per replica",
description="QPS for each replica.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_request_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(8, 0, 8, 8),
),
Panel(
id=3,
title="Error QPS per replica",
description="Error QPS for each replica.",
unit="qps",
targets=[
Target(
expr='sum(rate(ray_serve_deployment_error_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(16, 0, 8, 8),
),
Panel(
id=4,
title="P50 latency per replica",
description="P50 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 1, 8, 8),
),
Panel(
id=5,
title="P90 latency per replica",
description="P90 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 1, 8, 8),
),
Panel(
id=6,
title="P99 latency per replica",
description="P99 latency per replica.",
unit="ms",
targets=[
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
legend="{{application}}#{{deployment}}#{{replica}}",
),
Target(
expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",{global_filters}}}[5m])) by (le))',
legend="Total",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 1, 8, 8),
),
Panel(
id=7,
title="Queue size per deployment",
description='Number of requests queued per deployment. Ignores "Replica" and "Route" variable.',
unit="requests",
targets=[
Target(
expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 2, 12, 8),
),
Panel(
id=8,
title="Running requests per replica",
description="Current running requests for each replica.",
unit="requests",
targets=[
Target(
expr="sum(ray_serve_replica_processing_queries{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(12, 2, 12, 8),
),
Panel(
id=9,
title="Multiplexed models per replica",
description="The number of multiplexed models for each replica.",
unit="models",
targets=[
Target(
expr="sum(ray_serve_num_multiplexed_models{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 3, 8, 8),
),
Panel(
id=10,
title="Multiplexed model loads per replica",
description="The number of times of multiplexed models loaded for each replica.",
unit="times",
targets=[
Target(
expr="sum(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 3, 8, 8),
),
Panel(
id=11,
title="Multiplexed model unloads per replica",
description="The number of times of multiplexed models unloaded for each replica.",
unit="times",
targets=[
Target(
expr="sum(ray_serve_multiplexed_models_unload_counter_total{{{global_filters}}}) by (application, deployment, replica)",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(16, 3, 8, 8),
),
Panel(
id=12,
title="P99 latency of multiplexed model loads per replica",
description="P99 latency of multiplexed model load per replica.",
unit="ms",
targets=[
Target(
expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_load_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(0, 4, 8, 8),
),
Panel(
id=13,
title="P99 latency of multiplexed model unloads per replica",
description="P99 latency of multiplexed model unload per replica.",
unit="ms",
targets=[
Target(
expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_unload_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
fill=0,
stack=False,
grid_pos=GridPos(8, 4, 8, 8),
),
Panel(
id=14,
title="Multiplexed model ids per replica",
description="The ids of multiplexed models for each replica.",
unit="model",
targets=[
Target(
expr="ray_serve_registered_multiplexed_model_id{{{global_filters}}}",
legend="{{replica}}:{{model_id}}",
),
],
grid_pos=GridPos(16, 4, 8, 8),
stack=False,
),
Panel(
id=15,
title="Multiplexed model cache hit rate",
description="The cache hit rate of multiplexed models for the deployment.",
unit="%",
targets=[
Target(
expr="(1 - sum(rate(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}[5m]))/sum(rate(ray_serve_multiplexed_get_model_requests_counter_total{{{global_filters}}}[5m])))",
legend="{{application}}#{{deployment}}#{{replica}}",
),
],
grid_pos=GridPos(0, 5, 8, 8),
),
]
ids = []
for panel in SERVE_DEPLOYMENT_GRAFANA_PANELS:
ids.append(panel.id)
ids.sort()
assert len(ids) == len(
set(ids)
), f"Duplicated id found. Use unique id for each panel. {ids}"
serve_deployment_dashboard_config = DashboardConfig(
name="SERVE_DEPLOYMENT",
default_uid="rayServeDeploymentDashboard",
panels=SERVE_DEPLOYMENT_GRAFANA_PANELS,
standard_global_filters=[
'application=~"$Application"',
'deployment=~"$Deployment"',
'replica=~"$Replica"',
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_deployment_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/serve_deployment_grafana_dashboard_base.json
+208
View File
@@ -0,0 +1,208 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries to specific prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Application",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Deployment",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Replica",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Route",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve Deployment Dashboard",
"uid": "rayServeDeploymentDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/serve_grafana_dashboard_base.json
+177
View File
@@ -0,0 +1,177 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Application",
"options": [],
"query": {
"query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "HTTP Route",
"multi": true,
"name": "HTTP_Route",
"options": [],
"query": {
"query": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
},
"datasource": "${datasource}",
"definition": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": "gRPC Service Method",
"multi": true,
"name": "gRPC_Method",
"options": [],
"query": {
"query": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
"refId": "Prometheus-Instance-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve Dashboard",
"uid": "rayServeDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py
+810
View File
@@ -0,0 +1,810 @@
# ruff: noqa: E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
GridPos,
Panel,
PanelTemplate,
Row,
Target,
)
# ---------------------------------------------------------------------------
# Reusable PromQL fragments
# ---------------------------------------------------------------------------
# WorkerId join: attaches deployment + replica labels to vLLM-only metrics.
# The `* 0 + 1` trick turns the counter into a constant-1 lookup table keyed
# by WorkerId so the join resolves deployment + replica labels without
# affecting the numeric value of the left-hand side.
# Keep `{global_filters}` trailing so an empty substitution leaves a tolerated
# trailing comma instead of a leading/double comma that Prometheus rejects.
_WORKER_JOIN = (
"\n* on(WorkerId) group_left(deployment, replica)"
"\nmax by(WorkerId, deployment, replica) ("
'ray_serve_deployment_request_counter_total{{deployment=~"$deployment", {global_filters}}} * 0 + 1)'
)
# Standard vLLM metric filter
_VLLM_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", {global_filters}'
# vLLM filter scoped to a specific deployment (used for ray_serve_* metrics
# that also carry model_name / WorkerId labels).
_VLLM_DEPLOYMENT_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", deployment=~"$deployment", {global_filters}'
# Legend used by most per-worker panels
_DEP_REPLICA = "{{deployment}}: {{replica}}"
def _mean_with_join(metric_base: str) -> str:
"""Mean = sum(_sum) / sum(_count) with NaN guard + WorkerId join."""
return (
"(\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_sum{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" /\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
def _percentile_with_join(metric_base: str, quantile: float) -> str:
"""histogram_quantile with NaN guard + WorkerId join."""
return (
"(\n"
" histogram_quantile(\n"
f" {quantile},\n"
f" sum by (le, WorkerId) (rate({metric_base}_bucket{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
def _gauge_with_join(metric: str) -> str:
"""Simple gauge metric with WorkerId join (no rate, no guard)."""
return f"sum by(WorkerId) ({metric}{{{{{_VLLM_FILTER}}}}})" + _WORKER_JOIN
def _rate_with_join(metric: str, agg_fn: str = "rate") -> str:
"""rate() or increase() of a metric summed by WorkerId, with join."""
return (
f"sum by(WorkerId) ({agg_fn}({metric}{{{{{_VLLM_FILTER}}}}}[$interval]))"
+ _WORKER_JOIN
)
def _ratio_with_join_and_guard(
numerator_metric: str,
denominator_metric: str,
*,
scale: str = "",
guard_metric: str | None = None,
) -> str:
"""Ratio of two rate metrics with NaN guard + WorkerId join.
Optionally applies a scale factor (e.g. '* 1000' or '/ 1024 / 1024 / 1024').
"""
guard = guard_metric or denominator_metric
return (
"(\n"
" (\n"
f" sum by(WorkerId) (rate({numerator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
" /\n"
f" sum by(WorkerId) (rate({denominator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+ (f" {scale}\n" if scale else "")
+ " )\n"
" and on(WorkerId)\n"
" (\n"
f" sum by(WorkerId) (rate({guard}{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
" )\n"
")" + _WORKER_JOIN
)
# ---------------------------------------------------------------------------
# Histogram helper: generates Mean / P50 / P90 panels for a given metric
# ---------------------------------------------------------------------------
def _histogram_panels(
metric_base: str,
label: str,
ids: tuple,
y: int,
unit: str = "s",
linewidth: int = 2,
description: str = "",
) -> list:
"""Return [Mean, P50, P90] panels for a histogram metric."""
return [
Panel(
id=ids[0],
title=f"{label} -- Mean",
description=description,
unit=unit,
targets=[Target(expr=_mean_with_join(metric_base), legend=_DEP_REPLICA)],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(0, y, 8, 8),
),
Panel(
id=ids[1],
title=f"{label} -- P50",
description=description,
unit=unit,
targets=[
Target(
expr=_percentile_with_join(metric_base, 0.5), legend=_DEP_REPLICA
)
],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(8, y, 8, 8),
),
Panel(
id=ids[2],
title=f"{label} -- P90",
description=description,
unit=unit,
targets=[
Target(
expr=_percentile_with_join(metric_base, 0.9), legend=_DEP_REPLICA
)
],
fill=1,
linewidth=linewidth,
stack=False,
grid_pos=GridPos(16, y, 8, 8),
),
]
# ===================================================================
# Row 1: Throughput
# ===================================================================
_throughput_panels = [
Panel(
id=2,
title="Requests / s",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (deployment, replica) (rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
legend="Total QPS",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 1, 8, 8),
),
Panel(
id=3,
title="Prompt Tokens/s",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_prompt_tokens_sum"),
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_vllm_request_prompt_tokens_sum{{{{{_VLLM_FILTER}}}}}[$interval]))",
legend="Total",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(8, 1, 8, 8),
),
Panel(
id=4,
title="Generation Tokens/s",
description="Number of tokens processed per second",
unit="tokens/s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_generation_tokens_total"),
legend=_DEP_REPLICA,
),
Target(
expr=f"sum(rate(ray_vllm_generation_tokens_total{{{{{_VLLM_FILTER}}}}}[$interval]))",
legend="Total",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(16, 1, 8, 8),
),
]
# ===================================================================
# Row 2: Latency (3x3 grid)
# ===================================================================
_latency_panels_list = [
*_histogram_panels(
"ray_vllm_request_time_per_output_token_seconds", "TPOT", (6, 7, 8), 10
),
*_histogram_panels("ray_vllm_time_to_first_token_seconds", "TTFT", (9, 10, 11), 18),
*_histogram_panels(
"ray_vllm_e2e_request_latency_seconds",
"Request Latency",
(12, 13, 14),
26,
description="Latency from request start to first token returned (in seconds).",
),
]
# ===================================================================
# Row 3: Cache
# ===================================================================
_cache_panels = [
Panel(
id=16,
title="Cache Utilization",
description="Percentage of used cache blocks by vLLM.",
unit="percentunit",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_kv_cache_usage_perc"),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 35, 12, 8),
),
Panel(
id=17,
title="GPU KV Cache Hit Rate",
description="",
unit="percent",
targets=[
Target(
expr=(
f"100 * ("
f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{{{{_VLLM_FILTER}}}}}[$interval])) "
f"/ sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])))"
f" and on(WorkerId) "
f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])) > 0))"
+ _WORKER_JOIN
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 35, 12, 8),
),
]
# ===================================================================
# Row 4: Request Length
# ===================================================================
_request_length_panels = [
*_histogram_panels(
"ray_vllm_request_prompt_tokens",
"Prompt Length",
(19, 20, 21),
44,
unit="short",
linewidth=1,
),
*_histogram_panels(
"ray_vllm_request_generation_tokens",
"Generation Length",
(22, 23, 24),
52,
unit="short",
linewidth=1,
),
]
# ===================================================================
# Row 5: Scheduler
# ===================================================================
_scheduler_panels = [
Panel(
id=26,
title="Scheduler: Running",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_running"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 61, 8, 8),
),
Panel(
id=27,
title="Scheduler: Swapped",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_swapped"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 61, 8, 8),
),
Panel(
id=28,
title="Scheduler: Waiting",
description="",
unit="short",
targets=[
Target(
expr=_gauge_with_join("ray_vllm_num_requests_waiting"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 61, 8, 8),
),
Panel(
id=29,
title="Finish Reason",
description="Number of finished requests by their finish reason.",
unit="short",
targets=[
Target(
expr=(
f"sum by(finished_reason, WorkerId) (increase(ray_vllm_request_success_total{{{{{_VLLM_FILTER}}}}}[$interval]))"
+ _WORKER_JOIN
),
legend="{{finished_reason}} \u2014 {{deployment}}: {{replica}}",
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 69, 12, 8),
),
Panel(
id=30,
title="Queue Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_queue_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 69, 12, 8),
),
Panel(
id=31,
title="Prefill Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_prefill_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 77, 12, 8),
),
Panel(
id=32,
title="Decode Time",
description="",
unit="s",
targets=[
Target(
expr=_rate_with_join("ray_vllm_request_decode_time_seconds_sum"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(12, 77, 12, 8),
),
]
# ===================================================================
# Row 6: NIXL
# ===================================================================
_nixl_panels = [
Panel(
id=34,
title="NIXL: Transfer Latency",
description="Average NIXL KV cache transfer latency in milliseconds.",
unit="ms",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_xfer_time_seconds_sum",
"ray_vllm_nixl_xfer_time_seconds_count",
scale="* 1000",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 86, 8, 8),
),
Panel(
id=35,
title="NIXL: Transfer Throughput",
description="NIXL KV cache transfer throughput in GB/s.",
unit="GBs",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_bytes_transferred_sum",
"ray_vllm_nixl_xfer_time_seconds_sum",
scale="/ 1024 / 1024 / 1024",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 86, 8, 8),
),
Panel(
id=36,
title="NIXL: Transfer Rate",
description="Number of NIXL KV cache transfers per second.",
unit="ops",
targets=[
Target(
expr=_rate_with_join("ray_vllm_nixl_xfer_time_seconds_count"),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 86, 8, 8),
),
Panel(
id=37,
title="NIXL: Avg Post Time",
description="Average time to post/initiate a NIXL transfer in milliseconds.",
unit="ms",
targets=[
Target(
expr=_ratio_with_join_and_guard(
"ray_vllm_nixl_post_time_seconds_sum",
"ray_vllm_nixl_post_time_seconds_count",
scale="* 1000",
),
legend=_DEP_REPLICA,
),
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(0, 94, 8, 8),
),
Panel(
id=38,
title="NIXL: KV Transfer Failures",
description="Number of failed NIXL KV cache transfers. Any non-zero value is concerning and indicates RDMA transfer errors.",
unit="short",
targets=[
Target(
expr=_rate_with_join(
"ray_vllm_nixl_num_failed_transfers", agg_fn="increase"
),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(8, 94, 8, 8),
),
Panel(
id=39,
title="NIXL: KV Expired Requests",
description="Number of requests whose KV blocks expired before decode consumed them. Spikes indicate prefill is outrunning decode or the timeout is too short.",
unit="short",
targets=[
Target(
expr=_rate_with_join(
"ray_vllm_nixl_num_kv_expired_reqs_total", agg_fn="increase"
),
legend=_DEP_REPLICA,
)
],
fill=1,
linewidth=1,
stack=False,
grid_pos=GridPos(16, 94, 8, 8),
),
]
# ===================================================================
# Row 7: Token Distribution (collapsed)
# ===================================================================
_WORKERID_FILTER = 'WorkerId=~"$workerid", {global_filters}'
_token_distribution_panels = [
Panel(
id=41,
title="Tokens Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
legend="Input: {{model_name}}",
),
Target(
expr=f"(sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
legend="Generated: {{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 103, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=42,
title="Tokens Last Hour",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
legend="Input: {{model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
legend="Generated: {{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 103, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=43,
title="Ratio Input:Generated Tokens Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) / sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 111, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=44,
title="Distribution of Requests Per Model Last 24 Hours",
description="",
unit="Requests",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 111, 12, 8),
template=PanelTemplate.PIE_CHART,
),
Panel(
id=45,
title="Peak Tokens Per Second Per Model Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"max_over_time(sum by (model_name) (rate(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[2m]))[24h:1m])",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 119, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=46,
title="Tokens Per Model Last 24 Hours",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) + sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
legend="{{model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 119, 12, 8),
template=PanelTemplate.STAT,
),
Panel(
id=47,
title="Avg Total Tokens Per Request Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=(
f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) +\n"
f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
f" / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
),
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 127, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=48,
title="Requests Per Model Last Week",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 127, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=49,
title="Tokens Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="In: {{ model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="Out: {{ model_name }}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 135, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=50,
title="Avg Total Tokens Per Request Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=(
f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) "
f"+ sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
f"/ sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
),
legend="{{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(12, 135, 12, 8),
template=PanelTemplate.GAUGE,
),
Panel(
id=51,
title="Tokens Per Request Per Model Last 7 Days",
description="",
unit="short",
targets=[
Target(
expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="In: {{ model_name}}",
),
Target(
expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
legend="Out: {{ model_name}}",
),
],
fill=1,
linewidth=2,
stack=False,
grid_pos=GridPos(0, 143, 12, 8),
template=PanelTemplate.GAUGE,
),
]
# ===================================================================
# Assemble rows and config
# ===================================================================
_ALL_ROWS = [
Row(title="Throughput", id=501, panels=_throughput_panels),
Row(title="Latency", id=502, panels=_latency_panels_list),
Row(title="Cache", id=503, panels=_cache_panels),
Row(title="Request Length", id=504, panels=_request_length_panels),
Row(title="Scheduler", id=505, panels=_scheduler_panels),
Row(title="NIXL", id=506, panels=_nixl_panels),
Row(
title="Token Distribution",
id=507,
collapsed=True,
panels=_token_distribution_panels,
),
]
# Validate uniqueness of panel IDs across all rows
_all_ids = sorted(panel.id for row in _ALL_ROWS for panel in row.panels)
assert len(_all_ids) == len(
set(_all_ids)
), f"Duplicated id found. Use unique id for each panel. {_all_ids}"
serve_llm_dashboard_config = DashboardConfig(
name="SERVE_LLM",
default_uid="rayServeLlmDashboard",
standard_global_filters=[
'ray_io_cluster=~"$Cluster"',
],
base_json_file_name="serve_llm_grafana_dashboard_base.json",
rows=_ALL_ROWS,
)
python/ray/dashboard/modules/metrics/dashboards/serve_llm_grafana_dashboard_base.json
+199
View File
@@ -0,0 +1,199 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1667344411089,
"links": [],
"panels": [],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"selected": false
},
"description": "Filter queries of a specific Prometheus type.",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"name": "vllm_model_name",
"label": "vLLM Model Name",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "workerid",
"label": "Worker ID",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"query": {
"query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": false,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "deployment",
"label": "Deployment",
"type": "query",
"hide": 0,
"datasource": "${datasource}",
"definition": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
"query": {
"query": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"allValue": ".*",
"current": {
"selected": false
},
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Cluster",
"options": [],
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"name": "interval",
"label": "Interval",
"type": "custom",
"hide": 0,
"includeAll": false,
"multi": false,
"options": [
{
"selected": true,
"text": "30s",
"value": "30s"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
}
],
"current": {
"selected": true,
"text": "5m",
"value": "5m"
}
}
]
},
"rayMeta": ["excludesSystemRoutes"],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Serve LLM Dashboard",
"uid": "rayServeLlmDashboard",
"version": 1
}
python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py
+346
View File
@@ -0,0 +1,346 @@
# flake8: noqa E501
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
Panel,
Row,
Target,
)
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL = Panel(
id=1,
title="Controller State",
description="Current state of the Ray Train controller.",
unit="",
targets=[
Target(
expr='sum(ray_train_controller_state{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name, ray_train_controller_state)',
legend="Run Name: {{ray_train_run_name}}, Controller State: {{ray_train_controller_state}}",
),
],
)
CONTROLLER_OPERATION_TIME_PANEL = Panel(
id=2,
title="Cumulative Worker Group Start/Shutdown Time",
description="Cumulative time the controller spends starting and shutting down worker groups (re-created on worker failures and resizes).",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_worker_group_start_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
legend="Run Name: {{ray_train_run_name}}, Worker Group Start Time",
),
Target(
expr='sum(ray_train_worker_group_shutdown_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
legend="Run Name: {{ray_train_run_name}}, Worker Group Shutdown Time",
),
],
fill=0,
stack=False,
)
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL = Panel(
id=3,
title="Cumulative Time in ray.train.report",
description="Cumulative time workers spend blocked inside `ray.train.report()`. This includes the cross-rank checkpoint directory sync barrier, the checkpoint file transfer to storage, and the time waiting for the report queue ordering. See the Checkpoint Sync and Checkpoint Transfer panels for a breakdown.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_report_total_blocked_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
WORKER_CHECKPOINT_SYNC_TIME_PANEL = Panel(
id=16,
title="Cumulative Checkpoint Sync Time",
description="Cumulative time spent in the cross-rank barrier that synchronizes the checkpoint directory name across all workers. High values indicate workers are spending significant time waiting for each other to reach the synchronization point.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_checkpoint_sync_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL = Panel(
id=17,
title="Cumulative Checkpoint Transfer Time",
description="Cumulative time spent transferring checkpoint files to storage. High values indicate slow storage throughput or large checkpoint sizes.",
unit="seconds",
targets=[
Target(
expr='sum(ray_train_checkpoint_transfer_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
)
],
fill=0,
stack=False,
)
# Core System Resources
CPU_UTILIZATION_PANEL = Panel(
id=4,
title="CPU Usage",
description="CPU core utilization across all workers.",
unit="cores",
targets=[
Target(
expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType)',
legend="CPU Usage: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
MEMORY_UTILIZATION_PANEL = Panel(
id=5,
title="Total Memory Usage",
description="Total physical memory used vs total available memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Memory Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
MEMORY_DETAILED_PANEL = Panel(
id=6,
title="Memory Allocation Details",
description="Memory allocation details including available and shared memory.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_mem_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Available Memory: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Shared Memory: {{instance}} ({{RayNodeType}})",
),
],
)
# GPU Resources
# TODO: Add GPU Device/Index as a filter.
GPU_UTILIZATION_PANEL = Panel(
id=7,
title="GPU Usage",
description="GPU utilization across all workers.",
unit="GPUs",
targets=[
Target(
expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})',
legend="MAX",
),
],
)
GPU_MEMORY_UTILIZATION_PANEL = Panel(
id=8,
title="GPU Memory Usage",
description="GPU memory usage across all workers.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
),
Target(
expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})) * 1024 * 1024',
legend="MAX",
),
],
)
# Storage Resources
DISK_UTILIZATION_PANEL = Panel(
id=9,
title="Disk Space Usage",
description="Disk space usage across all workers.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Disk Used: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
legend="MAX",
),
],
)
DISK_THROUGHPUT_PANEL = Panel(
id=10,
title="Disk Throughput",
description="Current disk read/write throughput.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read Speed: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write Speed: {{instance}} ({{RayNodeType}})",
),
],
)
DISK_OPERATIONS_PANEL = Panel(
id=11,
title="Disk Operations",
description="Current disk read/write operations per second.",
unit="ops/s",
targets=[
Target(
expr='sum(ray_node_disk_read_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Read IOPS: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_disk_write_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Write IOPS: {{instance}} ({{RayNodeType}})",
),
],
)
# Network Resources
NETWORK_THROUGHPUT_PANEL = Panel(
id=12,
title="Network Throughput",
description="Current network send/receive throughput.",
unit="Bps",
targets=[
Target(
expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Receive Speed: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Send Speed: {{instance}} ({{RayNodeType}})",
),
],
)
NETWORK_TOTAL_PANEL = Panel(
id=13,
title="Network Total Traffic",
description="Total network traffic sent/received.",
unit="bytes",
targets=[
Target(
expr='sum(ray_node_network_sent{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Total Sent: {{instance}} ({{RayNodeType}})",
),
Target(
expr='sum(ray_node_network_received{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
legend="Total Received: {{instance}} ({{RayNodeType}})",
),
],
)
TRAIN_GRAFANA_PANELS = []
TRAIN_GRAFANA_ROWS = [
# Train Metrics Row
Row(
title="Train Metrics",
id=14,
panels=[
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL,
CONTROLLER_OPERATION_TIME_PANEL,
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
WORKER_CHECKPOINT_SYNC_TIME_PANEL,
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
],
collapsed=False,
),
# System Resources Row
Row(
title="Resource Utilization",
id=15,
panels=[
CPU_UTILIZATION_PANEL,
MEMORY_UTILIZATION_PANEL,
MEMORY_DETAILED_PANEL,
# GPU Resources
GPU_UTILIZATION_PANEL,
GPU_MEMORY_UTILIZATION_PANEL,
# Storage Resources
DISK_UTILIZATION_PANEL,
DISK_THROUGHPUT_PANEL,
DISK_OPERATIONS_PANEL,
# Network Resources
NETWORK_THROUGHPUT_PANEL,
NETWORK_TOTAL_PANEL,
],
collapsed=True,
),
]
TRAIN_RUN_PANELS = [
# Ray Train Metrics (Controller)
CONTROLLER_STATE_PANEL,
CONTROLLER_OPERATION_TIME_PANEL,
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
]
TRAIN_WORKER_PANELS = [
# Ray Train Metrics (Worker)
WORKER_TRAIN_REPORT_TIME_PANEL,
WORKER_CHECKPOINT_SYNC_TIME_PANEL,
WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
# Core System Resources
CPU_UTILIZATION_PANEL,
MEMORY_UTILIZATION_PANEL,
# GPU Resources
GPU_UTILIZATION_PANEL,
GPU_MEMORY_UTILIZATION_PANEL,
# Storage Resources
DISK_UTILIZATION_PANEL,
# Network Resources
NETWORK_THROUGHPUT_PANEL,
]
# Get all panel IDs from both top-level panels and panels within rows
all_panel_ids = [panel.id for panel in TRAIN_GRAFANA_PANELS]
for row in TRAIN_GRAFANA_ROWS:
all_panel_ids.append(row.id)
all_panel_ids.extend(panel.id for panel in row.panels)
all_panel_ids.sort()
assert len(all_panel_ids) == len(
set(all_panel_ids)
), f"Duplicated id found. Use unique id for each panel. {all_panel_ids}"
train_dashboard_config = DashboardConfig(
name="TRAIN",
default_uid="rayTrainDashboard",
rows=TRAIN_GRAFANA_ROWS,
standard_global_filters=['SessionName=~"$SessionName"'],
base_json_file_name="train_grafana_dashboard_base.json",
)
python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json
+267
View File
@@ -0,0 +1,267 @@
{
"title": "Train Dashboard",
"uid": "rayTrainDashboard",
"version": 1,
"schemaVersion": 27,
"style": "dark",
"editable": true,
"graphTooltip": 1,
"refresh": false,
"panels": [],
"time": {
"from": "now-30m",
"to": "now"
},
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"description": "Filter queries of a specific Prometheus type.",
"datasource": null,
"query": "prometheus",
"refresh": 1,
"hide": 2,
"includeAll": false,
"multi": false,
"current": {
"selected": false
}
},
{
"name": "SessionName",
"type": "query",
"description": "Filter queries to specific ray sessions.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainRunName",
"type": "query",
"description": "Filter queries to specific Ray Train run names.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainRunId",
"type": "query",
"description": "Filter queries to specific Ray Train run ids.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
"query": {
"query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainWorkerWorldRank",
"type": "query",
"description": "Filter queries to specific Ray Train worker world ranks.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
"query": {
"query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 0,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "TrainWorkerActorId",
"type": "query",
"description": "Filter queries to specific Ray Train worker actor ids.",
"datasource": "${datasource}",
"definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
"query": {
"query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "Instance",
"type": "query",
"description": "Filter queries to specific node instances.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
"query": {
"query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": false,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "GpuIndex",
"type": "query",
"description": "Filter queries to specific GPU indices.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
"query": {
"query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"name": "GpuDeviceName",
"type": "query",
"description": "Filter queries to specific GPU device names.",
"datasource": "${datasource}",
"definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
"query": {
"query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"hide": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"sort": 2,
"current": {
"selected": true,
"text": ["All"],
"value": ["$__all"]
}
},
{
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
},
"description": "Filter queries to specific Ray node types (head or worker).",
"includeAll": true,
"multi": true,
"name": "RayNodeType",
"options": [
{
"selected": false,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "Head Node",
"value": "head"
},
{
"selected": false,
"text": "Worker Node",
"value": "worker"
}
],
"query": "head, worker",
"type": "custom"
}
]
}
}
python/ray/dashboard/modules/metrics/default_impl.py
+12
View File
@@ -0,0 +1,12 @@
from ray.dashboard.modules.metrics.dashboards.common import DashboardConfig
def get_serve_dashboard_config() -> DashboardConfig:
from ray.dashboard.modules.metrics.dashboards.serve_dashboard_panels import (
serve_dashboard_config,
)
return serve_dashboard_config
# Anyscale overrides
python/ray/dashboard/modules/metrics/export/prometheus/prometheus.yml
+12
View File
@@ -0,0 +1,12 @@
# my global config
global:
scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute.
evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
scrape_configs:
# Scrape from each Ray node as defined in the service_discovery.json provided by Ray.
- job_name: 'ray'
file_sd_configs:
- files:
- '/tmp/ray/prom_metrics_service_discovery.json'
python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py
+541
View File
@@ -0,0 +1,541 @@
import copy
import json
import math
import os
import re
from dataclasses import asdict
from typing import List, Tuple
import ray
from ray.dashboard.modules.metrics.dashboards.common import (
DashboardConfig,
Panel,
PanelTemplate,
)
from ray.dashboard.modules.metrics.dashboards.data_dashboard_panels import (
data_dashboard_config,
)
from ray.dashboard.modules.metrics.dashboards.data_llm_dashboard_panels import (
data_llm_dashboard_config,
)
from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import (
default_dashboard_config,
)
from ray.dashboard.modules.metrics.dashboards.serve_deployment_dashboard_panels import (
serve_deployment_dashboard_config,
)
from ray.dashboard.modules.metrics.dashboards.serve_llm_dashboard_panels import (
serve_llm_dashboard_config,
)
from ray.dashboard.modules.metrics.dashboards.train_dashboard_panels import (
train_dashboard_config,
)
from ray.dashboard.modules.metrics.default_impl import get_serve_dashboard_config
GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE = "RAY_GRAFANA_{name}_DASHBOARD_UID"
GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE = (
"RAY_GRAFANA_{name}_DASHBOARD_GLOBAL_FILTERS"
)
GRAFANA_DASHBOARD_LOG_LINK_URL_ENV_VAR_TEMPLATE = "RAY_GRAFANA_{name}_LOG_LINK_URL"
# Grafana dashboard layout constants
# Dashboard uses a 24-column grid with 2-column panels
ROW_WIDTH = 24 # Full dashboard width
PANELS_PER_ROW = 2
PANEL_WIDTH = ROW_WIDTH // PANELS_PER_ROW # Width of each panel
PANEL_HEIGHT = 8 # Height of each panel
ROW_HEIGHT = 1 # Height of row container
def _read_configs_for_dashboard(
dashboard_config: DashboardConfig,
) -> Tuple[str, List[str], str]:
"""Reads environment variable configs for overriding uid, global_filters, and the log link URL for a given dashboard.
Args:
dashboard_config: The dashboard whose env-var overrides are read.
``dashboard_config.name`` selects the env-var suffix and
``default_uid`` is used as a fallback.
Returns:
Tuple with format uid, global_filters, log_link_url
"""
uid = (
os.environ.get(
GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE.format(
name=dashboard_config.name
)
)
or dashboard_config.default_uid
)
global_filters_str = (
os.environ.get(
GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE.format(
name=dashboard_config.name
)
)
or ""
)
if global_filters_str == "":
global_filters = []
else:
global_filters = global_filters_str.split(",")
log_link_url = (
os.environ.get(
GRAFANA_DASHBOARD_LOG_LINK_URL_ENV_VAR_TEMPLATE.format(
name=dashboard_config.name
)
)
or ""
)
return uid, global_filters, log_link_url
def generate_default_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the default dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(default_dashboard_config)
def generate_serve_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the serve dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(get_serve_dashboard_config())
def generate_serve_deployment_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the serve dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(serve_deployment_dashboard_config)
def generate_serve_llm_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the serve dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(serve_llm_dashboard_config)
def generate_data_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the data dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(data_dashboard_config)
def generate_data_llm_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the Data LLM dashboard and returns
both the content and the uid.
This dashboard provides vLLM metrics visibility for Ray Data LLM workloads,
including latency (TTFT, TPOT), throughput, cache utilization, and
prefix cache hit rate.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(data_llm_dashboard_config)
def generate_train_grafana_dashboard() -> Tuple[str, str]:
"""
Generates the dashboard output for the train dashboard and returns
both the content and the uid.
Returns:
Tuple with format content, uid
"""
return _generate_grafana_dashboard(train_dashboard_config)
def _generate_grafana_dashboard(dashboard_config: DashboardConfig) -> str:
"""Render the Grafana dashboard JSON for the given config.
Args:
dashboard_config: Configuration describing the panels and base
template JSON file to use for rendering.
Returns:
Tuple with format dashboard_content, uid
"""
uid, global_filters, log_link_url = _read_configs_for_dashboard(dashboard_config)
panels = _generate_grafana_panels(dashboard_config, global_filters, log_link_url)
base_file_name = dashboard_config.base_json_file_name
base_json = json.load(
open(os.path.join(os.path.dirname(__file__), "dashboards", base_file_name))
)
base_json["panels"] = panels
# Update variables to use global_filters
global_filters_str = ",".join(global_filters)
variables = base_json.get("templating", {}).get("list", [])
for variable in variables:
if "definition" not in variable:
continue
definition = variable["definition"].format(global_filters=global_filters_str)
query = variable["query"]["query"].format(global_filters=global_filters_str)
if not global_filters_str:
definition = _clean_empty_filters(definition)
query = _clean_empty_filters(query)
variable["definition"] = definition
variable["query"]["query"] = query
tags = base_json.get("tags", []) or []
tags.append(f"rayVersion:{ray.__version__}")
base_json["tags"] = tags
base_json["uid"] = uid
# Ray metadata can be used to put arbitrary metadata
ray_meta = base_json.get("rayMeta", []) or []
ray_meta.append("supportsGlobalFilterOverride")
base_json["rayMeta"] = ray_meta
return json.dumps(base_json, indent=4), uid
def _generate_panel_template(
panel: Panel,
panel_global_filters: List[str],
panel_index: int,
base_y_position: int,
log_link_url: str,
) -> dict:
"""
Helper method to generate a panel template with common configuration.
Args:
panel: The panel configuration
panel_global_filters: List of global filters to apply
panel_index: The index of the panel within its row (0-based)
base_y_position: The base y-coordinate for the row in the dashboard grid
log_link_url: The URL to the log link for the panel
Returns:
dict: The configured panel template
"""
# Create base template from panel configuration
template = copy.deepcopy(panel.template.value)
template.update(
{
"title": panel.title,
"description": panel.description,
"id": panel.id,
"targets": _generate_targets(panel, panel_global_filters),
}
)
# Set panel position and dimensions
if panel.grid_pos:
template["gridPos"] = asdict(panel.grid_pos)
else:
# Calculate panel position in 2-column grid layout
# x: 0 or 12 (left or right column)
# y: base position + (row number * panel height)
row_number = panel_index // PANELS_PER_ROW
template["gridPos"] = {
"h": PANEL_HEIGHT,
"w": PANEL_WIDTH,
"x": PANEL_WIDTH * (panel_index % PANELS_PER_ROW),
"y": base_y_position + (row_number * PANEL_HEIGHT),
}
# Set unit format for legacy graph-style panels (GRAPH, HEATMAP, STAT, GAUGE, PIE_CHART, BAR_CHART)
if panel.template in (
PanelTemplate.GRAPH,
PanelTemplate.HEATMAP,
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.PIE_CHART,
PanelTemplate.BAR_CHART,
):
template["yaxes"][0]["format"] = panel.unit
# Set fieldConfig unit (for newer panel types with fieldConfig.defaults)
if panel.template in (
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.HEATMAP,
PanelTemplate.PIE_CHART,
PanelTemplate.BAR_CHART,
PanelTemplate.TABLE,
PanelTemplate.GRAPH,
):
template["fieldConfig"]["defaults"]["unit"] = panel.unit
# Set fill, stack, linewidth, nullPointMode (only for GRAPH panels)
if panel.template == PanelTemplate.GRAPH:
template["fill"] = panel.fill
template["stack"] = panel.stack
template["linewidth"] = panel.linewidth
if panel.stack is True:
template["nullPointMode"] = "connected"
if panel.hideXAxis:
template.setdefault("xaxis", {})["show"] = False
# Handle optional panel customization fields
# Thresholds (for panels with fieldConfig.defaults.thresholds)
if panel.thresholds is not None:
if panel.template in (PanelTemplate.STAT, PanelTemplate.GAUGE):
template["fieldConfig"]["defaults"]["thresholds"][
"steps"
] = panel.thresholds
# Value mappings (for panels with fieldConfig.defaults.mappings)
if panel.value_mappings is not None:
if panel.template in (
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.TABLE,
):
template["fieldConfig"]["defaults"]["mappings"] = panel.value_mappings
# Color mode (for STAT panels with options.colorMode)
if panel.color_mode is not None:
if panel.template == PanelTemplate.STAT:
template["options"]["colorMode"] = panel.color_mode
# Legend mode
if panel.legend_mode is not None:
if panel.template in (PanelTemplate.GRAPH, PanelTemplate.BAR_CHART):
# For graph panels (legacy format with top-level legend object)
template["legend"]["show"] = panel.legend_mode != "hidden"
template["legend"]["alignAsTable"] = panel.legend_mode == "table"
elif panel.template == PanelTemplate.PIE_CHART:
# For PIE_CHART (options.legend.displayMode)
template["options"]["legend"]["displayMode"] = panel.legend_mode
# Min/max values (for panels with fieldConfig.defaults)
if panel.min_val is not None or panel.max_val is not None:
if panel.template in (
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.HEATMAP,
PanelTemplate.PIE_CHART,
PanelTemplate.BAR_CHART,
PanelTemplate.TABLE,
PanelTemplate.GRAPH,
):
if panel.min_val is not None:
template["fieldConfig"]["defaults"]["min"] = panel.min_val
if panel.max_val is not None:
template["fieldConfig"]["defaults"]["max"] = panel.max_val
# Reduce calculation (for panels with options.reduceOptions)
if panel.reduce_calc is not None:
if panel.template in (
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.PIE_CHART,
):
template["options"]["reduceOptions"]["calcs"] = [panel.reduce_calc]
# Handle heatmap-specific options
if panel.heatmap_color_scheme is not None:
if panel.template == PanelTemplate.HEATMAP:
template["options"]["color"]["scheme"] = panel.heatmap_color_scheme
if panel.heatmap_color_reverse is not None:
if panel.template == PanelTemplate.HEATMAP:
template["options"]["color"]["reverse"] = panel.heatmap_color_reverse
if panel.heatmap_yaxis_label is not None:
if panel.template in (
PanelTemplate.GRAPH,
PanelTemplate.HEATMAP,
PanelTemplate.STAT,
PanelTemplate.GAUGE,
PanelTemplate.PIE_CHART,
PanelTemplate.BAR_CHART,
):
template["yaxes"][0]["label"] = panel.heatmap_yaxis_label
# Add log link if URL is provided via environment variable.
if log_link_url:
template["links"] = [
{
"targetBlank": True,
"title": "View Logs",
"url": log_link_url,
}
]
return template
def _create_row_panel(row: Panel, y_position: int) -> dict:
"""
Creates a Grafana row panel that spans the full dashboard width.
Row panels can be collapsed to hide their contained panels.
Args:
row: Row config with title, id, and collapse state
y_position: Vertical position in dashboard grid
Returns:
Grafana row panel configuration
"""
return {
"collapsed": row.collapsed,
"gridPos": {"h": ROW_HEIGHT, "w": ROW_WIDTH, "x": 0, "y": y_position},
"id": row.id,
"title": row.title,
"type": "row",
"panels": [],
}
def _calculate_panel_heights(num_panels: int) -> int:
"""
Calculate the total height needed for a set of panels.
Args:
num_panels: Number of panels to position
Returns:
Total height needed for the panels
"""
rows_needed = math.ceil(num_panels / PANELS_PER_ROW)
return rows_needed * PANEL_HEIGHT
def _generate_grafana_panels(
config: DashboardConfig, global_filters: List[str], log_link_url: str
) -> List[dict]:
"""
Generates Grafana panel configurations for a dashboard.
The dashboard layout follows these rules:
- Panels are arranged in 2 columns (12 units wide each)
- Each panel is 8 units high
- Rows are 1 unit high and can be collapsed
- Panels within rows follow the same 2-column layout
- Panel positions can be overridden via panel.grid_pos or auto-calculated
Args:
config: Dashboard configuration containing panels and rows
global_filters: List of filters to apply to all panels
log_link_url: Optional URL for panel log links. When set, each panel
gets a "View Logs" link pointing to this URL.
Returns:
List of Grafana panel configurations for the dashboard
"""
panels = []
panel_global_filters = [*config.standard_global_filters, *global_filters]
current_y_position = 0
# Add top-level panels in 2-column grid
for panel_index, panel in enumerate(config.panels):
panel_template = _generate_panel_template(
panel, panel_global_filters, panel_index, current_y_position, log_link_url
)
panels.append(panel_template)
# Calculate space needed for top-level panels
current_y_position += _calculate_panel_heights(len(config.panels))
# Add rows and their panels
if not config.rows:
return panels
for row in config.rows:
# Create and add row panel
row_panel = _create_row_panel(row, current_y_position)
panels.append(row_panel)
current_y_position += ROW_HEIGHT
# Add panels within row using 2-column grid
for panel_index, panel in enumerate(row.panels):
panel_template = _generate_panel_template(
panel,
panel_global_filters,
panel_index,
current_y_position,
log_link_url,
)
# Add panel to row if collapsed, otherwise to main dashboard
if row.collapsed:
row_panel["panels"].append(panel_template)
else:
panels.append(panel_template)
# Update y position for next row based on actual panel positions
# when explicit grid_pos is used, or fallback to calculated height.
if any(p.grid_pos for p in row.panels):
max_y_bottom = max(
(p.grid_pos.y + p.grid_pos.h for p in row.panels if p.grid_pos),
default=current_y_position,
)
current_y_position = max_y_bottom
else:
current_y_position += _calculate_panel_heights(len(row.panels))
return panels
def _clean_empty_filters(expr: str) -> str:
"""Clean up malformed PromQL when global_filters is empty.
Removes artifacts like trailing/leading commas in label matchers:
", ," → ","
", }" → "}"
"{ ," → "{"
"""
expr = re.sub(r",\s*,", ",", expr)
expr = re.sub(r",\s*}", "}", expr)
expr = re.sub(r"{\s*,", "{", expr)
return expr
def gen_incrementing_alphabets(length):
assert 65 + length < 96, "we only support up to 26 targets at a time."
# 65: ascii code of 'A'.
return list(map(chr, range(65, 65 + length)))
def _generate_targets(panel: Panel, panel_global_filters: List[str]) -> List[dict]:
targets = []
for target, ref_id in zip(
panel.targets, gen_incrementing_alphabets(len(panel.targets))
):
template = copy.deepcopy(target.template.value)
global_filters_str = ",".join(panel_global_filters)
expr = target.expr.format(global_filters=global_filters_str)
if not global_filters_str:
expr = _clean_empty_filters(expr)
template.update(
{
"expr": expr,
"legendFormat": target.legend,
"refId": ref_id,
}
)
targets.append(template)
return targets
python/ray/dashboard/modules/metrics/install_and_start_prometheus.py
+203
View File
@@ -0,0 +1,203 @@
import logging
import os
import platform
import subprocess
import sys
import tarfile
from pathlib import Path
import requests
from ray.dashboard.consts import PROMETHEUS_CONFIG_INPUT_PATH
FALLBACK_PROMETHEUS_VERSION = "2.48.1"
DOWNLOAD_BLOCK_SIZE = 8192 # 8 KB
TEST_MODE_ENV_VAR = "RAY_PROMETHEUS_DOWNLOAD_TEST_MODE"
def get_system_info():
os_type = platform.system().lower()
architecture = platform.machine()
if architecture == "x86_64":
# In the Prometheus filename, it's called amd64
architecture = "amd64"
elif architecture == "aarch64":
# In the Prometheus filename, it's called arm64
architecture = "arm64"
return os_type, architecture
def download_file(url, filename):
logging.info(f"Downloading {url} to {Path(filename).absolute()}...")
try:
test_mode = os.environ.get(TEST_MODE_ENV_VAR, False)
request_method = requests.head if test_mode else requests.get
response = request_method(url, stream=True)
response.raise_for_status()
total_size_in_bytes = int(response.headers.get("content-length", 0))
total_size_in_mb = total_size_in_bytes / (1024 * 1024)
downloaded_size_in_mb = 0
block_size = DOWNLOAD_BLOCK_SIZE
with open(filename, "wb") as file:
for chunk in response.iter_content(chunk_size=block_size):
file.write(chunk)
downloaded_size_in_mb += len(chunk) / (1024 * 1024)
print(
f"Downloaded: {downloaded_size_in_mb:.2f} MB / "
f"{total_size_in_mb:.2f} MB",
end="\r",
)
print("\nDownload completed.")
return True
except requests.RequestException as e:
logging.error(f"Error downloading file: {e}")
return False
def install_prometheus(file_path):
try:
with tarfile.open(file_path) as tar:
tar.extractall()
logging.info("Prometheus installed successfully.")
return True
except Exception as e:
logging.error(f"Error installing Prometheus: {e}")
return False
def start_prometheus(prometheus_dir):
# The function assumes the Ray cluster to be monitored by Prometheus uses the
# default configuration with "/tmp/ray" as the default root temporary directory.
#
# This is to support the `ray metrics launch-prometheus` command, when a Ray cluster
# hasn't started yet and the user doesn't have a way to get a `--temp-dir`
# anywhere. So we choose to use a hardcoded default value.
config_file = Path(PROMETHEUS_CONFIG_INPUT_PATH)
if not config_file.exists():
raise FileNotFoundError(f"Prometheus config file not found: {config_file}")
prometheus_cmd = [
f"{prometheus_dir}/prometheus",
"--config.file",
str(config_file),
"--web.enable-lifecycle",
]
try:
process = subprocess.Popen(prometheus_cmd)
logging.info("Prometheus has started.")
return process
except Exception as e:
logging.error(f"Failed to start Prometheus: {e}")
return None
def print_shutdown_message(process_id):
message = (
f"Prometheus is running with PID {process_id}.\n"
"To stop Prometheus, use the command: "
"`ray metrics shutdown-prometheus`, "
f"'kill {process_id}', or if you need to force stop, "
f"use 'kill -9 {process_id}'."
)
print(message)
debug_message = (
"To list all processes running Prometheus, use the command: "
"'ps aux | grep prometheus'."
)
print(debug_message)
def get_latest_prometheus_version():
url = "https://api.github.com/repos/prometheus/prometheus/releases/latest"
try:
response = requests.get(url)
response.raise_for_status()
data = response.json()
# Remove the leading 'v' from the version number
return data["tag_name"].lstrip("v")
except requests.RequestException as e:
logging.error(f"Error fetching latest Prometheus version: {e}")
return None
def get_prometheus_filename(os_type=None, architecture=None, prometheus_version=None):
if os_type is None or architecture is None:
os_type, architecture = get_system_info()
if prometheus_version is None:
prometheus_version = get_latest_prometheus_version()
if prometheus_version is None:
logging.warning(
"Failed to retrieve the latest Prometheus version. Falling "
f"back to {FALLBACK_PROMETHEUS_VERSION}."
)
# Fall back to a hardcoded version
prometheus_version = FALLBACK_PROMETHEUS_VERSION
return (
f"prometheus-{prometheus_version}.{os_type}-{architecture}.tar.gz",
prometheus_version,
)
def get_prometheus_download_url(
os_type=None, architecture=None, prometheus_version=None
):
file_name, prometheus_version = get_prometheus_filename(
os_type, architecture, prometheus_version
)
return (
"https://github.com/prometheus/prometheus/releases/"
f"download/v{prometheus_version}/{file_name}"
)
def download_prometheus(os_type=None, architecture=None, prometheus_version=None):
file_name, _ = get_prometheus_filename(os_type, architecture, prometheus_version)
download_url = get_prometheus_download_url(
os_type, architecture, prometheus_version
)
return download_file(download_url, file_name), file_name
def main():
# Configure logging only when this script is run directly
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logging.warning("This script is not intended for production use.")
downloaded, file_name = download_prometheus()
if not downloaded:
logging.error("Failed to download Prometheus.")
sys.exit(1)
# TODO: Verify the checksum of the downloaded file
if not install_prometheus(file_name):
logging.error("Installation failed.")
sys.exit(1)
# TODO: Add a check to see if Prometheus is already running
assert file_name.endswith(".tar.gz")
process = start_prometheus(
# remove the .tar.gz extension
prometheus_dir=file_name.rstrip(".tar.gz")
)
if process:
print_shutdown_message(process.pid)
if __name__ == "__main__":
main()
python/ray/dashboard/modules/metrics/metrics_head.py
+436
View File
@@ -0,0 +1,436 @@
import json
import logging
import os
import shutil
from urllib.parse import quote
import aiohttp
import ray.dashboard.optional_utils as dashboard_optional_utils
import ray.dashboard.utils as dashboard_utils
from ray._private.ray_constants import (
PROMETHEUS_SERVICE_DISCOVERY_FILE,
SESSION_LATEST,
)
from ray.dashboard.modules.metrics.grafana_dashboard_factory import (
generate_data_grafana_dashboard,
generate_data_llm_grafana_dashboard,
generate_default_grafana_dashboard,
generate_serve_deployment_grafana_dashboard,
generate_serve_grafana_dashboard,
generate_serve_llm_grafana_dashboard,
generate_train_grafana_dashboard,
)
from ray.dashboard.modules.metrics.templates import (
DASHBOARD_PROVISIONING_TEMPLATE,
GRAFANA_DATASOURCE_TEMPLATE,
GRAFANA_INI_TEMPLATE,
PROMETHEUS_YML_TEMPLATE,
)
from ray.dashboard.subprocesses.module import SubprocessModule
from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
METRICS_OUTPUT_ROOT_ENV_VAR = "RAY_METRICS_OUTPUT_ROOT"
DEFAULT_PROMETHEUS_HOST = "http://localhost:9090"
PROMETHEUS_HOST_ENV_VAR = "RAY_PROMETHEUS_HOST"
DEFAULT_PROMETHEUS_HEADERS = "{}"
PROMETHEUS_HEADERS_ENV_VAR = "RAY_PROMETHEUS_HEADERS"
DEFAULT_PROMETHEUS_NAME = "Prometheus"
PROMETHEUS_NAME_ENV_VAR = "RAY_PROMETHEUS_NAME"
PROMETHEUS_HEALTHCHECK_PATH = "-/healthy"
DEFAULT_GRAFANA_HOST = "http://localhost:3000"
GRAFANA_HOST_ENV_VAR = "RAY_GRAFANA_HOST"
GRAFANA_ORG_ID_ENV_VAR = "RAY_GRAFANA_ORG_ID"
DEFAULT_GRAFANA_ORG_ID = "1"
GRAFANA_CLUSTER_FILTER_ENV_VAR = "RAY_GRAFANA_CLUSTER_FILTER"
GRAFANA_HOST_DISABLED_VALUE = "DISABLED"
GRAFANA_IFRAME_HOST_ENV_VAR = "RAY_GRAFANA_IFRAME_HOST"
GRAFANA_DASHBOARD_OUTPUT_DIR_ENV_VAR = "RAY_METRICS_GRAFANA_DASHBOARD_OUTPUT_DIR"
GRAFANA_HEALTHCHECK_PATH = "api/health"
# parse_prom_headers will make sure the input is in one of the following formats:
# 1. {"H1": "V1", "H2": "V2"}
# 2. [["H1", "V1"], ["H2", "V2"], ["H2", "V3"]]
def parse_prom_headers(prometheus_headers):
parsed = json.loads(prometheus_headers)
if isinstance(parsed, dict):
if all(isinstance(k, str) and isinstance(v, str) for k, v in parsed.items()):
return parsed
if isinstance(parsed, list):
if all(len(e) == 2 and all(isinstance(v, str) for v in e) for e in parsed):
return parsed
raise ValueError(
f"{PROMETHEUS_HEADERS_ENV_VAR} should be a JSON string in one of the formats:\n"
+ "1) An object with string keys and string values.\n"
+ "2) an array of string arrays with 2 string elements each.\n"
+ 'For example, {"H1": "V1", "H2": "V2"} and\n'
+ '[["H1", "V1"], ["H2", "V2"], ["H2", "V3"]] are valid.'
)
class PrometheusQueryError(Exception):
def __init__(self, status, message):
self.message = (
f"Error fetching data from prometheus. status: {status}, message: {message}"
)
super().__init__(self.message)
class MetricsHead(SubprocessModule):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.grafana_host = os.environ.get(GRAFANA_HOST_ENV_VAR, DEFAULT_GRAFANA_HOST)
self.prometheus_host = os.environ.get(
PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST
)
default_metrics_root = os.path.join(self.session_dir, "metrics")
self.prometheus_headers = parse_prom_headers(
os.environ.get(
PROMETHEUS_HEADERS_ENV_VAR,
DEFAULT_PROMETHEUS_HEADERS,
)
)
session_latest_metrics_root = os.path.join(
self.temp_dir, SESSION_LATEST, "metrics"
)
self._metrics_root = os.environ.get(
METRICS_OUTPUT_ROOT_ENV_VAR, default_metrics_root
)
self._metrics_root_session_latest = os.environ.get(
METRICS_OUTPUT_ROOT_ENV_VAR, session_latest_metrics_root
)
self._grafana_config_output_path = os.path.join(self._metrics_root, "grafana")
self._grafana_session_latest_config_output_path = os.path.join(
self._metrics_root_session_latest, "grafana"
)
self._grafana_dashboard_output_dir = os.environ.get(
GRAFANA_DASHBOARD_OUTPUT_DIR_ENV_VAR,
os.path.join(self._grafana_config_output_path, "dashboards"),
)
self._prometheus_name = os.environ.get(
PROMETHEUS_NAME_ENV_VAR, DEFAULT_PROMETHEUS_NAME
)
self._grafana_org_id = os.environ.get(
GRAFANA_ORG_ID_ENV_VAR, DEFAULT_GRAFANA_ORG_ID
)
self._grafana_cluster_filter = os.environ.get(GRAFANA_CLUSTER_FILTER_ENV_VAR)
# To be set later when dashboards gets generated
self._dashboard_uids = {}
@routes.get("/api/grafana_health")
async def grafana_health(self, req) -> aiohttp.web.Response:
"""
Endpoint that checks if Grafana is running
"""
# If disabled, we don't want to show the metrics tab at all.
if self.grafana_host == GRAFANA_HOST_DISABLED_VALUE:
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.OK,
message="Grafana disabled",
grafana_host=GRAFANA_HOST_DISABLED_VALUE,
)
grafana_iframe_host = os.environ.get(
GRAFANA_IFRAME_HOST_ENV_VAR, self.grafana_host
)
path = f"{self.grafana_host}/{GRAFANA_HEALTHCHECK_PATH}"
try:
async with self.http_session.get(path) as resp:
if resp.status != 200:
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
message="Grafana healthcheck failed",
status=resp.status,
)
json = await resp.json()
# Check if the required Grafana services are running.
if json["database"] != "ok":
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
message="Grafana healthcheck failed. Database not ok.",
status=resp.status,
json=json,
)
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.OK,
message="Grafana running",
grafana_host=grafana_iframe_host,
grafana_org_id=self._grafana_org_id,
session_name=self.session_name,
dashboard_uids=self._dashboard_uids,
dashboard_datasource=self._prometheus_name,
grafana_cluster_filter=self._grafana_cluster_filter,
)
except Exception as e:
logger.debug(
"Error fetching grafana endpoint. Is grafana running?", exc_info=e
)
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
message="Grafana healthcheck failed",
exception=str(e),
)
@routes.get("/api/prometheus_health")
async def prometheus_health(self, req):
try:
path = f"{self.prometheus_host}/{PROMETHEUS_HEALTHCHECK_PATH}"
async with self.http_session.get(
path, headers=self.prometheus_headers
) as resp:
if resp.status != 200:
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
message="prometheus healthcheck failed.",
status=resp.status,
)
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.OK,
message="prometheus running",
)
except Exception as e:
logger.debug(
"Error fetching prometheus endpoint. Is prometheus running?", exc_info=e
)
return dashboard_optional_utils.rest_response(
status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
message="prometheus healthcheck failed.",
reason=str(e),
)
def _create_default_grafana_configs(self):
"""
Creates the Grafana configurations that are by default provided by Ray.
"""
# Create Grafana configuration folder
if os.path.exists(self._grafana_config_output_path):
shutil.rmtree(self._grafana_config_output_path)
os.makedirs(self._grafana_config_output_path, exist_ok=True)
# Overwrite Grafana's configuration file
grafana_provisioning_folder = os.path.join(
self._grafana_config_output_path, "provisioning"
)
grafana_prov_folder_with_latest_session = os.path.join(
self._grafana_session_latest_config_output_path, "provisioning"
)
with open(
os.path.join(
self._grafana_config_output_path,
"grafana.ini",
),
"w",
) as f:
f.write(
GRAFANA_INI_TEMPLATE.format(
grafana_provisioning_folder=grafana_prov_folder_with_latest_session
)
)
# Overwrite Grafana's dashboard provisioning directory based on env var
dashboard_provisioning_path = os.path.join(
grafana_provisioning_folder, "dashboards"
)
os.makedirs(
dashboard_provisioning_path,
exist_ok=True,
)
with open(
os.path.join(
dashboard_provisioning_path,
"default.yml",
),
"w",
) as f:
f.write(
DASHBOARD_PROVISIONING_TEMPLATE.format(
dashboard_output_folder=self._grafana_dashboard_output_dir
)
)
# Overwrite Grafana's Prometheus datasource based on env var
prometheus_host = os.environ.get(
PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST
)
prometheus_headers = parse_prom_headers(
os.environ.get(PROMETHEUS_HEADERS_ENV_VAR, DEFAULT_PROMETHEUS_HEADERS)
)
# parse_prom_headers will make sure the prometheus_headers is either format of:
# 1. {"H1": "V1", "H2": "V2"} or
# 2. [["H1", "V1"], ["H2", "V2"], ["H2", "V3"]]
prometheus_header_pairs = []
if isinstance(prometheus_headers, list):
prometheus_header_pairs = prometheus_headers
elif isinstance(prometheus_headers, dict):
prometheus_header_pairs = list(prometheus_headers.items())
data_sources_path = os.path.join(grafana_provisioning_folder, "datasources")
os.makedirs(
data_sources_path,
exist_ok=True,
)
os.makedirs(
self._grafana_dashboard_output_dir,
exist_ok=True,
)
with open(
os.path.join(
data_sources_path,
"default.yml",
),
"w",
) as f:
f.write(
GRAFANA_DATASOURCE_TEMPLATE(
prometheus_host=prometheus_host,
prometheus_name=self._prometheus_name,
jsonData={
f"httpHeaderName{i + 1}": header
for i, (header, _) in enumerate(prometheus_header_pairs)
},
secureJsonData={
f"httpHeaderValue{i + 1}": value
for i, (_, value) in enumerate(prometheus_header_pairs)
},
)
)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"default_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["default"],
) = generate_default_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"serve_grafana_dashboard.json",
),
"w",
) as f:
content, self._dashboard_uids["serve"] = generate_serve_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"serve_deployment_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["serve_deployment"],
) = generate_serve_deployment_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"serve_llm_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["serve_llm"],
) = generate_serve_llm_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"data_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["data"],
) = generate_data_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"data_llm_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["data_llm"],
) = generate_data_llm_grafana_dashboard()
f.write(content)
with open(
os.path.join(
self._grafana_dashboard_output_dir,
"train_grafana_dashboard.json",
),
"w",
) as f:
(
content,
self._dashboard_uids["train"],
) = generate_train_grafana_dashboard()
f.write(content)
def _create_default_prometheus_configs(self):
"""
Creates the Prometheus configurations that are by default provided by Ray.
"""
prometheus_config_output_path = os.path.join(
self._metrics_root, "prometheus", "prometheus.yml"
)
# Generate the default Prometheus configurations
if os.path.exists(prometheus_config_output_path):
os.remove(prometheus_config_output_path)
os.makedirs(os.path.dirname(prometheus_config_output_path), exist_ok=True)
# This code generates the Prometheus config based on the custom temporary root
# path set by the user at Ray cluster start up (via --temp-dir). In contrast,
# start_prometheus in install_and_start_prometheus.py uses a hardcoded
# Prometheus config at PROMETHEUS_CONFIG_INPUT_PATH that always uses "/tmp/ray".
# Other than the root path, the config file generated here is identical to that
# hardcoded config file.
prom_discovery_file_path = os.path.join(
self.session_dir, PROMETHEUS_SERVICE_DISCOVERY_FILE
)
with open(prometheus_config_output_path, "w") as f:
f.write(
PROMETHEUS_YML_TEMPLATE.format(
prom_metrics_service_discovery_file_path=prom_discovery_file_path
)
)
async def run(self):
await super().run()
self._create_default_grafana_configs()
self._create_default_prometheus_configs()
async def _query_prometheus(self, query):
async with self.http_session.get(
f"{self.prometheus_host}/api/v1/query?query={quote(query)}",
headers=self.prometheus_headers,
) as resp:
if resp.status == 200:
prom_data = await resp.json()
return prom_data
message = await resp.text()
raise PrometheusQueryError(resp.status, message)
python/ray/dashboard/modules/metrics/templates.py
+63
View File
@@ -0,0 +1,63 @@
import yaml
GRAFANA_INI_TEMPLATE = """
[security]
allow_embedding = true
[auth.anonymous]
enabled = true
org_name = Main Org.
org_role = Viewer
[paths]
provisioning = {grafana_provisioning_folder}
"""
DASHBOARD_PROVISIONING_TEMPLATE = """
apiVersion: 1
providers:
- name: Ray # Default dashboards provided by OSS Ray
folder: Ray
type: file
options:
path: {dashboard_output_folder}
"""
def GRAFANA_DATASOURCE_TEMPLATE(
prometheus_name, prometheus_host, jsonData, secureJsonData
):
return yaml.safe_dump(
{
"apiVersion": 1,
"datasources": [
{
"name": prometheus_name,
"url": prometheus_host,
"type": "prometheus",
"isDefault": True,
"access": "proxy",
"jsonData": jsonData,
"secureJsonData": secureJsonData,
}
],
}
)
PROMETHEUS_YML_TEMPLATE = """# my global config
global:
scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every \
1 minute.
evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 \
minute.
# scrape_timeout is set to the global default (10s).
scrape_configs:
# Scrape from each Ray node as defined in the service_discovery.json provided by Ray.
- job_name: 'ray'
file_sd_configs:
- files:
- '{prom_metrics_service_discovery_file_path}'
"""
python/ray/dashboard/modules/metrics/tests/test_utils.py
+32
View File
@@ -0,0 +1,32 @@
import json
import sys
import pytest
from ray.dashboard.modules.metrics.metrics_head import (
DEFAULT_PROMETHEUS_HEADERS,
parse_prom_headers,
)
@pytest.mark.parametrize(
"headers, raises_error",
[
(DEFAULT_PROMETHEUS_HEADERS, False),
('{"H1": "V1", "H2": "V2"}', False),
('[["H1", "V1"], ["H2", "V2"], ["H2", "V3"]]', False),
('{"H1": "V1", "H2": ["V1", "V2"]}', True),
("not_json", True),
],
)
def test_parse_prom_headers(headers, raises_error):
if raises_error:
with pytest.raises(ValueError):
parse_prom_headers(headers)
else:
result = parse_prom_headers(headers)
assert result == json.loads(headers)
if __name__ == "__main__":
sys.exit(pytest.main(["-vv", __file__]))