wehub-resource-sync
251 lines
9.8 KiB
Python
251 lines
9.8 KiB
Python
# coding: utf-8
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
import pytest
|
|
|
|
from ray._common.utils import get_default_ray_temp_dir
|
|
from ray._private.ray_constants import SESSION_LATEST
|
|
from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import (
|
|
DEFAULT_GRAFANA_ROWS,
|
|
MAX_PERCENTAGE_EXPRESSION,
|
|
)
|
|
from ray.dashboard.modules.metrics.dashboards.serve_dashboard_panels import (
|
|
SERVE_GRAFANA_PANELS,
|
|
)
|
|
from ray.tests.conftest import _ray_start
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"is_temp_dir_set, temp_dir_val",
|
|
[
|
|
(False, ""),
|
|
pytest.param(
|
|
True,
|
|
"/tmp/test-temp-dir",
|
|
marks=pytest.mark.skipif(
|
|
sys.platform == "win32", reason="Path doesn't apply to Windows"
|
|
),
|
|
),
|
|
],
|
|
)
|
|
def test_metrics_folder_and_content(is_temp_dir_set, temp_dir_val):
|
|
"""
|
|
Tests that the default dashboard files get created. It also verifies paths in the
|
|
dashboard config files are set correctly.
|
|
|
|
It checks both the default case and the case where the _temp_dir is specified.
|
|
"""
|
|
with _ray_start(
|
|
include_dashboard=True, _temp_dir=temp_dir_val if is_temp_dir_set else None
|
|
) as context:
|
|
session_dir = context["session_dir"]
|
|
temp_dir = temp_dir_val if is_temp_dir_set else get_default_ray_temp_dir()
|
|
assert os.path.exists(
|
|
f"{session_dir}/metrics/grafana/provisioning/dashboards/default.yml"
|
|
)
|
|
with open(
|
|
f"{session_dir}/metrics/grafana/provisioning/dashboards/default.yml", "r"
|
|
) as f:
|
|
target_path = os.path.join(session_dir, "metrics", "grafana", "dashboards")
|
|
assert f"path: {target_path}" in f.read()
|
|
|
|
assert os.path.exists(
|
|
f"{session_dir}/metrics/grafana/dashboards"
|
|
"/default_grafana_dashboard.json"
|
|
)
|
|
assert os.path.exists(
|
|
f"{session_dir}/metrics/grafana/provisioning/datasources/default.yml"
|
|
)
|
|
|
|
assert os.path.exists(f"{session_dir}/metrics/grafana/grafana.ini")
|
|
with open(f"{session_dir}/metrics/grafana/grafana.ini", "r") as f:
|
|
target_path = os.path.join(
|
|
temp_dir, SESSION_LATEST, "metrics", "grafana", "provisioning"
|
|
)
|
|
assert f"provisioning = {target_path}" in f.read()
|
|
|
|
assert os.path.exists(f"{session_dir}/metrics/prometheus/prometheus.yml")
|
|
with open(f"{session_dir}/metrics/prometheus/prometheus.yml", "r") as f:
|
|
target_path = os.path.join(
|
|
session_dir, "prom_metrics_service_discovery.json"
|
|
)
|
|
assert f"- '{target_path}'" in f.read()
|
|
|
|
|
|
@pytest.fixture
|
|
def override_dashboard_dir():
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
os.environ["RAY_METRICS_GRAFANA_DASHBOARD_OUTPUT_DIR"] = tempdir
|
|
yield tempdir
|
|
del os.environ["RAY_METRICS_GRAFANA_DASHBOARD_OUTPUT_DIR"]
|
|
|
|
|
|
@pytest.fixture
|
|
def override_default_dashboard():
|
|
uid = "test_uid_ses_12345"
|
|
global_filters = 'TestVar="StaticValue"'
|
|
|
|
os.environ["RAY_GRAFANA_DEFAULT_DASHBOARD_UID"] = uid
|
|
os.environ["RAY_GRAFANA_DEFAULT_DASHBOARD_GLOBAL_FILTERS"] = global_filters
|
|
yield uid, global_filters
|
|
del os.environ["RAY_GRAFANA_DEFAULT_DASHBOARD_GLOBAL_FILTERS"]
|
|
del os.environ["RAY_GRAFANA_DEFAULT_DASHBOARD_UID"]
|
|
|
|
|
|
@pytest.fixture
|
|
def override_serve_dashboard():
|
|
uid = "test_serve_uid_ses_12345"
|
|
global_filters = 'TestVar="$TestVariableValue"'
|
|
|
|
os.environ["RAY_GRAFANA_SERVE_DASHBOARD_UID"] = uid
|
|
os.environ["RAY_GRAFANA_SERVE_DASHBOARD_GLOBAL_FILTERS"] = global_filters
|
|
yield uid, global_filters
|
|
del os.environ["RAY_GRAFANA_SERVE_DASHBOARD_GLOBAL_FILTERS"]
|
|
del os.environ["RAY_GRAFANA_SERVE_DASHBOARD_UID"]
|
|
|
|
|
|
def test_metrics_folder_with_dashboard_override(
|
|
override_dashboard_dir, override_default_dashboard, override_serve_dashboard
|
|
):
|
|
"""
|
|
Tests that the default dashboard files get created.
|
|
"""
|
|
uid, global_filters = override_default_dashboard
|
|
serve_uid, serve_global_filters = override_serve_dashboard
|
|
|
|
with _ray_start(include_dashboard=True) as context:
|
|
session_dir = context["session_dir"]
|
|
assert os.path.exists(
|
|
f"{override_dashboard_dir}/default_grafana_dashboard.json"
|
|
)
|
|
with open(
|
|
f"{session_dir}/metrics/grafana/provisioning/dashboards/default.yml"
|
|
) as f:
|
|
contents = f.read()
|
|
assert override_dashboard_dir in contents
|
|
|
|
# Default Dashboard
|
|
with open(f"{override_dashboard_dir}/default_grafana_dashboard.json") as f:
|
|
contents = json.loads(f.read())
|
|
assert contents["uid"] == uid
|
|
for panel in contents["panels"]:
|
|
if panel["type"] == "row":
|
|
# Row panels don't have targets
|
|
continue
|
|
for target in panel["targets"]:
|
|
if target["expr"] == MAX_PERCENTAGE_EXPRESSION:
|
|
# We skip expressions that are constant value targets
|
|
continue
|
|
# Check for standard_global_filters
|
|
assert 'SessionName=~"$SessionName"' in target["expr"]
|
|
# Check for custom global_filters
|
|
assert global_filters in target["expr"]
|
|
for variable in contents["templating"]["list"]:
|
|
if variable["name"] == "datasource":
|
|
continue
|
|
if variable["name"] == "RayNodeType":
|
|
# RayNodeType uses hardcoded values instead of a query
|
|
continue
|
|
assert global_filters in variable["definition"]
|
|
assert global_filters in variable["query"]["query"]
|
|
assert "supportsGlobalFilterOverride" in contents["rayMeta"]
|
|
# Check that panels have some width and height
|
|
for panel in contents["panels"]:
|
|
assert panel["gridPos"]["h"] > 0
|
|
assert panel["gridPos"]["w"] > 0
|
|
# Check for series overrides for MAX and MAX + PENDING
|
|
found_max = False
|
|
found_max_pending = False
|
|
for panel in contents["panels"]:
|
|
if panel["type"] == "row":
|
|
# Row panels don't have series overrides
|
|
continue
|
|
for override in panel.get("seriesOverrides", []):
|
|
if override.get("alias") == "MAX":
|
|
assert override["fill"] == 0
|
|
assert override["stack"] is False
|
|
found_max = True
|
|
if override.get("alias") == "MAX + PENDING":
|
|
assert override["fill"] == 0
|
|
assert override["stack"] is False
|
|
found_max_pending = True
|
|
assert found_max
|
|
assert found_max_pending
|
|
|
|
# Serve Dashboard
|
|
with open(f"{override_dashboard_dir}/serve_grafana_dashboard.json") as f:
|
|
contents = json.loads(f.read())
|
|
assert contents["uid"] == serve_uid
|
|
for panel in contents["panels"]:
|
|
if panel["type"] == "row":
|
|
# Row panels contain nested panels, not targets directly
|
|
for nested_panel in panel.get("panels", []):
|
|
for target in nested_panel["targets"]:
|
|
assert serve_global_filters in target["expr"]
|
|
else:
|
|
for target in panel["targets"]:
|
|
assert serve_global_filters in target["expr"]
|
|
for variable in contents["templating"]["list"]:
|
|
if variable["name"] == "datasource":
|
|
continue
|
|
assert serve_global_filters in variable["definition"]
|
|
assert serve_global_filters in variable["query"]["query"]
|
|
assert "supportsGlobalFilterOverride" in contents["rayMeta"]
|
|
|
|
# Serve Deployment Dashboard
|
|
with open(
|
|
f"{override_dashboard_dir}/serve_deployment_grafana_dashboard.json"
|
|
) as f:
|
|
contents = json.loads(f.read())
|
|
assert contents["uid"] == "rayServeDeploymentDashboard"
|
|
assert "supportsGlobalFilterOverride" in contents["rayMeta"]
|
|
|
|
|
|
def test_metrics_folder_when_dashboard_disabled():
|
|
"""
|
|
Tests that the default dashboard files do not get created when dashboard
|
|
is disabled.
|
|
"""
|
|
with _ray_start(include_dashboard=False) as context:
|
|
session_dir = context["session_dir"]
|
|
assert not os.path.exists(
|
|
f"{session_dir}/metrics/grafana/provisioning/dashboards/default.yml"
|
|
)
|
|
assert not os.path.exists(
|
|
f"{session_dir}/metrics/grafana/dashboards"
|
|
"/default_grafana_dashboard.json"
|
|
)
|
|
assert not os.path.exists(
|
|
f"{session_dir}/metrics/grafana/provisioning/datasources/default.yml"
|
|
)
|
|
assert not os.path.exists(f"{session_dir}/metrics/prometheus/prometheus.yml")
|
|
|
|
|
|
def test_default_dashboard_utilizes_global_filters():
|
|
for row in DEFAULT_GRAFANA_ROWS:
|
|
for panel in row.panels:
|
|
for target in panel.targets:
|
|
if target.legend == "MAX" and target.expr == MAX_PERCENTAGE_EXPRESSION:
|
|
# We skip expressions that are constant value targets serving as visual aids
|
|
continue
|
|
assert "{global_filters}" in target.expr
|
|
|
|
|
|
def test_serve_dashboard_utilizes_global_filters():
|
|
for panel in SERVE_GRAFANA_PANELS:
|
|
for target in panel.targets:
|
|
if target.legend == "MAX" and target.expr == MAX_PERCENTAGE_EXPRESSION:
|
|
# We skip expressions that are constant value targets serving as visual aids
|
|
continue
|
|
assert "{global_filters}" in target.expr
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-sv", __file__]))
|