chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"agent":{
|
||||
"metrics_collection_interval":60,
|
||||
"run_as_user":"root"
|
||||
},
|
||||
"logs":{
|
||||
"metrics_collected": {
|
||||
"prometheus": {
|
||||
"log_group_name": "{cluster_name}-ray-prometheus",
|
||||
"prometheus_config_path": "/opt/aws/amazon-cloudwatch-agent/etc/prometheus.yml",
|
||||
"emf_processor": {
|
||||
"metric_declaration_dedup": true,
|
||||
"metric_namespace": "{cluster_name}-ray-prometheus",
|
||||
"metric_unit":{
|
||||
"python_gc_collections_total": "Count",
|
||||
"python_gc_objects": "Count",
|
||||
"python_gc_objects_uncollectable_total": "Count",
|
||||
"python_gc_objects_collected_total": "Count",
|
||||
"ray_cluster_active_nodes": "Count",
|
||||
"ray_cluster_pending_nodes": "Count",
|
||||
"ray_node_cpu_count": "Count",
|
||||
"ray_node_cpu_utilization": "Percent",
|
||||
"ray_node_disk_free": "Bytes",
|
||||
"ray_node_disk_usage": "Bytes",
|
||||
"ray_node_disk_utilization_percentage": "Percent",
|
||||
"ray_node_mem_available": "Bytes",
|
||||
"ray_node_mem_total": "Bytes",
|
||||
"ray_node_mem_used": "Bytes",
|
||||
"ray_node_mem_total_host": "Bytes",
|
||||
"ray_node_mem_used_host": "Bytes",
|
||||
"ray_node_network_receive_speed": "Bytes",
|
||||
"ray_node_network_received": "Bytes",
|
||||
"ray_node_network_send_speed": "Bytes",
|
||||
"ray_node_network_sent": "Bytes",
|
||||
"ray_avg_num_executed_tasks": "Count",
|
||||
"ray_avg_num_scheduled_tasks": "Count",
|
||||
"ray_avg_num_spilled_back_tasks": "Count",
|
||||
"ray_object_manager_num_pull_requests": "Count",
|
||||
"ray_object_store_available_memory": "Bytes",
|
||||
"ray_object_store_used_memory": "Bytes",
|
||||
"ray_object_store_fallback_memory":"Bytes",
|
||||
"ray_object_store_num_local_objects": "Count",
|
||||
"ray_object_directory_subscriptions": "Count",
|
||||
"ray_object_directory_added_locations": "Count",
|
||||
"ray_object_directory_removed_locations": "Count",
|
||||
"ray_object_directory_lookups": "Count",
|
||||
"ray_object_directory_updates": "Count",
|
||||
"ray_pending_actors": "Count",
|
||||
"ray_pending_placement_groups": "Count",
|
||||
"ray_raylet_cpu": "Count",
|
||||
"ray_raylet_mem": "Bytes",
|
||||
"ray_raylet_mem_uss": "Bytes",
|
||||
"ray_workers_mem": "Bytes",
|
||||
"ray_workers_mem_uss": "Bytes",
|
||||
"ray_internal_num_spilled_tasks": "Count",
|
||||
"ray_internal_num_infeasible_tasks": "Count",
|
||||
"ray_internal_num_processes_started": "Count",
|
||||
"ray_internal_num_received_tasks": "Count",
|
||||
"ray_internal_num_dispatched_tasks": "Count",
|
||||
"process_max_fds": "Count",
|
||||
"process_open_fds": "Count",
|
||||
"process_resident_memory_bytes": "Bytes",
|
||||
"process_virtual_memory_bytes": "Bytes",
|
||||
"process_start_time_seconds": "Seconds",
|
||||
"process_cpu_seconds_total": "Seconds",
|
||||
"autoscaler_config_validation_exceptions": "Count",
|
||||
"autoscaler_node_launch_exceptions": "Count",
|
||||
"autoscaler_pending_nodes": "Count",
|
||||
"autoscaler_reset_exceptions": "Count",
|
||||
"autoscaler_running_workers": "Count",
|
||||
"autoscaler_started_nodes": "Count",
|
||||
"autoscaler_stopped_nodes": "Count",
|
||||
"autoscaler_update_loop_exceptions": "Count",
|
||||
"autoscaler_worker_create_node_time": "Seconds",
|
||||
"autoscaler_worker_update_time": "Seconds",
|
||||
"autoscaler_updating_nodes": "Count",
|
||||
"autoscaler_successful_updates": "Count",
|
||||
"autoscaler_failed_updates": "Count",
|
||||
"autoscaler_failed_create_nodes": "Count",
|
||||
"autoscaler_recovering_nodes": "Count",
|
||||
"autoscaler_successful_recoveries": "Count",
|
||||
"autoscaler_failed_recoveries": "Count"
|
||||
},
|
||||
"metric_declaration": [
|
||||
{
|
||||
"source_labels": [
|
||||
"job"
|
||||
],
|
||||
"label_matcher": "ray",
|
||||
"dimensions": [
|
||||
[
|
||||
"instance"
|
||||
]
|
||||
],
|
||||
"metric_selectors": [
|
||||
""
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"logs_collected":{
|
||||
"files":{
|
||||
"collect_list":[
|
||||
{
|
||||
"file_path":"/tmp/ray/session_*/logs/**.out",
|
||||
"log_group_name":"{cluster_name}-ray_logs_out",
|
||||
"log_stream_name":"{instance_id}"
|
||||
},
|
||||
{
|
||||
"file_path":"/tmp/ray/session_*/logs/**.err",
|
||||
"log_group_name":"{cluster_name}-ray_logs_err",
|
||||
"log_stream_name":"{instance_id}"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"namespace": "{cluster_name}-ray-CWAgent",
|
||||
"aggregation_dimensions": [
|
||||
[
|
||||
"InstanceId"
|
||||
]
|
||||
],
|
||||
"append_dimensions": {
|
||||
"AutoScalingGroupName": "${aws:AutoScalingGroupName}",
|
||||
"InstanceId": "${aws:InstanceId}"
|
||||
},
|
||||
"metrics_collected": {
|
||||
"collectd": {
|
||||
"metrics_aggregation_interval": 60
|
||||
},
|
||||
"cpu": {
|
||||
"measurement": [
|
||||
"usage_active",
|
||||
"usage_system",
|
||||
"usage_user",
|
||||
"usage_idle",
|
||||
"time_active",
|
||||
"time_system",
|
||||
"time_user",
|
||||
"time_idle"
|
||||
],
|
||||
"resources": [
|
||||
"*"
|
||||
]
|
||||
},
|
||||
"processes": {
|
||||
"measurement": [
|
||||
"processes_running",
|
||||
"processes_sleeping",
|
||||
"processes_zombies",
|
||||
"processes_dead",
|
||||
"processes_total"
|
||||
],
|
||||
"metrics_collection_interval": 60,
|
||||
"resources": [
|
||||
"*"
|
||||
]
|
||||
},
|
||||
"disk": {
|
||||
"measurement": [
|
||||
"disk_used_percent"
|
||||
],
|
||||
"metrics_collection_interval": 60,
|
||||
"resources": [
|
||||
"/"
|
||||
]
|
||||
},
|
||||
"mem": {
|
||||
"measurement": [
|
||||
"mem_used_percent"
|
||||
],
|
||||
"metrics_collection_interval": 60,
|
||||
"resources": [
|
||||
"*"
|
||||
]
|
||||
},
|
||||
"statsd": {
|
||||
"metrics_aggregation_interval": 60,
|
||||
"metrics_collection_interval": 10,
|
||||
"service_address": ":8125"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"EvaluationPeriods":1,
|
||||
"ComparisonOperator":"GreaterThanThreshold",
|
||||
"AlarmActions":[
|
||||
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
|
||||
],
|
||||
"Namespace":"{cluster_name}-ray-CWAgent",
|
||||
"AlarmDescription":"Memory used exceeds 90 percent for 5 minutes",
|
||||
"Period":300,
|
||||
"Threshold":90.0,
|
||||
"AlarmName":"high mem_used_percent_{instance_id}",
|
||||
"Dimensions":[
|
||||
{
|
||||
"Name":"InstanceId",
|
||||
"Value":"{instance_id}"
|
||||
}
|
||||
],
|
||||
"Statistic":"Average",
|
||||
"InsufficientDataActions":[
|
||||
|
||||
],
|
||||
"OKActions":[
|
||||
|
||||
],
|
||||
"ActionsEnabled":true,
|
||||
"MetricName":"mem_used_percent"
|
||||
},
|
||||
{
|
||||
"EvaluationPeriods":1,
|
||||
"ComparisonOperator":"GreaterThanThreshold",
|
||||
"AlarmActions":[
|
||||
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
|
||||
],
|
||||
"Namespace":"{cluster_name}-ray-CWAgent",
|
||||
"AlarmDescription":"Disk used exceeds 90 percent for five minutes",
|
||||
"Period":300,
|
||||
"Threshold":90.0,
|
||||
"AlarmName":"high disk_used_percent_{instance_id}",
|
||||
"Dimensions": [
|
||||
{
|
||||
"Name": "InstanceId",
|
||||
"Value": "{instance_id}"
|
||||
}
|
||||
],
|
||||
"Statistic":"Average",
|
||||
"InsufficientDataActions":[
|
||||
|
||||
],
|
||||
"OKActions":[
|
||||
|
||||
],
|
||||
"ActionsEnabled":true,
|
||||
"MetricName":"disk_used_percent"
|
||||
},
|
||||
{
|
||||
"EvaluationPeriods":1,
|
||||
"ComparisonOperator":"GreaterThanThreshold",
|
||||
"AlarmActions":[
|
||||
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
|
||||
],
|
||||
"Namespace":"AWS/EC2",
|
||||
"AlarmDescription":"CPU used exceeds 90 percent for 2 hours",
|
||||
"Period":7200,
|
||||
"Threshold":90.0,
|
||||
"AlarmName":"high_CPUUtilization_{instance_id}",
|
||||
"Dimensions":[
|
||||
{
|
||||
"Name":"InstanceId",
|
||||
"Value":"{instance_id}"
|
||||
}
|
||||
],
|
||||
"Statistic":"Average",
|
||||
"InsufficientDataActions":[
|
||||
|
||||
],
|
||||
"OKActions":[
|
||||
|
||||
],
|
||||
"ActionsEnabled":true,
|
||||
"MetricName":"CPUUtilization"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,237 @@
|
||||
[
|
||||
{
|
||||
"type":"explorer",
|
||||
"x":12,
|
||||
"y":18,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties": {
|
||||
"metrics": [
|
||||
{
|
||||
"metricName": "CPUUtilization",
|
||||
"resourceType": "AWS::EC2::Instance",
|
||||
"stat": "Average"
|
||||
}
|
||||
],
|
||||
"aggregateBy": {
|
||||
"key": "*",
|
||||
"func": "SUM"
|
||||
},
|
||||
"labels": [
|
||||
{
|
||||
"key": "cloudwatch-agent-installed",
|
||||
"value": "True"
|
||||
},
|
||||
{
|
||||
"key": "ray-cluster-name",
|
||||
"value": "{cluster_name}"
|
||||
}
|
||||
],
|
||||
"widgetOptions": {
|
||||
"legend": {
|
||||
"position": "bottom"
|
||||
},
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"rowsPerPage": 1,
|
||||
"widgetsPerRow": 1
|
||||
},
|
||||
"title":"Cluster CPU Utilization"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"explorer",
|
||||
"x":0,
|
||||
"y":18,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties": {
|
||||
"metrics": [
|
||||
{
|
||||
"metricName": "CPUUtilization",
|
||||
"resourceType": "AWS::EC2::Instance",
|
||||
"stat": "Average"
|
||||
}
|
||||
],
|
||||
"aggregateBy": {
|
||||
"key": "*",
|
||||
"func": "AVG"
|
||||
},
|
||||
"labels": [
|
||||
{
|
||||
"key": "cloudwatch-agent-installed",
|
||||
"value": "True"
|
||||
},
|
||||
{
|
||||
"key": "ray-cluster-name",
|
||||
"value": "{cluster_name}"
|
||||
}
|
||||
],
|
||||
"widgetOptions": {
|
||||
"legend": {
|
||||
"position": "bottom"
|
||||
},
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"rowsPerPage": 1,
|
||||
"widgetsPerRow": 1
|
||||
},
|
||||
"title":"Single Node CPU Utilization (Avg and Max)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":12,
|
||||
"y":6,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties":{
|
||||
"view":"timeSeries",
|
||||
"metrics":[
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process sum", "id": "e1" } ],
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process sum", "id": "e2" } ]
|
||||
],
|
||||
"region":"{region}",
|
||||
"stat":"Average",
|
||||
"period":60,
|
||||
"title":"Cluster Processes"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":0,
|
||||
"y":6,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties":{
|
||||
"view":"timeSeries",
|
||||
"metrics":[
|
||||
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process average", "id": "e3" } ],
|
||||
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process average", "id": "e4" } ],
|
||||
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process maximum", "id": "e5" } ],
|
||||
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process maximum", "id": "e6" } ]
|
||||
],
|
||||
"region":"{region}",
|
||||
"stat":"Average",
|
||||
"period":60,
|
||||
"title":"Single Node Processes (Avg and Max)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":12,
|
||||
"y":12,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties":{
|
||||
"view":"timeSeries",
|
||||
"stacked":false,
|
||||
"metrics":[
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Average', 300))", "label": "cluster disk used percent sum", "id": "e7", "period": 300 } ]
|
||||
|
||||
],
|
||||
"region":"{region}",
|
||||
"title":"Cluster Disk Usage"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":0,
|
||||
"y":12,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties":{
|
||||
"view":"timeSeries",
|
||||
"stacked":false,
|
||||
"metrics":[
|
||||
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Average', 300))", "id": "e8", "label": "cluster disk used percent average", "period": 300 } ],
|
||||
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Maximum', 300))", "id": "e9", "label": "cluster disk used percent maximum", "period": 300 } ]
|
||||
|
||||
],
|
||||
"region":"{region}",
|
||||
"title":"Single Node Disk Usage (Avg and Max)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":12,
|
||||
"y":18,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties": {
|
||||
"metrics": [
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Average', 300))", "id": "e10", "label": "cluster mem used percent sum", "period": 300 } ]
|
||||
|
||||
],
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"region": "{region}",
|
||||
"stat": "Maximum",
|
||||
"period": 300,
|
||||
"start": "-PT2H",
|
||||
"end": "P0D",
|
||||
"title": "Cluster Memory Usage"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type":"metric",
|
||||
"x":0,
|
||||
"y":18,
|
||||
"width":12,
|
||||
"height":6,
|
||||
"properties": {
|
||||
"metrics": [
|
||||
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Average', 300))", "id": "e11", "label": "cluster mem used percent average", "period": 300 } ],
|
||||
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Maximum', 300))", "id": "e12", "label": "cluster mem used percent maximum", "period": 300 } ]
|
||||
],
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"region": "{region}",
|
||||
"stat": "Maximum",
|
||||
"period": 300,
|
||||
"start": "-PT2H",
|
||||
"end": "P0D",
|
||||
"title": "Single Node Memory Usage (Avg and Max)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"height": 6,
|
||||
"width": 12,
|
||||
"y": 0,
|
||||
"x": 0,
|
||||
"type": "metric",
|
||||
"properties": {
|
||||
"metrics": [
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-prometheus,instance} ray_node_cpu_count', 'Maximum', 300))", "label": "cluster cpu sum", "id": "e13" } ]
|
||||
],
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"region": "{region}",
|
||||
"stat": "Maximum",
|
||||
"period": 300,
|
||||
"start": "-PT2H",
|
||||
"end": "P0D",
|
||||
"title": "Cluster CPUs"
|
||||
}
|
||||
},
|
||||
{
|
||||
"height": 6,
|
||||
"width": 12,
|
||||
"y": 0,
|
||||
"x": 12,
|
||||
"type": "metric",
|
||||
"properties": {
|
||||
"metrics": [
|
||||
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-prometheus,instance} object_store_available_memory', 'Average', 300))", "label": "cluster object store available memory sum", "id": "e14" } ]
|
||||
],
|
||||
"view": "timeSeries",
|
||||
"stacked": false,
|
||||
"region": "{region}",
|
||||
"stat": "Maximum",
|
||||
"period": 300,
|
||||
"start": "-PT2H",
|
||||
"end": "P0D",
|
||||
"title": "Cluster Object Store Available Memory"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
# Prometheus config file
|
||||
|
||||
# my global config
|
||||
global:
|
||||
scrape_interval: 10s
|
||||
evaluation_interval: 10s
|
||||
scrape_timeout: 10s
|
||||
|
||||
# use ray file-based service discovery file as scrape target.
|
||||
scrape_configs:
|
||||
- job_name: 'ray'
|
||||
file_sd_configs:
|
||||
- files:
|
||||
- '/tmp/ray/prom_metrics_service_discovery.json'
|
||||
refresh_interval: 1m
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
MAX_ATTEMPTS=120
|
||||
DELAY_SECONDS=10
|
||||
RAY_PROM_METRICS_FILE_PATH="/tmp/ray/prom_metrics_service_discovery.json"
|
||||
CLUSTER_NAME=$1
|
||||
while [ $MAX_ATTEMPTS -gt 0 ]; do
|
||||
if [ -f $RAY_PROM_METRICS_FILE_PATH ]; then
|
||||
echo "Ray Prometheus metrics service discovery file found at: $RAY_PROM_METRICS_FILE_PATH."
|
||||
echo "Restarting cloudwatch agent.This may take a few minutes..."
|
||||
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -m ec2 -a stop
|
||||
echo "Cloudwatch agent stopped, starting cloudwatch agent..."
|
||||
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:AmazonCloudWatch-ray_agent_config_$CLUSTER_NAME"
|
||||
echo "Cloudwatch agent successfully restarted!"
|
||||
exit 0
|
||||
else
|
||||
echo "Ray Prometheus metrics service discovery file not found at: $RAY_PROM_METRICS_FILE_PATH. Will check again in $DELAY_SECONDS seconds..."
|
||||
sleep $DELAY_SECONDS
|
||||
MAX_ATTEMPTS=$((MAX_ATTEMPTS-1))
|
||||
fi
|
||||
done
|
||||
echo "Ray Prometheus metrics service discovery file not found at: $RAY_PROM_METRICS_FILE_PATH. Ray system metrics will not be available in CloudWatch."
|
||||
exit 1
|
||||
Reference in New Issue
Block a user