{
   "agent":{
      "metrics_collection_interval":60,
      "run_as_user":"root"
   },
   "logs":{
      "metrics_collected": {
        "prometheus": {
          "log_group_name": "{cluster_name}-ray-prometheus",
          "prometheus_config_path": "/opt/aws/amazon-cloudwatch-agent/etc/prometheus.yml",
          "emf_processor": {
            "metric_declaration_dedup": true,
            "metric_namespace": "{cluster_name}-ray-prometheus",
            "metric_unit":{
              "python_gc_collections_total": "Count",
              "python_gc_objects": "Count",
              "python_gc_objects_uncollectable_total": "Count",
              "python_gc_objects_collected_total": "Count",
              "ray_cluster_active_nodes": "Count",
              "ray_cluster_pending_nodes": "Count",
              "ray_node_cpu_count": "Count",
              "ray_node_cpu_utilization": "Percent",
              "ray_node_disk_free": "Bytes",
              "ray_node_disk_usage": "Bytes",
              "ray_node_disk_utilization_percentage": "Percent",
              "ray_node_mem_available": "Bytes",
              "ray_node_mem_total": "Bytes",
              "ray_node_mem_used": "Bytes",
              "ray_node_mem_total_host": "Bytes",
              "ray_node_mem_used_host": "Bytes",
              "ray_node_network_receive_speed": "Bytes",
              "ray_node_network_received": "Bytes",
              "ray_node_network_send_speed": "Bytes",
              "ray_node_network_sent": "Bytes",
              "ray_avg_num_executed_tasks": "Count",
              "ray_avg_num_scheduled_tasks": "Count",
              "ray_avg_num_spilled_back_tasks": "Count",
              "ray_object_manager_num_pull_requests": "Count",
              "ray_object_store_available_memory": "Bytes",
              "ray_object_store_used_memory": "Bytes",
              "ray_object_store_fallback_memory":"Bytes",
              "ray_object_store_num_local_objects": "Count",
              "ray_object_directory_subscriptions": "Count",
              "ray_object_directory_added_locations": "Count",
              "ray_object_directory_removed_locations": "Count",
              "ray_object_directory_lookups": "Count",
              "ray_object_directory_updates": "Count",
              "ray_pending_actors": "Count",
              "ray_pending_placement_groups": "Count",
              "ray_raylet_cpu": "Count",
              "ray_raylet_mem": "Bytes",
              "ray_raylet_mem_uss": "Bytes",
              "ray_workers_mem": "Bytes",
              "ray_workers_mem_uss": "Bytes",
              "ray_internal_num_spilled_tasks": "Count",
              "ray_internal_num_infeasible_tasks": "Count",
              "ray_internal_num_processes_started": "Count",
              "ray_internal_num_received_tasks": "Count",
              "ray_internal_num_dispatched_tasks": "Count",
              "process_max_fds": "Count",
              "process_open_fds": "Count",
              "process_resident_memory_bytes": "Bytes",
              "process_virtual_memory_bytes": "Bytes",
              "process_start_time_seconds": "Seconds",
              "process_cpu_seconds_total": "Seconds",
              "autoscaler_config_validation_exceptions": "Count",
              "autoscaler_node_launch_exceptions": "Count",
              "autoscaler_pending_nodes": "Count",
              "autoscaler_reset_exceptions": "Count",
              "autoscaler_running_workers": "Count",
              "autoscaler_started_nodes": "Count",
              "autoscaler_stopped_nodes": "Count",
              "autoscaler_update_loop_exceptions": "Count",
              "autoscaler_worker_create_node_time": "Seconds",
              "autoscaler_worker_update_time": "Seconds",
              "autoscaler_updating_nodes": "Count",
              "autoscaler_successful_updates": "Count",
              "autoscaler_failed_updates": "Count",
              "autoscaler_failed_create_nodes": "Count",
              "autoscaler_recovering_nodes": "Count",
              "autoscaler_successful_recoveries": "Count",
              "autoscaler_failed_recoveries": "Count"
            },
            "metric_declaration": [
              {
                "source_labels": [
                  "job"
                ],
                "label_matcher": "ray",
                "dimensions": [
                  [
                    "instance"
                  ]
                ],
                "metric_selectors": [
                  ""
                ]
              }
            ]
          }
        }
      },
      "logs_collected":{
         "files":{
            "collect_list":[
               {
                  "file_path":"/tmp/ray/session_*/logs/**.out",
                  "log_group_name":"{cluster_name}-ray_logs_out",
                  "log_stream_name":"{instance_id}"
               },
               {
                  "file_path":"/tmp/ray/session_*/logs/**.err",
                  "log_group_name":"{cluster_name}-ray_logs_err",
                  "log_stream_name":"{instance_id}"
               }
            ]
         }
      }
   },
   "metrics": {
     "namespace": "{cluster_name}-ray-CWAgent",
     "aggregation_dimensions": [
       [
         "InstanceId"
       ]
     ],
     "append_dimensions": {
       "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
       "InstanceId": "${aws:InstanceId}"
     },
     "metrics_collected": {
       "collectd": {
         "metrics_aggregation_interval": 60
       },
       "cpu": {
         "measurement": [
           "usage_active",
           "usage_system",
           "usage_user",
           "usage_idle",
           "time_active",
           "time_system",
           "time_user",
           "time_idle"
         ],
         "resources": [
           "*"
         ]
       },
       "processes": {
         "measurement": [
           "processes_running",
           "processes_sleeping",
           "processes_zombies",
           "processes_dead",
           "processes_total"
         ],
         "metrics_collection_interval": 60,
         "resources": [
           "*"
         ]
       },
       "disk": {
         "measurement": [
           "disk_used_percent"
         ],
         "metrics_collection_interval": 60,
         "resources": [
           "/"
         ]
       },
       "mem": {
         "measurement": [
           "mem_used_percent"
         ],
         "metrics_collection_interval": 60,
         "resources": [
           "*"
         ]
       },
       "statsd": {
         "metrics_aggregation_interval": 60,
         "metrics_collection_interval": 10,
         "service_address": ":8125"
       }
     }
   }
}