wehub/ray-project--ray
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/autoscaler/aws/BUILD.bazel
+19
View File
@@ -0,0 +1,19 @@
filegroup(
name = "example",
data = glob(["example-*.yaml"]),
visibility = [
"//python/ray/tests:__pkg__",
],
)
filegroup(
name = "test_configs",
data = glob(["tests/*.yaml"]),
visibility = ["//release:__pkg__"],
)
filegroup(
name = "default_config",
srcs = ["defaults.yaml"],
visibility = ["//visibility:public"],
)
python/ray/autoscaler/aws/__init__.py
View File
python/ray/autoscaler/aws/cloudwatch/example-cloudwatch-agent-config.json
Executable
+188
View File
@@ -0,0 +1,188 @@
{
"agent":{
"metrics_collection_interval":60,
"run_as_user":"root"
},
"logs":{
"metrics_collected": {
"prometheus": {
"log_group_name": "{cluster_name}-ray-prometheus",
"prometheus_config_path": "/opt/aws/amazon-cloudwatch-agent/etc/prometheus.yml",
"emf_processor": {
"metric_declaration_dedup": true,
"metric_namespace": "{cluster_name}-ray-prometheus",
"metric_unit":{
"python_gc_collections_total": "Count",
"python_gc_objects": "Count",
"python_gc_objects_uncollectable_total": "Count",
"python_gc_objects_collected_total": "Count",
"ray_cluster_active_nodes": "Count",
"ray_cluster_pending_nodes": "Count",
"ray_node_cpu_count": "Count",
"ray_node_cpu_utilization": "Percent",
"ray_node_disk_free": "Bytes",
"ray_node_disk_usage": "Bytes",
"ray_node_disk_utilization_percentage": "Percent",
"ray_node_mem_available": "Bytes",
"ray_node_mem_total": "Bytes",
"ray_node_mem_used": "Bytes",
"ray_node_mem_total_host": "Bytes",
"ray_node_mem_used_host": "Bytes",
"ray_node_network_receive_speed": "Bytes",
"ray_node_network_received": "Bytes",
"ray_node_network_send_speed": "Bytes",
"ray_node_network_sent": "Bytes",
"ray_avg_num_executed_tasks": "Count",
"ray_avg_num_scheduled_tasks": "Count",
"ray_avg_num_spilled_back_tasks": "Count",
"ray_object_manager_num_pull_requests": "Count",
"ray_object_store_available_memory": "Bytes",
"ray_object_store_used_memory": "Bytes",
"ray_object_store_fallback_memory":"Bytes",
"ray_object_store_num_local_objects": "Count",
"ray_object_directory_subscriptions": "Count",
"ray_object_directory_added_locations": "Count",
"ray_object_directory_removed_locations": "Count",
"ray_object_directory_lookups": "Count",
"ray_object_directory_updates": "Count",
"ray_pending_actors": "Count",
"ray_pending_placement_groups": "Count",
"ray_raylet_cpu": "Count",
"ray_raylet_mem": "Bytes",
"ray_raylet_mem_uss": "Bytes",
"ray_workers_mem": "Bytes",
"ray_workers_mem_uss": "Bytes",
"ray_internal_num_spilled_tasks": "Count",
"ray_internal_num_infeasible_tasks": "Count",
"ray_internal_num_processes_started": "Count",
"ray_internal_num_received_tasks": "Count",
"ray_internal_num_dispatched_tasks": "Count",
"process_max_fds": "Count",
"process_open_fds": "Count",
"process_resident_memory_bytes": "Bytes",
"process_virtual_memory_bytes": "Bytes",
"process_start_time_seconds": "Seconds",
"process_cpu_seconds_total": "Seconds",
"autoscaler_config_validation_exceptions": "Count",
"autoscaler_node_launch_exceptions": "Count",
"autoscaler_pending_nodes": "Count",
"autoscaler_reset_exceptions": "Count",
"autoscaler_running_workers": "Count",
"autoscaler_started_nodes": "Count",
"autoscaler_stopped_nodes": "Count",
"autoscaler_update_loop_exceptions": "Count",
"autoscaler_worker_create_node_time": "Seconds",
"autoscaler_worker_update_time": "Seconds",
"autoscaler_updating_nodes": "Count",
"autoscaler_successful_updates": "Count",
"autoscaler_failed_updates": "Count",
"autoscaler_failed_create_nodes": "Count",
"autoscaler_recovering_nodes": "Count",
"autoscaler_successful_recoveries": "Count",
"autoscaler_failed_recoveries": "Count"
},
"metric_declaration": [
{
"source_labels": [
"job"
],
"label_matcher": "ray",
"dimensions": [
[
"instance"
]
],
"metric_selectors": [
""
]
}
]
}
}
},
"logs_collected":{
"files":{
"collect_list":[
{
"file_path":"/tmp/ray/session_*/logs/**.out",
"log_group_name":"{cluster_name}-ray_logs_out",
"log_stream_name":"{instance_id}"
},
{
"file_path":"/tmp/ray/session_*/logs/**.err",
"log_group_name":"{cluster_name}-ray_logs_err",
"log_stream_name":"{instance_id}"
}
]
}
}
},
"metrics": {
"namespace": "{cluster_name}-ray-CWAgent",
"aggregation_dimensions": [
[
"InstanceId"
]
],
"append_dimensions": {
"AutoScalingGroupName": "${aws:AutoScalingGroupName}",
"InstanceId": "${aws:InstanceId}"
},
"metrics_collected": {
"collectd": {
"metrics_aggregation_interval": 60
},
"cpu": {
"measurement": [
"usage_active",
"usage_system",
"usage_user",
"usage_idle",
"time_active",
"time_system",
"time_user",
"time_idle"
],
"resources": [
"*"
]
},
"processes": {
"measurement": [
"processes_running",
"processes_sleeping",
"processes_zombies",
"processes_dead",
"processes_total"
],
"metrics_collection_interval": 60,
"resources": [
"*"
]
},
"disk": {
"measurement": [
"disk_used_percent"
],
"metrics_collection_interval": 60,
"resources": [
"/"
]
},
"mem": {
"measurement": [
"mem_used_percent"
],
"metrics_collection_interval": 60,
"resources": [
"*"
]
},
"statsd": {
"metrics_aggregation_interval": 60,
"metrics_collection_interval": 10,
"service_address": ":8125"
}
}
}
}
python/ray/autoscaler/aws/cloudwatch/example-cloudwatch-alarm-config.json
+83
View File
@@ -0,0 +1,83 @@
[
{
"EvaluationPeriods":1,
"ComparisonOperator":"GreaterThanThreshold",
"AlarmActions":[
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
],
"Namespace":"{cluster_name}-ray-CWAgent",
"AlarmDescription":"Memory used exceeds 90 percent for 5 minutes",
"Period":300,
"Threshold":90.0,
"AlarmName":"high mem_used_percent_{instance_id}",
"Dimensions":[
{
"Name":"InstanceId",
"Value":"{instance_id}"
}
],
"Statistic":"Average",
"InsufficientDataActions":[
],
"OKActions":[
],
"ActionsEnabled":true,
"MetricName":"mem_used_percent"
},
{
"EvaluationPeriods":1,
"ComparisonOperator":"GreaterThanThreshold",
"AlarmActions":[
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
],
"Namespace":"{cluster_name}-ray-CWAgent",
"AlarmDescription":"Disk used exceeds 90 percent for five minutes",
"Period":300,
"Threshold":90.0,
"AlarmName":"high disk_used_percent_{instance_id}",
"Dimensions": [
{
"Name": "InstanceId",
"Value": "{instance_id}"
}
],
"Statistic":"Average",
"InsufficientDataActions":[
],
"OKActions":[
],
"ActionsEnabled":true,
"MetricName":"disk_used_percent"
},
{
"EvaluationPeriods":1,
"ComparisonOperator":"GreaterThanThreshold",
"AlarmActions":[
"TODO: Add alarm actions! See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html"
],
"Namespace":"AWS/EC2",
"AlarmDescription":"CPU used exceeds 90 percent for 2 hours",
"Period":7200,
"Threshold":90.0,
"AlarmName":"high_CPUUtilization_{instance_id}",
"Dimensions":[
{
"Name":"InstanceId",
"Value":"{instance_id}"
}
],
"Statistic":"Average",
"InsufficientDataActions":[
],
"OKActions":[
],
"ActionsEnabled":true,
"MetricName":"CPUUtilization"
}
]
python/ray/autoscaler/aws/cloudwatch/example-cloudwatch-dashboard-config.json
+237
View File
@@ -0,0 +1,237 @@
[
{
"type":"explorer",
"x":12,
"y":18,
"width":12,
"height":6,
"properties": {
"metrics": [
{
"metricName": "CPUUtilization",
"resourceType": "AWS::EC2::Instance",
"stat": "Average"
}
],
"aggregateBy": {
"key": "*",
"func": "SUM"
},
"labels": [
{
"key": "cloudwatch-agent-installed",
"value": "True"
},
{
"key": "ray-cluster-name",
"value": "{cluster_name}"
}
],
"widgetOptions": {
"legend": {
"position": "bottom"
},
"view": "timeSeries",
"stacked": false,
"rowsPerPage": 1,
"widgetsPerRow": 1
},
"title":"Cluster CPU Utilization"
}
},
{
"type":"explorer",
"x":0,
"y":18,
"width":12,
"height":6,
"properties": {
"metrics": [
{
"metricName": "CPUUtilization",
"resourceType": "AWS::EC2::Instance",
"stat": "Average"
}
],
"aggregateBy": {
"key": "*",
"func": "AVG"
},
"labels": [
{
"key": "cloudwatch-agent-installed",
"value": "True"
},
{
"key": "ray-cluster-name",
"value": "{cluster_name}"
}
],
"widgetOptions": {
"legend": {
"position": "bottom"
},
"view": "timeSeries",
"stacked": false,
"rowsPerPage": 1,
"widgetsPerRow": 1
},
"title":"Single Node CPU Utilization (Avg and Max)"
}
},
{
"type":"metric",
"x":12,
"y":6,
"width":12,
"height":6,
"properties":{
"view":"timeSeries",
"metrics":[
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process sum", "id": "e1" } ],
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process sum", "id": "e2" } ]
],
"region":"{region}",
"stat":"Average",
"period":60,
"title":"Cluster Processes"
}
},
{
"type":"metric",
"x":0,
"y":6,
"width":12,
"height":6,
"properties":{
"view":"timeSeries",
"metrics":[
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process average", "id": "e3" } ],
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process average", "id": "e4" } ],
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_running', 'Average', 300))", "label": "cluster running process maximum", "id": "e5" } ],
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} processes_sleeping', 'Average', 300))", "label": "cluster sleeping process maximum", "id": "e6" } ]
],
"region":"{region}",
"stat":"Average",
"period":60,
"title":"Single Node Processes (Avg and Max)"
}
},
{
"type":"metric",
"x":12,
"y":12,
"width":12,
"height":6,
"properties":{
"view":"timeSeries",
"stacked":false,
"metrics":[
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Average', 300))", "label": "cluster disk used percent sum", "id": "e7", "period": 300 } ]
],
"region":"{region}",
"title":"Cluster Disk Usage"
}
},
{
"type":"metric",
"x":0,
"y":12,
"width":12,
"height":6,
"properties":{
"view":"timeSeries",
"stacked":false,
"metrics":[
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Average', 300))", "id": "e8", "label": "cluster disk used percent average", "period": 300 } ],
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} disk_used_percent', 'Maximum', 300))", "id": "e9", "label": "cluster disk used percent maximum", "period": 300 } ]
],
"region":"{region}",
"title":"Single Node Disk Usage (Avg and Max)"
}
},
{
"type":"metric",
"x":12,
"y":18,
"width":12,
"height":6,
"properties": {
"metrics": [
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Average', 300))", "id": "e10", "label": "cluster mem used percent sum", "period": 300 } ]
],
"view": "timeSeries",
"stacked": false,
"region": "{region}",
"stat": "Maximum",
"period": 300,
"start": "-PT2H",
"end": "P0D",
"title": "Cluster Memory Usage"
}
},
{
"type":"metric",
"x":0,
"y":18,
"width":12,
"height":6,
"properties": {
"metrics": [
[ { "expression": "AVG(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Average', 300))", "id": "e11", "label": "cluster mem used percent average", "period": 300 } ],
[ { "expression": "MAX(SEARCH('{{cluster_name}-ray-CWAgent,InstanceId} mem_used_percent', 'Maximum', 300))", "id": "e12", "label": "cluster mem used percent maximum", "period": 300 } ]
],
"view": "timeSeries",
"stacked": false,
"region": "{region}",
"stat": "Maximum",
"period": 300,
"start": "-PT2H",
"end": "P0D",
"title": "Single Node Memory Usage (Avg and Max)"
}
},
{
"height": 6,
"width": 12,
"y": 0,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-prometheus,instance} ray_node_cpu_count', 'Maximum', 300))", "label": "cluster cpu sum", "id": "e13" } ]
],
"view": "timeSeries",
"stacked": false,
"region": "{region}",
"stat": "Maximum",
"period": 300,
"start": "-PT2H",
"end": "P0D",
"title": "Cluster CPUs"
}
},
{
"height": 6,
"width": 12,
"y": 0,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "SUM(SEARCH('{{cluster_name}-ray-prometheus,instance} object_store_available_memory', 'Average', 300))", "label": "cluster object store available memory sum", "id": "e14" } ]
],
"view": "timeSeries",
"stacked": false,
"region": "{region}",
"stat": "Maximum",
"period": 300,
"start": "-PT2H",
"end": "P0D",
"title": "Cluster Object Store Available Memory"
}
}
]
python/ray/autoscaler/aws/cloudwatch/prometheus.yml
+15
View File
@@ -0,0 +1,15 @@
# Prometheus config file
# my global config
global:
scrape_interval: 10s
evaluation_interval: 10s
scrape_timeout: 10s
# use ray file-based service discovery file as scrape target.
scrape_configs:
- job_name: 'ray'
file_sd_configs:
- files:
- '/tmp/ray/prom_metrics_service_discovery.json'
refresh_interval: 1m
python/ray/autoscaler/aws/cloudwatch/ray_prometheus_waiter.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
MAX_ATTEMPTS=120
DELAY_SECONDS=10
RAY_PROM_METRICS_FILE_PATH="/tmp/ray/prom_metrics_service_discovery.json"
CLUSTER_NAME=$1
while [ $MAX_ATTEMPTS -gt 0 ]; do
if [ -f $RAY_PROM_METRICS_FILE_PATH ]; then
echo "Ray Prometheus metrics service discovery file found at: $RAY_PROM_METRICS_FILE_PATH."
echo "Restarting cloudwatch agent.This may take a few minutes..."
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -m ec2 -a stop
echo "Cloudwatch agent stopped, starting cloudwatch agent..."
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:AmazonCloudWatch-ray_agent_config_$CLUSTER_NAME"
echo "Cloudwatch agent successfully restarted!"
exit 0
else
echo "Ray Prometheus metrics service discovery file not found at: $RAY_PROM_METRICS_FILE_PATH. Will check again in $DELAY_SECONDS seconds..."
sleep $DELAY_SECONDS
MAX_ATTEMPTS=$((MAX_ATTEMPTS-1))
fi
done
echo "Ray Prometheus metrics service discovery file not found at: $RAY_PROM_METRICS_FILE_PATH. Ray system metrics will not be available in CloudWatch."
exit 1
python/ray/autoscaler/aws/defaults.yaml
+144
View File
@@ -0,0 +1,144 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: default
# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: 2
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# This executes all commands on all nodes in the docker container,
# and opens all the necessary ports to support the Ray cluster.
# Empty string means disabled.
docker: {}
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# Availability zone(s), comma-separated, that nodes may be launched in.
# Nodes will be launched in the first listed availability zone and will
# be tried in the subsequent availability zones if launching fails.
availability_zone: us-west-2a,us-west-2b
# Whether to allow node reuse. If set to False, nodes will be terminated
# instead of stopped.
cache_stopped_nodes: True # If not present, the default is True.
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# By default Ray creates a new private keypair, but you can also use your own.
# If you do so, make sure to also set "KeyName" in the head and worker node
# configurations below.
# ssh_private_key: /path/to/your/key.pem
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
ray.head.default:
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
# You can provision additional disk space with a conf as follows
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 256
# Additional options in the boto docs.
ray.worker.default:
# The minimum number of nodes of this type to launch.
# This number should be >= 0.
min_workers: 0
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
# Run workers on spot by default. Comment this out to use on-demand.
InstanceMarketOptions:
MarketType: spot
# Additional options can be found in the boto docs, e.g.
# SpotOptions:
# MaxPrice: MAX_HOURLY_PRICE
# Additional options in the boto docs.
# Specify the node type of the head node (as configured above).
head_node_type: ray.head.default
# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {
# "/path1/on/remote/machine": "/path1/on/local/machine",
# "/path2/on/remote/machine": "/path2/on/local/machine",
}
# Files or directories to copy from the head node to the worker nodes. The format is a
# list of paths. The same path on the head node will be copied to the worker node.
# This behavior is a subset of the file_mounts behavior. In the vast majority of cases
# you should just use file_mounts. Only use this if you know what you're doing!
cluster_synced_files: []
# Whether changes to directories in file_mounts or cluster_synced_files in the head node
# should sync to the worker node continuously
file_mounts_sync_continuously: False
# Patterns for files to exclude when running rsync up or rsync down
rsync_exclude: []
# Pattern files to use for filtering out files when running rsync up or rsync down. The file is searched for
# in the source directory and recursively through all subdirectories. For example, if .gitignore is provided
# as a value, the behavior will match git's behavior for finding and using .gitignore files.
rsync_filter: []
# List of commands that will be run before `setup_commands`. If docker is
# enabled, these commands will run outside the container and before docker
# is setup.
initialization_commands: []
# List of shell commands to run to set up nodes.
setup_commands:
- >-
(stat $HOME/anaconda3/envs/tensorflow2_p310/ &> /dev/null &&
echo 'export PATH="$HOME/anaconda3/envs/tensorflow2_p310/bin:$PATH"' >> ~/.bashrc) || true
- which ray || pip install -U "ray[default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl"
# Custom commands that will be run on the head node after common setup.
head_setup_commands:
- pip install 'boto3>=1.4.8' # 1.4.8 adds InstanceMarketOptions
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host=0.0.0.0
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
python/ray/autoscaler/aws/development-example.yaml
+73
View File
@@ -0,0 +1,73 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: development
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
# Provider-specific config for the head node, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
ray.head.default:
node_config:
InstanceType: m4.16xlarge
ImageId: ami-0def3275 # Default Ubuntu 16.04 AMI.
# Set primary volume to 25 GiB
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 50
# List of shell commands to install ray from source.
setup_commands:
# Consider uncommenting these if you run into dpkg locking issues
# - sudo pkill -9 apt-get || true
# - sudo pkill -9 dpkg || true
# - sudo dpkg --configure -a
# Install basics.
- sudo apt-get update
- sudo apt-get install -y build-essential curl unzip
# Install Node.js in order to build the dashboard.
- curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash
- sudo apt-get install -y nodejs
# Install Anaconda.
- wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true
- bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
# Build Ray.
- git clone https://github.com/ray-project/ray || true
- ray/ci/env/install-bazel.sh
- cd ray/python/ray/dashboard/client; npm ci; npm run build
- pip install boto3>=1.4.8 cython==0.29.37 aiohttp grpcio psutil setproctitle
- cd ray/python; pip install -e . --verbose
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --head --port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379
python/ray/autoscaler/aws/example-cloudwatch.yaml
+112
View File
@@ -0,0 +1,112 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: cloudwatch
# The maximum number of workers nodes to launch in addition to the head node.
max_workers: 2
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
# Start by defining a `cloudwatch` section to enable CloudWatch integration with your Ray cluster.
cloudwatch:
# We depend on AWS Systems Manager (SSM) to deploy CloudWatch configuration updates to your cluster,
# with relevant configuration created or updated in the SSM Parameter Store during `ray up`.
# We support three CloudWatch related config type under this cloudwatch section: agent, dashboard and alarm.
# The `AmazonCloudWatch-ray_{config_type}_config_{cluster_name}` SSM Parameter Store Config Key is used to
# store a remote cache of the last Unified CloudWatch config applied.
# Every time you run `ray up` to update your cluster, we compare your local CloudWatch config file contents
# to the SSM Parameter Store's contents for that config and, if they differ, then the associated CloudWatch
# config will be applied and uploaded to the SSM Parameter Store.
# For CloudWatch Unified Agent config files, we will also replace references to
# `{instance_id}` with your head node's EC2 instance ID, `{region}` with your cluster's region name, and `{cluster_name}` with your cluster name.
agent:
# The Unified CloudWatch Agent is configured via the config file described
# at https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html.
# We've configured our `example-cloudwatch-agent-config.json` file to ship the following log files to CloudWatch:
# 1. `/tmp/ray/session_*/logs/**.out` are shipped to the `{cluster_name}-ray_logs_out` CloudWatch Log Group.
# 2. `/tmp/ray/session_*/logs/**.err` are shipped to the `{cluster_name}-ray_logs_err` CloudWatch Log Group.
# If enabled, Prometheus metrics can be found in the CloudWatch > Metrics > `{cluster-name}-ray-prometheus` namespace.
# CloudWatch Log Stream names will be the same as your cluster head node's EC2 instance ID.
# See https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html for ray logging system details.
# Path to Unified CloudWatch Agent config file
config: "cloudwatch/example-cloudwatch-agent-config.json"
retryer:
# Max allowed Unified CloudWatch Agent SSM config update attempts on any host.
max_attempts: 120
# Seconds to wait between each Unified CloudWatch Agent SSM config update attempt.
delay_seconds: 30
# For CloudWatch Dashboard config files, we will also replace references to
# `{region}` with your cluster's region name, and `{cluster_name}` with your cluster name.
dashboard:
# CloudWatch Dashboard name
# Per-cluster level dashboard is created and dashboard name will be
# `{your_cluster_name}-example-dashboard-name` as default
name: "example-dashboard-name"
# The CloudWatch Dashboard is defined via the config file described
# at https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Dashboard-Body-Structure.html.
# Path to the CloudWatch Dashboard config file
config: "cloudwatch/example-cloudwatch-dashboard-config.json"
# For CloudWatch Alarm config files, we will also replace references to
# `{instance_id}` with every cluster node's EC2 instance ID, `{region}` with your cluster's region name, and `{cluster_name}` with your cluster name.
alarm:
# The CloudWatch Alarm config file is defined via the config file described
# at https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_PutMetricAlarm.html.
# To allow per-node alarm being created and updated, `{instance_id}` is included as part of `AlarmName` in the following json config.
# Replace `AlarmActions` in the `example-cloudwatch-alarm-config.json` with actions you want to take described at
# https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html
# Path to CloudWatch Alarm config file
config: "cloudwatch/example-cloudwatch-alarm-config.json"
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
available_node_types:
ray.head.default:
node_config:
InstanceType: c5a.large
# Disclaimer: CloudWatch integration with Ray requires an AMI (or Docker image) with the Unified CloudWatch Agent pre-installed.
# The AMI below is provided by the Amazon Ray Team, is based on version 48 of the Ubuntu 18.04 AWS Deep Learning AMI,
# and ships with Unified CloudWatch Agent v1.247348.0b251302.
# Please direct any questions, comments, or issues to the Amazon Ray Team at https://github.com/amzn/amazon-ray/issues/new/choose.
# Up-to-date versions of this AMI and AMIs for other regions can be found at https://github.com/amzn/amazon-ray.
ImageId: ami-0d88d9cbe28fac870
resources: {}
ray.worker.default:
node_config:
InstanceType: c5a.large
ImageId: ami-0d88d9cbe28fac870
# Note: IamInstanceProfile is needed to grant worker nodes required permission to make boto3 call for Cloudwatch setup.
# Default IamInstanceProfile is `arn:aws:iam::{your_aws_account_number}:instance-profile/ray-autoscaler-cloudwatch-v1`.
IamInstanceProfile:
Name: ray-autoscaler-cloudwatch-v1
resources: {}
min_workers: 0
max_workers: 2
head_node_type: ray.head.default
# If you want to export Ray's Prometheus system metrics to CloudWatch, you should first ensure that your cluster has the
# Unified CloudWatch Agent and Ray Dashboard installed, then uncomment the `head_setup_commands` section below.
# Note that this relies on CloudWatch's Embedded Metric Format (EMF), and will thus incur CloudWatch log and metric costs in
# accordance with https://aws.amazon.com/cloudwatch/pricing/. Also note that we use the following files to enable this feature:
# 1. prometheus.yml: The configuration file that tells Prometheus which metrics to scrape. In this case, we've configured
# it to scrape all available Ray system metrics. For more information, see:
# https://prometheus.io/docs/prometheus/latest/configuration/configuration/.
# 2. ray_prometheus_waiter.sh: A bash script that waits for the Ray Prometheus service discovery file to appear at
# `/tmp/ray/prom_metrics_service_discovery.json`, then restarts the CloudWatch Agent to start capturing all scraped
# Prometheus metrics.
# See https://docs.ray.io/en/latest/ray-metrics.html for more details about exporting Ray Prometheus metrics.
#head_setup_commands:
# # Make `ray_prometheus_waiter.sh` executable.
# - RAY_INSTALL_DIR=`pip show ray | grep -Po "(?<=Location:).*"` && sudo chmod +x $RAY_INSTALL_DIR/ray/autoscaler/aws/cloudwatch/ray_prometheus_waiter.sh
# # Copy `prometheus.yml` to Unified CloudWatch Agent folder
# - RAY_INSTALL_DIR=`pip show ray | grep -Po "(?<=Location:).*"` && sudo cp -f $RAY_INSTALL_DIR/ray/autoscaler/aws/cloudwatch/prometheus.yml /opt/aws/amazon-cloudwatch-agent/etc
# # First get current cluster name, then let the Unified CloudWatch Agent restart and use `AmazonCloudWatch-ray_agent_config_{cluster_name}` parameter at SSM Parameter Store.
# - nohup sudo sh -c "`pip show ray | grep -Po "(?<=Location:).*"`/ray/autoscaler/aws/cloudwatch/ray_prometheus_waiter.sh `cat ~/ray_bootstrap_config.yaml | jq '.cluster_name'` >> '/opt/aws/amazon-cloudwatch-agent/logs/ray_prometheus_waiter.out' 2>> '/opt/aws/amazon-cloudwatch-agent/logs/ray_prometheus_waiter.err'" &
python/ray/autoscaler/aws/example-full.yaml
+178
View File
@@ -0,0 +1,178 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: default
# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: 2
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# This executes all commands on all nodes in the docker container,
# and opens all the necessary ports to support the Ray cluster.
# Empty string means disabled.
docker:
image: "rayproject/ray-ml:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup
# image: rayproject/ray:latest-cpu # use this one if you don't need ML dependencies, it's faster to pull
container_name: "ray_container"
# If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
# if no cached version is present.
pull_before_run: True
run_options: # Extra options to pass into "docker run"
- --ulimit nofile=65536:65536
# Example of running a GPU head with CPU workers
# head_image: "rayproject/ray-ml:latest-gpu"
# Allow Ray to automatically detect GPUs
# worker_image: "rayproject/ray-ml:latest-cpu"
# worker_run_options: []
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# Availability zone(s), comma-separated, that nodes may be launched in.
# Nodes will be launched in the first listed availability zone and will
# be tried in the subsequent availability zones if launching fails.
availability_zone: us-west-2a,us-west-2b
# Whether to allow node reuse. If set to False, nodes will be terminated
# instead of stopped.
cache_stopped_nodes: True # If not present, the default is True.
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# By default Ray creates a new private keypair, but you can also use your own.
# If you do so, make sure to also set "KeyName" in the head and worker node
# configurations below.
# ssh_private_key: /path/to/your/key.pem
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
ray.head.default:
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
# Default AMI for us-west-2.
# Check https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/_private/aws/config.py
# for default images for other zones.
ImageId: ami-0387d929287ab193e
# You can provision additional disk space with a conf as follows
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 140
VolumeType: gp3
# Additional options in the boto docs.
ray.worker.default:
# The minimum number of worker nodes of this type to launch.
# This number should be >= 0.
min_workers: 1
# The maximum number of worker nodes of this type to launch.
# This takes precedence over min_workers.
max_workers: 2
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
# Default AMI for us-west-2.
# Check https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/_private/aws/config.py
# for default images for other zones.
ImageId: ami-0387d929287ab193e
# Run workers on spot by default. Comment this out to use on-demand.
# NOTE: If relying on spot instances, it is best to specify multiple different instance
# types to avoid interruption when one instance type is experiencing heightened demand.
# Demand information can be found at https://aws.amazon.com/ec2/spot/instance-advisor/
InstanceMarketOptions:
MarketType: spot
# Additional options can be found in the boto docs, e.g.
# SpotOptions:
# MaxPrice: MAX_HOURLY_PRICE
# Additional options in the boto docs.
# Specify the node type of the head node (as configured above).
head_node_type: ray.head.default
# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {
# "/path1/on/remote/machine": "/path1/on/local/machine",
# "/path2/on/remote/machine": "/path2/on/local/machine",
}
# Files or directories to copy from the head node to the worker nodes. The format is a
# list of paths. The same path on the head node will be copied to the worker node.
# This behavior is a subset of the file_mounts behavior. In the vast majority of cases
# you should just use file_mounts. Only use this if you know what you're doing!
cluster_synced_files: []
# Whether changes to directories in file_mounts or cluster_synced_files in the head node
# should sync to the worker node continuously
file_mounts_sync_continuously: False
# Patterns for files to exclude when running rsync up or rsync down
rsync_exclude:
- "**/.git"
- "**/.git/**"
# Pattern files to use for filtering out files when running rsync up or rsync down. The file is searched for
# in the source directory and recursively through all subdirectories. For example, if .gitignore is provided
# as a value, the behavior will match git's behavior for finding and using .gitignore files.
rsync_filter:
- ".gitignore"
# List of commands that will be run before `setup_commands`. If docker is
# enabled, these commands will run outside the container and before docker
# is setup.
initialization_commands: []
# List of shell commands to run to set up nodes.
setup_commands: []
# Note: if you're developing Ray, you probably want to create a Docker image that
# has your Ray repo pre-cloned. Then, you can replace the pip installs
# below with a git checkout <your_sha> (and possibly a recompile).
# To run the nightly version of ray (as opposed to the latest), either use a rayproject docker image
# that has the "nightly" (e.g. "rayproject/ray-ml:nightly-gpu") or uncomment the following line:
# - pip install -U "ray[default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
# Custom commands that will be run on the head node after common setup.
head_setup_commands: []
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
- ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host=0.0.0.0
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
- ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
python/ray/autoscaler/aws/example-gpu-docker.yaml
+136
View File
@@ -0,0 +1,136 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: gpu-docker
# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: 2
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# This executes all commands on all nodes in the docker container,
# and opens all the necessary ports to support the Ray cluster.
# Empty string means disabled.
docker:
image: "rayproject/ray-ml:latest-gpu"
# image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull
container_name: "ray_nvidia_docker" # e.g. ray_docker
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# Availability zone(s), comma-separated, that nodes may be launched in.
# Nodes will be launched in the first listed availability zone and will
# be tried in the subsequent availability zones if launching fails.
availability_zone: us-west-2a,us-west-2b
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# By default Ray creates a new private keypair, but you can also use your own.
# If you do so, make sure to also set "KeyName" in the head and worker node
# configurations below.
# ssh_private_key: /path/to/your/key.pem
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
# GPU head node.
ray.head.gpu:
# worker_image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: p2.xlarge
# Default AMI. Uncomment to use a different AMI.
# ImageId:
# You can provision additional disk space with a conf as follows
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 140
# Additional options in the boto docs.
# CPU workers.
ray.worker.default:
# Override global docker setting.
# This node type will run a CPU image,
# rather than the GPU image specified in the global docker settings.
docker:
worker_image: "rayproject/ray-ml:latest-cpu"
# The minimum number of nodes of this type to launch.
# This number should be >= 0.
min_workers: 1
# The maximum number of workers nodes of this type to launch.
# This takes precedence over min_workers.
max_workers: 2
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
# Default AMI. Uncomment to use a different AMI.
# ImageId:
# Run workers on spot by default. Comment this out to use on-demand.
InstanceMarketOptions:
MarketType: spot
# Additional options can be found in the boto docs, e.g.
# SpotOptions:
# MaxPrice: MAX_HOURLY_PRICE
# Additional options in the boto docs.
# Specify the node type of the head node (as configured above).
head_node_type: ray.head.gpu
# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {
# "/path1/on/remote/machine": "/path1/on/local/machine",
# "/path2/on/remote/machine": "/path2/on/local/machine",
}
# List of shell commands to run to set up nodes.
# NOTE: rayproject/ray:latest has ray latest bundled
setup_commands: []
# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
# - pip install -U "ray[default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
# Custom commands that will be run on the head node after common setup.
head_setup_commands:
- pip install boto3>=1.4.8 # 1.4.8 adds InstanceMarketOptions
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
python/ray/autoscaler/aws/example-head-and-worker-security-group.yaml
+28
View File
@@ -0,0 +1,28 @@
cluster_name: sg
max_workers: 1
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
auth:
ssh_user: ubuntu
# If required, head and worker nodes can exist on subnets in different VPCs and
# communicate via VPC peering.
# VPC peering overview: https://docs.aws.amazon.com/vpc/latest/userguide/vpc-peering.html.
# Setup VPC peering: https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html.
# Configure VPC peering route tables: https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html.
available_node_types:
ray.head.default:
node_config:
SecurityGroupIds:
- sg-1234abcd # Replace with an actual security group id.
ray.worker.default:
node_config:
SecurityGroupIds:
- sg-1234abcd # Replace with an actual security group id.
python/ray/autoscaler/aws/example-java.yaml
+75
View File
@@ -0,0 +1,75 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: java
# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 1
# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers.
max_workers: 1
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# Availability zone(s), comma-separated, that nodes may be launched in.
# Nodes will be launched in the first listed availability zone and will
# be tried in the subsequent availability zones if launching fails.
availability_zone: us-west-2a,us-west-2b
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
ray.head.default:
node_config:
InstanceType: m4.4xlarge
ImageId: ami-06d51e91cea0dac8d # Ubuntu 18.04
# You can provision additional disk space with a conf as follows
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 20
# Additional options in the boto docs.
ray.worker.default:
node_config:
InstanceType: m4.4xlarge
ImageId: ami-06d51e91cea0dac8d # Ubuntu 18.04
# Run workers on spot by default. Comment this out to use on-demand.
InstanceMarketOptions:
MarketType: spot
# Additional options can be found in the boto docs, e.g.
# SpotOptions:
# MaxPrice: MAX_HOURLY_PRICE
# List of commands that will be run before `setup_commands`. If docker is
# enabled, these commands will run outside the container and before docker
# is setup.
initialization_commands:
- sudo apt-get update
- sudo apt-get install -y maven
- git clone https://github.com/wuisawesome/ray-word-count.git || (pushd ray-word-count; git pull; popd)
- pushd ray-word-count; mvn clean package; popd
- cp -rv ray-word-count/files ./
# List of shell commands to run to set up nodes.
setup_commands:
- sudo apt-get install -y python3 python3-pip
- python3 -m pip install --upgrade pip
- python3 -m pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
# Custom commands that will be run on the head node after common setup.
head_setup_commands:
- python3 -m pip install boto3>=1.4.8 # 1.4.8 adds InstanceMarketOptions
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- ray stop
- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
# To run the program, run `ray exec java.yaml "java -jar ray-word-count/target/ray-word-count-1.0-SNAPSHOT-jar-with-dependencies.jar -Dray.job.code-search-path=ray-word-count/target"`
python/ray/autoscaler/aws/example-launch-templates.yaml
+60
View File
@@ -0,0 +1,60 @@
cluster_name: launch_templates
max_workers: 2
provider:
type: aws
region: us-west-2
# Note that availability zones can be omitted when using custom launch
# templates that contain either pre-configured availability zones or custom
# network interfaces for all node types, since each node will always be
# launched in either the launch template's AZ or the AZ shared by its
# network interface subnets.
# If some of your node types have launch templates binding them to AZs and
# others do not, then node types without AZ bindings will be limited to
# launching only in subnets available in the below availability zones:
availability_zone: us-west-2a, us-west-2b, us-west-2c
auth:
ssh_user: ubuntu
# You can use EC2 launch templates to consolidate, re-use, and version common
# node configurations.
# For more information, see the documentation on EC2 Launch Templates at:
# https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html
available_node_types:
ray.head.default:
resources: {}
node_config:
# The launch template to use to launch the instances. Any parameters that
# you specify in node_config override the same parameters in the launch
# template. Tags will be merged by key, with node_config values overriding
# launch template values for the same key. You can specify either the name
# or ID of a launch template, but not both.
LaunchTemplate:
LaunchTemplateId: lt-00000000000000000
# Launch template versions can be set to a version number, "$Default"
# for the default launch template version, or "$Latest" for the latest
# launch template version. If the version is omitted, it will
# automatically resolve to the launch template's default version.
Version: $Latest
ImageId: latest_dlami
InstanceType: m5.large
ray.worker.default:
min_workers: 0
max_workers: 1
resources: {}
node_config:
LaunchTemplate:
LaunchTemplateName: ExampleLaunchTemplate
Version: 2
ImageId: latest_dlami
InstanceType: m5.large
head_node_type: ray.head.default
python/ray/autoscaler/aws/example-minimal.yaml
+48
View File
@@ -0,0 +1,48 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: aws-example-minimal
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# The maximum number of workers nodes to launch in addition to the head
# node.
max_workers: 3
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
ray.head.default:
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g., instance type. By default
# Ray auto-configures unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
ray.worker.default:
# The minimum number of worker nodes of this type to launch.
# This number should be >= 0.
min_workers: 3
# The maximum number of worker nodes of this type to launch.
# This parameter takes precedence over min_workers.
max_workers: 3
# The node type's CPU and GPU resources are auto-detected based on AWS instance type.
# If desired, you can override the autodetected CPU and GPU resources advertised to the autoscaler.
# You can also set custom resources.
# For example, to mark a node type as having 1 CPU, 1 GPU, and 5 units of a resource called "custom", set
# resources: {"CPU": 1, "GPU": 1, "custom": 5}
resources: {}
# Provider-specific config for this node type, e.g., instance type. By default
# Ray auto-configures unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
node_config:
InstanceType: m5.large
python/ray/autoscaler/aws/example-ml.yaml
+140
View File
@@ -0,0 +1,140 @@
# A cluster setup for ML / RLlib workloads. Note that this uses pytorch by default.
# If you want to use tensorflow, change pytorch_p36 to tensorflow_p36 below.
#
# Important: Make sure to run "source activate pytorch_p36" in your sessions to
# activate the right conda environment. Otherwise you won't be able to import ray.
#
cluster_name: ml
# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 0
# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers.
max_workers: 0
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# This executes all commands on all nodes in the docker container,
# and opens all the necessary ports to support the Ray cluster.
# Empty string means disabled.
docker:
image: "" # e.g., rayproject/ray-ml:latest
container_name: "" # e.g. ray_docker
# If true, pulls latest version of image. Otherwise, `docker run` will only pull the image
# if no cached version is present.
pull_before_run: True
run_options: [] # Extra options to pass into "docker run"
# Example of running a GPU head with CPU workers
# head_image: "rayproject/ray-ml:latest-gpu"
# worker_image: "rayproject/ray-ml:latest"
# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
# Availability zone(s), comma-separated, that nodes may be launched in.
# Nodes will be launched in the first listed availability zone and will
# be tried in the subsequent availability zones if launching fails.
availability_zone: us-west-2a,us-west-2b
# Whether to allow node reuse. If set to False, nodes will be terminated
# instead of stopped.
cache_stopped_nodes: True # If not present, the default is True.
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
# By default Ray creates a new private keypair, but you can also use your own.
# If you do so, make sure to also set "KeyName" in the head and worker node
# configurations below.
# ssh_private_key: /path/to/your/key.pem
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
ray.head.default:
node_config:
InstanceType: m4.16xlarge
ImageId: latest_dlami
# You can provision additional disk space with a conf as follows
BlockDeviceMappings:
- DeviceName: /dev/sda1
Ebs:
VolumeSize: 140
ray.worker.default:
node_config:
InstanceType: m4.16xlarge
ImageId: latest_dlami
# Comment this in to use spot nodes.
# InstanceMarketOptions:
# MarketType: spot
# # Additional options can be found in the boto docs, e.g.
# # SpotOptions:
# # MaxPrice: MAX_HOURLY_PRICE
#
# Additional options in the boto docs.
# Files or directories to copy to the head and worker nodes. The format is a
# dictionary from REMOTE_PATH: LOCAL_PATH, e.g.
file_mounts: {
# "/path1/on/remote/machine": "/path1/on/local/machine",
# "/path2/on/remote/machine": "/path2/on/local/machine",
}
# Files or directories to copy from the head node to the worker nodes. The format is a
# list of paths. The same path on the head node will be copied to the worker node.
# This behavior is a subset of the file_mounts behavior. In the vast majority of cases
# you should just use file_mounts. Only use this if you know what you're doing!
cluster_synced_files: []
# Whether changes to directories in file_mounts or cluster_synced_files in the head node
# should sync to the worker node continuously
file_mounts_sync_continuously: False
# List of commands that will be run before `setup_commands`. If docker is
# enabled, these commands will run outside the container and before docker
# is setup.
initialization_commands: []
# List of shell commands to run to set up nodes.
setup_commands:
# Note: if you're developing Ray, you probably want to create an AMI that
# has your Ray repo pre-cloned. Then, you can replace the pip installs
# below with a git checkout <your_sha> (and possibly a recompile).
- source activate pytorch_p36 && pip install -U ray
- source activate pytorch_p36 && pip install -U ray[rllib] ray[tune] ray
# Consider uncommenting these if you also want to run apt-get commands during setup
# - sudo pkill -9 apt-get || true
# - sudo pkill -9 dpkg || true
# - sudo dpkg --configure -a
# Custom commands that will be run on the head node after common setup.
head_setup_commands:
- pip install 'boto3>=1.4.8' # 1.4.8 adds InstanceMarketOptions
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
- source activate pytorch_p36 && ray stop
- ulimit -n 65536; source activate pytorch_p36 && ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
- source activate pytorch_p36 && ray stop
- ulimit -n 65536; source activate pytorch_p36 && ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
python/ray/autoscaler/aws/example-multi-node-type.yaml
+65
View File
@@ -0,0 +1,65 @@
# Experimental: an example of configuring a mixed-node-type cluster.
cluster_name: multi_node_type
max_workers: 40
# The autoscaler will scale up the cluster faster with higher upscaling speed.
# E.g., if the task requires adding more nodes then autoscaler will gradually
# scale up the cluster in chunks of upscaling_speed*currently_running_nodes.
# This number should be > 0.
upscaling_speed: 1.0
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
# Tell the autoscaler the allowed node types and the resources they provide.
# The key is the name of the node type, which is just for debugging purposes.
# The node config specifies the launch config and physical instance type.
available_node_types:
cpu_4_ondemand:
node_config:
InstanceType: m4.xlarge
ImageId: latest_dlami
# For AWS instances, autoscaler will automatically add the available
# CPUs/GPUs/accelerator_type ({"CPU": 4} for m4.xlarge) in "resources".
# resources: {"CPU": 4}
min_workers: 1
max_workers: 5
cpu_16_spot:
node_config:
InstanceType: m4.4xlarge
ImageId: latest_dlami
InstanceMarketOptions:
MarketType: spot
# Autoscaler will auto fill the CPU resources below.
resources: {"Custom1": 1, "is_spot": 1}
max_workers: 10
gpu_1_ondemand:
node_config:
InstanceType: p2.xlarge
ImageId: latest_dlami
# Autoscaler will auto fill the CPU/GPU resources below.
resources: {"Custom2": 2}
max_workers: 4
worker_setup_commands:
- pip install tensorflow-gpu # Example command.
gpu_8_ondemand:
node_config:
InstanceType: p3.8xlarge
ImageId: latest_dlami
# Autoscaler autofills the "resources" below.
# resources: {"CPU": 32, "GPU": 4, "accelerator_type:V100": 1}
max_workers: 2
worker_setup_commands:
- pip install tensorflow-gpu # Example command.
# Specify the node type of the head node (as configured above).
head_node_type: cpu_4_ondemand
idle_timeout_minutes: 2
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
python/ray/autoscaler/aws/example-network-interfaces.yaml
+128
View File
@@ -0,0 +1,128 @@
cluster_name: network_interfaces
max_workers: 2
provider:
type: aws
# Ensure that all Security Group IDs associated with your network interfaces
# below are available in this region. If you are using Elastic Fabric
# Adaptors (EFA) with your network interfaces, then ensure that the
# available instance types in this region support EFA. To see the available
# instance types that support EFA in a Region, use the
# describe-instance-types command with the --region option and the
# appropriate Region code:
# aws ec2 describe-instance-types --region us-east-2 --filters Name=network-info.efa-supported,Values=true --query "InstanceTypes[*].[InstanceType]" --output text
region: us-west-2
# Note that availability zones can be omitted when using custom network
# interfaces with all node types, since each node will always be launched in
# the availability zone shared by its network interface subnets.
# If some of your node types have network interfaces configured and others
# do not, then node types without network interfaces will be limited to
# launching only in subnets available in the given availability zones.
# availability_zone: us-west-2a, us-west-2b, us-west-2c
# The example network interfaces below don't associate public IP addresses
# with Ray cluster nodes, so we need to explicitly tell Ray to connect to
# them via their private IP addresses. This also means that any instance
# running "ray up" or otherwise communicating with cluster nodes must be
# located in the same VPC to succeed. This line should be omitted if your
# network interfaces use public IP addresses.
use_internal_ips: True
auth:
ssh_user: ubuntu
# One or more NetworkInterfaces may be optionally defined for both head and
# worker nodes. Each NetworkInterface must minimally contain an associated
# DeviceIndex, SubnetID, and SecurityGroupID.
# For more information, see the "NetworkInterfaces" parameter of:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
available_node_types:
ray.head.default:
resources: {}
node_config:
NetworkInterfaces:
- DeviceIndex: 0 # Primary network interface.
SubnetId: subnet-0000000 # Replace with your Subnet ID.
# Head node network interfaces can optionally associate fixed private
# addresses with the head node.
PrivateIpAddress: 172.31.64.10 # Replace with an IP in your subnet.
Groups:
- sg-00000000 # Replace with your Security Group ID.
# Multiple network interfaces can optionally be attached to a single
# node. Each interface can be assigned a different subnet, but each
# subnet should be in the same availability zone.
# For more information, see:
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html
# When assigning multiple network interfaces to a node, the network
# interfaces CANNOT have associated public IP addresses.
- DeviceIndex: 1 # Secondary network interface.
SubnetId: subnet-11111111 # Replace with your Subnet ID.
PrivateIpAddress: 172.31.16.10 # Replace with an IP in your subnet.
Groups:
- sg-11111111 # Replace with your Security Group ID.
- DeviceIndex: 2 # Tertiary network interface.
SubnetId: subnet-11111111 # (Same as deviceIndex-1)
PrivateIpAddress: 172.31.16.11 # Replace with an IP in your subnet.
Groups:
- sg-11111111 # (Same as deviceIndex-1)
# Use any node and instance type with default network interface types.
ImageId: latest_dlami
InstanceType: m5.large
ray.worker.efa:
min_workers: 0
max_workers: 1
resources: {}
node_config:
# Worker node network interfaces should always use auto-assigned private
# IP addresses from their associated subnets to avoid conflicts between
# multiple workers trying to use the same private IP.
NetworkInterfaces:
- DeviceIndex: 0 # Primary network interface.
NetworkCardIndex: 0 # NetworkCard index else defaults to ZERO by ec2 API
AssociatePublicIpAddress: False # Omit to let your Subnet auto-assign a public IP (if enabled).
SubnetId: subnet-22222222 # Replace with your actual Subnet ID
Groups:
- sg-22222222 # Replace with your actual Security Group ID.
InterfaceType: efa # Use EFA for higher throughput and lower latency.
- DeviceIndex: 1 # Secondary interface.
NetworkCardIndex: 1 # NetworkCard index else defaults to ZERO by ec2 API
AssociatePublicIpAddress: False # Omit to let your Subnet auto-assign a public IP (if enabled).
SubnetId: subnet-22222222 # (Must be same AZ, subnetId can be same)
Groups:
- sg-22222222 # (Must be self-referenced with ALL traffic)
InterfaceType: efa # Use EFA for higher throughput and lower latency.
# Use an AMI and instance type that supports Elastic Fabric Adapters (EFA).
# When using EFA, ideally all your cluster nodes should be Linux instances
# in the same subnet. This allows your EFA interfaces to leverage their
# OS-bypass capabilities and communicate directly with the device. See:
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html
# In this case, we'll use EFA with NCCL on the latest Ubuntu Deep Learning
# AMI and a supported network-optimized GPU instance type.
# See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start-nccl-dlami.html
ImageId: latest_dlami
InstanceType: p3dn.24xlarge
ray.worker.default:
min_workers: 0
max_workers: 1
resources: {}
node_config:
NetworkInterfaces:
- DeviceIndex: 0 # Primary network interface.
AssociatePublicIpAddress: False # Omit to let your Subnet auto-assign a public IP (if enabled).
SubnetId: subnet-33333333 # Replace with your actual Subnet ID
Groups:
- sg-33333333 # Replace with your actual Security Group ID.
ImageId: latest_dlami
InstanceType: m5.large
head_node_type: ray.head.default
python/ray/autoscaler/aws/example-security-group.yaml
+29
View File
@@ -0,0 +1,29 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: minimal
# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers. min_workers default to 0.
max_workers: 1
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
# Security group to create with custom in bound rules and name.
security_group:
GroupName: test_security_group_name
IpPermissions:
- FromPort: 443
ToPort: 443
IpProtocol: TCP
IpRanges:
- CidrIp: 0.0.0.0/0
- FromPort: 8265
ToPort: 8265
IpProtocol: TCP
IpRanges:
- CidrIp: 0.0.0.0/0
# How Ray will authenticate with newly launched nodes.
auth:
ssh_user: ubuntu
python/ray/autoscaler/aws/example-subnets.yaml
+32
View File
@@ -0,0 +1,32 @@
cluster_name: subnets
max_workers: 1
provider:
type: aws
region: us-west-2
availability_zone: us-west-2a
auth:
ssh_user: ubuntu
# If required, head and worker nodes can exist on subnets in different VPCs and
# communicate via VPC peering.
# VPC peering overview: https://docs.aws.amazon.com/vpc/latest/userguide/vpc-peering.html.
# Setup VPC peering: https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html.
# Configure VPC peering route tables: https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html.
available_node_types:
ray.head.default:
node_config:
# To enable external SSH connectivity, you should also ensure that your VPC
# is configured to assign public IPv4 addresses to every EC2 instance
# assigned to it.
SubnetIds:
- subnet-0000000 # Replace with your actual Head Node Subnet ID.
ray.worker.default:
node_config:
SubnetIds:
- subnet-fffffff # Replace with your actual Worker Node Subnet ID.
python/ray/autoscaler/aws/tests/aws_cluster.yaml
+22
View File
@@ -0,0 +1,22 @@
# An unique identifier for the head node and workers of this cluster.
cluster_name: nightly-test-minimal
max_workers: 1
idle_timeout_minutes: 2
# Cloud-provider specific configuration.
provider:
type: aws
region: us-west-2
cache_stopped_nodes: False
available_node_types:
ray.head.default:
resources: {}
node_config:
InstanceType: t3.large
ray.worker.default:
resources: {}
min_workers: 1
max_workers: 1
node_config:
InstanceType: t3.large
python/ray/autoscaler/aws/tests/aws_compute.yaml
+16
View File
@@ -0,0 +1,16 @@
cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
region: us-west-2
advanced_configurations_json:
IamInstanceProfile: {"Name": "ray-autoscaler-v1"}
head_node_type:
name: head_node
instance_type: t3.large
worker_node_types:
- name: worker_node
instance_type: t3.large
min_workers: 0
max_workers: 0
use_spot: false
python/ray/autoscaler/aws/tests/aws_config.yaml
+13
View File
@@ -0,0 +1,13 @@
base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] | default("anyscale/ray:nightly-py39") }}
debian_packages: []
env_vars:
RAY_WHEEL_URL: {{ env["RAY_WHEELS"] | default("") }}
python:
pip_packages: []
conda_packages: []
post_build_cmds:
- pip3 uninstall -y ray && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
- pip3 install -U ray[default]
- {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}