# An unique identifier for the head node and workers of this cluster. cluster_name: gpu-docker # The maximum number of workers nodes to launch in addition to the head # node. max_workers: 2 # The autoscaler will scale up the cluster faster with higher upscaling speed. # E.g., if the task requires adding more nodes then autoscaler will gradually # scale up the cluster in chunks of upscaling_speed*currently_running_nodes. # This number should be > 0. upscaling_speed: 1.0 # This executes all commands on all nodes in the docker container, # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: image: "rayproject/ray:latest-gpu" # image: rayproject/ray-ml:latest-gpu # use this one if you need ML dependencies, but it's slower to pull container_name: "ray_nvidia_docker" # e.g. ray_docker # # Example of running a GPU head with CPU workers # head_image: "rayproject/ray-ml:latest-gpu" # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 # Cloud-provider specific configuration. provider: type: gcp region: us-west1 availability_zone: us-west1-b project_id: null # Replace this with your globally unique project id # How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu # By default Ray creates a new private keypair, but you can also use your own. # If you do so, make sure to also set "KeyName" in the head and worker node # configurations below. This requires that you have added the key into the # project wide meta-data. # ssh_private_key: /path/to/your/key.pem # Tell the autoscaler the allowed node types and the resources they provide. # The key is the name of the node type, which is just for debugging purposes. # The node config specifies the launch config and physical instance type. available_node_types: ray_head_gpu: # The resources provided by this node type. resources: {"CPU": 6, "GPU": 1} # Provider-specific config for the head node, e.g. instance type. By default # Ray will auto-configure unspecified fields such as subnets and ssh-keys. # For more documentation on available fields, see: # https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert node_config: machineType: n1-standard-2 disks: - boot: true autoDelete: true type: PERSISTENT initializeParams: diskSizeGb: 50 # See https://cloud.google.com/compute/docs/images for more images sourceImage: projects/ml-images/global/images/c0-deeplearning-common-cu121-v20231209-debian-11 # Make sure to set scheduling->onHostMaintenance to TERMINATE when GPUs are present guestAccelerators: - acceleratorType: nvidia-tesla-t4 acceleratorCount: 1 metadata: items: - key: install-nvidia-driver value: "True" scheduling: - onHostMaintenance: TERMINATE ray_worker_gpu: # The minimum number of nodes of this type to launch. # This number should be >= 0. min_workers: 0 # The maximum number of workers nodes of this type to launch. # This takes precedence over min_workers. max_workers: 2 # The resources provided by this node type. resources: {"CPU": 2, "GPU": 1} # Provider-specific config for the head node, e.g. instance type. By default # Ray will auto-configure unspecified fields such as subnets and ssh-keys. # For more documentation on available fields, see: # https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert node_config: machineType: n1-standard-2 disks: - boot: true autoDelete: true type: PERSISTENT initializeParams: diskSizeGb: 50 # See https://cloud.google.com/compute/docs/images for more images sourceImage: projects/ml-images/global/images/c0-deeplearning-common-cu121-v20231209-debian-11 # Make sure to set scheduling->onHostMaintenance to TERMINATE when GPUs are present guestAccelerators: - acceleratorType: nvidia-tesla-t4 acceleratorCount: 1 metadata: items: - key: install-nvidia-driver value: "True" # Run workers on preemtible instance by default. # Comment this out to use on-demand. scheduling: - preemptible: true - onHostMaintenance: TERMINATE # Specify the node type of the head node (as configured above). head_node_type: ray_head_gpu # Files or directories to copy to the head and worker nodes. The format is a # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", } initialization_commands: # Wait until nvidia drivers are installed - >- timeout 300 bash -c " command -v nvidia-smi && nvidia-smi until [ \$? -eq 0 ]; do command -v nvidia-smi && nvidia-smi done" # List of shell commands to run to set up nodes. # NOTE: rayproject/ray-ml:latest has ray latest bundled setup_commands: [] # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # - pip install -U "ray[default] @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl" # Custom commands that will be run on the head node after common setup. head_setup_commands: - pip install google-api-python-client==1.7.8 # Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - ray stop - >- ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - ray stop - >- ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076