wehub-resource-sync
211 lines
8.5 KiB
YAML
211 lines
8.5 KiB
YAML
# This is adapted from https://github.com/ray-project/kuberay/blob/master/ray-operator/config/samples/ray-cluster.complete.yaml
|
|
# It is a general RayCluster that has most fields in it for maximum flexibility in the Ray/KubeRay integration MVP.
|
|
apiVersion: ray.io/v1alpha1
|
|
kind: RayCluster
|
|
metadata:
|
|
labels:
|
|
controller-tools.k8s.io: "1.0"
|
|
# An unique identifier for the head node and workers of this cluster.
|
|
name: raycluster-complete
|
|
spec:
|
|
rayVersion: '1.12.1'
|
|
# With enableInTreeAutoscaling: true, the operator will insert an autoscaler sidecar container into the Ray head pod.
|
|
enableInTreeAutoscaling: true
|
|
######################headGroupSpecs#################################
|
|
# head group template and specs, (perhaps 'group' is not needed in the name)
|
|
headGroupSpec:
|
|
# Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer'
|
|
serviceType: ClusterIP
|
|
# the pod replicas in this group typed head (assuming there could be more than 1 in the future)
|
|
replicas: 1
|
|
# logical group name, for this called headgroup, also can be functional
|
|
# pod type head or worker
|
|
# rayNodeType: head # Not needed since it is under the headgroup
|
|
# the following params are used to complete the ray start: ray start --head --block --port=6379 ...
|
|
rayStartParams:
|
|
# Flag "no-monitor" must be set when running the autoscaler in
|
|
# a sidecar container.
|
|
no-monitor: "true"
|
|
port: '6379'
|
|
object-manager-port: '9999'
|
|
node-manager-port: '9998'
|
|
object-store-memory: '100000000'
|
|
dashboard-host: '0.0.0.0'
|
|
node-ip-address: $MY_POD_IP # auto-completed as the head pod IP
|
|
block: 'true'
|
|
num-cpus: '1' # can be auto-completed from the limits
|
|
# Use `resources` to optionally specify custom resource annotations for the Ray node.
|
|
# The value of `resources` is a string-integer mapping.
|
|
# Currently, `resources` must be provided in the unfortunate format demonstrated below.
|
|
# Moreover, "CPU" and "GPU" should NOT be included in the `resources` arg.
|
|
# (Use `num-cpus` and `num-gpus` rayStartParams instead.)
|
|
resources: '"{\"Custom1\": 1, \"Custom2\": 5}"'
|
|
#pod template
|
|
template:
|
|
metadata:
|
|
labels:
|
|
# custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller.
|
|
# Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
|
|
rayCluster: raycluster-sample # will be injected if missing
|
|
rayNodeType: head # will be injected if missing, must be head or worker
|
|
groupName: headgroup # will be injected if missing
|
|
# annotations for pod
|
|
annotations:
|
|
key: value
|
|
spec:
|
|
containers:
|
|
# The Ray head pod
|
|
- name: ray-head
|
|
# All Ray pods in the RayCluster should use the same version of Ray.
|
|
image: rayproject/ray:1.12.1
|
|
imagePullPolicy: Always
|
|
# The KubeRay operator uses the ports specified on the ray-head container
|
|
# to configure a service targeting the ports.
|
|
# The name of the service is <ray cluster name>-head-svc.
|
|
ports:
|
|
- containerPort: 6379
|
|
name: gcs
|
|
- containerPort: 8265
|
|
name: dashboard
|
|
- containerPort: 10001
|
|
name: client
|
|
env:
|
|
- name: CPU_REQUEST
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-head
|
|
resource: requests.cpu
|
|
- name: CPU_LIMITS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-head
|
|
resource: limits.cpu
|
|
- name: MEMORY_LIMITS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-head
|
|
resource: limits.memory
|
|
- name: MEMORY_REQUESTS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-head
|
|
resource: requests.memory
|
|
- name: MY_POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
command: ["/bin/sh","-c","ray stop"]
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: "1G"
|
|
requests:
|
|
cpu: "500m"
|
|
memory: "512Mi"
|
|
workerGroupSpecs:
|
|
# the pod replicas in this group typed worker
|
|
- replicas: 1
|
|
minReplicas: 0
|
|
maxReplicas: 300
|
|
# logical group name, for this called small-group, also can be functional
|
|
groupName: small-group
|
|
# if worker pods need to be added, we can simply increment the replicas
|
|
# if worker pods need to be removed, we decrement the replicas, and populate the podsToDelete list
|
|
# the operator will remove pods from the list until the number of replicas is satisfied
|
|
# when a pod is confirmed to be deleted, its name will be removed from the list below
|
|
#scaleStrategy:
|
|
# workersToDelete:
|
|
# - raycluster-complete-worker-small-group-bdtwh
|
|
# - raycluster-complete-worker-small-group-hv457
|
|
# - raycluster-complete-worker-small-group-k8tj7
|
|
# the following params are used to complete the ray start: ray start --block --node-ip-address= ...
|
|
rayStartParams:
|
|
node-ip-address: $MY_POD_IP
|
|
block: 'true'
|
|
# Use `resources` to optionally specify custom resource annotations for the Ray node.
|
|
# The value of `resources` is a string-integer mapping.
|
|
# Currently, `resources` must be provided in the unfortunate format demonstrated below.
|
|
# Moreover, "CPU" and "GPU" should NOT be included in the `resources` arg.
|
|
# (Use `num-cpus` and `num-gpus` rayStartParams instead.)
|
|
resources: '"{\"Custom2\": 5, \"Custom3\": 1}"'
|
|
#pod template
|
|
template:
|
|
metadata:
|
|
labels:
|
|
key: value
|
|
# annotations for pod
|
|
annotations:
|
|
key: value
|
|
spec:
|
|
initContainers:
|
|
# the env var $RAY_IP is set by the operator if missing, with the value of the head service name
|
|
- name: init-myservice
|
|
image: busybox:1.28
|
|
command: ['sh', '-c', "until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"]
|
|
containers:
|
|
- name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc'
|
|
# All Ray pods in the RayCluster should use the same version of Ray.
|
|
image: rayproject/ray:1.12.1
|
|
# environment variables to set in the container.Optional.
|
|
# Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/
|
|
env:
|
|
- name: RAY_DISABLE_DOCKER_CPU_WARNING
|
|
value: "1"
|
|
- name: TYPE
|
|
value: "worker"
|
|
- name: CPU_REQUEST
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-worker
|
|
resource: requests.cpu
|
|
- name: CPU_LIMITS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-worker
|
|
resource: limits.cpu
|
|
- name: MEMORY_LIMITS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-worker
|
|
resource: limits.memory
|
|
- name: MEMORY_REQUESTS
|
|
valueFrom:
|
|
resourceFieldRef:
|
|
containerName: ray-worker
|
|
resource: requests.memory
|
|
- name: MY_POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: MY_POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
ports:
|
|
- containerPort: 80
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
command: ["/bin/sh","-c","ray stop"]
|
|
# use volumeMounts.Optional.
|
|
# Refer to https://kubernetes.io/docs/concepts/storage/volumes/
|
|
volumeMounts:
|
|
- mountPath: /var/log
|
|
name: log-volume
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
memory: "512Mi"
|
|
requests:
|
|
cpu: "500m"
|
|
memory: "256Mi"
|
|
# use volumes
|
|
# Refer to https://kubernetes.io/docs/concepts/storage/volumes/
|
|
volumes:
|
|
- name: log-volume
|
|
emptyDir: {}
|
|
######################status#################################
|