# This is adapted from https://github.com/ray-project/kuberay/blob/master/ray-operator/config/samples/ray-cluster.complete.yaml
# It is a general RayCluster that has most fields in it for maximum flexibility in the Ray/KubeRay integration MVP.
apiVersion: ray.io/v1alpha1
kind: RayCluster
metadata:
  labels:
    controller-tools.k8s.io: "1.0"
    # An unique identifier for the head node and workers of this cluster.
  name: raycluster-complete
spec:
  rayVersion: '1.12.1'
  # With enableInTreeAutoscaling: true, the operator will insert an autoscaler sidecar container into the Ray head pod.
  enableInTreeAutoscaling: true
  ######################headGroupSpecs#################################
  # head group template and specs, (perhaps 'group' is not needed in the name)
  headGroupSpec:
    # Kubernetes Service Type, valid values are 'ClusterIP', 'NodePort' and 'LoadBalancer'
    serviceType: ClusterIP
    # the pod replicas in this group typed head (assuming there could be more than 1 in the future)
    replicas: 1
    # logical group name, for this called headgroup, also can be functional
    # pod type head or worker
    # rayNodeType: head # Not needed since it is under the headgroup
    # the following params are used to complete the ray start: ray start --head --block --port=6379 ...
    rayStartParams:
      # Flag "no-monitor" must be set when running the autoscaler in
      # a sidecar container.
      no-monitor: "true"
      port: '6379'
      object-manager-port: '9999'
      node-manager-port: '9998'
      object-store-memory: '100000000'
      dashboard-host: '0.0.0.0'
      node-ip-address: $MY_POD_IP # auto-completed as the head pod IP
      block: 'true'
      num-cpus: '1' # can be auto-completed from the limits
      # Use `resources` to optionally specify custom resource annotations for the Ray node.
      # The value of `resources` is a string-integer mapping.
      # Currently, `resources` must be provided in the unfortunate format demonstrated below.
      # Moreover, "CPU" and "GPU" should NOT be included in the `resources` arg.
      # (Use `num-cpus` and `num-gpus` rayStartParams instead.)
      resources: '"{\"Custom1\": 1, \"Custom2\": 5}"'
    #pod template
    template:
      metadata:
        labels:
          # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller.
          # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
          rayCluster: raycluster-sample # will be injected if missing
          rayNodeType: head # will be injected if missing, must be head or worker
          groupName: headgroup # will be injected if missing
        # annotations for pod
        annotations:
          key: value
      spec:
        containers:
        # The Ray head pod
        - name: ray-head
          # All Ray pods in the RayCluster should use the same version of Ray.
          image: rayproject/ray:1.12.1
          imagePullPolicy: Always
          # The KubeRay operator uses the ports specified on the ray-head container
          # to configure a service targeting the ports.
          # The name of the service is <ray cluster name>-head-svc.
          ports:
          - containerPort: 6379
            name: gcs
          - containerPort: 8265
            name: dashboard
          - containerPort: 10001
            name: client
          env:
          - name: CPU_REQUEST
            valueFrom:
              resourceFieldRef:
                containerName: ray-head
                resource: requests.cpu
          - name: CPU_LIMITS
            valueFrom:
              resourceFieldRef:
                containerName: ray-head
                resource: limits.cpu
          - name: MEMORY_LIMITS
            valueFrom:
              resourceFieldRef:
                containerName: ray-head
                resource: limits.memory
          - name: MEMORY_REQUESTS
            valueFrom:
              resourceFieldRef:
                containerName: ray-head
                resource: requests.memory
          - name: MY_POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh","-c","ray stop"]
          resources:
            limits:
              cpu: "1"
              memory: "1G"
            requests:
              cpu: "500m"
              memory: "512Mi"
  workerGroupSpecs:
  # the pod replicas in this group typed worker
  - replicas: 1
    minReplicas: 0
    maxReplicas: 300
    # logical group name, for this called small-group, also can be functional
    groupName: small-group
    # if worker pods need to be added, we can simply increment the replicas
    # if worker pods need to be removed, we decrement the replicas, and populate the podsToDelete list
    # the operator will remove pods from the list until the number of replicas is satisfied
    # when a pod is confirmed to be deleted, its name will be removed from the list below
    #scaleStrategy:
    #  workersToDelete:
    #  - raycluster-complete-worker-small-group-bdtwh
    #  - raycluster-complete-worker-small-group-hv457
    #  - raycluster-complete-worker-small-group-k8tj7
    # the following params are used to complete the ray start: ray start --block --node-ip-address= ...
    rayStartParams:
      node-ip-address: $MY_POD_IP
      block: 'true'
      # Use `resources` to optionally specify custom resource annotations for the Ray node.
      # The value of `resources` is a string-integer mapping.
      # Currently, `resources` must be provided in the unfortunate format demonstrated below.
      # Moreover, "CPU" and "GPU" should NOT be included in the `resources` arg.
      # (Use `num-cpus` and `num-gpus` rayStartParams instead.)
      resources: '"{\"Custom2\": 5, \"Custom3\": 1}"'
    #pod template
    template:
      metadata:
        labels:
          key: value
        # annotations for pod
        annotations:
          key: value
      spec:
        initContainers:
        # the env var $RAY_IP is set by the operator if missing, with the value of the head service name
        - name: init-myservice
          image: busybox:1.28
          command: ['sh', '-c', "until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"]
        containers:
        - name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name',  or '123-abc'
          # All Ray pods in the RayCluster should use the same version of Ray.
          image: rayproject/ray:1.12.1
          # environment variables to set in the container.Optional.
          # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/
          env:
          - name:  RAY_DISABLE_DOCKER_CPU_WARNING
            value: "1"
          - name: TYPE
            value: "worker"
          - name: CPU_REQUEST
            valueFrom:
              resourceFieldRef:
                containerName: ray-worker
                resource: requests.cpu
          - name: CPU_LIMITS
            valueFrom:
              resourceFieldRef:
                containerName: ray-worker
                resource: limits.cpu
          - name: MEMORY_LIMITS
            valueFrom:
              resourceFieldRef:
                containerName: ray-worker
                resource: limits.memory
          - name: MEMORY_REQUESTS
            valueFrom:
              resourceFieldRef:
                containerName: ray-worker
                resource: requests.memory
          - name: MY_POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: MY_POD_IP
            valueFrom:
              fieldRef:
                fieldPath: status.podIP
          ports:
          - containerPort: 80
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh","-c","ray stop"]
          # use volumeMounts.Optional.
          # Refer to https://kubernetes.io/docs/concepts/storage/volumes/
          volumeMounts:
            - mountPath: /var/log
              name: log-volume
          resources:
            limits:
              cpu: "1"
              memory: "512Mi"
            requests:
              cpu: "500m"
              memory: "256Mi"
        # use volumes
        # Refer to https://kubernetes.io/docs/concepts/storage/volumes/
        volumes:
          - name: log-volume
            emptyDir: {}
######################status#################################