Files
2026-07-13 13:17:40 +08:00

1262 lines
38 KiB
YAML

apiVersion: v1
items:
- apiVersion: v1
kind: Pod
metadata:
annotations:
ray.io/ft-enabled: "false"
ray.io/health-state: ""
creationTimestamp: "2022-11-14T23:10:15Z"
generateName: raycluster-autoscaler-head-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
ray.io/cluster: raycluster-autoscaler
ray.io/cluster-dashboard: raycluster-autoscaler-dashboard
ray.io/group: headgroup
ray.io/identifier: raycluster-autoscaler-head
ray.io/is-ray-node: "yes"
ray.io/node-type: head
name: raycluster-autoscaler-head-8zsc8
namespace: default
ownerReferences:
- apiVersion: ray.io/v1alpha1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: ec79effb-0295-4f40-b08b-8633aa7f786a
resourceVersion: "4519"
uid: 539ea57c-8d51-4503-a395-08779efb3bf0
spec:
containers:
- args:
- 'ulimit -n 65536; ray start --head --resources="{\"Custom1\": 1, \"Custom2\":
5}" --block --dashboard-host=0.0.0.0 --metrics-export-port=8080 --no-monitor --num-cpus=1 --memory=1000000000 '
command:
- /bin/bash
- -c
- --
env:
- name: RAY_IP
value: 127.0.0.1
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: 127.0.0.1:6379
- name: REDIS_PASSWORD
image: gekho/ray
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- ray stop
name: ray-head
ports:
- containerPort: 6379
name: gcs
protocol: TCP
- containerPort: 8265
name: dashboard
protocol: TCP
- containerPort: 10001
name: client
protocol: TCP
- containerPort: 8080
name: metrics
protocol: TCP
resources:
limits:
cpu: "1"
memory: 1G
requests:
cpu: 500m
memory: 512Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /tmp/ray
name: ray-logs
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-tmxvr
readOnly: true
- args:
- kuberay-autoscaler
- --cluster-name
- $(RAY_CLUSTER_NAME)
- --cluster-namespace
- $(RAY_CLUSTER_NAMESPACE)
command:
- ray
env:
- name: RAY_CLUSTER_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.labels['ray.io/cluster']
- name: RAY_CLUSTER_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: gekho/ray
imagePullPolicy: Always
name: autoscaler
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 500m
memory: 512Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /tmp/ray
name: ray-logs
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-tmxvr
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: gke-cluster-1-default-pool-a5503908-181p
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: raycluster-autoscaler
serviceAccountName: raycluster-autoscaler
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- emptyDir:
medium: Memory
sizeLimit: 512Mi
name: shared-mem
- emptyDir: {}
name: ray-logs
- name: kube-api-access-tmxvr
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:10:15Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:11:23Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:11:23Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:10:15Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://0b008432be839bec8dd97437d3f2be9ac8d7f017b91067a46ec45a487f141ebf
image: docker.io/gekho/ray:latest
imageID: docker.io/gekho/ray@sha256:7859a78d1a089bb88691864d5c4a2aad529f5353d7d9c82cc0274842fbda242b
lastState: {}
name: autoscaler
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2022-11-14T23:11:23Z"
- containerID: containerd://b2aae80ed028cc41bad1e350bb70a0a4e8ea722df098b38781efabe54adbc5ec
image: docker.io/gekho/ray:latest
imageID: docker.io/gekho/ray@sha256:7859a78d1a089bb88691864d5c4a2aad529f5353d7d9c82cc0274842fbda242b
lastState: {}
name: ray-head
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2022-11-14T23:11:22Z"
hostIP: 10.128.0.45
phase: Running
podIP: 10.4.2.6
podIPs:
- ip: 10.4.2.6
qosClass: Burstable
startTime: "2022-11-14T23:10:15Z"
- apiVersion: v1
kind: Pod
metadata:
annotations:
key: value
ray.io/ft-enabled: "false"
ray.io/health-state: ""
creationTimestamp: "2022-11-14T23:13:47Z"
generateName: raycluster-autoscaler-worker-fake-gpu-group-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
key: value
ray.io/cluster: raycluster-autoscaler
ray.io/cluster-dashboard: raycluster-autoscaler-dashboard
ray.io/group: fake-gpu-group
ray.io/identifier: raycluster-autoscaler-worker
ray.io/is-ray-node: "yes"
ray.io/node-type: worker
name: raycluster-autoscaler-worker-fake-gpu-group-2qnhv
namespace: default
ownerReferences:
- apiVersion: ray.io/v1alpha1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: ec79effb-0295-4f40-b08b-8633aa7f786a
resourceVersion: "5805"
uid: f1cadfb8-b76f-467d-9c2b-a1a5eb7d1d3f
spec:
containers:
- args:
- 'ulimit -n 65536; ray start --address=raycluster-autoscaler-head-svc:6379 --metrics-export-port=8080 --num-cpus=1 --memory=536870912 --block --num-gpus=1 --resources="{\"Custom1\":
1, \"Custom2\": 5}" '
command:
- /bin/bash
- -c
- --
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: raycluster-autoscaler-head-svc:6379
- name: REDIS_PASSWORD
image: gekho/ray
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- ray stop
name: machine-learning
ports:
- containerPort: 8080
name: metrics
protocol: TCP
resources:
limits:
cpu: "1"
memory: 512Mi
requests:
cpu: 500m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-ghm2j
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
initContainers:
- command:
- sh
- -c
- until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local;
do echo waiting for myservice; sleep 2; done
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
image: busybox:1.28
imagePullPolicy: IfNotPresent
name: init-myservice
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-ghm2j
readOnly: true
nodeName: gke-cluster-1-default-pool-a5503908-dpst
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- emptyDir:
medium: Memory
sizeLimit: 256Mi
name: shared-mem
- name: kube-api-access-ghm2j
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:49Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:50Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:50Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:47Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://b6f4129ac590c62e41cc6db1a18fb051a29e3458fb47296086292a5765c36a6c
image: docker.io/gekho/ray:latest
imageID: docker.io/gekho/ray@sha256:7859a78d1a089bb88691864d5c4a2aad529f5353d7d9c82cc0274842fbda242b
lastState: {}
name: machine-learning
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2022-11-14T23:13:49Z"
hostIP: 10.128.0.31
initContainerStatuses:
- containerID: containerd://5c946a3d443f76d21a3c72244edb4f80fc420b13b223f3c6981e25e689d381cf
image: docker.io/library/busybox:1.28
imageID: docker.io/library/busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
lastState: {}
name: init-myservice
ready: true
restartCount: 0
state:
terminated:
containerID: containerd://5c946a3d443f76d21a3c72244edb4f80fc420b13b223f3c6981e25e689d381cf
exitCode: 0
finishedAt: "2022-11-14T23:13:48Z"
reason: Completed
startedAt: "2022-11-14T23:13:48Z"
phase: Running
podIP: 10.4.0.6
podIPs:
- ip: 10.4.0.6
qosClass: Burstable
startTime: "2022-11-14T23:13:47Z"
- apiVersion: v1
kind: Pod
metadata:
annotations:
key: value
ray.io/ft-enabled: "false"
ray.io/health-state: ""
creationTimestamp: "2022-11-14T23:11:50Z"
generateName: raycluster-autoscaler-worker-small-group-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
key: value
ray.io/cluster: raycluster-autoscaler
ray.io/cluster-dashboard: raycluster-autoscaler-dashboard
ray.io/group: small-group
ray.io/identifier: raycluster-autoscaler-worker
ray.io/is-ray-node: "yes"
ray.io/node-type: worker
name: raycluster-autoscaler-worker-small-group-dkz2r
namespace: default
ownerReferences:
- apiVersion: ray.io/v1alpha1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: ec79effb-0295-4f40-b08b-8633aa7f786a
resourceVersion: "5322"
uid: b4fb3233-6024-48a8-9f4f-a18f5e490629
spec:
containers:
- args:
- 'ulimit -n 65536; ray start --block --resources="{\"Custom1\": 1, \"Custom2\":
5}" --address=raycluster-autoscaler-head-svc:6379 --metrics-export-port=8080 --num-cpus=1 --memory=536870912 '
command:
- /bin/bash
- -c
- --
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: raycluster-autoscaler-head-svc:6379
- name: REDIS_PASSWORD
image: gekho/ray
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- ray stop
name: machine-learning
ports:
- containerPort: 8080
name: metrics
protocol: TCP
resources:
limits:
cpu: "1"
memory: 512Mi
requests:
cpu: 500m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-djtd9
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
initContainers:
- command:
- sh
- -c
- until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local;
do echo waiting for myservice; sleep 2; done
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
image: busybox:1.28
imagePullPolicy: IfNotPresent
name: init-myservice
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-djtd9
readOnly: true
nodeName: gke-cluster-1-default-pool-a5503908-j51d
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- emptyDir:
medium: Memory
sizeLimit: 256Mi
name: shared-mem
- name: kube-api-access-djtd9
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:11:51Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:12:55Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:12:55Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:11:50Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://91fe3b347c34ebf74e99b67accd0d2c7fcce64886525dedbbd9d735d531da8ca
image: docker.io/gekho/ray:latest
imageID: docker.io/gekho/ray@sha256:7859a78d1a089bb88691864d5c4a2aad529f5353d7d9c82cc0274842fbda242b
lastState: {}
name: machine-learning
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2022-11-14T23:12:55Z"
hostIP: 10.128.0.43
initContainerStatuses:
- containerID: containerd://672d9a5836e27a17f57a4e15e1d86431cfee6f2edef1210d60e864e3c510aac0
image: docker.io/library/busybox:1.28
imageID: docker.io/library/busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
lastState: {}
name: init-myservice
ready: true
restartCount: 0
state:
terminated:
containerID: containerd://672d9a5836e27a17f57a4e15e1d86431cfee6f2edef1210d60e864e3c510aac0
exitCode: 0
finishedAt: "2022-11-14T23:11:51Z"
reason: Completed
startedAt: "2022-11-14T23:11:51Z"
phase: Running
podIP: 10.4.1.8
podIPs:
- ip: 10.4.1.8
qosClass: Burstable
startTime: "2022-11-14T23:11:50Z"
- apiVersion: v1
kind: Pod
metadata:
annotations:
key: value
ray.io/ft-enabled: "false"
ray.io/health-state: ""
creationTimestamp: "2022-11-14T23:13:37Z"
generateName: raycluster-autoscaler-worker-small-group-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
key: value
ray.io/cluster: raycluster-autoscaler
ray.io/cluster-dashboard: raycluster-autoscaler-dashboard
ray.io/group: small-group
ray.io/identifier: raycluster-autoscaler-worker
ray.io/is-ray-node: "yes"
ray.io/node-type: worker
name: raycluster-autoscaler-worker-small-group-lbfm4
namespace: default
ownerReferences:
- apiVersion: ray.io/v1alpha1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: ec79effb-0295-4f40-b08b-8633aa7f786a
resourceVersion: "5708"
uid: b9a000d5-d1de-47b8-91c7-3c51150bc669
spec:
containers:
- args:
- 'ulimit -n 65536; ray start --resources="{\"Custom1\": 1, \"Custom2\": 5}" --address=raycluster-autoscaler-head-svc:6379 --metrics-export-port=8080 --num-cpus=1 --memory=536870912 --block '
command:
- /bin/bash
- -c
- --
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: raycluster-autoscaler-head-svc:6379
- name: REDIS_PASSWORD
image: gekho/ray
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- ray stop
name: machine-learning
ports:
- containerPort: 8080
name: metrics
protocol: TCP
resources:
limits:
cpu: "1"
memory: 512Mi
requests:
cpu: 500m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-bblr7
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
initContainers:
- command:
- sh
- -c
- until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local;
do echo waiting for myservice; sleep 2; done
env:
- name: RAY_IP
value: raycluster-autoscaler-head-svc
image: busybox:1.28
imagePullPolicy: IfNotPresent
name: init-myservice
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-bblr7
readOnly: true
nodeName: gke-cluster-1-default-pool-a5503908-dpst
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- emptyDir:
medium: Memory
sizeLimit: 256Mi
name: shared-mem
- name: kube-api-access-bblr7
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:39Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:40Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:40Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2022-11-14T23:13:37Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://5680b43a1990a57f0a9adc834e5c361e85716866ed7129304794ba42c141f483
image: docker.io/gekho/ray:latest
imageID: docker.io/gekho/ray@sha256:7859a78d1a089bb88691864d5c4a2aad529f5353d7d9c82cc0274842fbda242b
lastState: {}
name: machine-learning
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2022-11-14T23:13:39Z"
hostIP: 10.128.0.31
initContainerStatuses:
- containerID: containerd://57423dd9569b214256a2f051e0a449c8611dde0597893c69002569fd4c146390
image: docker.io/library/busybox:1.28
imageID: docker.io/library/busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
lastState: {}
name: init-myservice
ready: true
restartCount: 0
state:
terminated:
containerID: containerd://57423dd9569b214256a2f051e0a449c8611dde0597893c69002569fd4c146390
exitCode: 0
finishedAt: "2022-11-14T23:13:38Z"
reason: Completed
startedAt: "2022-11-14T23:13:38Z"
phase: Running
podIP: 10.4.0.5
podIPs:
- ip: 10.4.0.5
qosClass: Burstable
startTime: "2022-11-14T23:13:37Z"
- apiVersion: v1
kind: Pod
metadata:
annotations:
cloud.google.com/cluster_autoscaler_unhelpable_since: 2025-06-19T07:34:20+0000
cloud.google.com/cluster_autoscaler_unhelpable_until: Inf
ray.io/ft-enabled: "false"
creationTimestamp: "2025-06-19T07:34:20Z"
generateName: raycluster-autoscaler-worker-tpu-group-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
ray.io/cluster: raycluster-autoscaler
ray.io/group: tpu-group
ray.io/identifier: raycluster-autoscaler-worker
ray.io/is-ray-node: "yes"
ray.io/node-type: worker
replicaIndex: tpu-group-0
name: raycluster-autoscaler-tpu-group-worker-jd69f
namespace: default
ownerReferences:
- apiVersion: ray.io/v1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: 9022aa6b-0090-4707-9dd9-96ed4895e401
resourceVersion: "1750318759258511010"
uid: 95f0f375-8e99-40f7-b804-68912387555a
spec:
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: replicaIndex
operator: In
values:
- tpu-group-0
topologyKey: cloud.google.com/gke-nodepool
containers:
- args:
- 'ulimit -n 65536; ray start --address=raycluster-autoscaler-head-svc.default.svc.cluster.local:6379 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 --num-cpus=1 --memory=40000000000 '
command:
- /bin/bash
- -lc
- --
env:
- name: FQ_RAY_IP
value: raycluster-autoscaler-head-svc.default.svc.cluster.local
- name: RAY_IP
value: raycluster-autoscaler-head-svc
- name: RAY_CLUSTER_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.labels['ray.io/cluster']
- name: RAY_CLOUD_INSTANCE_ID
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: RAY_NODE_TYPE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.labels['ray.io/group']
- name: KUBERAY_GEN_RAY_START_CMD
value: 'ray start --address=raycluster-autoscaler-head-svc.default.svc.cluster.local:6379 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 --num-cpus=1 --memory=40000000000 '
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: raycluster-autoscaler-head-svc.default.svc.cluster.local:6379
- name: RAY_USAGE_STATS_KUBERAY_IN_USE
value: "1"
- name: REDIS_PASSWORD
- name: RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE
value: "1"
- name: TPU_WORKER_HOSTNAMES
value: tpu-group-0-0.raycluster-autoscaler-headless-worker-svc,tpu-group-0-1.raycluster-autoscaler-headless-worker-svc
- name: TPU_WORKER_ID
value: "1"
- name: TPU_NAME
value: tpu-group-0
image: rayproject/ray:2.46.0-py310
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- bash
- -c
- wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep
success
failureThreshold: 120
initialDelaySeconds: 30
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 2
name: ray-worker
ports:
- containerPort: 8080
name: metrics
protocol: TCP
readinessProbe:
exec:
command:
- bash
- -c
- wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep
success
failureThreshold: 10
initialDelaySeconds: 10
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 2
resources:
limits:
cpu: "1"
ephemeral-storage: 20Gi
google.com/tpu: "4"
memory: 40G
requests:
cpu: "1"
ephemeral-storage: 10Gi
google.com/tpu: "4"
memory: 40G
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-nxnbq
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
hostname: tpu-group-0-1
nodeName: gke-tpu-2b029696-76nc
nodeSelector:
cloud.google.com/gke-accelerator-count: "4"
cloud.google.com/gke-tpu-accelerator: tpu-v4-podslice
cloud.google.com/gke-tpu-topology: 2x2x2
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
subdomain: raycluster-autoscaler-headless-worker-svc
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
- effect: NoSchedule
key: google.com/tpu
operator: Exists
volumes:
- emptyDir:
medium: Memory
sizeLimit: 40G
name: shared-mem
- name: kube-api-access-nxnbq
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:08Z"
status: "True"
type: PodReadyToStartContainers
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:38:43Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:19Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:19Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:38:43Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://a5c129309656aa84e80174fab530b26e0a8664e5168ef6a917ed3519fb3e2978
image: docker.io/rayproject/ray:2.46.0-py310
imageID: docker.io/rayproject/ray@sha256:812a7c6439306dca5cffc49d58a2a29f7da108565e8d4a044f84d4fa75d39c42
lastState: {}
name: ray-worker
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2025-06-19T07:39:08Z"
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-nxnbq
readOnly: true
recursiveReadOnly: Disabled
hostIP: 10.130.0.88
hostIPs:
- ip: 10.130.0.88
phase: Running
podIP: 10.24.8.4
podIPs:
- ip: 10.24.8.4
qosClass: Guaranteed
startTime: "2025-06-19T07:38:43Z"
- apiVersion: v1
kind: Pod
metadata:
annotations:
cloud.google.com/cluster_autoscaler_unhelpable_since: 2025-06-19T07:34:19+0000
cloud.google.com/cluster_autoscaler_unhelpable_until: Inf
ray.io/ft-enabled: "false"
creationTimestamp: "2025-06-19T07:34:19Z"
generateName: raycluster-autoscaler-worker-tpu-group-
labels:
app.kubernetes.io/created-by: kuberay-operator
app.kubernetes.io/name: kuberay
ray.io/cluster: raycluster-autoscaler
ray.io/group: tpu-group
ray.io/identifier: raycluster-autoscaler-worker
ray.io/is-ray-node: "yes"
ray.io/node-type: worker
replicaIndex: tpu-group-0
name: raycluster-autoscaler-tpu-group-worker-s8jhq
namespace: default
ownerReferences:
- apiVersion: ray.io/v1
blockOwnerDeletion: true
controller: true
kind: RayCluster
name: raycluster-autoscaler
uid: 9022aa6b-0090-4707-9dd9-96ed4895e401
resourceVersion: "1750318759835855004"
uid: bab4b83e-539c-4206-afc7-32a837a2bc3c
spec:
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: replicaIndex
operator: In
values:
- tpu-group-0
topologyKey: cloud.google.com/gke-nodepool
containers:
- args:
- 'ulimit -n 65536; ray start --num-cpus=1 --memory=40000000000 --address=raycluster-autoscaler-head-svc.default.svc.cluster.local:6379 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 '
command:
- /bin/bash
- -lc
- --
env:
- name: FQ_RAY_IP
value: raycluster-autoscaler-head-svc.default.svc.cluster.local
- name: RAY_IP
value: raycluster-autoscaler-head-svc
- name: RAY_CLUSTER_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.labels['ray.io/cluster']
- name: RAY_CLOUD_INSTANCE_ID
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: RAY_NODE_TYPE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.labels['ray.io/group']
- name: KUBERAY_GEN_RAY_START_CMD
value: 'ray start --num-cpus=1 --memory=40000000000 --address=raycluster-autoscaler-head-svc.default.svc.cluster.local:6379 --metrics-export-port=8080 --block --dashboard-agent-listen-port=52365 '
- name: RAY_PORT
value: "6379"
- name: RAY_ADDRESS
value: raycluster-autoscaler-head-svc.default.svc.cluster.local:6379
- name: RAY_USAGE_STATS_KUBERAY_IN_USE
value: "1"
- name: REDIS_PASSWORD
- name: RAY_DASHBOARD_ENABLE_K8S_DISK_USAGE
value: "1"
- name: TPU_WORKER_HOSTNAMES
value: tpu-group-0-0.raycluster-autoscaler-headless-worker-svc,tpu-group-0-1.raycluster-autoscaler-headless-worker-svc
- name: TPU_WORKER_ID
value: "0"
- name: TPU_NAME
value: tpu-group-0
image: rayproject/ray:2.46.0-py310
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- bash
- -c
- wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep
success
failureThreshold: 120
initialDelaySeconds: 30
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 2
name: ray-worker
ports:
- containerPort: 8080
name: metrics
protocol: TCP
readinessProbe:
exec:
command:
- bash
- -c
- wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep
success
failureThreshold: 10
initialDelaySeconds: 10
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 2
resources:
limits:
cpu: "1"
ephemeral-storage: 20Gi
google.com/tpu: "4"
memory: 40G
requests:
cpu: "1"
ephemeral-storage: 10Gi
google.com/tpu: "4"
memory: 40G
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-f7wqp
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
hostname: tpu-group-0-0
nodeName: gke-tpu-2b029696-3gvz
nodeSelector:
cloud.google.com/gke-accelerator-count: "4"
cloud.google.com/gke-tpu-accelerator: tpu-v4-podslice
cloud.google.com/gke-tpu-topology: 2x2x2
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
subdomain: raycluster-autoscaler-headless-worker-svc
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
- effect: NoSchedule
key: google.com/tpu
operator: Exists
volumes:
- emptyDir:
medium: Memory
sizeLimit: 40G
name: shared-mem
- name: kube-api-access-f7wqp
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:06Z"
status: "True"
type: PodReadyToStartContainers
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:38:39Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:19Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:39:19Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2025-06-19T07:38:39Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://c4174cbb26bba07aa5aed928fedd5b24c8913d3d2826f8fca198277dfeeaa26e
image: docker.io/rayproject/ray:2.46.0-py310
imageID: docker.io/rayproject/ray@sha256:812a7c6439306dca5cffc49d58a2a29f7da108565e8d4a044f84d4fa75d39c42
lastState: {}
name: ray-worker
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2025-06-19T07:39:05Z"
volumeMounts:
- mountPath: /dev/shm
name: shared-mem
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-f7wqp
readOnly: true
recursiveReadOnly: Disabled
hostIP: 10.130.0.138
hostIPs:
- ip: 10.130.0.138
phase: Running
podIP: 10.24.9.4
podIPs:
- ip: 10.24.9.4
qosClass: Guaranteed
startTime: "2025-06-19T07:38:39Z"
kind: List
metadata:
resourceVersion: ""