opensandbox-group--opensandbox/server/opensandbox_server/services/k8s/client.py

# Copyright 2025 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Kubernetes client wrapper that provides a unified interface for all K8s resource
operations. All API access goes through this class.
"""

import logging
import threading
from functools import partial
from typing import Any, Dict, List, Optional, Tuple

from kubernetes import client, config
from kubernetes.client import ApiException, CoreV1Api, CustomObjectsApi, NodeV1Api

from opensandbox_server.config import KubernetesRuntimeConfig
from opensandbox_server.services.k8s.informer import WorkloadInformer
from opensandbox_server.services.k8s.label_selector import matches, parse_selector
from opensandbox_server.services.k8s.rate_limiter import TokenBucketRateLimiter

logger = logging.getLogger(__name__)

OPENSANDBOX_API_GROUP = "sandbox.opensandbox.io"
OPENSANDBOX_API_VERSION = "v1alpha1"
POOL_KIND = "Pool"
POOL_PLURAL = "pools"
POOL_AUTO_ASSIGN_REF = "*"

_InformerKey = Tuple[str, str, str, str]  # (group, version, plural, namespace)


class K8sClient:
    """
    Unified Kubernetes API client.

    Encapsulates all cluster resource operations (CustomObject, Secret, Pod,
    RuntimeClass). Callers never hold raw API handles directly.
    """

    def __init__(self, k8s_config: KubernetesRuntimeConfig):
        self.config = k8s_config
        self._load_config()
        self._core_v1_api: Optional[CoreV1Api] = None
        self._custom_objects_api: Optional[CustomObjectsApi] = None
        self._node_v1_api: Optional[NodeV1Api] = None
        self._informers: Dict[_InformerKey, WorkloadInformer] = {}
        self._informers_lock = threading.Lock()
        self._read_limiter: Optional[TokenBucketRateLimiter] = (
            TokenBucketRateLimiter(qps=k8s_config.read_qps, burst=k8s_config.read_burst)
            if k8s_config.read_qps > 0
            else None
        )
        self._write_limiter: Optional[TokenBucketRateLimiter] = (
            TokenBucketRateLimiter(qps=k8s_config.write_qps, burst=k8s_config.write_burst)
            if k8s_config.write_qps > 0
            else None
        )

    def _load_config(self) -> None:
        """Load kubeconfig from file path or in-cluster service account."""
        try:
            if self.config.kubeconfig_path:
                config.load_kube_config(config_file=self.config.kubeconfig_path)
            else:
                config.load_incluster_config()
        except Exception as e:
            raise Exception(f"Failed to load Kubernetes configuration: {e}") from e

    def get_core_v1_api(self) -> CoreV1Api:
        if self._core_v1_api is None:
            self._core_v1_api = client.CoreV1Api()
        return self._core_v1_api

    def get_custom_objects_api(self) -> CustomObjectsApi:
        if self._custom_objects_api is None:
            self._custom_objects_api = client.CustomObjectsApi()
        return self._custom_objects_api

    def get_node_v1_api(self) -> NodeV1Api:
        if self._node_v1_api is None:
            self._node_v1_api = client.NodeV1Api()
        return self._node_v1_api


    def _lookup_informer(self, group: str, version: str, plural: str, namespace: str) -> Optional[WorkloadInformer]:
        """Return an existing informer without starting one. Used by write paths
        to invalidate cache entries; never auto-create on writes since list paths
        own the lazy-start contract."""
        if not self.config.informer_enabled:
            return None
        key: _InformerKey = (group, version, plural, namespace)
        with self._informers_lock:
            return self._informers.get(key)

    def _get_informer(self, group: str, version: str, plural: str, namespace: str) -> Optional[WorkloadInformer]:
        """Return the informer for this resource+namespace, starting it lazily."""
        if not self.config.informer_enabled:
            return None

        key: _InformerKey = (group, version, plural, namespace)
        with self._informers_lock:
            informer = self._informers.get(key)
            if informer is None:
                list_fn = partial(
                    self.get_custom_objects_api().list_namespaced_custom_object,
                    group=group,
                    version=version,
                    namespace=namespace,
                    plural=plural,
                )
                informer = WorkloadInformer(
                    list_fn=list_fn,
                    resync_period_seconds=self.config.informer_resync_seconds,
                    watch_timeout_seconds=self.config.informer_watch_timeout_seconds,
                    thread_name=f"workload-informer-{plural}-{namespace}",
                )
                self._informers[key] = informer
                try:
                    informer.start()
                except Exception as exc:  # pragma: no cover - defensive
                    logger.warning(f"Failed to start informer for {plural}/{namespace}: {exc}")
                    self._informers.pop(key, None)
                    return None
        return informer


    def create_custom_object(
        self,
        group: str,
        version: str,
        namespace: str,
        plural: str,
        body: Dict[str, Any],
    ) -> Dict[str, Any]:
        """Create a namespaced custom resource."""
        if self._write_limiter:
            self._write_limiter.acquire()
        obj = self.get_custom_objects_api().create_namespaced_custom_object(
            group=group,
            version=version,
            namespace=namespace,
            plural=plural,
            body=body,
        )
        informer = self._lookup_informer(group, version, plural, namespace)
        if informer:
            informer.update_cache(obj)
        return obj

    def get_custom_object(
        self,
        group: str,
        version: str,
        namespace: str,
        plural: str,
        name: str,
    ) -> Optional[Dict[str, Any]]:
        """Get a namespaced custom resource by name.

        Tries the informer cache first when available and synced.
        Returns None on 404.
        """
        informer = self._get_informer(group, version, plural, namespace)
        if informer and informer.has_synced:
            cached = informer.get(name)
            if cached is not None:
                return cached

        if self._read_limiter:
            self._read_limiter.acquire()
        try:
            obj = self.get_custom_objects_api().get_namespaced_custom_object(
                group=group,
                version=version,
                namespace=namespace,
                plural=plural,
                name=name,
            )
            if informer:
                informer.update_cache(obj)
            return obj
        except ApiException as e:
            if e.status == 404:
                return None
            raise

    def list_custom_objects(
        self,
        group: str,
        version: str,
        namespace: str,
        plural: str,
        label_selector: str = "",
    ) -> List[Dict[str, Any]]:
        """List namespaced custom resources, returning the items list.

        Tries the informer cache first when available, synced, and the label
        selector falls within the supported in-memory grammar. Falls back to
        a direct API call (with rate limiting) otherwise.
        """
        informer = self._get_informer(group, version, plural, namespace)
        if informer and informer.has_synced:
            terms = parse_selector(label_selector)
            if terms is not None:
                cached = informer.list()
                if not terms:
                    return cached
                return [
                    obj
                    for obj in cached
                    if matches(obj.get("metadata", {}).get("labels") or {}, terms)
                ]

        if self._read_limiter:
            self._read_limiter.acquire()
        try:
            resp = self.get_custom_objects_api().list_namespaced_custom_object(
                group=group,
                version=version,
                namespace=namespace,
                plural=plural,
                label_selector=label_selector,
            )
            return resp.get("items", [])
        except ApiException as e:
            if e.status == 404:
                return []
            raise

    def delete_custom_object(
        self,
        group: str,
        version: str,
        namespace: str,
        plural: str,
        name: str,
        grace_period_seconds: int = 0,
    ) -> None:
        """Delete a namespaced custom resource."""
        if self._write_limiter:
            self._write_limiter.acquire()
        self.get_custom_objects_api().delete_namespaced_custom_object(
            group=group,
            version=version,
            namespace=namespace,
            plural=plural,
            name=name,
            grace_period_seconds=grace_period_seconds,
        )
        informer = self._lookup_informer(group, version, plural, namespace)
        if informer:
            informer.delete_from_cache(name)

    def patch_custom_object(
        self,
        group: str,
        version: str,
        namespace: str,
        plural: str,
        name: str,
        body: Dict[str, Any],
    ) -> Dict[str, Any]:
        """Patch a namespaced custom resource."""
        if self._write_limiter:
            self._write_limiter.acquire()
        obj = self.get_custom_objects_api().patch_namespaced_custom_object(
            group=group,
            version=version,
            namespace=namespace,
            plural=plural,
            name=name,
            body=body,
        )
        informer = self._lookup_informer(group, version, plural, namespace)
        if informer:
            informer.update_cache(obj)
        return obj

    # ------------------------------------------------------------------
    # PersistentVolumeClaim operations
    # ------------------------------------------------------------------

    def get_pvc(
        self,
        namespace: str,
        name: str,
    ) -> Optional[Any]:
        """Read a PersistentVolumeClaim by name. Returns None on 404."""
        if self._read_limiter:
            self._read_limiter.acquire()
        try:
            return self.get_core_v1_api().read_namespaced_persistent_volume_claim(
                name=name,
                namespace=namespace,
            )
        except ApiException as e:
            if e.status == 404:
                return None
            raise

    def create_pvc(
        self,
        namespace: str,
        body: Any,
    ) -> Any:
        """Create a PersistentVolumeClaim."""
        if self._write_limiter:
            self._write_limiter.acquire()
        return self.get_core_v1_api().create_namespaced_persistent_volume_claim(
            namespace=namespace,
            body=body,
        )

    def list_pvcs(
        self,
        namespace: str,
        label_selector: str = "",
    ) -> List[Any]:
        """List PersistentVolumeClaims in a namespace, returning the items list."""
        if self._read_limiter:
            self._read_limiter.acquire()
        result = self.get_core_v1_api().list_namespaced_persistent_volume_claim(
            namespace=namespace,
            label_selector=label_selector,
        )
        return list(getattr(result, "items", []) or [])

    def delete_pvc(
        self,
        namespace: str,
        name: str,
    ) -> None:
        """Delete a PersistentVolumeClaim by name. 404 is swallowed."""
        if self._write_limiter:
            self._write_limiter.acquire()
        try:
            self.get_core_v1_api().delete_namespaced_persistent_volume_claim(
                name=name,
                namespace=namespace,
            )
        except ApiException as e:
            if e.status == 404:
                return
            raise

    def patch_pvc(
        self,
        namespace: str,
        name: str,
        body: Any,
    ) -> Any:
        """Patch a PersistentVolumeClaim using strategic merge semantics.

        The pinned kubernetes-client picks ``application/json-patch+json`` by
        default (first entry in the generated content-type list) which would
        reject our merge-shaped body. Force strategic-merge so list fields
        like ``ownerReferences`` merge by key.
        """
        if self._write_limiter:
            self._write_limiter.acquire()
        return self.get_core_v1_api().patch_namespaced_persistent_volume_claim(
            name=name,
            namespace=namespace,
            body=body,
            _content_type="application/strategic-merge-patch+json",
        )

    # ------------------------------------------------------------------
    # Secret operations
    # ------------------------------------------------------------------

    def create_secret(self, namespace: str, body: Any) -> Any:
        """Create a namespaced Secret."""
        if self._write_limiter:
            self._write_limiter.acquire()
        return self.get_core_v1_api().create_namespaced_secret(
            namespace=namespace,
            body=body,
        )


    def list_pods(
        self,
        namespace: str,
        label_selector: str = "",
    ) -> List[Any]:
        """List pods in a namespace, returning the items list."""
        if self._read_limiter:
            self._read_limiter.acquire()
        resp = self.get_core_v1_api().list_namespaced_pod(
            namespace=namespace,
            label_selector=label_selector,
        )
        return resp.items

    def read_runtime_class(self, name: str) -> Any:
        """Read a RuntimeClass from the cluster."""
        if self._read_limiter:
            self._read_limiter.acquire()
        return self.get_node_v1_api().read_runtime_class(name)