chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
+177
View File
@@ -0,0 +1,177 @@
package cli
import (
"context"
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/pkg/errors"
"github.com/segmentio/encoding/json"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NamespaceAccessPoliciesDeleteNamespace removes stored policies associated with a given namespace
func (kcl *KubeClient) NamespaceAccessPoliciesDeleteNamespace(ns string) error {
kcl.mu.Lock()
defer kcl.mu.Unlock()
policies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return errors.WithMessage(err, "failed to fetch access policies")
}
if policies != nil {
delete(policies, ns)
return kcl.UpdateNamespaceAccessPolicies(policies)
}
return nil
}
// GetNamespaceAccessPolicies gets the namespace access policies
// from config maps in the portainer namespace
func (kcl *KubeClient) GetNamespaceAccessPolicies() (map[string]portainer.K8sNamespaceAccessPolicy, error) {
configMap, err := kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Get(context.TODO(), portainerConfigMapName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
accessData := configMap.Data[portainerConfigMapAccessPoliciesKey]
policies := map[string]portainer.K8sNamespaceAccessPolicy{}
err = json.Unmarshal([]byte(accessData), &policies)
if err != nil {
return nil, err
}
return policies, nil
}
func (kcl *KubeClient) setupNamespaceAccesses(userID int, teamIDs []int, serviceAccountName string, restrictDefaultNamespace bool) error {
accessPolicies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return err
}
namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return err
}
for _, namespace := range namespaces.Items {
if namespace.Name == defaultNamespace && !restrictDefaultNamespace {
err = kcl.ensureNamespaceAccessForServiceAccount(serviceAccountName, defaultNamespace)
if err != nil {
return err
}
continue
}
policies, ok := accessPolicies[namespace.Name]
if !ok || !hasUserAccessToNamespace(userID, teamIDs, policies) {
err = kcl.removeNamespaceAccessForServiceAccount(serviceAccountName, namespace.Name)
if err != nil {
return err
}
continue
}
err = kcl.ensureNamespaceAccessForServiceAccount(serviceAccountName, namespace.Name)
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
}
return nil
}
func hasUserAccessToNamespace(userID int, teamIDs []int, policies portainer.K8sNamespaceAccessPolicy) bool {
_, userAccess := policies.UserAccessPolicies[portainer.UserID(userID)]
if userAccess {
return true
}
for _, teamID := range teamIDs {
_, teamAccess := policies.TeamAccessPolicies[portainer.TeamID(teamID)]
if teamAccess {
return true
}
}
return false
}
// UpdateNamespaceAccessPolicies updates the namespace access policies
func (kcl *KubeClient) UpdateNamespaceAccessPolicies(accessPolicies map[string]portainer.K8sNamespaceAccessPolicy) error {
data, err := json.Marshal(accessPolicies)
if err != nil {
return err
}
configMap, err := kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Get(context.TODO(), portainerConfigMapName, metav1.GetOptions{})
if err != nil {
return err
}
configMap.Data[portainerConfigMapAccessPoliciesKey] = string(data)
_, err = kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
return err
}
// GetNonAdminNamespaces retrieves namespaces for a non-admin user, excluding the default namespace if restricted.
func (kcl *KubeClient) GetNonAdminNamespaces(userID int, teamIDs []int, isRestrictDefaultNamespace bool) ([]string, error) {
accessPolicies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return nil, fmt.Errorf("an error occurred during the getNonAdminNamespaces operation, unable to get namespace access policies via portainer-config. check if portainer-config configMap exists in the Kubernetes cluster: %w", err)
}
nonAdminNamespaces := []string{}
if !isRestrictDefaultNamespace {
nonAdminNamespaces = append(nonAdminNamespaces, defaultNamespace)
}
for namespace, accessPolicy := range accessPolicies {
if hasUserAccessToNamespace(userID, teamIDs, accessPolicy) {
nonAdminNamespaces = append(nonAdminNamespaces, namespace)
}
}
return nonAdminNamespaces, nil
}
// GetIsKubeAdmin retrieves true if client is admin
func (kcl *KubeClient) GetIsKubeAdmin() bool {
kcl.mu.Lock()
defer kcl.mu.Unlock()
return kcl.isKubeAdmin
}
// UpdateIsKubeAdmin sets whether the kube client is admin
func (kcl *KubeClient) SetIsKubeAdmin(isKubeAdmin bool) {
kcl.mu.Lock()
defer kcl.mu.Unlock()
kcl.isKubeAdmin = isKubeAdmin
}
// GetClientNonAdminNamespaces retrieves non-admin namespaces
func (kcl *KubeClient) GetClientNonAdminNamespaces() []string {
kcl.mu.Lock()
defer kcl.mu.Unlock()
return kcl.nonAdminNamespaces
}
// UpdateClientNonAdminNamespaces sets the client non admin namespace list
func (kcl *KubeClient) SetClientNonAdminNamespaces(nonAdminNamespaces []string) {
kcl.mu.Lock()
defer kcl.mu.Unlock()
kcl.nonAdminNamespaces = nonAdminNamespaces
}
+96
View File
@@ -0,0 +1,96 @@
package cli
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ktypes "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func Test_NamespaceAccessPoliciesDeleteNamespace_updatesPortainerConfig_whenConfigExists(t *testing.T) {
t.Parallel()
testcases := []struct {
name string
namespaceToDelete string
expectedConfig map[string]portainer.K8sNamespaceAccessPolicy
}{
{
name: "doesn't change config, when designated namespace absent",
namespaceToDelete: "missing-namespace",
expectedConfig: map[string]portainer.K8sNamespaceAccessPolicy{
"ns1": {UserAccessPolicies: portainer.UserAccessPolicies{2: {RoleID: 0}}},
"ns2": {UserAccessPolicies: portainer.UserAccessPolicies{2: {RoleID: 0}}},
},
},
{
name: "removes designated namespace from config, when namespace is present",
namespaceToDelete: "ns2",
expectedConfig: map[string]portainer.K8sNamespaceAccessPolicy{
"ns1": {UserAccessPolicies: portainer.UserAccessPolicies{2: {RoleID: 0}}},
},
},
}
for _, test := range testcases {
t.Run(test.name, func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "instance",
}
config := &ktypes.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: portainerConfigMapName,
Namespace: portainerNamespace,
},
Data: map[string]string{
"NamespaceAccessPolicies": `{"ns1":{"UserAccessPolicies":{"2":{"RoleId":0}}}, "ns2":{"UserAccessPolicies":{"2":{"RoleId":0}}}}`,
},
}
_, err := k.cli.CoreV1().ConfigMaps(portainerNamespace).Create(t.Context(), config, metav1.CreateOptions{})
require.NoError(t, err, "failed to create a portainer config")
defer func() {
err := k.cli.CoreV1().ConfigMaps(portainerNamespace).Delete(t.Context(), portainerConfigMapName, metav1.DeleteOptions{})
require.NoError(t, err)
}()
err = k.NamespaceAccessPoliciesDeleteNamespace(test.namespaceToDelete)
require.NoError(t, err, "failed to delete namespace")
policies, err := k.GetNamespaceAccessPolicies()
require.NoError(t, err, "failed to fetch policies")
assert.Equal(t, test.expectedConfig, policies)
})
}
}
func TestKubeAdmin(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
require.False(t, kcl.GetIsKubeAdmin())
kcl.SetIsKubeAdmin(true)
require.True(t, kcl.GetIsKubeAdmin())
kcl.SetIsKubeAdmin(false)
require.False(t, kcl.GetIsKubeAdmin())
}
func TestClientNonAdminNamespaces(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
require.Empty(t, kcl.GetClientNonAdminNamespaces())
nss := []string{"ns1", "ns2"}
kcl.SetClientNonAdminNamespaces(nss)
require.Equal(t, nss, kcl.GetClientNonAdminNamespaces())
kcl.SetClientNonAdminNamespaces([]string{})
require.Empty(t, kcl.GetClientNonAdminNamespaces())
}
+585
View File
@@ -0,0 +1,585 @@
package cli
import (
"context"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
autoscalingv2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
labels "k8s.io/apimachinery/pkg/labels"
)
// PortainerApplicationResources contains collections of various Kubernetes resources
// associated with a Portainer application.
type PortainerApplicationResources struct {
Pods []corev1.Pod
ReplicaSets []appsv1.ReplicaSet
Deployments []appsv1.Deployment
StatefulSets []appsv1.StatefulSet
DaemonSets []appsv1.DaemonSet
Services []corev1.Service
HorizontalPodAutoscalers []autoscalingv2.HorizontalPodAutoscaler
}
// GetAllKubernetesApplications gets a list of kubernetes workloads (or applications) across all namespaces in the cluster
// if the user is an admin, all namespaces in the current k8s environment(endpoint) are fetched using the fetchApplications function.
// otherwise, namespaces the non-admin user has access to will be used to filter the applications based on the allowed namespaces.
func (kcl *KubeClient) GetApplications(namespace, nodeName string) ([]models.K8sApplication, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchApplications(namespace, nodeName)
}
return kcl.fetchApplicationsForNonAdmin(namespace, nodeName)
}
// fetchApplications fetches the applications in the namespaces the user has access to.
// This function is called when the user is an admin.
func (kcl *KubeClient) fetchApplications(namespace, nodeName string) ([]models.K8sApplication, error) {
podListOptions := metav1.ListOptions{}
if nodeName != "" {
podListOptions.FieldSelector = "spec.nodeName=" + nodeName
}
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources(namespace, podListOptions)
if err != nil {
return nil, err
}
applications, err := kcl.convertPodsToApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
unhealthyApplications, err := fetchUnhealthyApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
return append(applications, unhealthyApplications...), nil
}
// fetchApplicationsForNonAdmin fetches the applications in the namespaces the user has access to.
// This function is called when the user is not an admin.
func (kcl *KubeClient) fetchApplicationsForNonAdmin(namespace, nodeName string) ([]models.K8sApplication, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching applications for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
podListOptions := metav1.ListOptions{}
if nodeName != "" {
podListOptions.FieldSelector = "spec.nodeName=" + nodeName
}
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources(namespace, podListOptions)
if err != nil {
return nil, err
}
applications, err := kcl.convertPodsToApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
unhealthyApplications, err := fetchUnhealthyApplications(portainerApplicationResources)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sApplication, 0)
for _, application := range append(applications, unhealthyApplications...) {
if _, ok := nonAdminNamespaceSet[application.ResourcePool]; ok {
results = append(results, application)
}
}
return results, nil
}
// convertPodsToApplications processes pods and converts them to applications, ensuring uniqueness by owner reference.
func (kcl *KubeClient) convertPodsToApplications(portainerApplicationResources PortainerApplicationResources) ([]models.K8sApplication, error) {
applications := []models.K8sApplication{}
processedOwners := make(map[string]struct{})
for _, pod := range portainerApplicationResources.Pods {
if len(pod.OwnerReferences) > 0 {
ownerUID := string(pod.OwnerReferences[0].UID)
if _, exists := processedOwners[ownerUID]; exists {
continue
}
processedOwners[ownerUID] = struct{}{}
}
application, err := kcl.ConvertPodToApplication(pod, portainerApplicationResources, true)
if err != nil {
return nil, err
}
if application != nil {
applications = append(applications, *application)
}
}
return applications, nil
}
// GetClusterApplicationsResource returns the total resource requests and limits for all applications in a namespace
// for a cluster level resource, set the namespace to ""
func (kcl *KubeClient) GetApplicationsResource(namespace, node string) (models.K8sApplicationResource, error) {
resource := models.K8sApplicationResource{}
podListOptions := metav1.ListOptions{}
if node != "" {
podListOptions.FieldSelector = "spec.nodeName=" + node
}
pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.Background(), podListOptions)
if err != nil {
return resource, err
}
for _, pod := range pods.Items {
podResources := calculatePodResourceUsage(pod)
resource.CPURequest += podResources.CPURequest
resource.CPULimit += podResources.CPULimit
resource.MemoryRequest += podResources.MemoryRequest
resource.MemoryLimit += podResources.MemoryLimit
}
return resource, nil
}
// ConvertPodToApplication converts a pod to an application, updating owner references if necessary
func (kcl *KubeClient) ConvertPodToApplication(pod corev1.Pod, portainerApplicationResources PortainerApplicationResources, withResource bool) (*models.K8sApplication, error) {
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, portainerApplicationResources.ReplicaSets)
}
application := createApplicationFromPod(&pod, portainerApplicationResources)
if application.ID == "" && application.Name == "" {
return nil, nil
}
if withResource {
podResources := calculatePodResourceUsage(pod)
// multiply by the number of requested pods in the application (not the running count)
application.Resource.CPURequest = podResources.CPURequest * float64(application.TotalPodsCount)
application.Resource.CPULimit = podResources.CPULimit * float64(application.TotalPodsCount)
application.Resource.MemoryRequest = podResources.MemoryRequest * int64(application.TotalPodsCount)
application.Resource.MemoryLimit = podResources.MemoryLimit * int64(application.TotalPodsCount)
}
return &application, nil
}
// createApplicationFromPod creates a K8sApplication object from a pod
// it sets the application name, namespace, kind, image, stack id, stack name, and labels
func createApplicationFromPod(pod *corev1.Pod, portainerApplicationResources PortainerApplicationResources) models.K8sApplication {
kind := "Pod"
name := pod.Name
if len(pod.OwnerReferences) > 0 {
kind = pod.OwnerReferences[0].Kind
name = pod.OwnerReferences[0].Name
}
application := models.K8sApplication{
Services: []corev1.Service{},
Metadata: &models.Metadata{},
}
switch kind {
case "Deployment":
for _, deployment := range portainerApplicationResources.Deployments {
if deployment.Name == name && deployment.Namespace == pod.Namespace {
populateApplicationFromDeployment(&application, deployment)
break
}
}
case "StatefulSet":
for _, statefulSet := range portainerApplicationResources.StatefulSets {
if statefulSet.Name == name && statefulSet.Namespace == pod.Namespace {
populateApplicationFromStatefulSet(&application, statefulSet)
break
}
}
case "DaemonSet":
for _, daemonSet := range portainerApplicationResources.DaemonSets {
if daemonSet.Name == name && daemonSet.Namespace == pod.Namespace {
populateApplicationFromDaemonSet(&application, daemonSet)
break
}
}
case "Pod":
populateApplicationFromPod(&application, *pod)
}
if application.ID != "" && application.Name != "" && len(portainerApplicationResources.Services) > 0 {
updateApplicationWithService(&application, portainerApplicationResources.Services)
}
if application.ID != "" && application.Name != "" && len(portainerApplicationResources.HorizontalPodAutoscalers) > 0 {
updateApplicationWithHorizontalPodAutoscaler(&application, portainerApplicationResources.HorizontalPodAutoscalers)
}
return application
}
// createApplicationFromDeployment creates a K8sApplication from a Deployment
func createApplicationFromDeployment(deployment appsv1.Deployment) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromDeployment(&app, deployment)
return app
}
// createApplicationFromStatefulSet creates a K8sApplication from a StatefulSet
func createApplicationFromStatefulSet(statefulSet appsv1.StatefulSet) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromStatefulSet(&app, statefulSet)
return app
}
// createApplicationFromDaemonSet creates a K8sApplication from a DaemonSet
func createApplicationFromDaemonSet(daemonSet appsv1.DaemonSet) models.K8sApplication {
var app models.K8sApplication
populateApplicationFromDaemonSet(&app, daemonSet)
return app
}
func populateApplicationFromDeployment(application *models.K8sApplication, deployment appsv1.Deployment) {
application.ApplicationType = "Deployment"
application.Kind = "Deployment"
application.ID = string(deployment.UID)
application.ResourcePool = deployment.Namespace
application.Name = deployment.Name
application.ApplicationOwner = deployment.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = deployment.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = deployment.Labels["io.portainer.kubernetes.application.stack"]
application.StackKind = deployment.Labels["io.portainer.kubernetes.application.stackKind"]
application.Labels = deployment.Labels
application.MatchLabels = deployment.Spec.Selector.MatchLabels
application.CreationDate = deployment.CreationTimestamp.Time
application.TotalPodsCount = 0
if deployment.Spec.Replicas != nil {
application.TotalPodsCount = int(*deployment.Spec.Replicas)
}
application.RunningPodsCount = int(deployment.Status.ReadyReplicas)
application.DeploymentType = "Replicated"
application.Metadata = &models.Metadata{
Labels: deployment.Labels,
Annotations: deployment.Annotations,
}
// If the deployment has containers, use the first container's image
if len(deployment.Spec.Template.Spec.Containers) > 0 {
application.Image = deployment.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromStatefulSet(application *models.K8sApplication, statefulSet appsv1.StatefulSet) {
application.Kind = "StatefulSet"
application.ApplicationType = "StatefulSet"
application.ID = string(statefulSet.UID)
application.ResourcePool = statefulSet.Namespace
application.Name = statefulSet.Name
application.ApplicationOwner = statefulSet.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = statefulSet.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = statefulSet.Labels["io.portainer.kubernetes.application.stack"]
application.StackKind = statefulSet.Labels["io.portainer.kubernetes.application.stackKind"]
application.Labels = statefulSet.Labels
application.MatchLabels = statefulSet.Spec.Selector.MatchLabels
application.CreationDate = statefulSet.CreationTimestamp.Time
application.TotalPodsCount = 0
if statefulSet.Spec.Replicas != nil {
application.TotalPodsCount = int(*statefulSet.Spec.Replicas)
}
application.RunningPodsCount = int(statefulSet.Status.ReadyReplicas)
application.DeploymentType = "Replicated"
application.Metadata = &models.Metadata{
Labels: statefulSet.Labels,
Annotations: statefulSet.Annotations,
}
// If the statefulSet has containers, use the first container's image
if len(statefulSet.Spec.Template.Spec.Containers) > 0 {
application.Image = statefulSet.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromDaemonSet(application *models.K8sApplication, daemonSet appsv1.DaemonSet) {
application.Kind = "DaemonSet"
application.ApplicationType = "DaemonSet"
application.ID = string(daemonSet.UID)
application.ResourcePool = daemonSet.Namespace
application.Name = daemonSet.Name
application.ApplicationOwner = daemonSet.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = daemonSet.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = daemonSet.Labels["io.portainer.kubernetes.application.stack"]
application.StackKind = daemonSet.Labels["io.portainer.kubernetes.application.stackKind"]
application.Labels = daemonSet.Labels
application.MatchLabels = daemonSet.Spec.Selector.MatchLabels
application.CreationDate = daemonSet.CreationTimestamp.Time
application.TotalPodsCount = int(daemonSet.Status.DesiredNumberScheduled)
application.RunningPodsCount = int(daemonSet.Status.NumberReady)
application.DeploymentType = "Global"
application.Metadata = &models.Metadata{
Labels: daemonSet.Labels,
Annotations: daemonSet.Annotations,
}
if len(daemonSet.Spec.Template.Spec.Containers) > 0 {
application.Image = daemonSet.Spec.Template.Spec.Containers[0].Image
}
}
func populateApplicationFromPod(application *models.K8sApplication, pod corev1.Pod) {
runningPodsCount := 1
if pod.Status.Phase != corev1.PodRunning {
runningPodsCount = 0
}
application.ApplicationType = "Pod"
application.Kind = "Pod"
application.ID = string(pod.UID)
application.ResourcePool = pod.Namespace
application.Name = pod.Name
application.ApplicationOwner = pod.Labels["io.portainer.kubernetes.application.owner"]
application.StackID = pod.Labels["io.portainer.kubernetes.application.stackid"]
application.StackName = pod.Labels["io.portainer.kubernetes.application.stack"]
application.StackKind = pod.Labels["io.portainer.kubernetes.application.stackKind"]
application.Labels = pod.Labels
application.MatchLabels = pod.Labels
application.CreationDate = pod.CreationTimestamp.Time
application.TotalPodsCount = 1
application.RunningPodsCount = runningPodsCount
application.DeploymentType = string(pod.Status.Phase)
application.Metadata = &models.Metadata{
Labels: pod.Labels,
Annotations: pod.Annotations,
}
// If the pod has containers, use the first container's image
if len(pod.Spec.Containers) > 0 {
application.Image = pod.Spec.Containers[0].Image
}
}
// updateApplicationWithService updates the application with the services that match the application's selector match labels
// and are in the same namespace as the application
func updateApplicationWithService(application *models.K8sApplication, services []corev1.Service) {
for _, service := range services {
serviceSelector := labels.SelectorFromSet(service.Spec.Selector)
if service.Namespace == application.ResourcePool && !serviceSelector.Empty() && serviceSelector.Matches(labels.Set(application.MatchLabels)) {
application.ServiceType = string(service.Spec.Type)
application.Services = append(application.Services, service)
}
}
}
func updateApplicationWithHorizontalPodAutoscaler(application *models.K8sApplication, hpas []autoscalingv2.HorizontalPodAutoscaler) {
for _, hpa := range hpas {
// Check if HPA is in the same namespace as the application
if hpa.Namespace != application.ResourcePool {
continue
}
// Check if the scale target ref matches the application
scaleTargetRef := hpa.Spec.ScaleTargetRef
if scaleTargetRef.Name == application.Name && scaleTargetRef.Kind == application.Kind {
hpaCopy := hpa // Create a local copy
application.HorizontalPodAutoscaler = &hpaCopy
break
}
}
}
// calculatePodResourceUsage calculates the resource usage for a pod in CPU cores and Bytes
func calculatePodResourceUsage(pod corev1.Pod) models.K8sApplicationResource {
resource := models.K8sApplicationResource{}
for _, container := range pod.Spec.Containers {
// CPU cores as a decimal
resource.CPURequest += float64(container.Resources.Requests.Cpu().MilliValue()) / 1000
resource.CPULimit += float64(container.Resources.Limits.Cpu().MilliValue()) / 1000
// Bytes
resource.MemoryRequest += container.Resources.Requests.Memory().Value()
resource.MemoryLimit += container.Resources.Limits.Memory().Value()
}
return resource
}
// GetApplicationFromServiceSelector gets applications based on service selectors
// it matches the service selector with the pod labels
func (kcl *KubeClient) GetApplicationFromServiceSelector(pods []corev1.Pod, service models.K8sServiceInfo, replicaSets []appsv1.ReplicaSet) (*models.K8sApplication, error) {
servicesSelector := labels.SelectorFromSet(service.Selector)
if servicesSelector.Empty() {
return nil, nil
}
for _, pod := range pods {
if servicesSelector.Matches(labels.Set(pod.Labels)) {
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, replicaSets)
}
if len(pod.OwnerReferences) == 0 {
return &models.K8sApplication{
Name: pod.Name,
Kind: "Pod",
}, nil
}
return &models.K8sApplication{
Name: pod.OwnerReferences[0].Name,
Kind: pod.OwnerReferences[0].Kind,
}, nil
}
}
return nil, nil
}
// GetApplicationConfigurationOwnersFromConfigMap gets a list of applications that use a specific ConfigMap
// by checking all pods in the same namespace as the ConfigMap
func (kcl *KubeClient) GetApplicationConfigurationOwnersFromConfigMap(configMap models.K8sConfigMap, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]models.K8sConfigurationOwnerResource, error) {
configurationOwners := []models.K8sConfigurationOwnerResource{}
for _, pod := range pods {
if isPodUsingConfigMap(&pod, configMap) {
kind := "Pod"
name := pod.Name
if len(pod.OwnerReferences) > 0 {
kind = pod.OwnerReferences[0].Kind
name = pod.OwnerReferences[0].Name
}
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, replicaSets)
}
configurationOwners = append(configurationOwners, models.K8sConfigurationOwnerResource{
Name: name,
ResourceKind: kind,
})
}
}
return configurationOwners, nil
}
// GetApplicationConfigurationOwnersFromSecret gets a list of applications that use a specific Secret
// by checking all pods in the same namespace as the Secret
func (kcl *KubeClient) GetApplicationConfigurationOwnersFromSecret(secret models.K8sSecret, pods []corev1.Pod, replicaSets []appsv1.ReplicaSet) ([]models.K8sConfigurationOwnerResource, error) {
configurationOwners := []models.K8sConfigurationOwnerResource{}
for _, pod := range pods {
if isPodUsingSecret(&pod, secret) {
kind := "Pod"
name := pod.Name
if len(pod.OwnerReferences) > 0 {
kind = pod.OwnerReferences[0].Kind
name = pod.OwnerReferences[0].Name
}
if isReplicaSetOwner(pod) {
updateOwnerReferenceToDeployment(&pod, replicaSets)
}
configurationOwners = append(configurationOwners, models.K8sConfigurationOwnerResource{
Name: name,
ResourceKind: kind,
})
}
}
return configurationOwners, nil
}
// fetchUnhealthyApplications fetches applications that failed to schedule any pods
// due to issues like missing resource limits or other scheduling constraints
func fetchUnhealthyApplications(resources PortainerApplicationResources) ([]models.K8sApplication, error) {
var unhealthyApplications []models.K8sApplication
// Process Deployments
for _, deployment := range resources.Deployments {
if hasNoScheduledPods(deployment) {
app := createApplicationFromDeployment(deployment)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
// Process StatefulSets
for _, statefulSet := range resources.StatefulSets {
if hasNoScheduledPods(statefulSet) {
app := createApplicationFromStatefulSet(statefulSet)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
// Process DaemonSets
for _, daemonSet := range resources.DaemonSets {
if hasNoScheduledPods(daemonSet) {
app := createApplicationFromDaemonSet(daemonSet)
addRelatedResourcesToApplication(&app, resources)
unhealthyApplications = append(unhealthyApplications, app)
}
}
return unhealthyApplications, nil
}
// addRelatedResourcesToApplication adds Services and HPA information to the application
func addRelatedResourcesToApplication(app *models.K8sApplication, resources PortainerApplicationResources) {
if app.ID == "" || app.Name == "" {
return
}
if len(resources.Services) > 0 {
updateApplicationWithService(app, resources.Services)
}
if len(resources.HorizontalPodAutoscalers) > 0 {
updateApplicationWithHorizontalPodAutoscaler(app, resources.HorizontalPodAutoscalers)
}
}
// hasNoScheduledPods checks if a workload has completely failed to schedule any pods
// it checks for no replicas desired, i.e. nothing to schedule and see if any pods are running
// if any pods exist at all (even if not ready), it returns false
func hasNoScheduledPods(obj any) bool {
switch resource := obj.(type) {
case appsv1.Deployment:
if resource.Status.Replicas > 0 {
return false
}
return resource.Status.ReadyReplicas == 0 && resource.Status.AvailableReplicas == 0
case appsv1.StatefulSet:
if resource.Status.Replicas > 0 {
return false
}
return resource.Status.ReadyReplicas == 0 && resource.Status.CurrentReplicas == 0
case appsv1.DaemonSet:
if resource.Status.CurrentNumberScheduled > 0 || resource.Status.NumberMisscheduled > 0 {
return false
}
return resource.Status.NumberReady == 0 && resource.Status.DesiredNumberScheduled > 0
default:
return false
}
}
+463
View File
@@ -0,0 +1,463 @@
package cli
import (
"testing"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake"
)
// Helper functions to create test resources
func createTestDeployment(name, namespace string, replicas int32) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("deploy-" + name),
Labels: map[string]string{
"app": name,
},
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": name,
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": name,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: name,
Image: "nginx:latest",
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
},
},
},
},
},
Status: appsv1.DeploymentStatus{
Replicas: replicas,
ReadyReplicas: replicas,
},
}
}
func createTestReplicaSet(name, namespace, deploymentName string) *appsv1.ReplicaSet {
return &appsv1.ReplicaSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("rs-" + name),
OwnerReferences: []metav1.OwnerReference{
{
Kind: "Deployment",
Name: deploymentName,
UID: types.UID("deploy-" + deploymentName),
},
},
},
Spec: appsv1.ReplicaSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": deploymentName,
},
},
},
}
}
func createTestStatefulSet(name, namespace string, replicas int32) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("sts-" + name),
Labels: map[string]string{
"app": name,
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": name,
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": name,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: name,
Image: "redis:latest",
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
},
},
},
},
},
Status: appsv1.StatefulSetStatus{
Replicas: replicas,
ReadyReplicas: replicas,
},
}
}
func createTestDaemonSet(name, namespace string) *appsv1.DaemonSet {
return &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("ds-" + name),
Labels: map[string]string{
"app": name,
},
},
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": name,
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": name,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: name,
Image: "fluentd:latest",
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
},
},
},
},
},
Status: appsv1.DaemonSetStatus{
DesiredNumberScheduled: 2,
NumberReady: 2,
},
}
}
func createTestPod(name, namespace, ownerKind, ownerName string, isRunning bool) *corev1.Pod {
phase := corev1.PodPending
if isRunning {
phase = corev1.PodRunning
}
var ownerReferences []metav1.OwnerReference
if ownerKind != "" && ownerName != "" {
ownerReferences = []metav1.OwnerReference{
{
Kind: ownerKind,
Name: ownerName,
UID: types.UID(ownerKind + "-" + ownerName),
},
}
}
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("pod-" + name),
OwnerReferences: ownerReferences,
Labels: map[string]string{
"app": ownerName,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container-" + name,
Image: "busybox:latest",
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{},
Requests: corev1.ResourceList{},
},
},
},
},
Status: corev1.PodStatus{
Phase: phase,
},
}
}
func createTestService(name, namespace string, selector map[string]string) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("svc-" + name),
},
Spec: corev1.ServiceSpec{
Selector: selector,
Type: corev1.ServiceTypeClusterIP,
},
}
}
func TestGetApplications(t *testing.T) {
t.Parallel()
t.Run("Admin user - Mix of deployments, statefulsets and daemonsets with and without pods", func(t *testing.T) {
// Create a fake K8s client
fakeClient := fake.NewSimpleClientset()
// Setup the test namespace
namespace := "test-namespace"
defaultNamespace := "default"
// Create resources in the test namespace
// 1. Deployment with pods
deployWithPods := createTestDeployment("deploy-with-pods", namespace, 2)
_, err := fakeClient.AppsV1().Deployments(namespace).Create(t.Context(), deployWithPods, metav1.CreateOptions{})
require.NoError(t, err)
replicaSet := createTestReplicaSet("rs-deploy-with-pods", namespace, "deploy-with-pods")
_, err = fakeClient.AppsV1().ReplicaSets(namespace).Create(t.Context(), replicaSet, metav1.CreateOptions{})
require.NoError(t, err)
pod1 := createTestPod("pod1-deploy", namespace, "ReplicaSet", "rs-deploy-with-pods", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod1, metav1.CreateOptions{})
require.NoError(t, err)
pod2 := createTestPod("pod2-deploy", namespace, "ReplicaSet", "rs-deploy-with-pods", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod2, metav1.CreateOptions{})
require.NoError(t, err)
// 2. Deployment without pods (scaled to 0)
deployNoPods := createTestDeployment("deploy-no-pods", namespace, 0)
_, err = fakeClient.AppsV1().Deployments(namespace).Create(t.Context(), deployNoPods, metav1.CreateOptions{})
require.NoError(t, err)
// 3. StatefulSet with pods
stsWithPods := createTestStatefulSet("sts-with-pods", namespace, 1)
_, err = fakeClient.AppsV1().StatefulSets(namespace).Create(t.Context(), stsWithPods, metav1.CreateOptions{})
require.NoError(t, err)
pod3 := createTestPod("pod1-sts", namespace, "StatefulSet", "sts-with-pods", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod3, metav1.CreateOptions{})
require.NoError(t, err)
// 4. StatefulSet without pods
stsNoPods := createTestStatefulSet("sts-no-pods", namespace, 0)
_, err = fakeClient.AppsV1().StatefulSets(namespace).Create(t.Context(), stsNoPods, metav1.CreateOptions{})
require.NoError(t, err)
// 5. DaemonSet with pods
dsWithPods := createTestDaemonSet("ds-with-pods", namespace)
_, err = fakeClient.AppsV1().DaemonSets(namespace).Create(t.Context(), dsWithPods, metav1.CreateOptions{})
require.NoError(t, err)
pod4 := createTestPod("pod1-ds", namespace, "DaemonSet", "ds-with-pods", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod4, metav1.CreateOptions{})
require.NoError(t, err)
pod5 := createTestPod("pod2-ds", namespace, "DaemonSet", "ds-with-pods", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod5, metav1.CreateOptions{})
require.NoError(t, err)
// 6. Naked Pod (no owner reference)
nakedPod := createTestPod("naked-pod", namespace, "", "", true)
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), nakedPod, metav1.CreateOptions{})
require.NoError(t, err)
// 7. Resources in another namespace
deployOtherNs := createTestDeployment("deploy-other-ns", defaultNamespace, 1)
_, err = fakeClient.AppsV1().Deployments(defaultNamespace).Create(t.Context(), deployOtherNs, metav1.CreateOptions{})
require.NoError(t, err)
podOtherNs := createTestPod("pod-other-ns", defaultNamespace, "Deployment", "deploy-other-ns", true)
_, err = fakeClient.CoreV1().Pods(defaultNamespace).Create(t.Context(), podOtherNs, metav1.CreateOptions{})
require.NoError(t, err)
// 8. Add a service (dependency)
service := createTestService("svc-deploy", namespace, map[string]string{"app": "deploy-with-pods"})
_, err = fakeClient.CoreV1().Services(namespace).Create(t.Context(), service, metav1.CreateOptions{})
require.NoError(t, err)
// Create the KubeClient with admin privileges
kubeClient := &KubeClient{
cli: fakeClient,
instanceID: "test-instance",
isKubeAdmin: true,
}
// Test cases
// 1. All resources, no filtering
t.Run("All resources with dependencies", func(t *testing.T) {
apps, err := kubeClient.GetApplications("", "")
require.NoError(t, err)
// We expect 7 resources: 2 deployments + 2 statefulsets + 1 daemonset + 1 naked pod + 1 deployment in other namespace
// Note: Each controller with pods should count once, not per pod
assert.Len(t, apps, 7)
// Verify one of the deployments has services attached
appsWithServices := []models.K8sApplication{}
for _, app := range apps {
if len(app.Services) > 0 {
appsWithServices = append(appsWithServices, app)
}
}
assert.Len(t, appsWithServices, 1)
assert.Equal(t, "deploy-with-pods", appsWithServices[0].Name)
})
// 2. Filter by namespace
t.Run("Filter by namespace", func(t *testing.T) {
apps, err := kubeClient.GetApplications(namespace, "")
require.NoError(t, err)
// We expect 6 resources in the test namespace
assert.Len(t, apps, 6)
// Verify resources from other namespaces are not included
for _, app := range apps {
assert.Equal(t, namespace, app.ResourcePool)
}
})
})
t.Run("Non-admin user - Resources filtered by accessible namespaces", func(t *testing.T) {
// Create a fake K8s client
fakeClient := fake.NewSimpleClientset()
// Setup the test namespaces
namespace1 := "allowed-ns"
namespace2 := "restricted-ns"
// Create resources in the allowed namespace
sts1 := createTestStatefulSet("sts-allowed", namespace1, 1)
_, err := fakeClient.AppsV1().StatefulSets(namespace1).Create(t.Context(), sts1, metav1.CreateOptions{})
require.NoError(t, err)
pod1 := createTestPod("pod-allowed", namespace1, "StatefulSet", "sts-allowed", true)
_, err = fakeClient.CoreV1().Pods(namespace1).Create(t.Context(), pod1, metav1.CreateOptions{})
require.NoError(t, err)
// Add a StatefulSet without pods in the allowed namespace
stsNoPods := createTestStatefulSet("sts-no-pods-allowed", namespace1, 0)
_, err = fakeClient.AppsV1().StatefulSets(namespace1).Create(t.Context(), stsNoPods, metav1.CreateOptions{})
require.NoError(t, err)
// Create resources in the restricted namespace
sts2 := createTestStatefulSet("sts-restricted", namespace2, 1)
_, err = fakeClient.AppsV1().StatefulSets(namespace2).Create(t.Context(), sts2, metav1.CreateOptions{})
require.NoError(t, err)
pod2 := createTestPod("pod-restricted", namespace2, "StatefulSet", "sts-restricted", true)
_, err = fakeClient.CoreV1().Pods(namespace2).Create(t.Context(), pod2, metav1.CreateOptions{})
require.NoError(t, err)
// Create the KubeClient with non-admin privileges (only allowed namespace1)
kubeClient := &KubeClient{
cli: fakeClient,
instanceID: "test-instance",
isKubeAdmin: false,
nonAdminNamespaces: []string{namespace1},
}
// Test that only resources from allowed namespace are returned
apps, err := kubeClient.GetApplications("", "")
require.NoError(t, err)
// We expect 2 resources from the allowed namespace (1 sts with pod + 1 sts without pod)
assert.Len(t, apps, 2)
// Verify resources are from the allowed namespace
for _, app := range apps {
assert.Equal(t, namespace1, app.ResourcePool)
assert.Equal(t, "StatefulSet", app.Kind)
}
// Verify names of returned resources
stsNames := make(map[string]bool)
for _, app := range apps {
stsNames[app.Name] = true
}
assert.True(t, stsNames["sts-allowed"], "Expected StatefulSet 'sts-allowed' was not found")
assert.True(t, stsNames["sts-no-pods-allowed"], "Expected StatefulSet 'sts-no-pods-allowed' was not found")
})
t.Run("Filter by node name", func(t *testing.T) {
// Create a fake K8s client
fakeClient := fake.NewSimpleClientset()
// Setup test namespace
namespace := "node-filter-ns"
nodeName := "worker-node-1"
// Create a deployment with pods on specific node
deploy := createTestDeployment("node-deploy", namespace, 2)
_, err := fakeClient.AppsV1().Deployments(namespace).Create(t.Context(), deploy, metav1.CreateOptions{})
require.NoError(t, err)
// Create ReplicaSet for the deployment
rs := createTestReplicaSet("rs-node-deploy", namespace, "node-deploy")
_, err = fakeClient.AppsV1().ReplicaSets(namespace).Create(t.Context(), rs, metav1.CreateOptions{})
require.NoError(t, err)
// Create 2 pods, one on the specified node, one on a different node
pod1 := createTestPod("pod-on-node", namespace, "ReplicaSet", "rs-node-deploy", true)
pod1.Spec.NodeName = nodeName
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod1, metav1.CreateOptions{})
require.NoError(t, err)
pod2 := createTestPod("pod-other-node", namespace, "ReplicaSet", "rs-node-deploy", true)
pod2.Spec.NodeName = "worker-node-2"
_, err = fakeClient.CoreV1().Pods(namespace).Create(t.Context(), pod2, metav1.CreateOptions{})
require.NoError(t, err)
// Create the KubeClient
kubeClient := &KubeClient{
cli: fakeClient,
instanceID: "test-instance",
isKubeAdmin: true,
}
// Test filtering by node name
apps, err := kubeClient.GetApplications(namespace, nodeName)
require.NoError(t, err)
// We expect to find only the pod on the specified node
assert.Len(t, apps, 1)
if len(apps) > 0 {
assert.Equal(t, "node-deploy", apps[0].Name)
}
})
}
+495
View File
@@ -0,0 +1,495 @@
package cli
import (
"errors"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"sync"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
"github.com/patrickmn/go-cache"
pkgerrors "github.com/pkg/errors"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
metricsv "k8s.io/metrics/pkg/client/clientset/versioned"
gatewaycliv1 "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/typed/apis/v1"
)
const (
defaultKubeClientQPS = 30
defaultKubeClientBurst = 100
maxConcurrency = 30
)
type (
// ClientFactory is used to create Kubernetes clients
ClientFactory struct {
dataStore dataservices.DataStore
reverseTunnelService portainer.ReverseTunnelService
signatureService portainer.DigitalSignatureService
instanceID string
endpointProxyClients *cache.Cache
AddrHTTPS string
}
// KubeClient represent a service used to execute Kubernetes operations
KubeClient struct {
cli kubernetes.Interface
gatewayCLI gatewaycliv1.GatewayV1Interface
instanceID string
mu sync.Mutex
isKubeAdmin bool
nonAdminNamespaces []string
}
)
// NewClientFactory returns a new instance of a ClientFactory
func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*ClientFactory, error) {
if userSessionTimeout == "" {
userSessionTimeout = portainer.DefaultUserSessionTimeout
}
timeout, err := time.ParseDuration(userSessionTimeout)
if err != nil {
return nil, err
}
return &ClientFactory{
dataStore: dataStore,
signatureService: signatureService,
reverseTunnelService: reverseTunnelService,
instanceID: instanceID,
endpointProxyClients: cache.New(timeout, timeout),
AddrHTTPS: addrHTTPS,
}, nil
}
func (factory *ClientFactory) GetInstanceID() (instanceID string) {
return factory.instanceID
}
// Clear removes all cached kube clients
func (factory *ClientFactory) ClearClientCache() {
log.Debug().Msgf("kubernetes namespace permissions have changed, clearing the client cache")
factory.endpointProxyClients.Flush()
}
// ClearClientCache removes all cached kube clients for a userId
func (factory *ClientFactory) ClearUserClientCache(userID string) {
for key := range factory.endpointProxyClients.Items() {
if strings.HasSuffix(key, "."+userID) {
factory.endpointProxyClients.Delete(key)
}
}
}
// Remove the cached kube client so a new one can be created
func (factory *ClientFactory) RemoveKubeClient(endpointID portainer.EndpointID) {
factory.endpointProxyClients.Delete(strconv.Itoa(int(endpointID)))
endpointPrefix := strconv.Itoa(int(endpointID)) + "."
for key := range factory.endpointProxyClients.Items() {
if strings.HasPrefix(key, endpointPrefix) {
factory.endpointProxyClients.Delete(key)
}
}
}
func (factory *ClientFactory) GetAddrHTTPS() string {
return factory.AddrHTTPS
}
// GetPrivilegedKubeClient checks if an existing client is already registered
// for the environment(endpoint) and returns it if one is found.
//
// If no client is registered, it will create a new client, register it, and return it.
func (factory *ClientFactory) GetPrivilegedKubeClient(endpoint *portainer.Endpoint) (*KubeClient, error) {
key := strconv.Itoa(int(endpoint.ID))
pcl, ok := factory.endpointProxyClients.Get(key)
if ok {
return pcl.(*KubeClient), nil
}
kcl, err := factory.createCachedPrivilegedKubeClient(endpoint)
if err != nil {
return nil, err
}
factory.endpointProxyClients.Set(key, kcl, cache.DefaultExpiration)
return kcl, nil
}
// GetPrivilegedUserKubeClient checks if an existing admin client is already
// registered for the environment(endpoint) and user and returns it if one is
// found.
//
// If no client is registered, it will create a new client, register it, and return it.
func (factory *ClientFactory) GetPrivilegedUserKubeClient(endpoint *portainer.Endpoint, userID portainer.UserID) (*KubeClient, error) {
key := strconv.Itoa(int(endpoint.ID)) + ".admin." + strconv.Itoa(int(userID))
pcl, ok := factory.endpointProxyClients.Get(key)
if ok {
return pcl.(*KubeClient), nil
}
kcl, err := factory.createCachedPrivilegedKubeClient(endpoint)
if err != nil {
return nil, err
}
factory.endpointProxyClients.Set(key, kcl, cache.DefaultExpiration)
return kcl, nil
}
// GetProxyKubeClient retrieves a KubeClient from the cache. You should be
// calling SetProxyKubeClient before first. It is normally, called the
// kubernetes middleware.
func (factory *ClientFactory) GetProxyKubeClient(endpointID, userID string) (*KubeClient, bool) {
client, ok := factory.endpointProxyClients.Get(endpointID + "." + userID)
if ok {
return client.(*KubeClient), true
}
return nil, false
}
// SetProxyKubeClient stores a kubeclient in the cache.
func (factory *ClientFactory) SetProxyKubeClient(endpointID, userID string, cli *KubeClient) {
factory.endpointProxyClients.Set(endpointID+"."+userID, cli, cache.DefaultExpiration)
}
// CreateKubeClientFromKubeConfig creates a KubeClient from a clusterID, and
// Kubernetes config.
func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, kubeConfig []byte, IsKubeAdmin bool, NonAdminNamespaces []string) (*KubeClient, error) {
config, err := clientcmd.NewClientConfigFromBytes(kubeConfig)
if err != nil {
return nil, fmt.Errorf("failed to create a client config from kubeconfig: %w", err)
}
clientConfig, err := config.ClientConfig()
if err != nil {
return nil, fmt.Errorf("failed to get the complete client config from kubeconfig: %w", err)
}
clientConfig.QPS = defaultKubeClientQPS
clientConfig.Burst = defaultKubeClientBurst
httpClient, err := rest.HTTPClientFor(clientConfig)
if err != nil {
return nil, fmt.Errorf("failed to create http client for the given config: %w", err)
}
cli, err := kubernetes.NewForConfigAndClient(clientConfig, httpClient)
if err != nil {
return nil, fmt.Errorf("failed to create clientset for the given config: %w", err)
}
gatewayCLI, err := gatewaycliv1.NewForConfigAndClient(clientConfig, httpClient)
if err != nil {
return nil, fmt.Errorf("failed to create gateway clientset for the given config: %w", err)
}
return &KubeClient{
cli: cli,
gatewayCLI: gatewayCLI,
instanceID: factory.instanceID,
isKubeAdmin: IsKubeAdmin,
nonAdminNamespaces: NonAdminNamespaces,
}, nil
}
func (factory *ClientFactory) createCachedPrivilegedKubeClient(endpoint *portainer.Endpoint) (*KubeClient, error) {
cli, gatewayCLI, err := factory.CreateClient(endpoint)
if err != nil {
return nil, err
}
return &KubeClient{
cli: cli,
gatewayCLI: gatewayCLI,
instanceID: factory.instanceID,
isKubeAdmin: true,
}, nil
}
// CreateClient returns a pointer to a new Kubernetes Core Clientset instance.
func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint) (*kubernetes.Clientset, *gatewaycliv1.GatewayV1Client, error) {
switch endpoint.Type {
case portainer.KubernetesLocalEnvironment, portainer.AgentOnKubernetesEnvironment, portainer.EdgeAgentOnKubernetesEnvironment:
c, err := factory.CreateConfig(endpoint)
if err != nil {
return nil, nil, err
}
httpClient, err := rest.HTTPClientFor(c)
if err != nil {
return nil, nil, fmt.Errorf("failed to create http client for the given config: %w", err)
}
cli, err := kubernetes.NewForConfigAndClient(c, httpClient)
if err != nil {
return nil, nil, fmt.Errorf("failed to create clientset for the given config: %w", err)
}
gatewayCLI, err := gatewaycliv1.NewForConfigAndClient(c, httpClient)
if err != nil {
return nil, nil, fmt.Errorf("failed to create gateway clientset for the given config: %w", err)
}
return cli, gatewayCLI, nil
}
return nil, nil, errors.New("unsupported environment type")
}
// CreateConfig returns a pointer to a new kubeconfig ready to create a client.
func (factory *ClientFactory) CreateConfig(endpoint *portainer.Endpoint) (*rest.Config, error) {
switch endpoint.Type {
case portainer.KubernetesLocalEnvironment:
return buildLocalConfig()
case portainer.AgentOnKubernetesEnvironment:
return factory.buildAgentConfig(endpoint)
case portainer.EdgeAgentOnKubernetesEnvironment:
return factory.buildEdgeConfig(endpoint)
}
return nil, errors.New("unsupported environment type")
}
// AgentHeaderRoundTripper decorates HTTP requests with Portainer agent
// authentication headers and retries once on stale-connection errors
type AgentHeaderRoundTripper struct {
signatureHeader string
publicKeyHeader string
roundTripper http.RoundTripper
}
// NewAgentHeaderRoundTripper creates a round tripper that attaches the agent
// authentication headers to every request
func NewAgentHeaderRoundTripper(signatureHeader, publicKeyHeader string, rt http.RoundTripper) *AgentHeaderRoundTripper {
return &AgentHeaderRoundTripper{
signatureHeader: signatureHeader,
publicKeyHeader: publicKeyHeader,
roundTripper: rt,
}
}
func (rt *AgentHeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Add(portainer.PortainerAgentPublicKeyHeader, rt.publicKeyHeader)
req.Header.Add(portainer.PortainerAgentSignatureHeader, rt.signatureHeader)
resp, err := rt.roundTripper.RoundTrip(req)
if err == nil {
return resp, nil
}
// Retry once if a stale pooled connection was reset (non-timeout transport error)
var opErr *net.OpError
if !errors.As(err, &opErr) || opErr.Timeout() {
return nil, err
}
if req.GetBody != nil {
body, bodyErr := req.GetBody()
if bodyErr != nil {
return nil, err
}
req.Body = body
} else if req.Body != nil {
return nil, err
}
return rt.roundTripper.RoundTrip(req)
}
func (factory *ClientFactory) buildAgentConfig(endpoint *portainer.Endpoint) (*rest.Config, error) {
var clientURL strings.Builder
if !strings.HasPrefix(endpoint.URL, "http") {
clientURL.WriteString("https://")
}
clientURL.WriteString(endpoint.URL)
clientURL.WriteString("/kubernetes")
signature, err := factory.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
if err != nil {
return nil, err
}
config, err := clientcmd.BuildConfigFromFlags(clientURL.String(), "")
if err != nil {
return nil, err
}
config.Insecure = true
config.QPS = defaultKubeClientQPS
config.Burst = defaultKubeClientBurst
config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
return NewAgentHeaderRoundTripper(signature, factory.signatureService.EncodedPublicKey(), rt)
})
return config, nil
}
func (factory *ClientFactory) buildEdgeConfig(endpoint *portainer.Endpoint) (*rest.Config, error) {
tunnelAddr, err := factory.reverseTunnelService.TunnelAddr(endpoint)
if err != nil {
return nil, pkgerrors.Wrap(err, "failed to activate the chisel reverse tunnel. check if the tunnel port is open at the portainer instance")
}
endpointURL := fmt.Sprintf("http://%s/kubernetes", tunnelAddr)
config, err := clientcmd.BuildConfigFromFlags(endpointURL, "")
if err != nil {
return nil, err
}
signature, err := factory.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
if err != nil {
return nil, err
}
config.Insecure = true
config.QPS = defaultKubeClientQPS
config.Burst = defaultKubeClientBurst
config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
return NewAgentHeaderRoundTripper(signature, factory.signatureService.EncodedPublicKey(), rt)
})
return config, nil
}
func (factory *ClientFactory) CreateRemoteMetricsClient(endpoint *portainer.Endpoint) (*metricsv.Clientset, error) {
config, err := factory.CreateConfig(endpoint)
if err != nil {
return nil, errors.New("failed to create metrics KubeConfig")
}
return metricsv.NewForConfig(config)
}
func buildLocalConfig() (*rest.Config, error) {
config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
config.QPS = defaultKubeClientQPS
config.Burst = defaultKubeClientBurst
return config, nil
}
func (factory *ClientFactory) MigrateEndpointIngresses(e *portainer.Endpoint, datastore dataservices.DataStore, cli *KubeClient) error {
return datastore.UpdateTx(func(tx dataservices.DataStoreTx) error {
environment, err := tx.Endpoint().Endpoint(e.ID)
if err != nil {
log.Error().Err(err).Msgf("Error retrieving environment %d", e.ID)
return err
}
// classes is a list of controllers which have been manually added to the
// cluster setup view. These need to all be allowed globally, but then
// blocked in specific namespaces which they were not previously allowed in.
classes := environment.Kubernetes.Configuration.IngressClasses
// In pre-2.16 versions of portainer, the namespace level permissions were stored by
// creating an actual ingress rule in the cluster with a particular
// annotation indicating that it's name (the class name) should be allowed.
detected, err := cli.GetIngressControllers()
if err != nil {
log.Error().Err(err).Msgf("Error getting ingress controllers in environment %d", environment.ID)
return err
}
// newControllers is a set of all currently detected controllers.
newControllers := make(map[string]struct{})
for _, controller := range detected {
newControllers[controller.ClassName] = struct{}{}
}
namespaces, err := cli.GetNamespaces()
if err != nil {
log.Error().Err(err).Msgf("Error getting namespaces in environment %d", environment.ID)
return err
}
// Set of namespaces, if any, in which "allow none" should be true.
allow := make(map[string]map[string]struct{})
for _, c := range classes {
allow[c.Name] = make(map[string]struct{})
}
allow["none"] = make(map[string]struct{})
for namespace := range namespaces {
// Compare old annotations with currently detected controllers.
ingresses, err := cli.GetIngresses(namespace)
if err != nil {
log.Error().Err(err).Msgf("Error getting ingresses in environment %d", environment.ID)
return err
}
for _, ingress := range ingresses {
oldController, ok := ingress.Annotations["ingress.portainer.io/ingress-type"]
if !ok {
// Skip rules without our old annotation.
continue
}
if _, ok := newControllers[oldController]; ok {
// Skip rules which match a detected controller.
// TODO: Allow this particular controller.
allow[oldController][ingress.Namespace] = struct{}{}
continue
}
allow["none"][ingress.Namespace] = struct{}{}
}
}
// Locally, disable "allow none" for namespaces not inside shouldAllowNone.
var newClasses []portainer.KubernetesIngressClassConfig
for _, c := range classes {
var blocked []string
for namespace := range namespaces {
if _, ok := allow[c.Name][namespace]; ok {
continue
}
blocked = append(blocked, namespace)
}
newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
Name: c.Name,
Type: c.Type,
GloballyBlocked: false,
BlockedNamespaces: blocked,
})
}
// Handle "none".
if len(allow["none"]) != 0 {
environment.Kubernetes.Configuration.AllowNoneIngressClass = true
var disallowNone []string
for namespace := range namespaces {
if _, ok := allow["none"][namespace]; ok {
continue
}
disallowNone = append(disallowNone, namespace)
}
newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{
Name: "none",
Type: "custom",
GloballyBlocked: false,
BlockedNamespaces: disallowNone,
})
}
environment.Kubernetes.Configuration.IngressClasses = newClasses
environment.PostInitMigrations.MigrateIngresses = false
return tx.Endpoint().UpdateEndpoint(environment.ID, environment)
})
}
+23
View File
@@ -0,0 +1,23 @@
package cli
import (
"testing"
)
func TestClearUserClientCache(t *testing.T) {
t.Parallel()
factory, _ := NewClientFactory(nil, nil, nil, "", "", "")
kcl := &KubeClient{}
factory.endpointProxyClients.Set("12.1", kcl, 0)
factory.endpointProxyClients.Set("12.12", kcl, 0)
factory.endpointProxyClients.Set("12", kcl, 0)
factory.ClearUserClientCache("12")
if len(factory.endpointProxyClients.Items()) != 2 {
t.Errorf("Incorrect clients cached after clearUserClientCache;\ngot=\n%d\nwant=\n%d", len(factory.endpointProxyClients.Items()), 2)
}
if _, ok := factory.GetProxyKubeClient("12", "12"); ok {
t.Errorf("Expected not to find client cache for user after clear")
}
}
+97
View File
@@ -0,0 +1,97 @@
package cli
import (
"context"
"errors"
"slices"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
rbacv1 "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetClusterRoles gets all the clusterRoles for at the cluster level in a k8s endpoint.
// It returns a list of K8sClusterRole objects.
func (kcl *KubeClient) GetClusterRoles() ([]models.K8sClusterRole, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchClusterRoles()
}
return []models.K8sClusterRole{}, errors.New("non-admin users are not allowed to access cluster roles")
}
// fetchClusterRoles returns a list of all Roles in the specified namespace.
func (kcl *KubeClient) fetchClusterRoles() ([]models.K8sClusterRole, error) {
clusterRoles, err := kcl.cli.RbacV1().ClusterRoles().List(context.TODO(), meta.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sClusterRole, 0)
for _, clusterRole := range clusterRoles.Items {
results = append(results, parseClusterRole(clusterRole))
}
return results, nil
}
// parseClusterRole converts a rbacv1.ClusterRole object to a models.K8sClusterRole object.
func parseClusterRole(clusterRole rbacv1.ClusterRole) models.K8sClusterRole {
return models.K8sClusterRole{
Name: clusterRole.Name,
CreationDate: clusterRole.CreationTimestamp.Time,
UID: clusterRole.UID,
IsSystem: isSystemClusterRole(&clusterRole),
}
}
func (kcl *KubeClient) DeleteClusterRoles(req models.K8sClusterRoleDeleteRequests) error {
var errs error
for _, name := range req {
client := kcl.cli.RbacV1().ClusterRoles()
clusterRole, err := client.Get(context.Background(), name, meta.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
// this is a more serious error to do with the client so we return right away
return err
}
if isSystemClusterRole(clusterRole) {
log.Warn().Str("role_name", name).Msg("ignoring delete of 'system' cluster role, not allowed")
}
err = client.Delete(context.Background(), name, meta.DeleteOptions{})
if err != nil {
log.Err(err).Str("role_name", name).Msg("unable to delete the cluster role")
errs = errors.Join(errs, err)
}
}
return errs
}
func isSystemClusterRole(role *rbacv1.ClusterRole) bool {
if role.Namespace == "kube-system" || role.Namespace == "kube-public" ||
role.Namespace == "kube-node-lease" || role.Namespace == "portainer" {
return true
}
if strings.HasPrefix(role.Name, "system:") {
return true
}
if role.Labels != nil {
if role.Labels["kubernetes.io/bootstrapping"] == "rbac-defaults" {
return true
}
}
roles := getPortainerDefaultK8sRoleNames()
return slices.Contains(roles, role.Name)
}
+110
View File
@@ -0,0 +1,110 @@
package cli
import (
"context"
"errors"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
rbacv1 "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetClusterRoleBindings gets all the clusterRoleBindings for at the cluster level in a k8s endpoint.
// It returns a list of K8sClusterRoleBinding objects.
func (kcl *KubeClient) GetClusterRoleBindings() ([]models.K8sClusterRoleBinding, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchClusterRoleBindings()
}
return []models.K8sClusterRoleBinding{}, errors.New("non-admin users are not allowed to access cluster role bindings")
}
// fetchClusterRoleBindings returns a list of all cluster roles in the cluster.
func (kcl *KubeClient) fetchClusterRoleBindings() ([]models.K8sClusterRoleBinding, error) {
clusterRoleBindings, err := kcl.cli.RbacV1().ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sClusterRoleBinding, 0)
for _, clusterRoleBinding := range clusterRoleBindings.Items {
results = append(results, parseClusterRoleBinding(clusterRoleBinding))
}
return results, nil
}
// parseClusterRoleBinding converts a rbacv1.ClusterRoleBinding object to a models.K8sClusterRoleBinding object.
func parseClusterRoleBinding(clusterRoleBinding rbacv1.ClusterRoleBinding) models.K8sClusterRoleBinding {
return models.K8sClusterRoleBinding{
Name: clusterRoleBinding.Name,
UID: clusterRoleBinding.UID,
Namespace: clusterRoleBinding.Namespace,
RoleRef: clusterRoleBinding.RoleRef,
Subjects: clusterRoleBinding.Subjects,
CreationDate: clusterRoleBinding.CreationTimestamp.Time,
IsSystem: isSystemClusterRoleBinding(&clusterRoleBinding),
}
}
// DeleteClusterRoleBindings processes a K8sClusterRoleBindingDeleteRequest
// by deleting each cluster role binding in its given namespace. If deleting a specific cluster role binding
// fails, the error is logged and we continue to delete the remaining cluster role bindings.
func (kcl *KubeClient) DeleteClusterRoleBindings(reqs models.K8sClusterRoleBindingDeleteRequests) error {
var errs error
for _, name := range reqs {
client := kcl.cli.RbacV1().ClusterRoleBindings()
clusterRoleBinding, err := client.Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
// This is a more serious error to do with the client so we return right away
return err
}
if isSystemClusterRoleBinding(clusterRoleBinding) {
log.Warn().Str("role_name", name).Msg("ignoring delete of 'system' cluster role binding, not allowed")
}
if err := client.Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
log.Err(err).Str("role_name", name).Msg("unable to delete the cluster role binding")
errs = errors.Join(errs, err)
}
}
return errs
}
func isSystemClusterRoleBinding(binding *rbacv1.ClusterRoleBinding) bool {
if strings.HasPrefix(binding.Name, "system:") {
return true
}
if binding.Labels != nil {
if binding.Labels["kubernetes.io/bootstrapping"] == "rbac-defaults" {
return true
}
}
for _, sub := range binding.Subjects {
if strings.HasPrefix(sub.Name, "system:") {
return true
}
if sub.Namespace == "kube-system" ||
sub.Namespace == "kube-public" ||
sub.Namespace == "kube-node-lease" ||
sub.Namespace == "portainer" {
return true
}
}
return false
}
+157
View File
@@ -0,0 +1,157 @@
package cli
import (
"context"
"fmt"
"time"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetConfigMaps gets all the ConfigMaps for a given namespace in a k8s endpoint.
// if the user is an admin, all configMaps in the current k8s environment(endpoint) are fetched using the fetchConfigMaps function.
// otherwise, namespaces the non-admin user has access to will be used to filter the configMaps based on the allowed namespaces.
func (kcl *KubeClient) GetConfigMaps(namespace string) ([]models.K8sConfigMap, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchConfigMaps(namespace)
}
return kcl.fetchConfigMapsForNonAdmin(namespace)
}
// fetchConfigMapsForNonAdmin fetches the configMaps in the namespaces the user has access to.
// This function is called when the user is not an admin.
func (kcl *KubeClient) fetchConfigMapsForNonAdmin(namespace string) ([]models.K8sConfigMap, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching configMaps for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
configMaps, err := kcl.fetchConfigMaps(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sConfigMap, 0)
for _, configMap := range configMaps {
if _, ok := nonAdminNamespaceSet[configMap.Namespace]; ok {
results = append(results, configMap)
}
}
return results, nil
}
// fetchConfigMaps gets all the ConfigMaps for a given namespace in a k8s endpoint.
// the result is a list of config maps parsed into a K8sConfigMap struct.
func (kcl *KubeClient) fetchConfigMaps(namespace string) ([]models.K8sConfigMap, error) {
configMaps, err := kcl.cli.CoreV1().ConfigMaps(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := []models.K8sConfigMap{}
for _, configMap := range configMaps.Items {
results = append(results, parseConfigMap(&configMap, false))
}
return results, nil
}
func (kcl *KubeClient) GetConfigMap(namespace, configMapName string) (models.K8sConfigMap, error) {
configMap, err := kcl.cli.CoreV1().ConfigMaps(namespace).Get(context.Background(), configMapName, metav1.GetOptions{})
if err != nil {
return models.K8sConfigMap{}, err
}
return parseConfigMap(configMap, true), nil
}
// parseConfigMap parses a k8s ConfigMap object into a K8sConfigMap struct.
// for get operation, withData will be set to true.
// otherwise, only metadata will be parsed.
func parseConfigMap(configMap *corev1.ConfigMap, withData bool) models.K8sConfigMap {
result := models.K8sConfigMap{
K8sConfiguration: models.K8sConfiguration{
UID: string(configMap.UID),
Name: configMap.Name,
Namespace: configMap.Namespace,
CreationDate: configMap.CreationTimestamp.Time.UTC().Format(time.RFC3339),
Annotations: configMap.Annotations,
Labels: configMap.Labels,
ConfigurationOwner: configMap.Labels[labelPortainerKubeConfigOwner],
ConfigurationOwnerId: configMap.Labels[labelPortainerKubeConfigOwnerId],
},
}
if withData {
result.Data = configMap.Data
}
return result
}
// SetConfigMapsIsUsed combines the config maps with the applications that use them.
// the function fetches all the pods and replica sets in the cluster and checks if the config map is used by any of the pods.
// if the config map is used by a pod, the application that uses the pod is added to the config map.
// otherwise, the config map is returned as is.
func (kcl *KubeClient) SetConfigMapsIsUsed(configMaps *[]models.K8sConfigMap) error {
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources("", metav1.ListOptions{})
if err != nil {
return fmt.Errorf("an error occurred during the SetConfigMapsIsUsed operation, unable to fetch Portainer application resources. Error: %w", err)
}
for i := range *configMaps {
configMap := &(*configMaps)[i]
for _, pod := range portainerApplicationResources.Pods {
if isPodUsingConfigMap(&pod, *configMap) {
configMap.IsUsed = true
break
}
}
}
return nil
}
// CombineConfigMapWithApplications combines the config map with the applications that use it.
// the function fetches all the pods in the cluster and checks if the config map is used by any of the pods.
// it needs to check if the pods are owned by a replica set to determine if the pod is part of a deployment.
func (kcl *KubeClient) CombineConfigMapWithApplications(configMap models.K8sConfigMap) (models.K8sConfigMap, error) {
pods, err := kcl.cli.CoreV1().Pods(configMap.Namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return models.K8sConfigMap{}, fmt.Errorf("an error occurred during the CombineConfigMapWithApplications operation, unable to get pods. Error: %w", err)
}
replicaSetsItems := []appsv1.ReplicaSet{}
if containsReplicaSetOwnerReference(pods) {
replicaSets, err := kcl.cli.AppsV1().ReplicaSets(configMap.Namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return models.K8sConfigMap{}, fmt.Errorf("an error occurred during the CombineConfigMapWithApplications operation, unable to get replica sets. Error: %w", err)
}
replicaSetsItems = replicaSets.Items
}
applicationConfigurationOwners, err := kcl.GetApplicationConfigurationOwnersFromConfigMap(configMap, pods.Items, replicaSetsItems)
if err != nil {
return models.K8sConfigMap{}, fmt.Errorf("an error occurred during the CombineConfigMapWithApplications operation, unable to get applications from config map. Error: %w", err)
}
if len(applicationConfigurationOwners) > 0 {
configMap.ConfigurationOwnerResources = applicationConfigurationOwners
configMap.IsUsed = true
}
return configMap, nil
}
+128
View File
@@ -0,0 +1,128 @@
package cli
import (
"context"
"errors"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
batchv1 "k8s.io/api/batch/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetCronJobs returns all cronjobs in the given namespace
// If the user is a kube admin, it returns all cronjobs in the namespace
// Otherwise, it returns only the cronjobs in the non-admin namespaces
func (kcl *KubeClient) GetCronJobs(namespace string) ([]models.K8sCronJob, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchCronJobs(namespace)
}
return kcl.fetchCronJobsForNonAdmin(namespace)
}
// fetchCronJobsForNonAdmin returns all cronjobs in the given namespace
// It returns only the cronjobs in the non-admin namespaces
func (kcl *KubeClient) fetchCronJobsForNonAdmin(namespace string) ([]models.K8sCronJob, error) {
cronJobs, err := kcl.fetchCronJobs(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sCronJob, 0)
for _, cronJob := range cronJobs {
if _, ok := nonAdminNamespaceSet[cronJob.Namespace]; ok {
results = append(results, cronJob)
}
}
return results, nil
}
// fetchCronJobs returns all cronjobs in the given namespace
// It returns all cronjobs in the namespace
func (kcl *KubeClient) fetchCronJobs(namespace string) ([]models.K8sCronJob, error) {
cronJobs, err := kcl.cli.BatchV1().CronJobs(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
jobs, err := kcl.cli.BatchV1().Jobs(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sCronJob, 0)
for _, cronJob := range cronJobs.Items {
results = append(results, kcl.parseCronJob(cronJob, jobs))
}
return results, nil
}
// parseCronJob converts a batchv1.CronJob object to a models.K8sCronJob object.
func (kcl *KubeClient) parseCronJob(cronJob batchv1.CronJob, jobsList *batchv1.JobList) models.K8sCronJob {
jobs, err := kcl.getCronJobExecutions(cronJob.Name, cronJob.Namespace, jobsList)
if err != nil {
return models.K8sCronJob{}
}
timezone := "<none>"
if cronJob.Spec.TimeZone != nil {
timezone = *cronJob.Spec.TimeZone
}
suspend := false
if cronJob.Spec.Suspend != nil {
suspend = *cronJob.Spec.Suspend
}
var command string
if len(cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers) > 0 {
command = strings.Join(cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Command, " ")
}
return models.K8sCronJob{
Id: string(cronJob.UID),
Name: cronJob.Name,
Namespace: cronJob.Namespace,
Command: command,
Schedule: cronJob.Spec.Schedule,
Timezone: timezone,
Suspend: suspend,
Jobs: jobs,
IsSystem: kcl.isSystemCronJob(cronJob.Namespace),
}
}
func (kcl *KubeClient) isSystemCronJob(namespace string) bool {
return kcl.isSystemNamespace(namespace)
}
// DeleteCronJobs deletes the provided list of cronjobs in its namespace
// it returns an error if any of the cronjobs are not found or if there is an error deleting the cronjobs
func (kcl *KubeClient) DeleteCronJobs(payload models.K8sCronJobDeleteRequests) error {
var errs error
for namespace := range payload {
for _, cronJobName := range payload[namespace] {
client := kcl.cli.BatchV1().CronJobs(namespace)
_, err := client.Get(context.Background(), cronJobName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
errs = errors.Join(errs, err)
}
if err := client.Delete(context.Background(), cronJobName, metav1.DeleteOptions{}); err != nil {
errs = errors.Join(errs, err)
}
}
}
return errs
}
+128
View File
@@ -0,0 +1,128 @@
package cli
import (
"testing"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
// TestFetchCronJobs tests the fetchCronJobs method for both admin and non-admin clients
// It creates a fake Kubernetes client and passes it to the fetchCronJobs method
// It then logs the fetched Cron Jobs
// non-admin client will have access to the default namespace only
func (kcl *KubeClient) TestFetchCronJobs(t *testing.T) {
t.Run("admin client can fetch Cron Jobs from all namespaces", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
kcl.isKubeAdmin = true
cronJobs, err := kcl.GetCronJobs("")
if err != nil {
t.Fatalf("Failed to fetch Cron Jobs: %v", err)
}
t.Logf("Fetched Cron Jobs: %v", cronJobs)
})
t.Run("non-admin client can fetch Cron Jobs from the default namespace only", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
kcl.isKubeAdmin = false
kcl.SetClientNonAdminNamespaces([]string{"default"})
cronJobs, err := kcl.GetCronJobs("")
if err != nil {
t.Fatalf("Failed to fetch Cron Jobs: %v", err)
}
t.Logf("Fetched Cron Jobs: %v", cronJobs)
})
t.Run("delete Cron Jobs", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
_, err := kcl.cli.BatchV1().CronJobs("default").Create(t.Context(), &batchv1.CronJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-cronjob"},
}, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create cron job: %v", err)
}
err = kcl.DeleteCronJobs(models.K8sCronJobDeleteRequests{
"default": []string{"test-cronjob"},
})
if err != nil {
t.Fatalf("Failed to delete Cron Jobs: %v", err)
}
t.Logf("Deleted Cron Jobs")
})
}
// TestGetCronJobExecutionsNamespaceFilter verifies that getCronJobExecutions only returns
// executions belonging to the CronJob's own namespace, even when same-named CronJobs
// exist across multiple namespaces.
func TestGetCronJobExecutionsNamespaceFilter(t *testing.T) {
t.Parallel()
backoffLimit := int32(3)
completions := int32(1)
makeJob := func(name, namespace, cronJobName string) batchv1.Job {
return batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
OwnerReferences: []metav1.OwnerReference{
{Kind: "CronJob", Name: cronJobName},
},
},
Spec: batchv1.JobSpec{
BackoffLimit: &backoffLimit,
Completions: &completions,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{{Name: "worker", Image: "busybox"}},
},
},
},
}
}
// Simulate the cross-namespace job list returned when fetchCronJobs is called with namespace=""
allJobs := &batchv1.JobList{
Items: []batchv1.Job{
makeJob("backup-prod-28001440", "ns-prod", "backup"),
makeJob("backup-test-28001441", "ns-test", "backup"),
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
isKubeAdmin: true,
}
t.Run("returns only executions from the matching namespace", func(t *testing.T) {
result, err := kcl.getCronJobExecutions("backup", "ns-prod", allJobs)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "ns-prod", result[0].Namespace)
assert.Equal(t, "backup-prod-28001440", result[0].Name)
})
t.Run("returns only executions from the other matching namespace", func(t *testing.T) {
result, err := kcl.getCronJobExecutions("backup", "ns-test", allJobs)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "ns-test", result[0].Namespace)
assert.Equal(t, "backup-test-28001441", result[0].Name)
})
}
+262
View File
@@ -0,0 +1,262 @@
package cli
import (
"context"
"github.com/portainer/portainer/api/concurrent"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (kcl *KubeClient) GetDashboard() (models.K8sDashboard, error) {
dashboardData := models.K8sDashboard{}
// Get a list of all the namespaces first
namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), v1.ListOptions{})
if err != nil {
return dashboardData, err
}
getNamespaceCounts := func(namespace string) concurrent.Func {
return func(ctx context.Context) (any, error) {
data := models.K8sDashboard{}
// apps (deployments, statefulsets, daemonsets)
applicationCount, err := getApplicationsCount(ctx, kcl, namespace)
if err != nil {
// skip namespaces we're not allowed access to. But don't return an error so that we
// can still count the other namespaces. Returning an error here will stop concurrent.Run
if errors.IsForbidden(err) {
return nil, nil
}
return nil, err
}
data.ApplicationsCount = applicationCount
// services
serviceCount, err := getServicesCount(ctx, kcl, namespace)
if err != nil {
return nil, err
}
data.ServicesCount = serviceCount
// ingresses
ingressesCount, err := getIngressesCount(ctx, kcl, namespace)
if err != nil {
return nil, err
}
data.IngressesCount = ingressesCount
// configmaps
configMapCount, err := getConfigMapsCount(ctx, kcl, namespace)
if err != nil {
return nil, err
}
data.ConfigMapsCount = configMapCount
// secrets
secretsCount, err := getSecretsCount(ctx, kcl, namespace)
if err != nil {
return nil, err
}
data.SecretsCount = secretsCount
// volumes
volumesCount, err := getVolumesCount(ctx, kcl, namespace)
if err != nil {
return nil, err
}
data.VolumesCount = volumesCount
// count this namespace for the user
data.NamespacesCount = 1
return data, nil
}
}
dashboardTasks := make([]concurrent.Func, 0)
for _, ns := range namespaces.Items {
dashboardTasks = append(dashboardTasks, getNamespaceCounts(ns.Name))
}
// Fetch all the data for each namespace concurrently
results, err := concurrent.Run(context.TODO(), maxConcurrency, dashboardTasks...)
if err != nil {
return dashboardData, err
}
// Sum up the results
for i := range results {
data, _ := results[i].Result.(models.K8sDashboard)
dashboardData.NamespacesCount += data.NamespacesCount
dashboardData.ApplicationsCount += data.ApplicationsCount
dashboardData.ServicesCount += data.ServicesCount
dashboardData.IngressesCount += data.IngressesCount
dashboardData.ConfigMapsCount += data.ConfigMapsCount
dashboardData.SecretsCount += data.SecretsCount
dashboardData.VolumesCount += data.VolumesCount
}
return dashboardData, nil
}
// Get applications excluding nakedpods
func getApplicationsCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
options := v1.ListOptions{Limit: 1}
count := int64(0)
// deployments
deployments, err := kcl.cli.AppsV1().Deployments(namespace).List(ctx, options)
if err != nil {
return 0, err
}
if len(deployments.Items) > 0 {
count = 1 // first deployment
remainingItemsCount := deployments.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining deployments if any
}
}
// StatefulSets
statefulSets, err := kcl.cli.AppsV1().StatefulSets(namespace).List(ctx, options)
if err != nil {
return 0, err
}
if len(statefulSets.Items) > 0 {
count += 1 // + first statefulset
remainingItemsCount := statefulSets.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining statefulsets if any
}
}
// Daemonsets
daemonsets, err := kcl.cli.AppsV1().DaemonSets(namespace).List(ctx, options)
if err != nil {
return 0, err
}
if len(daemonsets.Items) > 0 {
count += 1 // + first daemonset
remainingItemsCount := daemonsets.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining daemonsets if any
}
}
// + (naked pods)
// TODO: Implement fetching of naked pods
// This is to be reworked as part of the dashboard refactor
// nakedPods, err := kcl.GetApplications(namespace, "nakedpods")
// if err != nil {
// return 0, err
// }
// For now, we're not including naked pods in the count
return count, nil
}
// Get the total count of services for the given namespace
func getServicesCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
options := v1.ListOptions{
Limit: 1,
}
var count int64 = 0
services, err := kcl.cli.CoreV1().Services(namespace).List(ctx, options)
if err != nil {
return 0, err
}
if len(services.Items) > 0 {
count = 1 // first service
remainingItemsCount := services.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining services if any
}
}
return count, nil
}
// Get the total count of ingresses for the given namespace
func getIngressesCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
ingresses, err := kcl.cli.NetworkingV1().Ingresses(namespace).List(ctx, v1.ListOptions{Limit: 1})
if err != nil {
return 0, err
}
count := int64(0)
if len(ingresses.Items) > 0 {
count = 1 // first ingress
remainingItemsCount := ingresses.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining ingresses if any
}
}
return count, nil
}
// Get the total count of configMaps for the given namespace
func getConfigMapsCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
configMaps, err := kcl.cli.CoreV1().ConfigMaps(namespace).List(ctx, v1.ListOptions{Limit: 1})
if err != nil {
return 0, err
}
count := int64(0)
if len(configMaps.Items) > 0 {
count = 1 // first configmap
remainingItemsCount := configMaps.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining configmaps if any
}
}
return count, nil
}
// Get the total count of secrets for the given namespace
func getSecretsCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
secrets, err := kcl.cli.CoreV1().Secrets(namespace).List(ctx, v1.ListOptions{Limit: 1})
if err != nil {
return 0, err
}
count := int64(0)
if len(secrets.Items) > 0 {
count = 1 // first secret
remainingItemsCount := secrets.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining secrets if any
}
}
return count, nil
}
// Get the total count of volumes for the given namespace
func getVolumesCount(ctx context.Context, kcl *KubeClient, namespace string) (int64, error) {
volumes, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).List(ctx, v1.ListOptions{Limit: 1})
if err != nil {
return 0, err
}
count := int64(0)
if len(volumes.Items) > 0 {
count = 1 // first volume
remainingItemsCount := volumes.GetRemainingItemCount()
if remainingItemsCount != nil {
count += *remainingItemsCount // add the remaining volumes if any
}
}
return count, nil
}
+22
View File
@@ -0,0 +1,22 @@
package cli
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
labels "k8s.io/apimachinery/pkg/labels"
)
// HasStackName checks whether the given name is used in the given namespace.
func (kcl *KubeClient) HasStackName(namespace string, stackName string) (bool, error) {
querySet := labels.Set{"io.portainer.kubernetes.application.stack": stackName}
listOpts := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(querySet).String()}
list, err := kcl.cli.AppsV1().Deployments(namespace).List(context.TODO(), listOpts)
if err != nil {
return false, err
}
if len(list.Items) > 0 {
return false, nil
}
return true, nil
}
+7
View File
@@ -0,0 +1,7 @@
package cli
import "errors"
// ErrUnauthorized is returned when a non-admin user attempts to access a resource
// outside their permitted namespace scope.
var ErrUnauthorized = errors.New("unauthorized")
+93
View File
@@ -0,0 +1,93 @@
package cli
import (
"context"
models "github.com/portainer/portainer/api/http/models/kubernetes"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetEvents gets all the Events for a given namespace and resource
// If the user is a kube admin, it returns all events in the namespace
// Otherwise, it returns only the events in the non-admin namespaces
func (kcl *KubeClient) GetEvents(namespace string, resourceId string) ([]models.K8sEvent, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchAllEvents(namespace, resourceId)
}
return kcl.fetchEventsForNonAdmin(namespace, resourceId)
}
// fetchEventsForNonAdmin returns all events in the given namespace and resource
// It returns only the events in the non-admin namespaces
func (kcl *KubeClient) fetchEventsForNonAdmin(namespace string, resourceId string) ([]models.K8sEvent, error) {
if len(kcl.GetClientNonAdminNamespaces()) == 0 {
return nil, nil
}
events, err := kcl.fetchAllEvents(namespace, resourceId)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sEvent, 0)
for _, event := range events {
if _, ok := nonAdminNamespaceSet[event.Namespace]; ok {
results = append(results, event)
}
}
return results, nil
}
// fetchEventsForNonAdmin returns all events in the given namespace and resource
// It returns all events in the namespace and resource
func (kcl *KubeClient) fetchAllEvents(namespace string, resourceId string) ([]models.K8sEvent, error) {
options := metav1.ListOptions{}
if resourceId != "" {
options.FieldSelector = "involvedObject.uid=" + resourceId
}
list, err := kcl.cli.CoreV1().Events(namespace).List(context.TODO(), options)
if err != nil {
return nil, err
}
results := make([]models.K8sEvent, 0)
for _, event := range list.Items {
results = append(results, parseEvent(&event))
}
return results, nil
}
func parseEvent(event *corev1.Event) models.K8sEvent {
result := models.K8sEvent{
Type: event.Type,
Name: event.Name,
Message: event.Message,
Reason: event.Reason,
Namespace: event.Namespace,
EventTime: event.EventTime.UTC(),
Kind: event.Kind,
Count: event.Count,
UID: string(event.GetUID()),
InvolvedObjectKind: models.K8sEventInvolvedObject{
Kind: event.InvolvedObject.Kind,
UID: string(event.InvolvedObject.UID),
Name: event.InvolvedObject.Name,
Namespace: event.InvolvedObject.Namespace,
},
}
if !event.LastTimestamp.Time.IsZero() {
result.LastTimestamp = &event.LastTimestamp.Time
}
if !event.FirstTimestamp.Time.IsZero() {
result.FirstTimestamp = &event.FirstTimestamp.Time
}
return result
}
+101
View File
@@ -0,0 +1,101 @@
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
// TestGetEvents tests the GetEvents method
// It creates a fake Kubernetes client and passes it to the GetEvents method
// It then logs the fetched events and validated the data returned
func TestGetEvents(t *testing.T) {
t.Parallel()
t.Run("can get events for resource id when admin", func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "instance",
isKubeAdmin: true,
}
event := corev1.Event{
InvolvedObject: corev1.ObjectReference{UID: "resourceId"},
Action: "something",
ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "myEvent"},
EventTime: metav1.NowMicro(),
Type: "warning",
Message: "This event has a very serious warning",
}
_, err := kcl.cli.CoreV1().Events("default").Create(t.Context(), &event, metav1.CreateOptions{})
require.NoError(t, err, "Failed to create Event")
events, err := kcl.GetEvents("default", "resourceId")
require.NoError(t, err, "Failed to fetch Events")
t.Logf("Fetched Events: %v", events)
require.Len(t, events, 1, "Expected to return 1 event")
assert.Equal(t, event.Message, events[0].Message, "Expected Message to be equal to event message created")
assert.Equal(t, event.Type, events[0].Type, "Expected Type to be equal to event type created")
assert.Equal(t, event.EventTime.UTC(), events[0].EventTime, "Expected EventTime to be saved as a string from event time created")
})
t.Run("can get kubernetes events for non admin namespace when non admin", func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "instance",
isKubeAdmin: false,
nonAdminNamespaces: []string{"nonAdmin"},
}
event := corev1.Event{
InvolvedObject: corev1.ObjectReference{UID: "resourceId"},
Action: "something",
ObjectMeta: metav1.ObjectMeta{Namespace: "nonAdmin", Name: "myEvent"},
EventTime: metav1.NowMicro(),
Type: "warning",
Message: "This event has a very serious warning",
}
_, err := kcl.cli.CoreV1().Events("nonAdmin").Create(t.Context(), &event, metav1.CreateOptions{})
require.NoError(t, err, "Failed to create Event")
events, err := kcl.GetEvents("nonAdmin", "resourceId")
require.NoError(t, err, "Failed to fetch Cron Jobs")
t.Logf("Fetched Events: %v", events)
require.Len(t, events, 1, "Expected to return 1 event")
assert.Equal(t, event.Message, events[0].Message, "Expected Message to be equal to event message created")
assert.Equal(t, event.Type, events[0].Type, "Expected Type to be equal to event type created")
assert.Equal(t, event.EventTime.UTC(), events[0].EventTime, "Expected EventTime to be saved as a string from event time created")
})
t.Run("cannot get kubernetes events for admin namespace when non admin", func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "instance",
isKubeAdmin: false,
nonAdminNamespaces: []string{"nonAdmin"},
}
event := corev1.Event{
InvolvedObject: corev1.ObjectReference{UID: "resourceId"},
Action: "something",
ObjectMeta: metav1.ObjectMeta{Namespace: "admin", Name: "myEvent"},
EventTime: metav1.NowMicro(),
Type: "warning",
Message: "This event has a very serious warning",
}
_, err := kcl.cli.CoreV1().Events("admin").Create(t.Context(), &event, metav1.CreateOptions{})
require.NoError(t, err, "Failed to create Event")
events, err := kcl.GetEvents("admin", "resourceId")
require.NoError(t, err, "Failed to fetch Cron Jobs")
t.Logf("Fetched Events: %v", events)
assert.Empty(t, events, "Expected to return 0 events")
})
}
+101
View File
@@ -0,0 +1,101 @@
package cli
import (
"context"
"errors"
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
utilexec "k8s.io/client-go/util/exec"
)
var (
channelProtocolList = []string{
"v5.channel.k8s.io",
"v4.channel.k8s.io",
"v3.channel.k8s.io",
"v2.channel.k8s.io",
"channel.k8s.io",
}
)
// StartExecProcess starts an exec process inside a container using an in-cluster config.
// This is a blocking operation.
func (kcl *KubeClient) StartExecProcess(params portainer.KubeExecParams) {
config, err := rest.InClusterConfig()
if err != nil {
params.ErrChan <- err
return
}
if !params.UseAdminToken {
config.BearerToken = params.Token
config.BearerTokenFile = ""
}
req := kcl.cli.CoreV1().RESTClient().
Post().
Resource("pods").
Name(params.PodName).
Namespace(params.Namespace).
SubResource("exec")
req.VersionedParams(&v1.PodExecOptions{
Container: params.ContainerName,
Command: params.Command,
Stdin: true,
Stdout: true,
Stderr: true,
TTY: true,
}, scheme.ParameterCodec)
streamOpts := remotecommand.StreamOptions{
Stdin: params.Stdin,
Stdout: params.Stdout,
Tty: true,
TerminalSizeQueue: params.ResizeQueue,
}
// Try WebSocket executor first, fall back to SPDY if it fails
exec, err := remotecommand.NewWebSocketExecutorForProtocols(
config,
"GET", // WebSocket uses GET for the upgrade request
req.URL().String(),
channelProtocolList...,
)
if err == nil {
err = exec.StreamWithContext(context.TODO(), streamOpts)
if err == nil {
params.ErrChan <- nil
return
}
log.Warn().
Err(err).
Str("context", "StartExecProcess").
Msg("WebSocket exec failed, falling back to SPDY")
}
// Fall back to SPDY executor
exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL())
if err != nil {
params.ErrChan <- fmt.Errorf("unable to create SPDY executor: %w", err)
return
}
err = exec.StreamWithContext(context.TODO(), streamOpts)
if err != nil {
var exitError utilexec.ExitError
if !errors.As(err, &exitError) {
params.ErrChan <- fmt.Errorf("unable to start exec process: %w", err)
return
}
}
params.ErrChan <- nil
}
+371
View File
@@ -0,0 +1,371 @@
package cli
import (
"context"
"fmt"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/rs/zerolog/log"
netv1 "k8s.io/api/networking/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (kcl *KubeClient) GetIngressControllers() (models.K8sIngressControllers, error) {
classeses, err := kcl.cli.NetworkingV1().IngressClasses().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
ingresses, err := kcl.GetIngresses("")
if err != nil {
return nil, err
}
usedClasses := make(map[string]struct{})
for _, ingress := range ingresses {
usedClasses[ingress.ClassName] = struct{}{}
}
results := []models.K8sIngressController{}
for _, class := range classeses.Items {
ingressClass := parseIngressClass(class)
if _, ok := usedClasses[class.Name]; ok {
ingressClass.Used = true
}
results = append(results, ingressClass)
}
return results, nil
}
// fetchIngressClasses fetches all the ingress classes in a k8s endpoint.
func (kcl *KubeClient) fetchIngressClasses() ([]models.K8sIngressController, error) {
ingressClasses, err := kcl.cli.NetworkingV1().IngressClasses().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
var controllers []models.K8sIngressController
for _, ingressClass := range ingressClasses.Items {
controllers = append(controllers, parseIngressClass(ingressClass))
}
return controllers, nil
}
// parseIngressClass converts a k8s native ingress class object to a Portainer K8sIngressController object.
func parseIngressClass(ingressClasses netv1.IngressClass) models.K8sIngressController {
ingressContoller := models.K8sIngressController{
Name: ingressClasses.Spec.Controller,
ClassName: ingressClasses.Name,
}
switch {
case strings.Contains(ingressContoller.Name, "nginx"):
ingressContoller.Type = "nginx"
case strings.Contains(ingressContoller.Name, "traefik"):
ingressContoller.Type = "traefik"
default:
ingressContoller.Type = "other"
}
return ingressContoller
}
// GetIngress gets an ingress in a given namespace in a k8s endpoint.
func (kcl *KubeClient) GetIngress(namespace, ingressName string) (models.K8sIngressInfo, error) {
ingress, err := kcl.cli.NetworkingV1().Ingresses(namespace).Get(context.Background(), ingressName, metav1.GetOptions{})
if err != nil {
return models.K8sIngressInfo{}, err
}
return parseIngress(*ingress), nil
}
// GetIngresses gets all the ingresses for a given namespace in a k8s endpoint.
func (kcl *KubeClient) GetIngresses(namespace string) ([]models.K8sIngressInfo, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchIngresses(namespace)
}
return kcl.fetchIngressesForNonAdmin(namespace)
}
// fetchIngressesForNonAdmin gets all the ingresses for non-admin users in a k8s endpoint.
func (kcl *KubeClient) fetchIngressesForNonAdmin(namespace string) ([]models.K8sIngressInfo, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching ingresses for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
ingresses, err := kcl.fetchIngresses(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sIngressInfo, 0)
for _, ingress := range ingresses {
if _, ok := nonAdminNamespaceSet[ingress.Namespace]; ok {
results = append(results, ingress)
}
}
return results, nil
}
// fetchIngresses fetches all the ingresses for a given namespace in a k8s endpoint.
func (kcl *KubeClient) fetchIngresses(namespace string) ([]models.K8sIngressInfo, error) {
ingresses, err := kcl.cli.NetworkingV1().Ingresses(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
ingressClasses, err := kcl.fetchIngressClasses()
if err != nil {
return nil, err
}
results := []models.K8sIngressInfo{}
if len(ingresses.Items) == 0 {
return results, nil
}
for _, ingress := range ingresses.Items {
result := parseIngress(ingress)
if ingress.Spec.IngressClassName != nil {
result.Type = findUsedIngressFromIngressClasses(ingressClasses, *ingress.Spec.IngressClassName).Name
}
results = append(results, result)
}
return results, nil
}
// parseIngress converts a k8s native ingress object to a Portainer K8sIngressInfo object.
func parseIngress(ingress netv1.Ingress) models.K8sIngressInfo {
ingressClassName := ""
if ingress.Spec.IngressClassName != nil {
ingressClassName = *ingress.Spec.IngressClassName
}
result := models.K8sIngressInfo{
Name: ingress.Name,
Namespace: ingress.Namespace,
UID: string(ingress.UID),
Annotations: ingress.Annotations,
Labels: ingress.Labels,
CreationDate: ingress.CreationTimestamp.Time,
ClassName: ingressClassName,
}
for _, tls := range ingress.Spec.TLS {
result.TLS = append(result.TLS, models.K8sIngressTLS{
Hosts: tls.Hosts,
SecretName: tls.SecretName,
})
}
hosts := make(map[string]struct{})
for _, r := range ingress.Spec.Rules {
hosts[r.Host] = struct{}{}
if r.HTTP == nil {
continue
}
for _, p := range r.HTTP.Paths {
var path models.K8sIngressPath
path.IngressName = result.Name
path.Host = r.Host
path.Path = p.Path
if p.PathType != nil {
path.PathType = string(*p.PathType)
}
if p.Backend.Service != nil {
path.ServiceName = p.Backend.Service.Name
path.Port = int(p.Backend.Service.Port.Number)
}
result.Paths = append(result.Paths, path)
}
}
for host := range hosts {
result.Hosts = append(result.Hosts, host)
}
return result
}
// findUsedIngressFromIngressClasses searches for an ingress in a slice of ingress classes and returns the ingress if found.
func findUsedIngressFromIngressClasses(ingressClasses []models.K8sIngressController, className string) models.K8sIngressController {
for _, ingressClass := range ingressClasses {
if ingressClass.ClassName == className {
return ingressClass
}
}
return models.K8sIngressController{}
}
// CreateIngress creates a new ingress in a given namespace in a k8s endpoint.
func (kcl *KubeClient) CreateIngress(namespace string, info models.K8sIngressInfo, owner string) error {
ingress := kcl.convertToK8sIngress(info, owner)
_, err := kcl.cli.NetworkingV1().Ingresses(namespace).Create(context.Background(), &ingress, metav1.CreateOptions{})
if err != nil {
return err
}
return nil
}
// convertToK8sIngress converts a Portainer K8sIngressInfo object to a k8s native Ingress object.
// this is required for create and update operations.
func (kcl *KubeClient) convertToK8sIngress(info models.K8sIngressInfo, owner string) netv1.Ingress {
ingressSpec := netv1.IngressSpec{}
if info.ClassName != "" {
ingressSpec.IngressClassName = &info.ClassName
}
result := netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: info.Name,
Namespace: info.Namespace,
Annotations: info.Annotations,
},
Spec: ingressSpec,
}
labels := make(map[string]string)
labels["io.portainer.kubernetes.ingress.owner"] = stackutils.SanitizeLabel(owner)
result.Labels = labels
tls := []netv1.IngressTLS{}
for _, t := range info.TLS {
tls = append(tls, netv1.IngressTLS{
Hosts: t.Hosts,
SecretName: t.SecretName,
})
}
result.Spec.TLS = tls
rules := make(map[string][]netv1.HTTPIngressPath)
for _, path := range info.Paths {
pathType := netv1.PathType(path.PathType)
rules[path.Host] = append(rules[path.Host], netv1.HTTPIngressPath{
Path: path.Path,
PathType: &pathType,
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: path.ServiceName,
Port: netv1.ServiceBackendPort{
Number: int32(path.Port),
},
},
},
})
}
for rule, paths := range rules {
result.Spec.Rules = append(result.Spec.Rules, netv1.IngressRule{
Host: rule,
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: paths,
},
},
})
}
for _, host := range info.Hosts {
if _, ok := rules[host]; !ok {
result.Spec.Rules = append(result.Spec.Rules, netv1.IngressRule{
Host: host,
})
}
}
return result
}
// DeleteIngresses processes a K8sIngressDeleteRequest by deleting each ingress
// in its given namespace.
func (kcl *KubeClient) DeleteIngresses(reqs models.K8sIngressDeleteRequests) error {
for namespace := range reqs {
for _, ingress := range reqs[namespace] {
err := kcl.cli.NetworkingV1().Ingresses(namespace).Delete(
context.Background(),
ingress,
metav1.DeleteOptions{},
)
if err != nil {
return err
}
}
}
return nil
}
// UpdateIngress updates an existing ingress in a given namespace in a k8s endpoint.
func (kcl *KubeClient) UpdateIngress(namespace string, info models.K8sIngressInfo) error {
ingress := kcl.convertToK8sIngress(info, "")
_, err := kcl.cli.NetworkingV1().Ingresses(namespace).Update(context.Background(), &ingress, metav1.UpdateOptions{})
if err != nil {
return err
}
return nil
}
// CombineIngressWithService combines an ingress with a service that is being used by the ingress.
// this is required to display the service that is being used by the ingress in the UI edit view.
func (kcl *KubeClient) CombineIngressWithService(ingress models.K8sIngressInfo) (models.K8sIngressInfo, error) {
services, err := kcl.GetServices(ingress.Namespace)
if err != nil {
return models.K8sIngressInfo{}, fmt.Errorf("an error occurred during the CombineIngressWithService operation, unable to retrieve services from the Kubernetes for a namespace level user. Error: %w", err)
}
serviceMap := kcl.buildServicesMap(services)
for pathIndex, path := range ingress.Paths {
if _, ok := serviceMap[path.ServiceName]; ok {
ingress.Paths[pathIndex].HasService = true
}
}
return ingress, nil
}
// CombineIngressesWithServices combines a list of ingresses with a list of services that are being used by the ingresses.
// this is required to display the services that are being used by the ingresses in the UI list view.
func (kcl *KubeClient) CombineIngressesWithServices(ingresses []models.K8sIngressInfo) ([]models.K8sIngressInfo, error) {
services, err := kcl.GetServices("")
if err != nil {
if k8serrors.IsUnauthorized(err) {
return nil, fmt.Errorf("an error occurred during the CombineIngressesWithServices operation, unauthorized access to the Kubernetes API. Error: %w", err)
}
return nil, fmt.Errorf("an error occurred during the CombineIngressesWithServices operation, unable to retrieve services from the Kubernetes for a cluster level user. Error: %w", err)
}
serviceMap := kcl.buildServicesMap(services)
for ingressIndex, ingress := range ingresses {
for pathIndex, path := range ingress.Paths {
if _, ok := serviceMap[path.ServiceName]; ok {
(ingresses)[ingressIndex].Paths[pathIndex].HasService = true
}
}
}
return ingresses, nil
}
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetIngresses(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
ingresses, err := kcl.GetIngresses("default")
require.NoError(t, err)
require.Empty(t, ingresses)
}
+252
View File
@@ -0,0 +1,252 @@
package cli
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetJobs returns all jobs in the given namespace
// If the user is a kube admin, it returns all jobs in the namespace
// Otherwise, it returns only the jobs in the non-admin namespaces
func (kcl *KubeClient) GetJobs(namespace string, includeCronJobChildren bool) ([]models.K8sJob, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchJobs(namespace, includeCronJobChildren)
}
return kcl.fetchJobsForNonAdmin(namespace, includeCronJobChildren)
}
// fetchJobsForNonAdmin returns all jobs in the given namespace
// It returns only the jobs in the non-admin namespaces
func (kcl *KubeClient) fetchJobsForNonAdmin(namespace string, includeCronJobChildren bool) ([]models.K8sJob, error) {
jobs, err := kcl.fetchJobs(namespace, includeCronJobChildren)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sJob, 0)
for _, job := range jobs {
if _, ok := nonAdminNamespaceSet[job.Namespace]; ok {
results = append(results, job)
}
}
return results, nil
}
// fetchJobs returns all jobs in the given namespace
// It returns all jobs in the namespace
func (kcl *KubeClient) fetchJobs(namespace string, includeCronJobChildren bool) ([]models.K8sJob, error) {
jobs, err := kcl.cli.BatchV1().Jobs(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sJob, 0)
for _, job := range jobs.Items {
if !includeCronJobChildren && checkCronJobOwner(job) {
continue
}
results = append(results, kcl.parseJob(job))
}
return results, nil
}
// checkCronJobOwner checks if the job has a cronjob owner
// it returns true if the job has a cronjob owner
// otherwise, it returns false
func checkCronJobOwner(job batchv1.Job) bool {
for _, owner := range job.OwnerReferences {
if owner.Kind == "CronJob" {
return true
}
}
return false
}
// parseJob converts a batchv1.Job object to a models.K8sJob object.
func (kcl *KubeClient) parseJob(job batchv1.Job) models.K8sJob {
times := parseJobTimes(job)
status, failedReason := determineJobStatus(job)
podName := getJobPodName(kcl, job)
var command string
var container corev1.Container
if len(job.Spec.Template.Spec.Containers) > 0 {
command = strings.Join(job.Spec.Template.Spec.Containers[0].Command, " ")
container = job.Spec.Template.Spec.Containers[0]
}
var backoffLimit int32
if job.Spec.BackoffLimit != nil {
backoffLimit = *job.Spec.BackoffLimit
}
var completions int32
if job.Spec.Completions != nil {
completions = *job.Spec.Completions
}
return models.K8sJob{
ID: string(job.UID),
Namespace: job.Namespace,
Name: job.Name,
PodName: podName,
Command: command,
Container: container,
BackoffLimit: backoffLimit,
Completions: completions,
StartTime: times.start,
FinishTime: times.finish,
Duration: times.duration,
Status: status,
FailedReason: failedReason,
IsSystem: kcl.isSystemJob(job.Namespace),
}
}
func (kcl *KubeClient) isSystemJob(namespace string) bool {
return kcl.isSystemNamespace(namespace)
}
type jobTimes struct {
start string
finish string
duration string
}
func parseJobTimes(job batchv1.Job) jobTimes {
times := jobTimes{
start: "N/A",
finish: "N/A",
duration: "N/A",
}
st := job.Status.StartTime
if st == nil {
return times
}
times.start = st.Format(time.RFC3339)
times.duration = time.Since(st.Time).Truncate(time.Minute).String()
if ct := job.Status.CompletionTime; ct != nil {
times.finish = ct.Format(time.RFC3339)
times.duration = ct.Time.Sub(st.Time).String()
}
return times
}
func determineJobStatus(job batchv1.Job) (status, failedReason string) {
failedReason = "N/A"
switch {
case job.Status.Failed > 0:
return "Failed", getLatestJobCondition(job.Status.Conditions)
case job.Status.Succeeded > 0:
return "Succeeded", failedReason
case job.Status.Active == 0:
return "Completed", failedReason
default:
return "Running", failedReason
}
}
func getJobPodName(kcl *KubeClient, job batchv1.Job) string {
pod, err := kcl.getLatestJobPod(job.Namespace, job.Name)
if err != nil {
log.Warn().Err(err).
Str("job", job.Name).
Str("namespace", job.Namespace).
Msg("Failed to get latest job pod")
return ""
}
if pod != nil {
return pod.Name
}
return ""
}
// getCronJobExecutions returns the jobs for a given cronjob
// it returns the jobs for the cronjob
func (kcl *KubeClient) getCronJobExecutions(cronJobName string, cronJobNamespace string, jobs *batchv1.JobList) ([]models.K8sJob, error) {
maxItems := 5
results := make([]models.K8sJob, 0)
for _, job := range jobs.Items {
if job.Namespace != cronJobNamespace {
continue
}
for _, owner := range job.OwnerReferences {
if owner.Kind == "CronJob" && owner.Name == cronJobName {
results = append(results, kcl.parseJob(job))
if len(results) >= maxItems {
return results, nil
}
}
}
}
return results, nil
}
// DeleteJobs deletes the provided list of jobs
// it returns an error if any of the jobs are not found or if there is an error deleting the jobs
func (kcl *KubeClient) DeleteJobs(payload models.K8sJobDeleteRequests) error {
var errs error
for namespace := range payload {
for _, jobName := range payload[namespace] {
client := kcl.cli.BatchV1().Jobs(namespace)
_, err := client.Get(context.Background(), jobName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
errs = errors.Join(errs, err)
}
if err := client.Delete(context.Background(), jobName, metav1.DeleteOptions{}); err != nil {
errs = errors.Join(errs, err)
}
}
}
return errs
}
// getLatestJobCondition returns the latest condition of the job
// it returns the latest condition of the job
// this is only used for the failed reason
func getLatestJobCondition(conditions []batchv1.JobCondition) string {
if len(conditions) == 0 {
return "No conditions"
}
sort.Slice(conditions, func(i, j int) bool {
return conditions[i].LastTransitionTime.After(conditions[j].LastTransitionTime.Time)
})
latest := conditions[0]
return fmt.Sprintf("%s: %s", latest.Type, latest.Message)
}
+91
View File
@@ -0,0 +1,91 @@
package cli
import (
"testing"
"time"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/stretchr/testify/require"
batchv1 "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
// TestFetchJobs tests the fetchJobs method for both admin and non-admin clients
// It creates a fake Kubernetes client and passes it to the fetchJobs method
// It then logs the fetched jobs
// non-admin client will have access to the default namespace only
func (kcl *KubeClient) TestFetchJobs(t *testing.T) {
t.Run("admin client can fetch jobs from all namespaces", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
kcl.isKubeAdmin = true
jobs, err := kcl.GetJobs("", false)
if err != nil {
t.Fatalf("Failed to fetch jobs: %v", err)
}
t.Logf("Fetched jobs: %v", jobs)
})
t.Run("non-admin client can fetch jobs from the default namespace only", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
kcl.isKubeAdmin = false
kcl.SetClientNonAdminNamespaces([]string{"default"})
jobs, err := kcl.GetJobs("", false)
if err != nil {
t.Fatalf("Failed to fetch jobs: %v", err)
}
t.Logf("Fetched jobs: %v", jobs)
})
t.Run("delete jobs", func(t *testing.T) {
kcl.cli = kfake.NewSimpleClientset()
kcl.instanceID = "test"
_, err := kcl.cli.BatchV1().Jobs("default").Create(t.Context(), &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{Name: "test-job"},
}, metav1.CreateOptions{})
if err != nil {
t.Fatalf("Failed to create job: %v", err)
}
err = kcl.DeleteJobs(models.K8sJobDeleteRequests{
"default": []string{"test-job"},
})
if err != nil {
t.Fatalf("Failed to delete jobs: %v", err)
}
})
}
func TestParseJobTimes(t *testing.T) {
t.Parallel()
// Empty job
jobTimes := parseJobTimes(batchv1.Job{})
require.Equal(t, "N/A", jobTimes.duration)
require.Equal(t, "N/A", jobTimes.start)
require.Equal(t, "N/A", jobTimes.finish)
// Full job
now := time.Now()
completionTime := now.Add(10 * time.Minute)
jobTimes = parseJobTimes(batchv1.Job{
Status: batchv1.JobStatus{
StartTime: &metav1.Time{Time: now},
CompletionTime: &metav1.Time{Time: completionTime},
},
})
require.Equal(t, "10m0s", jobTimes.duration)
require.Equal(t, now.Format(time.RFC3339), jobTimes.start)
require.Equal(t, completionTime.Format(time.RFC3339), jobTimes.finish)
}
+41
View File
@@ -0,0 +1,41 @@
package cli
import (
"context"
"regexp"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/segmentio/encoding/json"
)
// featureGateEnabled reports whether the named Kubernetes feature gate is active
// on the API server by scraping /metrics and checking the
// kubernetes_feature_enabled gauge for that gate name.
func (kcl *KubeClient) featureGateEnabled(ctx context.Context, gate string) (bool, error) {
raw, err := kcl.cli.CoreV1().RESTClient().Get().
AbsPath("/metrics").
DoRaw(ctx)
if err != nil {
return false, err
}
re := regexp.MustCompile(`(?m)kubernetes_feature_enabled\{[^}]*name="` + regexp.QuoteMeta(gate) + `"[^}]*\}\s+1$`)
return re.Match(raw), nil
}
// SupportsPodRestart reports whether the RestartAllContainersOnContainerExits
// feature gate is active, which is required for the pods/restart subresource.
func (kcl *KubeClient) SupportsPodRestart(ctx context.Context) (bool, error) {
return kcl.featureGateEnabled(ctx, "RestartAllContainersOnContainerExits")
}
func (kcl *KubeClient) GetMetrics() (models.K8sMetrics, error) {
var metrics models.K8sMetrics
resp, err := kcl.cli.CoreV1().RESTClient().Get().AbsPath("apis/metrics.k8s.io/v1beta1/nodes").DoRaw(context.Background())
if err != nil {
return metrics, err
}
err = json.Unmarshal(resp, &metrics)
return metrics, err
}
+95
View File
@@ -0,0 +1,95 @@
package cli
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
func metricsKubeClient(t *testing.T, statusCode int, body string) *KubeClient {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(statusCode)
_, _ = fmt.Fprint(w, body)
}))
t.Cleanup(server.Close)
cli, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL})
require.NoError(t, err)
return &KubeClient{cli: cli}
}
func TestFeatureGateEnabled(t *testing.T) {
t.Parallel()
t.Run("returns true when feature gate is enabled", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{name=\"TestGate\",stage=\"ALPHA\"} 1\n")
ok, err := kcl.featureGateEnabled(t.Context(), "TestGate")
require.NoError(t, err)
assert.True(t, ok)
})
t.Run("returns true when labels are in different order", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{stage=\"ALPHA\",name=\"TestGate\"} 1\n")
ok, err := kcl.featureGateEnabled(t.Context(), "TestGate")
require.NoError(t, err)
assert.True(t, ok)
})
t.Run("returns false when feature gate is disabled", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{name=\"TestGate\",stage=\"ALPHA\"} 0\n")
ok, err := kcl.featureGateEnabled(t.Context(), "TestGate")
require.NoError(t, err)
assert.False(t, ok)
})
t.Run("returns false when feature gate is absent", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{name=\"OtherGate\",stage=\"ALPHA\"} 1\n")
ok, err := kcl.featureGateEnabled(t.Context(), "TestGate")
require.NoError(t, err)
assert.False(t, ok)
})
t.Run("returns error when metrics endpoint is unavailable", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusForbidden, "")
ok, err := kcl.featureGateEnabled(t.Context(), "TestGate")
require.Error(t, err)
assert.False(t, ok)
})
}
func TestSupportsPodRestart(t *testing.T) {
t.Parallel()
t.Run("returns true when feature gate is enabled", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{name=\"RestartAllContainersOnContainerExits\",stage=\"ALPHA\"} 1\n")
ok, err := kcl.SupportsPodRestart(t.Context())
require.NoError(t, err)
assert.True(t, ok)
})
t.Run("returns false when feature gate is absent", func(t *testing.T) {
t.Parallel()
kcl := metricsKubeClient(t, http.StatusOK,
"kubernetes_feature_enabled{name=\"OtherFeature\",stage=\"ALPHA\"} 1\n")
ok, err := kcl.SupportsPodRestart(t.Context())
require.NoError(t, err)
assert.False(t, ok)
})
}
+465
View File
@@ -0,0 +1,465 @@
package cli
import (
"context"
"fmt"
"net/http"
"sort"
"strconv"
"time"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
systemNamespaceLabel = "io.portainer.kubernetes.namespace.system"
namespaceOwnerLabel = "io.portainer.kubernetes.resourcepool.owner"
namespaceNameLabel = "io.portainer.kubernetes.resourcepool.name"
)
func defaultSystemNamespaces() map[string]struct{} {
return map[string]struct{}{
"kube-system": {},
"kube-public": {},
"kube-node-lease": {},
"portainer": {},
}
}
// GetNamespaces gets the namespaces in the current k8s environment(endpoint).
// if the user is an admin, all namespaces in the current k8s environment(endpoint) are fetched using the fetchNamespaces function.
// otherwise, namespaces the non-admin user has access to will be used to filter the namespaces based on the allowed namespaces.
func (kcl *KubeClient) GetNamespaces() (map[string]portainer.K8sNamespaceInfo, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchNamespaces()
}
return kcl.fetchNamespacesForNonAdmin()
}
// fetchNamespacesForNonAdmin gets the namespaces in the current k8s environment(endpoint) for the non-admin user.
func (kcl *KubeClient) fetchNamespacesForNonAdmin() (map[string]portainer.K8sNamespaceInfo, error) {
log.Debug().
Str("context", "fetchNamespacesForNonAdmin").
Msg("Fetching namespaces for non-admin user")
if len(kcl.GetClientNonAdminNamespaces()) == 0 {
return nil, nil
}
namespaces, err := kcl.fetchNamespaces()
if err != nil {
return nil, fmt.Errorf("an error occurred during the fetchNamespacesForNonAdmin operation, unable to list namespaces for the non-admin user: %w", err)
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make(map[string]portainer.K8sNamespaceInfo)
for _, namespace := range namespaces {
if _, exists := nonAdminNamespaceSet[namespace.Name]; exists {
results[namespace.Name] = namespace
}
}
return results, nil
}
// fetchNamespaces gets the namespaces in the current k8s environment(endpoint).
// this function is used by both admin and non-admin users.
// the result gets parsed to a map of namespace name to namespace info.
func (kcl *KubeClient) fetchNamespaces() (map[string]portainer.K8sNamespaceInfo, error) {
namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
if err != nil {
log.Error().
Str("context", "fetchNamespaces").
Err(err).
Msg("Failed to list namespaces")
return nil, fmt.Errorf("an error occurred during the fetchNamespacesForAdmin operation, unable to list namespaces for the admin user: %w", err)
}
results := make(map[string]portainer.K8sNamespaceInfo)
for _, namespace := range namespaces.Items {
results[namespace.Name] = parseNamespace(&namespace)
}
return results, nil
}
// parseNamespace converts a k8s namespace object to a portainer namespace object.
func parseNamespace(namespace *corev1.Namespace) portainer.K8sNamespaceInfo {
return portainer.K8sNamespaceInfo{
Id: string(namespace.UID),
Name: namespace.Name,
Status: namespace.Status,
Annotations: namespace.Annotations,
CreationDate: namespace.CreationTimestamp.Format(time.RFC3339),
NamespaceOwner: namespace.Labels[namespaceOwnerLabel],
IsSystem: isSystemNamespace(namespace),
IsDefault: namespace.Name == defaultNamespace,
}
}
// GetNamespace gets the namespace in the current k8s environment(endpoint).
func (kcl *KubeClient) GetNamespace(name string) (portainer.K8sNamespaceInfo, error) {
if !kcl.GetIsKubeAdmin() {
if _, allowed := kcl.buildNonAdminNamespacesMap()[name]; !allowed {
log.Warn().
Str("context", "GetNamespace").
Str("namespace", name).
Msg("Non-admin user denied access to namespace not in allowed list")
return portainer.K8sNamespaceInfo{}, k8serrors.NewForbidden(
corev1.Resource("namespaces"),
name,
errors.New("user does not have access to this namespace"),
)
}
}
namespace, err := kcl.cli.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
log.Error().
Str("context", "GetNamespace").
Str("namespace", name).
Err(err).
Msg("Failed to get namespace")
return portainer.K8sNamespaceInfo{}, err
}
return parseNamespace(namespace), nil
}
// CreateNamespace creates a new namespace in a k8s endpoint.
func (kcl *KubeClient) CreateNamespace(info models.K8sNamespaceDetails) (*corev1.Namespace, error) {
portainerLabels := map[string]string{
namespaceNameLabel: stackutils.SanitizeLabel(info.Name),
namespaceOwnerLabel: stackutils.SanitizeLabel(info.Owner),
}
var ns corev1.Namespace
ns.Name = info.Name
ns.Annotations = info.Annotations
ns.Labels = portainerLabels
namespace, err := kcl.cli.CoreV1().Namespaces().Create(context.Background(), &ns, metav1.CreateOptions{})
if err != nil {
log.Error().
Err(err).
Str("context", "CreateNamespace").
Str("Namespace", info.Name).
Msg("Failed to create the namespace")
return nil, err
}
if err := kcl.createOrUpdateNamespaceResourceQuota(info, portainerLabels); err != nil {
log.Error().
Err(err).
Str("context", "CreateNamespace").
Str("name", info.Name).
Msg("failed to create or update resource quota for namespace")
return nil, err
}
return namespace, nil
}
// UpdateNamespace updates a namespace in a k8s endpoint.
func (kcl *KubeClient) UpdateNamespace(info models.K8sNamespaceDetails) (*corev1.Namespace, error) {
portainerLabels := map[string]string{
namespaceNameLabel: stackutils.SanitizeLabel(info.Name),
namespaceOwnerLabel: stackutils.SanitizeLabel(info.Owner),
}
namespace := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: info.Name,
Annotations: info.Annotations,
},
}
updatedNamespace, err := kcl.cli.CoreV1().Namespaces().Update(context.Background(), &namespace, metav1.UpdateOptions{})
if err != nil {
log.Error().
Str("context", "UpdateNamespace").
Str("namespace", info.Name).
Err(err).
Msg("Failed to update namespace")
return nil, err
}
if err := kcl.createOrUpdateNamespaceResourceQuota(info, portainerLabels); err != nil {
log.Error().
Err(err).
Str("context", "UpdateNamespace").
Str("name", info.Name).
Msg("failed to create or update resource quota for namespace")
return nil, err
}
return updatedNamespace, nil
}
func (kcl *KubeClient) createOrUpdateNamespaceResourceQuota(info models.K8sNamespaceDetails, portainerLabels map[string]string) error {
if !info.ResourceQuota.Enabled {
if err := kcl.deleteNamespaceResourceQuota(info.Name); err != nil {
log.Debug().Err(err).Str("context", "createOrUpdateNamespaceResourceQuota").Str("name", info.Name).Msg("failed to delete resource quota for namespace")
}
return nil
}
resourceQuota := &corev1.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: "portainer-rq-" + info.Name,
Namespace: info.Name,
Labels: portainerLabels,
},
Spec: corev1.ResourceQuotaSpec{
Hard: corev1.ResourceList{},
},
}
if info.ResourceQuota.Enabled {
memory := resource.MustParse(info.ResourceQuota.Memory)
cpu := resource.MustParse(info.ResourceQuota.CPU)
if memory.Value() > 0 {
memQuota := memory
resourceQuota.Spec.Hard[corev1.ResourceLimitsMemory] = memQuota
resourceQuota.Spec.Hard[corev1.ResourceRequestsMemory] = memQuota
}
if cpu.Value() > 0 {
cpuQuota := cpu
resourceQuota.Spec.Hard[corev1.ResourceLimitsCPU] = cpuQuota
resourceQuota.Spec.Hard[corev1.ResourceRequestsCPU] = cpuQuota
}
}
_, err := kcl.cli.CoreV1().ResourceQuotas(info.Name).Update(context.Background(), resourceQuota, metav1.UpdateOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
log.Warn().
Str("context", "createOrUpdateNamespaceResourceQuota").
Str("name", info.Name).
Msg("resource quota not found, creating")
_, err = kcl.cli.CoreV1().ResourceQuotas(info.Name).Create(context.Background(), resourceQuota, metav1.CreateOptions{})
}
}
return err
}
func (kcl *KubeClient) deleteNamespaceResourceQuota(namespaceName string) error {
err := kcl.cli.CoreV1().ResourceQuotas(namespaceName).Delete(context.Background(), "portainer-rq-"+namespaceName, metav1.DeleteOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
log.Error().
Str("context", "deleteNamespaceResourceQuota").
Str("name", namespaceName).
Err(err).
Msg("failed to delete resource quota for namespace")
return err
}
log.Warn().
Str("context", "deleteNamespaceResourceQuota").
Str("name", namespaceName).
Msg("resource quota to delete not found")
return nil
}
func isSystemNamespace(namespace *corev1.Namespace) bool {
systemLabelValue, hasSystemLabel := namespace.Labels[systemNamespaceLabel]
if hasSystemLabel {
return systemLabelValue == "true"
}
return isSystemDefaultNamespace(namespace.Name)
}
func isSystemDefaultNamespace(namespace string) bool {
systemNamespaces := defaultSystemNamespaces()
_, isSystem := systemNamespaces[namespace]
return isSystem
}
func (kcl *KubeClient) isSystemNamespace(namespace string) bool {
ns, err := kcl.cli.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{})
if err != nil {
return false
}
return isSystemNamespace(ns)
}
// ToggleSystemState will set a namespace as a system namespace, or remove this state
// if isSystem is true it will set `systemNamespaceLabel` to "true" and false otherwise
// this will skip if namespace is "default" or if the required state is already set
func (kcl *KubeClient) ToggleSystemState(namespaceName string, isSystem bool) error {
if namespaceName == "default" {
return nil
}
namespace, err := kcl.cli.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{})
if err != nil {
log.Error().
Str("context", "ToggleSystemState").
Str("namespace", namespaceName).
Err(err).
Msg("failed to get namespace")
return errors.Wrap(err, "failed fetching namespace object")
}
if isSystemNamespace(namespace) == isSystem {
return nil
}
if namespace.Labels == nil {
namespace.Labels = map[string]string{}
}
namespace.Labels[systemNamespaceLabel] = strconv.FormatBool(isSystem)
if _, err := kcl.cli.CoreV1().Namespaces().Update(context.TODO(), namespace, metav1.UpdateOptions{}); err != nil {
log.Error().
Str("context", "ToggleSystemState").
Str("namespace", namespaceName).
Err(err).
Msg("failed updating namespace object")
return errors.Wrap(err, "failed updating namespace object")
}
if isSystem {
return kcl.NamespaceAccessPoliciesDeleteNamespace(namespaceName)
}
return nil
}
func (kcl *KubeClient) DeleteNamespace(namespaceName string) (*corev1.Namespace, error) {
namespace, err := kcl.cli.CoreV1().Namespaces().Get(context.Background(), namespaceName, metav1.GetOptions{})
if err != nil {
log.Error().
Str("context", "DeleteNamespace").
Str("namespace", namespaceName).
Err(err).
Msg("failed fetching namespace object")
return nil, err
}
err = kcl.cli.CoreV1().Namespaces().Delete(context.Background(), namespaceName, metav1.DeleteOptions{})
if err != nil {
log.Error().
Str("context", "DeleteNamespace").
Str("namespace", namespaceName).
Err(err).
Msg("failed deleting namespace object")
return nil, err
}
return namespace, nil
}
// CombineNamespacesWithUnhealthyEvents combines namespaces with unhealthy events across all namespaces
func (kcl *KubeClient) CombineNamespacesWithUnhealthyEvents(namespaces map[string]portainer.K8sNamespaceInfo) (map[string]portainer.K8sNamespaceInfo, error) {
allEvents, err := kcl.GetEvents("", "")
if err != nil && !k8serrors.IsNotFound(err) {
log.Error().
Str("context", "CombineNamespacesWithUnhealthyEvents").
Err(err).
Msg("unable to retrieve unhealthy events from the Kubernetes for an admin user")
return nil, err
}
unhealthyEventCounts := make(map[string]int)
for _, event := range allEvents {
if event.Type == "Warning" {
unhealthyEventCounts[event.Namespace]++
}
}
for namespaceName, namespace := range namespaces {
if count, exists := unhealthyEventCounts[namespaceName]; exists {
namespace.UnhealthyEventCount = count
namespaces[namespaceName] = namespace
}
}
return namespaces, nil
}
// CombineNamespacesWithResourceQuotas combines namespaces with resource quotas where matching is based on "portainer-rq-"+namespace.Name
func (kcl *KubeClient) CombineNamespacesWithResourceQuotas(namespaces map[string]portainer.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError {
resourceQuotas, err := kcl.GetResourceQuotas("")
if err != nil && !k8serrors.IsNotFound(err) {
log.Error().
Str("context", "CombineNamespacesWithResourceQuotas").
Err(err).
Msg("unable to retrieve resource quotas from the Kubernetes for an admin user")
return httperror.InternalServerError("an error occurred during the CombineNamespacesWithResourceQuotas operation, unable to retrieve resource quotas from the Kubernetes for an admin user. Error: ", err)
}
if len(*resourceQuotas) > 0 {
return response.JSON(w, kcl.UpdateNamespacesWithResourceQuotas(namespaces, *resourceQuotas))
}
return response.JSON(w, kcl.ConvertNamespaceMapToSlice(namespaces))
}
// CombineNamespaceWithResourceQuota combines a namespace with a resource quota prefixed with "portainer-rq-"+namespace.Name
func (kcl *KubeClient) CombineNamespaceWithResourceQuota(namespace portainer.K8sNamespaceInfo, w http.ResponseWriter) *httperror.HandlerError {
resourceQuota, err := kcl.GetPortainerResourceQuota(namespace.Name)
if err != nil && !k8serrors.IsNotFound(err) {
log.Error().
Str("context", "CombineNamespaceWithResourceQuota").
Str("namespace", namespace.Name).
Err(err).
Msg("unable to retrieve the resource quota associated with the namespace")
return httperror.InternalServerError(fmt.Sprintf("an error occurred during the CombineNamespaceWithResourceQuota operation, unable to retrieve the resource quota associated with the namespace: %s for a non-admin user. Error: ", namespace.Name), err)
}
if resourceQuota != nil {
namespace.ResourceQuota = resourceQuota
}
return response.JSON(w, namespace)
}
// buildNonAdminNamespacesMap builds a map of non-admin namespaces.
// the map is used to filter the namespaces based on the allowed namespaces.
func (kcl *KubeClient) buildNonAdminNamespacesMap() map[string]struct{} {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
nonAdminNamespaceSet := make(map[string]struct{}, len(nonAdminNamespaces))
for _, namespace := range nonAdminNamespaces {
if !isSystemDefaultNamespace(namespace) {
nonAdminNamespaceSet[namespace] = struct{}{}
}
}
return nonAdminNamespaceSet
}
// ConvertNamespaceMapToSlice converts the namespace map to a slice of namespaces.
// this is used to for the API response.
func (kcl *KubeClient) ConvertNamespaceMapToSlice(namespaces map[string]portainer.K8sNamespaceInfo) []portainer.K8sNamespaceInfo {
namespaceSlice := make([]portainer.K8sNamespaceInfo, 0, len(namespaces))
for _, namespace := range namespaces {
namespaceSlice = append(namespaceSlice, namespace)
}
// Sort namespaces by name
sort.Slice(namespaceSlice, func(i, j int) bool {
return namespaceSlice[i].Name < namespaceSlice[j].Name
})
return namespaceSlice
}
+244
View File
@@ -0,0 +1,244 @@
package cli
import (
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
core "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func Test_ToggleSystemState(t *testing.T) {
t.Parallel()
t.Run("should skip is default (exit without error)", func(t *testing.T) {
nsName := "default"
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}}),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, true)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
_, exists := ns.Labels[systemNamespaceLabel]
assert.False(t, exists, "system label should not exists")
})
t.Run("should fail if namespace doesn't exist", func(t *testing.T) {
nsName := "not-exist"
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, true)
require.Error(t, err)
})
t.Run("if called with the same state, should skip (exit without error)", func(t *testing.T) {
nsName := "namespace"
tests := []struct {
isSystem bool
}{
{isSystem: true},
{isSystem: false},
}
for _, test := range tests {
t.Run(strconv.FormatBool(test.isSystem), func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName, Labels: map[string]string{
systemNamespaceLabel: strconv.FormatBool(test.isSystem),
}}}),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, test.isSystem)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
assert.Equal(t, test.isSystem, isSystemNamespace(ns))
})
}
})
t.Run("for regular namespace if isSystem is true and doesn't have a label, should set the label to true", func(t *testing.T) {
nsName := "namespace"
config := &core.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: portainerConfigMapName,
Namespace: portainerNamespace,
},
Data: map[string]string{
"NamespaceAccessPolicies": `{"ns1":{"UserAccessPolicies":{"2":{"RoleId":0}}}, "ns2":{"UserAccessPolicies":{"2":{"RoleId":0}}}}`,
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}}, config),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, true)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
labelValue, exists := ns.Labels[systemNamespaceLabel]
assert.True(t, exists, "system label should exists")
assert.Equal(t, "true", labelValue)
})
t.Run("for default system namespace if isSystem is false and doesn't have a label, should set the label to false", func(t *testing.T) {
nsName := "portainer"
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}}),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, false)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
labelValue, exists := ns.Labels[systemNamespaceLabel]
assert.True(t, exists, "system label should exists")
assert.Equal(t, "false", labelValue)
})
t.Run("for system namespace (with label), if called with false, should set the label", func(t *testing.T) {
nsName := "namespace"
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName, Labels: map[string]string{
systemNamespaceLabel: "true",
}}}),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, false)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
labelValue, exists := ns.Labels[systemNamespaceLabel]
assert.True(t, exists, "system label should exists")
assert.Equal(t, "false", labelValue)
})
t.Run("for non system namespace (with label), if called with true, should set the label, and remove accesses", func(t *testing.T) {
nsName := "ns1"
namespace := &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName, Labels: map[string]string{
systemNamespaceLabel: "false",
}}}
config := &core.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: portainerConfigMapName,
Namespace: portainerNamespace,
},
Data: map[string]string{
"NamespaceAccessPolicies": `{"ns1":{"UserAccessPolicies":{"2":{"RoleId":0}}}, "ns2":{"UserAccessPolicies":{"2":{"RoleId":0}}}}`,
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(namespace, config),
instanceID: "instance",
}
err := kcl.ToggleSystemState(nsName, true)
require.NoError(t, err)
ns, err := kcl.cli.CoreV1().Namespaces().Get(t.Context(), nsName, metav1.GetOptions{})
require.NoError(t, err)
labelValue, exists := ns.Labels[systemNamespaceLabel]
assert.True(t, exists, "system label should exists")
assert.Equal(t, "true", labelValue)
expectedPolicies := map[string]portainer.K8sNamespaceAccessPolicy{
"ns2": {UserAccessPolicies: portainer.UserAccessPolicies{2: {RoleID: 0}}},
}
actualPolicies, err := kcl.GetNamespaceAccessPolicies()
require.NoError(t, err, "failed to fetch policies")
assert.Equal(t, expectedPolicies, actualPolicies)
})
}
func Test_GetNamespace(t *testing.T) {
t.Parallel()
newClient := func() *KubeClient {
return &KubeClient{
cli: kfake.NewSimpleClientset(
&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-1"}},
&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-2"}},
&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "kube-system"}},
),
instanceID: "instance",
}
}
t.Run("admin can fetch any namespace", func(t *testing.T) {
kcl := newClient()
kcl.SetIsKubeAdmin(true)
kcl.SetClientNonAdminNamespaces(nil)
for _, name := range []string{"ns-1", "ns-2", "kube-system"} {
ns, err := kcl.GetNamespace(name)
require.NoError(t, err)
assert.Equal(t, name, ns.Name)
}
})
t.Run("non-admin can fetch a namespace in their namespace access", func(t *testing.T) {
kcl := newClient()
kcl.SetIsKubeAdmin(false)
kcl.SetClientNonAdminNamespaces([]string{"ns-1"})
ns, err := kcl.GetNamespace("ns-1")
require.NoError(t, err)
assert.Equal(t, "ns-1", ns.Name)
})
t.Run("non-admin is forbidden from a namespace outside their namespace access", func(t *testing.T) {
kcl := newClient()
kcl.SetIsKubeAdmin(false)
kcl.SetClientNonAdminNamespaces([]string{"ns-1"})
_, err := kcl.GetNamespace("ns-2")
require.Error(t, err)
assert.True(t, k8serrors.IsForbidden(err), "expected a Forbidden error, got %v", err)
})
t.Run("non-admin with no namespace access is forbidden from any namespace", func(t *testing.T) {
kcl := newClient()
kcl.SetIsKubeAdmin(false)
kcl.SetClientNonAdminNamespaces(nil)
_, err := kcl.GetNamespace("ns-1")
require.Error(t, err)
assert.True(t, k8serrors.IsForbidden(err), "expected a Forbidden error, got %v", err)
})
}
+34
View File
@@ -0,0 +1,34 @@
package cli
import (
"fmt"
)
const (
defaultNamespace = "default"
portainerNamespace = "portainer"
portainerUserCRName = "portainer-cr-user"
portainerUserCRBName = "portainer-crb-user"
portainerClusterAdminServiceAccountName = "portainer-sa-clusteradmin"
portainerUserServiceAccountPrefix = "portainer-sa-user"
portainerRBPrefix = "portainer-rb"
portainerConfigMapName = "portainer-config"
portainerConfigMapAccessPoliciesKey = "NamespaceAccessPolicies"
portainerShellPodPrefix = "portainer-pod-kubectl-shell"
)
func UserServiceAccountName(userID int, instanceID string) string {
return fmt.Sprintf("%s-%s-%d", portainerUserServiceAccountPrefix, instanceID, userID)
}
func userServiceAccountTokenSecretName(serviceAccountName string, instanceID string) string {
return fmt.Sprintf("%s-%s-secret", instanceID, serviceAccountName)
}
func namespaceClusterRoleBindingName(namespace string, instanceID string) string {
return fmt.Sprintf("%s-%s-%s", portainerRBPrefix, instanceID, namespace)
}
func userShellPodPrefix(serviceAccountName string) string {
return fmt.Sprintf("%s-%s-", portainerShellPodPrefix, serviceAccountName)
}
+22
View File
@@ -0,0 +1,22 @@
package cli
import (
"context"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetClusterNodes returns all nodes from the Kubernetes cluster.
func (kcl *KubeClient) GetClusterNodes() ([]corev1.Node, error) {
nodes, err := kcl.cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
for i := range nodes.Items {
nodes.Items[i].ManagedFields = nil
}
return nodes.Items, nil
}
+104
View File
@@ -0,0 +1,104 @@
package cli
import (
"context"
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetNodesLimits gets the CPU and Memory limits(unused resources) of all nodes in the current k8s environment(endpoint) connection
func (kcl *KubeClient) GetNodesLimits() (portainer.K8sNodesLimits, error) {
nodesLimits := make(portainer.K8sNodesLimits)
nodes, err := kcl.cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
pods, err := kcl.cli.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
for _, item := range nodes.Items {
cpu := item.Status.Allocatable.Cpu().MilliValue()
memory := item.Status.Allocatable.Memory().Value() // bytes
nodesLimits[item.Name] = &portainer.K8sNodeLimits{
CPU: cpu,
Memory: memory,
}
}
for _, item := range pods.Items {
if nodeLimits, ok := nodesLimits[item.Spec.NodeName]; ok {
for _, container := range item.Spec.Containers {
nodeLimits.CPU -= container.Resources.Requests.Cpu().MilliValue()
nodeLimits.Memory -= container.Resources.Requests.Memory().Value()
}
}
}
return nodesLimits, nil
}
// GetMaxResourceLimits gets the maximum CPU and Memory limits(unused resources) of all nodes in the current k8s environment(endpoint) connection, minus the accumulated resourcequotas for all namespaces except the one we're editing (skipNamespace)
// if skipNamespace is set to "" then all namespaces are considered
func (kcl *KubeClient) GetMaxResourceLimits(skipNamespace string, overCommitEnabled bool, resourceOverCommitPercent int) (portainer.K8sNodeLimits, error) {
limits := portainer.K8sNodeLimits{}
nodes, err := kcl.cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return limits, err
}
// accumulated node limits
memory := int64(0)
for _, node := range nodes.Items {
limits.CPU += node.Status.Allocatable.Cpu().MilliValue()
memory += node.Status.Allocatable.Memory().Value() // bytes
}
limits.Memory = memory / 1000000 // B to MB
if !overCommitEnabled {
namespaces, err := kcl.cli.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return limits, err
}
reservedPercent := float64(resourceOverCommitPercent) / 100.0
reserved := portainer.K8sNodeLimits{}
for _, namespace := range namespaces.Items {
// skip the namespace we're editing
if namespace.Name == skipNamespace {
continue
}
// minus accumulated resourcequotas for all namespaces except the one we're editing
resourceQuota, err := kcl.cli.CoreV1().ResourceQuotas(namespace.Name).List(context.TODO(), metav1.ListOptions{})
if err != nil {
log.Debug().Msgf("error getting resourcequota for namespace %s: %s", namespace.Name, err)
continue // skip it
}
for _, rq := range resourceQuota.Items {
hardLimits := rq.Status.Hard
for resourceType, limit := range hardLimits {
switch resourceType {
case "limits.cpu":
reserved.CPU += limit.MilliValue()
case "limits.memory":
reserved.Memory += limit.ScaledValue(6) // MB
}
}
}
}
limits.CPU = limits.CPU - int64(float64(limits.CPU)*reservedPercent) - reserved.CPU
limits.Memory = limits.Memory - int64(float64(limits.Memory)*reservedPercent) - reserved.Memory
}
return limits, nil
}
+140
View File
@@ -0,0 +1,140 @@
package cli
import (
"reflect"
"testing"
portainer "github.com/portainer/portainer/api"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
kfake "k8s.io/client-go/kubernetes/fake"
)
func newNodes() *v1.NodeList {
return &v1.NodeList{
Items: []v1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "test-node-0",
},
Status: v1.NodeStatus{
Allocatable: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
v1.ResourceMemory: resource.MustParse("4M"),
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "test-node-1",
},
Status: v1.NodeStatus{
Allocatable: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("3"),
v1.ResourceMemory: resource.MustParse("6M"),
},
},
},
},
}
}
func newPods() *v1.PodList {
return &v1.PodList{
Items: []v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Name: "test-container-0",
Namespace: "test-namespace-0",
},
Spec: v1.PodSpec{
NodeName: "test-node-0",
Containers: []v1.Container{
{
Name: "test-container-0",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1"),
v1.ResourceMemory: resource.MustParse("2M"),
},
},
},
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "test-container-1",
Namespace: "test-namespace-1",
},
Spec: v1.PodSpec{
NodeName: "test-node-1",
Containers: []v1.Container{
{
Name: "test-container-1",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
v1.ResourceMemory: resource.MustParse("3M"),
},
},
},
},
},
},
},
}
}
func TestKubeClient_GetNodesLimits(t *testing.T) {
t.Parallel()
type fields struct {
cli kubernetes.Interface
}
fieldsInstance := fields{
cli: kfake.NewSimpleClientset(newNodes(), newPods()),
}
tests := []struct {
name string
fields fields
want portainer.K8sNodesLimits
wantErr bool
}{
{
name: "2 nodes 2 pods",
fields: fieldsInstance,
want: portainer.K8sNodesLimits{
"test-node-0": &portainer.K8sNodeLimits{
CPU: 1000,
Memory: 2000000,
},
"test-node-1": &portainer.K8sNodeLimits{
CPU: 1000,
Memory: 3000000,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
kcl := &KubeClient{
cli: tt.fields.cli,
}
got, err := kcl.GetNodesLimits()
if (err != nil) != tt.wantErr {
t.Errorf("GetNodesLimits() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetNodesLimits() got = %v, want %v", got, tt.want)
}
})
}
}
+64
View File
@@ -0,0 +1,64 @@
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func TestGetClusterNodes_ReturnsAllNodes(t *testing.T) {
t.Parallel()
nodeList := &corev1.NodeList{
Items: []corev1.Node{
{ObjectMeta: metav1.ObjectMeta{Name: "node-0"}},
{ObjectMeta: metav1.ObjectMeta{Name: "node-1"}},
},
}
kcl := &KubeClient{cli: kfake.NewSimpleClientset(nodeList)}
got, err := kcl.GetClusterNodes()
require.NoError(t, err)
assert.Len(t, got, 2)
assert.Equal(t, "node-0", got[0].Name)
assert.Equal(t, "node-1", got[1].Name)
}
func TestGetClusterNodes_EmptyCluster(t *testing.T) {
t.Parallel()
kcl := &KubeClient{cli: kfake.NewSimpleClientset()}
got, err := kcl.GetClusterNodes()
require.NoError(t, err)
assert.Empty(t, got)
}
func TestGetClusterNodes_StripsManagedFields(t *testing.T) {
t.Parallel()
nodeList := &corev1.NodeList{
Items: []corev1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-0",
ManagedFields: []metav1.ManagedFieldsEntry{
{Manager: "kubectl"},
},
},
},
},
}
kcl := &KubeClient{cli: kfake.NewSimpleClientset(nodeList)}
got, err := kcl.GetClusterNodes()
require.NoError(t, err)
require.Len(t, got, 1)
assert.Nil(t, got[0].ManagedFields)
}
@@ -0,0 +1,255 @@
package cli
import (
"context"
"errors"
"fmt"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
// GetPersistentVolumeClaims returns all PVCs in the given namespace (or all namespaces if empty).
// For non-admin users, results are filtered to their accessible namespaces.
func (kcl *KubeClient) GetPersistentVolumeClaims(namespace string) ([]models.K8sPersistentVolumeClaim, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchPersistentVolumeClaims(namespace)
}
return kcl.fetchPersistentVolumeClaimsForNonAdmin(namespace)
}
// GetPersistentVolumeClaim returns a single PVC by namespace and name.
func (kcl *KubeClient) GetPersistentVolumeClaim(namespace, name string) (*models.K8sPersistentVolumeClaim, error) {
if !kcl.GetIsKubeAdmin() {
allowedNamespaces := kcl.buildNonAdminNamespacesMap()
if _, ok := allowedNamespaces[namespace]; !ok {
return nil, fmt.Errorf("access to namespace %s is not authorized: %w", namespace, ErrUnauthorized)
}
}
pvc, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("unable to get persistent volume claim %s/%s. Error: %w", namespace, name, err)
}
result := parsePersistentVolumeClaimDetail(pvc)
return &result, nil
}
// DeletePersistentVolumeClaims deletes the specified PVCs.
func (kcl *KubeClient) DeletePersistentVolumeClaims(reqs models.K8sVolumeDeleteRequests) error {
for _, req := range reqs {
log.Debug().
Str("context", "DeletePersistentVolumeClaims").
Str("namespace", req.Namespace).
Str("name", req.Name).
Msg("Deleting persistent volume claim")
err := kcl.cli.CoreV1().PersistentVolumeClaims(req.Namespace).Delete(context.Background(), req.Name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("unable to delete persistent volume claim %s/%s. Error: %w", req.Namespace, req.Name, err)
}
}
return nil
}
// ResizePersistentVolumeClaim resizes a PVC to the given new size.
// The storage class must have AllowVolumeExpansion set to true.
func (kcl *KubeClient) ResizePersistentVolumeClaim(namespace, name, newSize string) error {
pvc, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("unable to get persistent volume claim %s/%s. Error: %w", namespace, name, err)
}
if pvc.Spec.StorageClassName != nil {
sc, err := kcl.GetStorageClass(*pvc.Spec.StorageClassName)
if err != nil {
return fmt.Errorf("unable to get storage class %s. Error: %w", *pvc.Spec.StorageClassName, err)
}
if sc.AllowVolumeExpansion == nil || !*sc.AllowVolumeExpansion {
return errors.New("storage class " + sc.Name + " does not allow volume expansion")
}
}
_, err = resource.ParseQuantity(newSize)
if err != nil {
return fmt.Errorf("invalid size format %q. Error: %w", newSize, err)
}
patch := fmt.Sprintf(`{"spec":{"resources":{"requests":{"storage":"%s"}}}}`, newSize)
_, err = kcl.cli.CoreV1().PersistentVolumeClaims(namespace).Patch(
context.Background(),
name,
types.MergePatchType,
[]byte(patch),
metav1.PatchOptions{},
)
if err != nil {
return fmt.Errorf("unable to resize persistent volume claim %s/%s. Error: %w", namespace, name, err)
}
return nil
}
func (kcl *KubeClient) fetchPersistentVolumeClaims(namespace string) ([]models.K8sPersistentVolumeClaim, error) {
pvcList, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("unable to list persistent volume claims. Error: %w", err)
}
storageClasses, err := kcl.GetStorageClasses()
if err != nil {
return nil, fmt.Errorf("unable to list storage classes for volume expansion lookup. Error: %w", err)
}
scExpansionMap := make(map[string]bool, len(storageClasses))
for _, sc := range storageClasses {
if sc.AllowVolumeExpansion != nil {
scExpansionMap[sc.Name] = *sc.AllowVolumeExpansion
}
}
results := make([]models.K8sPersistentVolumeClaim, 0, len(pvcList.Items))
for i := range pvcList.Items {
pvc := parsePersistentVolumeClaimDetail(&pvcList.Items[i])
if pvc.StorageClass != nil {
pvc.AllowVolumeExpansion = scExpansionMap[*pvc.StorageClass]
}
results = append(results, pvc)
}
return results, nil
}
func (kcl *KubeClient) fetchPersistentVolumeClaimsForNonAdmin(namespace string) ([]models.K8sPersistentVolumeClaim, error) {
pvcs, err := kcl.fetchPersistentVolumeClaims(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sPersistentVolumeClaim, 0)
for _, pvc := range pvcs {
if _, ok := nonAdminNamespaceSet[pvc.Namespace]; ok {
results = append(results, pvc)
}
}
return results, nil
}
// parsePersistentVolumeClaimDetail parses a PVC into the model with access modes and storage request string.
func parsePersistentVolumeClaimDetail(pvc *corev1.PersistentVolumeClaim) models.K8sPersistentVolumeClaim {
storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage]
return models.K8sPersistentVolumeClaim{
ID: string(pvc.UID),
Name: pvc.Name,
Namespace: pvc.Namespace,
CreationDate: pvc.CreationTimestamp.Time,
Storage: storage.Value(),
StorageRequest: storage.String(),
AccessModes: humanReadableAccessModes(pvc.Spec.AccessModes),
HumanReadableAccessModes: pvc.Spec.AccessModes,
VolumeName: pvc.Spec.VolumeName,
ResourcesRequests: &pvc.Spec.Resources.Requests,
StorageClass: pvc.Spec.StorageClassName,
VolumeMode: pvc.Spec.VolumeMode,
OwningApplications: nil,
Phase: pvc.Status.Phase,
Labels: pvc.Labels,
}
}
// CombineClaimsWithApplications enriches each PVC with the workloads that mount it.
func (kcl *KubeClient) CombineClaimsWithApplications(pvcs []models.K8sPersistentVolumeClaim) ([]models.K8sPersistentVolumeClaim, error) {
pods, err := kcl.cli.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return pvcs, nil
}
return nil, fmt.Errorf("failed to list pods: %w", err)
}
replicaSetItems := make([]appsv1.ReplicaSet, 0)
deploymentItems := make([]appsv1.Deployment, 0)
if containsReplicaSetOwnerReference(pods) {
replicaSets, err := kcl.cli.AppsV1().ReplicaSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list replica sets: %w", err)
}
replicaSetItems = replicaSets.Items
deployments, err := kcl.cli.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list deployments: %w", err)
}
deploymentItems = deployments.Items
}
statefulSetItems := make([]appsv1.StatefulSet, 0)
if containsStatefulSetOwnerReference(pods) {
statefulSets, err := kcl.cli.AppsV1().StatefulSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list stateful sets: %w", err)
}
statefulSetItems = statefulSets.Items
}
daemonSetItems := make([]appsv1.DaemonSet, 0)
if containsDaemonSetOwnerReference(pods) {
daemonSets, err := kcl.cli.AppsV1().DaemonSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list daemon sets: %w", err)
}
daemonSetItems = daemonSets.Items
}
resources := PortainerApplicationResources{
ReplicaSets: replicaSetItems,
Deployments: deploymentItems,
StatefulSets: statefulSetItems,
DaemonSets: daemonSetItems,
}
for i := range pvcs {
for _, pod := range pods.Items {
for _, podVolume := range pod.Spec.Volumes {
if podVolume.PersistentVolumeClaim == nil {
continue
}
if podVolume.PersistentVolumeClaim.ClaimName != pvcs[i].Name || pod.Namespace != pvcs[i].Namespace {
continue
}
application, err := kcl.ConvertPodToApplication(pod, resources, false)
if err != nil {
return nil, fmt.Errorf("failed to convert pod to application: %w", err)
}
if application == nil {
continue
}
alreadyAdded := false
for _, existing := range pvcs[i].OwningApplications {
if existing.Name == application.Name && existing.Namespace == application.Namespace {
alreadyAdded = true
break
}
}
if !alreadyAdded {
pvcs[i].OwningApplications = append(pvcs[i].OwningApplications, *application)
}
}
}
}
return pvcs, nil
}
@@ -0,0 +1,378 @@
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func makeStorageClass(name string, allowExpansion bool) *storagev1.StorageClass {
return &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: name},
Provisioner: "kubernetes.io/no-provisioner",
AllowVolumeExpansion: &allowExpansion,
}
}
func makePVC(namespace, name, scName string) *corev1.PersistentVolumeClaim {
storageRequest := resource.MustParse("1Gi")
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: corev1.PersistentVolumeClaimSpec{
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: storageRequest,
},
},
},
}
if scName != "" {
pvc.Spec.StorageClassName = &scName
}
return pvc
}
func TestResizePersistentVolumeClaim(t *testing.T) {
t.Parallel()
t.Run("returns error when storage class does not allow expansion", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := makeStorageClass("no-expand", false)
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
pvc := makePVC("default", "pvc-no-expand", "no-expand")
_, err = k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.ResizePersistentVolumeClaim("default", "pvc-no-expand", "2Gi")
require.Error(t, err)
assert.Contains(t, err.Error(), "does not allow volume expansion")
})
t.Run("returns error for invalid size format", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := makeStorageClass("expandable", true)
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
pvc := makePVC("default", "pvc-badsize", "expandable")
_, err = k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.ResizePersistentVolumeClaim("default", "pvc-badsize", "notasize")
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid size format")
})
t.Run("success when storage class allows expansion with valid size", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := makeStorageClass("expandable", true)
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
pvc := makePVC("default", "pvc-resize", "expandable")
_, err = k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.ResizePersistentVolumeClaim("default", "pvc-resize", "5Gi")
require.NoError(t, err)
})
t.Run("success when PVC has no storage class", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "pvc-no-sc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.ResizePersistentVolumeClaim("default", "pvc-no-sc", "10Gi")
require.NoError(t, err)
})
t.Run("returns error for non-existent PVC", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
err := k.ResizePersistentVolumeClaim("default", "does-not-exist", "2Gi")
require.Error(t, err)
})
}
func TestGetPersistentVolumeClaims_EnrichesExpansionFlag(t *testing.T) {
t.Parallel()
t.Run("PVC with AllowVolumeExpansion=true SC gets flag true", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := makeStorageClass("expandable", true)
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
pvc := makePVC("default", "pvc-expand", "expandable")
_, err = k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
require.Len(t, pvcs, 1)
assert.True(t, pvcs[0].AllowVolumeExpansion)
})
t.Run("PVC with AllowVolumeExpansion=false SC gets flag false", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := makeStorageClass("not-expandable", false)
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
pvc := makePVC("default", "pvc-no-expand", "not-expandable")
_, err = k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
require.Len(t, pvcs, 1)
assert.False(t, pvcs[0].AllowVolumeExpansion)
})
}
func makePod(namespace, name, pvcName string) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
},
},
},
},
Containers: []corev1.Container{{Name: "app", Image: "nginx"}},
},
}
}
func makePodWithDeploymentOwner(namespace, name, pvcName, rsName string) *corev1.Pod {
pod := makePod(namespace, name, pvcName)
pod.OwnerReferences = []metav1.OwnerReference{
{Kind: "ReplicaSet", Name: rsName, APIVersion: "apps/v1"},
}
return pod
}
func makeReplicaSet(namespace, name, deploymentName string) *appsv1.ReplicaSet {
return &appsv1.ReplicaSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
OwnerReferences: []metav1.OwnerReference{
{Kind: "Deployment", Name: deploymentName, APIVersion: "apps/v1"},
},
},
}
}
func makeDeployment(namespace, name string) *appsv1.Deployment {
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{},
},
}
}
func TestCombineClaimsWithApplications(t *testing.T) {
t.Parallel()
t.Run("no pods leaves owningApplications empty", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "my-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Empty(t, result[0].OwningApplications)
})
t.Run("pod with no PVC volumes leaves owningApplications empty", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "my-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "default"},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{Name: "app", Image: "nginx"}},
},
}
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
assert.Empty(t, result[0].OwningApplications)
})
t.Run("pod mounting PVC populates owningApplications", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "my-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pod := makePod("default", "my-pod", "my-pvc")
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
require.Len(t, result[0].OwningApplications, 1)
assert.Equal(t, "my-pod", result[0].OwningApplications[0].Name)
assert.Equal(t, "Pod", result[0].OwningApplications[0].ApplicationType)
assert.Equal(t, "default", result[0].OwningApplications[0].ResourcePool)
})
t.Run("pod in different namespace does not match PVC", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "my-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pod := makePod("other-ns", "other-pod", "my-pvc")
_, err = k.cli.CoreV1().Pods("other-ns").Create(t.Context(), pod, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
assert.Empty(t, result[0].OwningApplications)
})
t.Run("two pods mounting the same PVC from the same Deployment are deduplicated", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "shared-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
deploy := makeDeployment("default", "my-deploy")
_, err = k.cli.AppsV1().Deployments("default").Create(t.Context(), deploy, metav1.CreateOptions{})
require.NoError(t, err)
rs := makeReplicaSet("default", "my-deploy-rs", "my-deploy")
_, err = k.cli.AppsV1().ReplicaSets("default").Create(t.Context(), rs, metav1.CreateOptions{})
require.NoError(t, err)
pod1 := makePodWithDeploymentOwner("default", "my-deploy-pod-1", "shared-pvc", "my-deploy-rs")
pod2 := makePodWithDeploymentOwner("default", "my-deploy-pod-2", "shared-pvc", "my-deploy-rs")
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod1, metav1.CreateOptions{})
require.NoError(t, err)
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod2, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
require.Len(t, result[0].OwningApplications, 1)
assert.Equal(t, "my-deploy", result[0].OwningApplications[0].Name)
assert.Equal(t, "Deployment", result[0].OwningApplications[0].ApplicationType)
})
t.Run("pod owned by Deployment resolves application name to Deployment", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "app-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
deploy := makeDeployment("default", "web-app")
_, err = k.cli.AppsV1().Deployments("default").Create(t.Context(), deploy, metav1.CreateOptions{})
require.NoError(t, err)
rs := makeReplicaSet("default", "web-app-rs-abc", "web-app")
_, err = k.cli.AppsV1().ReplicaSets("default").Create(t.Context(), rs, metav1.CreateOptions{})
require.NoError(t, err)
pod := makePodWithDeploymentOwner("default", "web-app-pod-xyz", "app-pvc", "web-app-rs-abc")
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
require.Len(t, result[0].OwningApplications, 1)
assert.Equal(t, "web-app", result[0].OwningApplications[0].Name)
assert.Equal(t, "Deployment", result[0].OwningApplications[0].ApplicationType)
assert.Equal(t, "default", result[0].OwningApplications[0].ResourcePool)
})
t.Run("unrelated pod mounting a different PVC does not affect result", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvc := makePVC("default", "target-pvc", "")
_, err := k.cli.CoreV1().PersistentVolumeClaims("default").Create(t.Context(), pvc, metav1.CreateOptions{})
require.NoError(t, err)
pod := makePod("default", "unrelated-pod", "other-pvc")
_, err = k.cli.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{})
require.NoError(t, err)
pvcs, err := k.GetPersistentVolumeClaims("default")
require.NoError(t, err)
result, err := k.CombineClaimsWithApplications(pvcs)
require.NoError(t, err)
assert.Empty(t, result[0].OwningApplications)
})
}
+156
View File
@@ -0,0 +1,156 @@
package cli
import (
"context"
"fmt"
"github.com/segmentio/encoding/json"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
// GetPersistentVolumes returns all PersistentVolumes in the cluster.
// For non-admin users, results are filtered to PVs bound to their accessible namespaces.
func (kcl *KubeClient) GetPersistentVolumes() ([]models.K8sPersistentVolume, error) {
pvList, err := kcl.cli.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("unable to list persistent volumes. Error: %w", err)
}
results := make([]models.K8sPersistentVolume, 0)
if kcl.GetIsKubeAdmin() {
for i := range pvList.Items {
results = append(results, parsePersistentVolumeDetail(&pvList.Items[i]))
}
return results, nil
}
allowedNamespaces := kcl.buildNonAdminNamespacesMap()
for i := range pvList.Items {
pv := &pvList.Items[i]
if pv.Spec.ClaimRef != nil {
if _, ok := allowedNamespaces[pv.Spec.ClaimRef.Namespace]; ok {
results = append(results, parsePersistentVolumeDetail(pv))
}
}
}
return results, nil
}
// GetPersistentVolume returns a single PersistentVolume by name.
// For non-admin users, access is restricted to PVs bound to their accessible namespaces.
func (kcl *KubeClient) GetPersistentVolume(name string) (*models.K8sPersistentVolume, error) {
pv, err := kcl.cli.CoreV1().PersistentVolumes().Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("unable to get persistent volume %s. Error: %w", name, err)
}
if !kcl.GetIsKubeAdmin() {
allowedNamespaces := kcl.buildNonAdminNamespacesMap()
if pv.Spec.ClaimRef == nil {
return nil, fmt.Errorf("access to unbound persistent volume %s is not authorized: %w", name, ErrUnauthorized)
}
if _, ok := allowedNamespaces[pv.Spec.ClaimRef.Namespace]; !ok {
return nil, fmt.Errorf("access to persistent volume %s is not authorized: %w", name, ErrUnauthorized)
}
}
result := parsePersistentVolumeDetail(pv)
return &result, nil
}
// DeletePersistentVolumes deletes the specified PersistentVolumes by name.
func (kcl *KubeClient) DeletePersistentVolumes(names []string) error {
for _, name := range names {
log.Debug().
Str("context", "DeletePersistentVolumes").
Str("persistent_volume", name).
Msg("Deleting persistent volume")
err := kcl.cli.CoreV1().PersistentVolumes().Delete(context.Background(), name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("unable to delete persistent volume %s. Error: %w", name, err)
}
}
return nil
}
// UpdatePersistentVolumeReclaimPolicy updates the reclaim policy on a PV.
func (kcl *KubeClient) UpdatePersistentVolumeReclaimPolicy(name string, policy corev1.PersistentVolumeReclaimPolicy) error {
switch policy {
case corev1.PersistentVolumeReclaimRetain,
corev1.PersistentVolumeReclaimDelete,
corev1.PersistentVolumeReclaimRecycle:
// valid
default:
return fmt.Errorf("invalid reclaim policy %q: must be Retain, Delete, or Recycle", policy)
}
patch := map[string]any{
"spec": map[string]any{
"persistentVolumeReclaimPolicy": policy,
},
}
patchBytes, err := json.Marshal(patch)
if err != nil {
return fmt.Errorf("unable to marshal reclaim policy patch. Error: %w", err)
}
_, err = kcl.cli.CoreV1().PersistentVolumes().Patch(
context.Background(),
name,
types.MergePatchType,
patchBytes,
metav1.PatchOptions{},
)
if err != nil {
return fmt.Errorf("unable to update reclaim policy for persistent volume %s. Error: %w", name, err)
}
return nil
}
// parsePersistentVolumeDetail parses a full PV into the model with status and access modes.
func parsePersistentVolumeDetail(pv *corev1.PersistentVolume) models.K8sPersistentVolume {
return models.K8sPersistentVolume{
Name: pv.Name,
Annotations: pv.Annotations,
Labels: pv.Labels,
AccessModes: humanReadableAccessModes(pv.Spec.AccessModes),
HumanReadableAccessModes: pv.Spec.AccessModes,
Capacity: pv.Spec.Capacity,
ClaimRef: pv.Spec.ClaimRef,
StorageClassName: pv.Spec.StorageClassName,
PersistentVolumeReclaimPolicy: pv.Spec.PersistentVolumeReclaimPolicy,
VolumeMode: pv.Spec.VolumeMode,
CSI: pv.Spec.CSI,
Status: pv.Status.Phase,
CreationDate: pv.CreationTimestamp.Time,
}
}
// humanReadableAccessModes converts Kubernetes access modes to short abbreviations.
func humanReadableAccessModes(modes []corev1.PersistentVolumeAccessMode) []string {
readable := make([]string, 0, len(modes))
for _, mode := range modes {
switch mode {
case corev1.ReadWriteOnce:
readable = append(readable, "RWO")
case corev1.ReadOnlyMany:
readable = append(readable, "ROX")
case corev1.ReadWriteMany:
readable = append(readable, "RWX")
case corev1.ReadWriteOncePod:
readable = append(readable, "RWOP")
default:
readable = append(readable, string(mode))
}
}
return readable
}
@@ -0,0 +1,166 @@
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func TestHumanReadableAccessModes(t *testing.T) {
t.Parallel()
tests := []struct {
name string
input []corev1.PersistentVolumeAccessMode
want []string
}{
{
name: "ReadWriteOnce",
input: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
want: []string{"RWO"},
},
{
name: "ReadOnlyMany",
input: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany},
want: []string{"ROX"},
},
{
name: "ReadWriteMany",
input: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany},
want: []string{"RWX"},
},
{
name: "ReadWriteOncePod",
input: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod},
want: []string{"RWOP"},
},
{
name: "unknown mode passes through",
input: []corev1.PersistentVolumeAccessMode{"SomeUnknownMode"},
want: []string{"SomeUnknownMode"},
},
{
name: "empty slice returns empty slice",
input: []corev1.PersistentVolumeAccessMode{},
want: []string{},
},
{
name: "multiple modes in order",
input: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadOnlyMany, corev1.ReadWriteMany, corev1.ReadWriteOncePod},
want: []string{"RWO", "ROX", "RWX", "RWOP"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := humanReadableAccessModes(tt.input)
assert.Equal(t, tt.want, got)
})
}
}
func TestGetPersistentVolumes(t *testing.T) {
t.Parallel()
t.Run("empty cluster returns empty slice", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pvs, err := k.GetPersistentVolumes()
require.NoError(t, err)
assert.Empty(t, pvs)
})
t.Run("returns correctly parsed PVs", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pv := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "pv-one",
},
Spec: corev1.PersistentVolumeSpec{
StorageClassName: "standard",
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
},
Status: corev1.PersistentVolumeStatus{
Phase: corev1.VolumeBound,
},
}
_, err := k.cli.CoreV1().PersistentVolumes().Create(t.Context(), pv, metav1.CreateOptions{})
require.NoError(t, err)
pvs, err := k.GetPersistentVolumes()
require.NoError(t, err)
require.Len(t, pvs, 1)
got := pvs[0]
assert.Equal(t, "pv-one", got.Name)
assert.Equal(t, "standard", got.StorageClassName)
assert.Equal(t, corev1.VolumeBound, got.Status)
assert.Equal(t, []string{"RWO"}, got.AccessModes)
})
}
func TestDeletePersistentVolumes(t *testing.T) {
t.Parallel()
t.Run("successfully deletes specified PVs", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
for _, name := range []string{"pv-a", "pv-b", "pv-c"} {
pv := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: name},
}
_, err := k.cli.CoreV1().PersistentVolumes().Create(t.Context(), pv, metav1.CreateOptions{})
require.NoError(t, err)
}
err := k.DeletePersistentVolumes([]string{"pv-a", "pv-c"})
require.NoError(t, err)
pvs, err := k.GetPersistentVolumes()
require.NoError(t, err)
require.Len(t, pvs, 1)
assert.Equal(t, "pv-b", pvs[0].Name)
})
t.Run("returns error for non-existent PV", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
err := k.DeletePersistentVolumes([]string{"does-not-exist"})
assert.Error(t, err)
})
}
func TestUpdatePersistentVolumeReclaimPolicy(t *testing.T) {
t.Parallel()
t.Run("successfully patches reclaim policy on existing PV", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
pv := &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: "pv-reclaim"},
Spec: corev1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete,
},
}
_, err := k.cli.CoreV1().PersistentVolumes().Create(t.Context(), pv, metav1.CreateOptions{})
require.NoError(t, err)
err = k.UpdatePersistentVolumeReclaimPolicy("pv-reclaim", corev1.PersistentVolumeReclaimRetain)
require.NoError(t, err)
updated, err := k.GetPersistentVolume("pv-reclaim")
require.NoError(t, err)
assert.Equal(t, corev1.PersistentVolumeReclaimRetain, updated.PersistentVolumeReclaimPolicy)
})
}
+335
View File
@@ -0,0 +1,335 @@
package cli
import (
"context"
"fmt"
"strconv"
"time"
portainer "github.com/portainer/portainer/api"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (kcl *KubeClient) GetPods(namespace string) ([]corev1.Pod, error) {
pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return pods.Items, nil
}
// DeletePod deletes a single pod. The owning controller (Deployment,
// StatefulSet, DaemonSet, ...) is responsible for recreating it. For naked
// pods the pod is removed permanently.
func (kcl *KubeClient) DeletePod(namespace, name string) error {
return kcl.cli.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
}
// RestartPod restarts all containers inside a pod in place using the
// Kubernetes 1.35 alpha pod-restart subresource. The pod itself is preserved.
// Requires the cluster to expose the corresponding subresource (and feature
// gate). On clusters that don't, the API server typically returns 404 or 405
// which is surfaced to the caller.
func (kcl *KubeClient) RestartPod(namespace, name string) error {
return kcl.cli.CoreV1().RESTClient().Post().
Namespace(namespace).
Resource("pods").
Name(name).
SubResource("restart").
Do(context.TODO()).
Error()
}
// isReplicaSetOwner checks if the pod's owner reference is a ReplicaSet
func isReplicaSetOwner(pod corev1.Pod) bool {
return len(pod.OwnerReferences) > 0 && pod.OwnerReferences[0].Kind == "ReplicaSet"
}
// updateOwnerReferenceToDeployment updates the pod's owner reference to the Deployment if applicable
func updateOwnerReferenceToDeployment(pod *corev1.Pod, replicaSets []appsv1.ReplicaSet) {
for _, replicaSet := range replicaSets {
if pod.OwnerReferences[0].Name == replicaSet.Name {
if len(replicaSet.OwnerReferences) > 0 && replicaSet.OwnerReferences[0].Kind == "Deployment" {
pod.OwnerReferences[0].Kind = "Deployment"
pod.OwnerReferences[0].Name = replicaSet.OwnerReferences[0].Name
}
break
}
}
}
// containsStatefulSetOwnerReference checks if the pod list contains a pod with a StatefulSet owner reference
func containsStatefulSetOwnerReference(pods *corev1.PodList) bool {
for _, pod := range pods.Items {
if len(pod.OwnerReferences) > 0 && pod.OwnerReferences[0].Kind == "StatefulSet" {
return true
}
}
return false
}
// containsDaemonSetOwnerReference checks if the pod list contains a pod with a DaemonSet owner reference
func containsDaemonSetOwnerReference(pods *corev1.PodList) bool {
for _, pod := range pods.Items {
if len(pod.OwnerReferences) > 0 && pod.OwnerReferences[0].Kind == "DaemonSet" {
return true
}
}
return false
}
// containsReplicaSetOwnerReference checks if the pod list contains a pod with a ReplicaSet owner reference
func containsReplicaSetOwnerReference(pods *corev1.PodList) bool {
for _, pod := range pods.Items {
if len(pod.OwnerReferences) > 0 && pod.OwnerReferences[0].Kind == "ReplicaSet" {
return true
}
}
return false
}
// CreateUserShellPod will create a kubectl based shell for the specified user by mounting their respective service account.
// The lifecycle of the pod is managed in this function; this entails management of the following pod operations:
// - The shell pod will be scoped to specified service accounts access permissions
// - The shell pod will be automatically removed if it's not ready after specified period of time
// - The shell pod will be automatically removed after a specified max life (prevent zombie pods)
// - The shell pod will be automatically removed if request is cancelled (or client closes websocket connection)
func (kcl *KubeClient) CreateUserShellPod(ctx context.Context, serviceAccountName, shellPodImage string) (*portainer.KubernetesShellPod, error) {
maxPodKeepAliveSecondsStr := strconv.Itoa(int(portainer.WebSocketKeepAlive.Seconds()))
podPrefix := userShellPodPrefix(serviceAccountName)
podSpec := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: podPrefix,
Namespace: portainerNamespace,
Annotations: map[string]string{
"kubernetes.io/pod.type": "kubectl-shell",
},
},
Spec: corev1.PodSpec{
TerminationGracePeriodSeconds: new(int64),
ServiceAccountName: serviceAccountName,
Containers: []corev1.Container{
{
Name: "kubectl-shell-container",
Image: shellPodImage,
Command: []string{"sleep"},
// Specify sleep time to prevent zombie pods in case portainer process is terminated
Args: []string{maxPodKeepAliveSecondsStr},
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
}
shellPod, err := kcl.cli.CoreV1().Pods(portainerNamespace).Create(context.TODO(), podSpec, metav1.CreateOptions{})
if err != nil {
return nil, errors.Wrap(err, "error creating shell pod")
}
// Wait for pod to reach ready state
timeoutCtx, cancelFunc := context.WithTimeout(ctx, 20*time.Second)
defer cancelFunc()
if err := kcl.waitForPodStatus(timeoutCtx, corev1.PodRunning, shellPod); err != nil {
innerErr := kcl.cli.CoreV1().Pods(portainerNamespace).Delete(context.TODO(), shellPod.Name, metav1.DeleteOptions{})
if innerErr != nil {
log.Warn().Err(innerErr).Msg("error deleting shell pod after failing to wait for ready status")
}
return nil, errors.Wrap(err, "aborting pod creation; error waiting for shell pod ready status")
}
var containerName string
if len(shellPod.Spec.Containers) > 0 {
containerName = shellPod.Spec.Containers[0].Name
}
podData := &portainer.KubernetesShellPod{
Namespace: shellPod.Namespace,
PodName: shellPod.Name,
ContainerName: containerName,
ShellExecCommand: "env TERM=xterm-256color /bin/bash",
}
// Handle pod lifecycle/cleanup - terminate pod after maxPodKeepAlive or upon request (long-lived) cancellation
go func() {
select {
case <-time.After(portainer.WebSocketKeepAlive):
log.Debug().Msg("pod removal schedule duration exceeded")
if err := kcl.cli.CoreV1().Pods(portainerNamespace).Delete(context.TODO(), shellPod.Name, metav1.DeleteOptions{}); err != nil {
log.Warn().Err(err).Msg("error deleting shell pod after max keep alive duration exceeded")
}
case <-ctx.Done():
err := ctx.Err()
log.Debug().Err(err).Msg("context error")
if err := kcl.cli.CoreV1().Pods(portainerNamespace).Delete(context.TODO(), shellPod.Name, metav1.DeleteOptions{}); err != nil {
log.Warn().Err(err).Msg("error deleting shell pod after context cancellation")
}
}
}()
return podData, nil
}
// waitForPodStatus will wait until duration d (from now) for a pod to reach defined phase/status.
// The pod status will be polled at specified delay until the pod reaches ready state.
func (kcl *KubeClient) waitForPodStatus(ctx context.Context, phase corev1.PodPhase, pod *corev1.Pod) error {
log.Debug().Str("pod", pod.Name).Msg("waiting for pod ready")
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
pod, err := kcl.cli.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
if err != nil {
return err
}
if pod.Status.Phase == phase {
return nil
}
time.Sleep(500 * time.Millisecond)
}
}
}
// fetchAllApplicationsListResources fetches all pods, replica sets, stateful sets, and daemon sets across the cluster, i.e. all namespaces
// this is required for the applications list view
func (kcl *KubeClient) fetchAllApplicationsListResources(namespace string, podListOptions metav1.ListOptions) (PortainerApplicationResources, error) {
return kcl.fetchResourcesWithOwnerReferences(namespace, podListOptions, true, true)
}
// fetchResourcesWithOwnerReferences fetches pods and other resources based on owner references
func (kcl *KubeClient) fetchResourcesWithOwnerReferences(namespace string, podListOptions metav1.ListOptions, includeStatefulSets, includeDaemonSets bool) (PortainerApplicationResources, error) {
pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.Background(), podListOptions)
if err != nil {
if k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, nil
}
return PortainerApplicationResources{}, fmt.Errorf("unable to list pods across the cluster: %w", err)
}
portainerApplicationResources := PortainerApplicationResources{Pods: pods.Items}
replicaSets, err := kcl.cli.AppsV1().ReplicaSets(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list replica sets across the cluster: %w", err)
}
portainerApplicationResources.ReplicaSets = replicaSets.Items
deployments, err := kcl.cli.AppsV1().Deployments(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list deployments across the cluster: %w", err)
}
portainerApplicationResources.Deployments = deployments.Items
if includeStatefulSets {
statefulSets, err := kcl.cli.AppsV1().StatefulSets(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list stateful sets across the cluster: %w", err)
}
portainerApplicationResources.StatefulSets = statefulSets.Items
}
if includeDaemonSets {
daemonSets, err := kcl.cli.AppsV1().DaemonSets(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list daemon sets across the cluster: %w", err)
}
portainerApplicationResources.DaemonSets = daemonSets.Items
}
services, err := kcl.cli.CoreV1().Services(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list services across the cluster: %w", err)
}
portainerApplicationResources.Services = services.Items
hpas, err := kcl.cli.AutoscalingV2().HorizontalPodAutoscalers(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil && !k8serrors.IsNotFound(err) {
return PortainerApplicationResources{}, fmt.Errorf("unable to list horizontal pod autoscalers across the cluster: %w", err)
}
portainerApplicationResources.HorizontalPodAutoscalers = hpas.Items
return portainerApplicationResources, nil
}
// isPodUsingConfigMap checks if a pod is using a specific ConfigMap
func isPodUsingConfigMap(pod *corev1.Pod, configMap models.K8sConfigMap) bool {
if pod.Namespace != configMap.Namespace {
return false
}
for _, volume := range pod.Spec.Volumes {
if volume.ConfigMap != nil && volume.ConfigMap.Name == configMap.Name {
return true
}
}
for _, container := range pod.Spec.Containers {
for _, env := range container.Env {
if env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil && env.ValueFrom.ConfigMapKeyRef.Name == configMap.Name {
return true
}
}
}
return false
}
// isPodUsingSecret checks if a pod is using a specific Secret
func isPodUsingSecret(pod *corev1.Pod, secret models.K8sSecret) bool {
if pod.Namespace != secret.Namespace {
return false
}
for _, volume := range pod.Spec.Volumes {
if volume.Secret != nil && volume.Secret.SecretName == secret.Name {
return true
}
}
for _, container := range pod.Spec.Containers {
for _, env := range container.Env {
if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil && env.ValueFrom.SecretKeyRef.Name == secret.Name {
return true
}
}
}
return false
}
// getLatestJobPod returns the pods that are owned by a job
// it returns an error if there is an error fetching the pods
func (kcl *KubeClient) getLatestJobPod(namespace string, jobName string) (*corev1.Pod, error) {
pods, err := kcl.cli.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
for _, pod := range pods.Items {
for _, owner := range pod.OwnerReferences {
if owner.Kind == "Job" && owner.Name == jobName {
return &pod, nil
}
}
}
return nil, nil
}
+125
View File
@@ -0,0 +1,125 @@
package cli
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func TestDeletePod(t *testing.T) {
t.Parallel()
t.Run("deletes an existing pod", func(t *testing.T) {
t.Parallel()
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "my-pod", Namespace: "default"}}
kcl := &KubeClient{cli: kfake.NewSimpleClientset(pod)}
err := kcl.DeletePod("default", "my-pod")
require.NoError(t, err)
})
t.Run("returns not-found error for a missing pod", func(t *testing.T) {
t.Parallel()
kcl := &KubeClient{cli: kfake.NewSimpleClientset()}
err := kcl.DeletePod("default", "nonexistent")
require.Error(t, err)
assert.True(t, k8serrors.IsNotFound(err), "expected a not-found error, got: %v", err)
})
t.Run("deletes only the named pod leaving others intact", func(t *testing.T) {
t.Parallel()
podA := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-a", Namespace: "default"}}
podB := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-b", Namespace: "default"}}
fakeClient := kfake.NewSimpleClientset(podA, podB)
kcl := &KubeClient{cli: fakeClient}
err := kcl.DeletePod("default", "pod-a")
require.NoError(t, err)
_, err = fakeClient.CoreV1().Pods("default").Get(t.Context(), "pod-a", metav1.GetOptions{})
assert.True(t, k8serrors.IsNotFound(err), "pod-a should have been deleted")
_, err = fakeClient.CoreV1().Pods("default").Get(t.Context(), "pod-b", metav1.GetOptions{})
require.NoError(t, err, "pod-b should still exist")
})
t.Run("returns not-found when pod exists in a different namespace", func(t *testing.T) {
t.Parallel()
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "my-pod", Namespace: "other"}}
kcl := &KubeClient{cli: kfake.NewSimpleClientset(pod)}
err := kcl.DeletePod("default", "my-pod")
require.Error(t, err)
assert.True(t, k8serrors.IsNotFound(err))
})
}
func Test_waitForPodStatus(t *testing.T) {
t.Parallel()
t.Run("successfully errors on cancelled context", func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
podSpec := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: defaultNamespace},
Spec: v1.PodSpec{
Containers: []v1.Container{
{Name: "test-pod", Image: "containous/whoami"},
},
},
}
ctx, cancel := context.WithCancel(t.Context())
cancel()
err := k.waitForPodStatus(ctx, v1.PodRunning, podSpec)
if !errors.Is(err, context.Canceled) {
t.Errorf("waitForPodStatus should throw context cancellation error; err=%s", err)
}
})
t.Run("successfully errors on timeout", func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
podSpec := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: defaultNamespace},
Spec: v1.PodSpec{
Containers: []v1.Container{
{Name: "test-pod", Image: "containous/whoami"},
},
},
}
pod, err := k.cli.CoreV1().Pods(defaultNamespace).Create(t.Context(), podSpec, metav1.CreateOptions{})
if err != nil {
t.Errorf("failed to create pod; err=%s", err)
}
defer func() {
err := k.cli.CoreV1().Pods(defaultNamespace).Delete(t.Context(), pod.Name, metav1.DeleteOptions{})
require.NoError(t, err)
}()
ctx, cancelFunc := context.WithTimeout(t.Context(), 0*time.Second)
defer cancelFunc()
err = k.waitForPodStatus(ctx, v1.PodRunning, podSpec)
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("waitForPodStatus should throw deadline exceeded error; err=%s", err)
}
})
}
+201
View File
@@ -0,0 +1,201 @@
package cli
import (
"context"
"time"
"github.com/portainer/portainer/api/internal/randomstring"
"github.com/rs/zerolog/log"
authv1 "k8s.io/api/authorization/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
authv1types "k8s.io/client-go/kubernetes/typed/authorization/v1"
corev1types "k8s.io/client-go/kubernetes/typed/core/v1"
rbacv1types "k8s.io/client-go/kubernetes/typed/rbac/v1"
)
const maxRetries = 5
// IsRBACEnabled checks if RBAC is enabled in the cluster by creating a service account, then checking it's access to a resourcequota before and after setting a cluster role and cluster role binding
func (kcl *KubeClient) IsRBACEnabled() (bool, error) {
namespace := "default"
verb := "list"
resource := "resourcequotas"
saClient := kcl.cli.CoreV1().ServiceAccounts(namespace)
uniqueString := randomstring.RandomString(4) // Append a unique string to resource names, in case they already exist
saName := "portainer-rbac-test-sa-" + uniqueString
if err := createServiceAccount(saClient, saName, namespace); err != nil {
log.Error().Err(err).Msg("Error creating service account")
return false, err
}
defer deleteServiceAccount(saClient, saName)
accessReviewClient := kcl.cli.AuthorizationV1().LocalSubjectAccessReviews(namespace)
allowed, err := checkServiceAccountAccess(accessReviewClient, saName, verb, resource, namespace)
if err != nil {
log.Error().Err(err).Msg("Error checking service account access")
return false, err
}
// If the service account with no authorizations is allowed, RBAC must be disabled
if allowed {
return false, nil
}
// Otherwise give the service account an rbac authorisation and check again
roleClient := kcl.cli.RbacV1().Roles(namespace)
roleName := "portainer-rbac-test-role-" + uniqueString
if err := createRole(roleClient, roleName, verb, resource, namespace); err != nil {
log.Error().Err(err).Msg("Error creating role")
return false, err
}
defer deleteRole(roleClient, roleName)
roleBindingClient := kcl.cli.RbacV1().RoleBindings(namespace)
roleBindingName := "portainer-rbac-test-role-binding-" + uniqueString
if err := createRoleBinding(roleBindingClient, roleBindingName, roleName, saName, namespace); err != nil {
log.Error().Err(err).Msg("Error creating role binding")
return false, err
}
defer deleteRoleBinding(roleBindingClient, roleBindingName)
allowed, err = checkServiceAccountAccess(accessReviewClient, saName, verb, resource, namespace)
if err != nil {
log.Error().Err(err).Msg("Error checking service account access with authorizations added")
return false, err
}
// If the service account allowed to list resource quotas after given rbac role, then RBAC is enabled
return allowed, nil
}
func createServiceAccount(saClient corev1types.ServiceAccountInterface, name string, namespace string) error {
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
}
_, err := saClient.Create(context.Background(), serviceAccount, metav1.CreateOptions{})
return err
}
func deleteServiceAccount(saClient corev1types.ServiceAccountInterface, name string) {
if err := saClient.Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
log.Error().Err(err).Msg("Error deleting service account: " + name)
}
}
func createRole(roleClient rbacv1types.RoleInterface, name string, verb string, resource string, namespace string) error {
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Verbs: []string{verb},
Resources: []string{resource},
},
},
}
_, err := roleClient.Create(context.Background(), role, metav1.CreateOptions{})
return err
}
func deleteRole(roleClient rbacv1types.RoleInterface, name string) {
if err := roleClient.Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
log.Error().Err(err).Msg("Error deleting role: " + name)
}
}
func createRoleBinding(roleBindingClient rbacv1types.RoleBindingInterface, clusterRoleBindingName string, roleName string, serviceAccountName string, namespace string) error {
clusterRoleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleBindingName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: namespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "Role",
Name: roleName,
APIGroup: "rbac.authorization.k8s.io",
},
}
roleBinding, err := roleBindingClient.Create(context.Background(), clusterRoleBinding, metav1.CreateOptions{})
if err != nil {
log.Error().Err(err).Msg("Error creating role binding: " + clusterRoleBindingName)
return err
}
// Retry checkRoleBinding a maximum of 5 times with a 100ms wait after each attempt
for range maxRetries {
err = checkRoleBinding(roleBindingClient, roleBinding.Name)
time.Sleep(100 * time.Millisecond) // Wait for 100ms, even if the check passes
if err == nil {
break
}
}
return err
}
func checkRoleBinding(roleBindingClient rbacv1types.RoleBindingInterface, name string) error {
if _, err := roleBindingClient.Get(context.Background(), name, metav1.GetOptions{}); err != nil {
log.Error().Err(err).Msg("Error finding rolebinding: " + name)
return err
}
return nil
}
func deleteRoleBinding(roleBindingClient rbacv1types.RoleBindingInterface, name string) {
if err := roleBindingClient.Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
log.Error().Err(err).Msg("Error deleting role binding: " + name)
}
}
func checkServiceAccountAccess(accessReviewClient authv1types.LocalSubjectAccessReviewInterface, serviceAccountName string, verb string, resource string, namespace string) (bool, error) {
subjectAccessReview := &authv1.LocalSubjectAccessReview{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
},
Spec: authv1.SubjectAccessReviewSpec{
ResourceAttributes: &authv1.ResourceAttributes{
Namespace: namespace,
Verb: verb,
Resource: resource,
},
User: "system:serviceaccount:default:" + serviceAccountName, // a workaround to be able to use the service account as a user
},
}
result, err := accessReviewClient.Create(context.Background(), subjectAccessReview, metav1.CreateOptions{})
if err != nil {
return false, err
}
return result.Status.Allowed, nil
}
+104
View File
@@ -0,0 +1,104 @@
package cli
import (
"context"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/registryutils"
"github.com/pkg/errors"
"github.com/segmentio/encoding/json"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
secretDockerConfigKey = ".dockerconfigjson"
labelRegistryType = "io.portainer.kubernetes.registry.type"
annotationRegistryID = "portainer.io/registry.id"
)
type (
dockerConfig struct {
Auths map[string]registryDockerConfig `json:"auths"`
}
registryDockerConfig struct {
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`
}
)
func (kcl *KubeClient) DeleteRegistrySecret(registry portainer.RegistryID, namespace string) error {
if err := kcl.cli.CoreV1().Secrets(namespace).Delete(context.TODO(), registryutils.RegistrySecretName(registry), metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return errors.Wrap(err, "failed removing secret")
}
return nil
}
func (kcl *KubeClient) CreateRegistrySecret(registry *portainer.Registry, namespace string) error {
username, password, err := registryutils.GetRegEffectiveCredential(registry)
if err != nil {
return err
}
config := dockerConfig{
Auths: map[string]registryDockerConfig{
registry.URL: {
Username: username,
Password: password,
},
},
}
configByte, err := json.Marshal(config)
if err != nil {
return errors.Wrap(err, "failed marshal config")
}
secret := &v1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Name: registryutils.RegistrySecretName(registry.ID),
Labels: map[string]string{
labelRegistryType: strconv.Itoa(int(registry.Type)),
"app.kubernetes.io/managed-by": "portainer",
},
Annotations: map[string]string{
annotationRegistryID: strconv.Itoa(int(registry.ID)),
},
},
Data: map[string][]byte{
secretDockerConfigKey: configByte,
},
Type: v1.SecretTypeDockerConfigJson,
}
if _, err := kcl.cli.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil && !k8serrors.IsAlreadyExists(err) {
return errors.Wrap(err, "failed saving secret")
}
return nil
}
func (cli *KubeClient) IsRegistrySecret(namespace, secretName string) (bool, error) {
secret, err := cli.cli.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return false, nil
}
return false, err
}
isSecret := secret.Type == v1.SecretTypeDockerConfigJson
return isSecret, nil
}
+44
View File
@@ -0,0 +1,44 @@
package cli
import (
"sync"
"k8s.io/client-go/tools/remotecommand"
)
// TerminalSizeQueue implements remotecommand.TerminalSizeQueue for Kubernetes pod exec.
// Resize events are received via Push and forwarded to the Kubernetes API server.
type TerminalSizeQueue struct {
resizeChan chan *remotecommand.TerminalSize
done chan struct{}
closeOnce sync.Once
}
func NewTerminalSizeQueue() *TerminalSizeQueue {
return &TerminalSizeQueue{
resizeChan: make(chan *remotecommand.TerminalSize),
done: make(chan struct{}),
}
}
// Next blocks until the next terminal resize event or the queue is closed.
func (q *TerminalSizeQueue) Next() *remotecommand.TerminalSize {
return <-q.resizeChan
}
// Push queues a terminal resize.
// Push is safe to call after Close.
func (q *TerminalSizeQueue) Push(cols, rows uint16) {
select {
case <-q.done:
case q.resizeChan <- &remotecommand.TerminalSize{Width: cols, Height: rows}:
}
}
// Close shuts down the queue. Safe to call multiple times.
func (q *TerminalSizeQueue) Close() {
q.closeOnce.Do(func() {
close(q.done)
close(q.resizeChan)
})
}
+63
View File
@@ -0,0 +1,63 @@
package cli
import (
"testing"
"github.com/stretchr/testify/require"
"k8s.io/client-go/tools/remotecommand"
)
func Test_TerminalSizeQueue(t *testing.T) {
t.Parallel()
t.Run("Next returns pushed size", func(t *testing.T) {
q := NewTerminalSizeQueue()
defer q.Close()
go q.Push(80, 24)
size := q.Next()
require.NotNil(t, size)
require.Equal(t, uint16(80), size.Width)
require.Equal(t, uint16(24), size.Height)
})
t.Run("Next returns sizes in push order", func(t *testing.T) {
q := NewTerminalSizeQueue()
defer q.Close()
go q.Push(80, 24)
size := q.Next()
require.NotNil(t, size)
require.Equal(t, uint16(80), size.Width)
require.Equal(t, uint16(24), size.Height)
go q.Push(120, 40)
size = q.Next()
require.NotNil(t, size)
require.Equal(t, uint16(120), size.Width)
require.Equal(t, uint16(40), size.Height)
})
t.Run("Close causes Next to return nil", func(t *testing.T) {
q := NewTerminalSizeQueue()
q.Close()
size := q.Next()
require.Nil(t, size)
})
t.Run("Next unblocks when queue is closed", func(t *testing.T) {
q := NewTerminalSizeQueue()
result := make(chan *remotecommand.TerminalSize, 1)
go func() { result <- q.Next() }()
q.Close()
require.Nil(t, <-result)
})
t.Run("Close is idempotent", func(t *testing.T) {
q := NewTerminalSizeQueue()
q.Close()
q.Close()
})
}
+27
View File
@@ -0,0 +1,27 @@
package cli
import (
"bytes"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
)
func GenerateYAML(obj runtime.Object) (string, error) {
serializer := json.NewSerializerWithOptions(
json.DefaultMetaFactory, nil, nil,
json.SerializerOptions{
Yaml: true,
Pretty: true,
Strict: true,
},
)
b := new(bytes.Buffer)
err := serializer.Encode(obj, b)
if err != nil {
return "", err
}
return b.String(), nil
}
+100
View File
@@ -0,0 +1,100 @@
package cli
import (
"context"
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetResourceQuotas gets all resource quotas in the current k8s environment(endpoint).
// if the user is an admin, all resource quotas in all namespaces are fetched.
// otherwise, namespaces the non-admin user has access to will be used to filter the resource quotas.
func (kcl *KubeClient) GetResourceQuotas(namespace string) (*[]corev1.ResourceQuota, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchResourceQuotas(namespace)
}
return kcl.fetchResourceQuotasForNonAdmin(namespace)
}
// fetchResourceQuotasForNonAdmin gets the resource quotas in the current k8s environment(endpoint) for a non-admin user.
// the role of the user must have read access to the resource quotas in the defined namespaces.
func (kcl *KubeClient) fetchResourceQuotasForNonAdmin(namespace string) (*[]corev1.ResourceQuota, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching resource quotas for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
resourceQuotas, err := kcl.fetchResourceQuotas(namespace)
if err != nil && !k8serrors.IsNotFound(err) {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := []corev1.ResourceQuota{}
for _, resourceQuota := range *resourceQuotas {
if _, exists := nonAdminNamespaceSet[resourceQuota.Namespace]; exists {
results = append(results, resourceQuota)
}
}
return &results, nil
}
func (kcl *KubeClient) fetchResourceQuotas(namespace string) (*[]corev1.ResourceQuota, error) {
resourceQuotas, err := kcl.cli.CoreV1().ResourceQuotas(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("an error occurred, failed to list resource quotas for the admin user: %w", err)
}
return &resourceQuotas.Items, nil
}
// GetPortainerResourceQuota gets the resource quota for the portainer namespace.
// The resource quota is prefixed with "portainer-rq-".
func (kcl *KubeClient) GetPortainerResourceQuota(namespace string) (*corev1.ResourceQuota, error) {
return kcl.cli.CoreV1().ResourceQuotas(namespace).Get(context.TODO(), "portainer-rq-"+namespace, metav1.GetOptions{})
}
// GetResourceQuota gets a resource quota in a specific namespace.
func (kcl *KubeClient) GetResourceQuota(namespace, resourceQuota string) (*corev1.ResourceQuota, error) {
return kcl.cli.CoreV1().ResourceQuotas(namespace).Get(context.TODO(), resourceQuota, metav1.GetOptions{})
}
// UpdateNamespacesWithResourceQuotas updates the namespaces with the resource quotas.
// The resource quotas are matched with the namespaces by name.
func (kcl *KubeClient) UpdateNamespacesWithResourceQuotas(namespaces map[string]portainer.K8sNamespaceInfo, resourceQuotas []corev1.ResourceQuota) []portainer.K8sNamespaceInfo {
namespacesWithQuota := map[string]portainer.K8sNamespaceInfo{}
for _, namespace := range namespaces {
resourceQuota := kcl.GetResourceQuotaFromNamespace(namespace, resourceQuotas)
if resourceQuota != nil {
namespace.ResourceQuota = resourceQuota
}
namespacesWithQuota[namespace.Name] = namespace
}
return kcl.ConvertNamespaceMapToSlice(namespacesWithQuota)
}
// GetResourceQuotaFromNamespace gets the resource quota in a specific namespace where the resource quota's name is prefixed with "portainer-rq-".
func (kcl *KubeClient) GetResourceQuotaFromNamespace(namespace portainer.K8sNamespaceInfo, resourceQuotas []corev1.ResourceQuota) *corev1.ResourceQuota {
for _, resourceQuota := range resourceQuotas {
if resourceQuota.Namespace == namespace.Name && resourceQuota.Name == "portainer-rq-"+namespace.Name {
return &resourceQuota
}
}
return nil
}
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetResourceQuotas(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
resourceQuotas, err := kcl.GetResourceQuotas("default")
require.NoError(t, err)
require.Empty(t, resourceQuotas)
}
+131
View File
@@ -0,0 +1,131 @@
package cli
import (
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientV1 "k8s.io/client-go/tools/clientcmd/api/v1"
)
// compareYAMLStrings will compare 2 strings by stripping tabs, newlines and whitespaces from both strings
func compareYAMLStrings(in1, in2 string) int {
r := strings.NewReplacer("\t", "", "\n", "", " ", "")
in1 = r.Replace(in1)
in2 = r.Replace(in2)
return strings.Compare(in1, in2)
}
func Test_GenerateYAML(t *testing.T) {
t.Parallel()
resourceYAMLTests := []struct {
title string
resource runtime.Object
wantYAML string
}{
{
title: "Config",
resource: &clientV1.Config{
APIVersion: "v1",
Kind: "Config",
CurrentContext: "portainer-ctx",
Contexts: []clientV1.NamedContext{
{
Name: "portainer-ctx",
Context: clientV1.Context{
AuthInfo: "test-user",
Cluster: "portainer-cluster",
},
},
},
Clusters: []clientV1.NamedCluster{
{
Name: "portainer-cluster",
Cluster: clientV1.Cluster{
Server: "localhost",
InsecureSkipTLSVerify: true,
},
},
},
AuthInfos: []clientV1.NamedAuthInfo{
{
Name: "test-user",
AuthInfo: clientV1.AuthInfo{
Token: "test-token",
},
},
},
},
wantYAML: `
apiVersion: v1
clusters:
- cluster:
insecure-skip-tls-verify: true
server: localhost
name: portainer-cluster
contexts:
- context:
cluster: portainer-cluster
user: test-user
name: portainer-ctx
current-context: portainer-ctx
kind: Config
users:
- name: test-user
user:
token: test-token
`,
},
}
for _, ryt := range resourceYAMLTests {
t.Run(ryt.title, func(t *testing.T) {
yaml, err := GenerateYAML(ryt.resource)
if err != nil {
t.Errorf("generateYamlConfig failed; err=%s", err)
}
if compareYAMLStrings(yaml, ryt.wantYAML) != 0 {
t.Errorf("generateYamlConfig failed;\ngot=\n%s\nwant=\n%s", yaml, ryt.wantYAML)
}
})
}
}
func TestGetResourceQuotaFromNamespace(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
namespace := portainer.K8sNamespaceInfo{Name: "my-namespace"}
resourceQuotas := []v1.ResourceQuota{
{
ObjectMeta: metav1.ObjectMeta{
Name: "portainer-rq-" + namespace.Name + "-1",
Namespace: namespace.Name,
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "portainer-rq-" + namespace.Name,
Namespace: namespace.Name,
},
},
}
rq := kcl.GetResourceQuotaFromNamespace(namespace, resourceQuotas)
require.NotNil(t, rq)
require.Equal(t, namespace.Name, rq.Namespace)
// Empty cases
rq = kcl.GetResourceQuotaFromNamespace(namespace, nil)
require.Nil(t, rq)
namespace.Name = "another-namespace"
rq = kcl.GetResourceQuotaFromNamespace(namespace, resourceQuotas)
require.Nil(t, rq)
}
+160
View File
@@ -0,0 +1,160 @@
package cli
import (
"context"
"errors"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
rbacv1 "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetRoles gets all the roles for either at the cluster level or a given namespace in a k8s endpoint.
// It returns a list of K8sRole objects.
func (kcl *KubeClient) GetRoles(namespace string) ([]models.K8sRole, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchRoles(namespace)
}
return kcl.fetchRolesForNonAdmin(namespace)
}
// fetchRolesForNonAdmin gets all the roles for either at the cluster level or a given namespace in a k8s endpoint.
// the namespace will be coming from NonAdminNamespaces as non-admin users are restricted to certain namespaces.
// it returns a list of K8sRole objects.
func (kcl *KubeClient) fetchRolesForNonAdmin(namespace string) ([]models.K8sRole, error) {
roles, err := kcl.fetchRoles(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sRole, 0)
for _, role := range roles {
if _, ok := nonAdminNamespaceSet[role.Namespace]; ok {
results = append(results, role)
}
}
return results, nil
}
// fetchRoles returns a list of all Roles in the specified namespace.
func (kcl *KubeClient) fetchRoles(namespace string) ([]models.K8sRole, error) {
roles, err := kcl.cli.RbacV1().Roles(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sRole, 0)
for _, role := range roles.Items {
results = append(results, kcl.parseRole(role))
}
return results, nil
}
// parseRole converts a rbacv1.Role object to a models.K8sRole object.
func (kcl *KubeClient) parseRole(role rbacv1.Role) models.K8sRole {
return models.K8sRole{
Name: role.Name,
UID: role.UID,
Namespace: role.Namespace,
CreationDate: role.CreationTimestamp.Time,
IsSystem: kcl.isSystemRole(&role),
}
}
func getPortainerUserDefaultPolicies() []rbacv1.PolicyRule {
return []rbacv1.PolicyRule{
{
Verbs: []string{"list", "get"},
Resources: []string{"namespaces", "nodes", "endpoints"},
APIGroups: []string{""},
},
{
Verbs: []string{"list"},
Resources: []string{"storageclasses"},
APIGroups: []string{"storage.k8s.io"},
},
{
Verbs: []string{"list", "get"},
Resources: []string{"namespaces", "pods", "nodes"},
APIGroups: []string{"metrics.k8s.io"},
},
{
Verbs: []string{"list"},
Resources: []string{"ingressclasses"},
APIGroups: []string{"networking.k8s.io"},
},
}
}
func (kcl *KubeClient) upsertPortainerK8sClusterRoles() error {
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: portainerUserCRName,
},
Rules: getPortainerUserDefaultPolicies(),
}
_, err := kcl.cli.RbacV1().ClusterRoles().Create(context.TODO(), clusterRole, metav1.CreateOptions{})
if err != nil {
if k8serrors.IsAlreadyExists(err) {
_, err = kcl.cli.RbacV1().ClusterRoles().Update(context.TODO(), clusterRole, metav1.UpdateOptions{})
}
if err != nil {
return err
}
}
return nil
}
func getPortainerDefaultK8sRoleNames() []string {
return []string{
string(portainerUserCRName),
}
}
func (kcl *KubeClient) isSystemRole(role *rbacv1.Role) bool {
if strings.HasPrefix(role.Name, "system:") {
return true
}
return kcl.isSystemNamespace(role.Namespace)
}
// DeleteRoles processes a K8sServiceDeleteRequest by deleting each role
// in its given namespace.
func (kcl *KubeClient) DeleteRoles(reqs models.K8sRoleDeleteRequests) error {
var errs error
for namespace := range reqs {
for _, name := range reqs[namespace] {
client := kcl.cli.RbacV1().Roles(namespace)
role, err := client.Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
// This is a more serious error to do with the client so we return right away
return err
}
if kcl.isSystemRole(role) {
log.Error().Str("role_name", name).Msg("ignoring delete of 'system' role, not allowed")
}
if err := client.Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
errs = errors.Join(errs, err)
}
}
}
return errs
}
+132
View File
@@ -0,0 +1,132 @@
package cli
import (
"context"
"errors"
"strings"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
rbacv1 "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetRoleBindings gets all the roleBindings for either at the cluster level or a given namespace in a k8s endpoint.
// It returns a list of K8sRoleBinding objects.
func (kcl *KubeClient) GetRoleBindings(namespace string) ([]models.K8sRoleBinding, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchRoleBindings(namespace)
}
return kcl.fetchRolebindingsForNonAdmin(namespace)
}
// fetchRolebindingsForNonAdmin gets all the roleBindings for either at the cluster level or a given namespace in a k8s endpoint.
// the namespace will be coming from NonAdminNamespaces as non-admin users are restricted to certain namespaces.
// it returns a list of K8sRoleBinding objects.
func (kcl *KubeClient) fetchRolebindingsForNonAdmin(namespace string) ([]models.K8sRoleBinding, error) {
roleBindings, err := kcl.fetchRoleBindings(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sRoleBinding, 0)
for _, roleBinding := range roleBindings {
if _, ok := nonAdminNamespaceSet[roleBinding.Namespace]; ok {
results = append(results, roleBinding)
}
}
return results, nil
}
// fetchRoleBindings returns a list of all Roles in the specified namespace.
func (kcl *KubeClient) fetchRoleBindings(namespace string) ([]models.K8sRoleBinding, error) {
roleBindings, err := kcl.cli.RbacV1().RoleBindings(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sRoleBinding, 0)
for _, roleBinding := range roleBindings.Items {
results = append(results, kcl.parseRoleBinding(roleBinding))
}
return results, nil
}
// parseRoleBinding converts a rbacv1.RoleBinding object to a models.K8sRoleBinding object.
func (kcl *KubeClient) parseRoleBinding(roleBinding rbacv1.RoleBinding) models.K8sRoleBinding {
return models.K8sRoleBinding{
Name: roleBinding.Name,
UID: roleBinding.UID,
Namespace: roleBinding.Namespace,
RoleRef: roleBinding.RoleRef,
Subjects: roleBinding.Subjects,
CreationDate: roleBinding.CreationTimestamp.Time,
IsSystem: kcl.isSystemRoleBinding(&roleBinding),
}
}
func (kcl *KubeClient) isSystemRoleBinding(rb *rbacv1.RoleBinding) bool {
if strings.HasPrefix(rb.Name, "system:") {
return true
}
if rb.Labels != nil {
if rb.Labels["kubernetes.io/bootstrapping"] == "rbac-defaults" {
return true
}
}
if rb.RoleRef.Name != "" {
role, err := kcl.getRole(rb.Namespace, rb.RoleRef.Name)
if err != nil {
return false
}
// Linked to a role that is marked a system role
if kcl.isSystemRole(role) {
return true
}
}
return false
}
func (kcl *KubeClient) getRole(namespace, name string) (*rbacv1.Role, error) {
client := kcl.cli.RbacV1().Roles(namespace)
return client.Get(context.Background(), name, metav1.GetOptions{})
}
// DeleteRoleBindings processes a K8sServiceDeleteRequest by deleting each service
// in its given namespace.
func (kcl *KubeClient) DeleteRoleBindings(reqs models.K8sRoleBindingDeleteRequests) error {
var errs error
for namespace := range reqs {
for _, name := range reqs[namespace] {
client := kcl.cli.RbacV1().RoleBindings(namespace)
roleBinding, err := client.Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
// This is a more serious error to do with the client so we return right away
return err
}
if kcl.isSystemRoleBinding(roleBinding) {
log.Error().Str("role_name", name).Msg("ignoring delete of 'system' role binding, not allowed")
}
if err := client.Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
errs = errors.Join(errs, err)
}
}
}
return errs
}
+262
View File
@@ -0,0 +1,262 @@
package cli
import (
"context"
"errors"
"fmt"
"time"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
labelPortainerKubeConfigOwner = "io.portainer.kubernetes.configuration.owner"
labelPortainerKubeConfigOwnerId = "io.portainer.kubernetes.configuration.owner.id"
)
// GetSecrets gets all the Secrets for a given namespace in a k8s endpoint.
// if the user is an admin, all secrets in the current k8s environment(endpoint) are fetched using the getSecrets function.
// otherwise, namespaces the non-admin user has access to will be used to filter the secrets based on the allowed namespaces.
func (kcl *KubeClient) GetSecrets(namespace string) ([]models.K8sSecret, error) {
if kcl.GetIsKubeAdmin() {
return kcl.getSecrets(namespace)
}
return kcl.getSecretsForNonAdmin(namespace)
}
// getSecretsForNonAdmin fetches the secrets in the namespaces the user has access to.
// This function is called when the user is not an admin.
func (kcl *KubeClient) getSecretsForNonAdmin(namespace string) ([]models.K8sSecret, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching secrets for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
secrets, err := kcl.getSecrets(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sSecret, 0)
for _, secret := range secrets {
if _, ok := nonAdminNamespaceSet[secret.Namespace]; ok {
results = append(results, secret)
}
}
return results, nil
}
// getSecrets gets all the Secrets for a given namespace in a k8s endpoint.
// the result is a list of secrets parsed into a K8sSecret struct.
func (kcl *KubeClient) getSecrets(namespace string) ([]models.K8sSecret, error) {
secrets, err := kcl.cli.CoreV1().Secrets(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := []models.K8sSecret{}
for _, secret := range secrets.Items {
results = append(results, parseSecret(&secret, false))
}
return results, nil
}
// GetSecret gets a Secret by name for a given namespace.
// the result is a secret parsed into a K8sSecret struct.
func (kcl *KubeClient) GetSecret(namespace string, secretName string) (models.K8sSecret, error) {
secret, err := kcl.cli.CoreV1().Secrets(namespace).Get(context.Background(), secretName, metav1.GetOptions{})
if err != nil {
return models.K8sSecret{}, err
}
return parseSecret(secret, true), nil
}
// parseSecret parses a k8s Secret object into a K8sSecret struct.
// for get operation, withData will be set to true.
// otherwise, only metadata will be parsed.
func parseSecret(secret *corev1.Secret, withData bool) models.K8sSecret {
result := models.K8sSecret{
K8sConfiguration: models.K8sConfiguration{
UID: string(secret.UID),
Name: secret.Name,
Namespace: secret.Namespace,
CreationDate: secret.CreationTimestamp.Time.UTC().Format(time.RFC3339),
Annotations: secret.Annotations,
Labels: secret.Labels,
ConfigurationOwner: secret.Labels[labelPortainerKubeConfigOwner],
ConfigurationOwnerId: secret.Labels[labelPortainerKubeConfigOwnerId],
},
SecretType: string(secret.Type),
}
if withData {
secretData := secret.Data
secretDataMap := make(map[string]string, len(secretData))
for key, value := range secretData {
secretDataMap[key] = string(value)
}
result.Data = secretDataMap
}
return result
}
// SetSecretsIsUsed combines the secrets with the applications that use them.
// the function fetches all the pods and service accounts in the cluster and checks if the secret is used by any of them.
// if the secret is used by a pod or service account, the secret is marked as used.
// otherwise, the secret is returned as is.
func (kcl *KubeClient) SetSecretsIsUsed(secrets *[]models.K8sSecret) error {
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources("", metav1.ListOptions{})
if err != nil {
return fmt.Errorf("an error occurred during the SetSecretsIsUsed operation, unable to fetch Portainer application resources. Error: %w", err)
}
serviceAccounts, err := kcl.GetServiceAccounts("")
if err != nil {
return fmt.Errorf("an error occurred during the SetSecretsIsUsed operation, unable to fetch service accounts. Error: %w", err)
}
for i := range *secrets {
secret := &(*secrets)[i]
if isSecretUsedByServiceAccount(*secret, serviceAccounts) {
secret.IsUsed = true
continue
}
for _, pod := range portainerApplicationResources.Pods {
if isPodUsingSecret(&pod, *secret) {
secret.IsUsed = true
break
}
}
}
return nil
}
func isSecretUsedByServiceAccount(secret models.K8sSecret, serviceAccounts []models.K8sServiceAccount) bool {
for _, serviceAccount := range serviceAccounts {
if serviceAccount.Namespace != secret.Namespace {
continue
}
for _, imagePullSecret := range serviceAccount.ImagePullSecrets {
if imagePullSecret.Name == secret.Name {
return true
}
}
}
return false
}
// CombineSecretWithApplications combines the secret with the applications that use it.
// the function fetches all the pods in the cluster and checks if the secret is used by any of the pods.
// it needs to check if the pods are owned by a replica set to determine if the pod is part of a deployment.
func (kcl *KubeClient) CombineSecretWithApplications(secret models.K8sSecret) (models.K8sSecret, error) {
pods, err := kcl.cli.CoreV1().Pods(secret.Namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return models.K8sSecret{}, fmt.Errorf("an error occurred during the CombineSecretWithApplications operation, unable to get pods. Error: %w", err)
}
replicaSetsItems := []appsv1.ReplicaSet{}
if containsReplicaSetOwnerReference(pods) {
replicaSets, err := kcl.cli.AppsV1().ReplicaSets(secret.Namespace).List(context.Background(), metav1.ListOptions{})
if err != nil {
return models.K8sSecret{}, fmt.Errorf("an error occurred during the CombineSecretWithApplications operation, unable to get replica sets. Error: %w", err)
}
replicaSetsItems = replicaSets.Items
}
applicationConfigurationOwners, err := kcl.GetApplicationConfigurationOwnersFromSecret(secret, pods.Items, replicaSetsItems)
if err != nil {
return models.K8sSecret{}, fmt.Errorf("an error occurred during the CombineSecretWithApplications operation, unable to get applications from secret. Error: %w", err)
}
if len(applicationConfigurationOwners) > 0 {
secret.ConfigurationOwnerResources = applicationConfigurationOwners
secret.IsUsed = true
}
return secret, nil
}
func (kcl *KubeClient) createServiceAccountToken(serviceAccountName string) error {
serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID)
serviceAccountSecret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountSecretName,
Annotations: map[string]string{
"kubernetes.io/service-account.name": serviceAccountName,
},
},
Type: "kubernetes.io/service-account-token",
}
_, err := kcl.cli.CoreV1().Secrets(portainerNamespace).Create(context.TODO(), serviceAccountSecret, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (kcl *KubeClient) getServiceAccountToken(serviceAccountName string) (string, error) {
serviceAccountSecretName := userServiceAccountTokenSecretName(serviceAccountName, kcl.instanceID)
secret, err := kcl.cli.CoreV1().Secrets(portainerNamespace).Get(context.TODO(), serviceAccountSecretName, metav1.GetOptions{})
if err != nil {
return "", err
}
// API token secret is populated asynchronously.
// Is it created by the controller and will depend on the environment(endpoint)/secret-store:
// https://github.com/kubernetes/kubernetes/issues/67882#issuecomment-422026204
// as a work-around, we wait for up to 5 seconds for the secret to be populated.
timeout := time.After(5 * time.Second)
searchingForSecret := true
for searchingForSecret {
select {
case <-timeout:
return "", errors.New("unable to find secret token associated to user service account (timeout)")
default:
secret, err = kcl.cli.CoreV1().Secrets(portainerNamespace).Get(context.TODO(), serviceAccountSecretName, metav1.GetOptions{})
if err != nil {
return "", err
}
if len(secret.Data) > 0 {
searchingForSecret = false
break
}
time.Sleep(1 * time.Second)
}
}
secretTokenData, ok := secret.Data["token"]
if ok {
return string(secretTokenData), nil
}
return "", errors.New("unable to find secret token associated to user service account")
}
+152
View File
@@ -0,0 +1,152 @@
package cli
import (
"testing"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func Test_SetSecretsIsUsed_ServiceAccountImagePullSecret(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-secret",
Namespace: "default",
},
Type: corev1.SecretTypeDockerConfigJson,
}
_, err := k.cli.CoreV1().Secrets("default").Create(t.Context(), secret, metav1.CreateOptions{})
require.NoError(t, err)
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "app-sa",
Namespace: "default",
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-secret"},
},
}
_, err = k.cli.CoreV1().ServiceAccounts("default").Create(t.Context(), serviceAccount, metav1.CreateOptions{})
require.NoError(t, err)
secrets := []models.K8sSecret{parseSecret(secret, false)}
err = k.SetSecretsIsUsed(&secrets)
require.NoError(t, err)
assert.True(t, secrets[0].IsUsed)
}
func Test_SetSecretsIsUsed_NotReferencedByAnySA(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "unused-secret",
Namespace: "default",
},
Type: corev1.SecretTypeDockerConfigJson,
}
_, err := k.cli.CoreV1().Secrets("default").Create(t.Context(), secret, metav1.CreateOptions{})
require.NoError(t, err)
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "app-sa",
Namespace: "default",
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "other-secret"},
},
}
_, err = k.cli.CoreV1().ServiceAccounts("default").Create(t.Context(), serviceAccount, metav1.CreateOptions{})
require.NoError(t, err)
secrets := []models.K8sSecret{parseSecret(secret, false)}
err = k.SetSecretsIsUsed(&secrets)
require.NoError(t, err)
assert.False(t, secrets[0].IsUsed)
}
func Test_SetSecretsIsUsed_SAInDifferentNamespace(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
// Create a secret named "registry-secret" in the default namespace
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-secret",
Namespace: "default",
},
Type: corev1.SecretTypeDockerConfigJson,
}
_, err := k.cli.CoreV1().Secrets("default").Create(t.Context(), secret, metav1.CreateOptions{})
require.NoError(t, err)
// Create a service account in a different namespace that references a secret with the same name.
// In Kubernetes, secrets are namespace-scoped, so this SA references a different secret
// (one that doesn't exist in "other-namespace"), not the one we created in "default".
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "app-sa",
Namespace: "other-namespace",
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-secret"},
},
}
_, err = k.cli.CoreV1().ServiceAccounts("other-namespace").Create(t.Context(), serviceAccount, metav1.CreateOptions{})
require.NoError(t, err)
secrets := []models.K8sSecret{parseSecret(secret, false)}
err = k.SetSecretsIsUsed(&secrets)
// The secret in the default namespace should not be marked as used,
// since the SA reference is to a secret in a different namespace.
require.NoError(t, err)
assert.False(t, secrets[0].IsUsed)
}
func Test_SetSecretsIsUsed_SAWithEmptyImagePullSecrets(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-secret",
Namespace: "default",
},
Type: corev1.SecretTypeDockerConfigJson,
}
_, err := k.cli.CoreV1().Secrets("default").Create(t.Context(), secret, metav1.CreateOptions{})
require.NoError(t, err)
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "app-sa",
Namespace: "default",
},
}
_, err = k.cli.CoreV1().ServiceAccounts("default").Create(t.Context(), serviceAccount, metav1.CreateOptions{})
require.NoError(t, err)
secrets := []models.K8sSecret{parseSecret(secret, false)}
err = k.SetSecretsIsUsed(&secrets)
require.NoError(t, err)
assert.False(t, secrets[0].IsUsed)
}
+7
View File
@@ -0,0 +1,7 @@
package cli
import "k8s.io/apimachinery/pkg/version"
func (kcl *KubeClient) ServerVersion() (*version.Info, error) {
return kcl.cli.Discovery().ServerVersion()
}
+227
View File
@@ -0,0 +1,227 @@
package cli
import (
"context"
"fmt"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
// GetServices gets all the services for either at the cluster level or a given namespace in a k8s endpoint.
// It returns a list of K8sServiceInfo objects.
func (kcl *KubeClient) GetServices(namespace string) ([]models.K8sServiceInfo, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchServices(namespace)
}
return kcl.fetchServicesForNonAdmin(namespace)
}
// fetchServicesForNonAdmin gets all the services for either at the cluster level or a given namespace in a k8s endpoint.
// the namespace will be coming from NonAdminNamespaces as non-admin users are restricted to certain namespaces.
// it returns a list of K8sServiceInfo objects.
func (kcl *KubeClient) fetchServicesForNonAdmin(namespace string) ([]models.K8sServiceInfo, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching services for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
services, err := kcl.fetchServices(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sServiceInfo, 0)
for _, service := range services {
if _, ok := nonAdminNamespaceSet[service.Namespace]; ok {
results = append(results, service)
}
}
return results, nil
}
// fetchServices gets the services in a given namespace in a k8s endpoint.
// It returns a list of K8sServiceInfo objects.
func (kcl *KubeClient) fetchServices(namespace string) ([]models.K8sServiceInfo, error) {
services, err := kcl.cli.CoreV1().Services(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sServiceInfo, 0)
for _, service := range services.Items {
results = append(results, parseService(service))
}
return results, nil
}
// parseService converts a k8s native service object to a Portainer K8sServiceInfo object.
// service ports, ingress status, labels, annotations, cluster IPs, and external IPs are parsed.
// it returns a K8sServiceInfo object.
func parseService(service corev1.Service) models.K8sServiceInfo {
servicePorts := make([]models.K8sServicePort, 0)
for _, port := range service.Spec.Ports {
servicePorts = append(servicePorts, models.K8sServicePort{
Name: port.Name,
NodePort: int(port.NodePort),
Port: int(port.Port),
Protocol: string(port.Protocol),
TargetPort: port.TargetPort.String(),
})
}
ingressStatus := make([]models.K8sServiceIngress, 0)
for _, status := range service.Status.LoadBalancer.Ingress {
ingressStatus = append(ingressStatus, models.K8sServiceIngress{
IP: status.IP,
Hostname: status.Hostname,
})
}
return models.K8sServiceInfo{
Name: service.Name,
UID: string(service.GetUID()),
Type: string(service.Spec.Type),
Namespace: service.Namespace,
CreationDate: service.GetCreationTimestamp().String(),
AllocateLoadBalancerNodePorts: service.Spec.AllocateLoadBalancerNodePorts,
Ports: servicePorts,
IngressStatus: ingressStatus,
Labels: service.GetLabels(),
Annotations: service.GetAnnotations(),
ClusterIPs: service.Spec.ClusterIPs,
ExternalName: service.Spec.ExternalName,
ExternalIPs: service.Spec.ExternalIPs,
Selector: service.Spec.Selector,
}
}
// convertToK8sService converts a K8sServiceInfo object back to a k8s native service object.
// this is required for create and update operations.
// it returns a v1.Service object.
func (kcl *KubeClient) convertToK8sService(info models.K8sServiceInfo) corev1.Service {
service := corev1.Service{}
service.Name = info.Name
service.Spec.Type = corev1.ServiceType(info.Type)
service.Namespace = info.Namespace
service.Annotations = info.Annotations
service.Labels = info.Labels
service.Spec.AllocateLoadBalancerNodePorts = info.AllocateLoadBalancerNodePorts
service.Spec.Selector = info.Selector
for _, p := range info.Ports {
port := corev1.ServicePort{}
port.Name = p.Name
port.NodePort = int32(p.NodePort)
port.Port = int32(p.Port)
port.Protocol = corev1.Protocol(p.Protocol)
port.TargetPort = intstr.FromString(p.TargetPort)
service.Spec.Ports = append(service.Spec.Ports, port)
}
for _, i := range info.IngressStatus {
service.Status.LoadBalancer.Ingress = append(
service.Status.LoadBalancer.Ingress,
corev1.LoadBalancerIngress{IP: i.IP, Hostname: i.Hostname},
)
}
return service
}
// CreateService creates a new service in a given namespace in a k8s endpoint.
func (kcl *KubeClient) CreateService(namespace string, info models.K8sServiceInfo) error {
service := kcl.convertToK8sService(info)
_, err := kcl.cli.CoreV1().Services(namespace).Create(context.Background(), &service, metav1.CreateOptions{})
return err
}
// DeleteServices processes a K8sServiceDeleteRequest by deleting each service
// in its given namespace.
func (kcl *KubeClient) DeleteServices(reqs models.K8sServiceDeleteRequests) error {
for namespace := range reqs {
for _, service := range reqs[namespace] {
err := kcl.cli.CoreV1().Services(namespace).Delete(context.Background(), service, metav1.DeleteOptions{})
if err != nil {
return err
}
}
}
return nil
}
// UpdateService updates service in a given namespace in a k8s endpoint.
func (kcl *KubeClient) UpdateService(namespace string, info models.K8sServiceInfo) error {
service := kcl.convertToK8sService(info)
_, err := kcl.cli.CoreV1().Services(namespace).Update(context.Background(), &service, metav1.UpdateOptions{})
return err
}
// CombineServicesWithApplications retrieves applications based on service selectors in a given namespace
// for all services, it lists pods based on the service selector and converts the pod to an application
// if replicasets are found, it updates the owner reference to deployment
// it then combines the service with the application
// finally, it returns a list of K8sServiceInfo objects
func (kcl *KubeClient) CombineServicesWithApplications(services []models.K8sServiceInfo) ([]models.K8sServiceInfo, error) {
if containsServiceWithSelector(services) {
updatedServices := make([]models.K8sServiceInfo, len(services))
portainerApplicationResources, err := kcl.fetchAllApplicationsListResources("", metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("an error occurred during the CombineServicesWithApplications operation, unable to fetch pods and replica sets. Error: %w", err)
}
for index, service := range services {
updatedService := service
application, err := kcl.GetApplicationFromServiceSelector(portainerApplicationResources.Pods, service, portainerApplicationResources.ReplicaSets)
if err != nil {
return services, fmt.Errorf("an error occurred during the CombineServicesWithApplications operation, unable to get application from service. Error: %w", err)
}
if application != nil {
updatedService.Applications = append(updatedService.Applications, *application)
}
updatedServices[index] = updatedService
}
return updatedServices, nil
}
return services, nil
}
// containsServiceWithSelector checks if a list of services contains a service with a selector
// it returns true if any service has a selector, otherwise false
func containsServiceWithSelector(services []models.K8sServiceInfo) bool {
for _, service := range services {
if len(service.Selector) > 0 {
return true
}
}
return false
}
// buildServicesMap builds a map of service names from a list of K8sServiceInfo objects
// it returns a map of service names for lookups
func (kcl *KubeClient) buildServicesMap(services []models.K8sServiceInfo) map[string]struct{} {
serviceMap := make(map[string]struct{})
for _, service := range services {
serviceMap[service.Name] = struct{}{}
}
return serviceMap
}
+372
View File
@@ -0,0 +1,372 @@
package cli
import (
"context"
"errors"
"fmt"
portainer "github.com/portainer/portainer/api"
models "github.com/portainer/portainer/api/http/models/kubernetes"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetServiceAccounts gets all the service accounts for either at the cluster level or a given namespace in a k8s endpoint.
// It returns a list of K8sServiceAccount objects.
func (kcl *KubeClient) GetServiceAccounts(namespace string) ([]models.K8sServiceAccount, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchServiceAccounts(namespace)
}
return kcl.fetchServiceAccountsForNonAdmin(namespace)
}
// fetchServiceAccountsForNonAdmin gets all the service accounts for either at the cluster level or a given namespace in a k8s endpoint.
// the namespace will be coming from NonAdminNamespaces as non-admin users are restricted to certain namespaces.
// it returns a list of K8sServiceAccount objects.
func (kcl *KubeClient) fetchServiceAccountsForNonAdmin(namespace string) ([]models.K8sServiceAccount, error) {
serviceAccounts, err := kcl.fetchServiceAccounts(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sServiceAccount, 0)
for _, serviceAccount := range serviceAccounts {
if _, ok := nonAdminNamespaceSet[serviceAccount.Namespace]; ok {
results = append(results, serviceAccount)
}
}
return results, nil
}
// fetchServiceAccounts returns a list of all ServiceAccounts in the specified namespace.
func (kcl *KubeClient) fetchServiceAccounts(namespace string) ([]models.K8sServiceAccount, error) {
serviceAccounts, err := kcl.cli.CoreV1().ServiceAccounts(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
results := make([]models.K8sServiceAccount, 0)
for _, serviceAccount := range serviceAccounts.Items {
results = append(results, kcl.parseServiceAccount(serviceAccount))
}
return results, nil
}
// parseServiceAccount converts a corev1.ServiceAccount object to a models.K8sServiceAccount object.
func (kcl *KubeClient) parseServiceAccount(serviceAccount corev1.ServiceAccount) models.K8sServiceAccount {
return models.K8sServiceAccount{
Name: serviceAccount.Name,
UID: serviceAccount.UID,
Namespace: serviceAccount.Namespace,
CreationDate: serviceAccount.CreationTimestamp.Time,
IsSystem: kcl.isSystemServiceAccount(serviceAccount.Namespace),
ImagePullSecrets: serviceAccount.ImagePullSecrets,
}
}
// GetServiceAccount returns the details of a single service account in the given namespace.
func (kcl *KubeClient) GetServiceAccount(namespace, name string) (models.K8sServiceAccount, error) {
sa, err := kcl.cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return models.K8sServiceAccount{}, err
}
return models.K8sServiceAccount{
Name: sa.Name,
UID: sa.UID,
Namespace: sa.Namespace,
CreationDate: sa.CreationTimestamp.Time,
IsSystem: kcl.isSystemServiceAccount(sa.Namespace),
AutomountServiceAccountToken: sa.AutomountServiceAccountToken,
ImagePullSecrets: sa.ImagePullSecrets,
Labels: sa.Labels,
Annotations: sa.Annotations,
}, nil
}
// GetPortainerUserServiceAccount returns the portainer ServiceAccountName associated to the specified user.
func (kcl *KubeClient) GetPortainerUserServiceAccount(tokenData *portainer.TokenData) (*corev1.ServiceAccount, error) {
portainerUserServiceAccountName := UserServiceAccountName(int(tokenData.ID), kcl.instanceID)
if tokenData.Role == portainer.AdministratorRole {
portainerUserServiceAccountName = portainerClusterAdminServiceAccountName
}
// verify name exists as service account resource within portainer namespace
serviceAccount, err := kcl.cli.CoreV1().ServiceAccounts(portainerNamespace).Get(context.TODO(), portainerUserServiceAccountName, metav1.GetOptions{})
if err != nil {
return nil, err
}
return serviceAccount, nil
}
func (kcl *KubeClient) isSystemServiceAccount(namespace string) bool {
return kcl.isSystemNamespace(namespace)
}
// DeleteServices processes a K8sServiceDeleteRequest by deleting each service
// in its given namespace.
func (kcl *KubeClient) DeleteServiceAccounts(reqs models.K8sServiceAccountDeleteRequests) error {
var errs error
for namespace := range reqs {
for _, serviceName := range reqs[namespace] {
client := kcl.cli.CoreV1().ServiceAccounts(namespace)
sa, err := client.Get(context.Background(), serviceName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
continue
}
return err
}
if kcl.isSystemServiceAccount(sa.Namespace) {
return fmt.Errorf("cannot delete system service account %q", namespace+"/"+serviceName)
}
if err := client.Delete(context.Background(), serviceName, metav1.DeleteOptions{}); err != nil {
errs = errors.Join(errs, err)
}
}
}
return errs
}
// GetServiceAccountBearerToken returns the ServiceAccountToken associated to the specified user.
func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error) {
serviceAccountName := UserServiceAccountName(userID, kcl.instanceID)
return kcl.getServiceAccountToken(serviceAccountName)
}
// SetupUserServiceAccount will make sure that all the required resources are created inside the Kubernetes
// cluster before creating a ServiceAccount and a ServiceAccountToken for the specified Portainer user.
// It will also create required default RoleBinding and ClusterRoleBinding rules.
func (kcl *KubeClient) SetupUserServiceAccount(userID int, teamIDs []int, restrictDefaultNamespace bool) error {
serviceAccountName := UserServiceAccountName(userID, kcl.instanceID)
err := kcl.ensureRequiredResourcesExist()
if err != nil {
return err
}
err = kcl.createUserServiceAccount(portainerNamespace, serviceAccountName)
if err != nil {
return err
}
err = kcl.createServiceAccountToken(serviceAccountName)
if err != nil {
return err
}
err = kcl.ensureServiceAccountHasPortainerUserClusterRole(serviceAccountName)
if err != nil {
return err
}
return kcl.setupNamespaceAccesses(userID, teamIDs, serviceAccountName, restrictDefaultNamespace)
}
func (kcl *KubeClient) ensureRequiredResourcesExist() error {
return kcl.upsertPortainerK8sClusterRoles()
}
func (kcl *KubeClient) createUserServiceAccount(namespace, serviceAccountName string) error {
serviceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
},
}
_, err := kcl.cli.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), serviceAccount, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (kcl *KubeClient) ensureServiceAccountHasPortainerUserClusterRole(serviceAccountName string) error {
clusterRoleBinding, err := kcl.cli.RbacV1().ClusterRoleBindings().Get(context.TODO(), portainerUserCRBName, metav1.GetOptions{})
if k8serrors.IsNotFound(err) {
clusterRoleBinding = &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: portainerUserCRBName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: portainerNamespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "ClusterRole",
Name: portainerUserCRName,
},
}
_, err := kcl.cli.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterRoleBinding, metav1.CreateOptions{})
return err
} else if err != nil {
return err
}
for _, subject := range clusterRoleBinding.Subjects {
if subject.Name == serviceAccountName {
return nil
}
}
clusterRoleBinding.Subjects = append(clusterRoleBinding.Subjects, rbacv1.Subject{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: portainerNamespace,
})
_, err = kcl.cli.RbacV1().ClusterRoleBindings().Update(context.TODO(), clusterRoleBinding, metav1.UpdateOptions{})
return err
}
func (kcl *KubeClient) removeNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error {
roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID)
roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{})
if k8serrors.IsNotFound(err) {
return nil
} else if err != nil {
return err
}
updatedSubjects := roleBinding.Subjects[:0]
for _, subject := range roleBinding.Subjects {
if subject.Name != serviceAccountName {
updatedSubjects = append(updatedSubjects, subject)
}
}
roleBinding.Subjects = updatedSubjects
_, err = kcl.cli.RbacV1().RoleBindings(namespace).Update(context.TODO(), roleBinding, metav1.UpdateOptions{})
return err
}
func (kcl *KubeClient) AddImagePullSecretToServiceAccount(namespace, serviceAccountName, secretName string) error {
sa, err := kcl.cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), serviceAccountName, metav1.GetOptions{})
if err != nil {
return err
}
for _, ref := range sa.ImagePullSecrets {
if ref.Name == secretName {
return nil
}
}
sa.ImagePullSecrets = append(sa.ImagePullSecrets, corev1.LocalObjectReference{Name: secretName})
_, err = kcl.cli.CoreV1().ServiceAccounts(namespace).Update(context.TODO(), sa, metav1.UpdateOptions{})
return err
}
func (kcl *KubeClient) RemoveImagePullSecretFromServiceAccount(namespace, serviceAccountName, secretName string) error {
sa, err := kcl.cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), serviceAccountName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return err
}
updated := sa.ImagePullSecrets[:0]
changed := false
for _, ref := range sa.ImagePullSecrets {
if ref.Name == secretName {
changed = true
continue
}
updated = append(updated, ref)
}
if !changed {
return nil
}
sa.ImagePullSecrets = updated
_, err = kcl.cli.CoreV1().ServiceAccounts(namespace).Update(context.TODO(), sa, metav1.UpdateOptions{})
return err
}
func (kcl *KubeClient) UpdateServiceAccountImagePullSecrets(namespace, name string, secretNames []string) error {
sa, err := kcl.cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get service account %q: %w", name, err)
}
refs := make([]corev1.LocalObjectReference, 0, len(secretNames))
for _, s := range secretNames {
refs = append(refs, corev1.LocalObjectReference{Name: s})
}
sa.ImagePullSecrets = refs
_, err = kcl.cli.CoreV1().ServiceAccounts(namespace).Update(context.TODO(), sa, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to update service account %q: %w", name, err)
}
return nil
}
func (kcl *KubeClient) ensureNamespaceAccessForServiceAccount(serviceAccountName, namespace string) error {
roleBindingName := namespaceClusterRoleBindingName(namespace, kcl.instanceID)
roleBinding, err := kcl.cli.RbacV1().RoleBindings(namespace).Get(context.TODO(), roleBindingName, metav1.GetOptions{})
if k8serrors.IsNotFound(err) {
roleBinding = &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: portainerNamespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "ClusterRole",
Name: "edit",
},
}
_, err = kcl.cli.RbacV1().RoleBindings(namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{})
return err
} else if err != nil {
return err
}
for _, subject := range roleBinding.Subjects {
if subject.Name == serviceAccountName {
return nil
}
}
roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: portainerNamespace,
})
_, err = kcl.cli.RbacV1().RoleBindings(namespace).Update(context.TODO(), roleBinding, metav1.UpdateOptions{})
return err
}
+422
View File
@@ -0,0 +1,422 @@
package cli
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func Test_GetServiceAccount(t *testing.T) {
t.Parallel()
t.Run("returns error if non-existent", func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
tokenData := &portainer.TokenData{ID: 1}
_, err := k.GetPortainerUserServiceAccount(tokenData)
if err == nil {
t.Error("GetPortainerUserServiceAccount should fail with service account not found")
}
})
t.Run("succeeds for cluster admin role", func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
tokenData := &portainer.TokenData{
ID: 1,
Role: portainer.AdministratorRole,
Username: portainerClusterAdminServiceAccountName,
}
serviceAccount := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: tokenData.Username,
},
}
_, err := k.cli.CoreV1().ServiceAccounts(portainerNamespace).Create(t.Context(), serviceAccount, metav1.CreateOptions{})
if err != nil {
t.Errorf("failed to create service acount; err=%s", err)
}
defer func() {
err := k.cli.CoreV1().ServiceAccounts(portainerNamespace).Delete(t.Context(), serviceAccount.Name, metav1.DeleteOptions{})
require.NoError(t, err)
}()
sa, err := k.GetPortainerUserServiceAccount(tokenData)
if err != nil {
t.Errorf("GetPortainerUserServiceAccount should succeed; err=%s", err)
}
want := "portainer-sa-clusteradmin"
if sa.Name != want {
t.Errorf("GetServiceAccount should succeed and return correct sa name; got=%s want=%s", sa.Name, want)
}
})
t.Run("succeeds for standard user role", func(t *testing.T) {
k := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
tokenData := &portainer.TokenData{
ID: 1,
Role: portainer.StandardUserRole,
}
serviceAccountName := UserServiceAccountName(int(tokenData.ID), k.instanceID)
serviceAccount := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
},
}
_, err := k.cli.CoreV1().ServiceAccounts(portainerNamespace).Create(t.Context(), serviceAccount, metav1.CreateOptions{})
if err != nil {
t.Errorf("failed to create service acount; err=%s", err)
}
defer func() {
err := k.cli.CoreV1().ServiceAccounts(portainerNamespace).Delete(t.Context(), serviceAccount.Name, metav1.DeleteOptions{})
require.NoError(t, err)
}()
sa, err := k.GetPortainerUserServiceAccount(tokenData)
if err != nil {
t.Errorf("GetPortainerUserServiceAccount should succeed; err=%s", err)
}
want := "portainer-sa-user-test-1"
if sa.Name != want {
t.Errorf("GetPortainerUserServiceAccount should succeed and return correct sa name; got=%s want=%s", sa.Name, want)
}
})
}
func TestGetServiceAccountDetails(t *testing.T) {
t.Parallel()
t.Run("returns service account details", func(t *testing.T) {
automount := false
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "my-sa",
Namespace: "default",
Labels: map[string]string{"app": "web"},
},
AutomountServiceAccountToken: &automount,
ImagePullSecrets: []v1.LocalObjectReference{
{Name: "registry-secret"},
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(sa),
instanceID: "test",
}
result, err := kcl.GetServiceAccount("default", "my-sa")
require.NoError(t, err)
assert.Equal(t, "my-sa", result.Name)
assert.Equal(t, "default", result.Namespace)
assert.Equal(t, &automount, result.AutomountServiceAccountToken)
assert.Len(t, result.ImagePullSecrets, 1)
assert.Equal(t, "registry-secret", result.ImagePullSecrets[0].Name)
assert.Equal(t, map[string]string{"app": "web"}, result.Labels)
})
t.Run("returns error when service account not found", func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
_, err := kcl.GetServiceAccount("default", "does-not-exist")
require.Error(t, err)
})
t.Run("marks system namespace accounts as system", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "kube-system"},
}
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "kube-system"},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(sa, ns),
instanceID: "test",
}
result, err := kcl.GetServiceAccount("kube-system", "default")
require.NoError(t, err)
assert.True(t, result.IsSystem)
})
t.Run("returns nil automount when not set", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "my-sa", Namespace: "default"},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(sa),
instanceID: "test",
}
result, err := kcl.GetServiceAccount("default", "my-sa")
require.NoError(t, err)
assert.Nil(t, result.AutomountServiceAccountToken)
})
}
func TestAddImagePullSecretToServiceAccount(t *testing.T) {
t.Parallel()
newKCL := func(sa *v1.ServiceAccount) *KubeClient {
return &KubeClient{cli: kfake.NewSimpleClientset(sa), instanceID: "test"}
}
defaultSA := func(namespace string, refs ...string) *v1.ServiceAccount {
pullSecrets := make([]v1.LocalObjectReference, len(refs))
for i, r := range refs {
pullSecrets[i] = v1.LocalObjectReference{Name: r}
}
return &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: namespace},
ImagePullSecrets: pullSecrets,
}
}
t.Run("adds entry to SA with empty ImagePullSecrets", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a"))
require.NoError(t, kcl.AddImagePullSecretToServiceAccount("ns-a", "default", "registry-1"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, sa.ImagePullSecrets, 1)
assert.Equal(t, "registry-1", sa.ImagePullSecrets[0].Name)
})
t.Run("is idempotent when secret already present", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "registry-1"))
require.NoError(t, kcl.AddImagePullSecretToServiceAccount("ns-a", "default", "registry-1"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
assert.Len(t, sa.ImagePullSecrets, 1)
})
t.Run("preserves pre-existing pull secrets", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "other-1", "other-2"))
require.NoError(t, kcl.AddImagePullSecretToServiceAccount("ns-a", "default", "registry-3"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, sa.ImagePullSecrets, 3)
assert.Equal(t, "other-1", sa.ImagePullSecrets[0].Name)
assert.Equal(t, "other-2", sa.ImagePullSecrets[1].Name)
assert.Equal(t, "registry-3", sa.ImagePullSecrets[2].Name)
})
t.Run("returns error when SA does not exist", func(t *testing.T) {
kcl := &KubeClient{cli: kfake.NewSimpleClientset(), instanceID: "test"}
err := kcl.AddImagePullSecretToServiceAccount("ns-a", "default", "registry-1")
require.Error(t, err)
})
t.Run("works with non-default service account name", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "ns-a"},
}
kcl := newKCL(sa)
require.NoError(t, kcl.AddImagePullSecretToServiceAccount("ns-a", "custom", "registry-1"))
got, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "custom", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, got.ImagePullSecrets, 1)
assert.Equal(t, "registry-1", got.ImagePullSecrets[0].Name)
})
}
func TestRemoveImagePullSecretFromServiceAccount(t *testing.T) {
t.Parallel()
newKCL := func(sa *v1.ServiceAccount) *KubeClient {
return &KubeClient{cli: kfake.NewSimpleClientset(sa), instanceID: "test"}
}
defaultSA := func(namespace string, refs ...string) *v1.ServiceAccount {
pullSecrets := make([]v1.LocalObjectReference, len(refs))
for i, r := range refs {
pullSecrets[i] = v1.LocalObjectReference{Name: r}
}
return &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: namespace},
ImagePullSecrets: pullSecrets,
}
}
t.Run("removes target entry from ImagePullSecrets", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "registry-1"))
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-1"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
assert.Empty(t, sa.ImagePullSecrets)
})
t.Run("is idempotent when secret not in list", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "other-secret"))
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-1"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
assert.Len(t, sa.ImagePullSecrets, 1)
})
t.Run("preserves other pull secrets when removing target", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "other-1", "registry-2", "other-3"))
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-2"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, sa.ImagePullSecrets, 2)
assert.Equal(t, "other-1", sa.ImagePullSecrets[0].Name)
assert.Equal(t, "other-3", sa.ImagePullSecrets[1].Name)
})
t.Run("returns nil when SA does not exist", func(t *testing.T) {
kcl := &KubeClient{cli: kfake.NewSimpleClientset(), instanceID: "test"}
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-1"))
})
t.Run("returns nil when ImagePullSecrets is nil", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "ns-a"},
}
kcl := newKCL(sa)
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-1"))
})
t.Run("removes all occurrences when target appears more than once", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "registry-1", "registry-1"))
require.NoError(t, kcl.RemoveImagePullSecretFromServiceAccount("ns-a", "default", "registry-1"))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
assert.Empty(t, sa.ImagePullSecrets)
})
}
func TestGetServiceAccount_CreatesAndFetches(t *testing.T) {
t.Parallel()
t.Run("returns annotations when set", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "annotated-sa",
Namespace: "default",
Annotations: map[string]string{"example.com/key": "value"},
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(sa),
instanceID: "test",
}
result, err := kcl.GetServiceAccount("default", "annotated-sa")
require.NoError(t, err)
assert.Equal(t, map[string]string{"example.com/key": "value"}, result.Annotations)
})
t.Run("round-trips UID correctly", func(t *testing.T) {
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "uid-sa",
Namespace: "default",
UID: "abc-123-def",
},
}
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(sa),
instanceID: "test",
}
result, err := kcl.GetServiceAccount("default", "uid-sa")
require.NoError(t, err)
assert.Equal(t, "abc-123-def", string(result.UID))
})
t.Run("creates service account and fetches it back", func(t *testing.T) {
kcl := &KubeClient{
cli: kfake.NewSimpleClientset(),
instanceID: "test",
}
sa := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "fresh-sa", Namespace: "staging"},
}
_, err := kcl.cli.CoreV1().ServiceAccounts("staging").Create(t.Context(), sa, metav1.CreateOptions{})
require.NoError(t, err)
result, err := kcl.GetServiceAccount("staging", "fresh-sa")
require.NoError(t, err)
assert.Equal(t, "fresh-sa", result.Name)
assert.Equal(t, "staging", result.Namespace)
})
}
func TestUpdateServiceAccountImagePullSecrets(t *testing.T) {
t.Parallel()
newKCL := func(sa *v1.ServiceAccount) *KubeClient {
return &KubeClient{cli: kfake.NewSimpleClientset(sa), instanceID: "test"}
}
defaultSA := func(namespace string, refs ...string) *v1.ServiceAccount {
pullSecrets := make([]v1.LocalObjectReference, len(refs))
for i, r := range refs {
pullSecrets[i] = v1.LocalObjectReference{Name: r}
}
return &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: namespace},
ImagePullSecrets: pullSecrets,
}
}
t.Run("sets full list replacing existing", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "old-1", "old-2"))
require.NoError(t, kcl.UpdateServiceAccountImagePullSecrets("ns-a", "default", []string{"new-1"}))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, sa.ImagePullSecrets, 1)
assert.Equal(t, "new-1", sa.ImagePullSecrets[0].Name)
})
t.Run("clears list when secretNames is empty", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a", "secret-1", "secret-2"))
require.NoError(t, kcl.UpdateServiceAccountImagePullSecrets("ns-a", "default", []string{}))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
assert.Empty(t, sa.ImagePullSecrets)
})
t.Run("returns error when SA does not exist", func(t *testing.T) {
kcl := &KubeClient{cli: kfake.NewSimpleClientset(), instanceID: "test"}
err := kcl.UpdateServiceAccountImagePullSecrets("ns-a", "does-not-exist", []string{"secret-1"})
require.Error(t, err)
})
t.Run("sets list on SA with no existing pull secrets", func(t *testing.T) {
kcl := newKCL(defaultSA("ns-a"))
require.NoError(t, kcl.UpdateServiceAccountImagePullSecrets("ns-a", "default", []string{"s1", "s2"}))
sa, err := kcl.cli.CoreV1().ServiceAccounts("ns-a").Get(t.Context(), "default", metav1.GetOptions{})
require.NoError(t, err)
require.Len(t, sa.ImagePullSecrets, 2)
assert.Equal(t, "s1", sa.ImagePullSecrets[0].Name)
assert.Equal(t, "s2", sa.ImagePullSecrets[1].Name)
})
}
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetServices(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
services, err := kcl.GetServices("default")
require.NoError(t, err)
require.Empty(t, services)
}
+37
View File
@@ -0,0 +1,37 @@
package cli
import (
"context"
portainer "github.com/portainer/portainer/api"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (kcl *KubeClient) GetStorage() ([]portainer.KubernetesStorageClassConfig, error) {
var storages []portainer.KubernetesStorageClassConfig
storageClient := kcl.cli.StorageV1().StorageClasses()
storageList, err := storageClient.List(context.Background(), metav1.ListOptions{})
if err != nil {
return storages, err
}
for _, s := range storageList.Items {
var storage portainer.KubernetesStorageClassConfig
v, ok := s.Annotations["storageclass.kubernetes.io/is-default-class"]
if !ok || v != "true" {
continue
}
storage.Name = s.Name
storage.Provisioner = s.Provisioner
storage.AccessModes = []string{"RWO"}
if s.AllowVolumeExpansion != nil {
storage.AllowVolumeExpansion = *s.AllowVolumeExpansion
}
storages = append(storages, storage)
}
return storages, nil
}
+136
View File
@@ -0,0 +1,136 @@
package cli
import (
"context"
"fmt"
"github.com/segmentio/encoding/json"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const storageClassDefaultAnnotation = "storageclass.kubernetes.io/is-default-class"
// GetStorageClasses returns all StorageClasses in the cluster.
func (kcl *KubeClient) GetStorageClasses() ([]models.K8sStorageClass, error) {
scList, err := kcl.cli.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("unable to list storage classes. Error: %w", err)
}
results := make([]models.K8sStorageClass, 0, len(scList.Items))
for i := range scList.Items {
results = append(results, parseStorageClassDetail(&scList.Items[i]))
}
return results, nil
}
// GetStorageClass returns a single StorageClass by name.
func (kcl *KubeClient) GetStorageClass(name string) (*models.K8sStorageClass, error) {
sc, err := kcl.cli.StorageV1().StorageClasses().Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("unable to get storage class %s. Error: %w", name, err)
}
result := parseStorageClassDetail(sc)
return &result, nil
}
// DeleteStorageClasses deletes the specified StorageClasses by name.
func (kcl *KubeClient) DeleteStorageClasses(names []string) error {
for _, name := range names {
log.Debug().
Str("context", "DeleteStorageClasses").
Str("storage_class", name).
Msg("Deleting storage class")
err := kcl.cli.StorageV1().StorageClasses().Delete(context.Background(), name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("unable to delete storage class %s. Error: %w", name, err)
}
}
return nil
}
// SetDefaultStorageClass sets the specified StorageClass as the default.
// It removes the default annotation from any other StorageClass that is currently default.
func (kcl *KubeClient) SetDefaultStorageClass(name string) error {
_, err := kcl.cli.StorageV1().StorageClasses().Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("unable to find storage class %s. Error: %w", name, err)
}
scList, err := kcl.cli.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{})
if err != nil {
return fmt.Errorf("unable to list storage classes. Error: %w", err)
}
for i := range scList.Items {
sc := &scList.Items[i]
isCurrentDefault := sc.Annotations[storageClassDefaultAnnotation] == "true"
isTarget := sc.Name == name
if isTarget && !isCurrentDefault {
if err := kcl.patchStorageClassDefaultAnnotation(sc.Name, "true"); err != nil {
return err
}
} else if !isTarget && isCurrentDefault {
if err := kcl.patchStorageClassDefaultAnnotation(sc.Name, "false"); err != nil {
return err
}
}
}
return nil
}
func (kcl *KubeClient) patchStorageClassDefaultAnnotation(name, value string) error {
patch := map[string]any{
"metadata": map[string]any{
"annotations": map[string]string{
storageClassDefaultAnnotation: value,
},
},
}
patchBytes, err := json.Marshal(patch)
if err != nil {
return fmt.Errorf("unable to marshal patch for storage class %s. Error: %w", name, err)
}
_, err = kcl.cli.StorageV1().StorageClasses().Patch(
context.Background(),
name,
types.MergePatchType,
patchBytes,
metav1.PatchOptions{},
)
if err != nil {
return fmt.Errorf("unable to patch default annotation on storage class %s. Error: %w", name, err)
}
return nil
}
// parseStorageClassDetail parses a StorageClass with full detail including default status.
func parseStorageClassDetail(sc *storagev1.StorageClass) models.K8sStorageClass {
isDefault := sc.Annotations[storageClassDefaultAnnotation] == "true"
return models.K8sStorageClass{
Name: sc.Name,
Provisioner: sc.Provisioner,
ReclaimPolicy: sc.ReclaimPolicy,
AllowVolumeExpansion: sc.AllowVolumeExpansion,
IsDefault: isDefault,
Annotations: sc.Annotations,
Labels: sc.Labels,
CreationDate: sc.CreationTimestamp.Time,
Parameters: sc.Parameters,
MountOptions: sc.MountOptions,
}
}
+246
View File
@@ -0,0 +1,246 @@
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
)
func TestParseStorageClassDetail_IsDefault(t *testing.T) {
t.Parallel()
t.Run("annotation true sets isDefault", func(t *testing.T) {
t.Parallel()
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "fast",
Annotations: map[string]string{
storageClassDefaultAnnotation: "true",
},
},
}
result := parseStorageClassDetail(sc)
assert.True(t, result.IsDefault)
})
t.Run("annotation false gives isDefault=false", func(t *testing.T) {
t.Parallel()
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "slow",
Annotations: map[string]string{
storageClassDefaultAnnotation: "false",
},
},
}
result := parseStorageClassDetail(sc)
assert.False(t, result.IsDefault)
})
t.Run("missing annotation gives isDefault=false", func(t *testing.T) {
t.Parallel()
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: "no-annotation"},
}
result := parseStorageClassDetail(sc)
assert.False(t, result.IsDefault)
})
}
func TestSetDefaultStorageClass(t *testing.T) {
t.Parallel()
t.Run("sets default annotation on target SC", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: "standard"},
}
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.SetDefaultStorageClass("standard")
require.NoError(t, err)
result, err := k.GetStorageClass("standard")
require.NoError(t, err)
assert.True(t, result.IsDefault)
})
t.Run("removes default annotation from previously-default SC", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
oldDefault := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "old-default",
Annotations: map[string]string{
storageClassDefaultAnnotation: "true",
},
},
}
newDefault := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: "new-default"},
}
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), oldDefault, metav1.CreateOptions{})
require.NoError(t, err)
_, err = k.cli.StorageV1().StorageClasses().Create(t.Context(), newDefault, metav1.CreateOptions{})
require.NoError(t, err)
err = k.SetDefaultStorageClass("new-default")
require.NoError(t, err)
old, err := k.GetStorageClass("old-default")
require.NoError(t, err)
assert.False(t, old.IsDefault)
newSC, err := k.GetStorageClass("new-default")
require.NoError(t, err)
assert.True(t, newSC.IsDefault)
})
t.Run("no-op if target is already default", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "already-default",
Annotations: map[string]string{
storageClassDefaultAnnotation: "true",
},
},
}
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
err = k.SetDefaultStorageClass("already-default")
require.NoError(t, err)
result, err := k.GetStorageClass("already-default")
require.NoError(t, err)
assert.True(t, result.IsDefault)
})
t.Run("handles multiple SCs where only one was previously default", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
scs := []*storagev1.StorageClass{
{
ObjectMeta: metav1.ObjectMeta{
Name: "sc-default",
Annotations: map[string]string{
storageClassDefaultAnnotation: "true",
},
},
},
{ObjectMeta: metav1.ObjectMeta{Name: "sc-other-1"}},
{ObjectMeta: metav1.ObjectMeta{Name: "sc-other-2"}},
{ObjectMeta: metav1.ObjectMeta{Name: "sc-target"}},
}
for _, sc := range scs {
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
}
err := k.SetDefaultStorageClass("sc-target")
require.NoError(t, err)
target, err := k.GetStorageClass("sc-target")
require.NoError(t, err)
assert.True(t, target.IsDefault)
oldDefault, err := k.GetStorageClass("sc-default")
require.NoError(t, err)
assert.False(t, oldDefault.IsDefault)
for _, name := range []string{"sc-other-1", "sc-other-2"} {
sc, err := k.GetStorageClass(name)
require.NoError(t, err)
assert.False(t, sc.IsDefault)
}
})
}
func TestGetStorageClasses(t *testing.T) {
t.Parallel()
t.Run("empty cluster returns empty slice", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
scs, err := k.GetStorageClasses()
require.NoError(t, err)
assert.Empty(t, scs)
})
t.Run("returns all SCs with correct fields", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
allowExpansion := true
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "fast",
Annotations: map[string]string{
storageClassDefaultAnnotation: "true",
},
},
Provisioner: "kubernetes.io/aws-ebs",
AllowVolumeExpansion: &allowExpansion,
}
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
scs, err := k.GetStorageClasses()
require.NoError(t, err)
require.Len(t, scs, 1)
got := scs[0]
assert.Equal(t, "fast", got.Name)
assert.Equal(t, "kubernetes.io/aws-ebs", got.Provisioner)
assert.True(t, got.IsDefault)
require.NotNil(t, got.AllowVolumeExpansion)
assert.True(t, *got.AllowVolumeExpansion)
})
}
func TestDeleteStorageClasses(t *testing.T) {
t.Parallel()
t.Run("deletes specified SCs", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
for _, name := range []string{"sc-a", "sc-b", "sc-c"} {
sc := &storagev1.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: name},
}
_, err := k.cli.StorageV1().StorageClasses().Create(t.Context(), sc, metav1.CreateOptions{})
require.NoError(t, err)
}
err := k.DeleteStorageClasses([]string{"sc-a", "sc-c"})
require.NoError(t, err)
scs, err := k.GetStorageClasses()
require.NoError(t, err)
require.Len(t, scs, 1)
assert.Equal(t, "sc-b", scs[0].Name)
})
t.Run("returns error for non-existent SC", func(t *testing.T) {
t.Parallel()
k := NewTestKubeClient(kfake.NewClientset())
err := k.DeleteStorageClasses([]string{"does-not-exist"})
assert.Error(t, err)
})
}
+27
View File
@@ -0,0 +1,27 @@
package cli
import (
"strconv"
portainer "github.com/portainer/portainer/api"
"k8s.io/client-go/kubernetes"
)
// NewTestClientFactory creates a ClientFactory with a pre-seeded KubeClient for
// a specific endpoint ID. Intended for use in tests across packages that need to
// inject a fake Kubernetes client without a real cluster connection.
func NewTestClientFactory(endpointID portainer.EndpointID, kcl *KubeClient) *ClientFactory {
factory, _ := NewClientFactory(nil, nil, nil, "test", "", "")
factory.endpointProxyClients.Set(strconv.Itoa(int(endpointID)), kcl, 0)
return factory
}
// NewTestKubeClient creates a KubeClient backed by the provided kubernetes.Interface.
// Intended for use in tests.
func NewTestKubeClient(clientset kubernetes.Interface) *KubeClient {
return &KubeClient{
cli: clientset,
instanceID: "test",
isKubeAdmin: true,
}
}
+305
View File
@@ -0,0 +1,305 @@
package cli
import (
"context"
"fmt"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/rs/zerolog/log"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GetVolumes gets the volumes in the current k8s environment(endpoint).
// If the user is an admin, it fetches all the volumes in the cluster.
// If the user is not an admin, it fetches the volumes in the namespaces the user has access to.
// It returns a list of K8sVolumeInfo.
func (kcl *KubeClient) GetVolumes(namespace string) ([]models.K8sVolumeInfo, error) {
if kcl.GetIsKubeAdmin() {
return kcl.fetchVolumes(namespace)
}
return kcl.fetchVolumesForNonAdmin(namespace)
}
// GetVolume gets the volume with the given name and namespace.
func (kcl *KubeClient) GetVolume(namespace, volumeName string) (*models.K8sVolumeInfo, error) {
persistentVolumeClaim, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), volumeName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
persistentVolumesMap, storageClassesMap, err := kcl.fetchPersistentVolumesAndStorageClassesMap()
if err != nil {
return nil, err
}
volume := parseVolume(persistentVolumeClaim, persistentVolumesMap, storageClassesMap)
return &volume, nil
}
// fetchVolumesForNonAdmin fetches the volumes in the namespaces the user has access to.
// This function is called when the user is not an admin.
// It fetches all the persistent volume claims, persistent volumes and storage classes in the namespaces the user has access to.
func (kcl *KubeClient) fetchVolumesForNonAdmin(namespace string) ([]models.K8sVolumeInfo, error) {
nonAdminNamespaces := kcl.GetClientNonAdminNamespaces()
log.Debug().
Strs("non_admin_namespaces", nonAdminNamespaces).
Msg("fetching volumes for non-admin user")
if len(nonAdminNamespaces) == 0 {
return nil, nil
}
volumes, err := kcl.fetchVolumes(namespace)
if err != nil {
return nil, err
}
nonAdminNamespaceSet := kcl.buildNonAdminNamespacesMap()
results := make([]models.K8sVolumeInfo, 0)
for _, volume := range volumes {
if _, ok := nonAdminNamespaceSet[volume.PersistentVolumeClaim.Namespace]; ok {
results = append(results, volume)
}
}
return results, nil
}
// fetchVolumes fetches all the persistent volume claims, persistent volumes and storage classes in the given namespace.
// It returns a list of K8sVolumeInfo.
// This function is called by fetchVolumesForAdmin and fetchVolumesForNonAdmin.
func (kcl *KubeClient) fetchVolumes(namespace string) ([]models.K8sVolumeInfo, error) {
volumes := make([]models.K8sVolumeInfo, 0)
persistentVolumeClaims, err := kcl.cli.CoreV1().PersistentVolumeClaims(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
if len(persistentVolumeClaims.Items) > 0 {
persistentVolumesMap, storageClassesMap, err := kcl.fetchPersistentVolumesAndStorageClassesMap()
if err != nil {
return nil, err
}
for _, persistentVolumeClaim := range persistentVolumeClaims.Items {
volumes = append(volumes, parseVolume(&persistentVolumeClaim, persistentVolumesMap, storageClassesMap))
}
}
return volumes, nil
}
// parseVolume parses the given persistent volume claim and returns a K8sVolumeInfo.
// This function is called by fetchVolumes.
// It returns a K8sVolumeInfo.
func parseVolume(persistentVolumeClaim *corev1.PersistentVolumeClaim, persistentVolumesMap map[string]models.K8sPersistentVolume, storageClassesMap map[string]models.K8sStorageClass) models.K8sVolumeInfo {
volume := models.K8sVolumeInfo{}
volumeClaim := parsePersistentVolumeClaim(persistentVolumeClaim)
if volumeClaim.VolumeName != "" {
persistentVolume, ok := persistentVolumesMap[volumeClaim.VolumeName]
if ok {
volume.PersistentVolume = persistentVolume
}
}
if volumeClaim.StorageClass != nil {
storageClass, ok := storageClassesMap[*volumeClaim.StorageClass]
if ok {
volume.StorageClass = storageClass
}
}
volume.PersistentVolumeClaim = volumeClaim
return volume
}
// parsePersistentVolumeClaim parses the given persistent volume claim and returns a K8sPersistentVolumeClaim.
func parsePersistentVolumeClaim(volume *corev1.PersistentVolumeClaim) models.K8sPersistentVolumeClaim {
storage := volume.Spec.Resources.Requests[corev1.ResourceStorage]
return models.K8sPersistentVolumeClaim{
ID: string(volume.UID),
Name: volume.Name,
Namespace: volume.Namespace,
CreationDate: volume.CreationTimestamp.Time,
Storage: storage.Value(),
StorageRequest: storage.String(),
AccessModes: humanReadableAccessModes(volume.Spec.AccessModes),
HumanReadableAccessModes: volume.Spec.AccessModes,
VolumeName: volume.Spec.VolumeName,
ResourcesRequests: &volume.Spec.Resources.Requests,
StorageClass: volume.Spec.StorageClassName,
VolumeMode: volume.Spec.VolumeMode,
OwningApplications: nil,
Phase: volume.Status.Phase,
Labels: volume.Labels,
}
}
// parsePersistentVolume parses the given persistent volume and returns a K8sPersistentVolume.
func parsePersistentVolume(volume *corev1.PersistentVolume) models.K8sPersistentVolume {
return models.K8sPersistentVolume{
Name: volume.Name,
Annotations: volume.Annotations,
Labels: volume.Labels,
AccessModes: humanReadableAccessModes(volume.Spec.AccessModes),
HumanReadableAccessModes: volume.Spec.AccessModes,
Capacity: volume.Spec.Capacity,
ClaimRef: volume.Spec.ClaimRef,
StorageClassName: volume.Spec.StorageClassName,
PersistentVolumeReclaimPolicy: volume.Spec.PersistentVolumeReclaimPolicy,
VolumeMode: volume.Spec.VolumeMode,
CSI: volume.Spec.CSI,
Status: volume.Status.Phase,
CreationDate: volume.CreationTimestamp.Time,
}
}
// buildPersistentVolumesMap builds a map of persistent volumes.
func (kcl *KubeClient) buildPersistentVolumesMap(persistentVolumes *corev1.PersistentVolumeList) map[string]models.K8sPersistentVolume {
persistentVolumesMap := make(map[string]models.K8sPersistentVolume)
for _, persistentVolume := range persistentVolumes.Items {
persistentVolumesMap[persistentVolume.Name] = parsePersistentVolume(&persistentVolume)
}
return persistentVolumesMap
}
// parseStorageClass parses the given storage class and returns a K8sStorageClass.
func parseStorageClass(storageClass *storagev1.StorageClass) models.K8sStorageClass {
return models.K8sStorageClass{
Name: storageClass.Name,
Provisioner: storageClass.Provisioner,
ReclaimPolicy: storageClass.ReclaimPolicy,
AllowVolumeExpansion: storageClass.AllowVolumeExpansion,
}
}
// buildStorageClassesMap builds a map of storage classes.
func (kcl *KubeClient) buildStorageClassesMap(storageClasses *storagev1.StorageClassList) map[string]models.K8sStorageClass {
storageClassesMap := make(map[string]models.K8sStorageClass)
for _, storageClass := range storageClasses.Items {
storageClassesMap[storageClass.Name] = parseStorageClass(&storageClass)
}
return storageClassesMap
}
// fetchPersistentVolumesAndStorageClassesMap fetches all the persistent volumes and storage classes in the cluster.
// It returns a map of persistent volumes and a map of storage classes.
func (kcl *KubeClient) fetchPersistentVolumesAndStorageClassesMap() (map[string]models.K8sPersistentVolume, map[string]models.K8sStorageClass, error) {
persistentVolumes, err := kcl.cli.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, nil, err
}
persistentVolumesMap := kcl.buildPersistentVolumesMap(persistentVolumes)
storageClasses, err := kcl.cli.StorageV1().StorageClasses().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, nil, err
}
storageClassesMap := kcl.buildStorageClassesMap(storageClasses)
return persistentVolumesMap, storageClassesMap, nil
}
// CombineVolumesWithApplications combines the volumes with the applications that use them.
func (kcl *KubeClient) CombineVolumesWithApplications(volumes *[]models.K8sVolumeInfo) (*[]models.K8sVolumeInfo, error) {
pods, err := kcl.cli.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return volumes, nil
}
log.Error().Err(err).Msg("Failed to list pods across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineServicesWithApplications operation, unable to list pods across the cluster. Error: %w", err)
}
hasReplicaSetOwnerReference := containsReplicaSetOwnerReference(pods)
replicaSetItems := make([]appsv1.ReplicaSet, 0)
deploymentItems := make([]appsv1.Deployment, 0)
if hasReplicaSetOwnerReference {
replicaSets, err := kcl.cli.AppsV1().ReplicaSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list replica sets across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list replica sets across the cluster. Error: %w", err)
}
replicaSetItems = replicaSets.Items
deployments, err := kcl.cli.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list deployments across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list deployments across the cluster. Error: %w", err)
}
deploymentItems = deployments.Items
}
hasStatefulSetOwnerReference := containsStatefulSetOwnerReference(pods)
statefulSetItems := make([]appsv1.StatefulSet, 0)
if hasStatefulSetOwnerReference {
statefulSets, err := kcl.cli.AppsV1().StatefulSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list stateful sets across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list stateful sets across the cluster. Error: %w", err)
}
statefulSetItems = statefulSets.Items
}
hasDaemonSetOwnerReference := containsDaemonSetOwnerReference(pods)
daemonSetItems := make([]appsv1.DaemonSet, 0)
if hasDaemonSetOwnerReference {
daemonSets, err := kcl.cli.AppsV1().DaemonSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list daemon sets across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list daemon sets across the cluster. Error: %w", err)
}
daemonSetItems = daemonSets.Items
}
return kcl.updateVolumesWithOwningApplications(volumes, pods, deploymentItems, replicaSetItems, statefulSetItems, daemonSetItems)
}
// updateVolumesWithOwningApplications updates the volumes with the applications that use them.
func (kcl *KubeClient) updateVolumesWithOwningApplications(volumes *[]models.K8sVolumeInfo, pods *corev1.PodList, deploymentItems []appsv1.Deployment, replicaSetItems []appsv1.ReplicaSet, statefulSetItems []appsv1.StatefulSet, daemonSetItems []appsv1.DaemonSet) (*[]models.K8sVolumeInfo, error) {
for i, volume := range *volumes {
for _, pod := range pods.Items {
if pod.Spec.Volumes != nil {
for _, podVolume := range pod.Spec.Volumes {
if podVolume.PersistentVolumeClaim != nil && podVolume.PersistentVolumeClaim.ClaimName == volume.PersistentVolumeClaim.Name && pod.Namespace == volume.PersistentVolumeClaim.Namespace {
application, err := kcl.ConvertPodToApplication(pod, PortainerApplicationResources{
ReplicaSets: replicaSetItems,
Deployments: deploymentItems,
StatefulSets: statefulSetItems,
DaemonSets: daemonSetItems,
}, false)
if err != nil {
log.Error().Err(err).Msg("Failed to convert pod to application")
return nil, fmt.Errorf("an error occurred during the CombineServicesWithApplications operation, unable to convert pod to application. Error: %w", err)
}
// Check if the application already exists in the OwningApplications slice
exists := false
for _, existingApp := range (*volumes)[i].PersistentVolumeClaim.OwningApplications {
if existingApp.Name == application.Name && existingApp.Namespace == application.Namespace {
exists = true
break
}
}
if !exists && application != nil {
(*volumes)[i].PersistentVolumeClaim.OwningApplications = append((*volumes)[i].PersistentVolumeClaim.OwningApplications, *application)
}
}
}
}
}
}
return volumes, nil
}
+16
View File
@@ -0,0 +1,16 @@
package cli
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetVolumes(t *testing.T) {
t.Parallel()
kcl := &KubeClient{}
volumes, err := kcl.GetVolumes("default")
require.NoError(t, err)
require.Empty(t, volumes)
}