chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
@@ -0,0 +1,27 @@
package endpoints
/// This feature is implemented in the agent API and not directly here.
/// However, it's proxied. So we document it here.
// @summary Upload a file under a specific path on the file system of an environment (endpoint)
// @description Use this environment(endpoint) to upload TLS files.
// @description **Access policy**: administrator
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @accept multipart/form-data
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @param volumeID query string false "Optional volume identifier to upload the file"
// @param Path formData string true "The destination path to upload the file to"
// @param file formData file true "The file to upload"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /endpoints/{id}/docker/v2/browse/put [post]
//
//lint:ignore U1000 Ignore unused code, for documentation purposes
func _fileBrowseFileUploadV2() {
// dummy function to make swag pick up the above docs for the following REST call
// POST request on /browse/put?volumeID=:id
}
@@ -0,0 +1,50 @@
package endpoints
import (
"net/http"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/set"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id AgentVersions
// @summary List agent versions
// @description List all agent versions based on the current user authorizations and query parameters.
// @description **Access policy**: restricted
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @produce json
// @success 200 {array} string "List of available agent versions"
// @failure 500 "Server error"
// @router /endpoints/agent_versions [get]
func (handler *Handler) agentVersions(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointGroups, err := handler.DataStore.EndpointGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment groups from the database", err)
}
endpoints, err := handler.DataStore.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext)
agentVersions := set.Set[string]{}
for _, endpoint := range filteredEndpoints {
if endpoint.Agent.Version != "" {
agentVersions[endpoint.Agent.Version] = true
}
}
return response.JSON(w, agentVersions.Keys())
}
@@ -0,0 +1,87 @@
package endpoints
import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"regexp"
"strings"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EndpointAssociationDelete
// @summary De-association an edge environment(endpoint)
// @description De-association an edge environment(endpoint).
// @description **Access policy**: administrator
// @security ApiKeyAuth
// @security jwt
// @tags endpoints
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 404 "Environment(Endpoint) not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/association [put]
func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
if endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment && endpoint.Type != portainer.EdgeAgentOnDockerEnvironment {
return httperror.BadRequest("Invalid environment type", errors.New("Invalid environment type"))
}
endpoint.EdgeID = ""
endpoint.Snapshots = []portainer.DockerSnapshot{}
endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{}
endpoint.EdgeKey, err = handler.updateEdgeKey(endpoint.EdgeKey)
if err != nil {
return httperror.InternalServerError("Invalid EdgeKey", err)
}
err = handler.DataStore.Endpoint().UpdateEndpoint(portainer.EndpointID(endpointID), endpoint)
if err != nil {
return httperror.InternalServerError("Failed persisting environment in database", err)
}
return response.Empty(w)
}
func (handler *Handler) updateEdgeKey(edgeKey string) (string, error) {
oldEdgeKeyByte, err := base64.RawStdEncoding.DecodeString(edgeKey)
if err != nil {
return "", err
}
oldEdgeKeyStr := string(oldEdgeKeyByte)
httpPort := getPort(handler.BindAddress)
httpsPort := getPort(handler.BindAddressHTTPS)
// replace "http://" with "https://" and replace ":9000" with ":9443", in the case of default values
// oldEdgeKeyStr example: http://10.116.1.178:9000|10.116.1.178:8000|46:99:4a:8d:a6:de:6a:bd:d8:e2:1c:99:81:60:54:55|52
r := regexp.MustCompile(fmt.Sprintf("^(http://)([^|]+)(:%s)(|.*)", httpPort))
newEdgeKeyStr := r.ReplaceAllString(oldEdgeKeyStr, fmt.Sprintf("https://$2:%s$4", httpsPort))
return base64.RawStdEncoding.EncodeToString([]byte(newEdgeKeyStr)), nil
}
func getPort(url string) string {
items := strings.Split(url, ":")
return items[len(items)-1]
}
@@ -0,0 +1,598 @@
package endpoints
import (
"errors"
"net/http"
"runtime"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/agent"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/client"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
)
type endpointCreatePayload struct {
Name string
URL string
EndpointCreationType endpointCreationEnum
PublicURL string
Gpus []portainer.Pair
GroupID int
TLS bool
TLSSkipVerify bool
TLSSkipClientVerify bool
TLSCACertFile []byte
TLSCertFile []byte
TLSKeyFile []byte
AzureApplicationID string
AzureTenantID string
AzureAuthenticationKey string
TagIDs []portainer.TagID
EdgeCheckinInterval int
ContainerEngine string
}
type endpointCreationEnum int
const (
_ endpointCreationEnum = iota
localDockerEnvironment
agentEnvironment
azureEnvironment
edgeAgentEnvironment
localKubernetesEnvironment
)
func (payload *endpointCreatePayload) Validate(r *http.Request) error {
name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
if err != nil {
return errors.New("invalid environment name")
}
payload.Name = name
endpointCreationType, err := request.RetrieveNumericMultiPartFormValue(r, "EndpointCreationType", false)
if err != nil || endpointCreationType == 0 {
return errors.New("invalid environment type value. Value must be one of: 1 (Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge Agent environment) or 5 (Local Kubernetes environment)")
}
payload.EndpointCreationType = endpointCreationEnum(endpointCreationType)
payload.ContainerEngine, err = request.RetrieveMultiPartFormValue(r, "ContainerEngine", true)
if err != nil || (payload.ContainerEngine != "" && payload.ContainerEngine != portainer.ContainerEngineDocker && payload.ContainerEngine != portainer.ContainerEnginePodman) {
return errors.New("invalid container engine value. Value must be one of: 'docker' or 'podman'")
}
groupID, _ := request.RetrieveNumericMultiPartFormValue(r, "GroupID", true)
if groupID == 0 {
groupID = 1
}
payload.GroupID = groupID
var tagIDs []portainer.TagID
if err := request.RetrieveMultiPartFormJSONValue(r, "TagIds", &tagIDs, true); err != nil {
return errors.New("invalid TagIds parameter")
}
payload.TagIDs = tagIDs
if payload.TagIDs == nil {
payload.TagIDs = make([]portainer.TagID, 0)
}
useTLS, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLS", true)
payload.TLS = useTLS
if payload.TLS && payload.EndpointCreationType == edgeAgentEnvironment {
return errors.New("TLS is not supported for Edge Agent environments")
}
if payload.TLS {
skipTLSServerVerification, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLSSkipVerify", true)
payload.TLSSkipVerify = skipTLSServerVerification
skipTLSClientVerification, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLSSkipClientVerify", true)
payload.TLSSkipClientVerify = skipTLSClientVerification
if !payload.TLSSkipVerify {
caCert, _, err := request.RetrieveMultiPartFormFile(r, "TLSCACertFile")
if err != nil {
return errors.New("invalid CA certificate file. Ensure that the file is uploaded correctly")
}
payload.TLSCACertFile = caCert
}
if !payload.TLSSkipClientVerify {
cert, _, err := request.RetrieveMultiPartFormFile(r, "TLSCertFile")
if err != nil {
return errors.New("invalid certificate file. Ensure that the file is uploaded correctly")
}
payload.TLSCertFile = cert
key, _, err := request.RetrieveMultiPartFormFile(r, "TLSKeyFile")
if err != nil {
return errors.New("invalid key file. Ensure that the file is uploaded correctly")
}
payload.TLSKeyFile = key
}
}
switch payload.EndpointCreationType {
case azureEnvironment:
azureApplicationID, err := request.RetrieveMultiPartFormValue(r, "AzureApplicationID", false)
if err != nil {
return errors.New("invalid Azure application ID")
}
payload.AzureApplicationID = azureApplicationID
azureTenantID, err := request.RetrieveMultiPartFormValue(r, "AzureTenantID", false)
if err != nil {
return errors.New("invalid Azure tenant ID")
}
payload.AzureTenantID = azureTenantID
azureAuthenticationKey, err := request.RetrieveMultiPartFormValue(r, "AzureAuthenticationKey", false)
if err != nil {
return errors.New("invalid Azure authentication key")
}
payload.AzureAuthenticationKey = azureAuthenticationKey
case edgeAgentEnvironment:
endpointURL, err := request.RetrieveMultiPartFormValue(r, "URL", false)
if err != nil || strings.EqualFold("", strings.Trim(endpointURL, " ")) {
return errors.New("URL cannot be empty")
}
payload.URL = endpointURL
publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true)
payload.PublicURL = publicURL
default:
endpointURL, err := request.RetrieveMultiPartFormValue(r, "URL", true)
if err != nil {
return errors.New("invalid environment URL")
}
payload.URL = endpointURL
publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true)
payload.PublicURL = publicURL
}
gpus := make([]portainer.Pair, 0)
if err := request.RetrieveMultiPartFormJSONValue(r, "Gpus", &gpus, true); err != nil {
return errors.New("invalid Gpus parameter")
}
payload.Gpus = gpus
edgeCheckinInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "EdgeCheckinInterval", true)
if edgeCheckinInterval == 0 {
// deprecated CheckinInterval
edgeCheckinInterval, _ = request.RetrieveNumericMultiPartFormValue(r, "CheckinInterval", true)
}
payload.EdgeCheckinInterval = edgeCheckinInterval
return nil
}
// @id EndpointCreate
// @summary Create a new environment(endpoint)
// @description Create a new environment(endpoint) that will be used to manage an environment(endpoint).
// @description **Access policy**: administrator
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @accept multipart/form-data
// @produce json
// @param Name formData string true "Name that will be used to identify this environment(endpoint) (example: my-environment)"
// @param EndpointCreationType formData endpointCreationEnum true "Environment(Endpoint) type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment)" Enum(1,2,3,4,5)
// @param ContainerEngine formData string false "Container engine used by the environment(endpoint). Value must be one of: 'docker' or 'podman'"
// @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine). Cannot be empty if EndpointCreationType is set to 4 (Edge agent environment)"
// @param PublicURL formData string false "URL or IP address where exposed containers will be reachable. Defaults to URL if not specified (example: docker.mydomain.tld:2375)"
// @param GroupID formData int false "Environment(Endpoint) group identifier. If not specified will default to 1 (unassigned)."
// @param TLS formData bool false "Require TLS to connect against this environment(endpoint). Must be true if EndpointCreationType is set to 2 (Agent environment)"
// @param TLSSkipVerify formData bool false "Skip server verification when using TLS. Must be true if EndpointCreationType is set to 2 (Agent environment)"
// @param TLSSkipClientVerify formData bool false "Skip client verification when using TLS. Must be true if EndpointCreationType is set to 2 (Agent environment)"
// @param TLSCACertFile formData file false "TLS CA certificate file"
// @param TLSCertFile formData file false "TLS client certificate file"
// @param TLSKeyFile formData file false "TLS client key file"
// @param AzureApplicationID formData string false "Azure application ID. Required if environment(endpoint) type is set to 3"
// @param AzureTenantID formData string false "Azure tenant ID. Required if environment(endpoint) type is set to 3"
// @param AzureAuthenticationKey formData string false "Azure authentication key. Required if environment(endpoint) type is set to 3"
// @param TagIds formData string false "JSON-parsable array of tag identifiers to which this environment(endpoint) is associated"
// @param EdgeCheckinInterval formData int false "The check in interval for edge agent (in seconds)"
// @param EdgeTunnelServerAddress formData string false "URL or IP address that will be used to establish a reverse tunnel"
// @param Gpus formData string false "List of GPUs - json stringified array of {name, value} structs"
// @success 200 {object} portainer.Endpoint "Success"
// @failure 400 "Invalid request"
// @failure 409 "Name is not unique"
// @failure 500 "Server error"
// @router /endpoints [post]
func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
payload := &endpointCreatePayload{}
if err := payload.Validate(r); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
isUnique, err := handler.isNameUnique(payload.Name, 0)
if err != nil {
return httperror.InternalServerError("Unable to check if name is unique", err)
}
if !isUnique {
return httperror.Conflict("Name is not unique", nil)
}
endpoint, endpointCreationError := handler.createEndpoint(handler.DataStore, payload)
if endpointCreationError != nil {
return endpointCreationError
}
endpointGroup, err := handler.DataStore.EndpointGroup().Read(endpoint.GroupID)
if err != nil {
return httperror.InternalServerError("Unable to find an environment group inside the database", err)
}
edgeGroups, err := handler.DataStore.EdgeGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve edge groups from the database", err)
}
edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
if err != nil {
return httperror.InternalServerError("Unable to retrieve edge stacks from the database", err)
}
relationObject := &portainer.EndpointRelation{
EndpointID: endpoint.ID,
EdgeStacks: map[portainer.EdgeStackID]bool{},
}
if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
relatedEdgeStacks := edge.EndpointRelatedEdgeStacks(endpoint, endpointGroup, edgeGroups, edgeStacks)
for _, stackID := range relatedEdgeStacks {
relationObject.EdgeStacks[stackID] = true
}
} else if endpointutils.IsKubernetesEndpoint(endpoint) {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
endpointutils.InitialIngressClassDetection(tx, endpoint, handler.K8sClientFactory)
endpointutils.InitialMetricsDetection(tx, endpoint, handler.K8sClientFactory)
endpointutils.InitialStorageDetection(tx, handler.DataStore, endpoint, handler.K8sClientFactory)
return nil
}); err != nil {
log.Err(err).Msg("failed to persist initial kube detection")
}
}
if err := handler.DataStore.EndpointRelation().Create(relationObject); err != nil {
return httperror.InternalServerError("Unable to persist the relation object inside the database", err)
}
return response.JSON(w, endpoint)
}
func (handler *Handler) createEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
switch payload.EndpointCreationType {
case azureEnvironment:
return handler.createAzureEndpoint(tx, payload)
case edgeAgentEnvironment:
return handler.createEdgeAgentEndpoint(tx, payload)
case localKubernetesEnvironment:
return handler.createKubernetesEndpoint(tx, payload)
}
endpointType := portainer.DockerEnvironment
var agentVersion string
if payload.EndpointCreationType == agentEnvironment {
tlsConfig, err := crypto.CreateTLSConfigurationFromBytes(payload.TLS, payload.TLSCACertFile, payload.TLSCertFile, payload.TLSKeyFile, payload.TLSSkipVerify, payload.TLSSkipClientVerify)
if err != nil {
return nil, httperror.InternalServerError("Unable to create TLS configuration", err)
}
agentPlatform, version, err := agent.GetAgentVersionAndPlatform(payload.URL, tlsConfig)
if err != nil {
return nil, httperror.InternalServerError("Unable to get environment type", err)
}
agentVersion = version
if agentPlatform == portainer.AgentPlatformDocker {
endpointType = portainer.AgentOnDockerEnvironment
} else if agentPlatform == portainer.AgentPlatformKubernetes {
endpointType = portainer.AgentOnKubernetesEnvironment
payload.URL = strings.TrimPrefix(payload.URL, "tcp://")
}
}
if payload.TLS {
return handler.createTLSSecuredEndpoint(tx, payload, endpointType, agentVersion)
}
return handler.createUnsecuredEndpoint(tx, payload)
}
func (handler *Handler) createAzureEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
credentials := portainer.AzureCredentials{
ApplicationID: payload.AzureApplicationID,
TenantID: payload.AzureTenantID,
AuthenticationKey: payload.AzureAuthenticationKey,
}
httpClient := client.NewHTTPClient()
if _, err := httpClient.ExecuteAzureAuthenticationRequest(&credentials); err != nil {
return nil, httperror.InternalServerError("Unable to authenticate against Azure", err)
}
endpointID := tx.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: "https://management.azure.com",
Type: portainer.AzureEnvironment,
GroupID: portainer.EndpointGroupID(payload.GroupID),
PublicURL: payload.PublicURL,
Gpus: payload.Gpus,
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
AzureCredentials: credentials,
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
}
if err := handler.saveEndpointAndUpdateAuthorizations(tx, endpoint); err != nil {
return nil, httperror.InternalServerError("An error occurred while trying to create the environment", err)
}
return endpoint, nil
}
func (handler *Handler) createEdgeAgentEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
portainerHost, err := edge.ParseHostForEdge(payload.URL)
if err != nil {
return nil, httperror.BadRequest("Unable to parse host", err)
}
edgeKey := handler.ReverseTunnelService.GenerateEdgeKey(payload.URL, portainerHost, endpointID)
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: portainerHost,
Type: func() portainer.EndpointType {
// an empty container engine means that the endpoint is a Kubernetes endpoint
if payload.ContainerEngine == "" {
return portainer.EdgeAgentOnKubernetesEnvironment
}
return portainer.EdgeAgentOnDockerEnvironment
}(),
ContainerEngine: payload.ContainerEngine,
GroupID: portainer.EndpointGroupID(payload.GroupID),
Gpus: payload.Gpus,
TLSConfig: portainer.TLSConfiguration{
TLS: false,
},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
EdgeKey: edgeKey,
EdgeCheckinInterval: payload.EdgeCheckinInterval,
Kubernetes: portainer.KubernetesDefault(),
UserTrusted: true,
}
settings, err := handler.DataStore.Settings().Settings()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve the settings from the database", err)
}
if settings.EnforceEdgeID {
edgeID, err := uuid.NewRandom()
if err != nil {
return nil, httperror.InternalServerError("Cannot generate the Edge ID", err)
}
endpoint.EdgeID = edgeID.String()
}
if err := handler.saveEndpointAndUpdateAuthorizations(tx, endpoint); err != nil {
return nil, httperror.InternalServerError("An error occurred while trying to create the environment", err)
}
return endpoint, nil
}
func (handler *Handler) createUnsecuredEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
endpointType := portainer.DockerEnvironment
if payload.URL == "" {
payload.URL = "unix:///var/run/docker.sock"
if runtime.GOOS == "windows" {
payload.URL = "npipe:////./pipe/docker_engine"
}
}
endpointID := tx.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: payload.URL,
Type: endpointType,
ContainerEngine: payload.ContainerEngine,
GroupID: portainer.EndpointGroupID(payload.GroupID),
PublicURL: payload.PublicURL,
Gpus: payload.Gpus,
TLSConfig: portainer.TLSConfiguration{
TLS: false,
},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
}
if err := handler.snapshotAndPersistEndpoint(tx, endpoint); err != nil {
return nil, err
}
return endpoint, nil
}
func (handler *Handler) createKubernetesEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
if payload.URL == "" {
payload.URL = "https://kubernetes.default.svc"
}
endpointID := tx.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: payload.URL,
Type: portainer.KubernetesLocalEnvironment,
GroupID: portainer.EndpointGroupID(payload.GroupID),
PublicURL: payload.PublicURL,
Gpus: payload.Gpus,
TLSConfig: portainer.TLSConfiguration{
TLS: payload.TLS,
TLSSkipVerify: payload.TLSSkipVerify,
},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
}
if err := handler.snapshotAndPersistEndpoint(tx, endpoint); err != nil {
return nil, err
}
return endpoint, nil
}
func (handler *Handler) createTLSSecuredEndpoint(tx dataservices.DataStoreTx, payload *endpointCreatePayload, endpointType portainer.EndpointType, agentVersion string) (*portainer.Endpoint, *httperror.HandlerError) {
endpointID := tx.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: payload.URL,
Type: endpointType,
ContainerEngine: payload.ContainerEngine,
GroupID: portainer.EndpointGroupID(payload.GroupID),
PublicURL: payload.PublicURL,
Gpus: payload.Gpus,
TLSConfig: portainer.TLSConfiguration{
TLS: payload.TLS,
TLSSkipVerify: payload.TLSSkipVerify,
},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
}
endpoint.Agent.Version = agentVersion
if err := handler.storeTLSFiles(endpoint, payload); err != nil {
return nil, err
}
if err := handler.snapshotAndPersistEndpoint(tx, endpoint); err != nil {
return nil, err
}
return endpoint, nil
}
func (handler *Handler) snapshotAndPersistEndpoint(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) *httperror.HandlerError {
if err := handler.SnapshotService.SnapshotEndpoint(endpoint); err != nil {
if (endpoint.Type == portainer.AgentOnDockerEnvironment && strings.Contains(err.Error(), "Invalid request signature")) ||
(endpoint.Type == portainer.AgentOnKubernetesEnvironment && strings.Contains(err.Error(), "unknown")) {
err = errors.New("agent already paired with another Portainer instance")
}
return httperror.InternalServerError("Unable to initiate communications with environment", err)
}
if err := handler.saveEndpointAndUpdateAuthorizations(tx, endpoint); err != nil {
return httperror.InternalServerError("An error occurred while trying to create the environment", err)
}
return nil
}
func (handler *Handler) saveEndpointAndUpdateAuthorizations(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) error {
endpoint.SecuritySettings = portainer.DefaultEndpointSecuritySettings()
if err := tx.Endpoint().Create(endpoint); err != nil {
return err
}
if err := endpointutils.InitializeEdgeEndpointRelation(endpoint, tx); err != nil {
return err
}
for _, tagID := range endpoint.TagIDs {
if err := tx.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) {
tag.Endpoints[endpoint.ID] = true
}); err != nil {
return err
}
}
return nil
}
func (handler *Handler) storeTLSFiles(endpoint *portainer.Endpoint, payload *endpointCreatePayload) *httperror.HandlerError {
folder := strconv.Itoa(int(endpoint.ID))
if !payload.TLSSkipVerify {
caCertPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCA, payload.TLSCACertFile)
if err != nil {
return httperror.InternalServerError("Unable to persist TLS CA certificate file on disk", err)
}
endpoint.TLSConfig.TLSCACertPath = caCertPath
}
if payload.TLSSkipClientVerify {
return nil
}
certPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCert, payload.TLSCertFile)
if err != nil {
return httperror.InternalServerError("Unable to persist TLS certificate file on disk", err)
}
endpoint.TLSConfig.TLSCertPath = certPath
keyPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileKey, payload.TLSKeyFile)
if err != nil {
return httperror.InternalServerError("Unable to persist TLS key file on disk", err)
}
endpoint.TLSConfig.TLSKeyPath = keyPath
return nil
}
@@ -0,0 +1,37 @@
package endpoints
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type endpointCreateGlobalKeyResponse struct {
EndpointID portainer.EndpointID `json:"endpointID"`
}
// @id EndpointCreateGlobalKey
// @summary Create or retrieve the endpoint for an EdgeID
// @tags endpoints
// @success 200 {object} endpointCreateGlobalKeyResponse "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /endpoints/global-key [post]
func (handler *Handler) endpointCreateGlobalKey(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeID := r.Header.Get(portainer.PortainerAgentEdgeIDHeader)
if edgeID == "" {
return httperror.BadRequest("Invalid Edge ID", errors.New("the Edge ID cannot be empty"))
}
// Search for existing endpoints for the given edgeID
endpointID, ok := handler.DataStore.Endpoint().EndpointIDByEdgeID(edgeID)
if ok {
return response.JSON(w, endpointCreateGlobalKeyResponse{endpointID})
}
return httperror.NotFound("Unable to find the endpoint in the database", nil)
}
@@ -0,0 +1,29 @@
package endpoints
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/testhelpers"
)
func TestEmptyGlobalKey(t *testing.T) {
t.Parallel()
handler := NewHandler(testhelpers.NewTestRequestBouncer())
req, err := http.NewRequest(http.MethodPost, "https://portainer.io:9443/endpoints/global-key", nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, "")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatal("expected a 400 response, found:", rec.Code)
}
}
@@ -0,0 +1,230 @@
package endpoints
import (
"net/http"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/chisel"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// EE-only kubeconfig validation tests removed for CE
func TestSaveEndpointAndUpdateAuthorizations(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
endpointGroup := &portainer.EndpointGroup{
ID: 1,
Name: "test-endpoint-group",
}
err := store.EndpointGroup().Create(endpointGroup)
require.NoError(t, err)
h := &Handler{
DataStore: store,
}
testCases := []struct {
name string
endpointType portainer.EndpointType
expectRelation bool
}{
{
name: "create azure environment, expect no relation to be created",
endpointType: portainer.AzureEnvironment,
expectRelation: false,
},
{
name: "create edge agent environment, expect relation to be created",
endpointType: portainer.EdgeAgentOnDockerEnvironment,
expectRelation: true,
},
{
name: "create kubernetes environment, expect no relation to be created",
endpointType: portainer.KubernetesLocalEnvironment,
expectRelation: false,
},
{
name: "create kubeconfig environment, expect no relation to be created",
endpointType: portainer.AgentOnKubernetesEnvironment,
expectRelation: false,
},
{
name: "create agent docker environment, expect no relation to be created",
endpointType: portainer.AgentOnDockerEnvironment,
expectRelation: false,
},
{
name: "create unsecured environment, expect no relation to be created",
endpointType: portainer.DockerEnvironment,
expectRelation: false,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(store.Endpoint().GetNextIdentifier()),
Type: testCase.endpointType,
GroupID: endpointGroup.ID,
}
err := h.saveEndpointAndUpdateAuthorizations(store, endpoint)
require.NoError(t, err)
relation, relationErr := store.EndpointRelation().EndpointRelation(endpoint.ID)
if testCase.expectRelation {
require.NoError(t, relationErr)
require.NotNil(t, relation)
} else {
require.Error(t, relationErr)
require.True(t, store.IsErrObjectNotFound(relationErr))
require.Nil(t, relation)
}
})
}
}
func TestCreateEndpointFailure(t *testing.T) {
t.Parallel()
fips.InitFIPS(false)
_, store := datastore.MustNewTestStore(t, true, false)
h := NewHandler(testhelpers.NewTestRequestBouncer())
h.DataStore = store
payload := &endpointCreatePayload{
Name: "Test Endpoint",
EndpointCreationType: agentEnvironment,
TLS: true,
TLSCertFile: []byte("invalid data"),
TLSKeyFile: []byte("invalid data"),
}
endpoint, httpErr := h.createEndpoint(store, payload)
require.NotNil(t, httpErr)
require.Equal(t, http.StatusInternalServerError, httpErr.StatusCode)
require.Nil(t, endpoint)
}
func TestValidateEndpointCreatePayload_TLS(t *testing.T) {
t.Parallel()
fips.InitFIPS(false)
testCases := []struct {
name string
formValues map[string][]string
expectedError string
}{
{
name: "edge agent env with TLS rejected",
formValues: map[string][]string{
"Name": {"Test Endpoint"},
"EndpointCreationType": {"4"},
"TLS": {"true"},
},
expectedError: "TLS is not supported for Edge Agent environments",
},
{
name: "edge agent env without TLS succeeds",
formValues: map[string][]string{
"Name": {"Test Endpoint"},
"EndpointCreationType": {"4"},
"URL": {"https://portainer.example:9443"},
},
expectedError: "",
},
{
name: "non-edge agent env with TLS allowed",
formValues: map[string][]string{
"Name": {"Test Endpoint"},
"EndpointCreationType": {"2"},
"TLS": {"true"},
"TLSSkipVerify": {"true"},
"TLSSkipClientVerify": {"true"},
},
expectedError: "",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
r := http.Request{Form: tc.formValues}
p := &endpointCreatePayload{}
err := p.Validate(&r)
if tc.expectedError == "" {
require.NoError(t, err)
} else {
require.EqualError(t, err, tc.expectedError)
}
})
}
}
func TestCreateEdgeAgentEndpoint_ContainerEngineMapping(t *testing.T) {
t.Parallel()
fips.InitFIPS(false)
_, store := datastore.MustNewTestStore(t, true, false)
// required group for save flow
endpointGroup := &portainer.EndpointGroup{ID: 1, Name: "test-group"}
err := store.EndpointGroup().Create(endpointGroup)
require.NoError(t, err)
h := &Handler{
DataStore: store,
ReverseTunnelService: chisel.NewService(store, nil, nil),
}
tests := []struct {
name string
engine string
wantType portainer.EndpointType
}{
{
name: "empty engine -> EdgeAgentOnKubernetesEnvironment",
engine: "",
wantType: portainer.EdgeAgentOnKubernetesEnvironment,
},
{
name: "docker engine -> EdgeAgentOnDockerEnvironment",
engine: portainer.ContainerEngineDocker,
wantType: portainer.EdgeAgentOnDockerEnvironment,
},
{
name: "podman engine -> EdgeAgentOnDockerEnvironment",
engine: portainer.ContainerEnginePodman,
wantType: portainer.EdgeAgentOnDockerEnvironment,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
payload := &endpointCreatePayload{
Name: "edge-endpoint",
EndpointCreationType: edgeAgentEnvironment,
ContainerEngine: tc.engine,
GroupID: 1,
URL: "https://portainer.example:9443",
}
ep, httpErr := h.createEdgeAgentEndpoint(store, payload)
require.Nil(t, httpErr)
require.NotNil(t, ep)
assert.Equal(t, tc.wantType, ep.Type)
assert.Equal(t, tc.engine, ep.ContainerEngine)
})
}
}
@@ -0,0 +1,251 @@
package endpoints
import (
"errors"
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
type endpointDeleteRequest struct {
ID int `json:"id"`
DeleteCluster bool `json:"deleteCluster"`
}
type endpointDeleteBatchPayload struct {
Endpoints []endpointDeleteRequest `json:"endpoints"`
}
type endpointDeleteBatchPartialResponse struct {
Deleted []int `json:"deleted"`
Errors []int `json:"errors"`
}
func (payload *endpointDeleteBatchPayload) Validate(r *http.Request) error {
if payload == nil || len(payload.Endpoints) == 0 {
return errors.New("invalid request payload. You must provide a list of environments to delete")
}
return nil
}
// @id EndpointDelete
// @summary Remove an environment
// @description Remove the environment associated to the specified identifier and optionally clean-up associated resources.
// @description **Access policy**: Administrator only.
// @tags endpoints
// @security ApiKeyAuth || jwt
// @param id path int true "Environment(Endpoint) identifier"
// @success 204 "Environment successfully deleted."
// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
// @failure 403 "Unauthorized access or operation not allowed."
// @failure 404 "Unable to find the environment with the specified identifier inside the database."
// @failure 500 "Server error occurred while attempting to delete the environment."
// @router /endpoints/{id} [delete]
func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
// This is a Portainer provisioned cloud environment
deleteCluster, err := request.RetrieveBooleanQueryParameter(r, "deleteCluster", true)
if err != nil {
return httperror.BadRequest("Invalid boolean query parameter", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.deleteEndpoint(tx, portainer.EndpointID(endpointID), deleteCluster)
})
return response.TxEmptyResponse(w, err)
}
// @id EndpointDeleteBatch
// @summary Remove multiple environments
// @description Remove multiple environments and optionally clean-up associated resources.
// @description **Access policy**: Administrator only.
// @tags endpoints
// @security ApiKeyAuth || jwt
// @accept json
// @produce json
// @param body body endpointDeleteBatchPayload true "List of environments to delete, with optional deleteCluster flag to clean-up associated resources (cloud environments only)"
// @success 204 "Environment(s) successfully deleted."
// @failure 207 {object} endpointDeleteBatchPartialResponse "Partial success. Some environments were deleted successfully, while others failed."
// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
// @failure 403 "Unauthorized access or operation not allowed."
// @failure 500 "Server error occurred while attempting to delete the specified environments."
// @router /endpoints/delete [post]
func (handler *Handler) endpointDeleteBatch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var p endpointDeleteBatchPayload
if err := request.DecodeAndValidateJSONPayload(r, &p); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
resp := endpointDeleteBatchPartialResponse{
Deleted: []int{},
Errors: []int{},
}
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
for _, e := range p.Endpoints {
if err := handler.deleteEndpoint(tx, portainer.EndpointID(e.ID), e.DeleteCluster); err != nil {
resp.Errors = append(resp.Errors, e.ID)
log.Warn().Err(err).Int("environment_id", e.ID).Msg("Unable to remove environment")
continue
}
resp.Deleted = append(resp.Deleted, e.ID)
}
return nil
}); err != nil {
return httperror.InternalServerError("Unable to delete environments", err)
}
if len(resp.Errors) > 0 {
return response.JSONWithStatus(w, resp, http.StatusPartialContent)
}
return response.Empty(w)
}
// @id EndpointDeleteBatchDeprecated
// @summary Remove multiple environments
// @deprecated
// @description Deprecated: use the `POST` endpoint instead.
// @description Remove multiple environments and optionally clean-up associated resources.
// @description **Access policy**: Administrator only.
// @tags endpoints
// @security ApiKeyAuth || jwt
// @accept json
// @produce json
// @param body body endpointDeleteBatchPayload true "List of environments to delete, with optional deleteCluster flag to clean-up associated resources (cloud environments only)"
// @success 204 "Environment(s) successfully deleted."
// @failure 207 {object} endpointDeleteBatchPartialResponse "Partial success. Some environments were deleted successfully, while others failed."
// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
// @failure 403 "Unauthorized access or operation not allowed."
// @failure 500 "Server error occurred while attempting to delete the specified environments."
// @router /endpoints [delete]
func (handler *Handler) endpointDeleteBatchDeprecated(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
return handler.endpointDeleteBatch(w, r)
}
func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID portainer.EndpointID, deleteCluster bool) error {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to read the environment record from the database", err)
}
if endpoint.TLSConfig.TLS {
folder := strconv.Itoa(int(endpointID))
if err := handler.FileService.DeleteTLSFiles(folder); err != nil {
log.Error().Err(err).Msgf("Unable to remove TLS files from disk when deleting endpoint %d", endpointID)
}
}
if err := tx.Snapshot().Delete(endpointID); err != nil {
log.Warn().Err(err).Msg("Unable to remove the snapshot from the database")
}
handler.ProxyManager.DeleteEndpointProxy(endpoint.ID)
if err := tx.EndpointRelation().DeleteEndpointRelation(endpoint.ID); err != nil {
log.Warn().Err(err).Msg("Unable to remove environment relation from the database")
}
for _, tagID := range endpoint.TagIDs {
var tag *portainer.Tag
tag, err = tx.Tag().Read(tagID)
if err == nil {
delete(tag.Endpoints, endpoint.ID)
err = tx.Tag().Update(tagID, tag)
}
if tx.IsErrObjectNotFound(err) {
log.Warn().Err(err).Msg("Unable to find tag inside the database")
} else if err != nil {
log.Warn().Err(err).Msg("Unable to delete tag relation from the database")
}
}
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
log.Warn().Err(err).Msgf("Unable to retrieve edge groups from the database")
}
for _, edgeGroup := range edgeGroups {
edgeGroup.EndpointIDs.Remove(endpoint.ID)
if err := tx.EdgeGroup().Update(edgeGroup.ID, &edgeGroup); err != nil {
log.Warn().Err(err).Msg("Unable to update edge group")
}
}
edgeStacks, err := tx.EdgeStack().EdgeStacks()
if err != nil {
log.Warn().Err(err).Msg("Unable to retrieve edge stacks from the database")
}
for _, edgeStack := range edgeStacks {
if err := tx.EdgeStackStatus().Delete(edgeStack.ID, endpoint.ID); err != nil {
log.Warn().Err(err).Msg("Unable to delete edge stack status")
}
}
registries, err := tx.Registry().ReadAll()
if err != nil {
log.Warn().Err(err).Msg("Unable to retrieve registries from the database")
}
for idx := range registries {
registry := &registries[idx]
if _, ok := registry.RegistryAccesses[endpoint.ID]; ok {
delete(registry.RegistryAccesses, endpoint.ID)
if err := tx.Registry().Update(registry.ID, registry); err != nil {
log.Warn().Err(err).Msg("Unable to update registry accesses")
}
}
}
if endpointutils.IsEdgeEndpoint(endpoint) {
edgeJobs, err := tx.EdgeJob().ReadAll()
if err != nil {
log.Warn().Err(err).Msg("Unable to retrieve edge jobs from the database")
}
for idx := range edgeJobs {
edgeJob := &edgeJobs[idx]
if _, ok := edgeJob.Endpoints[endpoint.ID]; ok {
delete(edgeJob.Endpoints, endpoint.ID)
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
log.Warn().Err(err).Msg("Unable to update edge job")
}
}
}
}
// delete the pending actions
if err := tx.PendingActions().DeleteByEndpointID(endpoint.ID); err != nil {
log.Warn().Err(err).Int("endpointId", int(endpoint.ID)).Msg("Unable to delete pending actions")
}
if err := tx.Endpoint().DeleteEndpoint(endpointID); err != nil {
return httperror.InternalServerError("Unable to delete the environment from the database", err)
}
return nil
}
@@ -0,0 +1,86 @@
package endpoints
import (
"net/http"
"net/http/httptest"
"strconv"
"sync"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
)
func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
t.Parallel()
const endpointsCount = 100
_, store := datastore.MustNewTestStore(t, true, false)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
handler.ProxyManager = proxy.NewManager(nil)
handler.ProxyManager.NewProxyFactory(nil, nil, nil, nil, nil, nil, nil, nil, nil)
// Create all the environments and add them to the same edge group
var endpointIDs []portainer.EndpointID
for i := range endpointsCount {
endpointID := portainer.EndpointID(i) + 1
if err := store.Endpoint().Create(&portainer.Endpoint{
ID: endpointID,
Name: "env-" + strconv.Itoa(int(endpointID)),
Type: portainer.EdgeAgentOnDockerEnvironment,
}); err != nil {
t.Fatal("could not create endpoint:", err)
}
endpointIDs = append(endpointIDs, endpointID)
}
if err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "edgegroup-1",
EndpointIDs: roar.FromSlice(endpointIDs),
}); err != nil {
t.Fatal("could not create edge group:", err)
}
// Remove the environments concurrently
var wg sync.WaitGroup
wg.Add(len(endpointIDs))
for _, endpointID := range endpointIDs {
go func(ID portainer.EndpointID) {
defer wg.Done()
req, err := http.NewRequest(http.MethodDelete, "/endpoints/"+strconv.Itoa(int(ID)), nil)
if err != nil {
t.Fail()
return
}
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
}(endpointID)
}
wg.Wait()
// Check that the edge group is consistent
edgeGroup, err := handler.DataStore.EdgeGroup().Read(1)
if err != nil {
t.Fatal("could not retrieve the edge group:", err)
}
if edgeGroup.EndpointIDs.Len() > 0 {
t.Fatal("the edge group is not consistent")
}
}
@@ -0,0 +1,200 @@
package endpoints
import (
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/client"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
)
type dockerhubStatusResponse struct {
// Remaiming images to pull
Remaining int `json:"remaining"`
// Daily limit
Limit int `json:"limit"`
}
// @id endpointDockerhubStatus
// @summary fetch docker pull limits
// @description get docker pull limits for a docker hub registry in the environment
// @description **Access policy**:
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "endpoint ID"
// @param registryId path int true "registry ID"
// @success 200 {object} dockerhubStatusResponse "Success"
// @failure 400 "Invalid request"
// @failure 403 "Permission denied"
// @failure 404 "registry or endpoint not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/dockerhub/{registryId} [get]
func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
if !endpointutils.IsLocalEndpoint(endpoint) {
return httperror.BadRequest("Invalid environment type", errors.New("Invalid environment type"))
}
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return httperror.BadRequest("Invalid registry identifier route variable", err)
}
var registry *portainer.Registry
if registryID == 0 {
registry = &portainer.Registry{}
} else {
registry, err = handler.DataStore.Registry().Read(portainer.RegistryID(registryID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a registry with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a registry with the specified identifier inside the database", err)
}
if registry.Type != portainer.DockerHubRegistry {
return httperror.BadRequest("Invalid registry type", errors.New("Invalid registry type"))
}
}
if handler.PullLimitCheckDisabled {
return response.JSON(w, &dockerhubStatusResponse{
Limit: 10,
Remaining: 10,
})
}
httpClient := client.NewHTTPClient()
token, err := getDockerHubToken(httpClient, registry)
if err != nil {
return httperror.InternalServerError("Unable to retrieve DockerHub token from DockerHub", err)
}
resp, err := getDockerHubLimits(httpClient, token)
if err != nil {
return httperror.InternalServerError("Unable to retrieve DockerHub rate limits from DockerHub", err)
}
return response.JSON(w, resp)
}
func getDockerHubToken(httpClient *client.HTTPClient, registry *portainer.Registry) (string, error) {
type dockerhubTokenResponse struct {
Token string `json:"token"`
}
requestURL := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull"
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
return "", err
}
if registry.Authentication {
req.SetBasicAuth(registry.Username, registry.Password)
}
resp, err := httpClient.Do(req)
if err != nil {
return "", err
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
}()
if resp.StatusCode != http.StatusOK {
return "", errors.New("failed fetching dockerhub token")
}
var data dockerhubTokenResponse
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return "", err
}
return data.Token, nil
}
func getDockerHubLimits(httpClient *client.HTTPClient, token string) (*dockerhubStatusResponse, error) {
requestURL := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest"
req, err := http.NewRequest(http.MethodHead, requestURL, nil)
if err != nil {
return nil, err
}
req.Header.Add("Authorization", "Bearer "+token)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
_, _ = io.Copy(io.Discard, resp.Body)
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New("failed fetching dockerhub limits")
}
// An error with rateLimit-Limit or RateLimit-Remaining is likely for dockerhub pro accounts where there is no rate limit.
// In that specific case the headers will not be present. Don't bubble up the error as its normal
// See: https://docs.docker.com/docker-hub/download-rate-limit/
rateLimit, err := parseRateLimitHeader(resp.Header, "RateLimit-Limit")
if err != nil {
return nil, nil
}
rateLimitRemaining, err := parseRateLimitHeader(resp.Header, "RateLimit-Remaining")
if err != nil {
return nil, nil
}
return &dockerhubStatusResponse{
Limit: rateLimit,
Remaining: rateLimitRemaining,
}, nil
}
func parseRateLimitHeader(headers http.Header, headerKey string) (int, error) {
headerValue := headers.Get(headerKey)
if headerValue == "" {
return 0, fmt.Errorf("Missing %s header", headerKey)
}
matches := strings.Split(headerValue, ";")
value, err := strconv.Atoi(matches[0])
if err != nil {
return 0, err
}
return value, nil
}
@@ -0,0 +1,125 @@
package endpoints
import (
"context"
"net/http"
"strings"
portaineree "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/docker/consts"
"github.com/portainer/portainer/api/docker/images"
"github.com/portainer/portainer/api/logs"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
dockertypes "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
)
type forceUpdateServicePayload struct {
// ServiceId to update
ServiceID string
// PullImage if true will pull the image
PullImage bool
}
func (payload *forceUpdateServicePayload) Validate(r *http.Request) error {
return nil
}
// @id endpointForceUpdateService
// @summary force update a docker service
// @description force update a docker service
// @description **Access policy**: authenticated
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "endpoint identifier"
// @param body body forceUpdateServicePayload true "details"
// @success 200 {object} swarm.ServiceUpdateResponse "Success"
// @failure 400 "Invalid request"
// @failure 403 "Permission denied"
// @failure 404 "endpoint not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/forceupdateservice [put]
func (handler *Handler) endpointForceUpdateService(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
var payload forceUpdateServicePayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portaineree.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
if err != nil {
return httperror.Forbidden("Permission denied to force update service", err)
}
dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "", nil)
if err != nil {
return httperror.InternalServerError("Error creating docker client", err)
}
closeClient := true
defer func() {
if closeClient {
logs.CloseAndLogErr(dockerClient)
}
}()
service, _, err := dockerClient.ServiceInspectWithRaw(context.Background(), payload.ServiceID, dockertypes.ServiceInspectOptions{InsertDefaults: true})
if err != nil {
return httperror.InternalServerError("Error looking up service", err)
}
service.Spec.TaskTemplate.ForceUpdate++
if payload.PullImage {
service.Spec.TaskTemplate.ContainerSpec.Image = strings.Split(service.Spec.TaskTemplate.ContainerSpec.Image, "@sha")[0]
}
newService, err := dockerClient.ServiceUpdate(context.Background(), payload.ServiceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{QueryRegistry: true})
if err != nil {
return httperror.InternalServerError("Error force update service", err)
}
// The goroutine will close the client
closeClient = false
go func() {
defer logs.CloseAndLogErr(dockerClient)
images.EvictImageStatus(payload.ServiceID)
images.EvictImageStatus(service.Spec.Labels[consts.SwarmStackNameLabel])
// ignore errors from this cleanup function, log them instead
containers, err := dockerClient.ContainerList(context.TODO(), container.ListOptions{
All: true,
Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIDLabel+"="+payload.ServiceID)),
})
if err != nil {
log.Warn().Err(err).Str("Environment", endpoint.Name).Msg("Error listing containers")
}
for _, container := range containers {
images.EvictImageStatus(container.ID)
}
}()
return response.JSON(w, newService)
}
@@ -0,0 +1,88 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
// @id EndpointInspect
// @summary Inspect an environment(endpoint)
// @description Retrieve details about an environment(endpoint).
// @description **Access policy**: restricted
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @param excludeSnapshot query bool false "if true, the snapshot data won't be retrieved"
// @success 200 {object} portainer.Endpoint "Success"
// @failure 400 "Invalid request"
// @failure 404 "Environment(Endpoint) not found"
// @failure 500 "Server error"
// @router /endpoints/{id} [get]
func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
if err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
}
settings, err := handler.DataStore.Settings().Settings()
if err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
hideFields(endpoint)
endpointutils.UpdateEdgeEndpointHeartbeat(endpoint, settings)
endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true)
if !excludeSnapshot {
if err := handler.SnapshotService.FillSnapshotData(endpoint, false); err != nil {
return httperror.InternalServerError("Unable to add snapshot data", err)
}
}
if endpointutils.IsKubernetesEndpoint(endpoint) {
isServerMetricsDetected := endpoint.Kubernetes.Flags.IsServerMetricsDetected
isServerStorageDetected := endpoint.Kubernetes.Flags.IsServerStorageDetected
if (!isServerMetricsDetected || !isServerStorageDetected) && handler.K8sClientFactory != nil {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if !isServerMetricsDetected {
endpointutils.InitialMetricsDetection(tx, endpoint, handler.K8sClientFactory)
}
if !isServerStorageDetected {
endpointutils.InitialStorageDetection(tx, handler.DataStore, endpoint, handler.K8sClientFactory)
}
return nil
}); err != nil {
log.Err(err).Msg("failed to persist initial kube detection")
}
}
}
// Execute endpoint pending actions
handler.PendingActionsService.Execute(endpoint.ID)
return response.JSON(w, endpoint)
}
+157
View File
@@ -0,0 +1,157 @@
package endpoints
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
const (
EdgeDeviceIntervalMultiplier = 2
EdgeDeviceIntervalAdd = 20
)
// @id EndpointList
// @summary List environments(endpoints)
// @description List all environments(endpoints) based on the current user authorizations. Will
// @description return all environments(endpoints) if using an administrator or team leader account otherwise it will
// @description only return authorized environments(endpoints).
// @description **Access policy**: restricted
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param start query int false "Start searching from"
// @param limit query int false "Limit results to this value"
// @param sort query sortKey false "Sort results by this value" Enum("Name", "Group", "Status", "LastCheckIn", "EdgeID")
// @param order query string false "Order sorted results by desc/asc" Enum("asc", "desc")
// @param search query string false "Search query"
// @param groupIds query []int false "List environments(endpoints) of these groups"
// @param status query []int false "List environments(endpoints) by this status"
// @param types query []portainer.EndpointType false "List environments(endpoints) of this type"
// @param platformTypes query []portainer.PlatformType false "Filter environments by platform type"
// @param outdated query bool false "If true, return only environments with an outdated agent"
// @param excludeGroupIds query []int false "Exclude environments of these groups"
// @param tagIds query []int false "search environments(endpoints) with these tags (depends on tagsPartialMatch)"
// @param tagsPartialMatch query bool false "If true, will return environment(endpoint) which has one of tagIds, if false (or missing) will return only environments(endpoints) that has all the tags"
// @param endpointIds query []int false "will return only these environments(endpoints)"
// @param excludeIds query []int false "will exclude these environments(endpoints)"
// @param agentVersions query []string false "will return only environments with on of these agent versions"
// @param edgeAsync query bool false "if exists true show only edge async agents, false show only standard edge agents. if missing, will show both types (relevant only for edge agents)"
// @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
// @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)"
// @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved"
// @param name query string false "will return only environments(endpoints) with this name"
// @param edgeStackId query int false "will return the environments of the specified edge stack"
// @param edgeStackStatus query portainer.EdgeStackStatusType false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled")
// @param edgeGroupIds query []int false "List environments(endpoints) of these edge groups"
// @param excludeEdgeGroupIds query []int false "Exclude environments(endpoints) of these edge groups"
// @success 200 {array} portainer.Endpoint "Endpoints"
// @failure 500 "Server error"
// @router /endpoints [get]
func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
start, _ := request.RetrieveNumericQueryParameter(r, "start", true)
if start != 0 {
start--
}
limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true)
sortField, _ := request.RetrieveQueryParameter(r, "sort", true)
sortOrder, _ := request.RetrieveQueryParameter(r, "order", true)
endpointGroups, err := handler.DataStore.EndpointGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment groups from the database", err)
}
edgeGroups, err := handler.DataStore.EdgeGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve edge groups from the database", err)
}
endpoints, err := handler.DataStore.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
settings, err := handler.DataStore.Settings().Settings()
if err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
query, err := parseQuery(r)
if err != nil {
return httperror.BadRequest("Invalid query parameters", err)
}
filteredEndpoints, totalAvailableEndpoints, err := handler.filterEndpointsByQuery(endpoints, query, endpointGroups, edgeGroups, settings, securityContext)
if err != nil {
return httperror.InternalServerError("Unable to filter endpoints", err)
}
filteredEndpoints = security.FilterEndpoints(filteredEndpoints, endpointGroups, securityContext)
sortEnvironmentsByField(filteredEndpoints, endpointGroups, getSortKey(sortField), sortOrder == "desc", settings)
filteredEndpointCount := len(filteredEndpoints)
paginatedEndpoints := paginateEndpoints(filteredEndpoints, start, limit)
for idx := range paginatedEndpoints {
hideFields(&paginatedEndpoints[idx])
paginatedEndpoints[idx].ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
if paginatedEndpoints[idx].EdgeCheckinInterval == 0 {
paginatedEndpoints[idx].EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
endpointutils.UpdateEdgeEndpointHeartbeat(&paginatedEndpoints[idx], settings)
if !query.excludeSnapshots {
if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx], false); err != nil {
return httperror.InternalServerError("Unable to add snapshot data", err)
}
}
}
w.Header().Set("X-Total-Count", strconv.Itoa(filteredEndpointCount))
w.Header().Set("X-Total-Available", strconv.Itoa(totalAvailableEndpoints))
return response.JSON(w, paginatedEndpoints)
}
func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []portainer.Endpoint {
if limit == 0 {
return endpoints
}
endpointCount := len(endpoints)
start = min(max(start, 0), endpointCount)
end := min(start+limit, endpointCount)
return endpoints[start:end]
}
func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.EndpointGroup) portainer.EndpointGroup {
var endpointGroup portainer.EndpointGroup
for _, group := range groups {
if group.ID == groupID {
endpointGroup = group
break
}
}
return endpointGroup
}
@@ -0,0 +1,239 @@
package endpoints
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/snapshot"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type endpointListTest struct {
title string
expected []portainer.EndpointID
}
func Test_EndpointList_AgentVersion(t *testing.T) {
t.Parallel()
version1Endpoint := portainer.Endpoint{
ID: 1,
GroupID: 1,
Type: portainer.AgentOnDockerEnvironment,
Agent: portainer.EnvironmentAgentData{Version: "1.0.0"},
}
version2Endpoint := portainer.Endpoint{
ID: 2,
GroupID: 1,
Type: portainer.AgentOnDockerEnvironment,
Agent: portainer.EnvironmentAgentData{Version: "2.0.0"},
}
noVersionEndpoint := portainer.Endpoint{ID: 3, Type: portainer.AgentOnDockerEnvironment, GroupID: 1}
notAgentEnvironments := portainer.Endpoint{ID: 4, Type: portainer.DockerEnvironment, GroupID: 1}
handler := setupEndpointListHandler(t, []portainer.Endpoint{
notAgentEnvironments,
version1Endpoint,
version2Endpoint,
noVersionEndpoint,
})
type endpointListAgentVersionTest struct {
endpointListTest
filter []string
}
tests := []endpointListAgentVersionTest{
{
endpointListTest{
"should show version 1 agent endpoints and non-agent endpoints",
[]portainer.EndpointID{version1Endpoint.ID, notAgentEnvironments.ID},
},
[]string{version1Endpoint.Agent.Version},
},
{
endpointListTest{
"should show version 2 endpoints and non-agent endpoints",
[]portainer.EndpointID{version2Endpoint.ID, notAgentEnvironments.ID},
},
[]string{version2Endpoint.Agent.Version},
},
{
endpointListTest{
"should show version 1 and 2 endpoints and non-agent endpoints",
[]portainer.EndpointID{version2Endpoint.ID, notAgentEnvironments.ID, version1Endpoint.ID},
},
[]string{version2Endpoint.Agent.Version, version1Endpoint.Agent.Version},
},
}
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {
is := assert.New(t)
var query strings.Builder
for _, filter := range test.filter {
_, _ = query.WriteString("agentVersions[]=")
_, _ = query.WriteString(filter)
_, _ = query.WriteRune('&')
}
req := buildEndpointListRequest(query.String())
resp, err := doEndpointListRequest(req, handler, is)
require.NoError(t, err)
is.Len(resp, len(test.expected))
respIds := []portainer.EndpointID{}
for _, endpoint := range resp {
respIds = append(respIds, endpoint.ID)
}
is.ElementsMatch(test.expected, respIds)
})
}
}
func Test_endpointList_edgeFilter(t *testing.T) {
t.Parallel()
trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
handler := setupEndpointListHandler(t, []portainer.Endpoint{
trustedEdgeAsync,
untrustedEdgeAsync,
regularUntrustedEdgeStandard,
regularTrustedEdgeStandard,
regularEndpoint,
})
type endpointListEdgeTest struct {
endpointListTest
edgeAsync *bool
edgeDeviceUntrusted bool
}
tests := []endpointListEdgeTest{
{
endpointListTest: endpointListTest{
"should show all endpoints expect of the untrusted devices",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID, regularEndpoint.ID},
},
},
{
endpointListTest: endpointListTest{
"should show only trusted edge async agents and regular endpoints",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
},
edgeAsync: new(true),
},
{
endpointListTest: endpointListTest{
"should show only untrusted edge devices and regular endpoints",
[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
},
edgeAsync: new(true),
edgeDeviceUntrusted: true,
},
{
endpointListTest: endpointListTest{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
},
edgeAsync: new(false),
},
}
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {
is := assert.New(t)
query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted)
if test.edgeAsync != nil {
query += fmt.Sprintf("edgeAsync=%v&", *test.edgeAsync)
}
req := buildEndpointListRequest(query)
resp, err := doEndpointListRequest(req, handler, is)
require.NoError(t, err)
is.Len(resp, len(test.expected))
respIds := []portainer.EndpointID{}
for _, endpoint := range resp {
respIds = append(respIds, endpoint.ID)
}
is.ElementsMatch(test.expected, respIds)
})
}
}
func setupEndpointListHandler(t *testing.T, endpoints []portainer.Endpoint) *Handler {
_, store := datastore.MustNewTestStore(t, true, true)
for _, endpoint := range endpoints {
err := store.Endpoint().Create(&endpoint)
require.NoError(t, err, "error creating environment")
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
require.NoError(t, err, "error creating a user")
bouncer := testhelpers.NewTestRequestBouncer()
handler := NewHandler(bouncer)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()
handler.SnapshotService, _ = snapshot.NewService("1s", store, nil, nil, nil)
return handler
}
func buildEndpointListRequest(query string) *http.Request {
req := httptest.NewRequest(http.MethodGet, "/endpoints?"+query, nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)
restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
req = req.WithContext(restrictedCtx)
testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
return req
}
func doEndpointListRequest(req *http.Request, h *Handler, is *assert.Assertions) ([]portainer.Endpoint, error) {
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
is.Equal(http.StatusOK, rr.Code, "Status should be 200")
body, err := io.ReadAll(rr.Body)
if err != nil {
return nil, err
}
resp := []portainer.Endpoint{}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, err
}
return resp, nil
}
@@ -0,0 +1,219 @@
package endpoints
import (
"net/http"
"slices"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/kubernetes"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id endpointRegistriesList
// @summary List Registries on environment
// @description List all registries based on the current user authorizations in current environment.
// @description **Access policy**: authenticated
// @tags endpoints
// @param namespace query string false "required if kubernetes environment, will show registries by namespace"
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @success 200 {array} portainer.Registry "Success"
// @failure 500 "Server error"
// @router /endpoints/{id}/registries [get]
func (handler *Handler) endpointRegistriesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
var registries []portainer.Registry
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
registries, err = handler.listRegistries(tx, r, portainer.EndpointID(endpointID))
return err
})
return response.TxResponse(w, registries, err)
}
func (handler *Handler) listRegistries(tx dataservices.DataStoreTx, r *http.Request, endpointID portainer.EndpointID) ([]portainer.Registry, error) {
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve info from request context", err)
}
user, err := tx.User().Read(securityContext.UserID)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve user from the database", err)
}
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return nil, httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
isAdmin := securityContext.IsAdmin
registries, err := tx.Registry().ReadAll()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve registries from the database", err)
}
registries, handleError := handler.filterRegistriesByAccess(tx, r, registries, endpoint, user, securityContext.UserMemberships)
if handleError != nil {
return nil, handleError
}
for idx := range registries {
hideRegistryFields(&registries[idx], !isAdmin)
}
return registries, err
}
func (handler *Handler) filterRegistriesByAccess(tx dataservices.DataStoreTx, r *http.Request, registries []portainer.Registry, endpoint *portainer.Endpoint, user *portainer.User, memberships []portainer.TeamMembership) ([]portainer.Registry, *httperror.HandlerError) {
if !endpointutils.IsKubernetesEndpoint(endpoint) {
return security.FilterRegistries(registries, user, memberships, endpoint.ID), nil
}
return handler.filterKubernetesEndpointRegistries(tx, r, registries, endpoint, user, memberships)
}
func (handler *Handler) filterKubernetesEndpointRegistries(tx dataservices.DataStoreTx, r *http.Request, registries []portainer.Registry, endpoint *portainer.Endpoint, user *portainer.User, memberships []portainer.TeamMembership) ([]portainer.Registry, *httperror.HandlerError) {
namespaceParam, _ := request.RetrieveQueryParameter(r, "namespace", true)
isAdmin, err := security.IsAdmin(r)
if err != nil {
return nil, httperror.InternalServerError("Unable to check user role", err)
}
if namespaceParam != "" {
if authorized, err := handler.isNamespaceAuthorized(endpoint, namespaceParam, user.ID, memberships, isAdmin); err != nil {
return nil, httperror.NotFound("Unable to check for namespace authorization", err)
} else if !authorized {
return nil, httperror.Forbidden("User is not authorized to use namespace", errors.New("user is not authorized to use namespace"))
}
return filterRegistriesByNamespaces(registries, endpoint.ID, []string{namespaceParam}), nil
}
if isAdmin {
return registries, nil
}
return handler.filterKubernetesRegistriesByUserRole(tx, r, registries, endpoint, user)
}
func (handler *Handler) isNamespaceAuthorized(endpoint *portainer.Endpoint, namespace string, userId portainer.UserID, memberships []portainer.TeamMembership, isAdmin bool) (bool, error) {
if isAdmin || namespace == "" {
return true, nil
}
if !endpoint.Kubernetes.Configuration.RestrictDefaultNamespace && namespace == kubernetes.DefaultNamespace {
return true, nil
}
kcl, err := handler.K8sClientFactory.GetPrivilegedKubeClient(endpoint)
if err != nil {
return false, errors.Wrap(err, "unable to retrieve kubernetes client")
}
accessPolicies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return false, errors.Wrap(err, "unable to retrieve environment's namespaces policies")
}
namespacePolicy, ok := accessPolicies[namespace]
if !ok {
return false, nil
}
return security.AuthorizedAccess(userId, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies), nil
}
func filterRegistriesByNamespaces(registries []portainer.Registry, endpointId portainer.EndpointID, namespaces []string) []portainer.Registry {
filteredRegistries := []portainer.Registry{}
for _, registry := range registries {
if registryAccessPoliciesContainsNamespace(registry.RegistryAccesses[endpointId], namespaces) {
filteredRegistries = append(filteredRegistries, registry)
}
}
return filteredRegistries
}
func registryAccessPoliciesContainsNamespace(registryAccess portainer.RegistryAccessPolicies, namespaces []string) bool {
for _, authorizedNamespace := range registryAccess.Namespaces {
if slices.Contains(namespaces, authorizedNamespace) {
return true
}
}
return false
}
func (handler *Handler) filterKubernetesRegistriesByUserRole(tx dataservices.DataStoreTx, r *http.Request, registries []portainer.Registry, endpoint *portainer.Endpoint, user *portainer.User) ([]portainer.Registry, *httperror.HandlerError) {
err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
if errors.Is(err, security.ErrAuthorizationRequired) {
return nil, httperror.Forbidden("User is not authorized", err)
}
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve info from request context", err)
}
userNamespaces, err := handler.userNamespaces(tx, endpoint, user)
if err != nil {
return nil, httperror.InternalServerError("unable to retrieve user namespaces", err)
}
return filterRegistriesByNamespaces(registries, endpoint.ID, userNamespaces), nil
}
func (handler *Handler) userNamespaces(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, user *portainer.User) ([]string, error) {
kcl, err := handler.K8sClientFactory.GetPrivilegedKubeClient(endpoint)
if err != nil {
return nil, err
}
namespaceAuthorizations, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return nil, err
}
userMemberships, err := tx.TeamMembership().TeamMembershipsByUserID(user.ID)
if err != nil {
return nil, err
}
var userNamespaces []string
for namespace, namespaceAuthorization := range namespaceAuthorizations {
if _, ok := namespaceAuthorization.UserAccessPolicies[user.ID]; ok {
userNamespaces = append(userNamespaces, namespace)
continue
}
for _, userTeam := range userMemberships {
if _, ok := namespaceAuthorization.TeamAccessPolicies[userTeam.TeamID]; ok {
userNamespaces = append(userNamespaces, namespace)
continue
}
}
}
return userNamespaces, nil
}
func hideRegistryFields(registry *portainer.Registry, hideAccesses bool) {
registry.Password = ""
registry.ManagementConfiguration = nil
if hideAccesses {
registry.RegistryAccesses = nil
}
}
@@ -0,0 +1,186 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/registryutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type registryAccessPayload struct {
UserAccessPolicies portainer.UserAccessPolicies
TeamAccessPolicies portainer.TeamAccessPolicies
Namespaces []string
}
func (payload *registryAccessPayload) Validate(r *http.Request) error {
return nil
}
// @id endpointRegistryAccess
// @summary update registry access for environment
// @description **Access policy**: authenticated
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @param registryId path int true "Registry identifier"
// @param body body registryAccessPayload true "details"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 403 "Permission denied"
// @failure 404 "Endpoint not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/registries/{registryId} [put]
func (handler *Handler) endpointRegistryAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return httperror.BadRequest("Invalid registry identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.updateRegistryAccess(tx, r, portainer.EndpointID(endpointID), portainer.RegistryID(registryID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) updateRegistryAccess(tx dataservices.DataStoreTx, r *http.Request, endpointID portainer.EndpointID, registryID portainer.RegistryID) error {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
if err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
}
if !securityContext.IsAdmin {
return httperror.Forbidden("User is not authorized", err)
}
registry, err := tx.Registry().Read(registryID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
var payload registryAccessPayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
if registry.RegistryAccesses == nil {
registry.RegistryAccesses = portainer.RegistryAccesses{}
}
if _, ok := registry.RegistryAccesses[endpoint.ID]; !ok {
registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{}
}
registryAccess := registry.RegistryAccesses[endpoint.ID]
if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
err := handler.updateKubeAccess(endpoint, registry, registryAccess.Namespaces, payload.Namespaces)
if err != nil {
return httperror.InternalServerError("Unable to update kube access policies", err)
}
registryAccess.Namespaces = payload.Namespaces
} else {
registryAccess.UserAccessPolicies = payload.UserAccessPolicies
registryAccess.TeamAccessPolicies = payload.TeamAccessPolicies
}
registry.RegistryAccesses[endpointID] = registryAccess
return tx.Registry().Update(registry.ID, registry)
}
func (handler *Handler) updateKubeAccess(endpoint *portainer.Endpoint, registry *portainer.Registry, oldNamespaces, newNamespaces []string) error {
cli, err := handler.K8sClientFactory.GetPrivilegedKubeClient(endpoint)
if err != nil {
return err
}
return applyKubeRegistryAccess(cli, registry, oldNamespaces, newNamespaces)
}
func applyKubeRegistryAccess(cli portainer.KubeClient, registry *portainer.Registry, oldNamespaces, newNamespaces []string) error {
oldNamespacesSet := toSet(oldNamespaces)
newNamespacesSet := toSet(newNamespaces)
namespacesToRemove := setDifference(oldNamespacesSet, newNamespacesSet)
namespacesToAdd := setDifference(newNamespacesSet, oldNamespacesSet)
for namespace := range namespacesToRemove {
secretName := registryutils.RegistrySecretName(registry.ID)
if err := cli.RemoveImagePullSecretFromServiceAccount(namespace, "default", secretName); err != nil {
return err
}
if err := cli.DeleteRegistrySecret(registry.ID, namespace); err != nil {
return err
}
}
for namespace := range namespacesToAdd {
secretName := registryutils.RegistrySecretName(registry.ID)
if err := cli.CreateRegistrySecret(registry, namespace); err != nil {
return err
}
if err := cli.AddImagePullSecretToServiceAccount(namespace, "default", secretName); err != nil {
return err
}
}
return nil
}
type stringSet map[string]bool
func toSet(list []string) stringSet {
set := stringSet{}
for _, el := range list {
set[el] = true
}
return set
}
// setDifference returns the set difference tagsA - tagsB
func setDifference(setA stringSet, setB stringSet) stringSet {
set := stringSet{}
for el := range setA {
if !setB[el] {
set[el] = true
}
}
return set
}
@@ -0,0 +1,169 @@
package endpoints
import (
"errors"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// spyKubeClient implements portainer.KubeClient for testing applyKubeRegistryAccess.
// It embeds the interface so unimplemented methods panic, and overrides only the
// four methods exercised by applyKubeRegistryAccess.
type spyKubeClient struct {
portainer.KubeClient
createSecretErrors map[string]error
deleteSecretErrors map[string]error
addPullSecretErrors map[string]error
removePullSecretErrors map[string]error
createdSecrets []string
deletedSecrets []string
addedPullSecrets []string
removedPullSecrets []string
}
func newSpyKubeClient() *spyKubeClient {
return &spyKubeClient{
createSecretErrors: make(map[string]error),
deleteSecretErrors: make(map[string]error),
addPullSecretErrors: make(map[string]error),
removePullSecretErrors: make(map[string]error),
}
}
func (s *spyKubeClient) CreateRegistrySecret(_ *portainer.Registry, namespace string) error {
s.createdSecrets = append(s.createdSecrets, namespace)
return s.createSecretErrors[namespace]
}
func (s *spyKubeClient) DeleteRegistrySecret(_ portainer.RegistryID, namespace string) error {
s.deletedSecrets = append(s.deletedSecrets, namespace)
return s.deleteSecretErrors[namespace]
}
func (s *spyKubeClient) AddImagePullSecretToServiceAccount(namespace, _, _ string) error {
s.addedPullSecrets = append(s.addedPullSecrets, namespace)
return s.addPullSecretErrors[namespace]
}
func (s *spyKubeClient) RemoveImagePullSecretFromServiceAccount(namespace, _, _ string) error {
s.removedPullSecrets = append(s.removedPullSecrets, namespace)
return s.removePullSecretErrors[namespace]
}
var testRegistry = &portainer.Registry{ID: 3, URL: "registry.example.com"}
func TestApplyKubeRegistryAccess_Grant(t *testing.T) {
t.Parallel()
t.Run("single namespace granted creates secret then patches SA", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, nil, []string{"ns-a"})
require.NoError(t, err)
assert.Equal(t, []string{"ns-a"}, spy.createdSecrets)
assert.Equal(t, []string{"ns-a"}, spy.addedPullSecrets)
assert.Empty(t, spy.deletedSecrets)
assert.Empty(t, spy.removedPullSecrets)
})
t.Run("multiple namespaces granted applies to all", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, nil, []string{"ns-a", "ns-b"})
require.NoError(t, err)
assert.ElementsMatch(t, []string{"ns-a", "ns-b"}, spy.createdSecrets)
assert.ElementsMatch(t, []string{"ns-a", "ns-b"}, spy.addedPullSecrets)
})
t.Run("CreateRegistrySecret fails - AddImagePullSecret not called", func(t *testing.T) {
spy := newSpyKubeClient()
spy.createSecretErrors["ns-a"] = errors.New("secret create failed")
err := applyKubeRegistryAccess(spy, testRegistry, nil, []string{"ns-a"})
require.Error(t, err)
assert.Equal(t, []string{"ns-a"}, spy.createdSecrets)
assert.Empty(t, spy.addedPullSecrets)
})
t.Run("AddImagePullSecret fails after secret created - returns error", func(t *testing.T) {
spy := newSpyKubeClient()
spy.addPullSecretErrors["ns-a"] = errors.New("sa patch failed")
err := applyKubeRegistryAccess(spy, testRegistry, nil, []string{"ns-a"})
require.Error(t, err)
assert.Equal(t, []string{"ns-a"}, spy.createdSecrets)
assert.Equal(t, []string{"ns-a"}, spy.addedPullSecrets)
})
}
func TestApplyKubeRegistryAccess_Revoke(t *testing.T) {
t.Parallel()
t.Run("single namespace revoked removes from SA then deletes secret", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-a"}, nil)
require.NoError(t, err)
assert.Equal(t, []string{"ns-a"}, spy.removedPullSecrets)
assert.Equal(t, []string{"ns-a"}, spy.deletedSecrets)
assert.Empty(t, spy.createdSecrets)
assert.Empty(t, spy.addedPullSecrets)
})
t.Run("multiple namespaces revoked applies to all", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-a", "ns-b"}, nil)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"ns-a", "ns-b"}, spy.removedPullSecrets)
assert.ElementsMatch(t, []string{"ns-a", "ns-b"}, spy.deletedSecrets)
})
t.Run("RemoveImagePullSecret fails - DeleteRegistrySecret not called", func(t *testing.T) {
spy := newSpyKubeClient()
spy.removePullSecretErrors["ns-a"] = errors.New("sa remove failed")
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-a"}, nil)
require.Error(t, err)
assert.Equal(t, []string{"ns-a"}, spy.removedPullSecrets)
assert.Empty(t, spy.deletedSecrets)
})
t.Run("DeleteRegistrySecret fails after SA patched - returns error", func(t *testing.T) {
spy := newSpyKubeClient()
spy.deleteSecretErrors["ns-a"] = errors.New("secret delete failed")
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-a"}, nil)
require.Error(t, err)
assert.Equal(t, []string{"ns-a"}, spy.removedPullSecrets)
assert.Equal(t, []string{"ns-a"}, spy.deletedSecrets)
})
}
func TestApplyKubeRegistryAccess_Mixed(t *testing.T) {
t.Parallel()
t.Run("one namespace added and one removed in same call", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-old"}, []string{"ns-new"})
require.NoError(t, err)
assert.Equal(t, []string{"ns-old"}, spy.removedPullSecrets)
assert.Equal(t, []string{"ns-old"}, spy.deletedSecrets)
assert.Equal(t, []string{"ns-new"}, spy.createdSecrets)
assert.Equal(t, []string{"ns-new"}, spy.addedPullSecrets)
})
t.Run("empty old and new namespaces - no operations performed", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, nil, nil)
require.NoError(t, err)
assert.Empty(t, spy.createdSecrets)
assert.Empty(t, spy.deletedSecrets)
assert.Empty(t, spy.addedPullSecrets)
assert.Empty(t, spy.removedPullSecrets)
})
t.Run("namespace present in both old and new - no operations performed for it", func(t *testing.T) {
spy := newSpyKubeClient()
err := applyKubeRegistryAccess(spy, testRegistry, []string{"ns-keep"}, []string{"ns-keep"})
require.NoError(t, err)
assert.Empty(t, spy.createdSecrets)
assert.Empty(t, spy.deletedSecrets)
assert.Empty(t, spy.addedPullSecrets)
assert.Empty(t, spy.removedPullSecrets)
})
}
@@ -0,0 +1,136 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type endpointSettingsUpdatePayload struct {
// Whether non-administrator should be able to use bind mounts when creating containers
AllowBindMountsForRegularUsers *bool `json:"allowBindMountsForRegularUsers" example:"false"`
// Whether non-administrator should be able to use privileged mode when creating containers
AllowPrivilegedModeForRegularUsers *bool `json:"allowPrivilegedModeForRegularUsers" example:"false"`
// Whether non-administrator should be able to browse volumes
AllowVolumeBrowserForRegularUsers *bool `json:"allowVolumeBrowserForRegularUsers" example:"true"`
// Whether non-administrator should be able to use the host pid
AllowHostNamespaceForRegularUsers *bool `json:"allowHostNamespaceForRegularUsers" example:"true"`
// Whether non-administrator should be able to use device mapping
AllowDeviceMappingForRegularUsers *bool `json:"allowDeviceMappingForRegularUsers" example:"true"`
// Whether non-administrator should be able to manage stacks
AllowStackManagementForRegularUsers *bool `json:"allowStackManagementForRegularUsers" example:"true"`
// Whether non-administrator should be able to use container capabilities
AllowContainerCapabilitiesForRegularUsers *bool `json:"allowContainerCapabilitiesForRegularUsers" example:"true"`
// Whether non-administrator should be able to use sysctl settings
AllowSysctlSettingForRegularUsers *bool `json:"allowSysctlSettingForRegularUsers" example:"true"`
// Whether non-administrator should be able to use security-opt settings
AllowSecurityOptForRegularUsers *bool `json:"allowSecurityOptForRegularUsers" example:"true"`
// Whether host management features are enabled
EnableHostManagementFeatures *bool `json:"enableHostManagementFeatures" example:"true"`
EnableGPUManagement *bool `json:"enableGPUManagement" example:"false"`
Gpus []portainer.Pair `json:"gpus"`
}
func (payload *endpointSettingsUpdatePayload) Validate(r *http.Request) error {
return nil
}
// @id EndpointSettingsUpdate
// @summary Update settings for an environment(endpoint)
// @description Update settings for an environment(endpoint).
// @description **Access policy**: authenticated
// @security ApiKeyAuth
// @security jwt
// @tags endpoints
// @accept json
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @param body body endpointSettingsUpdatePayload true "Environment(Endpoint) details"
// @success 200 {object} portainer.Endpoint "Success"
// @failure 400 "Invalid request"
// @failure 404 "Environment(Endpoint) not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/settings [put]
func (handler *Handler) endpointSettingsUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
var payload endpointSettingsUpdatePayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
securitySettings := endpoint.SecuritySettings
if payload.AllowBindMountsForRegularUsers != nil {
securitySettings.AllowBindMountsForRegularUsers = *payload.AllowBindMountsForRegularUsers
}
if payload.AllowContainerCapabilitiesForRegularUsers != nil {
securitySettings.AllowContainerCapabilitiesForRegularUsers = *payload.AllowContainerCapabilitiesForRegularUsers
}
if payload.AllowDeviceMappingForRegularUsers != nil {
securitySettings.AllowDeviceMappingForRegularUsers = *payload.AllowDeviceMappingForRegularUsers
}
if payload.AllowHostNamespaceForRegularUsers != nil {
securitySettings.AllowHostNamespaceForRegularUsers = *payload.AllowHostNamespaceForRegularUsers
}
if payload.AllowPrivilegedModeForRegularUsers != nil {
securitySettings.AllowPrivilegedModeForRegularUsers = *payload.AllowPrivilegedModeForRegularUsers
}
if payload.AllowStackManagementForRegularUsers != nil {
securitySettings.AllowStackManagementForRegularUsers = *payload.AllowStackManagementForRegularUsers
}
if payload.AllowVolumeBrowserForRegularUsers != nil {
securitySettings.AllowVolumeBrowserForRegularUsers = *payload.AllowVolumeBrowserForRegularUsers
}
if payload.AllowSysctlSettingForRegularUsers != nil {
securitySettings.AllowSysctlSettingForRegularUsers = *payload.AllowSysctlSettingForRegularUsers
}
if payload.EnableHostManagementFeatures != nil {
securitySettings.EnableHostManagementFeatures = *payload.EnableHostManagementFeatures
}
if payload.AllowSecurityOptForRegularUsers != nil {
securitySettings.AllowSecurityOptForRegularUsers = *payload.AllowSecurityOptForRegularUsers
}
endpoint.SecuritySettings = securitySettings
if payload.EnableGPUManagement != nil {
endpoint.EnableGPUManagement = *payload.EnableGPUManagement
}
if payload.Gpus != nil {
endpoint.Gpus = payload.Gpus
}
err = handler.DataStore.Endpoint().UpdateEndpoint(portainer.EndpointID(endpointID), endpoint)
if err != nil {
return httperror.InternalServerError("Failed persisting environment in database", err)
}
return response.JSON(w, endpoint)
}
@@ -0,0 +1,64 @@
package endpoints
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/snapshot"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EndpointSnapshot
// @summary Snapshots an environment(endpoint)
// @description Snapshots an environment(endpoint)
// @description **Access policy**: administrator
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @param id path int true "Environment(Endpoint) identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 404 "Environment(Endpoint) not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/snapshot [post]
func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
if !snapshot.SupportDirectSnapshot(endpoint) {
return httperror.BadRequest("Snapshots not supported for this environment", errors.New("Snapshots not supported for this environment"))
}
snapshotError := handler.SnapshotService.SnapshotEndpoint(endpoint)
latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
if latestEndpointReference == nil {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
}
latestEndpointReference.Status = portainer.EndpointStatusUp
if snapshotError != nil {
latestEndpointReference.Status = portainer.EndpointStatusDown
}
latestEndpointReference.Agent.Version = endpoint.Agent.Version
err = handler.DataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
if err != nil {
return httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
return response.Empty(w)
}
@@ -0,0 +1,72 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/snapshot"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
// @id EndpointSnapshots
// @summary Snapshot all environments(endpoints)
// @description Snapshot all environments(endpoints)
// @description **Access policy**: administrator
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @success 204 "Success"
// @failure 500 "Server Error"
// @router /endpoints/snapshot [post]
func (handler *Handler) endpointSnapshots(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpoints, err := handler.DataStore.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
for _, endpoint := range endpoints {
if !snapshot.SupportDirectSnapshot(&endpoint) {
continue
}
if endpoint.URL == "" {
continue
}
snapshotError := handler.SnapshotService.SnapshotEndpoint(&endpoint)
latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
if latestEndpointReference == nil {
log.Debug().
Str("endpoint", endpoint.Name).
Str("URL", endpoint.URL).
Err(err).
Msg("background schedule error (environment snapshot), environment not found inside the database anymore")
continue
}
latestEndpointReference.Status = portainer.EndpointStatusUp
if snapshotError != nil {
log.Debug().
Str("endpoint", endpoint.Name).
Str("URL", endpoint.URL).
Err(snapshotError).
Msg("background schedule error (environment snapshot), unable to create snapshot")
latestEndpointReference.Status = portainer.EndpointStatusDown
}
latestEndpointReference.Agent.Version = endpoint.Agent.Version
err = handler.DataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
if err != nil {
return httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
}
return response.Empty(w)
}
@@ -0,0 +1,109 @@
package endpoints
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_endpointSnapshots(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
endpointID := portainer.EndpointID(123)
endpoint := &portainer.Endpoint{
ID: endpointID,
Name: "mock",
URL: "http://mock.example/",
Status: portainer.EndpointStatusDown, // starts in down state
}
err := store.Endpoint().Create(endpoint)
require.NoError(t, err, "error creating environment")
err = store.User().Create(
&portainer.User{
Username: "admin",
Role: portainer.AdministratorRole,
},
)
require.NoError(t, err, "error creating a user")
bouncer := testhelpers.NewTestRequestBouncer()
snapshotService := &mockSnapshotService{
snapshotEndpointShouldSucceed: atomic.Bool{},
}
snapshotService.snapshotEndpointShouldSucceed.Store(true)
h := NewHandler(bouncer)
h.DataStore = store
h.SnapshotService = snapshotService
doPostRequest := func() {
req := httptest.NewRequest(http.MethodPost, "/endpoints/snapshot", nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)
testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusNoContent, rr.Code, "Status should be 204")
_, err := io.ReadAll(rr.Body)
require.NoError(t, err, "ReadAll should not return error")
}
doPostRequest()
// check that the endpoint has been immediately set to up
endpoint, err = store.Endpoint().Endpoint(endpointID)
require.NoError(t, err, "error getting endpoint")
assert.Equal(t, portainer.EndpointStatusUp, endpoint.Status, "endpoint should be up (1) since mock snapshot returned ok")
// set the mock to return an error
snapshotService.snapshotEndpointShouldSucceed.Store(false)
doPostRequest()
// check that the endpoint has been immediately set to down
endpoint, err = store.Endpoint().Endpoint(endpointID)
require.NoError(t, err, "error getting endpoint")
assert.Equal(t, portainer.EndpointStatusDown, endpoint.Status, "endpoint should be down (2) since mock snapshot returned error")
}
var _ portainer.SnapshotService = &mockSnapshotService{}
type mockSnapshotService struct {
snapshotEndpointShouldSucceed atomic.Bool
}
func (s *mockSnapshotService) Start(_ context.Context) {
}
func (s *mockSnapshotService) SetSnapshotInterval(snapshotInterval string) error {
return nil
}
func (s *mockSnapshotService) SnapshotEndpoint(endpoint *portainer.Endpoint) error {
if s.snapshotEndpointShouldSucceed.Load() {
return nil
}
return errors.New("snapshot failed")
}
func (s *mockSnapshotService) FillSnapshotData(endpoint *portainer.Endpoint, includeRaw bool) error {
return nil
}
@@ -0,0 +1,21 @@
package endpoints
import (
"fmt"
"net/http"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
)
// DEPRECATED
func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
url := fmt.Sprintf("/api/endpoints/%d/edge/status", endpointID)
http.Redirect(w, r, url, http.StatusPermanentRedirect)
return nil
}
@@ -0,0 +1,246 @@
package endpoints
import (
"net/http"
"sort"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
"golang.org/x/mod/semver"
)
type groupCount struct {
GroupID int `json:"groupID"`
GroupName string `json:"groupName"`
Count int `json:"count"`
}
type platformCounts struct {
Docker int `json:"docker"`
Kubernetes int `json:"kubernetes"`
Azure int `json:"azure"`
Podman int `json:"podman"`
}
type healthCounts struct {
Down int `json:"down"`
Outdated int `json:"outdated"`
Up int `json:"up"`
Heartbeat int `json:"heartbeat"`
}
type EnvironmentSummaryCountsResponse struct {
Total int `json:"total"`
Up int `json:"up"`
Down int `json:"down"`
Outdated int `json:"outdated"`
Unassigned int `json:"unassigned"`
ByGroup []groupCount `json:"byGroup"`
ByPlatformType platformCounts `json:"byPlatformType"`
ByHealth healthCounts `json:"byHealth"`
}
const UnassignedGroupID = portainer.EndpointGroupID(1)
// @id EndpointSummaryCounts
// @summary Get environment summary counts
// @description Returns counts of environments by status (up, down) and ungrouped environments (unassigned), plus breakdowns by group, type, and health.
// @description **Access policy**: restricted
// @tags endpoints
// @security ApiKeyAuth
// @security jwt
// @produce json
// @success 200 {object} EnvironmentSummaryCountsResponse "Environment summary counts"
// @failure 500 "Server error"
// @router /endpoints/summary [get]
func (handler *Handler) endpointSummaryCounts(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var endpointGroups []portainer.EndpointGroup
var endpoints []portainer.Endpoint
var settings *portainer.Settings
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
endpointGroups, err = tx.EndpointGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment groups from the database", err)
}
endpoints, err = tx.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
settings, err = tx.Settings().Settings()
if err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
return nil
}); err != nil {
return response.TxErrorResponse(err)
}
// Refresh LastCheckInDate from the in-memory heartbeats map. Edge agents
// use ETag-based response caching: cache hits update the map but do not
// write LastCheckInDate back to the database, so the tx value grows stale.
// The tx path cannot access the in-memory map; this non-tx access is
// intentional.
endpointSvc := handler.DataStore.Endpoint()
for i := range endpoints {
if t, ok := endpointSvc.Heartbeat(endpoints[i].ID); ok {
endpoints[i].LastCheckInDate = t
}
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext)
trustedEndpoints := make([]portainer.Endpoint, 0, len(filteredEndpoints))
for i := range filteredEndpoints {
ep := &filteredEndpoints[i]
if endpointutils.IsEdgeEndpoint(ep) && !ep.UserTrusted {
continue
}
trustedEndpoints = append(trustedEndpoints, filteredEndpoints[i])
}
counts := EnvironmentSummaryCountsResponse{
Total: len(trustedEndpoints),
}
groupCounts := make(map[portainer.EndpointGroupID]int)
platformCounts := platformCounts{}
healthCounts := healthCounts{}
for i := range trustedEndpoints {
endpoint := &trustedEndpoints[i]
switch endpointutils.EndpointPlatformType(endpoint) {
case portainer.DockerPlatformType:
platformCounts.Docker++
case portainer.KubernetesPlatformType:
platformCounts.Kubernetes++
case portainer.AzurePlatformType:
platformCounts.Azure++
case portainer.PodmanPlatformType:
platformCounts.Podman++
case portainer.UnknownPlatformType:
log.Error().Int("endpoint_id", int(endpoint.ID)).Msg("Unknown platform type")
}
groupCounts[endpoint.GroupID]++
if endpoint.GroupID == UnassignedGroupID {
counts.Unassigned++
}
outdated := isOutdated(endpoint)
status := resolveEndpointStatus(endpoint, settings)
if outdated {
counts.Outdated++
healthCounts.Outdated++
}
switch status {
case statusHeartbeat:
healthCounts.Heartbeat++
healthCounts.Up++
counts.Up++
case statusUp:
healthCounts.Up++
counts.Up++
case statusDown:
healthCounts.Down++
counts.Down++
}
}
counts.ByGroup = parseGroupCounts(groupCounts, endpointGroups)
counts.ByPlatformType = platformCounts
counts.ByHealth = healthCounts
return response.JSON(w, counts)
}
// iota order overlaps with portainer.EndpointStatus (Up=1, Down=2) so non-edge
// endpoints can pass their Status straight through. statusHeartbeat (0) is
// edge-only.
const (
statusHeartbeat = iota
statusUp
statusDown
)
func resolveEndpointStatus(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
if endpointutils.IsEdgeEndpoint(endpoint) {
if endpointutils.GetHeartbeatStatus(endpoint, settings) {
return statusHeartbeat
}
return statusDown
}
return int(endpoint.Status)
}
func parseGroupCounts(counts map[portainer.EndpointGroupID]int, endpointGroups []portainer.EndpointGroup) []groupCount {
parsedGroupCounts := []groupCount{}
// Build group name lookup
groupNameByID := make(map[portainer.EndpointGroupID]string, len(endpointGroups))
for _, g := range endpointGroups {
groupNameByID[g.ID] = g.Name
}
for groupID, count := range counts {
parsedGroupCounts = append(parsedGroupCounts,
groupCount{
GroupID: int(groupID),
GroupName: groupNameByID[groupID],
Count: count,
})
}
sort.Slice(parsedGroupCounts, func(i, j int) bool {
return parsedGroupCounts[i].GroupID < parsedGroupCounts[j].GroupID
})
return parsedGroupCounts
}
// canonicalizeSemver ensures v has a "v" prefix as required by golang.org/x/mod/semver.
func canonicalizeSemver(v string) string {
v = strings.TrimSpace(v)
if v == "" || strings.HasPrefix(v, "v") {
return v
}
return "v" + v
}
func isOutdated(endpoint *portainer.Endpoint) bool {
if !endpointutils.IsAgentEndpoint(endpoint) {
return false
}
if endpoint.Agent.Version == "" {
edgeHasCheckedInWithoutVersion := endpointutils.IsEdgeEndpoint(endpoint) && endpoint.LastCheckInDate > 0
return edgeHasCheckedInWithoutVersion
}
latestVersion := canonicalizeSemver(portainer.APIVersion)
agentVersion := canonicalizeSemver(endpoint.Agent.Version)
return semver.Compare(agentVersion, latestVersion) < 0
}
@@ -0,0 +1,497 @@
package endpoints
import (
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSummaryCounts(t *testing.T) {
type testEndpoint struct {
endpointType portainer.EndpointType
status portainer.EndpointStatus
groupID portainer.EndpointGroupID
agentVersion string
containerEngine string
userTrusted bool
lastCheckInDate int64
}
currentVersion := portainer.APIVersion
tests := []struct {
name string
endpoints []testEndpoint
expectedCounts EnvironmentSummaryCountsResponse
}{
{
name: "all docker endpoints up",
endpoints: []testEndpoint{
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 2, Up: 2, Down: 0, Outdated: 0, Unassigned: 0,
// GroupID 2 has no matching EndpointGroup in the test store.
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 2}},
ByPlatformType: platformCounts{Docker: 2},
ByHealth: healthCounts{Up: 2},
},
},
{
name: "mix of up and down docker endpoints",
endpoints: []testEndpoint{
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: currentVersion},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 3, Up: 1, Down: 2, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 3}},
ByPlatformType: platformCounts{Docker: 3},
ByHealth: healthCounts{Down: 2, Up: 1},
},
},
{
name: "unassigned endpoints have groupID 1",
endpoints: []testEndpoint{
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 1, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 2, Up: 2, Down: 0, Outdated: 0, Unassigned: 1,
// GroupID 1 is the default "Unassigned" group; GroupID 2 has no match.
ByGroup: []groupCount{{GroupID: 1, GroupName: "Unassigned", Count: 1}, {GroupID: 2, GroupName: "", Count: 1}},
ByPlatformType: platformCounts{Docker: 2},
ByHealth: healthCounts{Up: 2},
},
},
{
name: "mixed scenario with docker and kubernetes types",
endpoints: []testEndpoint{
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusDown, groupID: 1, agentVersion: currentVersion},
{endpointType: portainer.KubernetesLocalEnvironment, status: portainer.EndpointStatusUp, groupID: 1, agentVersion: currentVersion},
{endpointType: portainer.AgentOnKubernetesEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: currentVersion},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 4, Up: 2, Down: 2, Outdated: 0, Unassigned: 2,
ByGroup: []groupCount{{GroupID: 1, GroupName: "Unassigned", Count: 2}, {GroupID: 2, GroupName: "", Count: 2}},
ByPlatformType: platformCounts{Docker: 2, Kubernetes: 2},
ByHealth: healthCounts{Down: 2, Up: 2},
},
},
{
name: "outdated endpoints count in both their connection bucket and Outdated",
endpoints: []testEndpoint{
{endpointType: portainer.AgentOnDockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: "2.0.0"},
{endpointType: portainer.AgentOnDockerEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: "2.0.0"},
{endpointType: portainer.AgentOnDockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 3, Up: 2, Down: 1, Outdated: 2, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 3}},
ByPlatformType: platformCounts{Docker: 3},
ByHealth: healthCounts{Outdated: 2, Up: 2, Down: 1},
},
},
{
name: "azure and podman endpoints counted in platform breakdown",
endpoints: []testEndpoint{
{endpointType: portainer.AzureEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion, containerEngine: portainer.ContainerEnginePodman},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 2, Up: 2, Down: 0, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 2}},
ByPlatformType: platformCounts{Azure: 1, Podman: 1},
ByHealth: healthCounts{Up: 2},
},
},
{
name: "untrusted edge endpoints are excluded from counts",
endpoints: []testEndpoint{
{endpointType: portainer.DockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion},
{endpointType: portainer.EdgeAgentOnDockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion, userTrusted: false},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 1, Up: 1, Down: 0, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 1}},
ByPlatformType: platformCounts{Docker: 1},
ByHealth: healthCounts{Up: 1},
},
},
{
name: "trusted edge endpoints classified by heartbeat, not stored status",
endpoints: []testEndpoint{
// Recent check-in: heartbeat alive → counted as Up + Heartbeat.
{endpointType: portainer.EdgeAgentOnDockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion, userTrusted: true, lastCheckInDate: time.Now().Unix()},
// Stored Up but never checked in: counted as Down.
{endpointType: portainer.EdgeAgentOnDockerEnvironment, status: portainer.EndpointStatusUp, groupID: 2, agentVersion: currentVersion, userTrusted: true, lastCheckInDate: 0},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 2, Up: 1, Down: 1, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 2}},
ByPlatformType: platformCounts{Docker: 2},
ByHealth: healthCounts{Up: 1, Down: 1, Heartbeat: 1},
},
},
{
name: "edge endpoint with unknown version and no check-in is not outdated",
endpoints: []testEndpoint{
{endpointType: portainer.EdgeAgentOnDockerEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: "", userTrusted: true, lastCheckInDate: 0},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 1, Up: 0, Down: 1, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 1}},
ByPlatformType: platformCounts{Docker: 1},
ByHealth: healthCounts{Down: 1},
},
},
{
name: "edge endpoint with unknown version but prior check-in is outdated",
endpoints: []testEndpoint{
{endpointType: portainer.EdgeAgentOnDockerEnvironment, status: portainer.EndpointStatusDown, groupID: 2, agentVersion: "", userTrusted: true, lastCheckInDate: time.Now().Add(-1 * time.Hour).Unix()},
},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 1, Up: 0, Down: 1, Outdated: 1, Unassigned: 0,
ByGroup: []groupCount{{GroupID: 2, GroupName: "", Count: 1}},
ByPlatformType: platformCounts{Docker: 1},
ByHealth: healthCounts{Down: 1, Outdated: 1},
},
},
{
name: "no endpoints returns all zeros",
endpoints: []testEndpoint{},
expectedCounts: EnvironmentSummaryCountsResponse{
Total: 0, Up: 0, Down: 0, Outdated: 0, Unassigned: 0,
ByGroup: []groupCount{},
ByPlatformType: platformCounts{},
ByHealth: healthCounts{},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
for i, ep := range tt.endpoints {
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "env",
Type: ep.endpointType,
Status: ep.status,
GroupID: ep.groupID,
ContainerEngine: ep.containerEngine,
UserTrusted: ep.userTrusted,
LastCheckInDate: ep.lastCheckInDate,
}
endpoint.Agent.Version = ep.agentVersion
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
require.NoError(t, err)
bouncer := testhelpers.NewTestRequestBouncer()
handler := NewHandler(bouncer)
handler.DataStore = store
req := httptest.NewRequest(http.MethodGet, "/endpoints/summary", nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)
restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
req = req.WithContext(restrictedCtx)
testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code, "expected 200 OK")
body, err := io.ReadAll(rr.Body)
require.NoError(t, err)
var counts EnvironmentSummaryCountsResponse
err = json.Unmarshal(body, &counts)
require.NoError(t, err)
assert.Equal(t, tt.expectedCounts.Total, counts.Total, "Total")
assert.Equal(t, tt.expectedCounts.Up, counts.Up, "Up")
assert.Equal(t, tt.expectedCounts.Down, counts.Down, "Down")
assert.Equal(t, tt.expectedCounts.Outdated, counts.Outdated, "Outdated")
assert.Equal(t, tt.expectedCounts.Unassigned, counts.Unassigned, "Unassigned")
assert.Equal(t, tt.expectedCounts.ByPlatformType, counts.ByPlatformType, "ByPlatformType")
assert.Equal(t, tt.expectedCounts.ByHealth, counts.ByHealth, "ByHealth")
// ByGroup is derived from map iteration so order is non-deterministic.
assert.ElementsMatch(t, tt.expectedCounts.ByGroup, counts.ByGroup, "ByGroup")
})
}
}
func TestSummaryCountsHealthBreakdown(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
currentVersion := portainer.APIVersion
type ep struct {
id portainer.EndpointID
name string
epType portainer.EndpointType
status portainer.EndpointStatus
lastCheckIn int64
trusted bool
agentVer string
}
for _, e := range []ep{
{1, "outdated-up", portainer.EdgeAgentOnDockerEnvironment, 0, time.Now().Unix(), true, "2.0.0"},
{2, "outdated-down", portainer.EdgeAgentOnDockerEnvironment, 0, time.Now().Add(-1 * time.Hour).Unix(), true, "2.0.0"},
{3, "up-current", portainer.DockerEnvironment, portainer.EndpointStatusUp, 0, false, currentVersion},
{4, "down-current", portainer.DockerEnvironment, portainer.EndpointStatusDown, 0, false, currentVersion},
} {
endpoint := &portainer.Endpoint{
ID: e.id,
Name: e.name,
Type: e.epType,
Status: e.status,
LastCheckInDate: e.lastCheckIn,
GroupID: 2,
UserTrusted: e.trusted,
}
endpoint.Agent.Version = e.agentVer
require.NoError(t, store.Endpoint().Create(endpoint))
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
require.NoError(t, err)
bouncer := testhelpers.NewTestRequestBouncer()
handler := NewHandler(bouncer)
handler.DataStore = store
req := httptest.NewRequest(http.MethodGet, "/endpoints/summary", nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)
restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
req = req.WithContext(restrictedCtx)
testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code)
body, err := io.ReadAll(rr.Body)
require.NoError(t, err)
var counts EnvironmentSummaryCountsResponse
require.NoError(t, json.Unmarshal(body, &counts))
assert.Equal(t, 4, counts.Total)
assert.Equal(t, 2, counts.Up, "outdated-up counts in both Up and Outdated")
assert.Equal(t, 2, counts.Down, "outdated-down counts in both Down and Outdated")
assert.Equal(t, 2, counts.Outdated)
assert.Equal(t, healthCounts{
Down: 2,
Outdated: 2,
Up: 2,
Heartbeat: 1,
}, counts.ByHealth)
}
func TestSummaryCountsHeartbeatRefreshedFromInMemoryMap(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
endpoint := &portainer.Endpoint{
ID: 1,
Name: "stale-edge",
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 2,
UserTrusted: true,
LastCheckInDate: time.Now().Add(-1 * time.Hour).Unix(),
}
endpoint.Agent.Version = portainer.APIVersion
require.NoError(t, store.Endpoint().Create(endpoint))
store.Endpoint().UpdateHeartbeat(1)
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
require.NoError(t, err)
bouncer := testhelpers.NewTestRequestBouncer()
handler := NewHandler(bouncer)
handler.DataStore = store
req := httptest.NewRequest(http.MethodGet, "/endpoints/summary", nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)
restrictedCtx := security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
req = req.WithContext(restrictedCtx)
testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code)
body, err := io.ReadAll(rr.Body)
require.NoError(t, err)
var counts EnvironmentSummaryCountsResponse
require.NoError(t, json.Unmarshal(body, &counts))
assert.Equal(t, 1, counts.Total)
assert.Equal(t, 1, counts.Up, "in-memory heartbeat should override stale DB LastCheckInDate")
assert.Equal(t, 0, counts.Down)
assert.Equal(t, healthCounts{Up: 1, Heartbeat: 1}, counts.ByHealth)
}
func TestResolveEndpointStatus(t *testing.T) {
settings := &portainer.Settings{EdgeAgentCheckinInterval: 60}
tests := []struct {
name string
endpoint *portainer.Endpoint
expectedStatus int
}{
{
name: "non-edge endpoint returns stored up status",
endpoint: &portainer.Endpoint{
Type: portainer.DockerEnvironment,
Status: portainer.EndpointStatusUp,
},
expectedStatus: statusUp,
},
{
name: "non-edge endpoint returns stored down status",
endpoint: &portainer.Endpoint{
Type: portainer.DockerEnvironment,
Status: portainer.EndpointStatusDown,
},
expectedStatus: statusDown,
},
{
name: "edge endpoint with recent check-in returns heartbeat",
endpoint: &portainer.Endpoint{
Type: portainer.EdgeAgentOnDockerEnvironment,
Status: portainer.EndpointStatusUp,
LastCheckInDate: time.Now().Unix(),
},
expectedStatus: statusHeartbeat,
},
{
name: "edge endpoint with stale check-in returns down regardless of stored status",
endpoint: &portainer.Endpoint{
Type: portainer.EdgeAgentOnDockerEnvironment,
Status: portainer.EndpointStatusUp,
LastCheckInDate: 0,
},
expectedStatus: statusDown,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expectedStatus, resolveEndpointStatus(tt.endpoint, settings))
})
}
}
func TestIsOutdated(t *testing.T) {
currentVersion := portainer.APIVersion
tests := []struct {
name string
version string
lastCheckInDate int64
expected bool
}{
{name: "empty version with prior check-in is outdated (old agent style)", version: "", lastCheckInDate: time.Now().Unix(), expected: true},
{name: "empty version with no check-in is not outdated (never connected)", version: "", lastCheckInDate: 0, expected: false},
{name: "old version is outdated", version: "2.0.0", expected: true},
{name: "v-prefixed old version is outdated", version: "v2.0.0", expected: true},
{name: "current version is not outdated", version: currentVersion, expected: false},
{name: "v-prefixed current version is not outdated", version: "v" + currentVersion, expected: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Empty-version logic only applies to edge endpoints; use EdgeAgentOnDocker for those cases.
epType := portainer.AgentOnDockerEnvironment
if tt.version == "" {
epType = portainer.EdgeAgentOnDockerEnvironment
}
ep := &portainer.Endpoint{Type: epType, LastCheckInDate: tt.lastCheckInDate}
ep.Agent.Version = tt.version
assert.Equal(t, tt.expected, isOutdated(ep))
})
}
}
func TestParseGroupCounts(t *testing.T) {
groups := []portainer.EndpointGroup{
{ID: 1, Name: "Unassigned"},
{ID: 3, Name: "Production"},
{ID: 2, Name: "Staging"},
}
tests := []struct {
name string
counts map[portainer.EndpointGroupID]int
expected []groupCount
}{
{
name: "empty counts returns empty slice",
counts: map[portainer.EndpointGroupID]int{},
expected: []groupCount{},
},
{
name: "results are sorted by GroupID ascending",
counts: map[portainer.EndpointGroupID]int{
3: 5,
1: 2,
2: 8,
},
expected: []groupCount{
{GroupID: 1, GroupName: "Unassigned", Count: 2},
{GroupID: 2, GroupName: "Staging", Count: 8},
{GroupID: 3, GroupName: "Production", Count: 5},
},
},
{
name: "group with no matching name gets empty string",
counts: map[portainer.EndpointGroupID]int{
99: 1,
},
expected: []groupCount{
{GroupID: 99, GroupName: "", Count: 1},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := parseGroupCounts(tt.counts, groups)
assert.Equal(t, tt.expected, got)
})
}
}
func TestCanonicalizeSemver(t *testing.T) {
assert.Equal(t, "v2.0.0", canonicalizeSemver("2.0.0"))
assert.Equal(t, "v2.0.0", canonicalizeSemver("v2.0.0"))
assert.Empty(t, canonicalizeSemver(""))
assert.Empty(t, canonicalizeSemver(" "))
}
@@ -0,0 +1,317 @@
package endpoints
import (
"cmp"
"net/http"
"reflect"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/client"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/pendingactions/handlers"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
type endpointUpdatePayload struct {
// Name that will be used to identify this environment(endpoint)
Name *string `example:"my-environment"`
// URL or IP address of a Docker host
URL *string `example:"docker.mydomain.tld:2375"`
// URL or IP address where exposed containers will be reachable.\
// Defaults to URL if not specified
PublicURL *string `example:"docker.mydomain.tld:2375"`
// GPUs information
Gpus []portainer.Pair
// Group identifier
GroupID *int `example:"1"`
// Require TLS to connect against this environment(endpoint)
TLS *bool `example:"true"`
// Skip server verification when using TLS
TLSSkipVerify *bool `example:"false"`
// Skip client verification when using TLS
TLSSkipClientVerify *bool `example:"false"`
// The status of the environment(endpoint) (1 - up, 2 - down)
Status *int `example:"1"`
// Azure application ID
AzureApplicationID *string `example:"eag7cdo9-o09l-9i83-9dO9-f0b23oe78db4"`
// Azure tenant ID
AzureTenantID *string `example:"34ddc78d-4fel-2358-8cc1-df84c8o839f5"`
// Azure authentication key
AzureAuthenticationKey *string `example:"cOrXoK/1D35w8YQ8nH1/8ZGwzz45JIYD5jxHKXEQknk="`
// List of tag identifiers to which this environment(endpoint) is associated
TagIDs []portainer.TagID `example:"1,2"`
UserAccessPolicies portainer.UserAccessPolicies
TeamAccessPolicies portainer.TeamAccessPolicies
// The check in interval for edge agent (in seconds)
EdgeCheckinInterval *int `example:"5"`
// Associated Kubernetes data
Kubernetes *portainer.KubernetesData
}
func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
return nil
}
// @id EndpointUpdate
// @summary Update an environment(endpoint)
// @description Update an environment(endpoint).
// @description **Access policy**: authenticated
// @security ApiKeyAuth
// @security jwt
// @tags endpoints
// @accept json
// @produce json
// @param id path int true "Environment(Endpoint) identifier"
// @param body body endpointUpdatePayload true "Environment(Endpoint) details"
// @success 200 {object} portainer.Endpoint "Success"
// @failure 400 "Invalid request"
// @failure 404 "Environment(Endpoint) not found"
// @failure 409 "Name is not unique"
// @failure 500 "Server error"
// @router /endpoints/{id} [put]
func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
var payload endpointUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
if payload.TLS != nil && *payload.TLS && endpointutils.IsEdgeEndpoint(endpoint) {
return httperror.BadRequest("TLS is not supported for Edge Agent environments", nil)
}
updateEndpointProxy := shouldReloadTLSConfiguration(endpoint, &payload)
if payload.Name != nil {
name := *payload.Name
if isUnique, err := handler.isNameUnique(name, endpoint.ID); err != nil {
return httperror.InternalServerError("Unable to check if name is unique", err)
} else if !isUnique {
return httperror.Conflict("Name is not unique", nil)
}
endpoint.Name = name
}
if payload.URL != nil && *payload.URL != endpoint.URL {
endpoint.URL = *payload.URL
updateEndpointProxy = true
}
if payload.Gpus != nil {
endpoint.Gpus = payload.Gpus
}
endpoint.PublicURL = *cmp.Or(payload.PublicURL, &endpoint.PublicURL)
endpoint.EdgeCheckinInterval = *cmp.Or(payload.EdgeCheckinInterval, &endpoint.EdgeCheckinInterval)
updateRelations := false
if payload.GroupID != nil {
groupID := portainer.EndpointGroupID(*payload.GroupID)
updateRelations = updateRelations || groupID != endpoint.GroupID
endpoint.GroupID = groupID
}
if payload.TagIDs != nil {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
tagsChanged, err := updateEnvironmentTags(tx, payload.TagIDs, endpoint.TagIDs, endpoint.ID)
if err != nil {
return err
}
endpoint.TagIDs = payload.TagIDs
updateRelations = updateRelations || tagsChanged
return nil
}); err != nil {
return httperror.InternalServerError("Unable to update environment tags", err)
}
}
updateAuthorizations := false
if payload.Kubernetes != nil {
if payload.Kubernetes.Configuration.RestrictDefaultNamespace !=
endpoint.Kubernetes.Configuration.RestrictDefaultNamespace {
updateAuthorizations = true
}
endpoint.Kubernetes = *payload.Kubernetes
}
if payload.UserAccessPolicies != nil && !reflect.DeepEqual(payload.UserAccessPolicies, endpoint.UserAccessPolicies) {
updateAuthorizations = true
endpoint.UserAccessPolicies = payload.UserAccessPolicies
}
if payload.TeamAccessPolicies != nil && !reflect.DeepEqual(payload.TeamAccessPolicies, endpoint.TeamAccessPolicies) {
updateAuthorizations = true
endpoint.TeamAccessPolicies = payload.TeamAccessPolicies
}
if payload.Status != nil {
switch *payload.Status {
case 1:
endpoint.Status = portainer.EndpointStatusUp
case 2:
endpoint.Status = portainer.EndpointStatusDown
}
}
if endpoint.Type == portainer.AzureEnvironment {
updateEndpointProxy = true
credentials := endpoint.AzureCredentials
if payload.AzureApplicationID != nil {
credentials.ApplicationID = *payload.AzureApplicationID
}
if payload.AzureTenantID != nil {
credentials.TenantID = *payload.AzureTenantID
}
if payload.AzureAuthenticationKey != nil {
credentials.AuthenticationKey = *payload.AzureAuthenticationKey
}
httpClient := client.NewHTTPClient()
if _, err := httpClient.ExecuteAzureAuthenticationRequest(&credentials); err != nil {
return httperror.InternalServerError("Unable to authenticate against Azure", err)
}
endpoint.AzureCredentials = credentials
}
if payload.TLS != nil {
folder := strconv.Itoa(endpointID)
if *payload.TLS {
endpoint.TLSConfig.TLS = true
if payload.TLSSkipVerify != nil {
endpoint.TLSConfig.TLSSkipVerify = *payload.TLSSkipVerify
if !*payload.TLSSkipVerify {
caCertPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileCA)
endpoint.TLSConfig.TLSCACertPath = caCertPath
} else {
endpoint.TLSConfig.TLSCACertPath = ""
if err := handler.FileService.DeleteTLSFile(folder, portainer.TLSFileCA); err != nil {
log.Warn().Err(err).Msg("Unable to remove CA cert from disk")
}
}
}
if payload.TLSSkipClientVerify != nil {
if !*payload.TLSSkipClientVerify {
certPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileCert)
endpoint.TLSConfig.TLSCertPath = certPath
keyPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileKey)
endpoint.TLSConfig.TLSKeyPath = keyPath
} else {
endpoint.TLSConfig.TLSCertPath = ""
if err := handler.FileService.DeleteTLSFile(folder, portainer.TLSFileCert); err != nil {
log.Warn().Err(err).Msg("Unable to remove TLS cert from disk")
}
endpoint.TLSConfig.TLSKeyPath = ""
if err := handler.FileService.DeleteTLSFile(folder, portainer.TLSFileKey); err != nil {
log.Warn().Err(err).Msg("Unable to remove TLS key from disk")
}
}
}
} else {
endpoint.TLSConfig.TLS = false
endpoint.TLSConfig.TLSSkipVerify = false
endpoint.TLSConfig.TLSCACertPath = ""
endpoint.TLSConfig.TLSCertPath = ""
endpoint.TLSConfig.TLSKeyPath = ""
if err := handler.FileService.DeleteTLSFiles(folder); err != nil {
return httperror.InternalServerError("Unable to remove TLS files from disk", err)
}
}
isStandardKubeAgent := !endpointutils.IsLocalEndpoint(endpoint) && endpointutils.IsKubernetesEndpoint(endpoint) && !endpointutils.IsEdgeEndpoint(endpoint)
if isStandardKubeAgent {
endpoint.TLSConfig.TLS = true
endpoint.TLSConfig.TLSSkipVerify = true
}
}
if updateEndpointProxy {
handler.ProxyManager.DeleteEndpointProxy(endpoint.ID)
if _, err := handler.ProxyManager.CreateAndRegisterEndpointProxy(endpoint); err != nil {
return httperror.InternalServerError("Unable to register HTTP proxy for the environment", err)
}
}
if updateAuthorizations && endpointutils.IsKubernetesEndpoint(endpoint) {
if err := handler.AuthorizationService.CleanNAPWithOverridePolicies(handler.DataStore, endpoint, nil); err != nil {
log.Warn().Err(err).Msgf("Unable to clean NAP with override policies for endpoint (%d). Will try to update when endpoint is online.", endpoint.ID)
if err := handler.PendingActionsService.Create(handler.DataStore, handlers.NewCleanNAPWithOverridePolicies(endpoint.ID, nil)); err != nil {
log.Warn().Err(err).Msg("unable to schedule pending action to clean NAP with override policies")
}
}
}
if err := handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
return httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
if updateRelations {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.updateEdgeRelations(tx, endpoint)
}); err != nil {
return httperror.InternalServerError("Unable to update environment relations", err)
}
}
if err := handler.SnapshotService.FillSnapshotData(endpoint, true); err != nil {
return httperror.InternalServerError("Unable to add snapshot data", err)
}
return response.JSON(w, endpoint)
}
func shouldReloadTLSConfiguration(endpoint *portainer.Endpoint, payload *endpointUpdatePayload) bool {
// If we change anything in the tls config then we need to reload the proxy
if payload.TLS != nil && endpoint.TLSConfig.TLS != *payload.TLS {
return true
}
// When updating Docker API environment, as long as TLS is true and TLSSkipVerify is false,
// we assume that new TLS files have been uploaded and we need to reload the TLS configuration.
if endpoint.Type != portainer.DockerEnvironment ||
(payload.URL != nil && !strings.HasPrefix(*payload.URL, "tcp://")) ||
payload.TLS == nil || !*payload.TLS {
return false
}
if payload.TLSSkipVerify != nil && !*payload.TLSSkipVerify {
return true
}
return payload.TLSSkipClientVerify != nil && !*payload.TLSSkipClientVerify
}
@@ -0,0 +1,109 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
)
type endpointUpdateRelationsPayload struct {
Relations map[portainer.EndpointID]struct {
EdgeGroups []portainer.EdgeGroupID
Tags []portainer.TagID
Group portainer.EndpointGroupID
}
}
func (payload *endpointUpdateRelationsPayload) Validate(r *http.Request) error {
for eID := range payload.Relations {
if eID == 0 {
return errors.New("Missing environment identifier")
}
}
return nil
}
// @id EndpointUpdateRelations
// @summary Update relations for a list of environments
// @description Update relations for a list of environments
// @description Edge groups, tags and environment group can be updated.
// @description
// @description **Access policy**: administrator
// @tags endpoints
// @security jwt
// @accept json
// @param body body endpointUpdateRelationsPayload true "Environment relations data"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 401 "Unauthorized"
// @failure 404 "Not found"
// @failure 500 "Server error"
// @router /endpoints/relations [put]
func (handler *Handler) updateRelations(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
payload, err := request.GetPayload[endpointUpdateRelationsPayload](r)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
for environmentID, relationPayload := range payload.Relations {
endpoint, err := tx.Endpoint().Endpoint(environmentID)
if err != nil {
return errors.WithMessage(err, "Unable to find an environment with the specified identifier inside the database")
}
updateRelations := false
if relationPayload.Group != 0 {
groupIDChanged := endpoint.GroupID != relationPayload.Group
endpoint.GroupID = relationPayload.Group
updateRelations = updateRelations || groupIDChanged
}
if relationPayload.Tags != nil {
tagsChanged, err := updateEnvironmentTags(tx, relationPayload.Tags, endpoint.TagIDs, endpoint.ID)
if err != nil {
return errors.WithMessage(err, "Unable to update environment tags")
}
endpoint.TagIDs = relationPayload.Tags
updateRelations = updateRelations || tagsChanged
}
if relationPayload.EdgeGroups != nil {
edgeGroupsChanged, err := updateEnvironmentEdgeGroups(tx, relationPayload.EdgeGroups, endpoint.ID)
if err != nil {
return errors.WithMessage(err, "Unable to update environment edge groups")
}
updateRelations = updateRelations || edgeGroupsChanged
}
if updateRelations {
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
return errors.WithMessage(err, "Unable to update environment")
}
if err := handler.updateEdgeRelations(tx, endpoint); err != nil {
return errors.WithMessage(err, "Unable to update environment relations")
}
}
}
return nil
})
if err != nil {
return httperror.InternalServerError("Unable to update environment relations", err)
}
return response.Empty(w)
}
@@ -0,0 +1,67 @@
package endpoints
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_endpointPut_TLSRejectedForEdgeEndpoint(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, true)
h := NewHandler(testhelpers.NewTestRequestBouncer())
h.DataStore = store
testCases := []struct {
name string
endpointType portainer.EndpointType
}{
{
name: "edge agent on docker rejects TLS",
endpointType: portainer.EdgeAgentOnDockerEnvironment,
},
{
name: "edge agent on kubernetes rejects TLS",
endpointType: portainer.EdgeAgentOnKubernetesEnvironment,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
endpointID := portainer.EndpointID(store.Endpoint().GetNextIdentifier())
err := store.Endpoint().Create(&portainer.Endpoint{
ID: endpointID,
Type: tc.endpointType,
})
require.NoError(t, err)
payload := &endpointUpdatePayload{TLS: new(true)}
bodyJSON, err := json.Marshal(payload)
require.NoError(t, err)
url := fmt.Sprintf("/endpoints/%d", endpointID)
req := httptest.NewRequest(http.MethodPut, url, bytes.NewBuffer(bodyJSON))
rctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: portainer.AdministratorRole})
req = req.WithContext(rctx)
req.Header.Set("Content-Type", "application/json")
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
assert.Equal(t, http.StatusBadRequest, rr.Code)
})
}
}
+748
View File
@@ -0,0 +1,748 @@
package endpoints
import (
"fmt"
"net/http"
"slices"
"strconv"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
"github.com/portainer/portainer/api/set"
"github.com/portainer/portainer/api/slicesx"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/pkg/errors"
)
type EnvironmentsQuery struct {
search string
groupIds []portainer.EndpointGroupID
status []portainer.EndpointStatus
types []portainer.EndpointType
platformTypes []portainer.PlatformType
outdated bool
excludeGroupIds []portainer.EndpointGroupID
tagIds []portainer.TagID
tagsPartialMatch bool
endpointIds []portainer.EndpointID
excludeIds []portainer.EndpointID
agentVersions []string
// if edgeAsync not nil, will filter edge endpoints based on this value
edgeAsync *bool
edgeDeviceUntrusted bool
edgeCheckInPassedSeconds int
excludeSnapshots bool
name string
edgeStackId portainer.EdgeStackID
edgeStackStatus *portainer.EdgeStackStatusType
edgeGroupIds []portainer.EdgeGroupID
excludeEdgeGroupIds []portainer.EdgeGroupID
}
func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
search, _ := request.RetrieveQueryParameter(r, "search", true)
if search != "" {
search = strings.ToLower(search)
}
status, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointStatus](r, "status")
if err != nil {
return EnvironmentsQuery{}, err
}
groupIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointGroupID](r, "groupIds")
if err != nil {
return EnvironmentsQuery{}, err
}
endpointTypes, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointType](r, "types")
if err != nil {
return EnvironmentsQuery{}, err
}
platformTypes, err := request.RetrieveNumberArrayQueryParameter[portainer.PlatformType](r, "platformTypes")
if err != nil {
return EnvironmentsQuery{}, err
}
tagIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.TagID](r, "tagIds")
if err != nil {
return EnvironmentsQuery{}, err
}
tagsPartialMatch, _ := request.RetrieveBooleanQueryParameter(r, "tagsPartialMatch", true)
endpointIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointID](r, "endpointIds")
if err != nil {
return EnvironmentsQuery{}, err
}
excludeIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointID](r, "excludeIds")
if err != nil {
return EnvironmentsQuery{}, err
}
excludeGroupIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.EndpointGroupID](r, "excludeGroupIds")
if err != nil {
return EnvironmentsQuery{}, err
}
edgeGroupIDs, err := request.RetrieveNumberArrayQueryParameter[portainer.EdgeGroupID](r, "edgeGroupIds")
if err != nil {
return EnvironmentsQuery{}, err
}
excludeEdgeGroupIds, err := request.RetrieveNumberArrayQueryParameter[portainer.EdgeGroupID](r, "excludeEdgeGroupIds")
if err != nil {
return EnvironmentsQuery{}, err
}
agentVersions := request.RetrieveArrayQueryParameter(r, "agentVersions")
outdated, _ := request.RetrieveBooleanQueryParameter(r, "outdated", true)
name, _ := request.RetrieveQueryParameter(r, "name", true)
var edgeAsync *bool
edgeAsyncParam, _ := request.RetrieveQueryParameter(r, "edgeAsync", true)
if edgeAsyncParam != "" {
edgeAsync = new(edgeAsyncParam == "true")
}
edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true)
excludeSnapshots, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshots", true)
edgeCheckInPassedSeconds, _ := request.RetrieveNumericQueryParameter(r, "edgeCheckInPassedSeconds", true)
edgeStackId, _ := request.RetrieveNumericQueryParameter(r, "edgeStackId", true)
edgeStackStatus, err := getEdgeStackStatusParam(r)
if err != nil {
return EnvironmentsQuery{}, err
}
return EnvironmentsQuery{
search: search,
types: endpointTypes,
platformTypes: platformTypes,
tagIds: tagIDs,
endpointIds: endpointIDs,
excludeIds: excludeIDs,
excludeGroupIds: excludeGroupIDs,
tagsPartialMatch: tagsPartialMatch,
groupIds: groupIDs,
status: status,
edgeAsync: edgeAsync,
edgeDeviceUntrusted: edgeDeviceUntrusted,
excludeSnapshots: excludeSnapshots,
name: name,
agentVersions: agentVersions,
outdated: outdated,
edgeCheckInPassedSeconds: edgeCheckInPassedSeconds,
edgeStackId: portainer.EdgeStackID(edgeStackId),
edgeStackStatus: edgeStackStatus,
edgeGroupIds: edgeGroupIDs,
excludeEdgeGroupIds: excludeEdgeGroupIds,
}, nil
}
func (handler *Handler) filterEndpointsByQuery(
filteredEndpoints []portainer.Endpoint,
query EnvironmentsQuery,
groups []portainer.EndpointGroup,
edgeGroups []portainer.EdgeGroup,
settings *portainer.Settings,
context *security.RestrictedRequestContext,
) ([]portainer.Endpoint, int, error) {
totalAvailableEndpoints := len(filteredEndpoints)
if len(query.endpointIds) > 0 {
endpointIDs := roar.FromSlice(query.endpointIds)
filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, endpointIDs)
}
if len(query.excludeIds) > 0 {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return !slices.Contains(query.excludeIds, endpoint.ID)
})
}
if len(query.excludeGroupIds) > 0 {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return !slices.Contains(query.excludeGroupIds, endpoint.GroupID)
})
}
if len(query.groupIds) > 0 {
filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, query.groupIds)
}
if len(query.edgeGroupIds) > 0 {
filteredEndpoints, edgeGroups = filterEndpointsByEdgeGroupIDs(filteredEndpoints, edgeGroups, query.edgeGroupIds)
}
if len(query.excludeEdgeGroupIds) > 0 {
filteredEndpoints, edgeGroups = filterEndpointsByExcludeEdgeGroupIDs(filteredEndpoints, edgeGroups, query.excludeEdgeGroupIds)
}
if query.name != "" {
filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
}
// filter async edge environments
if query.edgeAsync != nil {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
return endpoint.Edge.AsyncMode == *query.edgeAsync
})
}
// filter edge environments by trusted/untrusted
// only portainer admins are allowed to see untrusted environments
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
if query.edgeDeviceUntrusted {
return !endpoint.UserTrusted && context.IsAdmin
}
return endpoint.UserTrusted == !query.edgeDeviceUntrusted
})
if query.edgeCheckInPassedSeconds > 0 {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
// ignore non-edge endpoints
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
// filter out endpoints that have never checked in
if endpoint.LastCheckInDate == 0 {
return false
}
return time.Now().Unix()-endpoint.LastCheckInDate < int64(query.edgeCheckInPassedSeconds)
})
}
if len(query.status) > 0 {
filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings)
}
if query.search != "" {
tags, err := handler.DataStore.Tag().ReadAll()
if err != nil {
return nil, 0, errors.WithMessage(err, "Unable to retrieve tags from the database")
}
tagsMap := make(map[portainer.TagID]string, len(tags))
for _, tag := range tags {
tagsMap[tag.ID] = tag.Name
}
filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, groups, edgeGroups, tagsMap, query.search)
}
if len(query.types) > 0 {
filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, query.types)
}
if len(query.platformTypes) > 0 {
filteredEndpoints = filterEndpointsByPlatform(filteredEndpoints, query.platformTypes)
}
if len(query.tagIds) > 0 {
filteredEndpoints = filteredEndpointsByTags(filteredEndpoints, query.tagIds, groups, query.tagsPartialMatch)
}
if len(query.agentVersions) > 0 {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return !endpointutils.IsAgentEndpoint(&endpoint) || slices.Contains(query.agentVersions, endpoint.Agent.Version)
})
}
if query.outdated {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return isOutdated(&endpoint)
})
}
if query.edgeStackId != 0 {
f, err := filterEndpointsByEdgeStack(filteredEndpoints, query.edgeStackId, query.edgeStackStatus, handler.DataStore)
if err != nil {
return nil, 0, err
}
filteredEndpoints = f
}
return filteredEndpoints, totalAvailableEndpoints, nil
}
func endpointStatusInStackMatchesFilter(stackStatus *portainer.EdgeStackStatusForEnv, statusFilter portainer.EdgeStackStatusType) bool {
// consider that if the env has no status in the stack it is in Pending state
if statusFilter == portainer.EdgeStackStatusPending {
return stackStatus == nil || len(stackStatus.Status) == 0
}
if stackStatus == nil {
return false
}
return slices.ContainsFunc(stackStatus.Status, func(s portainer.EdgeStackDeploymentStatus) bool {
return s.Type == statusFilter
})
}
func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId portainer.EdgeStackID, statusFilter *portainer.EdgeStackStatusType, datastore dataservices.DataStore) ([]portainer.Endpoint, error) {
var filteredEndpoints []portainer.Endpoint
if err := datastore.ViewTx(func(tx dataservices.DataStoreTx) error {
stack, err := tx.EdgeStack().EdgeStack(edgeStackId)
if err != nil {
return errors.WithMessage(err, "Unable to retrieve edge stack from the database")
}
envIds := roar.Roar[portainer.EndpointID]{}
for _, edgeGroupId := range stack.EdgeGroups {
edgeGroup, err := tx.EdgeGroup().Read(edgeGroupId)
if err != nil {
return errors.WithMessage(err, "Unable to retrieve edge group from the database")
}
if edgeGroup.Dynamic {
endpointIDs, err := endpointutils.GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
if err != nil {
return errors.WithMessage(err, "Unable to retrieve environments and environment groups for Edge group")
}
edgeGroup.EndpointIDs = roar.FromSlice(endpointIDs)
}
envIds.Union(edgeGroup.EndpointIDs)
}
filteredEnvIds := roar.Roar[portainer.EndpointID]{}
filteredEnvIds.Union(envIds)
if statusFilter != nil {
var innerErr error
envIds.Iterate(func(envId portainer.EndpointID) bool {
edgeStackStatus, err := tx.EdgeStackStatus().Read(edgeStackId, envId)
if err != nil && !dataservices.IsErrObjectNotFound(err) {
innerErr = errors.WithMessagef(err, "Unable to retrieve edge stack status for environment %d", envId)
return false
}
if !endpointStatusInStackMatchesFilter(edgeStackStatus, *statusFilter) {
filteredEnvIds.Remove(envId)
}
return true
})
if innerErr != nil {
return innerErr
}
}
filteredEndpoints = filteredEndpointsByIds(endpoints, filteredEnvIds)
return nil
}); err != nil {
return nil, err
}
return filteredEndpoints, nil
}
func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []portainer.EndpointGroupID) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if slices.Contains(endpointGroupIDs, endpoint.GroupID) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func filterEndpointsByEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGroups []portainer.EdgeGroup, edgeGroupIDs []portainer.EdgeGroupID) ([]portainer.Endpoint, []portainer.EdgeGroup) {
edgeGroupIDFilterSet := make(map[portainer.EdgeGroupID]struct{}, len(edgeGroupIDs))
for _, id := range edgeGroupIDs {
edgeGroupIDFilterSet[id] = struct{}{}
}
n := 0
for _, edgeGroup := range edgeGroups {
if _, exists := edgeGroupIDFilterSet[edgeGroup.ID]; exists {
edgeGroups[n] = edgeGroup
n++
}
}
edgeGroups = edgeGroups[:n]
endpointIDSet := roar.Roar[portainer.EndpointID]{}
for _, edgeGroup := range edgeGroups {
endpointIDSet.Union(edgeGroup.EndpointIDs)
}
n = 0
for _, endpoint := range endpoints {
if endpointIDSet.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n], edgeGroups
}
func filterEndpointsByExcludeEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGroups []portainer.EdgeGroup, excludeEdgeGroupIds []portainer.EdgeGroupID) ([]portainer.Endpoint, []portainer.EdgeGroup) {
excludeEdgeGroupIDSet := make(map[portainer.EdgeGroupID]struct{}, len(excludeEdgeGroupIds))
for _, id := range excludeEdgeGroupIds {
excludeEdgeGroupIDSet[id] = struct{}{}
}
n := 0
excludeEndpointIDSet := roar.Roar[portainer.EndpointID]{}
for _, edgeGroup := range edgeGroups {
if _, ok := excludeEdgeGroupIDSet[edgeGroup.ID]; ok {
excludeEndpointIDSet.Union(edgeGroup.EndpointIDs)
} else {
edgeGroups[n] = edgeGroup
n++
}
}
edgeGroups = edgeGroups[:n]
n = 0
for _, endpoint := range endpoints {
if !excludeEndpointIDSet.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n], edgeGroups
}
func filterEndpointsBySearchCriteria(
endpoints []portainer.Endpoint,
endpointGroups []portainer.EndpointGroup,
edgeGroups []portainer.EdgeGroup,
tagsMap map[portainer.TagID]string,
searchCriteria string,
) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if endpointMatchSearchCriteria(&endpoint, tagsMap, searchCriteria) {
endpoints[n] = endpoint
n++
continue
}
if endpointGroupMatchSearchCriteria(&endpoint, endpointGroups, tagsMap, searchCriteria) {
endpoints[n] = endpoint
n++
continue
}
if edgeGroupMatchSearchCriteria(&endpoint, edgeGroups, searchCriteria, endpointGroups) {
endpoints[n] = endpoint
n++
continue
}
}
return endpoints[:n]
}
func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portainer.EndpointStatus, settings *portainer.Settings) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
status := endpoint.Status
if endpointutils.IsEdgeEndpoint(&endpoint) {
isCheckValid := false
edgeCheckinInterval := endpoint.EdgeCheckinInterval
if edgeCheckinInterval == 0 {
edgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
if endpoint.Edge.AsyncMode {
edgeCheckinInterval = getShortestAsyncInterval(&endpoint, settings)
}
if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 {
isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd)
}
status = portainer.EndpointStatusDown // Offline
if isCheckValid {
status = portainer.EndpointStatusUp // Online
}
}
if slices.Contains(statuses, status) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) {
return true
}
if strings.Contains(strings.ToLower(endpoint.URL), searchCriteria) {
return true
}
if endpoint.Status == portainer.EndpointStatusUp && searchCriteria == "up" {
return true
} else if endpoint.Status == portainer.EndpointStatusDown && searchCriteria == "down" {
return true
}
for _, tagID := range endpoint.TagIDs {
if strings.Contains(strings.ToLower(tagsMap[tagID]), searchCriteria) {
return true
}
}
return false
}
func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
for _, group := range endpointGroups {
if group.ID != endpoint.GroupID {
continue
}
if strings.Contains(strings.ToLower(group.Name), searchCriteria) {
return true
}
for _, tagID := range group.TagIDs {
if strings.Contains(strings.ToLower(tagsMap[tagID]), searchCriteria) {
return true
}
}
}
return false
}
// search endpoint's related edgegroups
func edgeGroupMatchSearchCriteria(
endpoint *portainer.Endpoint,
edgeGroups []portainer.EdgeGroup,
searchCriteria string,
endpointGroups []portainer.EndpointGroup,
) bool {
for _, edgeGroup := range edgeGroups {
relatedEndpointIDs := edge.EdgeGroupRelatedEndpoints(&edgeGroup, []portainer.Endpoint{*endpoint}, endpointGroups)
for _, endpointID := range relatedEndpointIDs {
if endpointID == endpoint.ID {
if strings.Contains(strings.ToLower(edgeGroup.Name), searchCriteria) {
return true
}
}
}
}
return false
}
func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []portainer.EndpointType) []portainer.Endpoint {
typeSet := set.ToSet(endpointTypes)
return slicesx.Filter(endpoints, func(e portainer.Endpoint) bool {
return typeSet[e.Type]
})
}
func filterEndpointsByPlatform(endpoints []portainer.Endpoint, platformTypes []portainer.PlatformType) []portainer.Endpoint {
typeSet := set.ToSet(platformTypes)
return slicesx.Filter(endpoints, func(e portainer.Endpoint) bool {
return typeSet[endpointutils.EndpointPlatformType(&e)]
})
}
func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
endpointGroup := getEndpointGroup(endpoint.GroupID, endpointGroups)
endpointMatched := false
if partialMatch {
endpointMatched = endpointPartialMatchTags(endpoint, endpointGroup, tagIDs)
} else {
endpointMatched = endpointFullMatchTags(endpoint, endpointGroup, tagIDs)
}
if endpointMatched {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func endpointPartialMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
tagSet := make(map[portainer.TagID]bool, len(tagIDs))
for _, tagID := range tagIDs {
tagSet[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if tagSet[tagID] {
return true
}
}
for _, tagID := range endpointGroup.TagIDs {
if tagSet[tagID] {
return true
}
}
return false
}
func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
missingTags := make(map[portainer.TagID]bool)
for _, tagID := range tagIDs {
missingTags[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
for _, tagID := range endpointGroup.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
return len(missingTags) == 0
}
func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids roar.Roar[portainer.EndpointID]) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if ids.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint {
if name == "" {
return endpoints
}
n := 0
for _, endpoint := range endpoints {
if endpoint.Name == name {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func filter(endpoints []portainer.Endpoint, predicate func(endpoint portainer.Endpoint) bool) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if predicate(endpoint) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func getEdgeStackStatusParam(r *http.Request) (*portainer.EdgeStackStatusType, error) {
edgeStackStatusQuery, _ := request.RetrieveQueryParameter(r, "edgeStackStatus", true)
if edgeStackStatusQuery == "" {
return nil, nil
}
edgeStackStatusNumber, err := strconv.Atoi(edgeStackStatusQuery)
edgeStackStatus := portainer.EdgeStackStatusType(edgeStackStatusNumber)
if err != nil {
return nil, fmt.Errorf("failed parsing edgeStackStatus: %w", err)
}
if !slices.Contains([]portainer.EdgeStackStatusType{
portainer.EdgeStackStatusPending,
portainer.EdgeStackStatusDeploymentReceived,
portainer.EdgeStackStatusError,
portainer.EdgeStackStatusAcknowledged,
portainer.EdgeStackStatusRemoved,
portainer.EdgeStackStatusRemoteUpdateSuccess,
portainer.EdgeStackStatusImagesPulled,
portainer.EdgeStackStatusRunning,
portainer.EdgeStackStatusDeploying,
portainer.EdgeStackStatusRemoving,
portainer.EdgeStackStatusCompleted,
}, edgeStackStatus) {
return nil, errors.New("invalid edgeStackStatus parameter")
}
return &edgeStackStatus, nil
}
func getShortestAsyncInterval(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
const edgeIntervalUseDefault = -1
pingInterval := endpoint.Edge.PingInterval
if pingInterval == edgeIntervalUseDefault {
pingInterval = settings.Edge.PingInterval
}
snapshotInterval := endpoint.Edge.SnapshotInterval
if snapshotInterval == edgeIntervalUseDefault {
snapshotInterval = settings.Edge.SnapshotInterval
}
commandInterval := endpoint.Edge.CommandInterval
if commandInterval == edgeIntervalUseDefault {
commandInterval = settings.Edge.CommandInterval
}
return min(pingInterval, snapshotInterval, commandInterval)
}
+754
View File
@@ -0,0 +1,754 @@
package endpoints
import (
"net/http"
"net/url"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/portainer/portainer/api/slicesx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type filterTest struct {
title string
expected []portainer.EndpointID
query EnvironmentsQuery
}
func Test_Filter_AgentVersion(t *testing.T) {
t.Parallel()
version1Endpoint := portainer.Endpoint{ID: 1, GroupID: 1,
Type: portainer.AgentOnDockerEnvironment,
Agent: portainer.EnvironmentAgentData{Version: "1.0.0"}}
version2Endpoint := portainer.Endpoint{ID: 2, GroupID: 1,
Type: portainer.AgentOnDockerEnvironment,
Agent: portainer.EnvironmentAgentData{Version: "2.0.0"}}
noVersionEndpoint := portainer.Endpoint{ID: 3, GroupID: 1,
Type: portainer.AgentOnDockerEnvironment,
}
notAgentEnvironments := portainer.Endpoint{ID: 4, Type: portainer.DockerEnvironment, GroupID: 1}
endpoints := []portainer.Endpoint{
version1Endpoint,
version2Endpoint,
noVersionEndpoint,
notAgentEnvironments,
}
handler := setupFilterTest(t, endpoints)
tests := []filterTest{
{
"should show version 1 endpoints",
[]portainer.EndpointID{version1Endpoint.ID},
EnvironmentsQuery{
agentVersions: []string{version1Endpoint.Agent.Version},
types: []portainer.EndpointType{portainer.AgentOnDockerEnvironment},
},
},
{
"should show version 2 endpoints",
[]portainer.EndpointID{version2Endpoint.ID},
EnvironmentsQuery{
agentVersions: []string{version2Endpoint.Agent.Version},
types: []portainer.EndpointType{portainer.AgentOnDockerEnvironment},
},
},
{
"should show version 1 and 2 endpoints",
[]portainer.EndpointID{version2Endpoint.ID, version1Endpoint.ID},
EnvironmentsQuery{
agentVersions: []string{version2Endpoint.Agent.Version, version1Endpoint.Agent.Version},
types: []portainer.EndpointType{portainer.AgentOnDockerEnvironment},
},
},
}
runTests(tests, t, handler, endpoints)
}
func Test_Filter_edgeFilter(t *testing.T) {
t.Parallel()
trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
endpoints := []portainer.Endpoint{
trustedEdgeAsync,
untrustedEdgeAsync,
regularUntrustedEdgeStandard,
regularTrustedEdgeStandard,
regularEndpoint,
}
handler := setupFilterTest(t, endpoints)
tests := []filterTest{
{
"should show all edge endpoints except of the untrusted edge",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID},
EnvironmentsQuery{
types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment},
},
},
{
"should show only trusted edge devices and other regular endpoints",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeAsync: new(true),
},
},
{
"should show only untrusted edge devices and other regular endpoints",
[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeAsync: new(true),
edgeDeviceUntrusted: true,
},
},
{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
EnvironmentsQuery{
edgeAsync: new(false),
},
},
}
runTests(tests, t, handler, endpoints)
}
func Test_Filter_excludeIDs(t *testing.T) {
t.Parallel()
ids := []portainer.EndpointID{1, 2, 3, 4, 5, 6, 7, 8, 9}
environments := slicesx.Map(ids, func(id portainer.EndpointID) portainer.Endpoint {
return portainer.Endpoint{ID: id, GroupID: 1, Type: portainer.DockerEnvironment}
})
handler := setupFilterTest(t, environments)
tests := []filterTest{
{
title: "should exclude IDs 2,5,8",
expected: []portainer.EndpointID{1, 3, 4, 6, 7, 9},
query: EnvironmentsQuery{
excludeIds: []portainer.EndpointID{2, 5, 8},
},
},
}
runTests(tests, t, handler, environments)
}
func Test_Filter_excludeGroupIDs(t *testing.T) {
t.Parallel()
groupA := portainer.EndpointGroupID(10)
groupB := portainer.EndpointGroupID(20)
groupC := portainer.EndpointGroupID(30)
endpoints := []portainer.Endpoint{
{ID: 1, GroupID: groupA, Type: portainer.DockerEnvironment},
{ID: 2, GroupID: groupA, Type: portainer.DockerEnvironment},
{ID: 3, GroupID: groupB, Type: portainer.DockerEnvironment},
{ID: 4, GroupID: groupB, Type: portainer.DockerEnvironment},
{ID: 5, GroupID: groupC, Type: portainer.DockerEnvironment},
}
handler := setupFilterTest(t, endpoints)
tests := []filterTest{
{
title: "should exclude endpoints in groupA",
expected: []portainer.EndpointID{3, 4, 5},
query: EnvironmentsQuery{
excludeGroupIds: []portainer.EndpointGroupID{groupA},
},
},
{
title: "should exclude endpoints in groupA and groupB",
expected: []portainer.EndpointID{5},
query: EnvironmentsQuery{
excludeGroupIds: []portainer.EndpointGroupID{groupA, groupB},
},
},
{
title: "should return all endpoints when excludeGroupIds is empty",
expected: []portainer.EndpointID{1, 2, 3, 4, 5},
query: EnvironmentsQuery{},
},
}
runTests(tests, t, handler, endpoints)
}
func BenchmarkFilterEndpointsBySearchCriteria_PartialMatch(b *testing.B) {
n := 10000
endpointIDs := []portainer.EndpointID{}
endpoints := []portainer.Endpoint{}
for i := range n {
endpoints = append(endpoints, portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "endpoint-" + strconv.Itoa(i+1),
GroupID: 1,
TagIDs: []portainer.TagID{1},
Type: portainer.EdgeAgentOnDockerEnvironment,
})
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
endpointGroups := []portainer.EndpointGroup{}
edgeGroups := []portainer.EdgeGroup{}
for i := range 1000 {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
EndpointIDs: roar.FromSlice(endpointIDs),
Dynamic: true,
TagIDs: []portainer.TagID{1, 2, 3},
PartialMatch: true,
})
}
tagsMap := map[portainer.TagID]string{}
for i := range 10 {
tagsMap[portainer.TagID(i+1)] = "tag-" + strconv.Itoa(i+1)
}
searchString := "edge-group"
for b.Loop() {
e := filterEndpointsBySearchCriteria(endpoints, endpointGroups, edgeGroups, tagsMap, searchString)
if len(e) != n {
b.FailNow()
}
}
}
func BenchmarkFilterEndpointsBySearchCriteria_FullMatch(b *testing.B) {
n := 10000
endpointIDs := []portainer.EndpointID{}
endpoints := []portainer.Endpoint{}
for i := range n {
endpoints = append(endpoints, portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "endpoint-" + strconv.Itoa(i+1),
GroupID: 1,
TagIDs: []portainer.TagID{1, 2, 3},
Type: portainer.EdgeAgentOnDockerEnvironment,
})
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
endpointGroups := []portainer.EndpointGroup{}
edgeGroups := []portainer.EdgeGroup{}
for i := range 1000 {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
EndpointIDs: roar.FromSlice(endpointIDs),
Dynamic: true,
TagIDs: []portainer.TagID{1},
})
}
tagsMap := map[portainer.TagID]string{}
for i := range 10 {
tagsMap[portainer.TagID(i+1)] = "tag-" + strconv.Itoa(i+1)
}
searchString := "edge-group"
for b.Loop() {
e := filterEndpointsBySearchCriteria(endpoints, endpointGroups, edgeGroups, tagsMap, searchString)
require.Len(b, e, n)
}
}
func runTests(tests []filterTest, t *testing.T, handler *Handler, endpoints []portainer.Endpoint) {
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {
runTest(t, test, handler, append([]portainer.Endpoint{}, endpoints...))
})
}
}
func runTest(t *testing.T, test filterTest, handler *Handler, endpoints []portainer.Endpoint) {
is := assert.New(t)
filteredEndpoints, _, err := handler.filterEndpointsByQuery(
endpoints,
test.query,
[]portainer.EndpointGroup{},
[]portainer.EdgeGroup{},
&portainer.Settings{},
&security.RestrictedRequestContext{IsAdmin: true},
)
require.NoError(t, err)
is.Len(filteredEndpoints, len(test.expected))
respIds := []portainer.EndpointID{}
for _, endpoint := range filteredEndpoints {
respIds = append(respIds, endpoint.ID)
}
is.ElementsMatch(test.expected, respIds)
}
func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) *Handler {
_, store := datastore.MustNewTestStore(t, true, true)
for _, endpoint := range endpoints {
err := store.Endpoint().Create(&endpoint)
require.NoError(t, err, "error creating environment")
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
require.NoError(t, err, "error creating a user")
bouncer := testhelpers.NewTestRequestBouncer()
handler := NewHandler(bouncer)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()
return handler
}
func TestFilterEndpointsByEdgeStack(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1", Type: portainer.EdgeAgentOnDockerEnvironment, UserTrusted: true},
{ID: 2, Name: "Endpoint 2", TagIDs: []portainer.TagID{1}, Type: portainer.EdgeAgentOnDockerEnvironment, UserTrusted: true},
{ID: 3, Name: "Endpoint 3", TagIDs: []portainer.TagID{1}, Type: portainer.EdgeAgentOnDockerEnvironment, UserTrusted: true},
{ID: 4, Name: "Endpoint 4"},
}
edgeStackId := portainer.EdgeStackID(1)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.Tag().Create(&portainer.Tag{ID: 1, Name: "tag", Endpoints: map[portainer.EndpointID]bool{2: true, 3: true}}))
for i := range endpoints {
require.NoError(t, tx.Endpoint().Create(&endpoints[i]))
}
require.NoError(t, tx.EdgeStack().Create(edgeStackId, &portainer.EdgeStack{
ID: edgeStackId,
Name: "Test Edge Stack",
EdgeGroups: []portainer.EdgeGroupID{1, 2},
}))
require.NoError(t, tx.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
}))
require.NoError(t, tx.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
Dynamic: true,
TagIDs: []portainer.TagID{1},
}))
require.NoError(t, tx.EdgeStackStatus().Create(edgeStackId, endpoints[0].ID, &portainer.EdgeStackStatusForEnv{
Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusAcknowledged}}}))
return nil
}))
test := func(status *portainer.EdgeStackStatusType, expected []portainer.Endpoint) {
tmp := make([]portainer.Endpoint, len(endpoints))
require.Equal(t, 4, copy(tmp, endpoints))
es, err := filterEndpointsByEdgeStack(tmp, edgeStackId, status, store)
require.NoError(t, err)
// validate that the len is the same
require.Len(t, es, len(expected))
// and that all items are the expected ones
for i := range expected {
require.Contains(t, es, expected[i])
}
}
test(nil, []portainer.Endpoint{endpoints[0], endpoints[1], endpoints[2]})
status := portainer.EdgeStackStatusPending
test(&status, []portainer.Endpoint{endpoints[1], endpoints[2]})
status = portainer.EdgeStackStatusCompleted
test(&status, []portainer.Endpoint{})
status = portainer.EdgeStackStatusAcknowledged
test(&status, []portainer.Endpoint{endpoints[0]}) // that's the only one with an edge stack status in DB
}
func TestErrorsFilterEndpointsByEdgeStack(t *testing.T) {
t.Parallel()
t.Run("must error by edge stack not found", func(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
require.NotNil(t, store)
_, err := filterEndpointsByEdgeStack([]portainer.Endpoint{}, 1, nil, store)
require.Error(t, err)
})
t.Run("must error by edge group not found", func(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
require.NotNil(t, store)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.EdgeStack().Create(1, &portainer.EdgeStack{ID: 1, Name: "1", EdgeGroups: []portainer.EdgeGroupID{1}}))
return nil
}))
_, err := filterEndpointsByEdgeStack([]portainer.Endpoint{}, 1, nil, store)
require.Error(t, err)
})
t.Run("must error by env tag not found", func(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
require.NotNil(t, store)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.EdgeStack().Create(1, &portainer.EdgeStack{ID: 1, Name: "1", EdgeGroups: []portainer.EdgeGroupID{1}}))
require.NoError(t, tx.EdgeGroup().Create(&portainer.EdgeGroup{ID: 1, Name: "edge group", Dynamic: true, TagIDs: []portainer.TagID{1}}))
return nil
}))
_, err := filterEndpointsByEdgeStack([]portainer.Endpoint{}, 1, nil, store)
require.Error(t, err)
})
}
func TestFilterEndpointsByEdgeGroup(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1"},
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{2, 3}),
})
require.NoError(t, err)
edgeGroups, err := store.EdgeGroup().ReadAll()
require.NoError(t, err)
es, egs := filterEndpointsByEdgeGroupIDs(endpoints, edgeGroups, []portainer.EdgeGroupID{1, 2})
require.NoError(t, err)
require.Len(t, es, 3)
require.Contains(t, es, endpoints[0]) // Endpoint 1
require.Contains(t, es, endpoints[1]) // Endpoint 2
require.Contains(t, es, endpoints[2]) // Endpoint 3
require.NotContains(t, es, endpoints[3]) // Endpoint 4
require.Len(t, egs, 2)
require.Equal(t, egs[0].ID, portainer.EdgeGroupID(1))
require.Equal(t, egs[1].ID, portainer.EdgeGroupID(2))
}
func TestFilterEndpointsByExcludeEdgeGroupIDs(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1"},
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{2, 3}),
})
require.NoError(t, err)
edgeGroups, err := store.EdgeGroup().ReadAll()
require.NoError(t, err)
es, egs := filterEndpointsByExcludeEdgeGroupIDs(endpoints, edgeGroups, []portainer.EdgeGroupID{1})
require.NoError(t, err)
require.Len(t, es, 3)
require.Equal(t, []portainer.Endpoint{
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}, es)
require.Len(t, egs, 1)
require.Equal(t, egs[0].ID, portainer.EdgeGroupID(2))
}
func TestGetShortestAsyncInterval(t *testing.T) {
t.Parallel()
endpoint := &portainer.Endpoint{
ID: 1,
Name: "Test Endpoint",
Edge: portainer.EnvironmentEdgeSettings{
PingInterval: -1,
SnapshotInterval: -1,
CommandInterval: -1,
},
}
settings := &portainer.Settings{
Edge: portainer.Edge{
PingInterval: 10,
SnapshotInterval: 20,
CommandInterval: 30,
},
}
require.Equal(t, 10, getShortestAsyncInterval(endpoint, settings))
}
func Test_filterEndpointsByPlatform(t *testing.T) {
ep := func(id portainer.EndpointID, epType portainer.EndpointType, containerEngine string) portainer.Endpoint {
return portainer.Endpoint{
ID: id,
Type: epType,
ContainerEngine: containerEngine,
}
}
docker := ep(1, portainer.DockerEnvironment, portainer.ContainerEngineDocker)
agentDocker := ep(2, portainer.AgentOnDockerEnvironment, portainer.ContainerEngineDocker)
edgeAgentDocker := ep(3, portainer.EdgeAgentOnDockerEnvironment, portainer.ContainerEngineDocker)
podman := ep(4, portainer.DockerEnvironment, portainer.ContainerEnginePodman)
agentPodman := ep(5, portainer.AgentOnDockerEnvironment, portainer.ContainerEnginePodman)
edgeAgentPodman := ep(6, portainer.EdgeAgentOnDockerEnvironment, portainer.ContainerEnginePodman)
k8sLocal := ep(7, portainer.KubernetesLocalEnvironment, "")
agentK8s := ep(8, portainer.AgentOnKubernetesEnvironment, "")
edgeAgentK8s := ep(9, portainer.EdgeAgentOnKubernetesEnvironment, "")
azure := ep(10, portainer.AzureEnvironment, "")
type args struct {
endpoints []portainer.Endpoint
platformTypes []portainer.PlatformType
}
tests := []struct {
name string
args args
want []portainer.Endpoint
}{
// Docker platform types
{
name: "DockerEnvironment is Docker platform",
args: args{endpoints: []portainer.Endpoint{docker}, platformTypes: []portainer.PlatformType{portainer.DockerPlatformType}},
want: []portainer.Endpoint{docker},
},
{
name: "AgentOnDockerEnvironment is Docker platform",
args: args{endpoints: []portainer.Endpoint{agentDocker}, platformTypes: []portainer.PlatformType{portainer.DockerPlatformType}},
want: []portainer.Endpoint{agentDocker},
},
{
name: "EdgeAgentOnDockerEnvironment is Docker platform",
args: args{endpoints: []portainer.Endpoint{edgeAgentDocker}, platformTypes: []portainer.PlatformType{portainer.DockerPlatformType}},
want: []portainer.Endpoint{edgeAgentDocker},
},
// Podman platform types
{
name: "DockerEnvironment with Podman engine is Podman platform",
args: args{endpoints: []portainer.Endpoint{podman}, platformTypes: []portainer.PlatformType{portainer.PodmanPlatformType}},
want: []portainer.Endpoint{podman},
},
{
name: "AgentOnDockerEnvironment with Podman engine is Podman platform",
args: args{endpoints: []portainer.Endpoint{agentPodman}, platformTypes: []portainer.PlatformType{portainer.PodmanPlatformType}},
want: []portainer.Endpoint{agentPodman},
},
{
name: "EdgeAgentOnDockerEnvironment with Podman engine is Podman platform",
args: args{endpoints: []portainer.Endpoint{edgeAgentPodman}, platformTypes: []portainer.PlatformType{portainer.PodmanPlatformType}},
want: []portainer.Endpoint{edgeAgentPodman},
},
// Kubernetes platform types
{
name: "KubernetesLocalEnvironment is Kubernetes platform",
args: args{endpoints: []portainer.Endpoint{k8sLocal}, platformTypes: []portainer.PlatformType{portainer.KubernetesPlatformType}},
want: []portainer.Endpoint{k8sLocal},
},
{
name: "AgentOnKubernetesEnvironment is Kubernetes platform",
args: args{endpoints: []portainer.Endpoint{agentK8s}, platformTypes: []portainer.PlatformType{portainer.KubernetesPlatformType}},
want: []portainer.Endpoint{agentK8s},
},
{
name: "EdgeAgentOnKubernetesEnvironment is Kubernetes platform",
args: args{endpoints: []portainer.Endpoint{edgeAgentK8s}, platformTypes: []portainer.PlatformType{portainer.KubernetesPlatformType}},
want: []portainer.Endpoint{edgeAgentK8s},
},
// Azure platform type
{
name: "AzureEnvironment is Azure platform",
args: args{endpoints: []portainer.Endpoint{azure}, platformTypes: []portainer.PlatformType{portainer.AzurePlatformType}},
want: []portainer.Endpoint{azure},
},
// Filter behaviour
{
name: "filters out non-matching platform types",
args: args{
endpoints: []portainer.Endpoint{docker, k8sLocal, azure},
platformTypes: []portainer.PlatformType{portainer.DockerPlatformType},
},
want: []portainer.Endpoint{docker},
},
{
name: "multiple platform types returns all matches",
args: args{
endpoints: []portainer.Endpoint{docker, agentDocker, edgeAgentDocker, podman, k8sLocal, agentK8s, edgeAgentK8s, azure},
platformTypes: []portainer.PlatformType{portainer.DockerPlatformType, portainer.KubernetesPlatformType},
},
want: []portainer.Endpoint{docker, agentDocker, edgeAgentDocker, k8sLocal, agentK8s, edgeAgentK8s},
},
{
name: "Podman endpoints not returned when filtering for Docker",
args: args{
endpoints: []portainer.Endpoint{docker, podman, agentPodman},
platformTypes: []portainer.PlatformType{portainer.DockerPlatformType},
},
want: []portainer.Endpoint{docker},
},
{
name: "returns empty when no endpoints match filter",
args: args{
endpoints: []portainer.Endpoint{k8sLocal, azure},
platformTypes: []portainer.PlatformType{portainer.DockerPlatformType},
},
want: []portainer.Endpoint{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, filterEndpointsByPlatform(tt.args.endpoints, tt.args.platformTypes), "filterEndpointsByPlatform(%v, %v)", tt.args.endpoints, tt.args.platformTypes)
})
}
}
func Test_FilterQuery_PlatformTypes(t *testing.T) {
t.Parallel()
dockerEndpoint := portainer.Endpoint{ID: 1, GroupID: 1, Type: portainer.DockerEnvironment}
kubernetesEndpoint := portainer.Endpoint{ID: 2, GroupID: 1, Type: portainer.KubernetesLocalEnvironment}
azureEndpoint := portainer.Endpoint{ID: 3, GroupID: 1, Type: portainer.AzureEnvironment}
endpoints := []portainer.Endpoint{dockerEndpoint, kubernetesEndpoint, azureEndpoint}
handler := setupFilterTest(t, endpoints)
tests := []filterTest{
{
title: "platformTypes filter returns only matching platform",
expected: []portainer.EndpointID{dockerEndpoint.ID},
query: EnvironmentsQuery{platformTypes: []portainer.PlatformType{portainer.DockerPlatformType}},
},
{
title: "multiple platformTypes returns all matching platforms",
expected: []portainer.EndpointID{dockerEndpoint.ID, kubernetesEndpoint.ID},
query: EnvironmentsQuery{platformTypes: []portainer.PlatformType{portainer.DockerPlatformType, portainer.KubernetesPlatformType}},
},
}
runTests(tests, t, handler, endpoints)
}
func Test_FilterQuery_Outdated(t *testing.T) {
t.Parallel()
currentVersion := portainer.APIVersion
upToDateEndpoint := portainer.Endpoint{ID: 1, GroupID: 1, Type: portainer.AgentOnDockerEnvironment}
upToDateEndpoint.Agent.Version = currentVersion
outdatedEndpoint := portainer.Endpoint{ID: 2, GroupID: 1, Type: portainer.AgentOnDockerEnvironment}
outdatedEndpoint.Agent.Version = "2.0.0"
endpoints := []portainer.Endpoint{upToDateEndpoint, outdatedEndpoint}
handler := setupFilterTest(t, endpoints)
tests := []filterTest{
{
title: "outdated filter returns only outdated endpoints",
expected: []portainer.EndpointID{outdatedEndpoint.ID},
query: EnvironmentsQuery{outdated: true},
},
{
title: "outdated=false returns all endpoints",
expected: []portainer.EndpointID{upToDateEndpoint.ID, outdatedEndpoint.ID},
query: EnvironmentsQuery{outdated: false},
},
}
runTests(tests, t, handler, endpoints)
}
func Test_parseQuery_emptyArrayParams(t *testing.T) {
t.Parallel()
makeRequest := func(rawQuery string) *http.Request {
r := &http.Request{URL: &url.URL{RawQuery: rawQuery}}
require.NoError(t, r.ParseForm())
return r
}
tests := []struct {
name string
query string
}{
{name: "empty status", query: "status="},
{name: "empty endpointIds", query: "endpointIds="},
{name: "empty groupIds", query: "groupIds="},
{name: "empty tagIds", query: "tagIds="},
{name: "multiple empty params", query: "status=&groupIds=&tagIds="},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseQuery(makeRequest(tt.query))
require.NoError(t, err)
})
}
}
+94
View File
@@ -0,0 +1,94 @@
package endpoints
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/pendingactions"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
func hideFields(endpoint *portainer.Endpoint) {
endpoint.AzureCredentials = portainer.AzureCredentials{}
if len(endpoint.Snapshots) > 0 {
endpoint.Snapshots[0].SnapshotRaw = portainer.DockerSnapshotRaw{}
}
}
// Handler is the HTTP handler used to handle environment(endpoint) operations.
type Handler struct {
*mux.Router
requestBouncer security.BouncerService
DataStore dataservices.DataStore
FileService portainer.FileService
ProxyManager *proxy.Manager
ReverseTunnelService portainer.ReverseTunnelService
SnapshotService portainer.SnapshotService
K8sClientFactory *cli.ClientFactory
ComposeStackManager portainer.ComposeStackManager
AuthorizationService *authorization.Service
DockerClientFactory *dockerclient.ClientFactory
BindAddress string
BindAddressHTTPS string
PendingActionsService *pendingactions.PendingActionsService
PullLimitCheckDisabled bool
}
// NewHandler creates a handler to manage environment(endpoint) operations.
func NewHandler(bouncer security.BouncerService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
requestBouncer: bouncer,
}
h.Handle("/endpoints",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointCreate))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/settings",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointSettingsUpdate))).Methods(http.MethodPut)
h.Handle("/endpoints/{id}/association",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointAssociationDelete))).Methods(http.MethodDelete)
h.Handle("/endpoints/snapshot",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointSnapshots))).Methods(http.MethodPost)
h.Handle("/endpoints",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointList))).Methods(http.MethodGet)
h.Handle("/endpoints/agent_versions",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.agentVersions))).Methods(http.MethodGet)
h.Handle("/endpoints/summary",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointSummaryCounts))).Methods(http.MethodGet)
h.Handle("/endpoints/relations", bouncer.AdminAccess(httperror.LoggerHandler(h.updateRelations))).Methods(http.MethodPut)
h.Handle("/endpoints/{id}",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointInspect))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
h.Handle("/endpoints/{id}",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
h.Handle("/endpoints/delete",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDeleteBatch))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/dockerhub/{registryId}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/snapshot",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointSnapshot))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/registries",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistriesList))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/registries/{registryId}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryAccess))).Methods(http.MethodPut)
h.Handle("/endpoints/global-key", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointCreateGlobalKey))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/forceupdateservice",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointForceUpdateService))).Methods(http.MethodPut)
// DEPRECATED
h.Handle("/endpoints/{id}/status", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)
h.Handle("/endpoints", bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDeleteBatchDeprecated))).Methods(http.MethodDelete)
return h
}
+124
View File
@@ -0,0 +1,124 @@
package endpoints
import (
"slices"
"github.com/fvbommel/sortorder"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/endpointutils"
)
type comp[T any] func(a, b T) int
func stringComp(a, b string) int {
if sortorder.NaturalLess(a, b) {
return -1
} else if sortorder.NaturalLess(b, a) {
return 1
} else {
return 0
}
}
func healthRank(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
status := resolveEndpointStatus(endpoint, settings)
if status == statusDown {
return 0
}
if isOutdated(endpoint) {
return 1
}
if status == statusHeartbeat {
return 2
}
return 3
}
func sortEnvironmentsByField(environments []portainer.Endpoint, environmentGroups []portainer.EndpointGroup, sortField sortKey, isSortDesc bool, settings *portainer.Settings) {
if sortField == "" {
return
}
var less comp[portainer.Endpoint]
switch sortField {
case sortKeyName:
less = func(a, b portainer.Endpoint) int {
return stringComp(a.Name, b.Name)
}
case sortKeyGroup:
environmentGroupNames := make(map[portainer.EndpointGroupID]string, 0)
for _, group := range environmentGroups {
environmentGroupNames[group.ID] = group.Name
}
// set the "unassigned" group name to be empty string
environmentGroupNames[1] = ""
less = func(a, b portainer.Endpoint) int {
aGroup := environmentGroupNames[a.GroupID]
bGroup := environmentGroupNames[b.GroupID]
return stringComp(aGroup, bGroup)
}
case sortKeyStatus:
less = func(a, b portainer.Endpoint) int {
return int(a.Status - b.Status)
}
case sortKeyLastCheckInDate:
less = func(a, b portainer.Endpoint) int {
return int(a.LastCheckInDate - b.LastCheckInDate)
}
case sortKeyEdgeID:
less = func(a, b portainer.Endpoint) int {
return stringComp(a.EdgeID, b.EdgeID)
}
case sortKeyPlatformType:
less = func(a, b portainer.Endpoint) int {
return int(endpointutils.EndpointPlatformType(&a) - endpointutils.EndpointPlatformType(&b))
}
case sortKeyHealth:
less = func(a, b portainer.Endpoint) int {
return healthRank(&a, settings) - healthRank(&b, settings)
}
case sortKeyId:
less = func(a, b portainer.Endpoint) int {
return int(a.ID - b.ID)
}
}
slices.SortStableFunc(environments, func(a, b portainer.Endpoint) int {
mul := 1
if isSortDesc {
mul = -1
}
return less(a, b) * mul
})
}
type sortKey string
const (
sortKeyName sortKey = "Name"
sortKeyGroup sortKey = "Group"
sortKeyStatus sortKey = "Status"
sortKeyLastCheckInDate sortKey = "LastCheckIn"
sortKeyEdgeID sortKey = "EdgeID"
sortKeyPlatformType sortKey = "PlatformType"
sortKeyHealth sortKey = "Health"
sortKeyId sortKey = "Id"
)
func getSortKey(sortField string) sortKey {
fieldAsSortKey := sortKey(sortField)
if slices.Contains([]sortKey{sortKeyName, sortKeyGroup, sortKeyStatus, sortKeyLastCheckInDate, sortKeyEdgeID, sortKeyPlatformType, sortKeyHealth, sortKeyId}, fieldAsSortKey) {
return fieldAsSortKey
}
return ""
}
+243
View File
@@ -0,0 +1,243 @@
package endpoints
import (
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/slicesx"
"github.com/stretchr/testify/assert"
)
func TestSortEndpointsByField(t *testing.T) {
t.Parallel()
environments := []portainer.Endpoint{
{ID: 0, Name: "Environment 1", GroupID: 1, Status: 1, LastCheckInDate: 3, EdgeID: "edge32"},
{ID: 1, Name: "Environment 2", GroupID: 2, Status: 2, LastCheckInDate: 6, EdgeID: "edge57"},
{ID: 2, Name: "Environment 3", GroupID: 1, Status: 3, LastCheckInDate: 2, EdgeID: "test87"},
{ID: 3, Name: "Environment 4", GroupID: 2, Status: 4, LastCheckInDate: 1, EdgeID: "abc123"},
}
environmentGroups := []portainer.EndpointGroup{
{ID: 1, Name: "Group 1"},
{ID: 2, Name: "Group 2"},
}
tests := []struct {
name string
sortField sortKey
isSortDesc bool
expected []portainer.EndpointID
}{
{
name: "sort without value",
sortField: "",
expected: []portainer.EndpointID{
environments[0].ID,
environments[1].ID,
environments[2].ID,
environments[3].ID,
},
},
{
name: "sort by name ascending",
sortField: "Name",
isSortDesc: false,
expected: []portainer.EndpointID{
environments[0].ID,
environments[1].ID,
environments[2].ID,
environments[3].ID,
},
},
{
name: "sort by name descending",
sortField: "Name",
isSortDesc: true,
expected: []portainer.EndpointID{
environments[3].ID,
environments[2].ID,
environments[1].ID,
environments[0].ID,
},
},
{
name: "sort by group name ascending",
sortField: "Group",
isSortDesc: false,
expected: []portainer.EndpointID{
environments[0].ID,
environments[2].ID,
environments[1].ID,
environments[3].ID,
},
},
{
name: "sort by group name descending",
sortField: "Group",
isSortDesc: true,
expected: []portainer.EndpointID{
environments[1].ID,
environments[3].ID,
environments[0].ID,
environments[2].ID,
},
},
{
name: "sort by status ascending",
sortField: "Status",
isSortDesc: false,
expected: []portainer.EndpointID{
environments[0].ID,
environments[1].ID,
environments[2].ID,
environments[3].ID,
},
},
{
name: "sort by status descending",
sortField: "Status",
isSortDesc: true,
expected: []portainer.EndpointID{
environments[3].ID,
environments[2].ID,
environments[1].ID,
environments[0].ID,
},
},
{
name: "sort by last check-in ascending",
sortField: "LastCheckIn",
isSortDesc: false,
expected: []portainer.EndpointID{
environments[3].ID,
environments[2].ID,
environments[0].ID,
environments[1].ID,
},
},
{
name: "sort by last check-in descending",
sortField: "LastCheckIn",
isSortDesc: true,
expected: []portainer.EndpointID{
environments[1].ID,
environments[0].ID,
environments[2].ID,
environments[3].ID,
},
},
{
name: "sort by edge ID ascending",
sortField: "EdgeID",
expected: []portainer.EndpointID{
environments[3].ID,
environments[0].ID,
environments[1].ID,
environments[2].ID,
},
},
{
name: "sort by edge ID descending",
sortField: "EdgeID",
isSortDesc: true,
expected: []portainer.EndpointID{
environments[2].ID,
environments[1].ID,
environments[0].ID,
environments[3].ID,
},
},
{
name: "sort by platform type ascending groups same-platform types together",
sortField: "PlatformType",
expected: []portainer.EndpointID{
environments[0].ID,
environments[1].ID,
environments[2].ID,
environments[3].ID,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
is := assert.New(t)
sortEnvironmentsByField(environments, environmentGroups, "Name", false, nil) // reset to default sort order
sortEnvironmentsByField(environments, environmentGroups, tt.sortField, tt.isSortDesc, nil)
is.Equal(tt.expected, getEndpointIDs(environments))
})
}
}
func getEndpointIDs(environments []portainer.Endpoint) []portainer.EndpointID {
return slicesx.Map(environments, func(environment portainer.Endpoint) portainer.EndpointID {
return environment.ID
})
}
func TestSortEndpointsByHealth(t *testing.T) {
t.Parallel()
settings := &portainer.Settings{EdgeAgentCheckinInterval: 30}
// Down: non-edge with Status=2
down := portainer.Endpoint{
ID: 0,
Name: "down-env",
Type: portainer.DockerEnvironment,
Status: portainer.EndpointStatusDown,
}
// Outdated: edge agent with empty version and a prior check-in (old agent style, no version reported)
outdated := portainer.Endpoint{
ID: 1,
Name: "outdated-env",
Type: portainer.EdgeAgentOnDockerEnvironment,
Status: portainer.EndpointStatusUp,
LastCheckInDate: time.Now().Unix(),
// IsEdgeEndpoint + Agent.Version == "" + LastCheckInDate > 0 → isOutdated returns true
}
// Heartbeat: edge that checked in recently with a current agent version (not outdated)
heartbeat := portainer.Endpoint{
ID: 2,
Name: "heartbeat-env",
Type: portainer.EdgeAgentOnDockerEnvironment,
LastCheckInDate: time.Now().Unix(),
}
heartbeat.Agent.Version = portainer.APIVersion
// Up: non-edge with Status=1
up := portainer.Endpoint{
ID: 3,
Name: "up-env",
Type: portainer.DockerEnvironment,
Status: portainer.EndpointStatusUp,
}
t.Run("ascending: Down → Outdated → Heartbeat → Up", func(t *testing.T) {
environments := []portainer.Endpoint{up, heartbeat, outdated, down}
sortEnvironmentsByField(environments, nil, sortKeyHealth, false, settings)
assert.Equal(t,
[]portainer.EndpointID{down.ID, outdated.ID, heartbeat.ID, up.ID},
getEndpointIDs(environments),
)
})
t.Run("descending: Up → Heartbeat → Outdated → Down", func(t *testing.T) {
environments := []portainer.Endpoint{down, outdated, heartbeat, up}
sortEnvironmentsByField(environments, nil, sortKeyHealth, true, settings)
assert.Equal(t,
[]portainer.EndpointID{up.ID, heartbeat.ID, outdated.ID, down.ID},
getEndpointIDs(environments),
)
})
}
func TestGetSortKey(t *testing.T) {
assert.Equal(t, sortKey("Name"), getSortKey("Name"))
assert.Equal(t, sortKey("PlatformType"), getSortKey("PlatformType"))
assert.Equal(t, sortKey(""), getSortKey("unknown"))
}
+18
View File
@@ -0,0 +1,18 @@
package endpoints
import portainer "github.com/portainer/portainer/api"
func (handler *Handler) isNameUnique(name string, endpointID portainer.EndpointID) (bool, error) {
endpoints, err := handler.DataStore.Endpoint().Endpoints()
if err != nil {
return false, err
}
for _, endpoint := range endpoints {
if endpoint.Name == name && (endpointID == 0 || endpoint.ID != endpointID) {
return false, nil
}
}
return true, nil
}
@@ -0,0 +1,48 @@
package endpoints
import (
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/set"
)
// updateEdgeRelations updates the edge stacks associated to an edge endpoint
func (handler *Handler) updateEdgeRelations(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) error {
if !endpointutils.IsEdgeEndpoint(endpoint) {
return nil
}
relation, err := tx.EndpointRelation().EndpointRelation(endpoint.ID)
if err != nil {
return errors.WithMessage(err, "Unable to retrieve environment relation inside the database")
}
endpointGroup, err := tx.EndpointGroup().Read(endpoint.GroupID)
if err != nil {
return errors.WithMessage(err, "Unable to find environment group inside the database")
}
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
return errors.WithMessage(err, "Unable to retrieve edge groups from the database")
}
edgeStacks, err := tx.EdgeStack().EdgeStacks()
if err != nil {
return errors.WithMessage(err, "Unable to retrieve edge stacks from the database")
}
currentEdgeStackSet := set.ToSet(edge.EndpointRelatedEdgeStacks(endpoint, endpointGroup, edgeGroups, edgeStacks))
relation.EdgeStacks = currentEdgeStackSet
err = tx.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
if err != nil {
return errors.WithMessage(err, "Unable to persist environment relation changes inside the database")
}
return nil
}
@@ -0,0 +1,66 @@
package endpoints
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/set"
"github.com/pkg/errors"
)
func updateEnvironmentEdgeGroups(tx dataservices.DataStoreTx, newEdgeGroups []portainer.EdgeGroupID, environmentID portainer.EndpointID) (bool, error) {
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
return false, errors.WithMessage(err, "Unable to retrieve edge groups from the database")
}
newEdgeGroupsSet := set.ToSet(newEdgeGroups)
environmentEdgeGroupsSet := set.Set[portainer.EdgeGroupID]{}
for _, edgeGroup := range edgeGroups {
if edgeGroup.EndpointIDs.Contains(environmentID) {
environmentEdgeGroupsSet[edgeGroup.ID] = true
}
}
union := set.Union(newEdgeGroupsSet, environmentEdgeGroupsSet)
intersection := set.Intersection(newEdgeGroupsSet, environmentEdgeGroupsSet)
if len(union) <= len(intersection) {
return false, nil
}
updateSet := func(groupIDs set.Set[portainer.EdgeGroupID], updateItem func(*portainer.EdgeGroup)) error {
for groupID := range groupIDs {
group, err := tx.EdgeGroup().Read(groupID)
if err != nil {
return errors.WithMessage(err, "Unable to find a Edge group inside the database")
}
updateItem(group)
err = tx.EdgeGroup().Update(groupID, group)
if err != nil {
return errors.WithMessage(err, "Unable to persist Edge group changes inside the database")
}
}
return nil
}
removeEdgeGroups := environmentEdgeGroupsSet.Difference(newEdgeGroupsSet)
if err := updateSet(removeEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.EndpointIDs.Remove(environmentID)
}); err != nil {
return false, err
}
addToEdgeGroups := newEdgeGroupsSet.Difference(environmentEdgeGroupsSet)
if err := updateSet(addToEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.EndpointIDs.Add(environmentID)
}); err != nil {
return false, err
}
return true, nil
}
@@ -0,0 +1,152 @@
package endpoints
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_updateEdgeGroups(t *testing.T) {
t.Parallel()
createGroups := func(store *datastore.Store, names []string) ([]portainer.EdgeGroup, error) {
groups := make([]portainer.EdgeGroup, len(names))
for index, name := range names {
group := &portainer.EdgeGroup{
Name: name,
Dynamic: false,
TagIDs: make([]portainer.TagID, 0),
}
if err := store.EdgeGroup().Create(group); err != nil {
return nil, err
}
groups[index] = *group
}
return groups, nil
}
checkGroups := func(store *datastore.Store, is *assert.Assertions, groupIDs []portainer.EdgeGroupID, endpointID portainer.EndpointID) {
for _, groupID := range groupIDs {
group, err := store.EdgeGroup().Read(groupID)
require.NoError(t, err)
is.True(group.EndpointIDs.Contains(endpointID),
"expected endpoint to be in group")
}
}
groupsByName := func(groups []portainer.EdgeGroup, groupNames []string) []portainer.EdgeGroup {
result := make([]portainer.EdgeGroup, len(groupNames))
for i, tagName := range groupNames {
for j, tag := range groups {
if tag.Name == tagName {
result[i] = groups[j]
break
}
}
}
return result
}
type testCase struct {
title string
endpoint *portainer.Endpoint
groupNames []string
endpointGroupNames []string
groupsToApply []string
shouldNotBeUpdated bool
}
testFn := func(t *testing.T, testCase testCase) {
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(testCase.endpoint)
require.NoError(t, err)
groups, err := createGroups(store, testCase.groupNames)
require.NoError(t, err)
endpointGroups := groupsByName(groups, testCase.endpointGroupNames)
for _, group := range endpointGroups {
group.EndpointIDs.Add(testCase.endpoint.ID)
err = store.EdgeGroup().Update(group.ID, &group)
require.NoError(t, err)
}
expectedGroups := groupsByName(groups, testCase.groupsToApply)
expectedIDs := make([]portainer.EdgeGroupID, len(expectedGroups))
for i, tag := range expectedGroups {
expectedIDs[i] = tag.ID
}
err = store.UpdateTx(func(tx dataservices.DataStoreTx) error {
updated, err := updateEnvironmentEdgeGroups(tx, expectedIDs, testCase.endpoint.ID)
require.NoError(t, err)
is.Equal(testCase.shouldNotBeUpdated, !updated)
return nil
})
require.NoError(t, err)
checkGroups(store, is, expectedIDs, testCase.endpoint.ID)
}
testCases := []testCase{
{
title: "applying edge groups to an endpoint without edge groups",
endpoint: &portainer.Endpoint{},
groupNames: []string{"edge group1", "edge group2", "edge group3"},
endpointGroupNames: []string{},
groupsToApply: []string{"edge group1", "edge group2", "edge group3"},
},
{
title: "applying edge groups to an endpoint with edge groups",
endpoint: &portainer.Endpoint{},
groupNames: []string{"edge group1", "edge group2", "edge group3", "edge group4", "edge group5", "edge group6"},
endpointGroupNames: []string{"edge group1", "edge group2", "edge group3"},
groupsToApply: []string{"edge group4", "edge group5", "edge group6"},
},
{
title: "applying edge groups to an endpoint with edge groups that are already applied",
endpoint: &portainer.Endpoint{},
groupNames: []string{"edge group1", "edge group2", "edge group3"},
endpointGroupNames: []string{"edge group1", "edge group2", "edge group3"},
groupsToApply: []string{"edge group1", "edge group2", "edge group3"},
shouldNotBeUpdated: true,
},
{
title: "adding new edge groups to an endpoint with edge groups ",
endpoint: &portainer.Endpoint{},
groupNames: []string{"edge group1", "edge group2", "edge group3", "edge group4", "edge group5", "edge group6"},
endpointGroupNames: []string{"edge group1", "edge group2", "edge group3"},
groupsToApply: []string{"edge group1", "edge group2", "edge group3", "edge group4", "edge group5", "edge group6"},
},
{
title: "mixing edge groups that are already applied and new edge groups",
endpoint: &portainer.Endpoint{},
groupNames: []string{"edge group1", "edge group2", "edge group3", "edge group4", "edge group5", "edge group6"},
endpointGroupNames: []string{"edge group1", "edge group2", "edge group3"},
groupsToApply: []string{"edge group2", "edge group4", "edge group5"},
},
}
for _, testCase := range testCases {
t.Run(testCase.title, func(t *testing.T) {
testFn(t, testCase)
})
}
}
@@ -0,0 +1,56 @@
package endpoints
import (
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/set"
)
// updateEnvironmentTags updates the tags associated to an environment
func updateEnvironmentTags(tx dataservices.DataStoreTx, newTags []portainer.TagID, oldTags []portainer.TagID, environmentID portainer.EndpointID) (bool, error) {
payloadTagSet := set.ToSet(newTags)
environmentTagSet := set.ToSet(oldTags)
union := set.Union(payloadTagSet, environmentTagSet)
intersection := set.Intersection(payloadTagSet, environmentTagSet)
if len(union) <= len(intersection) {
return false, nil
}
updateSet := func(tagIDs set.Set[portainer.TagID], updateItem func(*portainer.Tag)) error {
for tagID := range tagIDs {
tag, err := tx.Tag().Read(tagID)
if err != nil {
return errors.WithMessage(err, "Unable to find a tag inside the database")
}
updateItem(tag)
err = tx.Tag().Update(tagID, tag)
if err != nil {
return errors.WithMessage(err, "Unable to persist tag changes inside the database")
}
}
return nil
}
removeTags := environmentTagSet.Difference(payloadTagSet)
err := updateSet(removeTags, func(tag *portainer.Tag) {
delete(tag.Endpoints, environmentID)
})
if err != nil {
return false, err
}
addTags := payloadTagSet.Difference(environmentTagSet)
err = updateSet(addTags, func(tag *portainer.Tag) {
tag.Endpoints[environmentID] = true
})
if err != nil {
return false, err
}
return true, nil
}
@@ -0,0 +1,165 @@
package endpoints
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_updateTags(t *testing.T) {
t.Parallel()
createTags := func(store *datastore.Store, tagNames []string) ([]portainer.Tag, error) {
tags := make([]portainer.Tag, len(tagNames))
for index, tagName := range tagNames {
tag := &portainer.Tag{
Name: tagName,
Endpoints: make(map[portainer.EndpointID]bool),
EndpointGroups: make(map[portainer.EndpointGroupID]bool),
}
err := store.Tag().Create(tag)
if err != nil {
return nil, err
}
tags[index] = *tag
}
return tags, nil
}
checkTags := func(store *datastore.Store, is *assert.Assertions, tagIDs []portainer.TagID, endpointID portainer.EndpointID) {
for _, tagID := range tagIDs {
tag, err := store.Tag().Read(tagID)
require.NoError(t, err)
_, ok := tag.Endpoints[endpointID]
is.True(ok, "expected endpoint to be tagged")
}
}
tagsByName := func(tags []portainer.Tag, tagNames []string) []portainer.Tag {
result := make([]portainer.Tag, len(tagNames))
for i, tagName := range tagNames {
for j, tag := range tags {
if tag.Name == tagName {
result[i] = tags[j]
break
}
}
}
return result
}
getIDs := func(tags []portainer.Tag) []portainer.TagID {
ids := make([]portainer.TagID, len(tags))
for i, tag := range tags {
ids[i] = tag.ID
}
return ids
}
type testCase struct {
title string
endpoint *portainer.Endpoint
tagNames []string
endpointTagNames []string
tagsToApply []string
shouldNotBeUpdated bool
}
testFn := func(t *testing.T, testCase testCase) {
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(testCase.endpoint)
require.NoError(t, err)
tags, err := createTags(store, testCase.tagNames)
require.NoError(t, err)
endpointTags := tagsByName(tags, testCase.endpointTagNames)
for _, tag := range endpointTags {
tag.Endpoints[testCase.endpoint.ID] = true
err = store.Tag().Update(tag.ID, &tag)
require.NoError(t, err)
}
endpointTagIDs := getIDs(endpointTags)
testCase.endpoint.TagIDs = endpointTagIDs
err = store.Endpoint().UpdateEndpoint(testCase.endpoint.ID, testCase.endpoint)
require.NoError(t, err)
expectedTags := tagsByName(tags, testCase.tagsToApply)
expectedTagIDs := make([]portainer.TagID, len(expectedTags))
for i, tag := range expectedTags {
expectedTagIDs[i] = tag.ID
}
err = store.UpdateTx(func(tx dataservices.DataStoreTx) error {
updated, err := updateEnvironmentTags(tx, expectedTagIDs, testCase.endpoint.TagIDs, testCase.endpoint.ID)
require.NoError(t, err)
is.Equal(testCase.shouldNotBeUpdated, !updated)
return nil
})
require.NoError(t, err)
checkTags(store, is, expectedTagIDs, testCase.endpoint.ID)
}
testCases := []testCase{
{
title: "applying tags to an endpoint without tags",
endpoint: &portainer.Endpoint{},
tagNames: []string{"tag1", "tag2", "tag3"},
endpointTagNames: []string{},
tagsToApply: []string{"tag1", "tag2", "tag3"},
},
{
title: "applying tags to an endpoint with tags",
endpoint: &portainer.Endpoint{},
tagNames: []string{"tag1", "tag2", "tag3", "tag4", "tag5", "tag6"},
endpointTagNames: []string{"tag1", "tag2", "tag3"},
tagsToApply: []string{"tag4", "tag5", "tag6"},
},
{
title: "applying tags to an endpoint with tags that are already applied",
endpoint: &portainer.Endpoint{},
tagNames: []string{"tag1", "tag2", "tag3"},
endpointTagNames: []string{"tag1", "tag2", "tag3"},
tagsToApply: []string{"tag1", "tag2", "tag3"},
shouldNotBeUpdated: true,
},
{
title: "adding new tags to an endpoint with tags ",
endpoint: &portainer.Endpoint{},
tagNames: []string{"tag1", "tag2", "tag3", "tag4", "tag5", "tag6"},
endpointTagNames: []string{"tag1", "tag2", "tag3"},
tagsToApply: []string{"tag1", "tag2", "tag3", "tag4", "tag5", "tag6"},
},
{
title: "mixing tags that are already applied and new tags",
endpoint: &portainer.Endpoint{},
tagNames: []string{"tag1", "tag2", "tag3", "tag4", "tag5", "tag6"},
endpointTagNames: []string{"tag1", "tag2", "tag3"},
tagsToApply: []string{"tag2", "tag4", "tag5"},
},
}
for _, testCase := range testCases {
t.Run(testCase.title, func(t *testing.T) {
testFn(t, testCase)
})
}
}