chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
@@ -0,0 +1,151 @@
package endpointutils
import (
"context"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/client"
"github.com/rs/zerolog/log"
)
// InitEndpoint controls the workflow to initialize the primary endpoint.
// When installing Portainer in a Docker Cluster using the yaml file provided in the official
// documentation (https://docs.portainer.io/start/install/server/swarm/linux), the "primary"
// endpoint is initialized before the admin user is created. This triggers the creation of a
// snapshot of the environment in the background, which includes the agent saving the signature
// from the first request made by the server. However, if a user restores Portainer from a backup
// instead of creating a new admin user, the server will not be able to connect to the agent because
// the saved signature will not match. To solve this issue, this solution proposes to wait for
// the admin user to be created before initializing the primary endpoint. This way, the agent
// will save the signature from the first request after the admin user is created, ensuring that
// it matches in the event of a backup restoration.
func InitEndpoint(shutdownCtx context.Context, adminCreationDone <-chan struct{}, flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) {
select {
case <-shutdownCtx.Done():
log.Debug().Msg("shutdown endpoint initialization")
case <-adminCreationDone:
// Wait for the admin user to be created before initializing the primary endpoint
// The admin user can be created in two ways:
// 1. Using the CLI with the --admin-password flag
// 2. Using the API with the /api/users/admin/init endpoint
log.Debug().Msg("init primary endpoint")
if err := initEndpoint(flags, dataStore, snapshotService); err != nil {
log.Fatal().Err(err).Msg("failed initializing environment")
}
}
}
func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
if *flags.EndpointURL == "" {
return nil
}
endpoints, err := dataStore.Endpoint().Endpoints()
if err != nil {
return err
}
if len(endpoints) > 0 {
log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
return nil
}
if *flags.TLS || *flags.TLSSkipVerify {
return createTLSSecuredEndpoint(flags, dataStore, snapshotService)
}
return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
}
func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
tlsConfiguration := portainer.TLSConfiguration{
TLS: *flags.TLS,
TLSSkipVerify: *flags.TLSSkipVerify,
}
if *flags.TLS {
tlsConfiguration.TLSCACertPath = *flags.TLSCacert
tlsConfiguration.TLSCertPath = *flags.TLSCert
tlsConfiguration.TLSKeyPath = *flags.TLSKey
} else if !*flags.TLS && *flags.TLSSkipVerify {
tlsConfiguration.TLS = true
}
endpointID := dataStore.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: "primary",
URL: *flags.EndpointURL,
GroupID: portainer.EndpointGroupID(1),
Type: portainer.DockerEnvironment,
TLSConfig: tlsConfiguration,
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: []portainer.TagID{},
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
SecuritySettings: portainer.DefaultEndpointSecuritySettings(),
}
if strings.HasPrefix(endpoint.URL, "tcp://") {
agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfiguration)
if err != nil {
return err
}
if agentOnDockerEnvironment {
endpoint.Type = portainer.AgentOnDockerEnvironment
}
}
if err := snapshotService.SnapshotEndpoint(endpoint); err != nil {
log.Error().
Str("endpoint", endpoint.Name).
Str("URL", endpoint.URL).
Err(err).
Msg("environment snapshot error")
}
return dataStore.Endpoint().Create(endpoint)
}
func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
if strings.HasPrefix(endpointURL, "tcp://") {
if _, err := client.ExecutePingOperation(endpointURL, portainer.TLSConfiguration{}); err != nil {
return err
}
}
endpointID := dataStore.Endpoint().GetNextIdentifier()
endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: "primary",
URL: endpointURL,
GroupID: portainer.EndpointGroupID(1),
Type: portainer.DockerEnvironment,
TLSConfig: portainer.TLSConfiguration{},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: []portainer.TagID{},
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
SecuritySettings: portainer.DefaultEndpointSecuritySettings(),
}
if err := snapshotService.SnapshotEndpoint(endpoint); err != nil {
log.Error().
Str("endpoint", endpoint.Name).
Str("URL", endpoint.URL).Err(err).
Msg("environment snapshot error")
}
return dataStore.Endpoint().Create(endpoint)
}
@@ -0,0 +1,13 @@
package endpointutils
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestCreateOfflineUnsecuredEndpoint(t *testing.T) {
t.Parallel()
err := createUnsecuredEndpoint("tcp://localhost:1", nil, nil)
require.Error(t, err)
}
@@ -0,0 +1,83 @@
package endpointutils
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/set"
)
// GetEndpointsByTags returns the trusted edge endpoints matching tagIDs, unioned if partialMatch
// is set or intersected otherwise.
func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, partialMatch bool) ([]portainer.EndpointID, error) {
if len(tagIDs) == 0 {
return []portainer.EndpointID{}, nil
}
endpoints, err := tx.Endpoint().Endpoints()
if err != nil {
return nil, err
}
groupEndpoints := mapEndpointGroupToEndpoints(endpoints)
tags := []portainer.Tag{}
for _, tagID := range tagIDs {
tag, err := tx.Tag().Read(tagID)
if err != nil {
return nil, err
}
tags = append(tags, *tag)
}
setsOfEndpoints := mapTagsToEndpoints(tags, groupEndpoints)
var endpointSet set.Set[portainer.EndpointID]
if partialMatch {
endpointSet = set.Union(setsOfEndpoints...)
} else {
endpointSet = set.Intersection(setsOfEndpoints...)
}
results := []portainer.EndpointID{}
for _, endpoint := range endpoints {
if endpointSet.Contains(endpoint.ID) && IsEdgeEndpoint(&endpoint) && endpoint.UserTrusted {
results = append(results, endpoint.ID)
}
}
return results, nil
}
func mapEndpointGroupToEndpoints(endpoints []portainer.Endpoint) map[portainer.EndpointGroupID]set.Set[portainer.EndpointID] {
groupEndpoints := map[portainer.EndpointGroupID]set.Set[portainer.EndpointID]{}
for _, endpoint := range endpoints {
groupID := endpoint.GroupID
if groupEndpoints[groupID] == nil {
groupEndpoints[groupID] = set.Set[portainer.EndpointID]{}
}
groupEndpoints[groupID].Add(endpoint.ID)
}
return groupEndpoints
}
func mapTagsToEndpoints(tags []portainer.Tag, groupEndpoints map[portainer.EndpointGroupID]set.Set[portainer.EndpointID]) []set.Set[portainer.EndpointID] {
sets := make([]set.Set[portainer.EndpointID], 0, len(tags))
for _, tag := range tags {
s := set.Set[portainer.EndpointID](tag.Endpoints)
for groupID := range tag.EndpointGroups {
for endpointID := range groupEndpoints[groupID] {
s.Add(endpointID)
}
}
sets = append(sets, s)
}
return sets
}
+144
View File
@@ -0,0 +1,144 @@
package endpointutils
import (
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type isEndpointTypeTest struct {
endpointType portainer.EndpointType
expected bool
}
func Test_IsDockerEndpoint(t *testing.T) {
t.Parallel()
tests := []isEndpointTypeTest{
{endpointType: portainer.DockerEnvironment, expected: true},
{endpointType: portainer.AgentOnDockerEnvironment, expected: true},
{endpointType: portainer.AzureEnvironment, expected: false},
{endpointType: portainer.EdgeAgentOnDockerEnvironment, expected: true},
{endpointType: portainer.KubernetesLocalEnvironment, expected: false},
{endpointType: portainer.AgentOnKubernetesEnvironment, expected: false},
{endpointType: portainer.EdgeAgentOnKubernetesEnvironment, expected: false},
}
for _, test := range tests {
ans := IsDockerEndpoint(&portainer.Endpoint{Type: test.endpointType})
assert.Equal(t, test.expected, ans)
}
}
func Test_IsKubernetesEndpoint(t *testing.T) {
t.Parallel()
tests := []isEndpointTypeTest{
{endpointType: portainer.DockerEnvironment, expected: false},
{endpointType: portainer.AgentOnDockerEnvironment, expected: false},
{endpointType: portainer.AzureEnvironment, expected: false},
{endpointType: portainer.EdgeAgentOnDockerEnvironment, expected: false},
{endpointType: portainer.KubernetesLocalEnvironment, expected: true},
{endpointType: portainer.AgentOnKubernetesEnvironment, expected: true},
{endpointType: portainer.EdgeAgentOnKubernetesEnvironment, expected: true},
}
for _, test := range tests {
ans := IsKubernetesEndpoint(&portainer.Endpoint{Type: test.endpointType})
assert.Equal(t, test.expected, ans)
}
}
func Test_IsAgentEndpoint(t *testing.T) {
t.Parallel()
tests := []isEndpointTypeTest{
{endpointType: portainer.DockerEnvironment, expected: false},
{endpointType: portainer.AgentOnDockerEnvironment, expected: true},
{endpointType: portainer.AzureEnvironment, expected: false},
{endpointType: portainer.EdgeAgentOnDockerEnvironment, expected: true},
{endpointType: portainer.KubernetesLocalEnvironment, expected: false},
{endpointType: portainer.AgentOnKubernetesEnvironment, expected: true},
{endpointType: portainer.EdgeAgentOnKubernetesEnvironment, expected: true},
}
for _, test := range tests {
ans := IsAgentEndpoint(&portainer.Endpoint{Type: test.endpointType})
assert.Equal(t, test.expected, ans)
}
}
func Test_FilterByExcludeIDs(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inputArray []portainer.Endpoint
inputExcludeIDs []portainer.EndpointID
asserts func(*testing.T, []portainer.Endpoint)
}{
{
name: "filter endpoints",
inputArray: []portainer.Endpoint{
{ID: portainer.EndpointID(1)},
{ID: portainer.EndpointID(2)},
{ID: portainer.EndpointID(3)},
{ID: portainer.EndpointID(4)},
},
inputExcludeIDs: []portainer.EndpointID{
portainer.EndpointID(2),
portainer.EndpointID(3),
},
asserts: func(t *testing.T, output []portainer.Endpoint) {
assert.Contains(t, output, portainer.Endpoint{ID: portainer.EndpointID(1)})
assert.NotContains(t, output, portainer.Endpoint{ID: portainer.EndpointID(2)})
assert.NotContains(t, output, portainer.Endpoint{ID: portainer.EndpointID(3)})
assert.Contains(t, output, portainer.Endpoint{ID: portainer.EndpointID(4)})
},
},
{
name: "empty input",
inputArray: []portainer.Endpoint{},
inputExcludeIDs: []portainer.EndpointID{
portainer.EndpointID(2),
},
asserts: func(t *testing.T, output []portainer.Endpoint) {
assert.Empty(t, output)
},
},
{
name: "no filter",
inputArray: []portainer.Endpoint{
{ID: portainer.EndpointID(1)},
{ID: portainer.EndpointID(2)},
},
inputExcludeIDs: []portainer.EndpointID{},
asserts: func(t *testing.T, output []portainer.Endpoint) {
assert.Len(t, output, 2)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
output := FilterByExcludeIDs(tt.inputArray, tt.inputExcludeIDs)
tt.asserts(t, output)
})
}
}
func TestUpdateEdgeEndpointHeartbeat(t *testing.T) {
t.Parallel()
endpoint := &portainer.Endpoint{
Type: portainer.EdgeAgentOnDockerEnvironment,
LastCheckInDate: time.Now().Unix(),
EdgeCheckinInterval: 5,
}
UpdateEdgeEndpointHeartbeat(endpoint, &portainer.Settings{})
require.True(t, endpoint.Heartbeat)
endpoint.LastCheckInDate = time.Now().Add(-time.Minute).Unix()
UpdateEdgeEndpointHeartbeat(endpoint, &portainer.Settings{})
require.False(t, endpoint.Heartbeat)
}
+243
View File
@@ -0,0 +1,243 @@
package endpointutils
import (
"errors"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/pkg/endpoints"
log "github.com/rs/zerolog/log"
)
var (
IsLocalEndpoint = endpoints.IsLocalEndpoint
IsKubernetesEndpoint = endpoints.IsKubernetesEndpoint
IsDockerEndpoint = endpoints.IsDockerEndpoint
IsEdgeEndpoint = endpoints.IsEdgeEndpoint
IsAgentEndpoint = endpoints.IsAgentEndpoint
EndpointSet = endpoints.EndpointSet
)
// EndpointPlatformType returns the type of the endpoint based on the environment and container engine
func EndpointPlatformType(endpoint *portainer.Endpoint) portainer.PlatformType {
switch endpoint.Type {
case portainer.DockerEnvironment, portainer.AgentOnDockerEnvironment, portainer.EdgeAgentOnDockerEnvironment:
if endpoint.ContainerEngine == portainer.ContainerEnginePodman {
return portainer.PodmanPlatformType
}
return portainer.DockerPlatformType
case portainer.KubernetesLocalEnvironment, portainer.AgentOnKubernetesEnvironment, portainer.EdgeAgentOnKubernetesEnvironment:
return portainer.KubernetesPlatformType
case portainer.AzureEnvironment:
return portainer.AzurePlatformType
}
return portainer.UnknownPlatformType
}
// FilterByExcludeIDs receives an environment(endpoint) array and returns a filtered array using an excludeIds param
func FilterByExcludeIDs(endpoints []portainer.Endpoint, excludeIds []portainer.EndpointID) []portainer.Endpoint {
if len(excludeIds) == 0 {
return endpoints
}
filteredEndpoints := make([]portainer.Endpoint, 0)
idsSet := make(map[portainer.EndpointID]bool)
for _, id := range excludeIds {
idsSet[id] = true
}
for _, endpoint := range endpoints {
if !idsSet[endpoint.ID] {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func InitialIngressClassDetection(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, factory *cli.ClientFactory) {
if endpoint.Kubernetes.Flags.IsServerIngressClassDetected {
return
}
defer func() {
endpoint.Kubernetes.Flags.IsServerIngressClassDetected = true
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
log.Debug().Err(err).Msg("unable to store found IngressClasses inside the database")
}
}()
cli, err := factory.GetPrivilegedKubeClient(endpoint)
if err != nil {
log.Debug().Err(err).Msg("unable to create kubernetes client for ingress class detection")
return
}
controllers, err := cli.GetIngressControllers()
if err != nil {
log.Debug().Err(err).Msg("failed to fetch ingressclasses")
return
}
var updatedClasses []portainer.KubernetesIngressClassConfig
for i := range controllers {
var updatedClass portainer.KubernetesIngressClassConfig
updatedClass.Name = controllers[i].ClassName
updatedClass.Type = controllers[i].Type
updatedClasses = append(updatedClasses, updatedClass)
}
endpoint.Kubernetes.Configuration.IngressClasses = updatedClasses
}
func InitialMetricsDetection(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, factory *cli.ClientFactory) {
if endpoint.Kubernetes.Flags.IsServerMetricsDetected {
return
}
defer func() {
endpoint.Kubernetes.Flags.IsServerMetricsDetected = true
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
log.Debug().Err(err).Msg("unable to enable UseServerMetrics inside the database")
}
}()
cli, err := factory.GetPrivilegedKubeClient(endpoint)
if err != nil {
log.Debug().Err(err).Msg("unable to create kubernetes client for initial metrics detection")
return
}
if _, err := cli.GetMetrics(); err != nil {
log.Debug().Err(err).Msg("unable to fetch metrics: leaving metrics collection disabled.")
return
}
endpoint.Kubernetes.Configuration.UseServerMetrics = true
}
func storageDetect(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, factory *cli.ClientFactory) error {
if endpoint.Kubernetes.Flags.IsServerStorageDetected {
return nil
}
defer func() {
endpoint.Kubernetes.Flags.IsServerStorageDetected = true
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
log.Info().Err(err).Msg("unable to enable storage class inside the database")
}
}()
cli, err := factory.GetPrivilegedKubeClient(endpoint)
if err != nil {
log.Debug().Err(err).Msg("unable to create Kubernetes client for initial storage detection")
return err
}
storage, err := cli.GetStorage()
if err != nil {
log.Debug().Err(err).Msg("unable to fetch storage classes: leaving storage classes disabled")
return err
} else if len(storage) == 0 {
log.Info().Err(err).Msg("zero storage classes found: they may be still building, retrying in 30 seconds")
return errors.New("zero storage classes found: they may be still building, retrying in 30 seconds")
}
endpoint.Kubernetes.Configuration.StorageClasses = storage
return nil
}
func InitialStorageDetection(tx dataservices.DataStoreTx, dataStore dataservices.DataStore, endpoint *portainer.Endpoint, factory *cli.ClientFactory) {
log.Info().Msg("attempting to detect storage classes in the cluster")
err := storageDetect(tx, endpoint, factory)
if err == nil {
return
}
log.Err(err).Msg("error while detecting storage classes")
endpointID := endpoint.ID
go func() {
// Retry after 30 seconds if the initial detection failed.
log.Info().Msg("retrying storage detection in 30 seconds")
time.Sleep(30 * time.Second)
err := dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
return storageDetect(tx, endpoint, factory)
})
log.Err(err).Msg("final error while detecting storage classes")
}()
}
func UpdateEdgeEndpointHeartbeat(endpoint *portainer.Endpoint, settings *portainer.Settings) {
if !IsEdgeEndpoint(endpoint) {
return
}
endpoint.Heartbeat = GetHeartbeatStatus(endpoint, settings)
}
func GetHeartbeatStatus(endpoint *portainer.Endpoint, settings *portainer.Settings) bool {
checkInInterval := getEndpointCheckinInterval(endpoint, settings)
return time.Now().Unix()-endpoint.LastCheckInDate <= int64(checkInInterval*2+20)
}
func getEndpointCheckinInterval(endpoint *portainer.Endpoint, settings *portainer.Settings) int {
if !endpoint.Edge.AsyncMode {
if endpoint.EdgeCheckinInterval > 0 {
return endpoint.EdgeCheckinInterval
}
return settings.EdgeAgentCheckinInterval
}
defaultInterval := 60
intervals := [][]int{
{endpoint.Edge.PingInterval, settings.Edge.PingInterval},
{endpoint.Edge.CommandInterval, settings.Edge.CommandInterval},
{endpoint.Edge.SnapshotInterval, settings.Edge.SnapshotInterval},
}
for i := range intervals {
effectiveInterval := intervals[i][0]
if effectiveInterval <= 0 {
effectiveInterval = intervals[i][1]
}
if effectiveInterval > 0 && effectiveInterval < defaultInterval {
defaultInterval = effectiveInterval
}
}
return defaultInterval
}
func InitializeEdgeEndpointRelation(endpoint *portainer.Endpoint, tx dataservices.DataStoreTx) error {
if !IsEdgeEndpoint(endpoint) {
return nil
}
relation := &portainer.EndpointRelation{
EndpointID: endpoint.ID,
EdgeStacks: make(map[portainer.EdgeStackID]bool),
}
return tx.EndpointRelation().Create(relation)
}