chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
+242
View File
@@ -0,0 +1,242 @@
package auth
import (
"net/http"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type authenticatePayload struct {
// Username
Username string `example:"admin" validate:"required"`
// Password
Password string `example:"mypassword" validate:"required"`
}
type authenticateResponse struct {
// JWT token used to authenticate against the API
JWT string `json:"jwt" example:"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzAB"`
}
func (payload *authenticatePayload) Validate(r *http.Request) error {
if len(payload.Username) == 0 {
return errors.New("Invalid username")
}
if len(payload.Password) == 0 {
return errors.New("Invalid password")
}
return nil
}
// @id AuthenticateUser
// @summary Authenticate
// @description **Access policy**: public
// @description Use this environment(endpoint) to authenticate against Portainer using a username and password.
// @tags auth
// @accept json
// @produce json
// @param body body authenticatePayload true "Credentials used for authentication"
// @success 200 {object} authenticateResponse "Success"
// @failure 400 "Invalid request"
// @failure 422 "Invalid Credentials"
// @failure 500 "Server error"
// @router /auth [post]
func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload authenticatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var settings *portainer.Settings
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
settings, err = tx.Settings().Settings()
return err
}); err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
user, err := handler.DataStore.User().UserByUsername(payload.Username)
if err != nil {
if !handler.DataStore.IsErrObjectNotFound(err) {
return httperror.InternalServerError("Unable to retrieve a user with the specified username from the database", err)
}
if settings.AuthenticationMethod == portainer.AuthenticationInternal ||
settings.AuthenticationMethod == portainer.AuthenticationOAuth ||
(settings.AuthenticationMethod == portainer.AuthenticationLDAP && !settings.LDAPSettings.AutoCreateUsers) {
// avoid username enumeration timing attack by creating a fake user
// https://en.wikipedia.org/wiki/Timing_attack
user = &portainer.User{
Username: "portainer-fake-username",
Password: "$2a$10$abcdefghijklmnopqrstuvwx..ABCDEFGHIJKLMNOPQRSTUVWXYZ12", // fake but valid format bcrypt hash
}
}
}
// Clear any existing user caches
if user != nil {
handler.KubernetesClientFactory.ClearUserClientCache(strconv.Itoa(int(user.ID)))
}
if user != nil && isUserInitialAdmin(user) || settings.AuthenticationMethod == portainer.AuthenticationInternal {
return handler.authenticateInternal(rw, r, user, payload.Password, settings.ForceSecureCookies)
}
if settings.AuthenticationMethod == portainer.AuthenticationOAuth {
return httperror.NewError(http.StatusUnprocessableEntity, "Only initial admin is allowed to login without oauth", httperrors.ErrUnauthorized)
}
if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
return handler.authenticateLDAP(rw, r, user, payload.Username, payload.Password, &settings.LDAPSettings, settings.ForceSecureCookies)
}
return httperror.NewError(http.StatusUnprocessableEntity, "Login method is not supported", httperrors.ErrUnauthorized)
}
func isUserInitialAdmin(user *portainer.User) bool {
return int(user.ID) == 1
}
func (handler *Handler) authenticateInternal(w http.ResponseWriter, r *http.Request, user *portainer.User, password string, forceSecureCookies bool) *httperror.HandlerError {
if err := handler.CryptoService.CompareHashAndData(user.Password, password); err != nil {
return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
}
forceChangePassword := !handler.passwordStrengthChecker.Check(password)
return handler.writeToken(w, r, user, forceChangePassword, forceSecureCookies)
}
func (handler *Handler) authenticateLDAP(w http.ResponseWriter, r *http.Request, user *portainer.User, username, password string, ldapSettings *portainer.LDAPSettings, forceSecureCookies bool) *httperror.HandlerError {
if err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings); err != nil {
if errors.Is(err, httperrors.ErrUnauthorized) {
return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
}
return httperror.InternalServerError("Unable to authenticate user against LDAP", err)
}
if user == nil {
user = &portainer.User{
Username: username,
Role: portainer.StandardUserRole,
PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
}
if err := handler.DataStore.User().Create(user); err != nil {
return httperror.InternalServerError("Unable to persist user inside the database", err)
}
}
if err := handler.syncUserTeamsWithLDAPGroups(user, ldapSettings); err != nil {
log.Warn().Err(err).Msg("unable to automatically sync user teams with ldap")
}
return handler.writeToken(w, r, user, false, forceSecureCookies)
}
func (handler *Handler) writeToken(w http.ResponseWriter, r *http.Request, user *portainer.User, forceChangePassword bool, forceSecureCookies bool) *httperror.HandlerError {
tokenData := composeTokenData(user, forceChangePassword)
return handler.persistAndWriteToken(w, r, tokenData, forceSecureCookies)
}
func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, r *http.Request, tokenData *portainer.TokenData, forceSecureCookies bool) *httperror.HandlerError {
token, expirationTime, err := handler.JWTService.GenerateToken(tokenData)
if err != nil {
return httperror.InternalServerError("Unable to generate JWT token", err)
}
security.AddAuthCookie(w, token, expirationTime, handler.isSecureCookie(r, forceSecureCookies))
return response.JSON(w, &authenticateResponse{JWT: token})
}
func (handler *Handler) isSecureCookie(r *http.Request, forceSecureCookies bool) bool {
return r.TLS != nil || middlewares.IsHTTPSRequest(r) || forceSecureCookies
}
func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settings *portainer.LDAPSettings) error {
// only sync if there is a group base DN
if len(settings.GroupSearchSettings) == 0 || len(settings.GroupSearchSettings[0].GroupBaseDN) == 0 {
return nil
}
teams, err := handler.DataStore.Team().ReadAll()
if err != nil {
return err
}
userGroups, err := handler.LDAPService.GetUserGroups(user.Username, settings)
if err != nil {
return err
}
userMemberships, err := handler.DataStore.TeamMembership().TeamMembershipsByUserID(user.ID)
if err != nil {
return err
}
for _, team := range teams {
if !teamExists(team.Name, userGroups) || teamMembershipExists(team.ID, userMemberships) {
continue
}
membership := &portainer.TeamMembership{
UserID: user.ID,
TeamID: team.ID,
Role: portainer.TeamMember,
}
if err := handler.DataStore.TeamMembership().Create(membership); err != nil {
return err
}
}
return nil
}
func teamExists(teamName string, ldapGroups []string) bool {
for _, group := range ldapGroups {
if strings.EqualFold(group, teamName) {
return true
}
}
return false
}
func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamMembership) bool {
for _, membership := range memberships {
if membership.TeamID == teamID {
return true
}
}
return false
}
func composeTokenData(user *portainer.User, forceChangePassword bool) *portainer.TokenData {
return &portainer.TokenData{
ID: user.ID,
Username: user.Username,
Role: user.Role,
ForceChangePassword: forceChangePassword,
}
}
+122
View File
@@ -0,0 +1,122 @@
package auth
import (
"context"
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperrors "github.com/portainer/portainer/api/http/errors"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/rs/zerolog/log"
)
type oauthPayload struct {
// OAuth code returned from OAuth Provided
Code string
}
func (payload *oauthPayload) Validate(r *http.Request) error {
if len(payload.Code) == 0 {
return errors.New("Invalid OAuth authorization code")
}
return nil
}
func (handler *Handler) authenticateOAuth(ctx context.Context, code string, settings *portainer.OAuthSettings) (string, error) {
if code == "" {
return "", errors.New("Invalid OAuth authorization code")
}
if settings == nil {
return "", errors.New("Invalid OAuth configuration")
}
username, err := handler.OAuthService.Authenticate(ctx, code, settings)
if err != nil {
return "", err
}
return username, nil
}
// @id ValidateOAuth
// @summary Authenticate with OAuth
// @description **Access policy**: public
// @tags auth
// @accept json
// @produce json
// @param body body oauthPayload true "OAuth Credentials used for authentication"
// @success 200 {object} authenticateResponse "Success"
// @failure 400 "Invalid request"
// @failure 422 "Invalid Credentials"
// @failure 500 "Server error"
// @router /auth/oauth/validate [post]
func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload oauthPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var settings *portainer.Settings
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
settings, err = tx.Settings().Settings()
return err
}); err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
if settings.AuthenticationMethod != portainer.AuthenticationOAuth {
return httperror.Forbidden("OAuth authentication is not enabled", errors.New("OAuth authentication is not enabled"))
}
username, err := handler.authenticateOAuth(r.Context(), payload.Code, &settings.OAuthSettings)
if err != nil {
log.Debug().Err(err).Msg("OAuth authentication error")
return httperror.InternalServerError("Unable to authenticate through OAuth", httperrors.ErrUnauthorized)
}
user, err := handler.DataStore.User().UserByUsername(username)
if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
return httperror.InternalServerError("Unable to retrieve a user with the specified username from the database", err)
}
if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
return httperror.Forbidden("Account not created beforehand in Portainer and automatic user provisioning not enabled", httperrors.ErrUnauthorized)
}
if user == nil {
user = &portainer.User{
Username: username,
Role: portainer.StandardUserRole,
}
err = handler.DataStore.User().Create(user)
if err != nil {
return httperror.InternalServerError("Unable to persist user inside the database", err)
}
if settings.OAuthSettings.DefaultTeamID != 0 {
membership := &portainer.TeamMembership{
UserID: user.ID,
TeamID: settings.OAuthSettings.DefaultTeamID,
Role: portainer.TeamMember,
}
err = handler.DataStore.TeamMembership().Create(membership)
if err != nil {
return httperror.InternalServerError("Unable to persist team membership inside the database", err)
}
}
}
return handler.writeToken(w, r, user, false, settings.ForceSecureCookies)
}
+49
View File
@@ -0,0 +1,49 @@
package auth
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle authentication operations.
type Handler struct {
*mux.Router
DataStore dataservices.DataStore
CryptoService portainer.CryptoService
JWTService portainer.JWTService
LDAPService portainer.LDAPService
OAuthService portainer.OAuthService
ProxyManager *proxy.Manager
KubernetesTokenCacheManager *kubernetes.TokenCacheManager
KubernetesClientFactory *cli.ClientFactory
passwordStrengthChecker security.PasswordStrengthChecker
bouncer security.BouncerService
}
// NewHandler creates a handler to manage authentication operations.
func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimiter, passwordStrengthChecker security.PasswordStrengthChecker, kubernetesClientFactory *cli.ClientFactory) *Handler {
h := &Handler{
Router: mux.NewRouter(),
passwordStrengthChecker: passwordStrengthChecker,
bouncer: bouncer,
KubernetesClientFactory: kubernetesClientFactory,
}
h.Handle("/auth/oauth/validate",
rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.validateOAuth)))).Methods(http.MethodPost)
h.Handle("/auth",
rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
h.Handle("/auth/logout",
bouncer.PublicAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
return h
}
+46
View File
@@ -0,0 +1,46 @@
package auth
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/logoutcontext"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id Logout
// @summary Logout
// @description **Access policy**: public
// @security ApiKeyAuth
// @security jwt
// @tags auth
// @success 204 "Success"
// @failure 500 "Server error"
// @router /auth/logout [post]
func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
tokenData, _ := handler.bouncer.CookieAuthLookup(r)
if tokenData != nil {
handler.KubernetesTokenCacheManager.RemoveUserFromCache(tokenData.ID)
handler.KubernetesClientFactory.ClearUserClientCache(strconv.Itoa(int(tokenData.ID)))
logoutcontext.Cancel(tokenData.Token)
handler.bouncer.RevokeJWT(tokenData.Token)
}
var settings *portainer.Settings
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
settings, err = tx.Settings().Settings()
return err
}); err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
security.RemoveAuthCookie(w, handler.isSecureCookie(r, settings.ForceSecureCookies))
return response.Empty(w)
}
+109
View File
@@ -0,0 +1,109 @@
package auth
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/stretchr/testify/require"
)
type mockBouncer struct {
security.BouncerService
}
func NewMockBouncer() *mockBouncer {
return &mockBouncer{BouncerService: testhelpers.NewTestRequestBouncer()}
}
func (*mockBouncer) CookieAuthLookup(r *http.Request) (*portainer.TokenData, error) {
return &portainer.TokenData{
ID: 1,
Username: "testuser",
Token: "valid-token",
}, nil
}
func TestLogout(t *testing.T) {
t.Parallel()
h := NewHandler(NewMockBouncer(), nil, nil, nil)
h.KubernetesTokenCacheManager = kubernetes.NewTokenCacheManager()
_, h.DataStore = datastore.MustNewTestStore(t, true, false)
k, err := cli.NewClientFactory(nil, nil, nil, "", "", "")
require.NoError(t, err)
h.KubernetesClientFactory = k
rr := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/auth/logout", nil)
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusNoContent, rr.Code)
}
func TestLogoutNoPanic(t *testing.T) {
t.Parallel()
h := NewHandler(testhelpers.NewTestRequestBouncer(), nil, nil, nil)
_, h.DataStore = datastore.MustNewTestStore(t, true, false)
rr := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/auth/logout", nil)
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusNoContent, rr.Code)
}
func TestLogout_ClearsCookie(t *testing.T) {
tests := []struct {
name string
forceSecureCookies bool
wantSecure bool
}{
{name: "clears cookie without secure flag", forceSecureCookies: false, wantSecure: false},
{name: "clears cookie with secure flag when ForceSecureCookies is set", forceSecureCookies: true, wantSecure: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
h := NewHandler(NewMockBouncer(), nil, nil, nil)
h.KubernetesTokenCacheManager = kubernetes.NewTokenCacheManager()
_, h.DataStore = datastore.MustNewTestStore(t, true, false)
k, err := cli.NewClientFactory(nil, nil, nil, "", "", "")
require.NoError(t, err)
h.KubernetesClientFactory = k
err = h.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.Settings().UpdateSettings(&portainer.Settings{ForceSecureCookies: tc.forceSecureCookies})
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest("POST", "/auth/logout", nil)
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusNoContent, rr.Code)
cookies := rr.Result().Cookies()
var authCookie *http.Cookie
for _, c := range cookies {
if c.Name == portainer.AuthCookieKey {
authCookie = c
break
}
}
require.NotNil(t, authCookie, "expected auth cookie to be present in response so the browser can clear it")
require.Empty(t, authCookie.Value)
require.Equal(t, -1, authCookie.MaxAge)
require.Equal(t, tc.wantSecure, authCookie.Secure)
})
}
}
+58
View File
@@ -0,0 +1,58 @@
package backup
import (
"net/http"
"os"
"path/filepath"
operations "github.com/portainer/portainer/api/backup"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/rs/zerolog/log"
)
type (
backupPayload struct {
Password string
}
)
func (p *backupPayload) Validate(r *http.Request) error {
return nil
}
// @id Backup
// @summary Creates an archive with a system data snapshot that could be used to restore the system.
// @description Creates an archive with a system data snapshot that could be used to restore the system.
// @description **Access policy**: admin
// @tags backup
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce octet-stream
// @param body body backupPayload false "An object contains the password to encrypt the backup with"
// @success 200 "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /backup [post]
func (h *Handler) backup(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload backupPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
archivePath, err := operations.CreateBackupArchive(payload.Password, h.gate, h.dataStore, h.filestorePath)
if err != nil {
return httperror.InternalServerError("Failed to create backup", err)
}
defer func() {
if err := os.RemoveAll(filepath.Dir(archivePath)); err != nil {
log.Warn().Err(err).Msg("failed to remove backup temp folder")
}
}()
w.Header().Set("Content-Disposition", "attachment; filename=portainer-backup_"+filepath.Base(archivePath))
http.ServeFile(w, r, archivePath)
return nil
}
+166
View File
@@ -0,0 +1,166 @@
package backup
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// prepareFilestorePath copies the test assets to an isolated temp dir so
// parallel tests don't share the same filestorePath and interfere with each other.
func prepareFilestorePath(t *testing.T) string {
t.Helper()
tmpDir := t.TempDir()
err := os.CopyFS(tmpDir, os.DirFS("./test_assets/handler_test"))
require.NoError(t, err)
return tmpDir
}
func init() {
fips.InitFIPS(false)
}
func listFiles(t *testing.T, dir string) []string {
items := make([]string, 0)
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if path == dir {
return nil
}
items = append(items, path)
return nil
})
require.NoError(t, err)
return items
}
func contains(t *testing.T, list []string, path string) {
assert.Contains(t, list, path)
copyContent, err := os.ReadFile(path)
require.NoError(t, err)
assert.Equal(t, "content\n", string(copyContent))
}
func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T) {
t.Parallel()
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":""}`))
w := httptest.NewRecorder()
gate := offlinegate.NewOfflineGate()
adminMonitor := adminmonitor.New(time.Hour, nil)
handlerErr := NewHandler(
testhelpers.NewTestRequestBouncer(),
testhelpers.NewDatastore(),
gate,
prepareFilestorePath(t),
func() {},
adminMonitor).backup(w, r)
assert.Nil(t, handlerErr, "Handler should not fail")
response := w.Result()
body, err := io.ReadAll(response.Body)
require.NoError(t, err)
err = response.Body.Close()
require.NoError(t, err)
tmpdir := t.TempDir()
archivePath := filesystem.JoinPaths(tmpdir, "archive.tar.gz")
if err := os.WriteFile(archivePath, body, 0600); err != nil {
t.Fatal("Failed to save downloaded .tar.gz archive: ", err)
}
cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
if err := cmd.Run(); err != nil {
t.Fatal("Failed to extract archive: ", err)
}
createdFiles := listFiles(t, tmpdir)
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "portainer.key"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "portainer.pub"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "tls", "file1"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "tls", "file2"))
assert.NotContains(t, createdFiles, filesystem.JoinPaths(tmpdir, "extra_file"))
assert.NotContains(t, createdFiles, filesystem.JoinPaths(tmpdir, "extra_folder", "file1"))
}
func Test_backupHandlerWithPassword_shouldCreateEncryptedATarballArchive(t *testing.T) {
t.Parallel()
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":"secret"}`))
w := httptest.NewRecorder()
gate := offlinegate.NewOfflineGate()
adminMonitor := adminmonitor.New(time.Hour, nil)
handlerErr := NewHandler(
testhelpers.NewTestRequestBouncer(),
testhelpers.NewDatastore(),
gate,
prepareFilestorePath(t),
func() {},
adminMonitor).backup(w, r)
assert.Nil(t, handlerErr, "Handler should not fail")
response := w.Result()
body, _ := io.ReadAll(response.Body)
err := response.Body.Close()
require.NoError(t, err)
tmpdir := t.TempDir()
dr, err := crypto.AesDecrypt(bytes.NewReader(body), []byte("secret"))
if err != nil {
t.Fatal("Failed to decrypt archive")
}
archivePath := filesystem.JoinPaths(tmpdir, "archive.tag.gz")
archive, err := os.Create(archivePath)
require.NoError(t, err)
defer func() {
err := archive.Close()
require.NoError(t, err)
}()
_, err = io.Copy(archive, dr)
require.NoError(t, err)
cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
if err := cmd.Run(); err != nil {
t.Fatal("Failed to extract archive: ", err)
}
createdFiles := listFiles(t, tmpdir)
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "portainer.key"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "portainer.pub"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "tls", "file1"))
contains(t, createdFiles, filesystem.JoinPaths(tmpdir, "tls", "file2"))
assert.NotContains(t, createdFiles, filesystem.JoinPaths(tmpdir, "extra_file"))
assert.NotContains(t, createdFiles, filesystem.JoinPaths(tmpdir, "extra_folder", "file1"))
}
+68
View File
@@ -0,0 +1,68 @@
package backup
import (
"context"
"net/http"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is an http handler responsible for backup and restore portainer state
type Handler struct {
*mux.Router
bouncer security.BouncerService
dataStore dataservices.DataStore
gate *offlinegate.OfflineGate
filestorePath string
shutdownTrigger context.CancelFunc
adminMonitor *adminmonitor.Monitor
SetupToken string
}
// NewHandler creates an new instance of backup handler
func NewHandler(
bouncer security.BouncerService,
dataStore dataservices.DataStore,
gate *offlinegate.OfflineGate,
filestorePath string,
shutdownTrigger context.CancelFunc,
adminMonitor *adminmonitor.Monitor,
) *Handler {
h := &Handler{
Router: mux.NewRouter(),
bouncer: bouncer,
dataStore: dataStore,
gate: gate,
filestorePath: filestorePath,
shutdownTrigger: shutdownTrigger,
adminMonitor: adminMonitor,
}
h.Handle("/backup", bouncer.RestrictedAccess(adminAccess(httperror.LoggerHandler(h.backup)))).Methods(http.MethodPost)
h.Handle("/restore", bouncer.PublicAccess(httperror.LoggerHandler(h.restore))).Methods(http.MethodPost)
return h
}
func adminAccess(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user info from request context", err)
return
}
if !securityContext.IsAdmin {
httperror.WriteError(w, http.StatusUnauthorized, "User is not authorized to perform the action", nil)
return
}
next.ServeHTTP(w, r)
})
}
+74
View File
@@ -0,0 +1,74 @@
package backup
import (
"bytes"
"io"
"net/http"
"github.com/pkg/errors"
operations "github.com/portainer/portainer/api/backup"
"github.com/portainer/portainer/api/http/security/setuptoken"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
)
type restorePayload struct {
FileContent []byte
FileName string
Password string
}
// @id Restore
// @summary Triggers a system restore using provided backup file
// @description Triggers a system restore using provided backup file
// @description **Access policy**: public (requires the X-Setup-Token header on an uninitialized instance unless --no-setup-token is set)
// @tags backup
// @accept json
// @param X-Setup-Token header string false "Setup token (required when instance is uninitialized and --no-setup-token is not set)"
// @param restorePayload body restorePayload true "Restore request payload"
// @success 200 "Success"
// @failure 400 "Invalid request"
// @failure 403 "Access denied - invalid or missing setup token"
// @failure 500 "Server error"
// @router /restore [post]
func (h *Handler) restore(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
if err := setuptoken.Validate(r, h.SetupToken); err != nil {
return err
}
initialized, err := h.adminMonitor.WasInitialized()
if err != nil {
return httperror.InternalServerError("Failed to check system initialization", err)
}
if initialized {
return httperror.BadRequest("Cannot restore already initialized instance", errors.New("system already initialized"))
}
h.adminMonitor.Stop()
var payload restorePayload
err = decodeForm(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var archiveReader io.Reader = bytes.NewReader(payload.FileContent)
err = operations.RestoreArchive(archiveReader, payload.Password, h.filestorePath, h.gate, h.dataStore, h.shutdownTrigger)
if err != nil {
return httperror.InternalServerError("Failed to restore the backup", err)
}
return nil
}
func decodeForm(r *http.Request, p *restorePayload) error {
content, name, err := request.RetrieveMultiPartFormFile(r, "file")
if err != nil {
return err
}
p.FileContent = content
p.FileName = name
password, _ := request.RetrieveMultiPartFormValue(r, "password", true)
p.Password = password
return nil
}
+189
View File
@@ -0,0 +1,189 @@
package backup
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/api/http/security/setuptoken"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_restoreArchive_usingCombinationOfPasswords(t *testing.T) {
t.Parallel()
tests := []struct {
name string
backupPassword string
restorePassword string
fails bool
}{
{
name: "empty password to both encrypt and decrypt",
backupPassword: "",
restorePassword: "",
fails: false,
},
{
name: "same password to encrypt and decrypt",
backupPassword: "secret",
restorePassword: "secret",
fails: false,
},
{
name: "different passwords to encrypt and decrypt",
backupPassword: "secret",
restorePassword: "terces",
fails: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
datastore := testhelpers.NewDatastore(
testhelpers.WithUsers([]portainer.User{}),
testhelpers.WithEdgeJobs([]portainer.EdgeJob{}),
)
adminMonitor := adminmonitor.New(time.Hour, datastore)
h := NewHandler(
testhelpers.NewTestRequestBouncer(),
datastore,
offlinegate.NewOfflineGate(),
prepareFilestorePath(t),
func() {},
adminMonitor,
)
// backup
archive := backup(t, h, test.backupPassword)
// restore
w := httptest.NewRecorder()
r := prepareMultipartRequest(t, test.restorePassword, archive)
restoreErr := h.restore(w, r)
assert.Equal(t, test.fails, restoreErr != nil, "Didn't meet expectation of failing restore handler")
})
}
}
func Test_restoreArchive_shouldFailIfSystemWasAlreadyInitialized(t *testing.T) {
t.Parallel()
admin := portainer.User{
Role: portainer.AdministratorRole,
}
datastore := testhelpers.NewDatastore(
testhelpers.WithUsers([]portainer.User{admin}),
testhelpers.WithEdgeJobs([]portainer.EdgeJob{}),
)
adminMonitor := adminmonitor.New(time.Hour, datastore)
h := NewHandler(testhelpers.NewTestRequestBouncer(),
datastore,
offlinegate.NewOfflineGate(),
prepareFilestorePath(t),
func() {},
adminMonitor,
)
// backup
archive := backup(t, h, "password")
// restore
w := httptest.NewRecorder()
r := prepareMultipartRequest(t, "password", archive)
restoreErr := h.restore(w, r)
assert.NotNil(t, restoreErr, "Should fail, because system it already initialized")
assert.Equal(t, "Cannot restore already initialized instance", restoreErr.Message, "Should fail with certain error")
}
func backup(t *testing.T, h *Handler, password string) []byte {
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"password":"%s"}`, password)))
w := httptest.NewRecorder()
backupErr := h.backup(w, r)
assert.Nil(t, backupErr, "Backup should not fail")
response := w.Result()
archive, err := io.ReadAll(response.Body)
require.NoError(t, err)
err = response.Body.Close()
require.NoError(t, err)
return archive
}
func Test_restore_setupTokenGate(t *testing.T) {
t.Parallel()
datastore := testhelpers.NewDatastore(
testhelpers.WithUsers([]portainer.User{}),
testhelpers.WithEdgeJobs([]portainer.EdgeJob{}),
)
adminMonitor := adminmonitor.New(time.Hour, datastore)
h := NewHandler(
testhelpers.NewTestRequestBouncer(),
datastore,
offlinegate.NewOfflineGate(),
prepareFilestorePath(t),
func() {},
adminMonitor,
)
h.SetupToken = "secret-token"
t.Run("403 without token header", func(t *testing.T) {
err := h.restore(httptest.NewRecorder(), prepareMultipartRequest(t, "", []byte("x")))
require.Error(t, err)
assert.Equal(t, http.StatusForbidden, err.StatusCode)
})
t.Run("403 with wrong token", func(t *testing.T) {
r := prepareMultipartRequest(t, "", []byte("x"))
r.Header.Set(setuptoken.HeaderName, "wrong")
err := h.restore(httptest.NewRecorder(), r)
require.Error(t, err)
assert.Equal(t, http.StatusForbidden, err.StatusCode)
})
t.Run("passes gate with correct token", func(t *testing.T) {
archive := backup(t, h, "")
r := prepareMultipartRequest(t, "", archive)
r.Header.Set(setuptoken.HeaderName, "secret-token")
require.Nil(t, h.restore(httptest.NewRecorder(), r))
})
}
func prepareMultipartRequest(t *testing.T, password string, file []byte) *http.Request {
var body bytes.Buffer
w := multipart.NewWriter(&body)
err := w.WriteField("password", password)
require.NoError(t, err)
fw, err := w.CreateFormFile("file", "filename")
require.NoError(t, err)
_, err = io.Copy(fw, bytes.NewReader(file))
require.NoError(t, err)
r := httptest.NewRequest(http.MethodPost, "http://localhost/", &body)
r.Header.Set("Content-Type", w.FormDataContentType())
err = w.Close()
require.NoError(t, err)
return r
}
@@ -0,0 +1 @@
content
@@ -0,0 +1 @@
content
@@ -0,0 +1 @@
content
@@ -0,0 +1 @@
content
@@ -0,0 +1 @@
content
@@ -0,0 +1 @@
content
@@ -0,0 +1,530 @@
package customtemplates
import (
"context"
"errors"
"net/http"
"os"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/sources"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/portainer/portainer/pkg/validate"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
)
func (handler *Handler) customTemplateCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
method, err := request.RetrieveRouteVariableValue(r, "method")
if err != nil {
return httperror.BadRequest("Invalid query parameter: method", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
customTemplate, err := handler.createCustomTemplate(method, r)
if err != nil {
return httperror.InternalServerError("Unable to create custom template", err)
}
customTemplate.CreatedByUserID = securityContext.UserID
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return createCustomTemplateTx(tx, customTemplate, securityContext)
})
return response.TxResponse(w, customTemplate, err)
}
func createCustomTemplateTx(tx dataservices.DataStoreTx, customTemplate *portainer.CustomTemplate, sc *security.RestrictedRequestContext) error {
existingTemplates, err := tx.CustomTemplate().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom templates from the database", err)
}
for _, existing := range existingTemplates {
if existing.Title == customTemplate.Title {
return httperror.InternalServerError("Template name must be unique", errors.New("Template name must be unique"))
}
}
if err := tx.CustomTemplate().Create(customTemplate); err != nil {
return httperror.InternalServerError("Unable to create custom template", err)
}
resourceControl := authorization.NewPrivateResourceControl(strconv.Itoa(int(customTemplate.ID)), portainer.CustomTemplateResourceControl, sc.UserID)
if err := tx.ResourceControl().Create(resourceControl); err != nil {
return httperror.InternalServerError("Unable to persist resource control inside the database", err)
}
customTemplate.ResourceControl = resourceControl
userContext := source.NewUserContext(sc.User, sc.UserMemberships)
populateGitConfig(tx, userContext, customTemplate)
return nil
}
func (handler *Handler) createCustomTemplate(method string, r *http.Request) (*portainer.CustomTemplate, error) {
switch method {
case "string":
return handler.createCustomTemplateFromFileContent(r)
case "repository":
return handler.createCustomTemplateFromGitRepository(r)
case "file":
return handler.createCustomTemplateFromFileUpload(r)
}
return nil, errors.New("Invalid value for query parameter: method. Value must be one of: string, repository or file")
}
type customTemplateFromFileContentPayload struct {
// URL of the template's logo
Logo string `example:"https://portainer.io/img/logo.svg"`
// Title of the template
Title string `example:"Nginx" validate:"required"`
// Description of the template
Description string `example:"High performance web server" validate:"required"`
// A note that will be displayed in the UI. Supports HTML content
Note string `example:"This is my <b>custom</b> template"`
// Platform associated to the template.
// Valid values are: 1 - 'linux', 2 - 'windows'
// Required for Docker stacks
Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
// Type of created stack:
// * 1 - swarm
// * 2 - compose
// * 3 - kubernetes
Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
// Content of stack file
FileContent string `validate:"required"`
// Definitions of variables in the stack file
Variables []portainer.CustomTemplateVariableDefinition
// EdgeTemplate indicates if this template purpose for Edge Stack
EdgeTemplate bool `example:"false"`
}
func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) error {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if len(payload.FileContent) == 0 {
return errors.New("Invalid file content")
}
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
return errors.New("Invalid custom template type")
}
if !IsValidNote(payload.Note) {
return errors.New("Invalid note. <img> tag is not supported")
}
return ValidateVariablesDefinitions(payload.Variables)
}
// @id CustomTemplateCreateString
// @summary Create a custom template
// @description Create a custom template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body customTemplateFromFileContentPayload true "body"
// @success 200 {object} portainer.CustomTemplate
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /custom_templates/create/string [post]
func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*portainer.CustomTemplate, error) {
var payload customTemplateFromFileContentPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return nil, err
}
customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
customTemplate := &portainer.CustomTemplate{
ID: portainer.CustomTemplateID(customTemplateID),
Title: payload.Title,
EntryPoint: filesystem.ComposeFileDefaultName,
Description: payload.Description,
Note: payload.Note,
Platform: (payload.Platform),
Type: (payload.Type),
Logo: payload.Logo,
Variables: payload.Variables,
EdgeTemplate: payload.EdgeTemplate,
}
templateFolder := strconv.Itoa(customTemplateID)
projectPath, err := handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
if err != nil {
return nil, err
}
customTemplate.ProjectPath = projectPath
return customTemplate, nil
}
type customTemplateFromGitRepositoryPayload struct {
// URL of the template's logo
Logo string `example:"https://portainer.io/img/logo.svg"`
// Title of the template
Title string `example:"Nginx" validate:"required"`
// Description of the template
Description string `example:"High performance web server" validate:"required"`
// A note that will be displayed in the UI. Supports HTML content
Note string `example:"This is my <b>custom</b> template"`
// Platform associated to the template.
// Valid values are: 1 - 'linux', 2 - 'windows'
// Required for Docker stacks
Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
// Type of created stack:
// * 1 - swarm
// * 2 - compose
// * 3 - kubernetes
Type portainer.StackType `example:"1" enums:"1,2" validate:"required"`
// SourceID references an existing Source for git credentials/URL.
// When set, the inline URL and authentication fields are ignored.
SourceID portainer.SourceID `example:"1" validate:"required"`
// Deprecated: use SourceID instead. URL of a Git repository hosting the Stack file.
RepositoryURL string `example:"https://github.com/openfaas/faas"`
// Reference name of a Git repository hosting the Stack file
RepositoryReferenceName string `example:"refs/heads/master"`
// Deprecated: use SourceID instead. Use basic authentication to clone the Git repository.
RepositoryAuthentication bool `example:"true"`
// Deprecated: use SourceID instead. Username used in basic authentication. Required when RepositoryAuthentication is true.
RepositoryUsername string `example:"myGitUsername"`
// Deprecated: use SourceID instead. Password used in basic authentication. Required when RepositoryAuthentication is true.
RepositoryPassword string `example:"myGitPassword"`
// Path to the Stack file inside the Git repository
ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
// Definitions of variables in the stack file
Variables []portainer.CustomTemplateVariableDefinition
// Deprecated: use SourceID instead. TLSSkipVerify skips SSL verification when cloning the Git repository.
TLSSkipVerify bool `example:"false"`
// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
IsComposeFormat bool `example:"false"`
// EdgeTemplate indicates if this template purpose for Edge Stack
EdgeTemplate bool `example:"false"`
}
func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if payload.SourceID == 0 {
if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if payload.RepositoryAuthentication && (len(payload.RepositoryUsername) == 0 || len(payload.RepositoryPassword) == 0) {
return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
}
}
if len(payload.ComposeFilePathInRepository) == 0 {
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack {
return errors.New("Invalid custom template type")
}
if !IsValidNote(payload.Note) {
return errors.New("Invalid note. <img> tag is not supported")
}
return ValidateVariablesDefinitions(payload.Variables)
}
// @id CustomTemplateCreateRepository
// @summary Create a custom template
// @description Create a custom template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body customTemplateFromGitRepositoryPayload true "Required when using method=repository"
// @success 200 {object} portainer.CustomTemplate
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /custom_templates/create/repository [post]
func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (*portainer.CustomTemplate, error) {
var payload customTemplateFromGitRepositoryPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return nil, err
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve info from request context", err)
}
customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
customTemplate := &portainer.CustomTemplate{
ID: portainer.CustomTemplateID(customTemplateID),
Title: payload.Title,
Description: payload.Description,
Note: payload.Note,
Platform: payload.Platform,
Type: payload.Type,
Logo: payload.Logo,
Variables: payload.Variables,
IsComposeFormat: payload.IsComposeFormat,
EdgeTemplate: payload.EdgeTemplate,
}
getProjectPath := func() string {
return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
}
projectPath := getProjectPath()
customTemplate.ProjectPath = projectPath
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
gitConfig, httpErr := sources.ResolveRepoConfig(handler.DataStore, userContext, sources.RepoConfigInput{
SourceID: payload.SourceID,
ReferenceName: payload.RepositoryReferenceName,
ConfigFilePath: payload.ComposeFilePathInRepository,
RepositoryURL: payload.RepositoryURL,
TLSSkipVerify: payload.TLSSkipVerify,
RepositoryAuthentication: payload.RepositoryAuthentication,
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
})
if httpErr != nil {
return nil, httpErr
}
if err := ssrf.CheckURL(r.Context(), gitConfig.URL); err != nil {
return nil, err
}
commitHash, err := stackutils.DownloadGitRepository(context.TODO(), gitConfig, handler.GitService, getProjectPath)
if err != nil {
return nil, err
}
sourceID := payload.SourceID
if sourceID == 0 {
src, err := workflows.FindOrCreateGitSource(handler.DataStore, userContext, &portainer.Source{
Name: gittypes.RepoName(gitConfig.URL),
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: gitConfig.URL,
Authentication: gitConfig.Authentication,
TLSSkipVerify: gitConfig.TLSSkipVerify,
},
})
if err != nil {
return nil, err
}
sourceID = src.ID
}
customTemplate.Artifact = &portainer.Artifact{
Files: []portainer.ArtifactFile{{
SourceID: sourceID,
Path: gitConfig.ConfigFilePath,
Ref: gitConfig.ReferenceName,
Hash: commitHash,
}},
}
isValidProject := true
defer func() {
if !isValidProject {
if err := handler.FileService.RemoveDirectory(projectPath); err != nil {
log.Warn().Err(err).Msg("unable to remove git repository directory")
}
}
}()
entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath)
exists, err := handler.FileService.FileExists(entryPath)
if err != nil || !exists {
isValidProject = false
}
if err != nil {
return nil, err
}
if !exists {
if payload.Type == portainer.KubernetesStack {
return nil, errors.New("Invalid Manifest file, ensure that the Manifest file path is correct")
}
return nil, errors.New("Invalid Compose file, ensure that the Compose file path is correct")
}
info, err := os.Lstat(entryPath)
if err != nil {
isValidProject = false
return nil, err
}
if info.Mode()&os.ModeSymlink != 0 { // entry is a symlink
isValidProject = false
return nil, errors.New("Invalid Compose file, ensure that the Compose file is not a symbolic link")
}
return customTemplate, nil
}
type customTemplateFromFileUploadPayload struct {
Logo string
Title string
Description string
Note string
Platform portainer.CustomTemplatePlatform
// Type of created stack:
// * 1 - swarm
// * 2 - compose
// * 3 - kubernetes
Type portainer.StackType
FileContent []byte
// Definitions of variables in the stack file
Variables []portainer.CustomTemplateVariableDefinition
// EdgeTemplate indicates if this template purpose for Edge Stack
EdgeTemplate bool `example:"false"`
}
func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) error {
title, err := request.RetrieveMultiPartFormValue(r, "Title", false)
if err != nil {
return errors.New("Invalid custom template title")
}
payload.Title = title
description, err := request.RetrieveMultiPartFormValue(r, "Description", false)
if err != nil {
return errors.New("Invalid custom template description")
}
payload.Description = description
logo, _ := request.RetrieveMultiPartFormValue(r, "Logo", true)
payload.Logo = logo
note, _ := request.RetrieveMultiPartFormValue(r, "Note", true)
if !IsValidNote(note) {
return errors.New("Invalid note. <img> tag is not supported")
}
payload.Note = note
typeNumeral, _ := request.RetrieveNumericMultiPartFormValue(r, "Type", true)
templateType := portainer.StackType(typeNumeral)
if templateType != portainer.KubernetesStack && templateType != portainer.DockerSwarmStack && templateType != portainer.DockerComposeStack {
return errors.New("Invalid custom template type")
}
payload.Type = templateType
platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true)
templatePlatform := portainer.CustomTemplatePlatform(platform)
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if templateType != portainer.KubernetesStack && templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
payload.Platform = templatePlatform
composeFileContent, _, err := request.RetrieveMultiPartFormFile(r, "File")
if err != nil {
return errors.New("Invalid Compose file. Ensure that the Compose file is uploaded correctly")
}
payload.FileContent = composeFileContent
varsString, _ := request.RetrieveMultiPartFormValue(r, "Variables", true)
if varsString != "" {
if err := json.Unmarshal([]byte(varsString), &payload.Variables); err != nil {
return errors.New("Invalid variables. Ensure that the variables are valid JSON")
}
if err := ValidateVariablesDefinitions(payload.Variables); err != nil {
return err
}
}
edgeTemplate, _ := request.RetrieveBooleanMultiPartFormValue(r, "EdgeTemplate", true)
payload.EdgeTemplate = edgeTemplate
return nil
}
// @id CustomTemplateCreateFile
// @summary Create a custom template
// @description Create a custom template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @accept multipart/form-data
// @produce json
// @param Title formData string true "Title of the template"
// @param Description formData string true "Description of the template"
// @param Note formData string true "A note that will be displayed in the UI. Supports HTML content"
// @param Platform formData int true "Platform associated to the template (1 - 'linux', 2 - 'windows')" Enums(1,2)
// @param Type formData int true "Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)" Enums(1,2,3)
// @param File formData file true "File"
// @param Logo formData string false "URL of the template's logo" example:"https://portainer.io/img/logo.svg"
// @param Variables formData string false "A json array of variables definitions" example:"[{\"label\":\"image\",\"description\":\"Image name\",\"defaultValue\":\"nginx:latest\",\"name\":\"image\"}]"
// @success 200 {object} portainer.CustomTemplate
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /custom_templates/create/file [post]
func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*portainer.CustomTemplate, error) {
payload := &customTemplateFromFileUploadPayload{}
if err := payload.Validate(r); err != nil {
return nil, err
}
customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
customTemplate := &portainer.CustomTemplate{
ID: portainer.CustomTemplateID(customTemplateID),
Title: payload.Title,
Description: payload.Description,
Note: payload.Note,
Platform: payload.Platform,
Type: payload.Type,
Logo: payload.Logo,
EntryPoint: filesystem.ComposeFileDefaultName,
Variables: payload.Variables,
EdgeTemplate: payload.EdgeTemplate,
}
templateFolder := strconv.Itoa(customTemplateID)
projectPath, err := handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, payload.FileContent)
if err != nil {
return nil, err
}
customTemplate.ProjectPath = projectPath
return customTemplate, nil
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,79 @@
package customtemplates
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
// @id CustomTemplateDelete
// @summary Remove a template
// @description Remove a template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @param id path int true "Template identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 403 "Access denied to resource"
// @failure 404 "Template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id} [delete]
func (handler *Handler) customTemplateDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Custom template identifier route variable", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
customTemplate, err := handler.DataStore.CustomTemplate().Read(portainer.CustomTemplateID(customTemplateID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(strconv.Itoa(customTemplateID), portainer.CustomTemplateResourceControl)
if err != nil {
return httperror.InternalServerError("Unable to retrieve a resource control associated to the custom template", err)
}
customTemplate.ResourceControl = resourceControl
access := userCanEditTemplate(customTemplate, securityContext)
if !access {
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}
err = handler.DataStore.CustomTemplate().Delete(portainer.CustomTemplateID(customTemplateID))
if err != nil {
return httperror.InternalServerError("Unable to remove the custom template from the database", err)
}
err = handler.FileService.RemoveDirectory(customTemplate.ProjectPath)
if err != nil {
log.Warn().Err(err).Msg("Unable to remove custom template files from disk")
}
if resourceControl != nil {
err = handler.DataStore.ResourceControl().Delete(resourceControl.ID)
if err != nil {
return httperror.InternalServerError("Unable to remove the associated resource control from the database", err)
}
}
return response.Empty(w)
}
@@ -0,0 +1,296 @@
package customtemplates
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/gorilla/mux"
"github.com/stretchr/testify/require"
)
func TestCustomTemplateDelete_NotFound(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/99", nil)
r = mux.SetURLVars(r, map[string]string{"id": "99"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusNotFound, herr.StatusCode)
}
func TestCustomTemplateDelete_Forbidden(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 1,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
// User 2 did not create this template and is not an admin
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateDelete_CreatorDeniedWhenAdminOnly(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
// User 2 created the template but an admin later changed it to admins-only
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateDelete_CreatorDeniedWithoutResourceControl(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 2,
})
})
require.NoError(t, err)
// User 2 created this template but there is no resource control
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateDelete_Success(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
UserAccesses: []portainer.UserResourceAccess{{UserID: 2}},
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusNoContent, rr.Code)
err = store.ViewTx(func(tx dataservices.DataStoreTx) error {
_, err := tx.CustomTemplate().Read(1)
require.True(t, tx.IsErrObjectNotFound(err))
return nil
})
require.NoError(t, err)
}
func TestCustomTemplateDelete_AdminCanDeleteAdminOnly(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusNoContent, rr.Code)
}
func TestCustomTemplateDelete_PublicTemplateAllowsAnyUser(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 1,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
Public: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
// User 2 is not the creator but the template is public
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusNoContent, rr.Code)
}
func TestCustomTemplateDelete_NonCreatorForbiddenWithPrivateRC(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 1,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
UserAccesses: []portainer.UserResourceAccess{{UserID: 1}},
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
// User 2 is not the creator and the template has a private resource control
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateDelete_CreatorDeniedWithoutAccess(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
CreatedByUserID: 2,
})
require.NoError(t, err)
// RC exists but only grants access to user 3, not the creator (user 2)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
UserAccesses: []portainer.UserResourceAccess{{UserID: 3}},
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodDelete, "/custom_templates/1", nil)
r = mux.SetURLVars(r, map[string]string{"id": "1"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateDelete(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
@@ -0,0 +1,93 @@
package customtemplates
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type fileResponse struct {
FileContent string
}
// @id CustomTemplateFile
// @summary Get Template stack file content.
// @description Retrieve the content of the Stack file for the specified custom template
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Template identifier"
// @success 200 {object} fileResponse "Success"
// @failure 400 "Invalid request"
// @failure 404 "Custom template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id}/file [get]
func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid custom template identifier route variable", err)
}
var customTemplate *portainer.CustomTemplate
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
customTemplate, err = tx.CustomTemplate().Read(portainer.CustomTemplateID(customTemplateID))
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
resourceControl, err := tx.ResourceControl().ResourceControlByResourceIDAndType(strconv.Itoa(customTemplateID), portainer.CustomTemplateResourceControl)
if err != nil {
return httperror.InternalServerError("Unable to retrieve a resource control associated to the custom template", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user info from request context", err)
}
customTemplate.ResourceControl = resourceControl
canEdit := userCanEditTemplate(customTemplate, securityContext)
hasAccess := false
if resourceControl != nil {
teamIDs := slicesx.Map(securityContext.UserMemberships, func(m portainer.TeamMembership) portainer.TeamID {
return m.TeamID
})
hasAccess = authorization.UserCanAccessResource(securityContext.UserID, teamIDs, resourceControl)
}
if canEdit || hasAccess {
return nil
}
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}); err != nil {
return response.TxErrorResponse(err)
}
entryPath := customTemplate.EntryPoint
if customTemplate.Artifact != nil && len(customTemplate.Artifact.Files) > 0 {
entryPath = customTemplate.Artifact.Files[0].Path
}
fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath)
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
}
return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
}
@@ -0,0 +1,184 @@
package customtemplates
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestCustomTemplateFile(t *testing.T) {
t.Parallel()
handler, ds, fs := newTestHandler(t)
templateContent := "some template content"
templateEntrypoint := "entrypoint"
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
// template 1
path, err := fs.StoreCustomTemplateFileFromBytes("1", templateEntrypoint, []byte(templateContent))
require.NoError(t, err)
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{ID: 1, EntryPoint: templateEntrypoint, ProjectPath: path}))
// template 2
path, err = fs.StoreCustomTemplateFileFromBytes("2", templateEntrypoint, []byte(templateContent))
require.NoError(t, err)
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{ID: 2, EntryPoint: templateEntrypoint, ProjectPath: path}))
require.NoError(t, tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1, ResourceID: "2", Type: portainer.CustomTemplateResourceControl,
UserAccesses: []portainer.UserResourceAccess{{UserID: 2}},
TeamAccesses: []portainer.TeamResourceAccess{{TeamID: 1}},
}))
return nil
}))
test := func(templateID string, restrictedContext *security.RestrictedRequestContext) (*httptest.ResponseRecorder, *httperror.HandlerError) {
r := httptest.NewRequest(http.MethodGet, "/custom_templates/"+templateID+"/file", nil)
r = mux.SetURLVars(r, map[string]string{"id": templateID})
ctx := security.StoreRestrictedRequestContext(r, restrictedContext)
r = r.WithContext(ctx)
rr := httptest.NewRecorder()
return rr, handler.customTemplateFile(rr, r)
}
t.Run("unknown id should get not found error", func(t *testing.T) {
_, r := test("0", &security.RestrictedRequestContext{UserID: 1})
require.NotNil(t, r)
require.Equal(t, http.StatusNotFound, r.StatusCode)
})
t.Run("admin should access adminonly template", func(t *testing.T) {
rr, r := test("1", &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var res struct{ FileContent string }
require.NoError(t, json.NewDecoder(rr.Body).Decode(&res))
require.Equal(t, templateContent, res.FileContent)
})
t.Run("std should not access adminonly template", func(t *testing.T) {
_, r := test("1", &security.RestrictedRequestContext{UserID: 2})
require.NotNil(t, r)
require.Equal(t, http.StatusForbidden, r.StatusCode)
})
t.Run("std should access template via direct user access", func(t *testing.T) {
rr, r := test("2", &security.RestrictedRequestContext{UserID: 2})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var res struct{ FileContent string }
require.NoError(t, json.NewDecoder(rr.Body).Decode(&res))
require.Equal(t, templateContent, res.FileContent)
})
t.Run("std should access template via team access", func(t *testing.T) {
rr, r := test("2", &security.RestrictedRequestContext{UserID: 3, UserMemberships: []portainer.TeamMembership{{ID: 1, UserID: 3, TeamID: 1}}})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var res struct{ FileContent string }
require.NoError(t, json.NewDecoder(rr.Body).Decode(&res))
require.Equal(t, templateContent, res.FileContent)
})
t.Run("std should not access template without access", func(t *testing.T) {
_, r := test("2", &security.RestrictedRequestContext{UserID: 4})
require.NotNil(t, r)
require.Equal(t, http.StatusForbidden, r.StatusCode)
})
}
func TestCustomTemplateFile_CreatorDeniedWhenAdminOnly(t *testing.T) {
t.Parallel()
handler, store, fs := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
path, err := fs.StoreCustomTemplateFileFromBytes("5", "entrypoint", []byte("content"))
require.NoError(t, err)
err = tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 5,
EntryPoint: "entrypoint",
ProjectPath: path,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 5,
ResourceID: "5",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodGet, "/custom_templates/5/file", nil)
r = mux.SetURLVars(r, map[string]string{"id": "5"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2}))
rr := httptest.NewRecorder()
herr := handler.customTemplateFile(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateFile_GitTemplate(t *testing.T) {
t.Parallel()
handler, ds, fs := newTestHandler(t)
templateContent := "git template content"
configFilePath := "docker-compose.yml"
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://github.com/example/repo"},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
path, err := fs.StoreCustomTemplateFileFromBytes("10", configFilePath, []byte(templateContent))
require.NoError(t, err)
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 10,
EntryPoint: "should-not-be-used.yml",
ProjectPath: path,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{Path: configFilePath, SourceID: src.ID}},
},
})
}))
r := httptest.NewRequest(http.MethodGet, "/custom_templates/10/file", nil)
r = mux.SetURLVars(r, map[string]string{"id": "10"})
ctx := security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
r = r.WithContext(ctx)
rr := httptest.NewRecorder()
herr := handler.customTemplateFile(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var res struct{ FileContent string }
require.NoError(t, json.NewDecoder(rr.Body).Decode(&res))
require.Equal(t, templateContent, res.FileContent)
}
@@ -0,0 +1,162 @@
package customtemplates
import (
"context"
"net/http"
"os"
"sync"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
// @id CustomTemplateGitFetch
// @summary Fetch the latest config file content based on custom template's git repository configuration
// @description Retrieve details about a template created from git repository method.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Template identifier"
// @success 200 {object} fileResponse "Success"
// @failure 400 "Invalid request"
// @failure 404 "Custom template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id}/git_fetch [put]
func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Custom template identifier route variable", err)
}
customTemplate, err := handler.DataStore.CustomTemplate().Read(portainer.CustomTemplateID(customTemplateID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
if customTemplate.Artifact == nil || len(customTemplate.Artifact.Files) == 0 {
return httperror.BadRequest("Git configuration does not exist in this custom template", nil)
}
file := customTemplate.Artifact.Files[0]
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
var src *portainer.Source
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
src, err = tx.Source().Read(userContext, file.SourceID)
if err != nil {
return httperror.InternalServerError("Unable to retrieve git source for custom template", err)
}
return nil
}); err != nil {
return response.TxErrorResponse(err)
}
if src.Git == nil {
return httperror.InternalServerError("Source has no git configuration", nil)
}
gitConfig := &gittypes.RepoConfig{
URL: src.Git.URL,
Authentication: src.Git.Authentication,
TLSSkipVerify: src.Git.TLSSkipVerify,
ReferenceName: file.Ref,
ConfigFilePath: file.Path,
ConfigHash: file.Hash,
}
// If multiple users are trying to fetch the same custom template simultaneously, a lock needs to be added
mu, ok := handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)]
if !ok {
mu = &sync.Mutex{}
handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] = mu
}
mu.Lock()
defer mu.Unlock()
// back up the current custom template folder
backupPath, err := backupCustomTemplate(customTemplate.ProjectPath)
if err != nil {
return httperror.InternalServerError("Failed to backup the custom template folder", err)
}
// remove backup custom template folder
defer func() {
if err := cleanUpBackupCustomTemplate(backupPath); err != nil {
log.Warn().Err(err).Msg("failed to remove backup custom template folder")
}
}()
commitHash, err := stackutils.DownloadGitRepository(context.TODO(), *gitConfig, handler.GitService, func() string {
return customTemplate.ProjectPath
})
if err != nil {
log.Warn().Err(err).Msg("failed to download git repository")
if rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath); rbErr != nil {
return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr)
}
return httperror.InternalServerError("Failed to download git repository", err)
}
if customTemplate.Artifact.Files[0].Hash != commitHash {
customTemplate.Artifact.Files[0].Hash = commitHash
if err := handler.DataStore.CustomTemplate().Update(customTemplate.ID, customTemplate); err != nil {
return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
}
}
fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, gitConfig.ConfigFilePath)
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
}
return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
}
func backupCustomTemplate(projectPath string) (string, error) {
stat, err := os.Stat(projectPath)
if err != nil {
return "", err
}
backupPath := projectPath + "-backup"
if err := os.Rename(projectPath, backupPath); err != nil {
return "", err
}
return backupPath, os.Mkdir(projectPath, stat.Mode())
}
func rollbackCustomTemplate(backupPath, projectPath string) error {
if err := os.RemoveAll(projectPath); err != nil {
return err
}
return os.Rename(backupPath, projectPath)
}
func cleanUpBackupCustomTemplate(backupPath string) error {
return os.RemoveAll(backupPath)
}
@@ -0,0 +1,339 @@
package customtemplates
import (
"bytes"
"context"
"errors"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/pkg/fips"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
func init() {
fips.InitFIPS(false)
}
var testFileContent = "abcdefg"
type TestGitService struct {
portainer.GitService
targetFilePath string
}
func (g *TestGitService) CloneRepository(
_ context.Context,
destination string,
repositoryURL,
referenceName string,
username,
password string,
tlsSkipVerify bool,
) error {
time.Sleep(100 * time.Millisecond)
return createTestFile(g.targetFilePath)
}
func (g *TestGitService) LatestCommitID(
_ context.Context,
repositoryURL,
referenceName,
username,
password string,
tlsSkipVerify bool,
) (string, error) {
return "", nil
}
type TestFileService struct {
portainer.FileService
}
func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]byte, error) {
return os.ReadFile(filesystem.JoinPaths(projectPath, configFilePath))
}
type InvalidTestGitService struct {
portainer.GitService
targetFilePath string
}
func (g *InvalidTestGitService) CloneRepository(
_ context.Context,
dest,
repoUrl,
refName,
username,
password string,
tlsSkipVerify bool,
) error {
return errors.New("simulate network error")
}
func (g *InvalidTestGitService) LatestCommitID(
_ context.Context,
repositoryURL,
referenceName,
username,
password string,
tlsSkipVerify bool,
) (string, error) {
return "", nil
}
func createTestFile(targetPath string) error {
f, err := os.Create(targetPath)
if err != nil {
return err
}
defer logs.CloseAndLogErr(f)
_, err = f.WriteString(testFileContent)
return err
}
func prepareTestFolder(projectPath, filename string) error {
if err := os.MkdirAll(projectPath, fs.ModePerm); err != nil {
return err
}
return createTestFile(filesystem.JoinPaths(projectPath, filename))
}
func singleAPIRequest(h *Handler, jwt string, expect string) error {
type response struct {
FileContent string
}
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
testhelpers.AddTestSecurityCookie(req, jwt)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
return errors.New("unexpected status code: " + http.StatusText(rr.Code))
}
body, err := io.ReadAll(rr.Body)
if err != nil {
return err
}
var resp response
if err := json.Unmarshal(body, &resp); err != nil {
return err
}
if resp.FileContent != expect {
return errors.New("unexpected file content: " + resp.FileContent + ", expected: " + expect)
}
return nil
}
func Test_customTemplateGitFetch(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
// create user(s)
user1 := &portainer.User{ID: 1, Username: "user-1", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
err := store.User().Create(user1)
require.NoError(t, err, "error creating user 1")
user2 := &portainer.User{ID: 2, Username: "user-2", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
err = store.User().Create(user2)
require.NoError(t, err, "error creating user 2")
dir, err := os.Getwd()
require.NoError(t, err, "error to get working directory")
src := &portainer.Source{
ID: 1,
Type: portainer.SourceTypeGit,
Public: true,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
},
}
err = store.Source().Create(adminUserContext, src)
require.NoError(t, err, "error creating source")
const configFilePath = "test-config-path.txt"
template1 := &portainer.CustomTemplate{
ID: 1,
Title: "custom-template-1",
ProjectPath: filesystem.JoinPaths(dir, "fixtures/custom_template_1"),
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{Path: configFilePath, SourceID: src.ID}},
},
}
err = store.CustomTemplateService.Create(template1)
require.NoError(t, err, "error creating custom template 1")
// prepare testing folder
err = prepareTestFolder(template1.ProjectPath, configFilePath)
require.NoError(t, err, "error creating testing folder")
defer func() {
err := os.RemoveAll(filesystem.JoinPaths(dir, "fixtures"))
require.NoError(t, err)
}()
// setup services
jwtService, err := jwt.NewService("1h", store)
require.NoError(t, err, "Error initiating jwt service")
requestBouncer := security.NewRequestBouncer(t.Context(), store, jwtService, nil)
gitService := &TestGitService{
targetFilePath: filesystem.JoinPaths(template1.ProjectPath, configFilePath),
}
fileService := &TestFileService{}
h := NewHandler(requestBouncer, store, fileService, gitService)
// generate two standard users' tokens
jwt1, _, err := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
require.NoError(t, err)
jwt2, _, err := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
require.NoError(t, err)
t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
err := singleAPIRequest(h, jwt1, "abcdefg")
require.NoError(t, err)
})
t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) {
var g errgroup.Group
for range 5 {
g.Go(func() error {
return singleAPIRequest(h, jwt1, "abcdefg")
})
}
err := g.Wait()
require.NoError(t, err)
})
t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) {
var g errgroup.Group
for i := range 10 {
g.Go(func() error {
if i%2 == 0 {
return singleAPIRequest(h, jwt1, "abcdefg")
}
return singleAPIRequest(h, jwt2, "abcdefg")
})
}
err := g.Wait()
require.NoError(t, err)
})
t.Run("can return the expected file content after a new commit is made", func(t *testing.T) {
err := singleAPIRequest(h, jwt1, "abcdefg")
require.NoError(t, err)
testFileContent = "gfedcba"
err = singleAPIRequest(h, jwt2, "gfedcba")
require.NoError(t, err)
})
t.Run("restore git repository if it is failed to download the new git repository", func(t *testing.T) {
invalidGitService := &InvalidTestGitService{
targetFilePath: filesystem.JoinPaths(template1.ProjectPath, configFilePath),
}
h := NewHandler(requestBouncer, store, fileService, invalidGitService)
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
testhelpers.AddTestSecurityCookie(req, jwt1)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
is.Equal(http.StatusInternalServerError, rr.Code)
var errResp httperror.HandlerError
err = json.NewDecoder(rr.Body).Decode(&errResp)
require.NoError(t, err, "failed to parse error body")
assert.FileExists(t, gitService.targetFilePath, "previous git repository is not restored")
fileContent, err := os.ReadFile(gitService.targetFilePath)
require.NoError(t, err, "failed to read target file")
assert.Equal(t, "gfedcba", string(fileContent))
})
}
func TestCustomTemplateGitFetch_NilArtifactReturnsBadRequest(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
template := &portainer.CustomTemplate{ID: 1, Title: "no-git-template"}
err := store.CustomTemplateService.Create(template)
require.NoError(t, err)
h := NewHandler(testhelpers.NewTestRequestBouncer(), store, &TestFileService{}, &TestGitService{})
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusBadRequest, rr.Code)
}
func TestCustomTemplateGitFetch_EmptySourceIDsReturnsBadRequest(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
template := &portainer.CustomTemplate{
ID: 1,
Title: "empty-source-ids",
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{},
},
}
err := store.CustomTemplateService.Create(template)
require.NoError(t, err)
h := NewHandler(testhelpers.NewTestRequestBouncer(), store, &TestFileService{}, &TestGitService{})
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusBadRequest, rr.Code)
}
@@ -0,0 +1,81 @@
package customtemplates
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id CustomTemplateInspect
// @summary Inspect a custom template
// @description Retrieve details about a template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Template identifier"
// @success 200 {object} portainer.CustomTemplate "Success"
// @failure 400 "Invalid request"
// @failure 404 "Template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id} [get]
func (handler *Handler) customTemplateInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Custom template identifier route variable", err)
}
var customTemplate *portainer.CustomTemplate
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
customTemplate, err = tx.CustomTemplate().Read(portainer.CustomTemplateID(customTemplateID))
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
resourceControl, err := tx.ResourceControl().ResourceControlByResourceIDAndType(strconv.Itoa(customTemplateID), portainer.CustomTemplateResourceControl)
if err != nil {
return httperror.InternalServerError("Unable to retrieve a resource control associated to the custom template", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user info from request context", err)
}
customTemplate.ResourceControl = resourceControl
canEdit := userCanEditTemplate(customTemplate, securityContext)
hasAccess := false
if resourceControl != nil {
teamIDs := slicesx.Map(securityContext.UserMemberships, func(m portainer.TeamMembership) portainer.TeamID {
return m.TeamID
})
hasAccess = authorization.UserCanAccessResource(securityContext.UserID, teamIDs, resourceControl)
}
if !canEdit && !hasAccess {
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
populateGitConfig(tx, userContext, customTemplate)
return nil
})
return response.TxResponse(w, customTemplate, err)
}
@@ -0,0 +1,212 @@
package customtemplates
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// newTestHandler creates a handler with a test datastore, filesystem service, and common fixtures:
// admin user (ID=1), standard users (ID=2,3,4), endpoint (ID=1) with access for users 2 and 3,
// team (ID=1), and team membership for user 3.
func newTestHandler(t *testing.T) (*Handler, dataservices.DataStore, portainer.FileService) {
t.Helper()
_, ds := datastore.MustNewTestStore(t, true, false)
fs, err := filesystem.NewService(t.TempDir(), t.TempDir())
require.NoError(t, err)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole}))
require.NoError(t, tx.User().Create(&portainer.User{ID: 2, Username: "std2", Role: portainer.StandardUserRole}))
require.NoError(t, tx.User().Create(&portainer.User{ID: 3, Username: "std3", Role: portainer.StandardUserRole}))
require.NoError(t, tx.User().Create(&portainer.User{ID: 4, Username: "std4", Role: portainer.StandardUserRole}))
require.NoError(t, tx.Endpoint().Create(&portainer.Endpoint{
ID: 1,
UserAccessPolicies: portainer.UserAccessPolicies{
2: portainer.AccessPolicy{RoleID: 0},
3: portainer.AccessPolicy{RoleID: 0},
},
}))
require.NoError(t, tx.Team().Create(&portainer.Team{ID: 1}))
require.NoError(t, tx.TeamMembership().Create(&portainer.TeamMembership{ID: 1, UserID: 3, TeamID: 1, Role: portainer.TeamMember}))
return nil
}))
handler := NewHandler(testhelpers.NewTestRequestBouncer(), ds, fs, nil)
return handler, ds, fs
}
func TestInspectHandler(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{ID: 1}))
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{ID: 2}))
require.NoError(t, tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1, ResourceID: "2", Type: portainer.CustomTemplateResourceControl,
UserAccesses: []portainer.UserResourceAccess{{UserID: 2}},
TeamAccesses: []portainer.TeamResourceAccess{{TeamID: 1}},
}))
return nil
}))
test := func(templateID string, restrictedContext *security.RestrictedRequestContext) (*httptest.ResponseRecorder, *httperror.HandlerError) {
r := httptest.NewRequest(http.MethodGet, "/custom_templates/"+templateID, nil)
r = mux.SetURLVars(r, map[string]string{"id": templateID})
ctx := security.StoreRestrictedRequestContext(r, restrictedContext)
r = r.WithContext(ctx)
rr := httptest.NewRecorder()
return rr, handler.customTemplateInspect(rr, r)
}
t.Run("unknown id should get not found error", func(t *testing.T) {
_, r := test("0", &security.RestrictedRequestContext{UserID: 1, User: &portainer.User{ID: 1, Role: portainer.AdministratorRole}})
require.NotNil(t, r)
require.Equal(t, http.StatusNotFound, r.StatusCode)
})
t.Run("admin should access adminonly template", func(t *testing.T) {
rr, r := test("1", &security.RestrictedRequestContext{UserID: 1, IsAdmin: true, User: &portainer.User{ID: 1, Role: portainer.AdministratorRole}})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var template portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&template))
require.Equal(t, portainer.CustomTemplateID(1), template.ID)
})
t.Run("std should not access adminonly template", func(t *testing.T) {
_, r := test("1", &security.RestrictedRequestContext{UserID: 2, User: &portainer.User{ID: 2, Role: portainer.StandardUserRole}})
require.NotNil(t, r)
require.Equal(t, http.StatusForbidden, r.StatusCode)
})
t.Run("std should access template via direct user access", func(t *testing.T) {
rr, r := test("2", &security.RestrictedRequestContext{UserID: 2, User: &portainer.User{ID: 2, Role: portainer.StandardUserRole}})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var template portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&template))
require.Equal(t, portainer.CustomTemplateID(2), template.ID)
})
t.Run("std should access template via team access", func(t *testing.T) {
rr, r := test("2", &security.RestrictedRequestContext{UserID: 3, User: &portainer.User{ID: 3, Role: portainer.StandardUserRole}, UserMemberships: []portainer.TeamMembership{{ID: 1, UserID: 3, TeamID: 1}}})
require.Nil(t, r)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var template portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&template))
require.Equal(t, portainer.CustomTemplateID(2), template.ID)
})
t.Run("std should not access template without access", func(t *testing.T) {
_, r := test("2", &security.RestrictedRequestContext{UserID: 4, User: &portainer.User{ID: 4, Role: portainer.StandardUserRole}})
require.NotNil(t, r)
require.Equal(t, http.StatusForbidden, r.StatusCode)
})
}
func TestInspectHandler_CreatorDeniedWhenAdminOnly(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 5,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 5,
ResourceID: "5",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
r := httptest.NewRequest(http.MethodGet, "/custom_templates/5", nil)
r = mux.SetURLVars(r, map[string]string{"id": "5"})
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 2, User: &portainer.User{ID: 2, Role: portainer.StandardUserRole}}))
rr := httptest.NewRecorder()
herr := handler.customTemplateInspect(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestInspectHandler_GitConfigPopulatedFromSource(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
var srcID portainer.SourceID
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
TLSSkipVerify: true,
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 10,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{
Ref: "refs/heads/main",
Path: "docker-compose.yml",
Hash: "abc123",
SourceID: srcID,
}},
},
})
}))
r := httptest.NewRequest(http.MethodGet, "/custom_templates/10", nil)
r = mux.SetURLVars(r, map[string]string{"id": "10"})
ctx := security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true, User: &portainer.User{ID: 1, Role: portainer.AdministratorRole}})
r = r.WithContext(ctx)
rr := httptest.NewRecorder()
herr := handler.customTemplateInspect(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var template portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&template))
require.NotNil(t, template.GitConfig)
require.Equal(t, "https://github.com/example/repo", template.GitConfig.URL)
require.True(t, template.GitConfig.TLSSkipVerify)
require.Equal(t, "refs/heads/main", template.GitConfig.ReferenceName)
require.Equal(t, "docker-compose.yml", template.GitConfig.ConfigFilePath)
require.Equal(t, "abc123", template.GitConfig.ConfigHash)
}
@@ -0,0 +1,150 @@
package customtemplates
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
// @id CustomTemplateList
// @summary List available custom templates
// @description List available custom templates.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param type query []int true "Template types" Enums(1,2,3)
// @param edge query boolean false "Filter by edge templates"
// @success 200 {array} portainer.CustomTemplate "Success"
// @failure 500 "Server error"
// @router /custom_templates [get]
func (handler *Handler) customTemplateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
templateTypes, err := parseTemplateTypes(r)
if err != nil {
return httperror.BadRequest("Invalid Custom template type", err)
}
edge := retrieveEdgeParam(r)
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
var customTemplates []portainer.CustomTemplate
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
customTemplates, err = tx.CustomTemplate().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom templates from the database", err)
}
resourceControls, err := tx.ResourceControl().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve resource controls from the database", err)
}
customTemplates = authorization.DecorateCustomTemplates(customTemplates, resourceControls)
if !securityContext.IsAdmin {
user, err := tx.User().Read(securityContext.UserID)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user information from the database", err)
}
userTeamIDs := authorization.TeamIDs(securityContext.UserMemberships)
customTemplates = authorization.FilterAuthorizedCustomTemplates(customTemplates, user, userTeamIDs)
}
customTemplates = filterByType(customTemplates, templateTypes)
if edge != nil {
customTemplates = slicesx.FilterInPlace(customTemplates, func(customTemplate portainer.CustomTemplate) bool {
return customTemplate.EdgeTemplate == *edge
})
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
for i := range customTemplates {
populateGitConfig(tx, userContext, &customTemplates[i])
}
return nil
})
return response.TxResponse(w, customTemplates, err)
}
func retrieveEdgeParam(r *http.Request) *bool {
var edge *bool
edgeParam, _ := request.RetrieveQueryParameter(r, "edge", true)
if edgeParam != "" {
edgeVal, err := strconv.ParseBool(edgeParam)
if err != nil {
log.Warn().Err(err).Msg("failed parsing edge param")
return nil
}
edge = &edgeVal
}
return edge
}
func parseTemplateTypes(r *http.Request) ([]portainer.StackType, error) {
err := r.ParseForm()
if err != nil {
return nil, errors.WithMessage(err, "failed to parse request params")
}
types, exist := r.Form["type"]
if !exist {
return []portainer.StackType{}, nil
}
res := []portainer.StackType{}
for _, templateTypeStr := range types {
templateType, err := strconv.Atoi(templateTypeStr)
if err != nil {
return nil, errors.WithMessage(err, "failed parsing template type")
}
res = append(res, portainer.StackType(templateType))
}
return res, nil
}
func filterByType(customTemplates []portainer.CustomTemplate, templateTypes []portainer.StackType) []portainer.CustomTemplate {
if len(templateTypes) == 0 {
return customTemplates
}
typeSet := map[portainer.StackType]bool{}
for _, templateType := range templateTypes {
typeSet[templateType] = true
}
filtered := []portainer.CustomTemplate{}
for _, template := range customTemplates {
if typeSet[template.Type] {
filtered = append(filtered, template)
}
}
return filtered
}
@@ -0,0 +1,127 @@
package customtemplates
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestCustomTemplateList_PopulatesGitConfigFromSource(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
var srcID portainer.SourceID
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
TLSSkipVerify: true,
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{
Ref: "refs/heads/main",
Path: "docker-compose.yml",
Hash: "abc123",
SourceID: srcID,
}},
},
}))
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{ID: 2, EntryPoint: "docker-compose.yml"}))
return nil
}))
r := httptest.NewRequest(http.MethodGet, "/custom_templates", nil)
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true, User: &portainer.User{ID: 1, Role: portainer.AdministratorRole}}))
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, r)
require.Equal(t, http.StatusOK, rr.Code)
var templates []portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&templates))
var gitTemplate portainer.CustomTemplate
for _, tpl := range templates {
if tpl.ID == 1 {
gitTemplate = tpl
}
}
require.NotNil(t, gitTemplate.GitConfig)
require.Equal(t, "https://github.com/example/repo", gitTemplate.GitConfig.URL)
require.True(t, gitTemplate.GitConfig.TLSSkipVerify)
require.Equal(t, "refs/heads/main", gitTemplate.GitConfig.ReferenceName)
require.Equal(t, "docker-compose.yml", gitTemplate.GitConfig.ConfigFilePath)
require.Equal(t, "abc123", gitTemplate.GitConfig.ConfigHash)
var plainTemplate portainer.CustomTemplate
for _, tpl := range templates {
if tpl.ID == 2 {
plainTemplate = tpl
}
}
require.Nil(t, plainTemplate.GitConfig)
}
func TestCustomTemplateList_StripsPasswordFromGitConfig(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
var srcID portainer.SourceID
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
Authentication: &gittypes.GitAuthentication{
Username: "user",
Password: "topsecret",
},
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{SourceID: srcID}},
},
}))
return nil
}))
r := httptest.NewRequest(http.MethodGet, "/custom_templates", nil)
r = r.WithContext(security.StoreRestrictedRequestContext(r, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true, User: &portainer.User{ID: 1, Role: portainer.AdministratorRole}}))
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, r)
require.Equal(t, http.StatusOK, rr.Code)
var templates []portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&templates))
require.Len(t, templates, 1)
require.NotNil(t, templates[0].GitConfig)
require.NotNil(t, templates[0].GitConfig.Authentication)
require.Equal(t, "user", templates[0].GitConfig.Authentication.Username)
require.Empty(t, templates[0].GitConfig.Authentication.Password)
}
@@ -0,0 +1,284 @@
package customtemplates
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/sources"
"github.com/portainer/portainer/api/gitops/workflows"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
)
type customTemplateUpdatePayload struct {
// URL of the template's logo
Logo string `example:"https://portainer.io/img/logo.svg"`
// Title of the template
Title string `example:"Nginx" validate:"required"`
// Description of the template
Description string `example:"High performance web server" validate:"required"`
// A note that will be displayed in the UI. Supports HTML content
Note string `example:"This is my <b>custom</b> template"`
// Platform associated to the template.
// Valid values are: 1 - 'linux', 2 - 'windows'
// Required for Docker stacks
Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
// SourceID references an existing Source for git credentials/URL.
// When set, the inline URL and authentication fields are ignored.
SourceID portainer.SourceID `example:"1"`
// Deprecated: use SourceID instead. URL of a Git repository hosting the Stack file.
RepositoryURL string `example:"https://github.com/openfaas/faas"`
// Reference name of a Git repository hosting the Stack file
RepositoryReferenceName string `example:"refs/heads/master"`
// Deprecated: use SourceID instead. Use authentication to clone the Git repository.
RepositoryAuthentication bool `example:"true"`
// Deprecated: use SourceID instead. Username used in basic authentication. Required when RepositoryAuthentication is true.
RepositoryUsername string `example:"myGitUsername"`
// Deprecated: use SourceID instead. Password used in basic authentication or token used in token authentication. Required when RepositoryAuthentication is true.
RepositoryPassword string `example:"myGitPassword"`
// Path to the Stack file inside the Git repository
ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
// Content of stack file
FileContent string `validate:"required"`
// Definitions of variables in the stack file
Variables []portainer.CustomTemplateVariableDefinition
// Deprecated: use SourceID instead. TLSSkipVerify skips SSL verification when cloning the Git repository.
TLSSkipVerify bool `example:"false"`
// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
IsComposeFormat bool `example:"false"`
// EdgeTemplate indicates if this template purpose for Edge Stack
EdgeTemplate bool `example:"false"`
}
func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
return errors.New("Invalid custom template type")
}
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if !IsValidNote(payload.Note) {
return errors.New("Invalid note. <img> tag is not supported")
}
if len(payload.FileContent) == 0 && payload.SourceID == 0 {
if len(payload.RepositoryURL) == 0 {
return errors.New("Either file content, git repository url, or source ID need to be provided")
}
if !validate.IsURL(payload.RepositoryURL) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if payload.RepositoryAuthentication && (len(payload.RepositoryUsername) == 0 || len(payload.RepositoryPassword) == 0) {
return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
}
}
if len(payload.ComposeFilePathInRepository) == 0 {
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}
if err := ValidateVariablesDefinitions(payload.Variables); err != nil {
return err
}
return nil
}
// @id CustomTemplateUpdate
// @summary Update a template
// @description Update a template.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Template identifier"
// @param body body customTemplateUpdatePayload true "Template details"
// @success 200 {object} portainer.CustomTemplate "Success"
// @failure 400 "Invalid request"
// @failure 403 "Permission denied to access template"
// @failure 404 "Template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id} [put]
func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Custom template identifier route variable", err)
}
var payload customTemplateUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
duplicates, err := handler.DataStore.CustomTemplate().ReadAll(func(t portainer.CustomTemplate) bool {
return t.ID != portainer.CustomTemplateID(customTemplateID) && t.Title == payload.Title
})
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom templates from the database", err)
}
if len(duplicates) > 0 {
return httperror.InternalServerError("Template name must be unique", errors.New("Template name must be unique"))
}
customTemplate, err := handler.DataStore.CustomTemplate().Read(portainer.CustomTemplateID(customTemplateID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(strconv.Itoa(customTemplateID), portainer.CustomTemplateResourceControl)
if err != nil {
return httperror.InternalServerError("Unable to retrieve a resource control associated to the custom template", err)
}
customTemplate.ResourceControl = resourceControl
if !userCanEditTemplate(customTemplate, securityContext) {
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}
customTemplate.Title = payload.Title
customTemplate.Logo = payload.Logo
customTemplate.Description = payload.Description
customTemplate.Note = payload.Note
customTemplate.Platform = payload.Platform
customTemplate.Type = payload.Type
customTemplate.Variables = payload.Variables
customTemplate.IsComposeFormat = payload.IsComposeFormat
customTemplate.EdgeTemplate = payload.EdgeTemplate
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if payload.SourceID != 0 || payload.RepositoryURL != "" {
gitConfig, httpErr := sources.ResolveRepoConfig(handler.DataStore, userContext, sources.RepoConfigInput{
SourceID: payload.SourceID,
ReferenceName: payload.RepositoryReferenceName,
ConfigFilePath: payload.ComposeFilePathInRepository,
RepositoryURL: payload.RepositoryURL,
TLSSkipVerify: payload.TLSSkipVerify,
RepositoryAuthentication: payload.RepositoryAuthentication,
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
})
if httpErr != nil {
return httpErr
}
var username, password string
if gitConfig.Authentication != nil {
username = gitConfig.Authentication.Username
password = gitConfig.Authentication.Password
}
cleanBackup, err := git.CloneWithBackup(context.TODO(), handler.GitService, handler.FileService, git.CloneOptions{
ProjectPath: customTemplate.ProjectPath,
URL: gitConfig.URL,
ReferenceName: gitConfig.ReferenceName,
Username: username,
Password: password,
TLSSkipVerify: gitConfig.TLSSkipVerify,
})
if err != nil {
return httperror.InternalServerError("Unable to clone git repository directory", err)
}
defer cleanBackup()
commitHash, err := handler.GitService.LatestCommitID(
context.TODO(),
gitConfig.URL,
gitConfig.ReferenceName,
username,
password,
gitConfig.TLSSkipVerify,
)
if err != nil {
return httperror.InternalServerError("Unable get latest commit id", fmt.Errorf("failed to fetch latest commit id of the template %v: %w", customTemplate.ID, err))
}
sourceID := payload.SourceID
if sourceID == 0 {
src, err := workflows.FindOrCreateGitSource(handler.DataStore, userContext, &portainer.Source{
Name: gittypes.RepoName(gitConfig.URL),
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: gitConfig.URL,
Authentication: gitConfig.Authentication,
TLSSkipVerify: gitConfig.TLSSkipVerify,
},
})
if err != nil {
return httperror.InternalServerError("Unable to find or create git source", err)
}
sourceID = src.ID
}
customTemplate.Artifact = &portainer.Artifact{
Files: []portainer.ArtifactFile{{
SourceID: sourceID,
Path: gitConfig.ConfigFilePath,
Ref: gitConfig.ReferenceName,
Hash: commitHash,
}},
}
} else {
templateFolder := strconv.Itoa(customTemplateID)
projectPath, err := handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
if err != nil {
return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
}
customTemplate.ProjectPath = projectPath
customTemplate.Artifact = nil
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := tx.CustomTemplate().Update(customTemplate.ID, customTemplate); err != nil {
return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
populateGitConfig(tx, userContext, customTemplate)
return nil
})
return response.TxResponse(w, customTemplate, err)
}
@@ -0,0 +1,649 @@
package customtemplates
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/gorilla/mux"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func updateTemplateRequest(t *testing.T, templateID string, payload any, ctx *security.RestrictedRequestContext) *http.Request {
t.Helper()
body, err := json.Marshal(payload)
require.NoError(t, err)
r := httptest.NewRequest(http.MethodPut, "/custom_templates/"+templateID, bytes.NewReader(body))
r.Header.Set("Content-Type", "application/json")
r = mux.SetURLVars(r, map[string]string{"id": templateID})
if ctx.User == nil {
role := portainer.StandardUserRole
if ctx.IsAdmin {
role = portainer.AdministratorRole
}
ctx.User = &portainer.User{ID: ctx.UserID, Role: role}
}
return r.WithContext(security.StoreRestrictedRequestContext(r, ctx))
}
func TestCustomTemplateUpdate_NotFound(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "New Title",
Description: "New Description",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusNotFound, herr.StatusCode)
}
func TestCustomTemplateUpdate_Forbidden(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Original Title",
EntryPoint: filesystem.ComposeFileDefaultName,
CreatedByUserID: 1,
})
}))
payload := customTemplateUpdatePayload{
Title: "New Title",
Description: "New Description",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
// User 2 did not create this template and is not an admin
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 2})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateUpdate_DuplicateTitle(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Template One",
}))
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 2,
Title: "Template Two",
})
}))
payload := customTemplateUpdatePayload{
Title: "Template One",
Description: "Renamed",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "2", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusInternalServerError, herr.StatusCode)
}
func TestCustomTemplateUpdate_Success_FileContent(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Original Title",
Description: "Original Description",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 1,
})
}))
payload := customTemplateUpdatePayload{
Title: "Updated Title",
Description: "Updated Description",
FileContent: "version: '3'\nservices:\n app:\n image: alpine",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
var tmpl portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&tmpl))
require.Equal(t, "Updated Title", tmpl.Title)
require.Equal(t, "Updated Description", tmpl.Description)
err := ds.ViewTx(func(tx dataservices.DataStoreTx) error {
stored, err := tx.CustomTemplate().Read(1)
require.NoError(t, err)
require.Equal(t, "Updated Title", stored.Title)
require.Equal(t, "Updated Description", stored.Description)
return nil
})
require.NoError(t, err)
}
func TestCustomTemplateUpdate_SameTitleAllowed(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "My Template",
EntryPoint: filesystem.ComposeFileDefaultName,
})
}))
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "Updated description",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
err := ds.ViewTx(func(tx dataservices.DataStoreTx) error {
stored, err := tx.CustomTemplate().Read(1)
require.NoError(t, err)
require.Equal(t, "My Template", stored.Title)
require.Equal(t, "Updated description", stored.Description)
return nil
})
require.NoError(t, err)
}
func TestCustomTemplateUpdate_InvalidPayload(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "My Template",
EntryPoint: filesystem.ComposeFileDefaultName,
})
}))
payload := customTemplateUpdatePayload{
// Title is empty - invalid
Description: "A description",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_MissingDescription(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_BothContentAndRepoMissing(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "A description",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_InvalidPlatform(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "A description",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: 0,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_InvalidType(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "A description",
FileContent: "version: '3'",
Type: 0,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_NoteWithImage(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "A description",
FileContent: "version: '3'",
Note: `Some note <img src="x" onerror="alert(1)">`,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_Validation_AuthWithoutCredentials(t *testing.T) {
t.Parallel()
handler, _, _ := newTestHandler(t)
payload := customTemplateUpdatePayload{
Title: "My Template",
Description: "A description",
RepositoryURL: "https://github.com/example/repo",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
RepositoryAuthentication: true,
}
r := updateTemplateRequest(t, "99", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusBadRequest, herr.StatusCode)
}
func TestCustomTemplateUpdate_ClearsArtifact(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Git Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{},
},
})
}))
payload := customTemplateUpdatePayload{
Title: "Git Template",
Description: "Updated with file content",
FileContent: "version: '3'\nservices:\n app:\n image: alpine",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
var tmpl portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&tmpl))
require.Nil(t, tmpl.Artifact)
err := ds.ViewTx(func(tx dataservices.DataStoreTx) error {
stored, err := tx.CustomTemplate().Read(1)
require.NoError(t, err)
require.Nil(t, stored.Artifact)
return nil
})
require.NoError(t, err)
}
func TestCustomTemplateUpdate_CreatorDeniedWhenAdminOnly(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "User Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
payload := customTemplateUpdatePayload{
Title: "User Template Updated",
Description: "Attempted update by creator after adminonly change",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 2})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusForbidden, herr.StatusCode)
}
func TestCustomTemplateUpdate_WithSourceID_Success(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
handler.GitService = &gitServiceCreatingFile{}
projectDir := t.TempDir()
var srcID portainer.SourceID
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Source Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 1,
ProjectPath: projectDir,
}))
src := &portainer.Source{
Name: "example/repo",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return nil
}))
payload := customTemplateUpdatePayload{
Title: "Source Template",
Description: "Updated via source ID",
SourceID: srcID,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
var tmpl portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&tmpl))
require.NotNil(t, tmpl.Artifact)
require.Len(t, tmpl.Artifact.Files, 1)
require.Equal(t, srcID, tmpl.Artifact.Files[0].SourceID)
require.Equal(t, "deadbeef123", tmpl.Artifact.Files[0].Hash)
}
func TestCustomTemplateUpdate_WithSourceID_NonExistentSource(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
handler.GitService = &gitServiceCreatingFile{}
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Source Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 1,
})
}))
payload := customTemplateUpdatePayload{
Title: "Source Template",
Description: "Updated via non-existent source ID",
SourceID: 999,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.NotNil(t, herr)
require.Equal(t, http.StatusNotFound, herr.StatusCode)
}
func TestCustomTemplateUpdate_AdminCanUpdateAdminOnly(t *testing.T) {
t.Parallel()
handler, store, _ := newTestHandler(t)
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
err := tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "User Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 2,
})
require.NoError(t, err)
err = tx.ResourceControl().Create(&portainer.ResourceControl{
ID: 1,
ResourceID: "1",
Type: portainer.CustomTemplateResourceControl,
AdministratorsOnly: true,
})
require.NoError(t, err)
return nil
})
require.NoError(t, err)
payload := customTemplateUpdatePayload{
Title: "Updated by Admin",
Description: "Admin update of adminonly template",
FileContent: "version: '3'",
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
}
func TestCustomTemplateUpdate_GitRepository_Success(t *testing.T) {
t.Parallel()
handler, ds, _ := newTestHandler(t)
handler.GitService = &gitServiceCreatingFile{}
projectDir := t.TempDir()
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.CustomTemplate().Create(&portainer.CustomTemplate{
ID: 1,
Title: "Git Template",
EntryPoint: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
CreatedByUserID: 1,
ProjectPath: projectDir,
})
}))
payload := customTemplateUpdatePayload{
Title: "Git Template",
Description: "Updated via git",
RepositoryURL: "https://github.com/example/repo",
RepositoryReferenceName: "refs/heads/main",
ComposeFilePathInRepository: filesystem.ComposeFileDefaultName,
Type: portainer.DockerComposeStack,
Platform: portainer.CustomTemplatePlatformLinux,
}
r := updateTemplateRequest(t, "1", payload, &security.RestrictedRequestContext{UserID: 1, IsAdmin: true})
rr := httptest.NewRecorder()
herr := handler.customTemplateUpdate(rr, r)
require.Nil(t, herr)
require.Equal(t, http.StatusOK, rr.Code)
var tmpl portainer.CustomTemplate
require.NoError(t, json.NewDecoder(rr.Body).Decode(&tmpl))
require.NotNil(t, tmpl.Artifact)
require.Len(t, tmpl.Artifact.Files, 1)
require.Equal(t, "deadbeef123", tmpl.Artifact.Files[0].Hash)
err := ds.ViewTx(func(tx dataservices.DataStoreTx) error {
stored, err := tx.CustomTemplate().Read(1)
require.NoError(t, err)
require.NotNil(t, stored.Artifact)
src, err := tx.Source().Read(adminUserContext, stored.Artifact.Files[0].SourceID)
require.NoError(t, err)
require.Equal(t, portainer.SourceTypeGit, src.Type)
require.Equal(t, "https://github.com/example/repo", src.Git.URL)
return nil
})
require.NoError(t, err)
}
@@ -0,0 +1,77 @@
package customtemplates
import (
"net/http"
"sync"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle environment(endpoint) group operations.
type Handler struct {
*mux.Router
DataStore dataservices.DataStore
FileService portainer.FileService
GitService portainer.GitService
gitFetchMutexs map[portainer.TemplateID]*sync.Mutex
}
// NewHandler creates a handler to manage environment(endpoint) group operations.
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, fileService portainer.FileService, gitService portainer.GitService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
DataStore: dataStore,
FileService: fileService,
GitService: gitService,
gitFetchMutexs: make(map[portainer.TemplateID]*sync.Mutex),
}
h.Handle("/custom_templates/create/{method}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateCreate))).Methods(http.MethodPost)
h.Handle("/custom_templates",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateList))).Methods(http.MethodGet)
h.Handle("/custom_templates/{id}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateInspect))).Methods(http.MethodGet)
h.Handle("/custom_templates/{id}/file",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateFile))).Methods(http.MethodGet)
h.Handle("/custom_templates/{id}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut)
h.Handle("/custom_templates/{id}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete)
h.Handle("/custom_templates/{id}/git_fetch",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut)
return h
}
func userCanEditTemplate(customTemplate *portainer.CustomTemplate, securityContext *security.RestrictedRequestContext) bool {
resourceControl := customTemplate.ResourceControl
if securityContext.IsAdmin {
return true
}
if resourceControl == nil || resourceControl.AdministratorsOnly {
return false
}
if resourceControl.Public {
return true
}
if customTemplate.CreatedByUserID != securityContext.UserID {
return false
}
teamIDs := slicesx.Map(securityContext.UserMemberships, func(m portainer.TeamMembership) portainer.TeamID {
return m.TeamID
})
return authorization.UserCanAccessResource(securityContext.UserID, teamIDs, resourceControl)
}
@@ -0,0 +1,5 @@
package customtemplates
import "github.com/portainer/portainer/api/dataservices/source"
var adminUserContext = source.InsecureNewAdminContext()
+63
View File
@@ -0,0 +1,63 @@
package customtemplates
import (
"errors"
"regexp"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
)
func populateGitConfig(tx dataservices.DataStoreTx, userContext source.UserContext, template *portainer.CustomTemplate) {
if template.Artifact == nil || len(template.Artifact.Files) == 0 {
return
}
file := template.Artifact.Files[0]
src, err := tx.Source().Read(userContext, file.SourceID)
if err != nil || src.Git == nil {
return
}
cfg := &gittypes.RepoConfig{
URL: src.Git.URL,
Authentication: src.Git.Authentication,
TLSSkipVerify: src.Git.TLSSkipVerify,
ReferenceName: file.Ref,
ConfigFilePath: file.Path,
ConfigHash: file.Hash,
}
if cfg.Authentication != nil {
sanitized := *cfg.Authentication
sanitized.Password = ""
cfg.Authentication = &sanitized
}
template.GitConfig = cfg
}
// IsValidNote reports whether note is safe to display. Notes containing <img> tags are rejected.
func IsValidNote(note string) bool {
if len(note) == 0 {
return true
}
match, _ := regexp.MatchString("<img", note)
return !match
}
// ValidateVariablesDefinitions returns an error if any variable definition is missing a required field.
func ValidateVariablesDefinitions(variables []portainer.CustomTemplateVariableDefinition) error {
for _, variable := range variables {
if variable.Name == "" {
return errors.New("variable name is required")
}
if variable.Label == "" {
return errors.New("variable label is required")
}
}
return nil
}
@@ -0,0 +1,142 @@
package customtemplates
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/require"
)
func TestPopulateGitConfig_NilArtifactIsNoOp(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
template := &portainer.CustomTemplate{ID: 1}
err := store.ViewTx(func(tx dataservices.DataStoreTx) error {
populateGitConfig(tx, adminUserContext, template)
return nil
})
require.NoError(t, err)
require.Nil(t, template.GitConfig)
}
func TestPopulateGitConfig_EmptySourceIDsIsNoOp(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
template := &portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{},
},
}
err := store.ViewTx(func(tx dataservices.DataStoreTx) error {
populateGitConfig(tx, adminUserContext, template)
return nil
})
require.NoError(t, err)
require.Nil(t, template.GitConfig)
}
func TestPopulateGitConfig_PopulatesFromSourceAndArtifact(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
TLSSkipVerify: true,
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return nil
})
require.NoError(t, err)
template := &portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{
Ref: "refs/heads/main",
Path: "docker-compose.yml",
Hash: "abc123",
SourceID: srcID,
}},
},
}
err = store.ViewTx(func(tx dataservices.DataStoreTx) error {
populateGitConfig(tx, adminUserContext, template)
return nil
})
require.NoError(t, err)
require.NotNil(t, template.GitConfig)
require.Equal(t, "https://github.com/example/repo", template.GitConfig.URL)
require.True(t, template.GitConfig.TLSSkipVerify)
require.Equal(t, "refs/heads/main", template.GitConfig.ReferenceName)
require.Equal(t, "docker-compose.yml", template.GitConfig.ConfigFilePath)
require.Equal(t, "abc123", template.GitConfig.ConfigHash)
}
func TestPopulateGitConfig_StripsPassword(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
err := store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/example/repo",
Authentication: &gittypes.GitAuthentication{
Username: "user",
Password: "secret",
},
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return nil
})
require.NoError(t, err)
template := &portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{SourceID: srcID}},
},
}
err = store.ViewTx(func(tx dataservices.DataStoreTx) error {
populateGitConfig(tx, adminUserContext, template)
return nil
})
require.NoError(t, err)
require.NotNil(t, template.GitConfig)
require.NotNil(t, template.GitConfig.Authentication)
require.Equal(t, "user", template.GitConfig.Authentication.Username)
require.Empty(t, template.GitConfig.Authentication.Password)
}
@@ -0,0 +1,87 @@
package containers
import (
"net/http"
"slices"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/middlewares"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
containertypes "github.com/docker/docker/api/types/container"
)
type containerGpusResponse struct {
Gpus string `json:"gpus"`
}
// @id dockerContainerGpusInspect
// @summary Fetch container gpus data
// @description
// @description **Access policy**:
// @tags docker
// @security jwt
// @accept json
// @produce json
// @param environmentId path int true "Environment identifier"
// @param containerId path int true "Container identifier"
// @success 200 {object} containerGpusResponse "Success"
// @failure 404 "Environment or container not found"
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @router /docker/{environmentId}/containers/{containerId}/gpus [get]
func (handler *Handler) containerGpusInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
containerId, err := request.RetrieveRouteVariableValue(r, "containerId")
if err != nil {
return httperror.BadRequest("Invalid container identifier route variable", err)
}
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return httperror.NotFound("Unable to find an environment on request context", err)
}
agentTargetHeader := r.Header.Get(portainer.PortainerAgentTargetHeader)
cli, err := handler.dockerClientFactory.CreateClient(endpoint, agentTargetHeader, nil)
if err != nil {
return httperror.InternalServerError("Unable to connect to the Docker daemon", err)
}
container, err := cli.ContainerInspect(r.Context(), containerId)
if err != nil {
return httperror.NotFound("Unable to find the container", err)
}
if container.HostConfig == nil {
return httperror.NotFound("Unable to find the container host config", err)
}
gpuOptionsIndex := slices.IndexFunc(container.HostConfig.DeviceRequests, func(opt containertypes.DeviceRequest) bool {
if opt.Driver == "nvidia" {
return true
}
if len(opt.Capabilities) == 0 || len(opt.Capabilities[0]) == 0 {
return false
}
return opt.Capabilities[0][0] == "gpu"
})
if gpuOptionsIndex == -1 {
return response.JSON(w, containerGpusResponse{Gpus: "none"})
}
gpuOptions := container.HostConfig.DeviceRequests[gpuOptionsIndex]
gpu := "all"
if gpuOptions.Count != -1 {
gpu = "id:" + strings.Join(gpuOptions.DeviceIDs, ",")
}
return response.JSON(w, containerGpusResponse{Gpus: gpu})
}
@@ -0,0 +1,40 @@
package containers
import (
"net/http"
"github.com/gorilla/mux"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
type Handler struct {
*mux.Router
dockerClientFactory *dockerclient.ClientFactory
dataStore dataservices.DataStore
containerService *docker.ContainerService
bouncer security.BouncerService
}
// NewHandler creates a handler to process non-proxied requests to docker APIs directly.
func NewHandler(routePrefix string, bouncer security.BouncerService, dataStore dataservices.DataStore, dockerClientFactory *dockerclient.ClientFactory, containerService *docker.ContainerService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dataStore: dataStore,
dockerClientFactory: dockerClientFactory,
containerService: containerService,
bouncer: bouncer,
}
router := h.PathPrefix(routePrefix).Subrouter()
router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
router.Handle("/{containerId}/gpus", httperror.LoggerHandler(h.containerGpusInspect)).Methods(http.MethodGet)
router.Handle("/{containerId}/recreate", httperror.LoggerHandler(h.recreate)).Methods(http.MethodPost)
return h
}
@@ -0,0 +1,98 @@
package containers
import (
"context"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/docker/consts"
"github.com/portainer/portainer/api/docker/images"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/internal/authorization"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
type RecreatePayload struct {
// PullImage if true will pull the image
PullImage bool `json:"PullImage"`
}
func (r RecreatePayload) Validate(request *http.Request) error {
return nil
}
func (handler *Handler) recreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
containerID, err := request.RetrieveRouteVariableValue(r, "containerId")
if err != nil {
return httperror.BadRequest("Invalid containerId", err)
}
var payload RecreatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return httperror.NotFound("Unable to find an environment on request context", err)
}
if err := handler.bouncer.AuthorizedEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to force update service", err)
}
agentTargetHeader := r.Header.Get(portainer.PortainerAgentTargetHeader)
newContainer, err := handler.containerService.Recreate(context.TODO(), endpoint, containerID, payload.PullImage, "", agentTargetHeader)
if err != nil {
return httperror.InternalServerError("Error recreating container", err)
}
handler.updateWebhook(containerID, newContainer.ID)
handler.createResourceControl(containerID, newContainer.ID)
go func() {
images.EvictImageStatus(containerID)
images.EvictImageStatus(newContainer.Config.Labels[consts.ComposeStackNameLabel])
images.EvictImageStatus(newContainer.Config.Labels[consts.SwarmServiceIDLabel])
}()
return response.JSON(w, newContainer)
}
func (handler *Handler) createResourceControl(oldContainerId string, newContainerId string) {
resourceControls, err := handler.dataStore.ResourceControl().ReadAll()
if err != nil {
log.Error().Err(err).Msg("Exporting Resource Controls")
return
}
resourceControl := authorization.GetResourceControlByResourceIDAndType(oldContainerId, portainer.ContainerResourceControl, resourceControls)
if resourceControl == nil {
return
}
resourceControl.ResourceID = newContainerId
if err := handler.dataStore.ResourceControl().Create(resourceControl); err != nil {
log.Error().Err(err).Str("containerId", newContainerId).Msg("Failed to create new resource control for container")
}
}
func (handler *Handler) updateWebhook(oldContainerId string, newContainerId string) {
webhook, err := handler.dataStore.Webhook().WebhookByResourceID(oldContainerId)
if err != nil {
log.Error().Err(err).Str("containerId", oldContainerId).Msg("cannot find webhook by containerId")
return
}
webhook.ResourceID = newContainerId
if err := handler.dataStore.Webhook().Update(webhook.ID, webhook); err != nil {
log.Error().Err(err).Int("webhookId", int(webhook.ID)).Msg("cannot update webhook")
}
}
+170
View File
@@ -0,0 +1,170 @@
package docker
import (
"errors"
"net/http"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/api/types/volume"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker/stats"
"github.com/portainer/portainer/api/http/handler/docker/utils"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/uac"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type imagesCounters struct {
Total int `json:"total"`
Size int64 `json:"size"`
}
type dashboardResponse struct {
Containers stats.ContainerStats `json:"containers"`
Services int `json:"services"`
Images imagesCounters `json:"images"`
Volumes int `json:"volumes"`
Networks int `json:"networks"`
Stacks int `json:"stacks"`
}
// @id dockerDashboard
// @summary Get counters for the dashboard
// @description **Access policy**: restricted
// @tags docker
// @security jwt
// @param environmentId path int true "Environment identifier"
// @accept json
// @produce json
// @success 200 {object} dashboardResponse "Success"
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @router /docker/{environmentId}/dashboard [get]
func (h *Handler) dashboard(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var resp dashboardResponse
err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
cli, httpErr := utils.GetClient(r, h.dockerClientFactory)
if httpErr != nil {
return httpErr
}
context, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user details from request context", err)
}
user, err := tx.User().Read(context.UserID)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user", err)
}
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return err
}
containers, err := cli.ContainerList(r.Context(), container.ListOptions{All: true})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker containers", err)
}
if containers, err = uac.FilterByResourceControl(containers, user, context.UserMemberships, uac.ContainerResourceControlGetter(tx, endpoint.ID)); err != nil {
return err
}
images, err := cli.ImageList(r.Context(), image.ListOptions{})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker images", err)
}
var totalSize int64
for _, image := range images {
totalSize += image.Size
}
info, err := cli.Info(r.Context())
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker info", err)
}
isSwarmManager := info.Swarm.ControlAvailable && info.Swarm.NodeID != ""
var services []swarm.Service
if isSwarmManager {
servicesRes, err := cli.ServiceList(r.Context(), types.ServiceListOptions{})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker services", err)
}
if services, err = uac.FilterByResourceControl(servicesRes, user, context.UserMemberships, uac.ServiceResourceControlGetter(tx, endpoint.ID)); err != nil {
return err
}
}
volumesRes, err := cli.VolumeList(r.Context(), volume.ListOptions{})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker volumes", err)
}
var volumes []*volume.Volume
if volumes, err = uac.FilterByResourceControl(volumesRes.Volumes, user, context.UserMemberships, func(item *volume.Volume) (*portainer.ResourceControl, error) {
if item == nil {
return nil, errors.New("Found nil volume in volumes list")
}
return uac.VolumeResourceControlGetter(tx, endpoint.ID)(*item)
}); err != nil {
return err
}
networks, err := cli.NetworkList(r.Context(), network.ListOptions{})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker networks", err)
}
if networks, err = uac.FilterByResourceControl(networks, user, context.UserMemberships, uac.NetworkResourceControlGetter(tx, endpoint.ID)); err != nil {
return err
}
environment, err := middlewares.FetchEndpoint(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment", err)
}
stackCount := 0
if environment.SecuritySettings.AllowStackManagementForRegularUsers || context.IsAdmin {
stacks, err := utils.GetDockerStacks(tx, context, environment.ID, containers, services)
if err != nil {
return httperror.InternalServerError("Unable to retrieve stacks", err)
}
stackCount = len(stacks)
}
containersStats, err := stats.CalculateContainerStats(r.Context(), cli, info.Swarm.ControlAvailable, containers)
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker containers stats", err)
}
resp = dashboardResponse{
Images: imagesCounters{
Total: len(images),
Size: totalSize,
},
Services: len(services),
Containers: containersStats,
Networks: len(networks),
Volumes: len(volumes),
Stacks: stackCount,
}
return nil
})
return response.TxResponse(w, resp, err)
}
@@ -0,0 +1,95 @@
package docker
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/apikey"
"github.com/portainer/portainer/api/datastore"
dockerdomain "github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// unreachableDockerURL points the test environment at a port that refuses connections, so an
// authorized caller fails fast when the handler builds a docker client rather than blocking on a
// real daemon. The authorization middleware runs before this, which is what these tests assert.
const unreachableDockerURL = "tcp://127.0.0.1:1"
func newDashboardAuthTestHandler(t *testing.T) (*Handler, *jwt.Service, *datastore.Store) {
t.Helper()
fips.InitFIPS(false)
_, store := datastore.MustNewTestStore(t, true, true)
require.NoError(t, store.Endpoint().Create(&portainer.Endpoint{
ID: 1, Name: "docker-env", Type: portainer.DockerEnvironment, URL: unreachableDockerURL,
}))
jwtService, err := jwt.NewService("1h", store)
require.NoError(t, err)
bouncer := security.NewRequestBouncer(t.Context(), store, jwtService, apikey.NewAPIKeyService(nil, nil))
factory := dockerclient.NewClientFactory(nil, nil)
authorizationService := authorization.NewService(store)
containerService := dockerdomain.NewContainerService(factory, store)
handler := NewHandler(bouncer, authorizationService, store, factory, containerService)
return handler, jwtService, store
}
func dashboardRequest(t *testing.T, handler *Handler, jwtService *jwt.Service, user *portainer.User) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/docker/1/dashboard", nil)
tk, _, err := jwtService.GenerateToken(&portainer.TokenData{ID: user.ID, Username: user.Username, Role: user.Role})
require.NoError(t, err)
testhelpers.AddTestSecurityCookie(req, tk)
rr := httptest.NewRecorder()
handler.ServeHTTP(rr, req)
return rr
}
// TestEndpointAuthorization_DeniedUser_Returns403 verifies that the docker /dashboard
// route rejects users with no access policy for the target environment (R8S-1057).
func TestEndpointAuthorization_DeniedUser_Returns403(t *testing.T) {
handler, jwtService, store := newDashboardAuthTestHandler(t)
noAccessUser := &portainer.User{
Username: "no-access",
Role: portainer.StandardUserRole,
PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
}
require.NoError(t, store.User().Create(noAccessUser))
// A standard user with no access policy must be rejected before the dashboard handler
// builds a docker client for the environment.
rr := dashboardRequest(t, handler, jwtService, noAccessUser)
assert.Equal(t, http.StatusForbidden, rr.Code)
}
// TestEndpointAuthorization_AuthorizedUser_NotForbidden verifies that the docker /dashboard
// route lets an authorized caller through the authorization middleware (R8S-1057). The request
// fails later when the handler cannot reach the docker daemon, but it must not be rejected with 403.
func TestEndpointAuthorization_AuthorizedUser_NotForbidden(t *testing.T) {
handler, jwtService, store := newDashboardAuthTestHandler(t)
adminUser := &portainer.User{Username: "admin", Role: portainer.AdministratorRole}
require.NoError(t, store.User().Create(adminUser))
rr := dashboardRequest(t, handler, jwtService, adminUser)
assert.NotEqual(t, http.StatusForbidden, rr.Code)
}
+74
View File
@@ -0,0 +1,74 @@
package docker
import (
"errors"
"net/http"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/handler/docker/containers"
"github.com/portainer/portainer/api/http/handler/docker/images"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler which will natively deal with to external environments(endpoints).
type Handler struct {
*mux.Router
requestBouncer security.BouncerService
dataStore dataservices.DataStore
dockerClientFactory *dockerclient.ClientFactory
authorizationService *authorization.Service
containerService *docker.ContainerService
}
// NewHandler creates a handler to process non-proxied requests to docker APIs directly.
func NewHandler(bouncer security.BouncerService, authorizationService *authorization.Service, dataStore dataservices.DataStore, dockerClientFactory *dockerclient.ClientFactory, containerService *docker.ContainerService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
requestBouncer: bouncer,
authorizationService: authorizationService,
dataStore: dataStore,
dockerClientFactory: dockerClientFactory,
containerService: containerService,
}
// endpoints
endpointRouter := h.PathPrefix("/docker/{id}").Subrouter()
endpointRouter.Use(bouncer.AuthenticatedAccess)
endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"), dockerOnlyMiddleware)
// /dashboard is the only route on this router without its own endpoint authorization;
// the containers/images sub-routers already apply CheckEndpointAuthorization.
endpointRouter.Handle("/dashboard", middlewares.CheckEndpointAuthorization(bouncer)(httperror.LoggerHandler(h.dashboard))).Methods(http.MethodGet)
containersHandler := containers.NewHandler("/docker/{id}/containers", bouncer, dataStore, dockerClientFactory, containerService)
endpointRouter.PathPrefix("/containers").Handler(containersHandler)
imagesHandler := images.NewHandler("/docker/{id}/images", bouncer, dockerClientFactory)
endpointRouter.PathPrefix("/images").Handler(imagesHandler)
return h
}
func dockerOnlyMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {
endpoint, err := middlewares.FetchEndpoint(request)
if err != nil {
httperror.WriteError(rw, http.StatusInternalServerError, "Unable to find an environment on request context", err)
return
}
if !endpointutils.IsDockerEndpoint(endpoint) {
errMessage := "environment is not a docker environment"
httperror.WriteError(rw, http.StatusBadRequest, errMessage, errors.New(errMessage))
return
}
next.ServeHTTP(rw, request)
})
}
+33
View File
@@ -0,0 +1,33 @@
package images
import (
"net/http"
"github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
type Handler struct {
*mux.Router
dockerClientFactory *client.ClientFactory
bouncer security.BouncerService
}
// NewHandler creates a handler to process non-proxied requests to docker APIs directly.
func NewHandler(routePrefix string, bouncer security.BouncerService, dockerClientFactory *client.ClientFactory) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dockerClientFactory: dockerClientFactory,
bouncer: bouncer,
}
router := h.PathPrefix(routePrefix).Subrouter()
router.Use(bouncer.AuthenticatedAccess, middlewares.CheckEndpointAuthorization(bouncer))
router.Handle("", httperror.LoggerHandler(h.imagesList)).Methods(http.MethodGet)
return h
}
@@ -0,0 +1,103 @@
package images
import (
"context"
"fmt"
"net/http"
"strings"
"github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/handler/docker/utils"
"github.com/portainer/portainer/api/set"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
)
type ImageResponse struct {
Created int64 `json:"created"`
NodeName string `json:"nodeName"`
ID string `json:"id"`
Size int64 `json:"size"`
Tags []string `json:"tags"`
// Used is true if the image is used by at least one container
// supplied only when withUsage is true
Used bool `json:"used"`
}
// @id dockerImagesList
// @summary Fetch images
// @description
// @description **Access policy**:
// @tags docker
// @security jwt
// @param environmentId path int true "Environment identifier"
// @param withUsage query boolean false "Include image usage information"
// @produce json
// @success 200 {array} ImageResponse "Success"
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @router /docker/{environmentId}/images [get]
func (handler *Handler) imagesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
cli, httpErr := utils.GetClient(r, handler.dockerClientFactory)
if httpErr != nil {
return httpErr
}
nodeNames := make(map[string]string)
// Pass the node names map to the context so the custom NodeNameTransport can use it
ctx := context.WithValue(r.Context(), client.NodeNamesCtxKey{}, nodeNames)
images, err := cli.ImageList(ctx, image.ListOptions{})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker images", err)
}
withUsage, err := request.RetrieveBooleanQueryParameter(r, "withUsage", true)
if err != nil {
return httperror.BadRequest("Invalid query parameter: withUsage", err)
}
imageUsageSet := set.Set[string]{}
if withUsage {
containers, err := cli.ContainerList(r.Context(), container.ListOptions{
All: true,
})
if err != nil {
return httperror.InternalServerError("Unable to retrieve Docker containers", err)
}
for _, container := range containers {
imageUsageSet.Add(container.ImageID)
}
}
imagesList := make([]ImageResponse, len(images))
for i, image := range images {
if len(image.RepoTags) == 0 && len(image.RepoDigests) > 0 {
for _, repoDigest := range image.RepoDigests {
image.RepoTags = append(image.RepoTags, repoDigest[0:strings.Index(repoDigest, "@")]+":<none>")
}
}
imagesList[i] = ImageResponse{
Created: image.Created,
// Only works if the order of `images` is not changed between unmarshaling the agent's response
// in NodeNameTransport.RoundTrip() (api/docker/client/client.go)
// and docker's cli.ImageList()
// As both functions unmarshal the same response body, the resulting array will be ordered the same way.
NodeName: nodeNames[fmt.Sprintf("%s-%d", image.ID, i)],
ID: image.ID,
Size: image.Size,
Tags: image.RepoTags,
Used: imageUsageSet.Contains(image.ID),
}
}
return response.JSON(w, imagesList)
}
@@ -0,0 +1,28 @@
package utils
import (
"net/http"
dockerclient "github.com/docker/docker/client"
portainer "github.com/portainer/portainer/api"
prclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/http/middlewares"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
// GetClient returns a Docker client based on the request context
func GetClient(r *http.Request, dockerClientFactory *prclient.ClientFactory) (*dockerclient.Client, *httperror.HandlerError) {
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return nil, httperror.NotFound("Unable to find an environment on request context", err)
}
agentTargetHeader := r.Header.Get(portainer.PortainerAgentTargetHeader)
cli, err := dockerClientFactory.CreateClient(endpoint, agentTargetHeader, nil)
if err != nil {
return nil, httperror.InternalServerError("Unable to connect to the Docker daemon", err)
}
return cli, nil
}
@@ -0,0 +1,95 @@
package utils
import (
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dockerconsts "github.com/portainer/portainer/api/docker/consts"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/uac"
)
type StackViewModel struct {
InternalStack *portainer.Stack
ID portainer.StackID
Name string
IsExternal bool
Type portainer.StackType
Labels map[string]string
}
// GetDockerStacks retrieves all the stacks associated to a specific environment filtered by the user's access.
func GetDockerStacks(tx dataservices.DataStoreTx, securityContext *security.RestrictedRequestContext, environmentID portainer.EndpointID, containers []types.Container, services []swarm.Service) ([]StackViewModel, error) {
stacks, err := tx.Stack().ReadAll()
if err != nil {
return nil, fmt.Errorf("Unable to retrieve stacks: %w", err)
}
user, err := tx.User().Read(securityContext.UserID)
if err != nil {
return nil, fmt.Errorf("Unable to retrieve user: %w", err)
}
stacksNameSet := map[string]*StackViewModel{}
for i := range stacks {
stack := stacks[i]
if stack.EndpointID == environmentID {
stacksNameSet[stack.Name] = &StackViewModel{
InternalStack: &stack,
ID: stack.ID,
Name: stack.Name,
IsExternal: false,
Type: stack.Type,
}
}
}
for _, container := range containers {
name := container.Labels[dockerconsts.ComposeStackNameLabel]
if name != "" && stacksNameSet[name] == nil && !isHiddenStack(container.Labels) {
stacksNameSet[name] = &StackViewModel{
Name: name,
IsExternal: true,
Type: portainer.DockerComposeStack,
Labels: container.Labels,
}
}
}
for _, service := range services {
name := service.Spec.Labels[dockerconsts.SwarmStackNameLabel]
if name != "" && stacksNameSet[name] == nil && !isHiddenStack(service.Spec.Labels) {
stacksNameSet[name] = &StackViewModel{
Name: name,
IsExternal: true,
Type: portainer.DockerSwarmStack,
Labels: service.Spec.Labels,
}
}
}
stacksList := make([]StackViewModel, 0)
for _, stack := range stacksNameSet {
stacksList = append(stacksList, *stack)
}
return uac.FilterByResourceControl(stacksList, user, securityContext.UserMemberships,
func(item StackViewModel) (*portainer.ResourceControl, error) {
if item.InternalStack != nil {
return uac.StackResourceControlGetter(tx, environmentID)(*item.InternalStack)
}
return uac.ExternalStackResourceControlGetter(tx, environmentID)(uac.ExternalStack{Labels: item.Labels})
},
)
}
func isHiddenStack(labels map[string]string) bool {
return labels[dockerconsts.HideStackLabel] != ""
}
@@ -0,0 +1,144 @@
package utils
import (
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/docker/consts"
"github.com/portainer/portainer/api/http/security"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHandler_getDockerStacks(t *testing.T) {
t.Parallel()
is := require.New(t)
environment := &portainer.Endpoint{
ID: 1,
SecuritySettings: portainer.EndpointSecuritySettings{
AllowStackManagementForRegularUsers: true,
},
}
containers := []types.Container{
{
Labels: map[string]string{
consts.ComposeStackNameLabel: "stack1",
},
},
{
Labels: map[string]string{
consts.ComposeStackNameLabel: "stack2",
"io.portainer.accesscontrol.public": "true",
},
},
}
services := []swarm.Service{
{
Spec: swarm.ServiceSpec{
Annotations: swarm.Annotations{
Labels: map[string]string{
consts.SwarmStackNameLabel: "stack3",
},
},
},
},
}
stack1 := portainer.Stack{
ID: 1,
Name: "stack1",
EndpointID: 1,
Type: portainer.DockerComposeStack,
}
ok, store := datastore.MustNewTestStore(t, false, false)
is.True(ok)
is.NoError(store.UpdateTx(func(tx dataservices.DataStoreTx) error {
is.NoError(tx.Endpoint().Create(environment))
is.NoError(tx.Stack().Create(&stack1))
is.NoError(tx.Stack().Create(&portainer.Stack{
ID: 2,
Name: "stack2", // stack 2 on env 2
EndpointID: 2,
Type: portainer.DockerSwarmStack,
}))
is.NoError(tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}))
is.NoError(tx.User().Create(&portainer.User{ID: 2, Role: portainer.StandardUserRole}))
return nil
}))
// testing admin user
is.NoError(store.ViewTx(func(tx dataservices.DataStoreTx) error {
stacksList, err := GetDockerStacks(tx, &security.RestrictedRequestContext{
IsAdmin: true,
UserID: 1,
}, environment.ID, containers, services)
require.NoError(t, err)
assert.Len(t, stacksList, 3)
expectedStacks := []StackViewModel{
{
InternalStack: &stack1,
ID: 1,
Name: "stack1",
IsExternal: false,
Type: portainer.DockerComposeStack,
},
{
Name: "stack2",
IsExternal: true,
Type: portainer.DockerComposeStack,
Labels: map[string]string{
consts.ComposeStackNameLabel: "stack2",
"io.portainer.accesscontrol.public": "true",
},
},
{
Name: "stack3",
IsExternal: true,
Type: portainer.DockerSwarmStack,
Labels: map[string]string{
consts.SwarmStackNameLabel: "stack3",
},
},
}
assert.ElementsMatch(t, expectedStacks, stacksList)
return nil
}))
// testing standard user
is.NoError(store.ViewTx(func(tx dataservices.DataStoreTx) error {
stacksList, err := GetDockerStacks(tx, &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 2,
}, environment.ID, containers, services)
require.NoError(t, err)
assert.Len(t, stacksList, 1)
expectedStacks := []StackViewModel{
{
Name: "stack2",
IsExternal: true,
Type: portainer.DockerComposeStack,
Labels: map[string]string{
consts.ComposeStackNameLabel: "stack2",
"io.portainer.accesscontrol.public": "true",
},
},
}
assert.ElementsMatch(t, expectedStacks, stacksList)
return nil
}))
}
@@ -0,0 +1,32 @@
package edgegroups
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/roar"
)
func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs roar.Roar[portainer.EndpointID]) ([]portainer.EndpointID, error) {
var innerErr error
results := []portainer.EndpointID{}
endpointIDs.Iterate(func(endpointID portainer.EndpointID) bool {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
innerErr = err
return false
}
if !endpoint.UserTrusted {
return true
}
results = append(results, endpoint.ID)
return true
})
return results, innerErr
}
@@ -0,0 +1,117 @@
package edgegroups
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type edgeGroupCreatePayload struct {
Name string
Dynamic bool
TagIDs []portainer.TagID
Endpoints []portainer.EndpointID
PartialMatch bool
}
func (payload *edgeGroupCreatePayload) Validate(r *http.Request) error {
if len(payload.Name) == 0 {
return errors.New("invalid Edge group name")
}
if payload.Dynamic && len(payload.TagIDs) == 0 {
return errors.New("tagIDs is mandatory for a dynamic Edge group")
}
return nil
}
func calculateEndpointsOrTags(tx dataservices.DataStoreTx, edgeGroup *portainer.EdgeGroup, endpoints []portainer.EndpointID, tagIDs []portainer.TagID) error {
if edgeGroup.Dynamic {
edgeGroup.TagIDs = tagIDs
return nil
}
endpointIDs := []portainer.EndpointID{}
for _, endpointID := range endpoints {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment from the database", err)
}
if endpointutils.IsEdgeEndpoint(endpoint) {
endpointIDs = append(endpointIDs, endpoint.ID)
}
}
edgeGroup.Endpoints = endpointIDs
edgeGroup.EndpointIDs = roar.FromSlice(endpointIDs)
return nil
}
// @id EdgeGroupCreate
// @summary Create an EdgeGroup
// @description **Access policy**: administrator
// @tags edge_groups
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body edgeGroupCreatePayload true "EdgeGroup data"
// @success 200 {object} portainer.EdgeGroup
// @failure 503 "Edge compute features are disabled"
// @failure 500
// @router /edge_groups [post]
func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload edgeGroupCreatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var shadowEdgeGroup shadowedEdgeGroup
err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge groups from the database", err)
}
for _, edgeGroup := range edgeGroups {
if edgeGroup.Name == payload.Name {
return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
}
}
edgeGroup := &portainer.EdgeGroup{
Name: payload.Name,
Dynamic: payload.Dynamic,
TagIDs: []portainer.TagID{},
Endpoints: []portainer.EndpointID{},
EndpointIDs: roar.Roar[portainer.EndpointID]{},
PartialMatch: payload.PartialMatch,
}
if err := calculateEndpointsOrTags(tx, edgeGroup, payload.Endpoints, payload.TagIDs); err != nil {
return err
}
if err := tx.EdgeGroup().Create(edgeGroup); err != nil {
return httperror.InternalServerError("Unable to persist the Edge group inside the database", err)
}
shadowEdgeGroup = shadowedEdgeGroup{EdgeGroup: *edgeGroup}
return nil
})
return response.TxResponse(w, shadowEdgeGroup, err)
}
@@ -0,0 +1,57 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestEdgeGroupCreateHandler(t *testing.T) {
t.Parallel()
handler, _ := newHandlerWithEdgeEndpoints(t)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPost,
"/edge_groups",
strings.NewReader(`{"Name": "New Edge Group", "Endpoints": [1, 2, 3]}`),
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err := json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}
func TestEdgeGroupCreatePanic(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EdgeGroup().Create(&portainer.EdgeGroup{ID: 1, Name: "New Edge Group"})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost,
"/edge_groups",
strings.NewReader(`{"Name": "New Edge Group", "Endpoints": [1, 2, 3]}`),
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusBadRequest, rr.Result().StatusCode)
}
@@ -0,0 +1,77 @@
package edgegroups
import (
"errors"
"net/http"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeGroupDelete
// @summary Deletes an EdgeGroup
// @description **Access policy**: administrator
// @tags edge_groups
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EdgeGroup Id"
// @success 204
// @failure 409 "Edge group is in use by an Edge stack or Edge job"
// @failure 503 "Edge compute features are disabled"
// @failure 500 "Server error"
// @router /edge_groups/{id} [delete]
func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge group identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return deleteEdgeGroup(tx, portainer.EdgeGroupID(edgeGroupID))
})
return response.TxEmptyResponse(w, err)
}
func deleteEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) error {
ok, err := tx.EdgeGroup().Exists(ID)
if !ok {
return httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", dserrors.ErrObjectNotFound)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err)
}
edgeStacks, err := tx.EdgeStack().EdgeStacks()
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge stacks from the database", err)
}
for _, edgeStack := range edgeStacks {
if slices.Contains(edgeStack.EdgeGroups, ID) {
return httperror.Conflict("Edge group is used by an Edge stack", errors.New("edge group is used by an Edge stack"))
}
}
edgeJobs, err := tx.EdgeJob().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err)
}
for _, edgeJob := range edgeJobs {
if slices.Contains(edgeJob.EdgeGroups, ID) {
return httperror.Conflict("Edge group is used by an Edge job", errors.New("edge group is used by an Edge job"))
}
}
err = tx.EdgeGroup().Delete(ID)
if err != nil {
return httperror.InternalServerError("Unable to remove the Edge group from the database", err)
}
return nil
}
@@ -0,0 +1,68 @@
package edgegroups
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeGroupInspect
// @summary Inspects an EdgeGroup
// @description **Access policy**: administrator
// @tags edge_groups
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeGroup Id"
// @success 200 {object} portainer.EdgeGroup
// @failure 503 "Edge compute features are disabled"
// @failure 500
// @router /edge_groups/{id} [get]
func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge group identifier route variable", err)
}
var shadowEdgeGroup shadowedEdgeGroup
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
edgeGroup, err := getEdgeGroup(tx, portainer.EdgeGroupID(edgeGroupID))
if err != nil {
return err
}
edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
shadowEdgeGroup = shadowedEdgeGroup{EdgeGroup: *edgeGroup}
return nil
})
return response.TxResponse(w, shadowEdgeGroup, err)
}
func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
edgeGroup, err := tx.EdgeGroup().Read(ID)
if tx.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", err)
} else if err != nil {
return nil, httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err)
}
if edgeGroup.Dynamic {
endpoints, err := endpointutils.GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
}
edgeGroup.EndpointIDs = roar.FromSlice(endpoints)
}
return edgeGroup, err
}
@@ -0,0 +1,201 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newHandlerWithEdgeEndpoints(t *testing.T) (*Handler, *datastore.Store) {
t.Helper()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
return handler, store
}
func TestEdgeGroupInspectHandler(t *testing.T) {
t.Parallel()
handler, store := newHandlerWithEdgeEndpoints(t)
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups/1",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
assert.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}
func TestEmptyEdgeGroupInspectHandler(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.Roar[portainer.EndpointID]{},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups/1",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
// Make sure the frontend does not get a null value but a [] instead
require.NotNil(t, responseGroup.Endpoints)
require.Empty(t, responseGroup.Endpoints)
}
func TestDynamicEdgeGroupInspectHandler(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
err = store.Tag().Create(&portainer.Tag{
ID: 1,
Name: "Test Tag",
Endpoints: map[portainer.EndpointID]bool{
1: true,
2: true,
3: true,
},
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
TagIDs: []portainer.TagID{1},
UserTrusted: true,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
Dynamic: true,
TagIDs: []portainer.TagID{1},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups/1",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}
func TestEdgeGroupInspectPanic(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/edge_groups/1", nil)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusNotFound, rr.Result().StatusCode)
}
@@ -0,0 +1,151 @@
package edgegroups
import (
"fmt"
"net/http"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type shadowedEdgeGroup struct {
portainer.EdgeGroup
EndpointIds int `json:"EndpointIds,omitempty"` // Shadow to avoid exposing in the API
}
type decoratedEdgeGroup struct {
shadowedEdgeGroup
HasEdgeStack bool `json:"HasEdgeStack"`
HasEdgeJob bool `json:"HasEdgeJob"`
EndpointTypes []portainer.EndpointType
TrustedEndpoints []portainer.EndpointID `json:"TrustedEndpoints"`
}
// @id EdgeGroupList
// @summary list EdgeGroups
// @description **Access policy**: administrator
// @tags edge_groups
// @security ApiKeyAuth
// @security jwt
// @produce json
// @success 200 {array} decoratedEdgeGroup "EdgeGroups"
// @failure 500
// @failure 503 "Edge compute features are disabled"
// @router /edge_groups [get]
func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var decoratedEdgeGroups []decoratedEdgeGroup
var err error
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
decoratedEdgeGroups, err = getEdgeGroupList(tx)
return err
})
return response.TxResponse(w, decoratedEdgeGroups, err)
}
func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error) {
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve Edge groups from the database", err)
}
edgeStacks, err := tx.EdgeStack().EdgeStacks()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve Edge stacks from the database", err)
}
usedEdgeGroups := make(map[portainer.EdgeGroupID]bool)
for _, stack := range edgeStacks {
for _, groupID := range stack.EdgeGroups {
usedEdgeGroups[groupID] = true
}
}
edgeJobs, err := tx.EdgeJob().ReadAll()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err)
}
decoratedEdgeGroups := []decoratedEdgeGroup{}
for _, orgEdgeGroup := range edgeGroups {
usedByEdgeJob := false
for _, edgeJob := range edgeJobs {
if slices.Contains(edgeJob.EdgeGroups, orgEdgeGroup.ID) {
usedByEdgeJob = true
break
}
}
edgeGroup := decoratedEdgeGroup{
shadowedEdgeGroup: shadowedEdgeGroup{EdgeGroup: orgEdgeGroup},
EndpointTypes: []portainer.EndpointType{},
}
if edgeGroup.Dynamic {
endpointIDs, err := endpointutils.GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
}
edgeGroup.Endpoints = endpointIDs
edgeGroup.TrustedEndpoints = endpointIDs
} else {
trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.EndpointIDs)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments for Edge group", err)
}
edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
edgeGroup.TrustedEndpoints = trustedEndpoints
}
endpointTypes, err := getEndpointTypes(tx, edgeGroup.EndpointIDs)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environment types for Edge group", err)
}
edgeGroup.EndpointTypes = endpointTypes
edgeGroup.HasEdgeStack = usedEdgeGroups[edgeGroup.ID]
edgeGroup.HasEdgeJob = usedByEdgeJob
decoratedEdgeGroups = append(decoratedEdgeGroups, edgeGroup)
}
return decoratedEdgeGroups, nil
}
func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds roar.Roar[portainer.EndpointID]) ([]portainer.EndpointType, error) {
var innerErr error
typeSet := map[portainer.EndpointType]bool{}
endpointIds.Iterate(func(endpointID portainer.EndpointID) bool {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
innerErr = fmt.Errorf("failed fetching environment: %w", err)
return false
}
typeSet[endpoint.Type] = true
return true
})
if innerErr != nil {
return nil, innerErr
}
endpointTypes := make([]portainer.EndpointType, 0, len(typeSet))
for endpointType := range typeSet {
endpointTypes = append(endpointTypes, endpointType)
}
return endpointTypes, nil
}
@@ -0,0 +1,92 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_getEndpointTypes(t *testing.T) {
t.Parallel()
endpoints := []portainer.Endpoint{
{ID: 1, Type: portainer.DockerEnvironment},
{ID: 2, Type: portainer.AgentOnDockerEnvironment},
{ID: 3, Type: portainer.AzureEnvironment},
{ID: 4, Type: portainer.EdgeAgentOnDockerEnvironment},
{ID: 5, Type: portainer.KubernetesLocalEnvironment},
{ID: 6, Type: portainer.AgentOnKubernetesEnvironment},
{ID: 7, Type: portainer.EdgeAgentOnKubernetesEnvironment},
}
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints(endpoints))
tests := []struct {
endpointIds []portainer.EndpointID
expected []portainer.EndpointType
}{
{endpointIds: []portainer.EndpointID{1}, expected: []portainer.EndpointType{portainer.DockerEnvironment}},
{endpointIds: []portainer.EndpointID{2}, expected: []portainer.EndpointType{portainer.AgentOnDockerEnvironment}},
{endpointIds: []portainer.EndpointID{3}, expected: []portainer.EndpointType{portainer.AzureEnvironment}},
{endpointIds: []portainer.EndpointID{4}, expected: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment}},
{endpointIds: []portainer.EndpointID{5}, expected: []portainer.EndpointType{portainer.KubernetesLocalEnvironment}},
{endpointIds: []portainer.EndpointID{6}, expected: []portainer.EndpointType{portainer.AgentOnKubernetesEnvironment}},
{endpointIds: []portainer.EndpointID{7}, expected: []portainer.EndpointType{portainer.EdgeAgentOnKubernetesEnvironment}},
{endpointIds: []portainer.EndpointID{7, 2}, expected: []portainer.EndpointType{portainer.EdgeAgentOnKubernetesEnvironment, portainer.AgentOnDockerEnvironment}},
{endpointIds: []portainer.EndpointID{6, 4, 1}, expected: []portainer.EndpointType{portainer.AgentOnKubernetesEnvironment, portainer.EdgeAgentOnDockerEnvironment, portainer.DockerEnvironment}},
{endpointIds: []portainer.EndpointID{1, 2, 3}, expected: []portainer.EndpointType{portainer.DockerEnvironment, portainer.AgentOnDockerEnvironment, portainer.AzureEnvironment}},
}
for _, test := range tests {
ans, err := getEndpointTypes(datastore, roar.FromSlice(test.endpointIds))
require.NoError(t, err, "getEndpointTypes shouldn't fail")
assert.ElementsMatch(t, test.expected, ans, "getEndpointTypes expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
}
}
func Test_getEndpointTypes_failWhenEndpointDontExist(t *testing.T) {
t.Parallel()
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
_, err := getEndpointTypes(datastore, roar.FromSlice([]portainer.EndpointID{1}))
require.Error(t, err, "getEndpointTypes should fail")
}
func TestEdgeGroupListHandler(t *testing.T) {
t.Parallel()
handler, store := newHandlerWithEdgeEndpoints(t)
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroups []decoratedEdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroups)
require.NoError(t, err)
require.Len(t, responseGroups, 1)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroups[0].Endpoints)
require.Empty(t, responseGroups[0].TrustedEndpoints)
}
@@ -0,0 +1,203 @@
package edgegroups
import (
"errors"
"net/http"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type edgeGroupUpdatePayload struct {
Name string
Dynamic bool
TagIDs []portainer.TagID
Endpoints []portainer.EndpointID
PartialMatch *bool
}
func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
if payload.Dynamic && len(payload.TagIDs) == 0 {
return errors.New("tagIDs is mandatory for a dynamic Edge group")
}
return nil
}
// @id EdgeGroupUpdate
// @summary Updates an EdgeGroup
// @description **Access policy**: administrator
// @tags edge_groups
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "EdgeGroup Id"
// @param body body edgeGroupUpdatePayload true "EdgeGroup data"
// @success 200 {object} portainer.EdgeGroup
// @failure 503 "Edge compute features are disabled"
// @failure 500
// @router /edge_groups/{id} [put]
func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge group identifier route variable", err)
}
var payload edgeGroupUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var shadowEdgeGroup shadowedEdgeGroup
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeGroup, err := tx.EdgeGroup().Read(portainer.EdgeGroupID(edgeGroupID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge group with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge group with the specified identifier inside the database", err)
}
edgeGroups, err := tx.EdgeGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge groups from the database", err)
}
if payload.Name != "" {
for _, edgeGroup := range edgeGroups {
if edgeGroup.Name == payload.Name && edgeGroup.ID != portainer.EdgeGroupID(edgeGroupID) {
return httperror.BadRequest("Edge group name must be unique", errors.New("edge group name must be unique"))
}
}
edgeGroup.Name = payload.Name
}
endpoints, err := tx.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environments from database", err)
}
endpointGroups, err := tx.EndpointGroup().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment groups from database", err)
}
oldRelatedEndpoints := edge.EdgeGroupRelatedEndpoints(edgeGroup, endpoints, endpointGroups)
edgeGroup.Dynamic = payload.Dynamic
if err := calculateEndpointsOrTags(tx, edgeGroup, payload.Endpoints, payload.TagIDs); err != nil {
return err
}
if payload.PartialMatch != nil {
edgeGroup.PartialMatch = *payload.PartialMatch
}
if err := tx.EdgeGroup().Update(edgeGroup.ID, edgeGroup); err != nil {
return httperror.InternalServerError("Unable to persist Edge group changes inside the database", err)
}
newRelatedEndpoints := edge.EdgeGroupRelatedEndpoints(edgeGroup, endpoints, endpointGroups)
endpointsToUpdate := slicesx.Unique(append(newRelatedEndpoints, oldRelatedEndpoints...))
edgeJobs, err := tx.EdgeJob().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to fetch Edge jobs", err)
}
edgeStacks, err := tx.EdgeStack().EdgeStacks()
if err != nil {
return err
}
// Update the edgeGroups with the modified edgeGroup for updateEndpointStacks()
for i := range edgeGroups {
if edgeGroups[i].ID == edgeGroup.ID {
edgeGroups[i] = *edgeGroup
}
}
for _, endpointID := range endpointsToUpdate {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return httperror.InternalServerError("Unable to get Environment from database", err)
}
if err := handler.updateEndpointStacks(tx, endpoint, edgeGroups, edgeStacks); err != nil {
return httperror.InternalServerError("Unable to persist Environment relation changes inside the database", err)
}
if !endpointutils.IsEdgeEndpoint(endpoint) {
continue
}
var operation string
if slices.Contains(newRelatedEndpoints, endpointID) && slices.Contains(oldRelatedEndpoints, endpointID) {
continue
} else if slices.Contains(newRelatedEndpoints, endpointID) {
operation = "add"
} else if slices.Contains(oldRelatedEndpoints, endpointID) {
operation = "remove"
} else {
continue
}
if err := handler.updateEndpointEdgeJobs(edgeGroup.ID, endpoint, edgeJobs, operation); err != nil {
return httperror.InternalServerError("Unable to persist Environment Edge Jobs changes inside the database", err)
}
}
shadowEdgeGroup = shadowedEdgeGroup{EdgeGroup: *edgeGroup}
return nil
})
return response.TxResponse(w, shadowEdgeGroup, err)
}
func (handler *Handler) updateEndpointStacks(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) error {
relation, err := tx.EndpointRelation().EndpointRelation(endpoint.ID)
if err != nil {
return err
}
endpointGroup, err := tx.EndpointGroup().Read(endpoint.GroupID)
if err != nil {
return err
}
edgeStackSet := map[portainer.EdgeStackID]bool{}
endpointEdgeStacks := edge.EndpointRelatedEdgeStacks(endpoint, endpointGroup, edgeGroups, edgeStacks)
for _, edgeStackID := range endpointEdgeStacks {
edgeStackSet[edgeStackID] = true
}
relation.EdgeStacks = edgeStackSet
return tx.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
}
func (handler *Handler) updateEndpointEdgeJobs(edgeGroupID portainer.EdgeGroupID, endpoint *portainer.Endpoint, edgeJobs []portainer.EdgeJob, operation string) error {
for _, edgeJob := range edgeJobs {
if !slices.Contains(edgeJob.EdgeGroups, edgeGroupID) {
continue
}
switch operation {
case "add", "remove":
cache.Del(endpoint.ID)
}
}
return nil
}
@@ -0,0 +1,59 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestEdgeGroupUpdateHandler(t *testing.T) {
t.Parallel()
handler, store := newHandlerWithEdgeEndpoints(t)
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPut,
"/edge_groups/1",
strings.NewReader(`{"Endpoints": [1, 2, 3]}`),
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}
func TestEdgeGroupUpdatePanic(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPut, "/edge_groups/1", strings.NewReader("{}"))
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusNotFound, rr.Result().StatusCode)
}
+38
View File
@@ -0,0 +1,38 @@
package edgegroups
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle environment(endpoint) group operations.
type Handler struct {
*mux.Router
DataStore dataservices.DataStore
ReverseTunnelService portainer.ReverseTunnelService
}
// NewHandler creates a handler to manage environment(endpoint) group operations.
func NewHandler(bouncer security.BouncerService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
}
h.Handle("/edge_groups",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupCreate)))).Methods(http.MethodPost)
h.Handle("/edge_groups",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupList)))).Methods(http.MethodGet)
h.Handle("/edge_groups/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupInspect)))).Methods(http.MethodGet)
h.Handle("/edge_groups/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupUpdate)))).Methods(http.MethodPut)
h.Handle("/edge_groups/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeGroupDelete)))).Methods(http.MethodDelete)
return h
}
+271
View File
@@ -0,0 +1,271 @@
package edgejobs
import (
"errors"
"maps"
"net/http"
"strconv"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
)
type edgeJobBasePayload struct {
Name string
CronExpression string
Recurring bool
Endpoints []portainer.EndpointID
EdgeGroups []portainer.EdgeGroupID
}
func (handler *Handler) edgeJobCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
method, err := request.RetrieveRouteVariableValue(r, "method")
if err != nil {
return httperror.BadRequest("Invalid query parameter: method. Valid values are: file or string", err)
}
switch method {
case "string":
return handler.createEdgeJobFromFileContent(w, r)
case "file":
return handler.createEdgeJobFromFile(w, r)
default:
return httperror.BadRequest("Invalid query parameter: method. Valid values are: file or string", errors.New(strings.ToLower(request.ErrInvalidQueryParameter)))
}
}
type edgeJobCreateFromFileContentPayload struct {
edgeJobBasePayload
FileContent string
}
func (payload *edgeJobCreateFromFileContentPayload) Validate(r *http.Request) error {
if len(payload.Name) == 0 {
return errors.New("invalid Edge job name")
}
if !validate.Matches(payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) {
return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
}
if len(payload.CronExpression) == 0 {
return errors.New("invalid cron expression")
}
if len(payload.Endpoints) == 0 && len(payload.EdgeGroups) == 0 {
return errors.New("no environments or groups have been provided")
}
if len(payload.FileContent) == 0 {
return errors.New("invalid script file content")
}
return nil
}
// @id EdgeJobCreateString
// @summary Create an EdgeJob from a text
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param body body edgeJobCreateFromFileContentPayload true "EdgeGroup data when method is string"
// @success 200 {object} portainer.EdgeGroup
// @failure 503 "Edge compute features are disabled"
// @failure 500
// @router /edge_jobs/create/string [post]
func (handler *Handler) createEdgeJobFromFileContent(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload edgeJobCreateFromFileContentPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var edgeJob *portainer.EdgeJob
var err error
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeJob, err = handler.createEdgeJob(tx, &payload.edgeJobBasePayload, []byte(payload.FileContent))
return err
})
return response.TxResponse(w, edgeJob, err)
}
func (handler *Handler) createEdgeJob(tx dataservices.DataStoreTx, payload *edgeJobBasePayload, fileContent []byte) (*portainer.EdgeJob, error) {
var err error
edgeJob := handler.createEdgeJobObjectFromPayload(tx, payload)
var endpoints []portainer.EndpointID
if len(edgeJob.EdgeGroups) > 0 {
endpoints, err = edge.GetEndpointsFromEdgeGroups(payload.EdgeGroups, tx)
if err != nil {
return nil, httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
}
if err := handler.addAndPersistEdgeJob(tx, edgeJob, fileContent, endpoints); err != nil {
return nil, httperror.InternalServerError("Unable to schedule Edge job", err)
}
for _, endpointID := range endpoints {
cache.Del(endpointID)
}
return edgeJob, nil
}
type edgeJobCreateFromFilePayload struct {
edgeJobBasePayload
File []byte
}
func (payload *edgeJobCreateFromFilePayload) Validate(r *http.Request) error {
name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
if err != nil {
return errors.New("invalid Edge job name")
}
if !validate.Matches(name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
}
payload.Name = name
cronExpression, err := request.RetrieveMultiPartFormValue(r, "CronExpression", false)
if err != nil {
return errors.New("invalid cron expression")
}
payload.CronExpression = cronExpression
var endpoints []portainer.EndpointID
if err := request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, true); err != nil {
return errors.New("invalid environments")
}
payload.Endpoints = endpoints
var edgeGroups []portainer.EdgeGroupID
if err := request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, true); err != nil {
return errors.New("invalid edge groups")
}
payload.EdgeGroups = edgeGroups
if len(payload.Endpoints) == 0 && len(payload.EdgeGroups) == 0 {
return errors.New("no environments or groups have been provided")
}
file, _, err := request.RetrieveMultiPartFormFile(r, "file")
if err != nil {
return errors.New("invalid script file. Ensure that the file is uploaded correctly")
}
payload.File = file
return nil
}
// @id EdgeJobCreateFile
// @summary Create an EdgeJob from a file
// @description **Access policy**: administrator
// @tags edge_jobs
// @accept multipart/form-data
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param file formData file true "Content of the Stack file"
// @param Name formData string true "Name of the stack"
// @param CronExpression formData string true "A cron expression to schedule this job"
// @param EdgeGroups formData string true "JSON stringified array of Edge Groups ids"
// @param Endpoints formData string true "JSON stringified array of Environment ids"
// @param Recurring formData bool false "If recurring"
// @success 200 {object} portainer.EdgeGroup
// @failure 503 "Edge compute features are disabled"
// @failure 500
// @router /edge_jobs/create/file [post]
func (handler *Handler) createEdgeJobFromFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
payload := &edgeJobCreateFromFilePayload{}
if err := payload.Validate(r); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var edgeJob *portainer.EdgeJob
var err error
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeJob, err = handler.createEdgeJob(tx, &payload.edgeJobBasePayload, payload.File)
return err
})
return response.TxResponse(w, edgeJob, err)
}
func (handler *Handler) createEdgeJobObjectFromPayload(tx dataservices.DataStoreTx, payload *edgeJobBasePayload) *portainer.EdgeJob {
return &portainer.EdgeJob{
ID: portainer.EdgeJobID(tx.EdgeJob().GetNextIdentifier()),
Name: payload.Name,
CronExpression: payload.CronExpression,
Recurring: payload.Recurring,
Created: time.Now().Unix(),
Endpoints: convertEndpointsToMetaObject(payload.Endpoints),
EdgeGroups: payload.EdgeGroups,
Version: 1,
GroupLogsCollection: map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{},
}
}
func (handler *Handler) addAndPersistEdgeJob(tx dataservices.DataStoreTx, edgeJob *portainer.EdgeJob, file []byte, endpointsFromGroups []portainer.EndpointID) error {
edgeCronExpression := strings.Split(edgeJob.CronExpression, " ")
if len(edgeCronExpression) == 6 {
edgeCronExpression = edgeCronExpression[1:]
}
edgeJob.CronExpression = strings.Join(edgeCronExpression, " ")
for ID := range edgeJob.Endpoints {
endpoint, err := tx.Endpoint().Endpoint(ID)
if err != nil {
return err
}
if !endpointutils.IsEdgeEndpoint(endpoint) {
delete(edgeJob.Endpoints, ID)
}
}
scriptPath, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), file)
if err != nil {
return err
}
edgeJob.ScriptPath = scriptPath
var endpointsMap map[portainer.EndpointID]portainer.EdgeJobEndpointMeta
if len(endpointsFromGroups) > 0 {
endpointsMap = convertEndpointsToMetaObject(endpointsFromGroups)
for ID := range endpointsMap {
endpoint, err := tx.Endpoint().Endpoint(ID)
if err != nil {
return err
}
if !endpointutils.IsEdgeEndpoint(endpoint) {
delete(endpointsMap, ID)
}
}
maps.Copy(endpointsMap, edgeJob.Endpoints)
} else {
endpointsMap = edgeJob.Endpoints
}
if len(endpointsMap) == 0 {
return errors.New("environments or edge groups are mandatory for an Edge job")
}
return tx.EdgeJob().CreateWithID(edgeJob.ID, edgeJob)
}
@@ -0,0 +1,160 @@
package edgejobs
import (
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
type mockFileService struct {
mock.Mock
portainer.FileService
}
func (m *mockFileService) StoreEdgeJobFileFromBytes(id string, file []byte) (string, error) {
args := m.Called(id, file)
return args.String(0), args.Error(1)
}
func (m *mockFileService) GetEdgeJobFolder(id string) string {
args := m.Called(id)
return args.String(0)
}
func (m *mockFileService) RemoveDirectory(path string) error {
args := m.Called(path)
return args.Error(0)
}
func initStore(t *testing.T) *datastore.Store {
_, store := datastore.MustNewTestStore(t, true, true)
require.NotNil(t, store)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.Endpoint().Create(&portainer.Endpoint{
ID: 1,
Name: "endpoint-1",
EdgeID: "edge-id-1",
GroupID: 1,
Type: portainer.EdgeAgentOnDockerEnvironment,
UserTrusted: true,
}))
require.NoError(t, tx.Endpoint().Create(&portainer.Endpoint{
ID: 2,
Name: "endpoint-2",
EdgeID: "edge-id-2",
GroupID: 1,
Type: portainer.EdgeAgentOnDockerEnvironment,
UserTrusted: false,
}))
return nil
}))
return store
}
func Test_edgeJobCreate_StringMethod_Success(t *testing.T) {
t.Parallel()
store := initStore(t)
fileService := &mockFileService{}
fileService.On("StoreEdgeJobFileFromBytes", mock.Anything, mock.Anything).Return("testfile.txt", nil)
handler := &Handler{
DataStore: store,
FileService: fileService,
}
payload := edgeJobCreateFromFileContentPayload{
edgeJobBasePayload: edgeJobBasePayload{
Name: "testjob",
CronExpression: "* * * * *",
Endpoints: []portainer.EndpointID{1, 2},
},
FileContent: "echo hello",
}
body, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, "/edge_jobs/create/string", bytes.NewReader(body))
req = mux.SetURLVars(req, map[string]string{"method": "string"})
w := httptest.NewRecorder()
// Call handler
errh := handler.edgeJobCreate(w, req)
require.Nil(t, errh)
require.Equal(t, http.StatusOK, w.Result().StatusCode)
// Get edge job ID from response
var resp struct {
ID int `json:"Id"`
}
require.NoError(t, json.NewDecoder(w.Body).Decode(&resp))
edgeJob, err := store.EdgeJob().Read(portainer.EdgeJobID(resp.ID))
require.NoError(t, err)
require.Len(t, edgeJob.Endpoints, 2)
require.Contains(t, edgeJob.Endpoints, portainer.EndpointID(1))
}
func Test_edgeJobCreate_FileMethod_Success(t *testing.T) {
t.Parallel()
store := initStore(t)
fileService := &mockFileService{}
fileService.On("StoreEdgeJobFileFromBytes", mock.Anything, mock.Anything).Return("testfile.txt", nil)
handler := &Handler{
DataStore: store,
FileService: fileService,
}
var body bytes.Buffer
writer := multipart.NewWriter(&body)
require.NoError(t, writer.WriteField("Name", "testjob"))
require.NoError(t, writer.WriteField("CronExpression", "* * * * *"))
require.NoError(t, writer.WriteField("Endpoints", "[1,2]"))
fileWriter, err := writer.CreateFormFile("file", "test.txt")
require.NoError(t, err)
_, err = io.Copy(fileWriter, strings.NewReader("echo hello"))
require.NoError(t, err)
require.NoError(t, writer.Close())
req := httptest.NewRequest(http.MethodPost, "/edge_jobs/create/file", &body)
req = mux.SetURLVars(req, map[string]string{"method": "file"})
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
handlerErr := handler.edgeJobCreate(w, req)
require.Nil(t, handlerErr)
require.Equal(t, http.StatusOK, w.Result().StatusCode)
var resp struct {
ID int `json:"Id"`
}
require.NoError(t, json.NewDecoder(w.Body).Decode(&resp))
edgeJob, err := store.EdgeJob().Read(portainer.EdgeJobID(resp.ID))
require.NoError(t, err)
require.Len(t, edgeJob.Endpoints, 2)
require.Contains(t, edgeJob.Endpoints, portainer.EndpointID(1))
}
@@ -0,0 +1,79 @@
package edgejobs
import (
"maps"
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
// @id EdgeJobDelete
// @summary Delete an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EdgeJob Id"
// @success 204
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id} [delete]
func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.deleteEdgeJob(tx, portainer.EdgeJobID(edgeJobID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) deleteEdgeJob(tx dataservices.DataStoreTx, edgeJobID portainer.EdgeJobID) error {
edgeJob, err := tx.EdgeJob().Read(edgeJobID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
edgeJobFolder := handler.FileService.GetEdgeJobFolder(strconv.Itoa(int(edgeJobID)))
if err := handler.FileService.RemoveDirectory(edgeJobFolder); err != nil {
log.Warn().Err(err).Msg("Unable to remove the files associated to the Edge job on the filesystem")
}
var endpointsMap map[portainer.EndpointID]portainer.EdgeJobEndpointMeta
if len(edgeJob.EdgeGroups) > 0 {
endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
if err != nil {
return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
endpointsMap = convertEndpointsToMetaObject(endpoints)
maps.Copy(endpointsMap, edgeJob.Endpoints)
} else {
endpointsMap = edgeJob.Endpoints
}
for endpointID := range endpointsMap {
cache.Del(endpointID)
}
if err := tx.EdgeJob().Delete(edgeJob.ID); err != nil {
return httperror.InternalServerError("Unable to remove the Edge job from the database", err)
}
return nil
}
+48
View File
@@ -0,0 +1,48 @@
package edgejobs
import (
"net/http"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type edgeJobFileResponse struct {
FileContent string `json:"FileContent"`
}
// @id EdgeJobFile
// @summary Fetch a file of an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @success 200 {object} edgeJobFileResponse
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id}/file [get]
func (handler *Handler) edgeJobFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
edgeJob, err := handler.DataStore.EdgeJob().Read(portainer.EdgeJobID(edgeJobID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
edgeJobFileContent, err := handler.FileService.GetFileContent(edgeJob.ScriptPath, "")
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge job script file from disk", err)
}
return response.JSON(w, &edgeJobFileResponse{FileContent: string(edgeJobFileContent)})
}
@@ -0,0 +1,52 @@
package edgejobs
import (
"net/http"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type edgeJobInspectResponse struct {
*portainer.EdgeJob
Endpoints []portainer.EndpointID
}
// @id EdgeJobInspect
// @summary Inspect an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @success 200 {object} portainer.EdgeJob
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id} [get]
func (handler *Handler) edgeJobInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
edgeJob, err := handler.DataStore.EdgeJob().Read(portainer.EdgeJobID(edgeJobID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
responseObj := edgeJobInspectResponse{
EdgeJob: edgeJob,
}
for endpointID := range edgeJob.Endpoints {
responseObj.Endpoints = append(responseObj.Endpoints, endpointID)
}
return response.JSON(w, responseObj)
}
+30
View File
@@ -0,0 +1,30 @@
package edgejobs
import (
"net/http"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeJobList
// @summary Fetch EdgeJobs list
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @success 200 {array} portainer.EdgeJob
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs [get]
// GET request on /api/edge_jobs
func (handler *Handler) edgeJobList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobs, err := handler.DataStore.EdgeJob().ReadAll()
if err != nil {
return httperror.InternalServerError("Unable to retrieve Edge jobs from the database", err)
}
return response.JSON(w, edgeJobs)
}
@@ -0,0 +1,93 @@
package edgejobs
import (
"net/http"
"slices"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeJobTasksClear
// @summary Clear the log for a specifc task on an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @param taskID path int true "Task Id"
// @success 204
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id}/tasks/{taskID}/logs [delete]
func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
taskID, err := request.RetrieveNumericRouteVariableValue(r, "taskID")
if err != nil {
return httperror.BadRequest("Invalid Task identifier route variable", err)
}
mutationFn := func(edgeJob *portainer.EdgeJob, endpointID portainer.EndpointID, endpointsFromGroups []portainer.EndpointID) {
if slices.Contains(endpointsFromGroups, endpointID) {
edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
CollectLogs: false,
LogsStatus: portainer.EdgeJobLogsStatusIdle,
}
} else {
meta := edgeJob.Endpoints[endpointID]
meta.CollectLogs = false
meta.LogsStatus = portainer.EdgeJobLogsStatusIdle
edgeJob.Endpoints[endpointID] = meta
}
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
updateEdgeJobFn := func(edgeJob *portainer.EdgeJob, endpointID portainer.EndpointID, endpointsFromGroups []portainer.EndpointID) error {
mutationFn(edgeJob, endpointID, endpointsFromGroups)
return tx.EdgeJob().Update(edgeJob.ID, edgeJob)
}
return handler.clearEdgeJobTaskLogs(tx, portainer.EdgeJobID(edgeJobID), portainer.EndpointID(taskID), updateEdgeJobFn)
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) clearEdgeJobTaskLogs(tx dataservices.DataStoreTx, edgeJobID portainer.EdgeJobID, endpointID portainer.EndpointID, updateEdgeJob func(*portainer.EdgeJob, portainer.EndpointID, []portainer.EndpointID) error) error {
edgeJob, err := tx.EdgeJob().Read(edgeJobID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
if err := handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(int(edgeJobID)), strconv.Itoa(int(endpointID))); err != nil {
return httperror.InternalServerError("Unable to clear log file from disk", err)
}
endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
if err != nil {
return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
if err := updateEdgeJob(edgeJob, endpointID, endpointsFromGroups); err != nil {
return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
}
cache.Del(endpointID)
return nil
}
@@ -0,0 +1,86 @@
package edgejobs
import (
"net/http"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeJobTasksCollect
// @summary Collect the log for a specifc task on an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @param taskID path int true "Task Id"
// @success 204
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id}/tasks/{taskID}/logs [post]
func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
taskID, err := request.RetrieveNumericRouteVariableValue(r, "taskID")
if err != nil {
return httperror.BadRequest("Invalid Task identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeJob, err := tx.EdgeJob().Read(portainer.EdgeJobID(edgeJobID))
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
endpointID := portainer.EndpointID(taskID)
endpointsFromGroups, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
if err != nil {
return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
if slices.Contains(endpointsFromGroups, endpointID) {
edgeJob.GroupLogsCollection[endpointID] = portainer.EdgeJobEndpointMeta{
CollectLogs: true,
LogsStatus: portainer.EdgeJobLogsStatusPending,
}
} else {
meta := edgeJob.Endpoints[endpointID]
meta.CollectLogs = true
meta.LogsStatus = portainer.EdgeJobLogsStatusPending
edgeJob.Endpoints[endpointID] = meta
}
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
}
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment from the database", err)
}
cache.Del(endpointID)
if endpoint.Edge.AsyncMode {
return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
}
return nil
})
return response.TxEmptyResponse(w, err)
}
@@ -0,0 +1,47 @@
package edgejobs
import (
"net/http"
"strconv"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type fileResponse struct {
FileContent string `json:"FileContent"`
}
// @id EdgeJobTaskLogsInspect
// @summary Fetch the log for a specifc task on an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @param taskID path int true "Task Id"
// @success 200 {object} fileResponse
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id}/tasks/{taskID}/logs [get]
func (handler *Handler) edgeJobTaskLogsInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
taskID, err := request.RetrieveNumericRouteVariableValue(r, "taskID")
if err != nil {
return httperror.BadRequest("Invalid Task identifier route variable", err)
}
logFileContent, err := handler.FileService.GetEdgeJobTaskLogFileContent(strconv.Itoa(edgeJobID), strconv.Itoa(taskID))
if err != nil {
return httperror.InternalServerError("Unable to retrieve log file from disk", err)
}
return response.JSON(w, &fileResponse{FileContent: logFileContent})
}
@@ -0,0 +1,128 @@
package edgejobs
import (
"errors"
"fmt"
"maps"
"net/http"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/utils/filters"
"github.com/portainer/portainer/api/internal/edge"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type taskContainer struct {
ID string `json:"Id"`
EndpointID portainer.EndpointID `json:"EndpointId"`
EndpointName string `json:"EndpointName"`
LogsStatus portainer.EdgeJobLogsStatus `json:"LogsStatus"`
}
// @id EdgeJobTasksList
// @summary Fetch the list of tasks on an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeJob Id"
// @success 200 {array} taskContainer
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id}/tasks [get]
func (handler *Handler) edgeJobTasksList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
params := filters.ExtractListModifiersQueryParams(r)
var tasks []*taskContainer
err = handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
tasks, err = listEdgeJobTasks(tx, portainer.EdgeJobID(edgeJobID))
return err
})
return response.TxFuncResponse(err, func() *httperror.HandlerError {
results := filters.SearchOrderAndPaginate(tasks, params, filters.Config[*taskContainer]{
SearchAccessors: []filters.SearchAccessor[*taskContainer]{
func(tc *taskContainer) (string, error) {
switch tc.LogsStatus {
case portainer.EdgeJobLogsStatusPending:
return "pending", nil
case 0, portainer.EdgeJobLogsStatusIdle:
return "idle", nil
case portainer.EdgeJobLogsStatusCollected:
return "collected", nil
}
return "", errors.New("unknown state")
},
func(tc *taskContainer) (string, error) {
return tc.EndpointName, nil
},
},
SortBindings: []filters.SortBinding[*taskContainer]{
{Key: "EndpointName", Fn: func(a, b *taskContainer) int { return strings.Compare(a.EndpointName, b.EndpointName) }},
},
})
filters.ApplyFilterResultsHeaders(&w, results)
return response.JSON(w, results.Items)
})
}
func listEdgeJobTasks(tx dataservices.DataStoreTx, edgeJobID portainer.EdgeJobID) ([]*taskContainer, error) {
edgeJob, err := tx.EdgeJob().Read(edgeJobID)
if tx.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return nil, httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
endpoints, err := tx.Endpoint().Endpoints()
if err != nil {
return nil, err
}
tasks := make([]*taskContainer, 0)
endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
if len(edgeJob.EdgeGroups) > 0 {
endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
if err != nil {
return nil, httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
endpointsMap = convertEndpointsToMetaObject(endpoints)
maps.Copy(endpointsMap, edgeJob.GroupLogsCollection)
}
maps.Copy(endpointsMap, edgeJob.Endpoints)
for endpointID, meta := range endpointsMap {
endpointName := ""
for idx := range endpoints {
if endpoints[idx].ID == endpointID {
endpointName = endpoints[idx].Name
}
}
tasks = append(tasks, &taskContainer{
ID: fmt.Sprintf("edgejob_task_%d_%d", edgeJob.ID, endpointID),
EndpointID: endpointID,
EndpointName: endpointName,
LogsStatus: meta.LogsStatus,
})
}
return tasks, nil
}
@@ -0,0 +1,135 @@
package edgejobs
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_EdgeJobTasksListHandler(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
addEnv := func(env *portainer.Endpoint) {
err := store.EndpointService.Create(env)
require.NoError(t, err)
}
addEdgeGroup := func(group *portainer.EdgeGroup) {
err := store.EdgeGroupService.Create(group)
require.NoError(t, err)
}
addJob := func(job *portainer.EdgeJob) {
err := store.EdgeJobService.Create(job)
require.NoError(t, err)
}
envCount := 6
for i := range envCount {
addEnv(&portainer.Endpoint{ID: portainer.EndpointID(i + 1), Name: "env_" + strconv.Itoa(i+1)})
}
addEdgeGroup(&portainer.EdgeGroup{ID: 1, Name: "edge_group_1", EndpointIDs: roar.FromSlice([]portainer.EndpointID{5, 6})})
addJob(&portainer.EdgeJob{
ID: 1,
Endpoints: map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{
1: {},
2: {LogsStatus: portainer.EdgeJobLogsStatusIdle},
3: {LogsStatus: portainer.EdgeJobLogsStatusPending},
4: {LogsStatus: portainer.EdgeJobLogsStatusCollected}},
EdgeGroups: []portainer.EdgeGroupID{1},
})
test := func(params string, expect []taskContainer, expectedCount int) {
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/edge_jobs/1/tasks"+params, nil)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var response []taskContainer
err := json.NewDecoder(rr.Body).Decode(&response)
require.NoError(t, err)
assert.ElementsMatch(t, expect, response)
tcStr := rr.Header().Get("x-total-count")
assert.NotEmpty(t, tcStr)
totalCount, err := strconv.Atoi(tcStr)
require.NoError(t, err)
assert.Equal(t, expectedCount, totalCount)
taStr := rr.Header().Get("x-total-available")
assert.NotEmpty(t, taStr)
totalAvailable, err := strconv.Atoi(taStr)
require.NoError(t, err)
assert.Equal(t, envCount, totalAvailable)
}
tasks := []taskContainer{
{},
{"edgejob_task_1_1", 1, "env_1", 0},
{"edgejob_task_1_2", 2, "env_2", portainer.EdgeJobLogsStatusIdle},
{"edgejob_task_1_3", 3, "env_3", portainer.EdgeJobLogsStatusPending},
{"edgejob_task_1_4", 4, "env_4", portainer.EdgeJobLogsStatusCollected},
{"edgejob_task_1_5", 5, "env_5", 0},
{"edgejob_task_1_6", 6, "env_6", 0},
}
t.Run("should return no results", func(t *testing.T) {
test("?search=foo", []taskContainer{}, 0) // unknown search
test("?start=100&limit=1", []taskContainer{}, 6) // overflowing start. Still return the correct count header
})
t.Run("should return one element", func(t *testing.T) {
// limit the *returned* results but not the total count
test("?start=0&limit=1&sort=EndpointName&order=asc", []taskContainer{tasks[1]}, envCount) // limit
test("?start=5&limit=10&sort=EndpointName&order=asc", []taskContainer{tasks[6]}, envCount) // start = last element + overflowing limit
// limit the number of results
test("?search=env_1", []taskContainer{tasks[1]}, 1) // only 1 result
})
t.Run("should filter by status", func(t *testing.T) {
test("?search=idle", []taskContainer{tasks[1], tasks[2], tasks[5], tasks[6]}, 4) // 0 (default value) is IDLE
test("?search=pending", []taskContainer{tasks[3]}, 1)
test("?search=collected", []taskContainer{tasks[4]}, 1)
})
t.Run("should return all elements", func(t *testing.T) {
test("", tasks[1:], envCount) // default
test("?some=invalid_param", tasks[1:], envCount) // unknown query params
test("?limit=-1", tasks[1:], envCount) // underflowing limit
test("?start=100", tasks[1:], envCount) // overflowing start without limit
test("?search=env", tasks[1:], envCount) // search in a match-all keyword
})
testError := func(params string, status int) {
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/edge_jobs/2/tasks"+params, nil)
handler.ServeHTTP(rr, req)
require.Equal(t, status, rr.Result().StatusCode)
}
t.Run("errors", func(t *testing.T) {
testError("", http.StatusNotFound) // unknown job id
})
}
+233
View File
@@ -0,0 +1,233 @@
package edgejobs
import (
"errors"
"maps"
"net/http"
"slices"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
)
type edgeJobUpdatePayload struct {
Name *string
CronExpression *string
Recurring *bool
Endpoints []portainer.EndpointID
EdgeGroups []portainer.EdgeGroupID
FileContent *string
}
func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
if payload.Name != nil && !validate.Matches(*payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
}
return nil
}
// @id EdgeJobUpdate
// @summary Update an EdgeJob
// @description **Access policy**: administrator
// @tags edge_jobs
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "EdgeJob Id"
// @param body body edgeJobUpdatePayload true "EdgeGroup data"
// @success 200 {object} portainer.EdgeJob
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_jobs/{id} [put]
func (handler *Handler) edgeJobUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
var payload edgeJobUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var edgeJob *portainer.EdgeJob
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeJob, err = handler.updateEdgeJob(tx, portainer.EdgeJobID(edgeJobID), payload)
return err
})
return response.TxResponse(w, edgeJob, err)
}
func (handler *Handler) updateEdgeJob(tx dataservices.DataStoreTx, edgeJobID portainer.EdgeJobID, payload edgeJobUpdatePayload) (*portainer.EdgeJob, error) {
edgeJob, err := tx.EdgeJob().Read(edgeJobID)
if tx.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
} else if err != nil {
return nil, httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
if err := handler.updateEdgeSchedule(tx, edgeJob, &payload); err != nil {
return nil, httperror.InternalServerError("Unable to update Edge job", err)
}
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
return nil, httperror.InternalServerError("Unable to persist Edge job changes inside the database", err)
}
return edgeJob, nil
}
func (handler *Handler) updateEdgeSchedule(tx dataservices.DataStoreTx, edgeJob *portainer.EdgeJob, payload *edgeJobUpdatePayload) error {
if payload.Name != nil {
edgeJob.Name = *payload.Name
}
endpointsToAdd := map[portainer.EndpointID]bool{}
endpointsToRemove := map[portainer.EndpointID]bool{}
if payload.Endpoints != nil {
endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
newEndpoints := endpointutils.EndpointSet(payload.Endpoints)
for endpointID := range edgeJob.Endpoints {
if !newEndpoints[endpointID] {
endpointsToRemove[endpointID] = true
}
}
for _, endpointID := range payload.Endpoints {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
if !endpointutils.IsEdgeEndpoint(endpoint) {
continue
}
if meta, exists := edgeJob.Endpoints[endpointID]; exists {
endpointsMap[endpointID] = meta
} else {
endpointsMap[endpointID] = portainer.EdgeJobEndpointMeta{}
endpointsToAdd[endpointID] = true
}
}
edgeJob.Endpoints = endpointsMap
}
if len(payload.EdgeGroups) == 0 && len(edgeJob.EdgeGroups) > 0 {
endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
if err != nil {
return errors.New("unable to get endpoints from edge groups")
}
for _, endpointID := range endpoints {
endpointsToRemove[endpointID] = true
}
edgeJob.EdgeGroups = nil
}
edgeGroupsToAdd := []portainer.EdgeGroupID{}
edgeGroupsToRemove := []portainer.EdgeGroupID{}
endpointsFromGroupsToAddMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
if len(payload.EdgeGroups) > 0 {
for _, edgeGroupID := range payload.EdgeGroups {
if ok, err := tx.EdgeGroup().Exists(edgeGroupID); !ok {
return dserrors.ErrObjectNotFound
} else if err != nil {
return err
}
if !slices.Contains(edgeJob.EdgeGroups, edgeGroupID) {
edgeGroupsToAdd = append(edgeGroupsToAdd, edgeGroupID)
}
}
endpointsFromGroupsToAdd, err := edge.GetEndpointsFromEdgeGroups(edgeGroupsToAdd, tx)
if err != nil {
return errors.New("unable to get endpoints from edge groups")
}
endpointsFromGroupsToAddMap = convertEndpointsToMetaObject(endpointsFromGroupsToAdd)
for endpointID := range endpointsFromGroupsToAddMap {
endpointsToAdd[endpointID] = true
}
newEdgeGroups := edge.EdgeGroupSet(payload.EdgeGroups)
for _, edgeGroupID := range edgeJob.EdgeGroups {
if !newEdgeGroups[edgeGroupID] {
edgeGroupsToRemove = append(edgeGroupsToRemove, edgeGroupID)
}
}
endpointsFromGroupsToRemove, err := edge.GetEndpointsFromEdgeGroups(edgeGroupsToRemove, tx)
if err != nil {
return errors.New("unable to get endpoints from edge groups")
}
endpointsToRemoveMap := convertEndpointsToMetaObject(endpointsFromGroupsToRemove)
for endpointID := range endpointsToRemoveMap {
endpointsToRemove[endpointID] = true
}
edgeJob.EdgeGroups = payload.EdgeGroups
}
updateVersion := false
if payload.CronExpression != nil && *payload.CronExpression != edgeJob.CronExpression {
edgeJob.CronExpression = *payload.CronExpression
updateVersion = true
}
fileContent, err := handler.FileService.GetFileContent(edgeJob.ScriptPath, "")
if err != nil {
return err
}
if payload.FileContent != nil && *payload.FileContent != string(fileContent) {
fileContent = []byte(*payload.FileContent)
if _, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), fileContent); err != nil {
return err
}
updateVersion = true
}
if payload.Recurring != nil && *payload.Recurring != edgeJob.Recurring {
edgeJob.Recurring = *payload.Recurring
updateVersion = true
}
if updateVersion {
edgeJob.Version++
}
maps.Copy(endpointsFromGroupsToAddMap, edgeJob.Endpoints)
for endpointID := range endpointsFromGroupsToAddMap {
cache.Del(endpointID)
}
for endpointID := range endpointsToRemove {
cache.Del(endpointID)
}
return nil
}
+60
View File
@@ -0,0 +1,60 @@
package edgejobs
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle Edge job operations.
type Handler struct {
*mux.Router
DataStore dataservices.DataStore
FileService portainer.FileService
ReverseTunnelService portainer.ReverseTunnelService
}
// NewHandler creates a handler to manage Edge job operations.
func NewHandler(bouncer security.BouncerService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
}
h.Handle("/edge_jobs",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobList)))).Methods(http.MethodGet)
h.Handle("/edge_jobs/create/{method}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobCreate)))).Methods(http.MethodPost)
h.Handle("/edge_jobs/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobInspect)))).Methods(http.MethodGet)
h.Handle("/edge_jobs/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobUpdate)))).Methods(http.MethodPut)
h.Handle("/edge_jobs/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobDelete)))).Methods(http.MethodDelete)
h.Handle("/edge_jobs/{id}/file",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobFile)))).Methods(http.MethodGet)
h.Handle("/edge_jobs/{id}/tasks",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobTasksList)))).Methods(http.MethodGet)
h.Handle("/edge_jobs/{id}/tasks/{taskID}/logs",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobTaskLogsInspect)))).Methods(http.MethodGet)
h.Handle("/edge_jobs/{id}/tasks/{taskID}/logs",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobTasksCollect)))).Methods(http.MethodPost)
h.Handle("/edge_jobs/{id}/tasks/{taskID}/logs",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeJobTasksClear)))).Methods(http.MethodDelete)
return h
}
func convertEndpointsToMetaObject(endpoints []portainer.EndpointID) map[portainer.EndpointID]portainer.EdgeJobEndpointMeta {
endpointsMap := map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{}
for _, endpointID := range endpoints {
endpointsMap[endpointID] = portainer.EdgeJobEndpointMeta{}
}
return endpointsMap
}
@@ -0,0 +1,57 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
method, err := request.RetrieveRouteVariableValue(r, "method")
if err != nil {
return httperror.BadRequest("Invalid query parameter: method", err)
}
dryrun, _ := request.RetrieveBooleanQueryParameter(r, "dryrun", true)
tokenData, err := security.RetrieveTokenData(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
}
var edgeStack *portainer.EdgeStack
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeStack, err = handler.createSwarmStack(tx, method, dryrun, tokenData, r)
return err
}); err != nil {
switch {
case httperrors.IsInvalidPayloadError(err):
return httperror.BadRequest("Invalid payload", err)
case httperrors.IsConflictError(err):
return httperror.Conflict(err.Error(), err)
default:
return httperror.InternalServerError("Unable to create Edge stack", err)
}
}
return response.JSON(w, edgeStack)
}
func (handler *Handler) createSwarmStack(tx dataservices.DataStoreTx, method string, dryrun bool, tokenData *portainer.TokenData, r *http.Request) (*portainer.EdgeStack, error) {
switch method {
case "string":
return handler.createEdgeStackFromFileContent(r, tx, tokenData, dryrun)
case "repository":
return handler.createEdgeStackFromGitRepository(r, tx, tokenData, dryrun)
case "file":
return handler.createEdgeStackFromFileUpload(r, tx, tokenData, dryrun)
}
return nil, httperrors.NewInvalidPayloadError("Invalid value for query parameter: method. Value must be one of: string, repository or file")
}
@@ -0,0 +1,129 @@
package edgestacks
import (
"fmt"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/portainer/portainer/pkg/edge"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/pkg/errors"
)
type edgeStackFromFileUploadPayload struct {
// Name of the stack
// Max length: 255
// Name must only contains lowercase characters, numbers, hyphens, or underscores
// Name must start with a lowercase character or number
// Example: stack-name or stack_123 or stackName
Name string
StackFileContent []byte
EdgeGroups []portainer.EdgeGroupID
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes'
// compose is enabled only for docker environments
// kubernetes is enabled only for kubernetes environments
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
Registries []portainer.RegistryID
// Uses the manifest's namespaces instead of the default one
UseManifestNamespaces bool
}
func (payload *edgeStackFromFileUploadPayload) Validate(r *http.Request) error {
name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
if err != nil {
return httperrors.NewInvalidPayloadError("Invalid stack name")
}
payload.Name = name
if !edge.IsValidEdgeStackName(payload.Name) {
return httperrors.NewInvalidPayloadError("Invalid stack name. Stack name must only consist of lowercase alpha characters, numbers, hyphens, or underscores as well as start with a lowercase character or number")
}
composeFileContent, _, err := request.RetrieveMultiPartFormFile(r, "file")
if err != nil {
return httperrors.NewInvalidPayloadError("Invalid Compose file. Ensure that the Compose file is uploaded correctly")
}
payload.StackFileContent = composeFileContent
var edgeGroups []portainer.EdgeGroupID
err = request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, false)
if err != nil || len(edgeGroups) == 0 {
return httperrors.NewInvalidPayloadError("Edge Groups are mandatory for an Edge stack")
}
payload.EdgeGroups = edgeGroups
deploymentType, err := request.RetrieveNumericMultiPartFormValue(r, "DeploymentType", false)
if err != nil {
return httperrors.NewInvalidPayloadError("Invalid deployment type")
}
payload.DeploymentType = portainer.EdgeStackDeploymentType(deploymentType)
if payload.DeploymentType != portainer.EdgeStackDeploymentCompose && payload.DeploymentType != portainer.EdgeStackDeploymentKubernetes {
return httperrors.NewInvalidPayloadError("Invalid deployment type")
}
var registries []portainer.RegistryID
err = request.RetrieveMultiPartFormJSONValue(r, "Registries", &registries, true)
if err != nil {
return httperrors.NewInvalidPayloadError("Invalid registry type")
}
payload.Registries = registries
useManifestNamespaces, _ := request.RetrieveBooleanMultiPartFormValue(r, "UseManifestNamespaces", true)
payload.UseManifestNamespaces = useManifestNamespaces
return nil
}
// @id EdgeStackCreateFile
// @summary Create an EdgeStack from file
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @accept multipart/form-data
// @produce json
// @param Name formData string true "Name of the stack. it must only consist of lowercase alphanumeric characters, hyphens, or underscores as well as start with a letter or number"
// @param file formData file true "Content of the Stack file"
// @param EdgeGroups formData string true "JSON stringified array of Edge Groups ids"
// @param DeploymentType formData int true "deploy type 0 - 'compose', 1 - 'kubernetes'"
// @param Registries formData string false "JSON stringified array of Registry ids to use for this stack"
// @param UseManifestNamespaces formData bool false "Uses the manifest's namespaces instead of the default one, relevant only for kube environments"
// @param PrePullImage formData bool false "Pre Pull image"
// @param RetryDeploy formData bool false "Retry deploy"
// @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
// @success 200 {object} portainer.EdgeStack
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/create/file [post]
func (handler *Handler) createEdgeStackFromFileUpload(r *http.Request, tx dataservices.DataStoreTx, tokenData *portainer.TokenData, dryrun bool) (*portainer.EdgeStack, error) {
payload := &edgeStackFromFileUploadPayload{}
if err := payload.Validate(r); err != nil {
return nil, err
}
stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
if err != nil {
return nil, errors.Wrap(err, "failed to create edge stack object")
}
if dryrun {
return stack, nil
}
if err := stackutils.ValidateEdgeStackComposeContent(r.Context(), payload.DeploymentType, payload.StackFileContent); err != nil {
return nil, httperrors.NewInvalidPayloadError(err.Error())
}
stack.CreatedByUserId = fmt.Sprintf("%d", tokenData.ID)
stack.CreatedBy = stackutils.SanitizeLabel(tokenData.Username)
return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeFileContent(tx, stackFolder, payload.DeploymentType, relatedEndpointIds, payload.StackFileContent)
})
}
@@ -0,0 +1,208 @@
package edgestacks
import (
"context"
"fmt"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/sources"
"github.com/portainer/portainer/api/gitops/workflows"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/portainer/portainer/pkg/edge"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/portainer/portainer/pkg/validate"
"github.com/pkg/errors"
)
type edgeStackFromGitRepositoryPayload struct {
// Name of the stack
// Max length: 255
// Name must only contains lowercase characters, numbers, hyphens, or underscores
// Name must start with a lowercase character or number
// Example: stack-name or stack_123 or stackName
Name string `example:"stack-name" validate:"required"`
// SourceID references an existing Source for git credentials/URL.
// When set, the inline URL and authentication fields are ignored.
SourceID portainer.SourceID `example:"1"`
// Deprecated: Use SourceID instead. URL of a Git repository hosting the Stack file.
RepositoryURL string `example:"https://github.com/openfaas/faas"`
// Reference name of a Git repository hosting the Stack file
RepositoryReferenceName string `example:"refs/heads/master"`
// Deprecated: Use SourceID instead. Use basic authentication to clone the Git repository.
RepositoryAuthentication bool `example:"true"`
// Deprecated: Use SourceID instead. Username used in basic authentication.
RepositoryUsername string `example:"myGitUsername"`
// Deprecated: Use SourceID instead. Password used in basic authentication.
RepositoryPassword string `example:"myGitPassword"`
// Path to the Stack file inside the Git repository
FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
// List of identifiers of EdgeGroups
EdgeGroups []portainer.EdgeGroupID `example:"1" validate:"required"`
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes'
// compose is enabled only for docker environments
// kubernetes is enabled only for kubernetes environments
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
// List of Registries to use for this stack
Registries []portainer.RegistryID
// Uses the manifest's namespaces instead of the default one
UseManifestNamespaces bool
// Deprecated: Use SourceID instead. TLSSkipVerify skips SSL verification when cloning the Git repository.
TLSSkipVerify bool `example:"false"`
}
func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) error {
if len(payload.Name) == 0 {
return httperrors.NewInvalidPayloadError("Invalid stack name")
}
if !edge.IsValidEdgeStackName(payload.Name) {
return httperrors.NewInvalidPayloadError("Invalid stack name. Stack name must only consist of lowercase alpha characters, numbers, hyphens, or underscores as well as start with a lowercase character or number")
}
if payload.SourceID == 0 {
if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
}
if payload.RepositoryAuthentication && len(payload.RepositoryPassword) == 0 {
return httperrors.NewInvalidPayloadError("Invalid repository credentials. Password must be specified when authentication is enabled")
}
}
if payload.DeploymentType != portainer.EdgeStackDeploymentCompose && payload.DeploymentType != portainer.EdgeStackDeploymentKubernetes {
return httperrors.NewInvalidPayloadError("Invalid deployment type")
}
if len(payload.FilePathInRepository) == 0 {
switch payload.DeploymentType {
case portainer.EdgeStackDeploymentCompose:
payload.FilePathInRepository = filesystem.ComposeFileDefaultName
case portainer.EdgeStackDeploymentKubernetes:
payload.FilePathInRepository = filesystem.ManifestFileDefaultName
}
}
if len(payload.EdgeGroups) == 0 {
return httperrors.NewInvalidPayloadError("Invalid edge groups. At least one edge group must be specified")
}
return nil
}
// @id EdgeStackCreateRepository
// @summary Create an EdgeStack from a git repository
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param body body edgeStackFromGitRepositoryPayload true "stack config"
// @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
// @success 200 {object} portainer.EdgeStack
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/create/repository [post]
func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, tx dataservices.DataStoreTx, tokenData *portainer.TokenData, dryrun bool) (*portainer.EdgeStack, error) {
var payload edgeStackFromGitRepositoryPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return nil, err
}
stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
if err != nil {
return nil, errors.Wrap(err, "failed to create edge stack object")
}
if dryrun {
return stack, nil
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve user info from request context", err)
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
repoConfig, httpErr := sources.ResolveRepoConfig(tx, userContext, sources.RepoConfigInput{
SourceID: payload.SourceID,
ReferenceName: payload.RepositoryReferenceName,
ConfigFilePath: payload.FilePathInRepository,
RepositoryURL: payload.RepositoryURL,
TLSSkipVerify: payload.TLSSkipVerify,
RepositoryAuthentication: payload.RepositoryAuthentication,
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
})
if httpErr != nil {
return nil, httpErr
}
if err := ssrf.CheckURL(r.Context(), repoConfig.URL); err != nil {
return nil, errors.Wrap(err, "repository URL blocked by SSRF policy")
}
stack.CreatedByUserId = fmt.Sprintf("%d", tokenData.ID)
stack.CreatedBy = stackutils.SanitizeLabel(tokenData.Username)
return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeManifestFromGitRepository(context.TODO(), tx, userContext, payload.SourceID, stackFolder, relatedEndpointIds, payload.DeploymentType, tokenData.ID, repoConfig)
})
}
func (handler *Handler) storeManifestFromGitRepository(ctx context.Context, tx dataservices.DataStoreTx, userContext source.UserContext, sourceID portainer.SourceID, stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) {
if hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType); err != nil {
return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err)
} else if hasWrongType {
return "", "", "", errors.New("edge stack with config do not match the environment type")
}
projectPath = handler.FileService.GetEdgeStackProjectPath(stackFolder)
repositoryUsername := ""
repositoryPassword := ""
if repositoryConfig.Authentication != nil && repositoryConfig.Authentication.Password != "" {
repositoryUsername = repositoryConfig.Authentication.Username
repositoryPassword = repositoryConfig.Authentication.Password
}
if err := handler.GitService.CloneRepository(
ctx,
projectPath,
repositoryConfig.URL,
repositoryConfig.ReferenceName,
repositoryUsername,
repositoryPassword,
repositoryConfig.TLSSkipVerify,
); err != nil {
if statusErr := workflows.SaveSourceStatus(tx, userContext, sourceID, err); statusErr != nil {
return "", "", "", fmt.Errorf("%w (and failed to persist status: %w)", err, statusErr)
}
return "", "", "", err
}
if err := workflows.SaveSourceStatus(tx, userContext, sourceID, nil); err != nil {
return "", "", "", fmt.Errorf("failed to persist source sync status: %w", err)
}
if deploymentType == portainer.EdgeStackDeploymentCompose {
return repositoryConfig.ConfigFilePath, "", projectPath, nil
}
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
return "", repositoryConfig.ConfigFilePath, projectPath, nil
}
errMessage := fmt.Sprintf("unknown deployment type: %d", deploymentType)
return "", "", "", httperrors.NewInvalidPayloadError(errMessage)
}
@@ -0,0 +1,136 @@
package edgestacks
import (
"fmt"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/portainer/portainer/pkg/edge"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/pkg/errors"
)
type edgeStackFromStringPayload struct {
// Name of the stack
// Max length: 255
// Name must only contains lowercase characters, numbers, hyphens, or underscores
// Name must start with a lowercase character or number
// Example: stack-name or stack_123 or stackName
Name string `example:"stack-name" validate:"required"`
// Content of the Stack file
StackFileContent string `example:"version: 3\n services:\n web:\n image:nginx" validate:"required"`
// List of identifiers of EdgeGroups
EdgeGroups []portainer.EdgeGroupID `example:"1"`
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes'
// compose is enabled only for docker environments
// kubernetes is enabled only for kubernetes environments
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
// List of Registries to use for this stack
Registries []portainer.RegistryID
// Uses the manifest's namespaces instead of the default one
UseManifestNamespaces bool
}
func (payload *edgeStackFromStringPayload) Validate(r *http.Request) error {
if len(payload.Name) == 0 {
return httperrors.NewInvalidPayloadError("Invalid stack name")
}
if !edge.IsValidEdgeStackName(payload.Name) {
return httperrors.NewInvalidPayloadError("Invalid stack name. Stack name must only consist of lowercase alpha characters, numbers, hyphens, or underscores as well as start with a lowercase character or number")
}
if len(payload.StackFileContent) == 0 {
return httperrors.NewInvalidPayloadError("Invalid stack file content")
}
if len(payload.EdgeGroups) == 0 {
return httperrors.NewInvalidPayloadError("Edge Groups are mandatory for an Edge stack")
}
if payload.DeploymentType != portainer.EdgeStackDeploymentCompose && payload.DeploymentType != portainer.EdgeStackDeploymentKubernetes {
return httperrors.NewInvalidPayloadError("Invalid deployment type")
}
return nil
}
// @id EdgeStackCreateString
// @summary Create an EdgeStack from a text
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param body body edgeStackFromStringPayload true "stack config"
// @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
// @success 200 {object} portainer.EdgeStack
// @failure 400 "Bad request"
// @failure 500 "Internal server error"
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/create/string [post]
func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, tx dataservices.DataStoreTx, tokenData *portainer.TokenData, dryrun bool) (*portainer.EdgeStack, error) {
var payload edgeStackFromStringPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return nil, err
}
stack, err := handler.edgeStacksService.BuildEdgeStack(tx, payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries, payload.UseManifestNamespaces)
if err != nil {
return nil, errors.Wrap(err, "failed to create Edge stack object")
}
stack.CreatedByUserId = fmt.Sprintf("%d", tokenData.ID)
stack.CreatedBy = stackutils.SanitizeLabel(tokenData.Username)
if dryrun {
return stack, nil
}
if err := stackutils.ValidateEdgeStackComposeContent(r.Context(), payload.DeploymentType, []byte(payload.StackFileContent)); err != nil {
return nil, httperrors.NewInvalidPayloadError(err.Error())
}
return handler.edgeStacksService.PersistEdgeStack(tx, stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeFileContent(tx, stackFolder, payload.DeploymentType, relatedEndpointIds, []byte(payload.StackFileContent))
})
}
func (handler *Handler) storeFileContent(tx dataservices.DataStoreTx, stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) {
if hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType); err != nil {
return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err)
} else if hasWrongType {
return "", "", "", errors.New("edge stack with config do not match the environment type")
}
if deploymentType == portainer.EdgeStackDeploymentCompose {
composePath = filesystem.ComposeFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, composePath, fileContent)
if err != nil {
return "", "", "", err
}
return composePath, "", projectPath, nil
}
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
manifestPath = filesystem.ManifestFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, manifestPath, fileContent)
if err != nil {
return "", "", "", err
}
return "", manifestPath, projectPath, nil
}
errMessage := fmt.Sprintf("invalid deployment type: %d", deploymentType)
return "", "", "", httperrors.NewInvalidPayloadError(errMessage)
}
@@ -0,0 +1,216 @@
package edgestacks
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// Create
func TestCreateAndInspect(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
// Create Endpoint, EdgeGroup and EndpointRelation
endpoint := createEndpoint(t, handler.DataStore)
edgeGroup := portainer.EdgeGroup{
ID: 1,
Name: "EdgeGroup 1",
Dynamic: false,
TagIDs: nil,
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpoint.ID}),
PartialMatch: false,
}
err := handler.DataStore.EdgeGroup().Create(&edgeGroup)
require.NoError(t, err)
endpointRelation := portainer.EndpointRelation{
EndpointID: endpoint.ID,
EdgeStacks: map[portainer.EdgeStackID]bool{},
}
err = handler.DataStore.EndpointRelation().Create(&endpointRelation)
require.NoError(t, err)
payload := edgeStackFromStringPayload{
Name: "test-stack",
StackFileContent: "stack content",
EdgeGroups: []portainer.EdgeGroupID{1},
DeploymentType: portainer.EdgeStackDeploymentCompose,
}
jsonPayload, err := json.Marshal(payload)
require.NoError(t, err)
r := bytes.NewBuffer(jsonPayload)
// Create EdgeStack
req, err := http.NewRequest(http.MethodPost, "/edge_stacks/create/string", r)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
data := portainer.EdgeStack{}
err = json.NewDecoder(rec.Body).Decode(&data)
require.NoError(t, err)
// Inspect
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", data.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
data = portainer.EdgeStack{}
err = json.NewDecoder(rec.Body).Decode(&data)
require.NoError(t, err)
if payload.Name != data.Name {
t.Fatalf("expected EdgeStack Name %s, found %s", payload.Name, data.Name)
}
}
func TestCreateWithInvalidPayload(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
cases := []struct {
Name string
Payload any
ExpectedStatusCode int
Method string
}{
{
Name: "Invalid method parameter",
Payload: edgeStackFromStringPayload{},
Method: "invalid",
ExpectedStatusCode: 400,
},
{
Name: "Empty edgeStackFromStringPayload with string method",
Payload: edgeStackFromStringPayload{},
Method: "string",
ExpectedStatusCode: 400,
},
{
Name: "Empty edgeStackFromStringPayload with repository method",
Payload: edgeStackFromStringPayload{},
Method: "repository",
ExpectedStatusCode: 400,
},
{
Name: "Empty edgeStackFromStringPayload with file method",
Payload: edgeStackFromStringPayload{},
Method: "file",
ExpectedStatusCode: 400,
},
{
Name: "Duplicated EdgeStack Name",
Payload: edgeStackFromStringPayload{
Name: edgeStack.Name,
StackFileContent: "content",
EdgeGroups: edgeStack.EdgeGroups,
DeploymentType: edgeStack.DeploymentType,
},
Method: "string",
ExpectedStatusCode: http.StatusConflict,
},
{
Name: "Empty EdgeStack Groups",
Payload: edgeStackFromStringPayload{
Name: edgeStack.Name,
StackFileContent: "content",
EdgeGroups: []portainer.EdgeGroupID{},
DeploymentType: edgeStack.DeploymentType,
},
Method: "string",
ExpectedStatusCode: 400,
},
{
Name: "EdgeStackDeploymentKubernetes with Docker endpoint",
Payload: edgeStackFromStringPayload{
Name: "stack-name",
StackFileContent: "content",
EdgeGroups: []portainer.EdgeGroupID{1},
DeploymentType: portainer.EdgeStackDeploymentKubernetes,
},
Method: "string",
ExpectedStatusCode: 500,
},
{
Name: "Empty Stack File Content",
Payload: edgeStackFromStringPayload{
Name: "stack-name",
StackFileContent: "",
EdgeGroups: []portainer.EdgeGroupID{1},
DeploymentType: portainer.EdgeStackDeploymentCompose,
},
Method: "string",
ExpectedStatusCode: 400,
},
{
Name: "Clone Git repository error",
Payload: edgeStackFromGitRepositoryPayload{
Name: "stack-name",
RepositoryURL: "github.com/portainer/portainer",
RepositoryReferenceName: "ref name",
RepositoryAuthentication: false,
RepositoryUsername: "",
RepositoryPassword: "",
FilePathInRepository: "/file/path",
EdgeGroups: []portainer.EdgeGroupID{1},
DeploymentType: portainer.EdgeStackDeploymentCompose,
},
Method: "repository",
ExpectedStatusCode: 500,
},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
jsonPayload, err := json.Marshal(tc.Payload)
if err != nil {
t.Fatal("JSON marshal error:", err)
}
r := bytes.NewBuffer(jsonPayload)
// Create EdgeStack
req, err := http.NewRequest(http.MethodPost, "/edge_stacks/create/"+tc.Method, r)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
@@ -0,0 +1,57 @@
package edgestacks
import (
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeStackDelete
// @summary Delete an EdgeStack
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EdgeStack Id"
// @success 204
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/{id} [delete]
func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid edge stack identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.deleteEdgeStack(tx, portainer.EdgeStackID(edgeStackID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) deleteEdgeStack(tx dataservices.DataStoreTx, edgeStackID portainer.EdgeStackID) error {
edgeStack, err := tx.EdgeStack().EdgeStack(edgeStackID)
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an edge stack with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", err)
}
if err := handler.edgeStacksService.DeleteEdgeStack(tx, edgeStack.ID, edgeStack.EdgeGroups); err != nil {
return httperror.InternalServerError("Unable to delete edge stack", err)
}
stackFolder := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(edgeStack.ID)))
if err := handler.FileService.RemoveDirectory(stackFolder); err != nil {
return httperror.InternalServerError("Unable to remove edge stack project folder", err)
}
return nil
}
@@ -0,0 +1,146 @@
package edgestacks
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Delete
func TestDeleteAndInspect(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
// Create
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Inspect
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
data := portainer.EdgeStack{}
err = json.NewDecoder(rec.Body).Decode(&data)
require.NoError(t, err)
if data.ID != edgeStack.ID {
t.Fatalf("expected EdgeStackID %d, found %d", int(edgeStack.ID), data.ID)
}
// Delete
req, err = http.NewRequest(http.MethodDelete, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("expected a %d response, found: %d", http.StatusNoContent, rec.Code)
}
// Inspect
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("expected a %d response, found: %d", http.StatusNotFound, rec.Code)
}
}
func TestDeleteInvalidEdgeStack(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
cases := []struct {
Name string
URL string
ExpectedStatusCode int
}{
{Name: "Non-existing EdgeStackID", URL: "/edge_stacks/-1", ExpectedStatusCode: http.StatusNotFound},
{Name: "Invalid EdgeStackID", URL: "/edge_stacks/aaaaaaa", ExpectedStatusCode: http.StatusBadRequest},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
req, err := http.NewRequest(http.MethodDelete, tc.URL, nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
func TestDeleteEdgeStack_RemoveProjectFolder(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
edgeGroup := createEdgeGroup(t, handler.DataStore)
payload := edgeStackFromStringPayload{
Name: "test-stack",
DeploymentType: portainer.EdgeStackDeploymentCompose,
EdgeGroups: []portainer.EdgeGroupID{edgeGroup.ID},
StackFileContent: "version: '3.7'\nservices:\n test:\n image: test",
}
var buf bytes.Buffer
err := json.NewEncoder(&buf).Encode(payload)
require.NoError(t, err)
// Create
req, err := http.NewRequest(http.MethodPost, "/edge_stacks/create/string", &buf)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusNoContent, rec.Code)
}
assert.DirExists(t, handler.FileService.GetEdgeStackProjectPath("1"))
// Delete
req, err = http.NewRequest(http.MethodDelete, "/edge_stacks/1", nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("expected a %d response, found: %d", http.StatusNoContent, rec.Code)
}
assert.NoDirExists(t, handler.FileService.GetEdgeStackProjectPath("1"))
}
@@ -0,0 +1,51 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type stackFileResponse struct {
StackFileContent string `json:"StackFileContent"`
}
// @id EdgeStackFile
// @summary Fetches the stack file for an EdgeStack
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeStack Id"
// @success 200 {object} stackFileResponse
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/{id}/file [get]
func (handler *Handler) edgeStackFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid edge stack identifier route variable", err)
}
stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
if err != nil {
return handlerDBErr(err, "Unable to find an edge stack with the specified identifier inside the database")
}
fileName := stack.EntryPoint
if stack.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
fileName = stack.ManifestPath
}
stackFileContent, err := handler.FileService.GetFileContent(stack.ProjectPath, fileName)
if err != nil {
return httperror.InternalServerError("Unable to retrieve stack file from disk", err)
}
return response.JSON(w, &stackFileResponse{StackFileContent: string(stackFileContent)})
}
@@ -0,0 +1,68 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EdgeStackInspect
// @summary Inspect an EdgeStack
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "EdgeStack Id"
// @success 200 {object} portainer.EdgeStack
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/{id} [get]
func (handler *Handler) edgeStackInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid edge stack identifier route variable", err)
}
edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
if err != nil {
return handlerDBErr(err, "Unable to find an edge stack with the specified identifier inside the database")
}
if err := fillEdgeStackStatus(handler.DataStore, edgeStack); err != nil {
return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
}
return response.JSON(w, edgeStack)
}
func fillEdgeStackStatus(tx dataservices.DataStoreTx, edgeStack *portainer.EdgeStack) error {
status, err := tx.EdgeStackStatus().ReadAll(edgeStack.ID)
if err != nil {
return err
}
edgeStack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus, len(status))
emptyStatus := make([]portainer.EdgeStackDeploymentStatus, 0)
for _, s := range status {
if s.Status == nil {
s.Status = emptyStatus
}
edgeStack.Status[s.EndpointID] = portainer.EdgeStackStatus{
Status: s.Status,
EndpointID: s.EndpointID,
DeploymentInfo: s.DeploymentInfo,
ReadyRePullImage: s.ReadyRePullImage,
}
}
return nil
}
@@ -0,0 +1,39 @@
package edgestacks
import (
"net/http"
"net/http/httptest"
"testing"
)
// Inspect
func TestInspectInvalidEdgeID(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
cases := []struct {
Name string
EdgeStackID string
ExpectedStatusCode int
}{
{"Invalid EdgeStackID", "x", 400},
{"Non-existing EdgeStackID", "5", 404},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, "/edge_stacks/"+tc.EdgeStackID, nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
@@ -0,0 +1,164 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type aggregatedStatusesMap map[portainer.EdgeStackStatusType]int
type SummarizedStatus string
const (
sumStatusUnavailable SummarizedStatus = "Unavailable"
sumStatusDeploying SummarizedStatus = "Deploying"
sumStatusFailed SummarizedStatus = "Failed"
sumStatusPaused SummarizedStatus = "Paused"
sumStatusPartiallyRunning SummarizedStatus = "PartiallyRunning"
sumStatusCompleted SummarizedStatus = "Completed"
sumStatusRunning SummarizedStatus = "Running"
)
type edgeStackStatusSummary struct {
AggregatedStatus aggregatedStatusesMap
Status SummarizedStatus
Reason string
}
type edgeStackListResponseItem struct {
portainer.EdgeStack
StatusSummary edgeStackStatusSummary
}
// @id EdgeStackList
// @summary Fetches the list of EdgeStacks
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param summarizeStatuses query boolean false "will summarize the statuses"
// @success 200 {array} portainer.EdgeStack
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks [get]
func (handler *Handler) edgeStackList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
summarizeStatuses, _ := request.RetrieveBooleanQueryParameter(r, "summarizeStatuses", true)
edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
if err != nil {
return httperror.InternalServerError("Unable to retrieve edge stacks from the database", err)
}
res := make([]edgeStackListResponseItem, len(edgeStacks))
for i := range edgeStacks {
res[i].EdgeStack = edgeStacks[i]
if summarizeStatuses {
if err := fillStatusSummary(handler.DataStore, &res[i]); err != nil {
return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
}
} else if err := fillEdgeStackStatus(handler.DataStore, &res[i].EdgeStack); err != nil {
return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
}
}
return response.JSON(w, res)
}
func fillStatusSummary(tx dataservices.DataStoreTx, edgeStack *edgeStackListResponseItem) error {
statuses, err := tx.EdgeStackStatus().ReadAll(edgeStack.ID)
if err != nil {
return err
}
aggregated := make(aggregatedStatusesMap)
for _, envStatus := range statuses {
for _, status := range envStatus.Status {
aggregated[status.Type]++
}
}
status, reason := SummarizeStatuses(statuses, edgeStack.NumDeployments)
edgeStack.StatusSummary = edgeStackStatusSummary{
AggregatedStatus: aggregated,
Status: status,
Reason: reason,
}
edgeStack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
return nil
}
func SummarizeStatuses(statuses []portainer.EdgeStackStatusForEnv, numDeployments int) (SummarizedStatus, string) {
if numDeployments == 0 {
return sumStatusUnavailable, "Your edge stack is currently unavailable due to the absence of an available environment in your edge group"
}
allStatuses := slicesx.FlatMap(statuses, func(x portainer.EdgeStackStatusForEnv) []portainer.EdgeStackDeploymentStatus {
return x.Status
})
lastStatuses := slicesx.Map(
slicesx.Filter(
statuses,
func(s portainer.EdgeStackStatusForEnv) bool {
return len(s.Status) > 0
},
),
func(x portainer.EdgeStackStatusForEnv) portainer.EdgeStackDeploymentStatus {
return x.Status[len(x.Status)-1]
},
)
if len(lastStatuses) == 0 {
return sumStatusDeploying, ""
}
if allFailed := slicesx.Every(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool {
return s.Type == portainer.EdgeStackStatusError
}); allFailed {
return sumStatusFailed, ""
}
if hasPaused := slicesx.Some(allStatuses, func(s portainer.EdgeStackDeploymentStatus) bool {
return s.Type == portainer.EdgeStackStatusPausedDeploying
}); hasPaused {
return sumStatusPaused, ""
}
if len(lastStatuses) < numDeployments {
return sumStatusDeploying, ""
}
hasDeploying := slicesx.Some(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool { return s.Type == portainer.EdgeStackStatusDeploying })
hasRunning := slicesx.Some(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool { return s.Type == portainer.EdgeStackStatusRunning })
hasFailed := slicesx.Some(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool { return s.Type == portainer.EdgeStackStatusError })
if hasRunning && hasFailed && !hasDeploying {
return sumStatusPartiallyRunning, ""
}
if allCompleted := slicesx.Every(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool { return s.Type == portainer.EdgeStackStatusCompleted }); allCompleted {
return sumStatusCompleted, ""
}
if allRunning := slicesx.Every(lastStatuses, func(s portainer.EdgeStackDeploymentStatus) bool {
return s.Type == portainer.EdgeStackStatusRunning
}); allRunning {
return sumStatusRunning, ""
}
return sumStatusDeploying, ""
}
@@ -0,0 +1,147 @@
package edgestacks
import (
"errors"
"fmt"
"net/http"
"slices"
"strconv"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type updateStatusPayload struct {
Error string
Status *portainer.EdgeStackStatusType
EndpointID portainer.EndpointID
Time int64
Version int
}
func (payload *updateStatusPayload) Validate(r *http.Request) error {
if payload.Status == nil {
return errors.New("invalid status")
}
if payload.EndpointID == 0 {
return errors.New("invalid EnvironmentID")
}
if *payload.Status == portainer.EdgeStackStatusError && len(payload.Error) == 0 {
return errors.New("error message is mandatory when status is error")
}
if payload.Time == 0 {
payload.Time = time.Now().Unix()
}
return nil
}
// @id EdgeStackStatusUpdate
// @summary Update an EdgeStack status
// @description Authorized only if the request is done by an Edge Environment(Endpoint)
// @tags edge_stacks
// @accept json
// @produce json
// @param id path int true "EdgeStack Id"
// @param body body updateStatusPayload true "EdgeStack status payload"
// @success 200 {object} portainer.EdgeStack
// @failure 500
// @failure 400
// @failure 404
// @failure 403
// @router /edge_stacks/{id}/status [put]
func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid stack identifier route variable", err)
}
var payload updateStatusPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", fmt.Errorf("edge polling error: %w. Environment ID: %d", err, payload.EndpointID))
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(payload.EndpointID)
if err != nil {
return handlerDBErr(fmt.Errorf("unable to find the environment from the database: %w. Environment ID: %d", err, payload.EndpointID), "unable to find the environment")
}
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment ID: %d", err, payload.EndpointID))
}
var stack *portainer.EdgeStack
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
var err error
stack, err = tx.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
if err != nil {
if dataservices.IsErrObjectNotFound(err) {
return nil
}
return httperror.InternalServerError("Unable to retrieve Edge stack from the database", err)
}
if err := handler.updateEdgeStackStatus(tx, stack, stack.ID, payload); err != nil {
return httperror.InternalServerError("Unable to update Edge stack status", err)
}
return nil
}); err != nil {
return response.TxErrorResponse(err)
}
if ok, _ := strconv.ParseBool(r.Header.Get("X-Portainer-No-Body")); ok {
return nil
}
if err := fillEdgeStackStatus(handler.DataStore, stack); err != nil {
return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
}
return response.JSON(w, stack)
}
func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, stack *portainer.EdgeStack, stackID portainer.EdgeStackID, payload updateStatusPayload) error {
if payload.Version > 0 && payload.Version < stack.Version {
return nil
}
status := *payload.Status
deploymentStatus := portainer.EdgeStackDeploymentStatus{
Type: status,
Error: payload.Error,
Time: payload.Time,
}
if deploymentStatus.Type == portainer.EdgeStackStatusRemoved {
return tx.EdgeStackStatus().Delete(stackID, payload.EndpointID)
}
environmentStatus, err := tx.EdgeStackStatus().Read(stackID, payload.EndpointID)
if err != nil && !tx.IsErrObjectNotFound(err) {
return err
} else if tx.IsErrObjectNotFound(err) {
environmentStatus = &portainer.EdgeStackStatusForEnv{
EndpointID: payload.EndpointID,
Status: []portainer.EdgeStackDeploymentStatus{},
}
}
if containsStatus := slices.ContainsFunc(environmentStatus.Status, func(e portainer.EdgeStackDeploymentStatus) bool {
return e.Type == deploymentStatus.Type
}); !containsStatus {
environmentStatus.Status = append(environmentStatus.Status, deploymentStatus)
}
return tx.EdgeStackStatus().Update(stackID, payload.EndpointID, environmentStatus)
}
@@ -0,0 +1,147 @@
package edgestacks
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// Update Status
func TestUpdateStatusAndInspect(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Update edge stack status
newStatus := portainer.EdgeStackStatusError
payload := updateStatusPayload{
Error: "test-error",
Status: &newStatus,
EndpointID: endpoint.ID,
}
jsonPayload, err := json.Marshal(payload)
require.NoError(t, err)
r := bytes.NewBuffer(jsonPayload)
req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d/status", edgeStack.ID), r)
require.NoError(t, err)
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, endpoint.EdgeID)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
// Get updated edge stack
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
updatedStack := portainer.EdgeStack{}
err = json.NewDecoder(rec.Body).Decode(&updatedStack)
require.NoError(t, err)
endpointStatus, ok := updatedStack.Status[payload.EndpointID]
require.True(t, ok)
lastStatus := endpointStatus.Status[len(endpointStatus.Status)-1]
if len(endpointStatus.Status) == len(edgeStack.Status[payload.EndpointID].Status) {
t.Fatal("expected status array to be updated")
}
if lastStatus.Type != *payload.Status {
t.Fatalf("expected EdgeStackStatusType %d, found %d", *payload.Status, lastStatus.Type)
}
if endpointStatus.EndpointID != payload.EndpointID {
t.Fatalf("expected EndpointID %d, found %d", payload.EndpointID, endpointStatus.EndpointID)
}
}
func TestUpdateStatusWithInvalidPayload(t *testing.T) {
t.Parallel()
handler, _ := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Update edge stack status
statusError := portainer.EdgeStackStatusError
statusOk := portainer.EdgeStackStatusDeploymentReceived
cases := []struct {
Name string
Payload updateStatusPayload
ExpectedErrorMessage string
ExpectedStatusCode int
}{
{
"Update with nil Status",
updateStatusPayload{
Error: "test-error",
Status: nil,
EndpointID: endpoint.ID,
},
"Invalid status",
400,
},
{
"Update with error status and empty error message",
updateStatusPayload{
Error: "",
Status: &statusError,
EndpointID: endpoint.ID,
},
"Error message is mandatory when status is error",
400,
},
{
"Update with missing EndpointID",
updateStatusPayload{
Error: "",
Status: &statusOk,
EndpointID: 0,
},
"Invalid EnvironmentID",
400,
},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
jsonPayload, err := json.Marshal(tc.Payload)
require.NoError(t, err)
r := bytes.NewBuffer(jsonPayload)
req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d/status", edgeStack.ID), r)
require.NoError(t, err)
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, endpoint.EdgeID)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
@@ -0,0 +1,147 @@
package edgestacks
import (
"strconv"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/apikey"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/edge/edgestacks"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/roar"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
// Helpers
func setupHandler(t *testing.T) (*Handler, string) {
t.Helper()
_, store := datastore.MustNewTestStore(t, true, true)
jwtService, err := jwt.NewService("1h", store)
if err != nil {
t.Fatal(err)
}
user := &portainer.User{ID: 2, Username: "admin", Role: portainer.AdministratorRole}
if err := store.User().Create(user); err != nil {
t.Fatal(err)
}
apiKeyService := apikey.NewAPIKeyService(store.APIKeyRepository(), store.User())
rawAPIKey, _, err := apiKeyService.GenerateApiKey(*user, "test")
if err != nil {
t.Fatal(err)
}
fs, err := filesystem.NewService(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
handler := NewHandler(
security.NewRequestBouncer(t.Context(), store, jwtService, apiKeyService),
store,
edgestacks.NewService(store),
)
handler.FileService = fs
settings, err := handler.DataStore.Settings().Settings()
require.NoError(t, err)
settings.EnableEdgeComputeFeatures = true
err = handler.DataStore.Settings().UpdateSettings(settings)
require.NoError(t, err)
handler.GitService = testhelpers.NewGitService(errors.New("Clone error"), "git-service-id")
return handler, rawAPIKey
}
func createEndpointWithId(t *testing.T, store dataservices.DataStore, endpointID portainer.EndpointID) portainer.Endpoint {
t.Helper()
endpoint := portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint-" + strconv.Itoa(int(endpointID)),
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
LastCheckInDate: time.Now().Unix(),
}
err := store.Endpoint().Create(&endpoint)
require.NoError(t, err)
return endpoint
}
func createEndpoint(t *testing.T, store dataservices.DataStore) portainer.Endpoint {
return createEndpointWithId(t, store, 5)
}
func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID portainer.EndpointID) portainer.EdgeStack {
t.Helper()
edgeGroup := portainer.EdgeGroup{
ID: 1,
Name: "EdgeGroup 1",
Dynamic: false,
TagIDs: nil,
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpointID}),
PartialMatch: false,
}
err := store.EdgeGroup().Create(&edgeGroup)
require.NoError(t, err)
edgeStackID := portainer.EdgeStackID(14)
edgeStack := portainer.EdgeStack{
ID: edgeStackID,
Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)),
CreationDate: time.Now().Unix(),
EdgeGroups: []portainer.EdgeGroupID{edgeGroup.ID},
ProjectPath: "/project/path",
EntryPoint: "entrypoint",
Version: 237,
ManifestPath: "/manifest/path",
DeploymentType: portainer.EdgeStackDeploymentKubernetes,
}
endpointRelation := portainer.EndpointRelation{
EndpointID: endpointID,
EdgeStacks: map[portainer.EdgeStackID]bool{
edgeStack.ID: true,
},
}
err = store.EdgeStack().Create(edgeStack.ID, &edgeStack)
require.NoError(t, err)
err = store.EndpointRelation().Create(&endpointRelation)
require.NoError(t, err)
return edgeStack
}
func createEdgeGroup(t *testing.T, store dataservices.DataStore) portainer.EdgeGroup {
edgeGroup := portainer.EdgeGroup{
ID: 1,
Name: "EdgeGroup 1",
}
err := store.EdgeGroup().Create(&edgeGroup)
require.NoError(t, err)
return edgeGroup
}
@@ -0,0 +1,164 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/set"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
)
type updateEdgeStackPayload struct {
StackFileContent string
UpdateVersion bool
EdgeGroups []portainer.EdgeGroupID
DeploymentType portainer.EdgeStackDeploymentType
// Uses the manifest's namespaces instead of the default one
UseManifestNamespaces bool
}
func (payload *updateEdgeStackPayload) Validate(r *http.Request) error {
if payload.StackFileContent == "" {
return errors.New("invalid stack file content")
}
if len(payload.EdgeGroups) == 0 {
return errors.New("edge Groups are mandatory for an Edge stack")
}
return nil
}
// @id EdgeStackUpdate
// @summary Update an EdgeStack
// @description **Access policy**: administrator
// @tags edge_stacks
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "EdgeStack Id"
// @param body body updateEdgeStackPayload true "EdgeStack data"
// @success 200 {object} portainer.EdgeStack
// @failure 500
// @failure 400
// @failure 503 "Edge compute features are disabled"
// @router /edge_stacks/{id} [put]
func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid stack identifier route variable", err)
}
var payload updateEdgeStackPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
if err := stackutils.ValidateEdgeStackComposeContent(r.Context(), payload.DeploymentType, []byte(payload.StackFileContent)); err != nil {
return httperror.BadRequest("Stack file contains a URL blocked by the SSRF policy", err)
}
var stack *portainer.EdgeStack
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack, err = handler.updateEdgeStack(tx, portainer.EdgeStackID(stackID), payload)
return err
}); err != nil {
return response.TxErrorResponse(err)
}
if err := fillEdgeStackStatus(handler.DataStore, stack); err != nil {
return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
}
return response.JSON(w, stack)
}
func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID portainer.EdgeStackID, payload updateEdgeStackPayload) (*portainer.EdgeStack, error) {
stack, err := tx.EdgeStack().EdgeStack(stackID)
if err != nil {
return nil, handlerDBErr(err, "Unable to find a stack with the specified identifier inside the database")
}
relationConfig, err := edge.FetchEndpointRelationsConfig(tx)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments relations config from database", err)
}
relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
}
groupsIds := stack.EdgeGroups
if payload.EdgeGroups != nil {
newRelated, _, err := handler.handleChangeEdgeGroups(tx, stack, payload.EdgeGroups, relatedEndpointIds, relationConfig)
if err != nil {
return nil, httperror.InternalServerError("Unable to handle edge groups change", err)
}
groupsIds = payload.EdgeGroups
relatedEndpointIds = newRelated
}
hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, payload.DeploymentType)
if err != nil {
return nil, httperror.InternalServerError("unable to check for existence of non fitting environments: %w", err)
}
if hasWrongType {
return nil, httperror.BadRequest("edge stack with config do not match the environment type", nil)
}
stack.NumDeployments = len(relatedEndpointIds)
stack.UseManifestNamespaces = payload.UseManifestNamespaces
stack.EdgeGroups = groupsIds
if payload.UpdateVersion {
if err := handler.updateStackVersion(tx, stack, payload.DeploymentType, []byte(payload.StackFileContent), "", relatedEndpointIds); err != nil {
return nil, httperror.InternalServerError("Unable to update stack version", err)
}
}
if err := tx.EdgeStack().UpdateEdgeStack(stack.ID, stack); err != nil {
return nil, httperror.InternalServerError("Unable to persist the stack changes inside the database", err)
}
return stack, nil
}
func (handler *Handler) handleChangeEdgeGroups(tx dataservices.DataStoreTx, edgeStack *portainer.EdgeStack, newEdgeGroupsIDs []portainer.EdgeGroupID, oldRelatedEnvironmentIDs []portainer.EndpointID, relationConfig *edge.EndpointRelationsConfig) ([]portainer.EndpointID, set.Set[portainer.EndpointID], error) {
newRelatedEnvironmentIDs, err := edge.EdgeStackRelatedEndpoints(newEdgeGroupsIDs, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
if err != nil {
return nil, nil, errors.WithMessage(err, "Unable to retrieve edge stack related environments from database")
}
oldRelatedEnvironmentsSet := set.ToSet(oldRelatedEnvironmentIDs)
newRelatedEnvironmentsSet := set.ToSet(newRelatedEnvironmentIDs)
relatedEnvironmentsToAdd := newRelatedEnvironmentsSet.Difference(oldRelatedEnvironmentsSet)
relatedEnvironmentsToRemove := oldRelatedEnvironmentsSet.Difference(newRelatedEnvironmentsSet)
if len(relatedEnvironmentsToRemove) > 0 {
if err := tx.EndpointRelation().RemoveEndpointRelationsForEdgeStack(relatedEnvironmentsToRemove.Keys(), edgeStack.ID); err != nil {
return nil, nil, errors.WithMessage(err, "Unable to remove edge stack relations from the database")
}
}
if len(relatedEnvironmentsToAdd) > 0 {
if err := tx.EndpointRelation().AddEndpointRelationsForEdgeStack(relatedEnvironmentsToAdd.Keys(), edgeStack); err != nil {
return nil, nil, errors.WithMessage(err, "Unable to add edge stack relations to the database")
}
}
return newRelatedEnvironmentIDs, relatedEnvironmentsToAdd, nil
}
@@ -0,0 +1,238 @@
package edgestacks
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// Update
func TestUpdateAndInspect(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
// Update edge stack: create new Endpoint, EndpointRelation and EdgeGroup
endpointID := portainer.EndpointID(6)
newEndpoint := createEndpointWithId(t, handler.DataStore, endpointID)
err := handler.DataStore.Endpoint().Create(&newEndpoint)
require.NoError(t, err)
endpointRelation := portainer.EndpointRelation{
EndpointID: endpointID,
EdgeStacks: map[portainer.EdgeStackID]bool{
edgeStack.ID: true,
},
}
err = handler.DataStore.EndpointRelation().Create(&endpointRelation)
require.NoError(t, err)
newEdgeGroup := portainer.EdgeGroup{
ID: 2,
Name: "EdgeGroup 2",
Dynamic: false,
TagIDs: nil,
EndpointIDs: roar.FromSlice([]portainer.EndpointID{newEndpoint.ID}),
PartialMatch: false,
}
err = handler.DataStore.EdgeGroup().Create(&newEdgeGroup)
require.NoError(t, err)
payload := updateEdgeStackPayload{
StackFileContent: "update-test",
UpdateVersion: true,
EdgeGroups: append(edgeStack.EdgeGroups, newEdgeGroup.ID),
DeploymentType: portainer.EdgeStackDeploymentCompose,
}
jsonPayload, err := json.Marshal(payload)
require.NoError(t, err)
r := bytes.NewBuffer(jsonPayload)
req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), r)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
// Get updated edge stack
req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
updatedStack := portainer.EdgeStack{}
err = json.NewDecoder(rec.Body).Decode(&updatedStack)
require.NoError(t, err)
if payload.UpdateVersion && updatedStack.Version != edgeStack.Version+1 {
t.Fatalf("expected EdgeStack version %d, found %d", edgeStack.Version+1, updatedStack.Version+1)
}
if updatedStack.DeploymentType != payload.DeploymentType {
t.Fatalf("expected DeploymentType %d, found %d", edgeStack.DeploymentType, updatedStack.DeploymentType)
}
if !reflect.DeepEqual(updatedStack.EdgeGroups, payload.EdgeGroups) {
t.Fatalf("expected EdgeGroups to be equal")
}
}
func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
newEdgeGroup := portainer.EdgeGroup{
ID: 2,
Name: "EdgeGroup 2",
Dynamic: false,
TagIDs: nil,
EndpointIDs: roar.FromSlice([]portainer.EndpointID{8889}),
PartialMatch: false,
}
err := handler.DataStore.EdgeGroup().Create(&newEdgeGroup)
require.NoError(t, err)
cases := []struct {
Name string
Payload updateEdgeStackPayload
ExpectedStatusCode int
}{
{
"Update with non-existing EdgeGroupID",
updateEdgeStackPayload{
StackFileContent: "error-test",
UpdateVersion: true,
EdgeGroups: []portainer.EdgeGroupID{9999},
DeploymentType: edgeStack.DeploymentType,
},
http.StatusInternalServerError,
},
{
"Update with invalid EdgeGroup (non-existing Endpoint)",
updateEdgeStackPayload{
StackFileContent: "error-test",
UpdateVersion: true,
EdgeGroups: []portainer.EdgeGroupID{2},
DeploymentType: edgeStack.DeploymentType,
},
http.StatusInternalServerError,
},
{
"Update DeploymentType from Docker to Kubernetes",
updateEdgeStackPayload{
StackFileContent: "error-test",
UpdateVersion: true,
EdgeGroups: []portainer.EdgeGroupID{1},
DeploymentType: portainer.EdgeStackDeploymentKubernetes,
},
http.StatusBadRequest,
},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
jsonPayload, err := json.Marshal(tc.Payload)
if err != nil {
t.Fatal("JSON marshal error:", err)
}
r := bytes.NewBuffer(jsonPayload)
req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), r)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
func TestUpdateWithInvalidPayload(t *testing.T) {
t.Parallel()
handler, rawAPIKey := setupHandler(t)
endpoint := createEndpoint(t, handler.DataStore)
edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
cases := []struct {
Name string
Payload updateEdgeStackPayload
ExpectedStatusCode int
}{
{
"Update with empty StackFileContent",
updateEdgeStackPayload{
StackFileContent: "",
UpdateVersion: true,
EdgeGroups: edgeStack.EdgeGroups,
DeploymentType: edgeStack.DeploymentType,
},
http.StatusBadRequest,
},
{
"Update with empty EdgeGroups",
updateEdgeStackPayload{
StackFileContent: "error-test",
UpdateVersion: true,
EdgeGroups: []portainer.EdgeGroupID{},
DeploymentType: edgeStack.DeploymentType,
},
http.StatusBadRequest,
},
}
for _, tc := range cases {
t.Run(tc.Name, func(t *testing.T) {
jsonPayload, err := json.Marshal(tc.Payload)
require.NoError(t, err)
r := bytes.NewBuffer(jsonPayload)
req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), r)
require.NoError(t, err)
req.Header.Add("x-api-key", rawAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != tc.ExpectedStatusCode {
t.Fatalf("expected a %d response, found: %d", tc.ExpectedStatusCode, rec.Code)
}
})
}
}
+45
View File
@@ -0,0 +1,45 @@
package edgestacks
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
)
func hasKubeEndpoint(endpointService dataservices.EndpointService, endpointIDs []portainer.EndpointID) (bool, error) {
return hasEndpointPredicate(endpointService, endpointIDs, endpointutils.IsKubernetesEndpoint)
}
func hasDockerEndpoint(endpointService dataservices.EndpointService, endpointIDs []portainer.EndpointID) (bool, error) {
return hasEndpointPredicate(endpointService, endpointIDs, endpointutils.IsDockerEndpoint)
}
func hasEndpointPredicate(endpointService dataservices.EndpointService, endpointIDs []portainer.EndpointID, predicate func(*portainer.Endpoint) bool) (bool, error) {
for _, endpointID := range endpointIDs {
endpoint, err := endpointService.Endpoint(endpointID)
if err != nil {
return false, fmt.Errorf("failed to retrieve environment from database: %w", err)
}
if predicate(endpoint) {
return true, nil
}
}
return false, nil
}
func hasWrongEnvironmentType(endpointService dataservices.EndpointService, endpointIDs []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType) (bool, error) {
return hasEndpointPredicate(endpointService, endpointIDs, func(e *portainer.Endpoint) bool {
switch deploymentType {
case portainer.EdgeStackDeploymentKubernetes:
return !endpointutils.IsKubernetesEndpoint(e)
case portainer.EdgeStackDeploymentCompose:
return !endpointutils.IsDockerEndpoint(e)
default:
return true
}
})
}
@@ -0,0 +1,105 @@
package edgestacks
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_hasKubeEndpoint(t *testing.T) {
t.Parallel()
endpoints := []portainer.Endpoint{
{ID: 1, Type: portainer.DockerEnvironment},
{ID: 2, Type: portainer.AgentOnDockerEnvironment},
{ID: 3, Type: portainer.AzureEnvironment},
{ID: 4, Type: portainer.EdgeAgentOnDockerEnvironment},
{ID: 5, Type: portainer.KubernetesLocalEnvironment},
{ID: 6, Type: portainer.AgentOnKubernetesEnvironment},
{ID: 7, Type: portainer.EdgeAgentOnKubernetesEnvironment},
}
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints(endpoints))
tests := []struct {
endpointIds []portainer.EndpointID
expected bool
}{
{endpointIds: []portainer.EndpointID{1}, expected: false},
{endpointIds: []portainer.EndpointID{2}, expected: false},
{endpointIds: []portainer.EndpointID{3}, expected: false},
{endpointIds: []portainer.EndpointID{4}, expected: false},
{endpointIds: []portainer.EndpointID{5}, expected: true},
{endpointIds: []portainer.EndpointID{6}, expected: true},
{endpointIds: []portainer.EndpointID{7}, expected: true},
{endpointIds: []portainer.EndpointID{7, 2}, expected: true},
{endpointIds: []portainer.EndpointID{6, 4, 1}, expected: true},
{endpointIds: []portainer.EndpointID{1, 2, 3}, expected: false},
}
for _, test := range tests {
ans, err := hasKubeEndpoint(datastore.Endpoint(), test.endpointIds)
require.NoError(t, err, "hasKubeEndpoint shouldn't fail")
assert.Equal(t, test.expected, ans, "hasKubeEndpoint expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
}
}
func Test_hasKubeEndpoint_failWhenEndpointDontExist(t *testing.T) {
t.Parallel()
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
_, err := hasKubeEndpoint(datastore.Endpoint(), []portainer.EndpointID{1})
require.Error(t, err, "hasKubeEndpoint should fail")
}
func Test_hasDockerEndpoint(t *testing.T) {
t.Parallel()
endpoints := []portainer.Endpoint{
{ID: 1, Type: portainer.DockerEnvironment},
{ID: 2, Type: portainer.AgentOnDockerEnvironment},
{ID: 3, Type: portainer.AzureEnvironment},
{ID: 4, Type: portainer.EdgeAgentOnDockerEnvironment},
{ID: 5, Type: portainer.KubernetesLocalEnvironment},
{ID: 6, Type: portainer.AgentOnKubernetesEnvironment},
{ID: 7, Type: portainer.EdgeAgentOnKubernetesEnvironment},
}
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints(endpoints))
tests := []struct {
endpointIds []portainer.EndpointID
expected bool
}{
{endpointIds: []portainer.EndpointID{1}, expected: true},
{endpointIds: []portainer.EndpointID{2}, expected: true},
{endpointIds: []portainer.EndpointID{3}, expected: false},
{endpointIds: []portainer.EndpointID{4}, expected: true},
{endpointIds: []portainer.EndpointID{5}, expected: false},
{endpointIds: []portainer.EndpointID{6}, expected: false},
{endpointIds: []portainer.EndpointID{7}, expected: false},
{endpointIds: []portainer.EndpointID{7, 2}, expected: true},
{endpointIds: []portainer.EndpointID{6, 4, 1}, expected: true},
{endpointIds: []portainer.EndpointID{1, 2, 3}, expected: true},
}
for _, test := range tests {
ans, err := hasDockerEndpoint(datastore.Endpoint(), test.endpointIds)
require.NoError(t, err, "hasDockerEndpoint shouldn't fail")
assert.Equal(t, test.expected, ans, "hasDockerEndpoint expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
}
}
func Test_hasDockerEndpoint_failWhenEndpointDontExist(t *testing.T) {
t.Parallel()
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
_, err := hasDockerEndpoint(datastore.Endpoint(), []portainer.EndpointID{1})
require.Error(t, err, "hasDockerEndpoint should fail")
}
+65
View File
@@ -0,0 +1,65 @@
package edgestacks
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
edgestackservice "github.com/portainer/portainer/api/internal/edge/edgestacks"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle environment(endpoint) group operations.
type Handler struct {
*mux.Router
requestBouncer security.BouncerService
DataStore dataservices.DataStore
FileService portainer.FileService
GitService portainer.GitService
edgeStacksService *edgestackservice.Service
KubernetesDeployer portainer.KubernetesDeployer
}
// NewHandler creates a handler to manage environment(endpoint) group operations.
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service) *Handler {
h := &Handler{
Router: mux.NewRouter(),
requestBouncer: bouncer,
DataStore: dataStore,
edgeStacksService: edgeStacksService,
}
h.Handle("/edge_stacks/create/{method}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackCreate)))).Methods(http.MethodPost)
h.Handle("/edge_stacks",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackList)))).Methods(http.MethodGet)
h.Handle("/edge_stacks/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackInspect)))).Methods(http.MethodGet)
h.Handle("/edge_stacks/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackUpdate)))).Methods(http.MethodPut)
h.Handle("/edge_stacks/{id}",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackDelete)))).Methods(http.MethodDelete)
h.Handle("/edge_stacks/{id}/file",
bouncer.AdminAccess(bouncer.EdgeComputeOperation(httperror.LoggerHandler(h.edgeStackFile)))).Methods(http.MethodGet)
h.Handle("/edge_stacks/{id}/status",
bouncer.PublicAccess(httperror.LoggerHandler(h.edgeStackStatusUpdate))).Methods(http.MethodPut)
edgeStackStatusRouter := h.NewRoute().Subrouter()
edgeStackStatusRouter.Use(middlewares.WithEndpoint(h.DataStore.Endpoint(), "endpoint_id"))
return h
}
func handlerDBErr(err error, msg string) *httperror.HandlerError {
httpErr := httperror.InternalServerError(msg, err)
if dataservices.IsErrObjectNotFound(err) {
httpErr.StatusCode = http.StatusNotFound
}
return httpErr
}
@@ -0,0 +1,59 @@
package edgestacks
import (
"fmt"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
"github.com/rs/zerolog/log"
)
func (handler *Handler) updateStackVersion(tx dataservices.DataStoreTx, stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte, oldGitHash string, relatedEnvironmentsIDs []portainer.EndpointID) error {
stack.Version++
if err := tx.EdgeStackStatus().Clear(stack.ID, relatedEnvironmentsIDs); err != nil {
return err
}
return handler.storeStackFile(stack, deploymentType, config)
}
func (handler *Handler) storeStackFile(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte) error {
if deploymentType != stack.DeploymentType {
// deployment type was changed - need to delete all old files
if err := handler.FileService.RemoveDirectory(stack.ProjectPath); err != nil {
log.Warn().Err(err).Msg("Unable to clear old files")
}
stack.EntryPoint = ""
stack.ManifestPath = ""
stack.DeploymentType = deploymentType
}
stackFolder := strconv.Itoa(int(stack.ID))
entryPoint := ""
if deploymentType == portainer.EdgeStackDeploymentCompose {
if stack.EntryPoint == "" {
stack.EntryPoint = filesystem.ComposeFileDefaultName
}
entryPoint = stack.EntryPoint
}
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
if stack.ManifestPath == "" {
stack.ManifestPath = filesystem.ManifestFileDefaultName
}
entryPoint = stack.ManifestPath
}
if _, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, entryPoint, config); err != nil {
return fmt.Errorf("unable to persist updated Compose file with version on disk: %w", err)
}
return nil
}
@@ -0,0 +1,112 @@
package endpointedge
import (
"errors"
"fmt"
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type logsPayload struct {
FileContent string
}
func (payload *logsPayload) Validate(r *http.Request) error {
return nil
}
// endpointEdgeJobsLogs
// @summary Update the logs collected from an Edge Job
// @description **Access policy**: Edge agent only — requires X-PortainerAgent-EdgeID header
// @tags edge_agent
// @accept json
// @produce json
// @param id path int true "environment Id"
// @param jobID path int true "Job Id"
// @success 200
// @failure 500
// @failure 400
// @failure 403
// @router /endpoints/{id}/edge/jobs/{jobID}/logs [post]
func (handler *Handler) endpointEdgeJobsLogs(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return httperror.BadRequest("Unable to find an environment on request context", err)
}
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment ID: %d", err, endpoint.ID))
}
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "jobID")
if err != nil {
return httperror.BadRequest("Invalid edge job identifier route variable", fmt.Errorf("invalid Edge job route variable: %w. Environment ID: %d", err, endpoint.ID))
}
var payload logsPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", fmt.Errorf("invalid Edge job request payload: %w. Environment ID: %d", err, endpoint.ID))
}
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.updateEdgeJobLogs(tx, endpoint.ID, portainer.EdgeJobID(edgeJobID), payload)
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
httpErr.Err = fmt.Errorf("edge polling error: %w. Environment ID: %d", httpErr.Err, endpoint.ID)
return httpErr
}
return httperror.InternalServerError("Unexpected error", fmt.Errorf("edge polling error: %w. Environment ID: %d", err, endpoint.ID))
}
return response.JSON(w, nil)
}
func (handler *Handler) updateEdgeJobLogs(tx dataservices.DataStoreTx, endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID, payload logsPayload) error {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
edgeJob, err := tx.EdgeJob().Read(edgeJobID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an edge job with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an edge job with the specified identifier inside the database", err)
}
if resp, err := handler.buildSchedules(tx, endpoint, []portainer.EdgeJob{*edgeJob}); err != nil || len(resp) == 0 {
return httperror.InternalServerError("Unable to verify if the edge job is assigned to the environment",
fmt.Errorf("unable to verify if the edge job is assigned to the environment: %w. Environment name: %s", err, endpoint.Name))
}
if err := handler.FileService.StoreEdgeJobTaskLogFileFromBytes(strconv.Itoa(int(edgeJobID)), strconv.Itoa(int(endpoint.ID)), []byte(payload.FileContent)); err != nil {
return httperror.InternalServerError("Unable to save task log to the filesystem", err)
}
meta := portainer.EdgeJobEndpointMeta{CollectLogs: false, LogsStatus: portainer.EdgeJobLogsStatusCollected}
if _, ok := edgeJob.GroupLogsCollection[endpoint.ID]; ok {
edgeJob.GroupLogsCollection[endpoint.ID] = meta
} else {
edgeJob.Endpoints[endpoint.ID] = meta
}
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
return httperror.InternalServerError("Unable to persist edge job changes to the database", err)
}
cache.Del(endpointID)
return nil
}
@@ -0,0 +1,41 @@
package endpointedge
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/require"
)
func TestUpdateUnrelatedEdgeJobLogs(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
h := &Handler{DataStore: store}
endpointID := portainer.EndpointID(2)
edgeJobID := portainer.EdgeJobID(3)
payload := logsPayload{FileContent: "log content"}
err := store.Endpoint().Create(&portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint",
})
require.NoError(t, err)
err = store.EdgeJob().CreateWithID(edgeJobID, &portainer.EdgeJob{
ID: edgeJobID,
Name: "test-edge-job",
})
require.NoError(t, err)
// There is no relation between the edge job and the endpoint, so the
// update must fail
err = store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return h.updateEdgeJobLogs(tx, endpointID, edgeJobID, payload)
})
require.Error(t, err)
}
@@ -0,0 +1,112 @@
package endpointedge
import (
"errors"
"fmt"
"net/http"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/edge"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/kubernetes"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"golang.org/x/sync/singleflight"
)
var edgeStackSingleFlightGroup = singleflight.Group{}
// @summary Inspect an Edge Stack for an Environment
// @description **Access policy**: Edge agent only — requires X-PortainerAgent-EdgeID header
// @tags edge_agent, edge_stacks
// @accept json
// @produce json
// @param id path int true "environment Id"
// @param stackId path int true "EdgeStack Id"
// @success 200 {object} edge.StackPayload
// @failure 500
// @failure 400
// @failure 404
// @router /endpoints/{id}/edge/stacks/{stackId} [get]
func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpoint, err := middlewares.FetchEndpoint(r)
if err != nil {
return httperror.BadRequest("Unable to find an environment on request context", err)
}
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment ID: %d", err, endpoint.ID))
}
edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "stackId")
if err != nil {
return httperror.BadRequest("Invalid edge stack identifier route variable", fmt.Errorf("invalid Edge stack route variable: %w. Environment ID: %d", err, endpoint.ID))
}
s, err, _ := edgeStackSingleFlightGroup.Do(strconv.Itoa(edgeStackID), func() (any, error) {
edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
if handler.DataStore.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Unable to find an edge stack with the specified identifier inside the database", fmt.Errorf("unable to find the Edge stack from database: %w. Environment ID: %d", err, endpoint.ID))
}
return edgeStack, err
})
if err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
return httpErr
}
return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", fmt.Errorf("failed to find Edge stack from the database: %w. Environment ID: %d", err, endpoint.ID))
}
// WARNING: this variable must not be mutated
edgeStack := s.(*portainer.EdgeStack)
fileName := edgeStack.EntryPoint
if endpointutils.IsDockerEndpoint(endpoint) {
if fileName == "" {
return httperror.BadRequest("Docker is not supported by this stack", fmt.Errorf("no filename is provided for the Docker endpoint. Environment ID: %d", endpoint.ID))
}
}
namespace := ""
if !edgeStack.UseManifestNamespaces {
namespace = kubernetes.DefaultNamespace
}
if endpointutils.IsKubernetesEndpoint(endpoint) {
fileName = edgeStack.ManifestPath
if fileName == "" {
return httperror.BadRequest("Kubernetes is not supported by this stack", fmt.Errorf("no filename is provided for the Kubernetes endpoint. Environment ID: %d", endpoint.ID))
}
}
dirEntries, err := filesystem.LoadDir(edgeStack.ProjectPath)
if err != nil {
return httperror.InternalServerError("Unable to load repository", fmt.Errorf("failed to load project directory: %w. Environment ID: %d", err, endpoint.ID))
}
fileContent, err := filesystem.FilterDirForCompatibility(dirEntries, fileName, endpoint.Agent.Version)
if err != nil {
return httperror.InternalServerError("File not found", fmt.Errorf("unable to find file: %w. Environment ID: %d", err, endpoint.ID))
}
dirEntries = filesystem.FilterDirForEntryFile(dirEntries, fileName)
return response.JSON(w, edge.StackPayload{
DirEntries: dirEntries,
EntryFileName: fileName,
StackFileContent: fileContent,
Name: edgeStack.Name,
Namespace: namespace,
CreatedBy: edgeStack.CreatedBy,
CreatedByUserId: edgeStack.CreatedByUserId,
})
}
@@ -0,0 +1,351 @@
package endpointedge
import (
"bytes"
"cmp"
"encoding/base64"
"errors"
"fmt"
"hash/fnv"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
type stackStatusResponse struct {
// EdgeStack Identifier
ID portainer.EdgeStackID `example:"1"`
// Version of this stack
Version int `example:"3"`
}
type edgeJobResponse struct {
// EdgeJob Identifier
ID portainer.EdgeJobID `json:"Id" example:"2"`
// Whether to collect logs
CollectLogs bool `json:"CollectLogs" example:"true"`
// A cron expression to schedule this job
CronExpression string `json:"CronExpression" example:"* * * * *"`
// Script to run
Script string `json:"Script" example:"echo hello"`
// Version of this EdgeJob
Version int `json:"Version" example:"2"`
}
type endpointEdgeStatusInspectResponse struct {
// Status represents the environment(endpoint) status
Status string `json:"status" example:"REQUIRED"`
// The tunnel port
Port int `json:"port" example:"8732"`
// List of requests for jobs to run on the environment(endpoint)
Schedules []edgeJobResponse `json:"schedules"`
// The current value of CheckinInterval
CheckinInterval int `json:"checkin" example:"5"`
//
Credentials string `json:"credentials"`
// List of stacks to be deployed on the environments(endpoints)
Stacks []stackStatusResponse `json:"stacks"`
}
// @id EndpointEdgeStatusInspect
// @summary Get environment status
// @description Endpoint for edge agent to check status of environment
// @description **Access policy**: Edge agent only — requires X-PortainerAgent-EdgeID header
// @tags edge_agent
// @security ApiKeyAuth
// @security jwt
// @param id path int true "Environment identifier"
// @success 200 {object} endpointEdgeStatusInspectResponse "Success"
// @failure 400 "Invalid request"
// @failure 403 "Permission denied to access environment"
// @failure 404 "Environment not found"
// @failure 500 "Server error"
// @router /endpoints/{id}/edge/status [get]
func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
if cachedResp := handler.respondFromCache(w, r, portainer.EndpointID(endpointID)); cachedResp {
return nil
}
if _, ok := handler.DataStore.Endpoint().Heartbeat(portainer.EndpointID(endpointID)); !ok {
// EE-5190
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unable to retrieve endpoint heartbeat. Environment ID: %d", endpointID))
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if err != nil {
// EE-5190
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unable to retrieve endpoint from database: %w. Environment ID: %d", err, endpointID))
}
firstConn := endpoint.LastCheckInDate == 0
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unauthorized Edge endpoint operation: %w. Environment ID: %d", err, endpoint.ID))
}
handler.DataStore.Endpoint().UpdateHeartbeat(endpoint.ID)
if err := handler.requestBouncer.TrustedEdgeEnvironmentAccess(handler.DataStore, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("untrusted Edge environment access: %w. Environment ID: %d", err, endpoint.ID))
}
var statusResponse *endpointEdgeStatusInspectResponse
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
statusResponse, err = handler.inspectStatus(tx, r, portainer.EndpointID(endpointID), firstConn)
return err
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
httpErr.Err = fmt.Errorf("edge polling error: %w. Environment ID: %d", httpErr.Err, endpoint.ID)
return httpErr
}
return httperror.InternalServerError("Unexpected error", fmt.Errorf("edge polling error: %w. Environment ID: %d", err, endpoint.ID))
}
return cacheResponse(w, endpoint.ID, *statusResponse)
}
func (handler *Handler) parseHeaders(r *http.Request, endpoint *portainer.Endpoint) error {
endpoint.EdgeID = cmp.Or(endpoint.EdgeID, r.Header.Get(portainer.PortainerAgentEdgeIDHeader))
agentPlatform, agentPlatformErr := parseAgentPlatform(r)
if agentPlatformErr != nil {
return httperror.BadRequest("agent platform header is not valid", agentPlatformErr)
}
endpoint.Type = agentPlatform
version := r.Header.Get(portainer.PortainerAgentHeader)
endpoint.Agent.Version = version
if gpuOperatorHeader := r.Header.Get(portainer.HTTPResponseAgentGPUOperator); gpuOperatorHeader != "" {
endpoint.Kubernetes.Flags.GPUOperator = gpuOperatorHeader == "true"
}
return nil
}
func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Request, endpointID portainer.EndpointID, firstConn bool) (*endpointEdgeStatusInspectResponse, error) {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return nil, err
}
if err := handler.parseHeaders(r, endpoint); err != nil {
return nil, err
}
// Take an initial snapshot
if firstConn {
if err := handler.ReverseTunnelService.Open(endpoint); err != nil {
log.Error().Err(err).Msg("could not open the tunnel")
}
}
endpoint.LastCheckInDate = time.Now().Unix()
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint); err != nil {
return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
tunnel := handler.ReverseTunnelService.Config(endpoint.ID)
statusResponse := endpointEdgeStatusInspectResponse{
Status: tunnel.Status,
Port: tunnel.Port,
CheckinInterval: edge.EffectiveCheckinInterval(tx, endpoint),
Credentials: tunnel.Credentials,
}
schedules, handlerErr := handler.buildAllSchedules(tx, endpoint)
if handlerErr != nil {
return nil, handlerErr
}
statusResponse.Schedules = schedules
edgeStacksStatus, handlerErr := handler.buildEdgeStacks(tx, endpoint.ID)
if handlerErr != nil {
return nil, handlerErr
}
statusResponse.Stacks = edgeStacksStatus
return &statusResponse, nil
}
func parseAgentPlatform(r *http.Request) (portainer.EndpointType, error) {
agentPlatformHeader := r.Header.Get(portainer.HTTPResponseAgentPlatform)
if agentPlatformHeader == "" {
return 0, errors.New("agent platform header is missing")
}
agentPlatformNumber, err := strconv.Atoi(agentPlatformHeader)
if err != nil {
return 0, err
}
agentPlatform := portainer.AgentPlatform(agentPlatformNumber)
switch agentPlatform {
case portainer.AgentPlatformDocker:
return portainer.EdgeAgentOnDockerEnvironment, nil
case portainer.AgentPlatformKubernetes:
return portainer.EdgeAgentOnKubernetesEnvironment, nil
default:
return 0, fmt.Errorf("agent platform %v is not valid", agentPlatform)
}
}
func (handler *Handler) buildAllSchedules(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) ([]edgeJobResponse, *httperror.HandlerError) {
edgeJobs, err := tx.EdgeJob().ReadAll()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve Edge Jobs", err)
}
return handler.buildSchedules(tx, endpoint, edgeJobs)
}
func (handler *Handler) buildSchedules(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, edgeJobs []portainer.EdgeJob) ([]edgeJobResponse, *httperror.HandlerError) {
schedules := []edgeJobResponse{}
endpointGroups, err := tx.EndpointGroup().ReadAll()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve endpoint groups", err)
}
for _, job := range edgeJobs {
_, endpointHasJob := job.Endpoints[endpoint.ID]
if !endpointHasJob {
for _, edgeGroupID := range job.EdgeGroups {
member, _, err := edge.EndpointInEdgeGroup(tx, endpoint, edgeGroupID, endpointGroups)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve relations", err)
} else if member {
endpointHasJob = true
break
}
}
}
if !endpointHasJob {
continue
}
schedule := edgeJobResponse{
ID: job.ID,
CronExpression: job.CronExpression,
CollectLogs: job.GroupLogsCollection[endpoint.ID].CollectLogs || job.Endpoints[endpoint.ID].CollectLogs,
Version: job.Version,
}
file, err := handler.FileService.GetFileContent(job.ScriptPath, "")
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve Edge job script file", err)
}
schedule.Script = base64.RawStdEncoding.EncodeToString(file)
schedules = append(schedules, schedule)
}
return schedules, nil
}
func (handler *Handler) buildEdgeStacks(tx dataservices.DataStoreTx, endpointID portainer.EndpointID) ([]stackStatusResponse, *httperror.HandlerError) {
relation, err := tx.EndpointRelation().EndpointRelation(endpointID)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve relation object from the database", err)
}
edgeStacksStatus := []stackStatusResponse{}
for stackID := range relation.EdgeStacks {
version, ok := tx.EdgeStack().EdgeStackVersion(stackID)
if !ok {
return nil, httperror.InternalServerError("Unable to retrieve edge stack from the database", err)
}
stackStatus := stackStatusResponse{
ID: stackID,
Version: version,
}
edgeStacksStatus = append(edgeStacksStatus, stackStatus)
}
return edgeStacksStatus, nil
}
func cacheResponse(w http.ResponseWriter, endpointID portainer.EndpointID, statusResponse endpointEdgeStatusInspectResponse) *httperror.HandlerError {
rr := httptest.NewRecorder()
if err := response.JSON(rr, statusResponse); err != nil {
return err
}
h := fnv.New32a()
h.Write(rr.Body.Bytes())
etag := strconv.FormatUint(uint64(h.Sum32()), 16)
cache.Set(endpointID, []byte(etag))
resp := rr.Result()
for k, vs := range resp.Header {
for _, v := range vs {
w.Header().Add(k, v)
}
}
w.Header().Set("ETag", etag)
if _, err := io.Copy(w, resp.Body); err != nil {
log.Warn().Err(err).Msg("failed to copy response body")
}
return nil
}
func (handler *Handler) respondFromCache(w http.ResponseWriter, r *http.Request, endpointID portainer.EndpointID) bool {
inmHeader := r.Header.Get("If-None-Match")
etags := strings.Split(inmHeader, ",")
if len(inmHeader) == 0 || etags[0] == "" {
return false
}
cachedETag, ok := cache.Get(endpointID)
if !ok {
return false
}
for _, etag := range etags {
if !bytes.Equal([]byte(etag), cachedETag) {
continue
}
handler.DataStore.Endpoint().UpdateHeartbeat(endpointID)
w.Header().Set("ETag", etag)
w.WriteHeader(http.StatusNotModified)
return true
}
return false
}
@@ -0,0 +1,440 @@
package endpointedge
import (
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"testing/synctest"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/apikey"
"github.com/portainer/portainer/api/chisel"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type endpointTestCase struct {
endpoint portainer.Endpoint
endpointRelation portainer.EndpointRelation
expectedStatusCode int
}
var endpointTestCases = []endpointTestCase{
{
portainer.Endpoint{},
portainer.EndpointRelation{},
http.StatusForbidden,
},
{
portainer.Endpoint{
ID: -1,
Name: "endpoint-id-1",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
},
portainer.EndpointRelation{
EndpointID: -1,
},
http.StatusForbidden,
},
{
portainer.Endpoint{
ID: 2,
Name: "endpoint-id-2",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "",
},
portainer.EndpointRelation{
EndpointID: 2,
},
http.StatusForbidden,
},
{
portainer.Endpoint{
ID: 4,
Name: "endpoint-id-4",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
},
portainer.EndpointRelation{
EndpointID: 4,
},
http.StatusOK,
},
}
func mustSetupHandler(t *testing.T) *Handler {
tmpDir := t.TempDir()
fs, err := filesystem.NewService(tmpDir, "")
if err != nil {
t.Fatalf("could not start a new filesystem service: %s", err)
}
_, store := datastore.MustNewTestStore(t, true, true)
jwtService, err := jwt.NewService("1h", store)
if err != nil {
t.Fatalf("could not start a new JWT service: %s", err)
}
apiKeyService := apikey.NewAPIKeyService(nil, nil)
settings, err := store.Settings().Settings()
if err != nil {
t.Fatalf("could not create new settings: %s", err)
}
settings.TrustOnFirstConnect = true
if err = store.Settings().UpdateSettings(settings); err != nil {
t.Fatalf("could not update settings: %s", err)
}
handler := NewHandler(
security.NewRequestBouncer(t.Context(), store, jwtService, apiKeyService),
store,
fs,
chisel.NewService(store, t.Context(), nil),
)
handler.ReverseTunnelService = chisel.NewService(store, t.Context(), nil)
return handler
}
func createEndpoint(handler *Handler, endpoint portainer.Endpoint, endpointRelation portainer.EndpointRelation) (err error) {
// Avoid setting ID below 0 to generate invalid test cases
if endpoint.ID <= 0 {
return nil
}
if err := handler.DataStore.Endpoint().Create(&endpoint); err != nil {
return err
}
return handler.DataStore.EndpointRelation().Create(&endpointRelation)
}
func TestMissingEdgeIdentifier(t *testing.T) {
t.Parallel()
handler := mustSetupHandler(t)
endpointID := portainer.EndpointID(45)
if err := createEndpoint(handler, portainer.Endpoint{
ID: endpointID,
Name: "endpoint-id-45",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
}, portainer.EndpointRelation{EndpointID: endpointID}); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpointID), nil)
if err != nil {
t.Fatal("request error:", err)
}
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("expected a %d response, found: %d without Edge identifier", http.StatusForbidden, rec.Code)
}
}
func TestWithEndpoints(t *testing.T) {
t.Parallel()
handler := mustSetupHandler(t)
for _, test := range endpointTestCases {
err := createEndpoint(handler, test.endpoint, test.endpointRelation)
if err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", test.endpoint.ID), nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, test.endpoint.EdgeID)
req.Header.Set(portainer.HTTPResponseAgentPlatform, "1")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != test.expectedStatusCode {
t.Fatalf("expected a %d response, found: %d for endpoint ID: %d", test.expectedStatusCode, rec.Code, test.endpoint.ID)
}
}
}
func TestLastCheckInDateIncreases(t *testing.T) {
t.Parallel()
synctest.Test(t, testLastCheckInDateIncreases)
}
func testLastCheckInDateIncreases(t *testing.T) {
handler := mustSetupHandler(t)
endpointID := portainer.EndpointID(56)
endpoint := portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint-56",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
LastCheckInDate: time.Now().Unix(),
}
endpointRelation := portainer.EndpointRelation{
EndpointID: endpoint.ID,
}
if err := createEndpoint(handler, endpoint, endpointRelation); err != nil {
t.Fatal(err)
}
time.Sleep(1 * time.Second)
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpoint.ID), nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, "edge-id")
req.Header.Set(portainer.HTTPResponseAgentPlatform, "1")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
updatedEndpoint, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
if err != nil {
t.Fatal(err)
}
assert.Greater(t, updatedEndpoint.LastCheckInDate, endpoint.LastCheckInDate)
}
func TestEmptyEdgeIdWithAgentPlatformHeader(t *testing.T) {
t.Parallel()
handler := mustSetupHandler(t)
endpointID := portainer.EndpointID(44)
edgeId := "edge-id"
endpoint := portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint-44",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "",
}
endpointRelation := portainer.EndpointRelation{
EndpointID: endpoint.ID,
}
if err := createEndpoint(handler, endpoint, endpointRelation); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpoint.ID), nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, edgeId)
req.Header.Set(portainer.HTTPResponseAgentPlatform, "1")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d with empty edge ID", http.StatusOK, rec.Code)
}
updatedEndpoint, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, updatedEndpoint.EdgeID, edgeId)
}
func TestEdgeStackStatus(t *testing.T) {
t.Parallel()
handler := mustSetupHandler(t)
endpointID := portainer.EndpointID(7)
endpoint := portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint-7",
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id",
LastCheckInDate: time.Now().Unix(),
}
edgeStackID := portainer.EdgeStackID(17)
edgeStack := portainer.EdgeStack{
ID: edgeStackID,
Name: "test-edge-stack-17",
CreationDate: time.Now().Unix(),
EdgeGroups: []portainer.EdgeGroupID{1, 2},
ProjectPath: "/project/path",
EntryPoint: "entrypoint",
Version: 237,
ManifestPath: "/manifest/path",
DeploymentType: 1,
}
endpointRelation := portainer.EndpointRelation{
EndpointID: endpoint.ID,
EdgeStacks: map[portainer.EdgeStackID]bool{
edgeStack.ID: true,
},
}
err := handler.DataStore.EdgeStack().Create(edgeStack.ID, &edgeStack)
require.NoError(t, err)
if err := createEndpoint(handler, endpoint, endpointRelation); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpoint.ID), nil)
if err != nil {
t.Fatal("request error:", err)
}
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, "edge-id")
req.Header.Set(portainer.HTTPResponseAgentPlatform, "1")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected a %d response, found: %d", http.StatusOK, rec.Code)
}
var data endpointEdgeStatusInspectResponse
if err := json.NewDecoder(rec.Body).Decode(&data); err != nil {
t.Fatal("error decoding response:", err)
}
assert.Len(t, data.Stacks, 1)
assert.Equal(t, edgeStack.ID, data.Stacks[0].ID)
assert.Equal(t, edgeStack.Version, data.Stacks[0].Version)
}
func TestEdgeJobsResponse(t *testing.T) {
t.Parallel()
handler := mustSetupHandler(t)
localCreateEndpoint := func(endpointID portainer.EndpointID, tagIDs []portainer.TagID) *portainer.Endpoint {
endpoint := portainer.Endpoint{
ID: endpointID,
Name: "test-endpoint-" + strconv.Itoa(int(endpointID)),
Type: portainer.EdgeAgentOnDockerEnvironment,
URL: "https://portainer.io:9443",
EdgeID: "edge-id-" + strconv.Itoa(int(endpointID)),
TagIDs: tagIDs,
LastCheckInDate: time.Now().Unix(),
UserTrusted: true,
}
err := createEndpoint(handler, endpoint,
portainer.EndpointRelation{EndpointID: endpointID})
require.NoError(t, err)
return &endpoint
}
dynamicGroupTags := []portainer.TagID{1, 2, 3}
endpoint := localCreateEndpoint(77, nil)
endpointFromStaticEdgeGroup := localCreateEndpoint(78, nil)
endpointFromDynamicEdgeGroup := localCreateEndpoint(79, dynamicGroupTags)
unrelatedEndpoint := localCreateEndpoint(80, nil)
staticEdgeGroup := portainer.EdgeGroup{
ID: 1,
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpointFromStaticEdgeGroup.ID}),
}
err := handler.DataStore.EdgeGroup().Create(&staticEdgeGroup)
require.NoError(t, err)
dynamicEdgeGroup := portainer.EdgeGroup{
ID: 2,
Dynamic: true,
TagIDs: dynamicGroupTags,
}
err = handler.DataStore.EdgeGroup().Create(&dynamicEdgeGroup)
require.NoError(t, err)
path, err := handler.FileService.StoreEdgeJobFileFromBytes("test-script", []byte("pwd"))
require.NoError(t, err)
edgeJobID := portainer.EdgeJobID(35)
edgeJob := portainer.EdgeJob{
ID: edgeJobID,
Created: time.Now().Unix(),
CronExpression: "* * * * *",
Name: "test-edge-job",
ScriptPath: path,
Recurring: true,
Version: 57,
Endpoints: map[portainer.EndpointID]portainer.EdgeJobEndpointMeta{
endpoint.ID: {},
},
EdgeGroups: []portainer.EdgeGroupID{staticEdgeGroup.ID, dynamicEdgeGroup.ID},
}
err = handler.DataStore.EdgeJob().Create(&edgeJob)
require.NoError(t, err)
f := func(endpoint *portainer.Endpoint, scheduleLen int) {
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/api/endpoints/%d/edge/status", endpoint.ID), nil)
require.NoError(t, err)
req.Header.Set(portainer.PortainerAgentEdgeIDHeader, endpoint.EdgeID)
req.Header.Set(portainer.HTTPResponseAgentPlatform, "1")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code)
var data endpointEdgeStatusInspectResponse
err = json.NewDecoder(rec.Body).Decode(&data)
require.NoError(t, err)
require.Len(t, data.Schedules, scheduleLen)
if scheduleLen > 0 {
require.Equal(t, edgeJob.ID, data.Schedules[0].ID)
require.Equal(t, edgeJob.CronExpression, data.Schedules[0].CronExpression)
require.Equal(t, edgeJob.Version, data.Schedules[0].Version)
}
}
f(endpoint, 1)
f(endpointFromStaticEdgeGroup, 1)
f(endpointFromDynamicEdgeGroup, 1)
f(unrelatedEndpoint, 0)
}
+46
View File
@@ -0,0 +1,46 @@
package endpointedge
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/middlewares"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
// Handler is the HTTP handler used to handle edge environment(endpoint) operations.
type Handler struct {
*mux.Router
requestBouncer security.BouncerService
DataStore dataservices.DataStore
FileService portainer.FileService
ReverseTunnelService portainer.ReverseTunnelService
}
// NewHandler creates a handler to manage environment(endpoint) operations.
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
requestBouncer: bouncer,
DataStore: dataStore,
FileService: fileService,
ReverseTunnelService: reverseTunnelService,
}
h.Handle("/api/endpoints/{id}/edge/status", bouncer.PublicAccess(httperror.LoggerHandler(h.endpointEdgeStatusInspect))).Methods(http.MethodGet)
endpointRouter := h.PathPrefix("/api/endpoints/{id}").Subrouter()
endpointRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
endpointRouter.PathPrefix("/edge/stacks/{stackId}").Handler(
bouncer.PublicAccess(httperror.LoggerHandler(h.endpointEdgeStackInspect))).Methods(http.MethodGet)
endpointRouter.PathPrefix("/edge/jobs/{jobID}/logs").Handler(
bouncer.PublicAccess(httperror.LoggerHandler(h.endpointEdgeJobsLogs))).Methods(http.MethodPost)
return h
}
@@ -0,0 +1,120 @@
package endpointgroups
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type endpointGroupCreatePayload struct {
// Environment(Endpoint) group name
Name string `validate:"required" example:"my-environment-group"`
// Environment(Endpoint) group description
Description string `example:"description"`
// List of environment(endpoint) identifiers that will be part of this group
AssociatedEndpoints []portainer.EndpointID `example:"1,3"`
// List of tag identifiers to which this environment(endpoint) group is associated
TagIDs []portainer.TagID `example:"1,2"`
}
func (payload *endpointGroupCreatePayload) Validate(r *http.Request) error {
if len(payload.Name) == 0 {
return errors.New("invalid environment group name")
}
if payload.TagIDs == nil {
payload.TagIDs = []portainer.TagID{}
}
return nil
}
// @summary Create an Environment(Endpoint) Group
// @description Create a new environment(endpoint) group.
// @description **Access policy**: administrator
// @tags endpoint_groups
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body endpointGroupCreatePayload true "Environment(Endpoint) Group details"
// @success 200 {object} portainer.EndpointGroup "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /endpoint_groups [post]
func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload endpointGroupCreatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
var endpointGroup *portainer.EndpointGroup
var err error
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
endpointGroup, err = handler.createEndpointGroup(tx, payload)
return err
})
return response.TxResponse(w, endpointGroup, err)
}
func (handler *Handler) createEndpointGroup(tx dataservices.DataStoreTx, payload endpointGroupCreatePayload) (*portainer.EndpointGroup, error) {
endpointGroup := &portainer.EndpointGroup{
Name: payload.Name,
Description: payload.Description,
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: payload.TagIDs,
}
err := tx.EndpointGroup().Create(endpointGroup)
if err != nil {
return nil, httperror.InternalServerError("Unable to persist the environment group inside the database", err)
}
endpoints, err := tx.Endpoint().Endpoints()
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
for _, id := range payload.AssociatedEndpoints {
for _, endpoint := range endpoints {
if endpoint.ID == id {
endpoint.GroupID = endpointGroup.ID
err := tx.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
if err != nil {
return nil, httperror.InternalServerError("Unable to update environment", err)
}
err = handler.updateEndpointRelations(tx, &endpoint, endpointGroup)
if err != nil {
return nil, httperror.InternalServerError("Unable to persist environment relations changes inside the database", err)
}
break
}
}
}
for _, tagID := range endpointGroup.TagIDs {
tag, err := tx.Tag().Read(tagID)
if err != nil {
return nil, httperror.InternalServerError("Unable to find a tag inside the database", err)
}
tag.EndpointGroups[endpointGroup.ID] = true
err = tx.Tag().Update(tagID, tag)
if err != nil {
return nil, httperror.InternalServerError("Unable to persist tag changes inside the database", err)
}
}
return endpointGroup, nil
}
@@ -0,0 +1,90 @@
package endpointgroups
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EndpointGroupDelete
// @summary Remove an environment(endpoint) group
// @description Remove an environment(endpoint) group.
// @description **Access policy**: administrator
// @tags endpoint_groups
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EndpointGroup identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 404 "EndpointGroup not found"
// @failure 500 "Server error"
// @router /endpoint_groups/{id} [delete]
func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment group identifier route variable", err)
}
if endpointGroupID == 1 {
return httperror.Forbidden("Unable to remove the default 'Unassigned' group", errors.New("Cannot remove the default environment group"))
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.deleteEndpointGroup(tx, portainer.EndpointGroupID(endpointGroupID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) deleteEndpointGroup(tx dataservices.DataStoreTx, endpointGroupID portainer.EndpointGroupID) error {
endpointGroup, err := tx.EndpointGroup().Read(endpointGroupID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment group with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment group with the specified identifier inside the database", err)
}
if err := tx.EndpointGroup().Delete(endpointGroupID); err != nil {
return httperror.InternalServerError("Unable to remove the environment group from the database", err)
}
endpoints, err := tx.Endpoint().Endpoints()
if err != nil {
return httperror.InternalServerError("Unable to retrieve environment from the database", err)
}
for _, endpoint := range endpoints {
if endpoint.GroupID != endpointGroupID {
continue
}
endpoint.GroupID = portainer.EndpointGroupID(1)
if err := tx.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint); err != nil {
return httperror.InternalServerError("Unable to update environment", err)
}
if err := handler.updateEndpointRelations(tx, &endpoint, nil); err != nil {
return httperror.InternalServerError("Unable to persist environment relations changes inside the database", err)
}
}
for _, tagID := range endpointGroup.TagIDs {
tag, err := tx.Tag().Read(tagID)
if err != nil {
return httperror.InternalServerError("Unable to find a tag inside the database", err)
}
delete(tag.EndpointGroups, endpointGroup.ID)
if err := tx.Tag().Update(tagID, tag); err != nil {
return httperror.InternalServerError("Unable to persist tag changes inside the database", err)
}
}
return nil
}
@@ -0,0 +1,73 @@
package endpointgroups
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EndpointGroupAddEndpoint
// @summary Add an environment(endpoint) to an environment(endpoint) group
// @description Add an environment(endpoint) to an environment(endpoint) group
// @description **Access policy**: administrator
// @tags endpoint_groups
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EndpointGroup identifier"
// @param endpointId path int true "Environment(Endpoint) identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 404 "EndpointGroup not found"
// @failure 500 "Server error"
// @router /endpoint_groups/{id}/endpoints/{endpointId} [put]
func (handler *Handler) endpointGroupAddEndpoint(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment group identifier route variable", err)
}
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "endpointId")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.addEndpoint(tx, portainer.EndpointGroupID(endpointGroupID), portainer.EndpointID(endpointID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) addEndpoint(tx dataservices.DataStoreTx, endpointGroupID portainer.EndpointGroupID, endpointID portainer.EndpointID) error {
endpointGroup, err := tx.EndpointGroup().Read(endpointGroupID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment group with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment group with the specified identifier inside the database", err)
}
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
endpoint.GroupID = endpointGroup.ID
err = tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
if err != nil {
return httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
err = handler.updateEndpointRelations(tx, endpoint, endpointGroup)
if err != nil {
return httperror.InternalServerError("Unable to persist environment relations changes inside the database", err)
}
return nil
}
@@ -0,0 +1,73 @@
package endpointgroups
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id EndpointGroupDeleteEndpoint
// @summary Removes environment(endpoint) from an environment(endpoint) group
// @description **Access policy**: administrator
// @tags endpoint_groups
// @security ApiKeyAuth
// @security jwt
// @param id path int true "EndpointGroup identifier"
// @param endpointId path int true "Environment(Endpoint) identifier"
// @success 204 "Success"
// @failure 400 "Invalid request"
// @failure 404 "EndpointGroup not found"
// @failure 500 "Server error"
// @router /endpoint_groups/{id}/endpoints/{endpointId} [delete]
func (handler *Handler) endpointGroupDeleteEndpoint(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment group identifier route variable", err)
}
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "endpointId")
if err != nil {
return httperror.BadRequest("Invalid environment identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.removeEndpoint(tx, portainer.EndpointGroupID(endpointGroupID), portainer.EndpointID(endpointID))
})
return response.TxEmptyResponse(w, err)
}
func (handler *Handler) removeEndpoint(tx dataservices.DataStoreTx, endpointGroupID portainer.EndpointGroupID, endpointID portainer.EndpointID) error {
ok, err := tx.EndpointGroup().Exists(endpointGroupID)
if !ok {
return httperror.NotFound("Unable to find an environment group with the specified identifier inside the database", dserrors.ErrObjectNotFound)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment group with the specified identifier inside the database", err)
}
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
}
endpoint.GroupID = portainer.EndpointGroupID(1)
err = tx.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
if err != nil {
return httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
err = handler.updateEndpointRelations(tx, endpoint, nil)
if err != nil {
return httperror.InternalServerError("Unable to persist environment relations changes inside the database", err)
}
return nil
}
@@ -0,0 +1,72 @@
package endpointgroups
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @summary Inspect an Environment(Endpoint) group
// @description Retrieve details abont an environment(endpoint) group.
// @description **Access policy**: administrator
// @tags endpoint_groups
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Environment(Endpoint) group identifier"
// @param size query boolean false "If true, include the number of environments and breakdown by type"
// @success 200 {object} EndpointGroupResponse "Success"
// @failure 400 "Invalid request"
// @failure 404 "EndpointGroup not found"
// @failure 500 "Server error"
// @router /endpoint_groups/{id} [get]
func (handler *Handler) endpointGroupInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid environment group identifier route variable", err)
}
includeSize, err := request.RetrieveBooleanQueryParameter(r, "size", true)
if err != nil {
return httperror.BadRequest("Invalid query parameter: size", err)
}
groupID := portainer.EndpointGroupID(endpointGroupID)
var endpointGroup *portainer.EndpointGroup
var endpoints []portainer.Endpoint
if err := handler.DataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
endpointGroup, err = tx.EndpointGroup().Read(groupID)
if err != nil {
return err
}
if includeSize {
endpoints, err = tx.Endpoint().Endpoints()
}
return err
}); err != nil {
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an environment group with the specified identifier inside the database", err)
}
return httperror.InternalServerError("Unable to retrieve environment group details", err)
}
resp := EndpointGroupResponse{
EndpointGroup: *endpointGroup,
}
if includeSize {
countMap, typeInfoMap := computeGroupSizeInfo([]portainer.EndpointGroup{*endpointGroup}, endpoints)
resp.Total = countMap[groupID]
resp.TypeInfo = typeInfoMap[groupID]
}
return response.JSON(w, resp)
}

Some files were not shown because too many files have changed in this diff Show More