chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
@@ -0,0 +1,172 @@
package sources
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Each test below primes the source cache with a read before mutating, then
// reads again and asserts the change is reflected. The handler is built with a
// non-expiring cache (newTestHandlerNoCacheExpiry), so a fresh result can only
// come from invalidation on the write — not from the TTL elapsing.
func TestGitSourceCreate_InvalidatesCache(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
user := &portainer.User{ID: 1, Role: portainer.AdministratorRole}
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(user)
}))
h := newTestHandlerNoCacheExpiry(t, store)
// prime the cache with the (empty) list
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
require.Empty(t, decodeSources(t, rr))
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Name: "my-source",
})
require.NoError(t, err)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, user.ID, body))
require.Equal(t, http.StatusCreated, rr.Code)
// the new source must appear despite the primed empty cache
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
got := decodeSources(t, rr)
require.Len(t, got, 1)
require.Equal(t, "my-source", got[0].Name)
}
func TestSourceDelete_InvalidatesCache(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
user := &portainer.User{ID: 1, Role: portainer.AdministratorRole}
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "to-delete", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "https://github.com/org/repo"}}
require.NoError(t, tx.Source().Create(adminUserContext, src))
srcID = src.ID
return tx.User().Create(user)
}))
h := newTestHandlerNoCacheExpiry(t, store)
// prime the cache with the source present
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
require.Len(t, decodeSources(t, rr), 1)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReq(t, user.ID, int(srcID)))
require.Equal(t, http.StatusNoContent, rr.Code)
// the deleted source must be gone despite the primed cache
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
require.Empty(t, decodeSources(t, rr))
}
func TestGitSourceUpdate_InvalidatesCache(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
user := &portainer.User{ID: 1, Role: portainer.AdministratorRole}
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "old-name", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "https://github.com/org/repo"}}
require.NoError(t, tx.Source().Create(adminUserContext, src))
srcID = src.ID
return tx.User().Create(user)
}))
h := newTestHandlerNoCacheExpiry(t, store)
// prime the cache with the original name
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
primed := decodeSources(t, rr)
require.Len(t, primed, 1)
require.Equal(t, "old-name", primed[0].Name)
body, err := json.Marshal(GitSourceUpdatePayload{Name: new("new-name")})
require.NoError(t, err)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, user.ID, int(srcID), body))
require.Equal(t, http.StatusOK, rr.Code)
// the updated name must be reflected despite the primed cache
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, user.ID, ""))
got := decodeSources(t, rr)
require.Len(t, got, 1)
require.Equal(t, "new-name", got[0].Name)
}
// TestSourceMutation_InvalidatesSummaryCache verifies the summary endpoint, which
// shares the cache entry with the list via getSources, also sees fresh data.
func TestSourceMutation_InvalidatesSummaryCache(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
user := &portainer.User{ID: 1, Role: portainer.AdministratorRole}
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(user)
}))
h := newTestHandlerNoCacheExpiry(t, store)
// prime the cache with the empty summary
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildSummaryReq(t, user.ID))
require.Equal(t, 0, summaryTotal(t, rr))
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Name: "my-source",
})
require.NoError(t, err)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, user.ID, body))
require.Equal(t, http.StatusCreated, rr.Code)
// the summary count must reflect the new source despite the primed cache
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildSummaryReq(t, user.ID))
assert.Equal(t, 1, summaryTotal(t, rr))
}
func summaryTotal(t *testing.T, rr *httptest.ResponseRecorder) int {
t.Helper()
require.Equal(t, http.StatusOK, rr.Code, "unexpected status: %s", rr.Body.String())
var summary map[string]int
require.NoError(t, json.NewDecoder(rr.Body).Decode(&summary))
total := 0
for _, count := range summary {
total += count
}
return total
}
@@ -0,0 +1,144 @@
package sources
import (
"errors"
"net/http"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
)
// GitAuthenticationPayload holds authentication parameters for a git source
type GitAuthenticationPayload struct {
Username string `json:"username"`
Password string `json:"password"`
}
type SourceAccessControlPayload struct {
Public bool `json:"public" example:"true"`
AdministratorsOnly bool `json:"administratorsOnly" example:"true"`
UserAccesses []portainer.UserID `json:"userAccesses"`
TeamAccesses []portainer.TeamID `json:"teamAccesses"`
}
// GitSourceCreatePayload holds the parameters for creating a git-backed source
type GitSourceCreatePayload struct {
SourceAccessControlPayload
Name string `json:"name"`
URL string `json:"url" validate:"required"`
TLSSkipVerify bool `json:"tlsSkipVerify"`
Authentication *GitAuthenticationPayload `json:"authentication"`
}
// Validate implements the portainer.Validatable interface
func (payload *GitSourceCreatePayload) Validate(_ *http.Request) error {
if !validate.IsURL(payload.URL) {
return errors.New("invalid repository URL. Must correspond to a valid URL format")
}
return nil
}
// @id GitOpsSourcesCreateGit
// @summary Create a Git source
// @description Creates a new GitOps source backed by a Git repository.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body GitSourceCreatePayload true "Git source details"
// @success 201 {object} portainer.Source
// @failure 400 "Invalid request payload"
// @failure 403 "Access denied"
// @failure 409 "A source with this URL and credentials already exists"
// @failure 500 "Server error"
// @router /gitops/sources/git [post]
func (h *Handler) gitSourceCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload GitSourceCreatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
src, err := BuildGitSource(payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.Source().Create(userContext, src)
}); errors.Is(err, source.ErrDuplicateSource) {
return httperror.Conflict("A source with this URL and credentials already exists", err)
} else if err != nil {
return httperror.InternalServerError("Unable to create source", err)
}
if src, err = h.testAndSaveSourceConnection(r.Context(), userContext, src); err != nil {
return httperror.InternalServerError("Unable to persist source status", err)
}
h.invalidateCache()
src.Git = gittypes.SanitizeGitSource(src.Git)
return response.JSONWithStatus(w, src, http.StatusCreated)
}
// BuildGitSource constructs a portainer.Source from a GitSourceCreatePayload
func BuildGitSource(payload GitSourceCreatePayload) (*portainer.Source, error) {
src := BuildBaseGitSource(payload)
src.Git.Authentication = BuildAuth(payload.Authentication)
return src, nil
}
// BuildBaseGitSource constructs the source skeleton (name, URL, TLS, accesses) without authentication.
func BuildBaseGitSource(payload GitSourceCreatePayload) *portainer.Source {
name := payload.Name
if strings.TrimSpace(name) == "" {
name = gittypes.RepoName(payload.URL)
}
return &portainer.Source{
Name: name,
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: payload.URL,
TLSSkipVerify: payload.TLSSkipVerify,
},
UserAccesses: payload.UserAccesses,
TeamAccesses: payload.TeamAccesses,
Public: payload.Public,
AdministratorsOnly: payload.AdministratorsOnly,
}
}
// BuildAuth constructs basic git authentication from the payload, returning nil
// when no authentication is provided.
func BuildAuth(payload *GitAuthenticationPayload) *gittypes.GitAuthentication {
if payload == nil {
return nil
}
return &gittypes.GitAuthentication{
Username: payload.Username,
Password: payload.Password,
}
}
@@ -0,0 +1,291 @@
package sources
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestBuildGitSource_DerivesNameFromURL(t *testing.T) {
t.Parallel()
src, err := BuildGitSource(GitSourceCreatePayload{
URL: "https://github.com/org/my-repo.git",
})
require.NoError(t, err)
require.Equal(t, "my-repo", src.Name)
require.Equal(t, portainer.SourceTypeGit, src.Type)
require.Nil(t, src.Git.Authentication)
}
func TestBuildGitSource_UsesExplicitName(t *testing.T) {
t.Parallel()
src, err := BuildGitSource(GitSourceCreatePayload{
Name: "custom-name",
URL: "https://github.com/org/repo.git",
})
require.NoError(t, err)
require.Equal(t, "custom-name", src.Name)
}
func TestBuildGitSource_WithAuthentication(t *testing.T) {
t.Parallel()
src, err := BuildGitSource(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "alice",
Password: "secret",
},
})
require.NoError(t, err)
require.NotNil(t, src.Git.Authentication)
require.Equal(t, "alice", src.Git.Authentication.Username)
require.Equal(t, "secret", src.Git.Authentication.Password)
}
func TestGitSourceCreatePayload_Validate_EmptyURL(t *testing.T) {
t.Parallel()
err := (&GitSourceCreatePayload{}).Validate(nil)
require.Error(t, err)
}
func TestGitSourceCreatePayload_Validate_ValidURL(t *testing.T) {
t.Parallel()
err := (&GitSourceCreatePayload{URL: "https://github.com/org/repo.git"}).Validate(nil)
require.NoError(t, err)
}
func TestGitSourceCreate_Success(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Name: "my-source",
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusCreated, rr.Code)
var src portainer.Source
err = json.NewDecoder(rr.Body).Decode(&src)
require.NoError(t, err)
require.Equal(t, "my-source", src.Name)
require.Equal(t, portainer.SourceTypeGit, src.Type)
require.NotZero(t, src.ID)
require.NotNil(t, src.Git)
require.Equal(t, "https://github.com/org/repo", src.Git.URL)
}
func TestGitSourceCreate_SanitizesCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "alice",
Password: "secret",
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusCreated, rr.Code)
var src portainer.Source
err = json.NewDecoder(rr.Body).Decode(&src)
require.NoError(t, err)
require.NotNil(t, src.Git)
require.NotNil(t, src.Git.Authentication)
require.Equal(t, "alice", src.Git.Authentication.Username)
require.Empty(t, src.Git.Authentication.Password)
}
func TestGitSourceCreate_MissingURL(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceCreatePayload{Name: "no-url"})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusBadRequest, rr.Code)
}
func TestGitSourceCreate_ConflictOnDuplicateURLAndCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "alice",
Password: "secret",
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusCreated, rr.Code)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusConflict, rr.Code)
}
func TestGitSourceCreate_AllowsDuplicateURLWithDifferentCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
first, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "alice",
Password: "secret",
},
})
require.NoError(t, err)
second, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "bob",
Password: "other",
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, first))
require.Equal(t, http.StatusCreated, rr.Code)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, second))
require.Equal(t, http.StatusCreated, rr.Code)
}
func TestGitSourceCreate_ConflictOnDuplicateAuthlessSource(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusCreated, rr.Code)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, body))
require.Equal(t, http.StatusConflict, rr.Code)
}
func TestGitSourceCreate_AllowsAuthlessAndAuthenticatedSameURL(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
authless, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
})
require.NoError(t, err)
authenticated, err := json.Marshal(GitSourceCreatePayload{
URL: "https://github.com/org/repo.git",
Authentication: &GitAuthenticationPayload{
Username: "alice",
Password: "secret",
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, authless))
require.Equal(t, http.StatusCreated, rr.Code)
rr = httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, authenticated))
require.Equal(t, http.StatusCreated, rr.Code)
}
func TestGitSourceCreate_MalformedJSON(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildCreateReq(t, 1, []byte("not-valid-json{")))
require.Equal(t, http.StatusBadRequest, rr.Code)
}
+97
View File
@@ -0,0 +1,97 @@
package sources
import (
"errors"
"net/http"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
var ErrSourceInUse = errors.New("source is used by one or more workflows or custom templates")
// @id GitOpsSourcesDelete
// @summary Delete a source
// @description Deletes an existing GitOps source. Returns 409 if the source is referenced by any workflow or custom template.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @param id path int true "Source identifier"
// @success 204 "Source deleted"
// @failure 400 "Invalid request"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 409 "Source is in use by one or more workflows or custom templates"
// @failure 500 "Server error"
// @router /gitops/sources/{id} [delete]
func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
sourceID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if exists, err := tx.Source().Exists(userContext, portainer.SourceID(sourceID)); err != nil {
return err
} else if !exists {
return dserrors.ErrObjectNotFound
}
workflows, err := tx.Workflow().ReadAll()
if err != nil {
return err
}
for _, wf := range workflows {
if slices.ContainsFunc(wf.Artifacts, func(as portainer.Artifact) bool {
return slices.ContainsFunc(as.Files, func(f portainer.ArtifactFile) bool {
return f.SourceID == portainer.SourceID(sourceID)
})
}) {
return ErrSourceInUse
}
}
templates, err := tx.CustomTemplate().ReadAll(func(t portainer.CustomTemplate) bool {
return t.Artifact != nil && slices.ContainsFunc(t.Artifact.Files, func(f portainer.ArtifactFile) bool {
return f.SourceID == portainer.SourceID(sourceID)
})
})
if err != nil {
return err
}
if len(templates) > 0 {
return ErrSourceInUse
}
return tx.Source().Delete(userContext, portainer.SourceID(sourceID))
}); h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a source with the specified identifier", err)
} else if errors.Is(err, ErrSourceInUse) {
return httperror.Conflict("Source is used by one or more workflows or custom templates", err)
} else if errors.Is(err, source.ErrNotEnoughPermission) {
return httperror.Forbidden("Not enough permissions to delete source", err)
} else if err != nil {
return httperror.InternalServerError("Unable to delete source", err)
}
h.invalidateCache()
return response.Empty(w)
}
@@ -0,0 +1,125 @@
package sources
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/require"
)
func TestSourceDelete_Success(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "to-delete", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID)))
require.Equal(t, http.StatusNoContent, rr.Code)
}
func TestSourceDelete_NotFound(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReq(t, 1, 99))
require.Equal(t, http.StatusNotFound, rr.Code)
}
func TestSourceDelete_InUse(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "in-use", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
err = tx.Workflow().Create(wf)
require.NoError(t, err)
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID)))
require.Equal(t, http.StatusConflict, rr.Code)
}
func TestSourceDelete_NonNumericID(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReqWithRawID(t, 1, "not-a-number"))
require.Equal(t, http.StatusBadRequest, rr.Code)
}
func TestSourceDelete_InUseByCustomTemplate(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "in-use-by-template", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
ct := &portainer.CustomTemplate{
ID: 1,
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
},
}
err = tx.CustomTemplate().Create(ct)
require.NoError(t, err)
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID)))
require.Equal(t, http.StatusConflict, rr.Code)
}
+77
View File
@@ -0,0 +1,77 @@
package sources
import (
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
ce "github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/set"
"github.com/portainer/portainer/api/slicesx"
)
// FetchSourceWorkflows returns the workflows and stats for a single source.
func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([]ce.Workflow, ce.SourceStats, error) {
wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool {
return slices.ContainsFunc(wf.Artifacts, func(artifact portainer.Artifact) bool {
return slices.ContainsFunc(artifact.Files, func(f portainer.ArtifactFile) bool {
return f.SourceID == src.ID
})
})
})
if err != nil {
return nil, ce.SourceStats{}, err
}
if len(wfs) == 0 {
return nil, ce.SourceStats{}, nil
}
wfIDSet := set.ToSet(slicesx.Map(wfs, func(wf portainer.Workflow) portainer.WorkflowID { return wf.ID }))
stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool {
_, ok := wfIDSet[s.WorkflowID]
return ok
})
if err != nil {
return nil, ce.SourceStats{}, err
}
artifactByStack := make(map[portainer.StackID]portainer.ArtifactFile)
for _, wf := range wfs {
for _, artifact := range wf.Artifacts {
if artifact.StackID == 0 {
continue
}
if _, exists := artifactByStack[artifact.StackID]; exists {
continue
}
for _, file := range artifact.Files {
if file.SourceID == src.ID {
artifactByStack[artifact.StackID] = file
break
}
}
}
}
unknown := ce.WorkflowPhaseStatus{Status: ce.StatusUnknown}
items := make([]ce.Workflow, 0, len(stacks))
stats := ce.SourceStats{EndpointIDs: set.Set[portainer.EndpointID]{}}
for _, stack := range stacks {
cfg := src.Git.ToRepoConfig()
if file, ok := artifactByStack[stack.ID]; ok {
cfg.ReferenceName = file.Ref
cfg.ConfigFilePath = file.Path
cfg.ConfigHash = file.Hash
}
items = append(items, ce.MapStackToWorkflow(stack, src.ID, cfg, unknown, unknown))
stats.WorkflowCount++
if stack.EndpointID != 0 {
stats.EndpointIDs.Add(stack.EndpointID)
}
}
return items, stats, nil
}
+179
View File
@@ -0,0 +1,179 @@
package sources
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
sourceDS "github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type gitAuthInfo struct {
Username string `json:"username"`
}
type connectionInfo struct {
TLSSkipVerify bool `json:"tlsSkipVerify"`
Authentication *gitAuthInfo `json:"authentication,omitempty"`
}
type AutoUpdateInfo struct {
Mechanism string `json:"mechanism,omitempty"`
FetchInterval string `json:"fetchInterval,omitempty"`
}
type SourceAccess struct {
Public bool `json:"public,omitempty"`
Users []portainer.UserID `json:"users,omitempty"`
Teams []portainer.TeamID `json:"teams,omitempty"`
}
// SourceDetail extends Source with connection settings and linked workflows.
type SourceDetail struct {
Source
Connection connectionInfo `json:"connection" validate:"required"`
AutoUpdate *AutoUpdateInfo `json:"autoUpdate,omitempty"`
Workflows []workflows.Workflow `json:"workflows"`
Access SourceAccess `json:"access"`
}
// @id GitOpsSourceGet
// @summary Get a GitOps source by ID
// @description Returns a single GitOps source with its connection settings and linked workflows.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Source identifier"
// @success 200 {object} SourceDetail
// @failure 400 "Invalid request"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 500 "Server error"
// @router /gitops/sources/{id} [get]
func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
srcID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
sourceID := portainer.SourceID(srcID)
var source *portainer.Source
var sourceWfs []workflows.Workflow
var stats workflows.SourceStats
err = h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
userContext := sourceDS.NewUserContext(securityContext.User, securityContext.UserMemberships)
source, err = tx.Source().Read(userContext, sourceID)
if err != nil {
return err
}
sourceWfs, stats, err = FetchSourceWorkflows(tx, source)
return err
})
if h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Source not found", err)
} else if errors.Is(err, sourceDS.ErrNotEnoughPermission) {
return httperror.Forbidden("Not enough permissions to retrieve source", err)
} else if err != nil {
return httperror.InternalServerError("Unable to retrieve source", err)
}
access := BuildSourceAccess(source)
detail := BuildSourceDetail(h.buildSource(source, stats), source.Git, sourceWfs, access)
return response.JSON(w, detail)
}
func BuildSourceDetail(baseSource Source, cfg *gittypes.GitSource, sourceWfs []workflows.Workflow, access SourceAccess) SourceDetail {
var autoUpdate *AutoUpdateInfo
if len(sourceWfs) > 0 {
autoUpdate = BuildAutoUpdateInfo(sourceWfs[0].AutoUpdate)
}
return SourceDetail{
Source: baseSource,
Connection: buildConnectionInfo(cfg),
AutoUpdate: autoUpdate,
Workflows: redactWorkflowCredentials(sourceWfs),
Access: access,
}
}
func BuildSourceAccess(source *portainer.Source) SourceAccess {
if source == nil {
return SourceAccess{}
}
if source.AdministratorsOnly {
return SourceAccess{}
}
if source.Public {
return SourceAccess{
Public: true,
}
}
return SourceAccess{
Public: source.Public,
Users: source.UserAccesses,
Teams: source.TeamAccesses,
}
}
func buildConnectionInfo(cfg *gittypes.GitSource) connectionInfo {
if cfg == nil {
return connectionInfo{}
}
return connectionInfo{
TLSSkipVerify: cfg.TLSSkipVerify,
Authentication: buildGitAuthInfo(cfg.Authentication),
}
}
func buildGitAuthInfo(auth *gittypes.GitAuthentication) *gitAuthInfo {
if auth == nil {
return nil
}
return &gitAuthInfo{
Username: auth.Username,
}
}
func BuildAutoUpdateInfo(autoUpdate *portainer.AutoUpdateSettings) *AutoUpdateInfo {
if autoUpdate == nil {
return nil
}
switch {
case autoUpdate.Interval != "":
return &AutoUpdateInfo{
Mechanism: "Interval",
FetchInterval: autoUpdate.Interval,
}
case autoUpdate.Webhook != "":
return &AutoUpdateInfo{
Mechanism: "Webhook",
}
default:
return nil
}
}
+117
View File
@@ -0,0 +1,117 @@
package sources
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetSource_NotFound(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, "999"))
assert.Equal(t, http.StatusNotFound, rr.Code)
}
func TestGetSource_ReturnsDetail(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := &gittypes.GitSource{
URL: "https://github.com/org/repo",
TLSSkipVerify: true,
}
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1, Name: "my-stack"}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID))))
detail := decodeSourceDetail(t, rr)
assert.Equal(t, srcID, detail.ID)
assert.Equal(t, "repo", detail.Name)
assert.Equal(t, 1, detail.UsedBy)
assert.True(t, detail.Connection.TLSSkipVerify)
require.Len(t, detail.Workflows, 1)
assert.Equal(t, "my-stack", detail.Workflows[0].Name)
}
func TestGetSource_RedactsCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := &gittypes.GitSource{
URL: "https://github.com/org/secure",
Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"},
}
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1, Name: "secure-stack"}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID))))
detail := decodeSourceDetail(t, rr)
require.Len(t, detail.Workflows, 1)
require.NotNil(t, detail.Workflows[0].GitConfig)
require.NotNil(t, detail.Workflows[0].GitConfig.Authentication)
assert.Equal(t, "user", detail.Workflows[0].GitConfig.Authentication.Username)
assert.Empty(t, detail.Workflows[0].GitConfig.Authentication.Password)
}
func TestGetSource_AutoUpdate(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := gitCfg("https://github.com/org/polled")
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{
ID: 1,
Name: "polled-stack",
AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"},
}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID))))
detail := decodeSourceDetail(t, rr)
require.NotNil(t, detail.AutoUpdate)
assert.Equal(t, "Interval", detail.AutoUpdate.Mechanism)
assert.Equal(t, "5m", detail.AutoUpdate.FetchInterval)
}
@@ -0,0 +1,62 @@
package sources
import (
"net/http"
"time"
gocache "github.com/patrickmn/go-cache"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/gorilla/mux"
)
const (
cacheTTL = 30 * time.Second
cacheCleanupInterval = 10 * time.Minute
)
// Handler is the HTTP handler for the GitOps sources API.
type Handler struct {
*mux.Router
dataStore dataservices.DataStore
gitService portainer.GitService
cache *gocache.Cache
k8sFactory *cli.ClientFactory
}
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, gitService portainer.GitService, k8sFactory *cli.ClientFactory) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dataStore: dataStore,
gitService: gitService,
cache: gocache.New(cacheTTL, cacheCleanupInterval),
k8sFactory: k8sFactory,
}
authenticatedRouter := h.PathPrefix("/gitops/sources").Subrouter()
authenticatedRouter.Use(bouncer.AuthenticatedAccess)
authenticatedRouter.Handle("", httperror.LoggerHandler(h.list)).Methods(http.MethodGet)
authenticatedRouter.Handle("/summary", httperror.LoggerHandler(h.summary)).Methods(http.MethodGet)
authenticatedRouter.Handle("/{id}", httperror.LoggerHandler(h.getSource)).Methods(http.MethodGet)
authenticatedRouter.Handle("/git", httperror.LoggerHandler(h.gitSourceCreate)).Methods(http.MethodPost)
authenticatedRouter.Handle("/test", httperror.LoggerHandler(h.gitSourceTest)).Methods(http.MethodPost)
authenticatedRouter.Handle("/{id}", httperror.LoggerHandler(h.gitSourceUpdate)).Methods(http.MethodPut)
authenticatedRouter.Handle("/{id}", httperror.LoggerHandler(h.sourceDelete)).Methods(http.MethodDelete)
authenticatedRouter.Handle("/{id}/test", httperror.LoggerHandler(h.sourceTestConnection)).Methods(http.MethodPost)
adminRouter := h.PathPrefix("/gitops/sources").Subrouter()
adminRouter.Use(bouncer.AdminAccess)
adminRouter.Handle("/{id}/access", httperror.LoggerHandler(h.gitSourceUpdateAccess)).Methods(http.MethodPut)
return h
}
// invalidateCache clears the cached source lists so the next read reflects
// the latest datastore state. Called after any mutating operation.
func (h *Handler) invalidateCache() {
h.cache.Flush()
}
@@ -0,0 +1,150 @@
package sources
import (
"bytes"
"fmt"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
gocache "github.com/patrickmn/go-cache"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
var adminUserContext = source.InsecureNewAdminContext()
// createGitWorkflow creates a Source and Workflow for the given config and
// wires them up by setting stack.WorkflowID before creating the stack.
func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack, cfg *gittypes.GitSource) portainer.SourceID {
t.Helper()
src := &portainer.Source{
Name: gittypes.RepoName(cfg.URL),
Type: portainer.SourceTypeGit,
Git: cfg,
}
require.NoError(t, tx.Source().Create(adminUserContext, src))
wf := &portainer.Workflow{
Artifacts: []portainer.Artifact{{
StackID: stack.ID,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}},
}
require.NoError(t, tx.Workflow().Create(wf))
stack.WorkflowID = wf.ID
return src.ID
}
func newTestHandler(t *testing.T, store dataservices.DataStore) *Handler {
t.Helper()
return NewHandler(testhelpers.NewTestRequestBouncer(), store, nil, nil)
}
// newTestHandlerNoCacheExpiry returns a handler whose source cache never expires,
// so cache-invalidation tests can prove a write clears the cache rather than the
// TTL simply elapsing between requests.
func newTestHandlerNoCacheExpiry(t *testing.T, store dataservices.DataStore) *Handler {
t.Helper()
h := newTestHandler(t, store)
h.cache = gocache.New(gocache.NoExpiration, cacheCleanupInterval)
return h
}
func adminRestrictedContext(userID portainer.UserID) *security.RestrictedRequestContext {
return &security.RestrictedRequestContext{
UserID: userID,
IsAdmin: true,
User: &portainer.User{ID: userID, Role: portainer.AdministratorRole},
}
}
// withSecurityContext attaches the admin token data and restricted request
// context for userID to req, mirroring what the auth middleware sets up in
// production.
func withSecurityContext(req *http.Request, userID portainer.UserID) *http.Request {
req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID}))
req = req.WithContext(security.StoreRestrictedRequestContext(req, adminRestrictedContext(userID)))
return req
}
func buildListReq(t *testing.T, userID portainer.UserID, query string) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/gitops/sources?"+query, nil)
return withSecurityContext(req, userID)
}
func buildGetReq(t *testing.T, userID portainer.UserID, id string) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/gitops/sources/"+id, nil)
return withSecurityContext(req, userID)
}
func decodeSources(t *testing.T, rr *httptest.ResponseRecorder) []Source {
t.Helper()
require.Equal(t, http.StatusOK, rr.Code, "unexpected status: %s", rr.Body.String())
var items []Source
require.NoError(t, json.NewDecoder(rr.Body).Decode(&items))
return items
}
func decodeSourceDetail(t *testing.T, rr *httptest.ResponseRecorder) SourceDetail {
t.Helper()
require.Equal(t, http.StatusOK, rr.Code, "unexpected status: %s", rr.Body.String())
var item SourceDetail
require.NoError(t, json.NewDecoder(rr.Body).Decode(&item))
return item
}
func gitCfg(url string) *gittypes.GitSource {
return &gittypes.GitSource{URL: url}
}
func buildCreateReq(t *testing.T, userID portainer.UserID, body []byte) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/gitops/sources/git", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
return withSecurityContext(req, userID)
}
func buildUpdateReq(t *testing.T, userID portainer.UserID, id int, body []byte) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/gitops/sources/%d", id), bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
return withSecurityContext(req, userID)
}
func buildDeleteReq(t *testing.T, userID portainer.UserID, id int) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodDelete, fmt.Sprintf("/gitops/sources/%d", id), nil)
return withSecurityContext(req, userID)
}
func buildSummaryReq(t *testing.T, userID portainer.UserID) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/gitops/sources/summary", nil)
return withSecurityContext(req, userID)
}
func buildUpdateReqWithRawID(t *testing.T, userID portainer.UserID, id string, body []byte) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodPut, "/gitops/sources/"+id, bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
return withSecurityContext(req, userID)
}
func buildDeleteReqWithRawID(t *testing.T, userID portainer.UserID, id string) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodDelete, "/gitops/sources/"+id, nil)
return withSecurityContext(req, userID)
}
+135
View File
@@ -0,0 +1,135 @@
package sources
import (
"context"
"net/http"
"slices"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/http/utils/filters"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
gocache "github.com/patrickmn/go-cache"
)
// @id GitOpsSourcesList
// @summary List all GitOps sources
// @description Returns a deduplicated list of git repositories used across all GitOps workflows.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param search query string false "Search term (matches URL)"
// @param sort query string false "Sort field: name | status | type"
// @param order query string false "Sort order: asc or desc"
// @param start query int false "Pagination start index"
// @param limit query int false "Pagination limit (0 = unlimited)"
// @param status query string false "Filter by status: healthy | syncing | error | paused | unknown"
// @param type query SourceType false "Filter by source type: git | oci | helm"
// @success 200 {array} Source
// @failure 400 "Invalid status parameter"
// @failure 403 "Access denied"
// @failure 500 "Server error"
// @router /gitops/sources [get]
func (h *Handler) list(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
params := filters.ExtractListModifiersQueryParams(r)
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
key := cacheKey(securityContext)
sources, err := h.getSources(r.Context(), key, securityContext)
if err != nil {
return httperror.InternalServerError("Unable to retrieve sources", err)
}
if status, _ := request.RetrieveQueryParameter(r, "status", true); status != "" {
s, err := workflows.ParseStatus(status)
if err != nil {
return httperror.BadRequest("Invalid status parameter", err)
}
sources = slicesx.FilterInPlace(sources, func(i Source) bool { return i.Status == s })
}
if sourceType, _ := request.RetrieveQueryParameter(r, "type", true); sourceType != "" {
t, err := parseSourceType(sourceType)
if err != nil {
return httperror.BadRequest("Invalid type parameter", err)
}
sources = slicesx.FilterInPlace(sources, func(i Source) bool { return i.Type == t })
}
results := filters.SearchOrderAndPaginate(sources, params, filters.Config[Source]{
SearchAccessors: []filters.SearchAccessor[Source]{
func(s Source) (string, error) { return s.URL, nil },
},
SortBindings: []filters.SortBinding[Source]{
{Key: "name", Fn: func(a, b Source) int { return strings.Compare(a.Name, b.Name) }},
{Key: "status", Fn: func(a, b Source) int { return strings.Compare(string(a.Status), string(b.Status)) }},
{Key: "type", Fn: func(a, b Source) int { return strings.Compare(string(a.Type), string(b.Type)) }},
},
})
filters.ApplyFilterResultsHeaders(&w, results)
return response.JSON(w, results.Items)
}
func (h *Handler) getSources(ctx context.Context, key string, sc *security.RestrictedRequestContext) ([]Source, error) {
if cached, ok := h.cache.Get(key); ok {
return slices.Clone(cached.([]Source)), nil
}
result, err := h.fetchSources(ctx, sc)
if err != nil {
return nil, err
}
h.cache.Set(key, result, gocache.DefaultExpiration)
return slices.Clone(result), nil
}
func cacheKey(sc *security.RestrictedRequestContext) string {
teamIDs := make([]string, len(sc.UserMemberships))
for i, membership := range sc.UserMemberships {
teamIDs[i] = strconv.Itoa(int(membership.TeamID))
}
slices.Sort(teamIDs)
return strconv.Itoa(int(sc.UserID)) + ":" + strconv.FormatBool(sc.IsAdmin) + ":" + strings.Join(teamIDs, ",")
}
func (h *Handler) fetchSources(ctx context.Context, sc *security.RestrictedRequestContext) ([]Source, error) {
var allSrcs []portainer.Source
var stats map[portainer.SourceID]workflows.SourceStats
if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
allSrcs, stats, err = workflows.FetchSourceStats(tx, h.k8sFactory, sc)
return err
}); err != nil {
return nil, err
}
result := make([]Source, 0, len(allSrcs))
for _, src := range allSrcs {
stat, ok := stats[src.ID]
if !ok {
stat = workflows.SourceStats{}
}
result = append(result, h.buildSource(&src, stat))
}
return result, nil
}
@@ -0,0 +1,127 @@
package sources
import (
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSourcesList_GroupsByURLAndCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
cfg := gitCfg("https://github.com/org/repo")
src := &portainer.Source{Name: "repo", Type: portainer.SourceTypeGit, Git: cfg}
require.NoError(t, tx.Source().Create(adminUserContext, src))
wfA := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
require.NoError(t, tx.Workflow().Create(wfA))
wfB := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
require.NoError(t, tx.Workflow().Create(wfB))
require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "stack-a", WorkflowID: wfA.ID}))
require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "stack-b", WorkflowID: wfB.ID}))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, 1, ""))
sources := decodeSources(t, rr)
require.Len(t, sources, 1)
assert.Equal(t, 2, sources[0].UsedBy)
}
func TestSourcesList_SeparatesCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
cfg1 := gitCfg("https://github.com/org/repo")
cfg1.Authentication = &gittypes.GitAuthentication{Username: "alice", Password: "pass1"}
cfg2 := gitCfg("https://github.com/org/repo")
cfg2.Authentication = &gittypes.GitAuthentication{Username: "bob", Password: "pass2"}
stackA := &portainer.Stack{ID: 1, Name: "stack-a"}
createGitWorkflow(t, tx, stackA, cfg1)
require.NoError(t, tx.Stack().Create(stackA))
stackB := &portainer.Stack{ID: 2, Name: "stack-b"}
createGitWorkflow(t, tx, stackB, cfg2)
require.NoError(t, tx.Stack().Create(stackB))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, 1, ""))
sources := decodeSources(t, rr)
assert.Len(t, sources, 2)
}
func TestSourcesList_StatusFilter(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
// With nil gitService, source git-phase status is always StatusUnknown.
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1}
createGitWorkflow(t, tx, stack, gitCfg("https://github.com/org/app"))
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
t.Run("status=unknown matches sources with unknown status", func(t *testing.T) {
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, 1, "status=unknown"))
sources := decodeSources(t, rr)
assert.Len(t, sources, 1)
})
t.Run("status=healthy excludes sources with unknown status", func(t *testing.T) {
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, 1, "status=healthy"))
sources := decodeSources(t, rr)
assert.Empty(t, sources)
})
}
func TestSourcesList_SearchByURL(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stackA := &portainer.Stack{ID: 1}
createGitWorkflow(t, tx, stackA, gitCfg("https://github.com/org/app"))
require.NoError(t, tx.Stack().Create(stackA))
stackB := &portainer.Stack{ID: 2}
createGitWorkflow(t, tx, stackB, gitCfg("https://github.com/org/infra"))
require.NoError(t, tx.Stack().Create(stackB))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildListReq(t, 1, "search=app"))
sources := decodeSources(t, rr)
require.Len(t, sources, 1)
assert.Equal(t, "app", sources[0].Name)
}
@@ -0,0 +1,162 @@
package sources
import (
"context"
"errors"
"io"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id GitOpsSourcesTestById
// @summary Test the connection of a stored source
// @description Tests connectivity for a GitOps source, applying optional overrides to the stored configuration.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Source identifier"
// @param body body GitSourceUpdatePayload false "Optional connection overrides; omitted fields fall back to stored values"
// @success 200 {object} ConnectionTestResult "Connection test result"
// @failure 400 "Invalid request payload"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 500 "Server error"
// @router /gitops/sources/{id}/test [post]
func (h *Handler) sourceTestConnection(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
sourceID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
var payload GitSourceUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil && !errors.Is(err, io.EOF) {
return httperror.BadRequest("Invalid request payload", err)
}
var src *portainer.Source
if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
src, err = tx.Source().Read(userContext, portainer.SourceID(sourceID))
return err
}); h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a source with the specified identifier", err)
} else if errors.Is(err, source.ErrNotEnoughPermission) {
return httperror.Forbidden("Not enough permissions to retrieve source", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find source", err)
}
if err := ApplyGitSourceChanges(src, payload); errors.Is(err, ErrNotGitSource) {
return httperror.BadRequest("Source is not a Git source", err)
} else if err != nil {
return httperror.InternalServerError("Unable to apply source changes", err)
}
if src.Git == nil {
return httperror.InternalServerError("Source has no git configuration", nil)
}
result := testSourceConnection(r.Context(), h.gitService, src.Git)
return response.JSON(w, result)
}
type ConnectionTestResult struct {
Success bool `json:"success"`
Error string `json:"error,omitempty"`
}
// @id GitOpsSourcesTest
// @summary Test a Git source connection
// @description Tests connectivity for Git connection details that have not been persisted yet.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param body body GitSourceCreatePayload true "Git connection details"
// @success 200 {object} ConnectionTestResult "Connection test result"
// @failure 400 "Invalid request payload"
// @failure 403 "Access denied"
// @failure 500 "Server error"
// @router /gitops/sources/test [post]
func (h *Handler) gitSourceTest(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload GitSourceCreatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
src, err := BuildGitSource(payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
if src.Git == nil {
return httperror.InternalServerError("Source has no git configuration", nil)
}
result := testSourceConnection(r.Context(), h.gitService, src.Git)
return response.JSON(w, result)
}
// testSourceConnection verifies that a git repository is reachable with the given config.
func testSourceConnection(ctx context.Context, gitService portainer.GitService, config *gittypes.GitSource) ConnectionTestResult {
var username, password string
if config.Authentication != nil {
username = config.Authentication.Username
password = config.Authentication.Password
}
_, err := gitService.ListRefs(ctx, config.URL, username, password, false, config.TLSSkipVerify)
if err != nil {
return ConnectionTestResult{Success: false, Error: err.Error()}
}
return ConnectionTestResult{Success: true}
}
func (h *Handler) testAndSaveSourceConnection(ctx context.Context, userContext source.UserContext, src *portainer.Source) (*portainer.Source, error) {
if h.gitService == nil || src.Git == nil {
return src, nil
}
result := testSourceConnection(ctx, h.gitService, src.Git)
var checkErr error
if !result.Success {
checkErr = errors.New(result.Error)
}
var updated *portainer.Source
if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := workflows.SaveSourceStatus(tx, userContext, src.ID, checkErr); err != nil {
return err
}
var readErr error
updated, readErr = tx.Source().Read(userContext, src.ID)
return readErr
}); err != nil {
return nil, err
}
return updated, nil
}
@@ -0,0 +1,54 @@
package sources
import (
"net/http"
ce "github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id GitOpsSourcesSummary
// @summary Summarize GitOps source status counts
// @description Returns a count of sources per status.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @produce json
// @success 200 {object} ce.StatusSummary
// @failure 403 "Access denied"
// @failure 500 "Server error"
// @router /gitops/sources/summary [get]
func (h *Handler) summary(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
key := cacheKey(securityContext)
sources, err := h.getSources(r.Context(), key, securityContext)
if err != nil {
return httperror.InternalServerError("Unable to retrieve sources", err)
}
summary := ce.StatusSummary{}
for _, s := range sources {
switch s.Status {
case ce.StatusHealthy:
summary.Healthy++
case ce.StatusSyncing:
summary.Syncing++
case ce.StatusError:
summary.Error++
case ce.StatusPaused:
summary.Paused++
default:
summary.Unknown++
}
}
return response.JSON(w, summary)
}
@@ -0,0 +1,66 @@
package sources
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
ceWorkflows "github.com/portainer/portainer/api/gitops/workflows"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestSourcesSummary_Empty(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildSummaryReq(t, 1))
require.Equal(t, http.StatusOK, rr.Code)
var summary ceWorkflows.StatusSummary
err := json.NewDecoder(rr.Body).Decode(&summary)
require.NoError(t, err)
require.Equal(t, ceWorkflows.StatusSummary{}, summary)
}
func TestSourcesSummary_CountsByStatus(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
for idx, name := range []string{"source-a", "source-b", "source-c"} {
err := tx.Source().Create(adminUserContext, &portainer.Source{Name: name, Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: fmt.Sprintf("http://github.com/org/repo%d", idx)}})
require.NoError(t, err)
}
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildSummaryReq(t, 1))
require.Equal(t, http.StatusOK, rr.Code)
var summary ceWorkflows.StatusSummary
err := json.NewDecoder(rr.Body).Decode(&summary)
require.NoError(t, err)
require.Equal(t, 3, summary.Unknown)
require.Zero(t, summary.Healthy)
require.Zero(t, summary.Error)
require.Zero(t, summary.Syncing)
require.Zero(t, summary.Paused)
}
+51
View File
@@ -0,0 +1,51 @@
package sources
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/gitops/workflows"
)
// Source represents a unique git repository used as a GitOps source across one or more workflows.
type Source struct {
ID portainer.SourceID `json:"id" validate:"required"`
Name string `json:"name" validate:"required"`
Type SourceType `json:"type" validate:"required"`
URL string `json:"url" validate:"required"`
Status workflows.Status `json:"status" validate:"required"`
Error string `json:"error,omitempty"`
UsedBy int `json:"usedBy"`
Environments int `json:"environments"`
LastSync int64 `json:"lastSync"`
}
type SourceType string
const (
SourceTypeGit SourceType = "git"
SourceTypeHelm SourceType = "helm"
SourceTypeOCI SourceType = "oci"
)
func parseSourceType(s string) (SourceType, error) {
switch SourceType(s) {
case SourceTypeGit, SourceTypeHelm, SourceTypeOCI:
return SourceType(s), nil
default:
return "", fmt.Errorf("invalid source type %q: must be git, helm, or oci", s)
}
}
func sourceTypeString(t portainer.SourceType) SourceType {
switch t {
case portainer.SourceTypeGit:
return SourceTypeGit
case portainer.SourceTypeHelm:
return SourceTypeHelm
case portainer.SourceTypeRegistry:
return SourceTypeOCI
default:
return SourceTypeGit
}
}
@@ -0,0 +1,103 @@
package sources
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
type SourceAccessUpdatePayload struct {
Public bool `json:"public"`
Users []portainer.UserID `json:"users,omitempty"`
Teams []portainer.TeamID `json:"teams,omitempty"`
}
// @id GitOpsSourcesUpdateAccess
// @summary Update a GitOps source's access control
// @description Updates the access control settings for an existing GitOps source.
// @description **Access policy**: administrator
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Source identifier"
// @param body body SourceAccessUpdatePayload true "Source access control"
// @success 200 {object} portainer.Source
// @failure 400 "Invalid request payload"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 500 "Server error"
// @router /gitops/sources/{id}/access [put]
func (h *Handler) gitSourceUpdateAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
id, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
var payload SourceAccessUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
sourceID := portainer.SourceID(id)
var src *portainer.Source
if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
var err error
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if src, err = tx.Source().Read(userContext, sourceID); err != nil {
return err
}
ApplySourceAccessChanges(src, payload)
return tx.Source().Update(userContext, src.ID, src)
}); h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a source with the specified identifier", err)
} else if err != nil {
return httperror.InternalServerError("Unable to update source access", err)
}
h.invalidateCache()
return response.JSON(w, src)
}
// Validate implements the portainer.Validatable interface
func (payload *SourceAccessUpdatePayload) Validate(_ *http.Request) error {
return nil
}
// ApplySourceAccessChanges applies the payload access changes to the source in place.
func ApplySourceAccessChanges(src *portainer.Source, payload SourceAccessUpdatePayload) {
src.Public = payload.Public
if payload.Public {
src.AdministratorsOnly = false
src.UserAccesses = []portainer.UserID{}
src.TeamAccesses = []portainer.TeamID{}
} else if len(payload.Users) == 0 && len(payload.Teams) == 0 {
src.AdministratorsOnly = true
src.UserAccesses = []portainer.UserID{}
src.TeamAccesses = []portainer.TeamID{}
} else {
src.AdministratorsOnly = false
src.UserAccesses = payload.Users
src.TeamAccesses = payload.Teams
}
}
@@ -0,0 +1,187 @@
package sources
import (
"errors"
"net/http"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
)
var (
ErrNotGitSource = errors.New("source is not a Git source")
)
// GitSourceUpdatePayload holds the parameters for updating a git-backed source
type GitSourceUpdatePayload struct {
Name *string `json:"name"`
URL *string `json:"url"`
TLSSkipVerify *bool `json:"tlsSkipVerify"`
Authentication *GitAuthenticationUpdatePayload `json:"authentication"`
}
type GitAuthenticationUpdatePayload struct {
Username *string `json:"username"`
Password *string `json:"password"`
}
// Validate implements the portainer.Validatable interface
func (payload *GitSourceUpdatePayload) Validate(_ *http.Request) error {
if payload.URL != nil && !validate.IsURL(*payload.URL) {
return errors.New("invalid repository URL. Must correspond to a valid URL format")
}
return nil
}
// @id GitOpsSourcesUpdateGit
// @summary Update a Git source
// @description Updates an existing GitOps source backed by a Git repository.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @accept json
// @produce json
// @param id path int true "Source identifier"
// @param body body GitSourceUpdatePayload true "Git source details"
// @success 200 {object} portainer.Source
// @failure 400 "Invalid request payload"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 409 "A source with this URL and credentials already exists"
// @failure 500 "Server error"
// @router /gitops/sources/{id} [put]
func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
id, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
var payload GitSourceUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
sourceID := portainer.SourceID(id)
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
var src *portainer.Source
if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
var err error
if src, err = tx.Source().Read(userContext, sourceID); err != nil {
return err
}
if err := ApplyGitSourceChanges(src, payload); err != nil {
return err
}
src.Status = portainer.SourceStatusUnknown
src.StatusError = ""
return tx.Source().Update(userContext, src.ID, src)
}); h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a source with the specified identifier", err)
} else if errors.Is(err, ErrNotGitSource) {
return httperror.BadRequest("Source is not a Git source", err)
} else if errors.Is(err, source.ErrNotEnoughPermission) {
return httperror.Forbidden("Not enough permissions to update source", err)
} else if errors.Is(err, source.ErrDuplicateSource) {
return httperror.Conflict("A source with this URL and credentials already exists", err)
} else if err != nil {
return httperror.InternalServerError("Unable to update source", err)
}
if src, err = h.testAndSaveSourceConnection(r.Context(), userContext, src); err != nil {
return httperror.InternalServerError("Unable to persist source status", err)
}
h.invalidateCache()
src.Git = gittypes.SanitizeGitSource(src.Git)
return response.JSON(w, src)
}
// ApplyGitSourceChanges applies the payload changes to the source in place
func ApplyGitSourceChanges(src *portainer.Source, payload GitSourceUpdatePayload) error {
if err := ApplyBaseGitSourceChanges(src, payload); err != nil {
return err
}
if payload.Authentication == nil {
return nil
}
if *payload.Authentication == (GitAuthenticationUpdatePayload{}) {
src.Git.Authentication = nil
return nil
}
src.Git.Authentication = ApplyAuthChanges(src.Git.Authentication, *payload.Authentication)
return nil
}
// ApplyBaseGitSourceChanges applies the non-authentication field changes (name,
// URL, reference, TLS) to the source in place, ensuring src.Git is set
func ApplyBaseGitSourceChanges(src *portainer.Source, payload GitSourceUpdatePayload) error {
if src.Type != portainer.SourceTypeGit {
return ErrNotGitSource
}
if payload.Name != nil && strings.TrimSpace(*payload.Name) != "" {
src.Name = *payload.Name
}
if src.Git == nil {
src.Git = &gittypes.GitSource{}
}
if payload.URL != nil {
src.Git.URL = *payload.URL
}
if payload.TLSSkipVerify != nil {
src.Git.TLSSkipVerify = *payload.TLSSkipVerify
}
return nil
}
// ApplyAuthChanges returns a copy of the existing authentication (or a fresh
// one) with the basic credential changes applied.
func ApplyAuthChanges(existing *gittypes.GitAuthentication, payload GitAuthenticationUpdatePayload) *gittypes.GitAuthentication {
auth := &gittypes.GitAuthentication{}
if existing != nil {
copied := *existing
auth = &copied
}
if payload.Username != nil {
auth.Username = *payload.Username
}
if payload.Password != nil {
auth.Password = *payload.Password
}
return auth
}
@@ -0,0 +1,347 @@
package sources
import (
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestGitSourceUpdate_Success(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "old-name", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{
URL: new("https://github.com/org/new.git"),
Name: new("new-name"),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusOK, rr.Code)
var src portainer.Source
err = json.NewDecoder(rr.Body).Decode(&src)
require.NoError(t, err)
require.Equal(t, "new-name", src.Name)
require.NotNil(t, src.Git)
require.Equal(t, "https://github.com/org/new", src.Git.URL)
}
func TestGitSourceUpdate_PreservesAuthWhenNotProvided(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Name: "auth-source",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/org/repo.git",
Authentication: &gittypes.GitAuthentication{
Username: "alice",
Password: "secret",
},
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{
URL: new("https://github.com/org/repo.git"),
Name: new("renamed"),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusOK, rr.Code)
var stored *portainer.Source
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
stored, err = tx.Source().Read(adminUserContext, srcID)
return err
}))
require.NotNil(t, stored.Git)
require.NotNil(t, stored.Git.Authentication)
require.Equal(t, "alice", stored.Git.Authentication.Username)
require.Equal(t, "secret", stored.Git.Authentication.Password)
}
func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Name: "auth-source",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/org/repo.git",
Authentication: &gittypes.GitAuthentication{
Username: "alice",
Password: "secret",
},
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{
URL: new("https://github.com/org/repo.git"),
Authentication: &GitAuthenticationUpdatePayload{},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusOK, rr.Code)
var stored *portainer.Source
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
stored, err = tx.Source().Read(adminUserContext, srcID)
return err
}))
require.NotNil(t, stored.Git)
require.Nil(t, stored.Git.Authentication)
}
func TestGitSourceUpdate_ReplacesAuthWhenProvided(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Name: "auth-source",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/org/repo.git",
Authentication: &gittypes.GitAuthentication{
Username: "alice",
Password: "secret",
},
},
}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{
URL: new("https://github.com/org/repo.git"),
Authentication: &GitAuthenticationUpdatePayload{
Username: new("bob"),
Password: new("new-secret"),
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusOK, rr.Code)
var stored *portainer.Source
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
stored, err = tx.Source().Read(adminUserContext, srcID)
return err
}))
require.NotNil(t, stored.Git)
require.NotNil(t, stored.Git.Authentication)
require.Equal(t, "bob", stored.Git.Authentication.Username)
require.Equal(t, "new-secret", stored.Git.Authentication.Password)
}
func TestGitSourceUpdate_NotFound(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{URL: new("https://github.com/org/repo.git")})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, 99, body))
require.Equal(t, http.StatusNotFound, rr.Code)
}
func TestGitSourceUpdate_ConflictOnDuplicateURL(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
existing := &portainer.Source{
Name: "existing",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/org/existing.git",
},
}
err := tx.Source().Create(adminUserContext, existing)
require.NoError(t, err)
src := &portainer.Source{Name: "other", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err = tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{
URL: new("https://github.com/org/existing.git"),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusConflict, rr.Code)
}
func TestGitSourceUpdate_MalformedJSON(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "src", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo"}}
err := tx.Source().Create(adminUserContext, src)
require.NoError(t, err)
srcID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), []byte("not-valid-json{")))
require.Equal(t, http.StatusBadRequest, rr.Code)
}
func TestGitSourceUpdate_ConflictWhenAuthChangesMatchAnotherSource(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
existing := &portainer.Source{
Name: "existing",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{
URL: "https://github.com/org/repo.git",
Authentication: &gittypes.GitAuthentication{
Username: "alice",
Password: "secret",
},
},
}
if err := tx.Source().Create(adminUserContext, existing); err != nil {
return err
}
other := &portainer.Source{
Name: "other",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://github.com/org/repo.git"},
}
if err := tx.Source().Create(adminUserContext, other); err != nil {
return err
}
srcID = other.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
alice := "alice"
secret := "secret"
body, err := json.Marshal(GitSourceUpdatePayload{
Authentication: &GitAuthenticationUpdatePayload{
Username: &alice,
Password: &secret,
},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body))
require.Equal(t, http.StatusConflict, rr.Code)
}
func TestGitSourceUpdate_NonNumericID(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
body, err := json.Marshal(GitSourceUpdatePayload{URL: new("https://github.com/org/repo.git")})
require.NoError(t, err)
req := buildUpdateReqWithRawID(t, 1, "not-a-number", body)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
require.Equal(t, http.StatusBadRequest, rr.Code)
}
+43
View File
@@ -0,0 +1,43 @@
package sources
import (
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
ce "github.com/portainer/portainer/api/gitops/workflows"
)
func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Source {
phase := ce.SourceStatusToPhase(src.Status, src.StatusError)
url := ""
if src.Git != nil {
url = gittypes.SanitizeURL(src.Git.URL)
}
return Source{
ID: src.ID,
Name: src.Name,
Type: sourceTypeString(src.Type),
URL: url,
Status: phase.Status,
Error: phase.Error,
UsedBy: stats.WorkflowCount,
Environments: len(stats.EndpointIDs),
LastSync: src.LastSync,
}
}
func redactWorkflowCredentials(wfs []ce.Workflow) []ce.Workflow {
redacted := make([]ce.Workflow, len(wfs))
for i, wf := range wfs {
redacted[i] = wf
if wf.GitConfig != nil && wf.GitConfig.Authentication != nil {
cfg := *wf.GitConfig
auth := *wf.GitConfig.Authentication
auth.Password = ""
cfg.Authentication = &auth
redacted[i].GitConfig = &cfg
}
}
return redacted
}
@@ -0,0 +1,82 @@
package sources
import (
"testing"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
ce "github.com/portainer/portainer/api/gitops/workflows"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRedactWorkflowCredentials(t *testing.T) {
t.Parallel()
t.Run("clears password and preserves username", func(t *testing.T) {
t.Parallel()
wfs := []ce.Workflow{{GitConfig: &gittypes.RepoConfig{
Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"},
}}}
got := redactWorkflowCredentials(wfs)
require.NotNil(t, got[0].GitConfig.Authentication)
assert.Equal(t, "user", got[0].GitConfig.Authentication.Username)
assert.Empty(t, got[0].GitConfig.Authentication.Password)
})
t.Run("does not mutate the original slice", func(t *testing.T) {
t.Parallel()
wfs := []ce.Workflow{{GitConfig: &gittypes.RepoConfig{
Authentication: &gittypes.GitAuthentication{Password: "s3cr3t"},
}}}
_ = redactWorkflowCredentials(wfs)
assert.Equal(t, "s3cr3t", wfs[0].GitConfig.Authentication.Password)
})
t.Run("nil GitConfig is safe", func(t *testing.T) {
t.Parallel()
assert.NotPanics(t, func() { redactWorkflowCredentials([]ce.Workflow{{}}) })
})
t.Run("nil Authentication is safe", func(t *testing.T) {
t.Parallel()
wfs := []ce.Workflow{{GitConfig: &gittypes.RepoConfig{}}}
assert.NotPanics(t, func() { redactWorkflowCredentials(wfs) })
})
}
func TestBuildAutoUpdateInfo(t *testing.T) {
t.Parallel()
assert.Nil(t, BuildAutoUpdateInfo(nil))
assert.Nil(t, BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{}))
got := BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{Interval: "5m"})
require.NotNil(t, got)
assert.Equal(t, "Interval", got.Mechanism)
assert.Equal(t, "5m", got.FetchInterval)
got = BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{Webhook: "abc123"})
require.NotNil(t, got)
assert.Equal(t, "Webhook", got.Mechanism)
assert.Empty(t, got.FetchInterval)
}
func TestBuildConnectionInfo(t *testing.T) {
t.Parallel()
assert.Equal(t, connectionInfo{}, buildConnectionInfo(nil))
cfg := &gittypes.GitSource{
TLSSkipVerify: true,
Authentication: &gittypes.GitAuthentication{Username: "user"},
}
got := buildConnectionInfo(cfg)
assert.True(t, got.TLSSkipVerify)
require.NotNil(t, got.Authentication)
assert.Equal(t, "user", got.Authentication.Username)
got = buildConnectionInfo(&gittypes.GitSource{})
assert.Nil(t, got.Authentication)
}