wehub/portainer--portainer
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/git/azure.go
+589
View File
@@ -0,0 +1,589 @@
package git
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/crypto"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/filemode"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
)
const (
azureDevOpsHost = "dev.azure.com"
visualStudioHostSuffix = ".visualstudio.com"
)
func IsAzureUrl(s string) bool {
return strings.Contains(s, azureDevOpsHost) ||
strings.Contains(s, visualStudioHostSuffix)
}
type azureOptions struct {
organisation, project, repository string
// a user may pass credentials in a repository URL,
// for example https://<username>:<password>@<domain>/<path>
username, password string
}
// azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs
type azureRef struct {
Name string `json:"name"`
ObjectID string `json:"objectId"`
}
// azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download
type azureItem struct {
ObjectID string `json:"objectId"`
CommitId string `json:"commitId"`
Path string `json:"path"`
}
type azureClient struct {
baseUrl string
}
func NewAzureClient() *azureClient {
return &azureClient{
baseUrl: "https://dev.azure.com",
}
}
func newHttpClientForAzure(insecureSkipVerify bool) *http.Client {
return &http.Client{
Transport: ssrf.NewTransport(crypto.CreateTLSConfiguration(insecureSkipVerify)),
Timeout: 300 * time.Second,
}
}
func (a *azureClient) Download(ctx context.Context, destination string, opt *git.CloneOptions) error {
if opt == nil {
return errors.New("options cannot be nil")
}
zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt)
if err != nil {
return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
}
defer func() {
if err := os.Remove(zipFilepath); err != nil {
log.Warn().Err(err).Msg("failed to remove temporary zip file")
}
}()
if err := archive.UnzipFile(zipFilepath, destination); err != nil {
return errors.Wrap(err, "failed to unzip file")
}
return nil
}
func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt *git.CloneOptions) (string, error) {
config, err := parseUrl(opt.URL)
if err != nil {
return "", errors.WithMessage(err, "failed to parse url")
}
downloadUrl, err := a.buildDownloadUrl(config, string(opt.ReferenceName))
if err != nil {
return "", errors.WithMessage(err, "failed to build download url")
}
zipFile, err := os.CreateTemp("", "azure-git-repo-*.zip")
if err != nil {
return "", errors.WithMessage(err, "failed to create temp file")
}
defer logs.CloseAndLogErr(zipFile)
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return "", errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
if basicAuth != nil {
req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return "", errors.WithMessage(err, "failed to create a new HTTP request")
}
client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections()
res, err := client.Do(req)
if err != nil {
return "", errors.WithMessage(err, "failed to make an HTTP request")
}
defer func() {
if err := res.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
}()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("failed to download zip with a status \"%v\"", res.Status)
}
if _, err := io.Copy(zipFile, res.Body); err != nil {
return "", errors.WithMessage(err, "failed to save HTTP response to a file")
}
return zipFile.Name(), nil
}
func (a *azureClient) LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error) {
if opt == nil {
return "", errors.New("options cannot be nil")
}
rootItem, err := a.getRootItem(ctx, repositoryUrl, referenceName, opt)
if err != nil {
return "", err
}
return rootItem.CommitId, nil
}
func (a *azureClient) getRootItem(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (*azureItem, error) {
config, err := parseUrl(repositoryUrl)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
rootItemUrl, err := a.buildRootItemUrl(config, referenceName)
if err != nil {
return nil, errors.WithMessage(err, "failed to build azure root item url")
}
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
if basicAuth != nil {
req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections()
resp, err := client.Do(req)
if err != nil {
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
}()
if resp.StatusCode != http.StatusOK {
return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode)
}
var items struct {
Value []azureItem
}
if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
return nil, errors.Wrap(err, "could not parse Azure items response")
}
if len(items.Value) == 0 || items.Value[0].CommitId == "" {
return nil, errors.Errorf("failed to get latest commitID in the repository")
}
return &items.Value[0], nil
}
func parseUrl(rawUrl string) (*azureOptions, error) {
if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
return parseHttpUrl(rawUrl)
}
if strings.HasPrefix(rawUrl, "git@ssh") {
return parseSshUrl(rawUrl)
}
if strings.HasPrefix(rawUrl, "ssh://") {
r := []rune(rawUrl)
return parseSshUrl(string(r[6:])) // remove the prefix
}
return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl)
}
const expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
func parseSshUrl(rawUrl string) (*azureOptions, error) {
path := strings.Split(rawUrl, "/")
unexpectedUrlErr := errors.Errorf("want url %s, got %s", expectedSshUrl, rawUrl)
if len(path) != 4 {
return nil, unexpectedUrlErr
}
return &azureOptions{
organisation: path[1],
project: path[2],
repository: path[3],
}, nil
}
const (
expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository"
expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository"
)
func parseHttpUrl(rawUrl string) (*azureOptions, error) {
u, err := url.Parse(rawUrl)
if err != nil {
return nil, errors.Wrap(err, "failed to parse HTTP url")
}
opt := azureOptions{}
switch {
case u.Host == azureDevOpsHost:
path := strings.Split(u.Path, "/")
if len(path) != 5 {
return nil, errors.Errorf("want url %s, got %s", expectedAzureDevOpsHttpUrl, u)
}
opt.organisation = path[1]
opt.project = path[2]
opt.repository = path[4]
case strings.HasSuffix(u.Host, visualStudioHostSuffix):
path := strings.Split(u.Path, "/")
if len(path) != 4 {
return nil, errors.Errorf("want url %s, got %s", expectedVisualStudioHttpUrl, u)
}
opt.organisation = strings.TrimSuffix(u.Host, visualStudioHostSuffix)
opt.project = path[1]
opt.repository = path[3]
default:
return nil, errors.Errorf("unknown azure host in url \"%s\"", rawUrl)
}
opt.username = u.User.Username()
opt.password, _ = u.User.Password()
return &opt, nil
}
func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository))
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl)
}
q := u.Query()
// scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0
q.Set("scopePath", "/")
q.Set("download", "true")
if referenceName != "" {
q.Set("versionDescriptor.versionType", getVersionType(referenceName))
q.Set("versionDescriptor.version", formatReferenceName(referenceName))
}
q.Set("$format", "zip")
q.Set("recursionLevel", "full")
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository))
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse root item url path %s", rawUrl)
}
q := u.Query()
q.Set("scopePath", "/")
if referenceName != "" {
q.Set("versionDescriptor.versionType", getVersionType(referenceName))
q.Set("versionDescriptor.version", formatReferenceName(referenceName))
}
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) {
// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository))
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl)
}
q := u.Query()
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) {
// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0
rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s",
a.baseUrl,
url.PathEscape(config.organisation),
url.PathEscape(config.project),
url.PathEscape(config.repository),
url.PathEscape(rootObjectHash),
)
u, err := url.Parse(rawUrl)
if err != nil {
return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
}
q := u.Query()
// projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0
q.Set("recursive", "true")
q.Set("api-version", "6.0")
u.RawQuery = q.Encode()
return u.String(), nil
}
const (
branchPrefix = "refs/heads/"
tagPrefix = "refs/tags/"
)
func formatReferenceName(name string) string {
if after, ok := strings.CutPrefix(name, branchPrefix); ok {
return after
}
if after, ok := strings.CutPrefix(name, tagPrefix); ok {
return after
}
return name
}
func getVersionType(name string) string {
if strings.HasPrefix(name, branchPrefix) {
return "branch"
}
if strings.HasPrefix(name, tagPrefix) {
return "tag"
}
return "commit"
}
func (a *azureClient) ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error) {
if opt == nil {
return nil, errors.New("options cannot be nil")
}
config, err := parseUrl(repositoryUrl)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
listRefsUrl, err := a.buildRefsUrl(config)
if err != nil {
return nil, errors.WithMessage(err, "failed to build list refs url")
}
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil)
if basicAuth != nil {
req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections()
resp, err := client.Do(req)
if err != nil {
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
}()
if resp.StatusCode != http.StatusOK {
return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode)
}
var refs struct {
Value []azureRef
}
if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
return nil, errors.Wrap(err, "could not parse Azure refs response")
}
var ret []string
for _, value := range refs.Value {
if value.Name == "HEAD" {
continue
}
ret = append(ret, value.Name)
}
return ret, nil
}
// listFiles list all filenames under the specific repository
func (a *azureClient) ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error) {
if opt == nil {
return nil, errors.New("options cannot be nil")
}
listOptions := &git.ListOptions{
Auth: opt.Auth,
InsecureSkipTLS: opt.InsecureSkipTLS,
}
rootItem, err := a.getRootItem(ctx, opt.URL, string(opt.ReferenceName), listOptions)
if err != nil {
return nil, err
}
config, err := parseUrl(opt.URL)
if err != nil {
return nil, errors.WithMessage(err, "failed to parse url")
}
listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID)
if err != nil {
return nil, errors.WithMessage(err, "failed to build list tree url")
}
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil)
if basicAuth != nil {
req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password)
}
if err != nil {
return nil, errors.WithMessage(err, "failed to create a new HTTP request")
}
client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections()
resp, err := client.Do(req)
if err != nil {
return nil, errors.WithMessage(err, "failed to make an HTTP request")
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
}()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status)
}
var tree struct {
TreeEntries []struct {
RelativePath string `json:"relativePath"`
Mode string `json:"mode"`
} `json:"treeEntries"`
}
if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil {
return nil, errors.Wrap(err, "could not parse Azure tree response")
}
var allPaths []string
for _, treeEntry := range tree.TreeEntries {
mode, _ := filemode.New(treeEntry.Mode)
isDir := filemode.Dir == mode
if dirOnly == isDir {
allPaths = append(allPaths, treeEntry.RelativePath)
}
}
return allPaths, nil
}
func checkAzureStatusCode(err error, code int) error {
if code == http.StatusNotFound {
return gittypes.ErrIncorrectRepositoryURL
} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
return gittypes.ErrAuthenticationFailure
}
return err
}
api/git/azure_integration_test.go
+362
View File
@@ -0,0 +1,362 @@
package git
import (
"fmt"
"os"
"testing"
"time"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
_ "github.com/joho/godotenv/autoload"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
func TestService_ClonePublicRepository_Azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService(t.Context())
type args struct {
repositoryURLFormat string
referenceName string
username string
password string
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Clone Azure DevOps repo branch",
args: args{
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
referenceName: "refs/heads/main",
username: "",
password: pat,
},
wantErr: false,
},
{
name: "Clone Azure DevOps repo tag",
args: args{
repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
referenceName: "refs/heads/tags/v1.1",
username: "",
password: pat,
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dst := t.TempDir()
repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password)
err := service.CloneRepository(
t.Context(),
dst,
repositoryUrl,
tt.args.referenceName,
"",
"",
false,
)
require.NoError(t, err)
assert.FileExists(t, filesystem.JoinPaths(dst, "README.md"))
})
}
}
func TestService_ClonePrivateRepository_Azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService(t.Context())
dst := t.TempDir()
err := service.CloneRepository(
t.Context(),
dst,
privateAzureRepoURL,
"refs/heads/main",
"",
pat,
false,
)
require.NoError(t, err)
assert.FileExists(t, filesystem.JoinPaths(dst, "README.md"))
}
func TestService_LatestCommitID_Azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
service := NewService(t.Context())
id, err := service.LatestCommitID(
t.Context(),
privateAzureRepoURL,
"refs/heads/main",
"",
pat,
false,
)
require.NoError(t, err)
assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
}
func TestService_ListRefs_Azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := NewService(t.Context())
refs, err := service.ListRefs(
t.Context(),
privateAzureRepoURL,
username,
accessToken,
false,
false,
)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
}
func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(t.Context(), repositoryCacheSize, 200*time.Millisecond)
go func() {
_, _ = service.ListRefs(t.Context(), privateAzureRepoURL, username, accessToken, false, false)
}()
_, err := service.ListRefs(t.Context(), privateAzureRepoURL, username, accessToken, false, false)
require.NoError(t, err)
time.Sleep(2 * time.Second)
}
func TestService_ListFiles_Azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
type args struct {
repositoryUrl string
referenceName string
username string
password string
extensions []string
}
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
service := newService(t.Context(), 0, 0)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: "",
password: "",
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
err: nil,
matchedCount: 19,
},
},
{
name: "list tree with real repository and head ref and existing file extension",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"yml"},
},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository and head ref and non-existing file extension",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"hcl"},
},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository but non-existing ref",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: args{
repositoryUrl: privateAzureRepoURL + "fake",
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(
t.Context(),
tt.args.repositoryUrl,
tt.args.referenceName,
tt.args.username,
tt.args.password,
false,
false,
tt.args.extensions,
false,
)
if tt.expect.shouldFail {
require.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
require.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.NotEmpty(t, paths)
}
}
})
}
}
func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(t.Context(), repositoryCacheSize, 200*time.Millisecond)
go func() {
_, _ = service.ListFiles(
t.Context(),
privateAzureRepoURL,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
}()
_, err := service.ListFiles(
t.Context(),
privateAzureRepoURL,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
time.Sleep(2 * time.Second)
}
func getRequiredValue(t *testing.T, name string) string {
value, ok := os.LookupEnv(name)
if !ok {
t.Fatalf("can't find required env var \"%s\"", name)
}
return value
}
func ensureIntegrationTest(t *testing.T) {
if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
t.Skip("skip an integration test")
}
}
api/git/azure_test.go
+662
View File
@@ -0,0 +1,662 @@
package git
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_buildDownloadUrl(t *testing.T) {
t.Parallel()
a := NewAzureClient()
u, err := a.buildDownloadUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
}, "refs/heads/main")
require.NoError(t, err)
expectedUrl, err := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&download=true&versionDescriptor.version=main&$format=zip&recursionLevel=full&api-version=6.0&versionDescriptor.versionType=branch")
require.NoError(t, err)
actualUrl, err := url.Parse(u)
require.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_buildRootItemUrl(t *testing.T) {
t.Parallel()
a := NewAzureClient()
u, err := a.buildRootItemUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
}, "refs/heads/main")
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/items?scopePath=/&api-version=6.0&versionDescriptor.version=main&versionDescriptor.versionType=branch")
actualUrl, _ := url.Parse(u)
require.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_buildRefsUrl(t *testing.T) {
t.Parallel()
a := NewAzureClient()
u, err := a.buildRefsUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
})
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?api-version=6.0")
actualUrl, _ := url.Parse(u)
require.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_buildTreeUrl(t *testing.T) {
t.Parallel()
a := NewAzureClient()
u, err := a.buildTreeUrl(&azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
}, "sha1")
expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/trees/sha1?api-version=6.0&recursive=true")
actualUrl, _ := url.Parse(u)
require.NoError(t, err)
assert.Equal(t, expectedUrl.Host, actualUrl.Host)
assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
assert.Equal(t, expectedUrl.Path, actualUrl.Path)
assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
}
func Test_parseAzureUrl(t *testing.T) {
t.Parallel()
type args struct {
url string
}
tests := []struct {
name string
args args
want *azureOptions
wantErr bool
}{
{
name: "Expected SSH URL format starting with ssh://",
args: args{
url: "ssh://git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
},
want: &azureOptions{
organisation: "Organisation",
project: "Project",
repository: "Repository",
},
wantErr: false,
},
{
name: "Expected SSH URL format starting with git@ssh",
args: args{
url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
},
want: &azureOptions{
organisation: "Organisation",
project: "Project",
repository: "Repository",
},
wantErr: false,
},
{
name: "Unexpected SSH URL format",
args: args{
url: "git@ssh.dev.azure.com:v3/Organisation/Repository",
},
wantErr: true,
},
{
name: "Expected HTTPS URL format",
args: args{
url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
},
want: &azureOptions{
organisation: "Organisation",
project: "Project",
repository: "Repository",
username: "Organisation",
},
wantErr: false,
},
{
name: "HTTPS URL with credentials",
args: args{
url: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
},
want: &azureOptions{
organisation: "Organisation",
project: "Project",
repository: "Repository",
username: "username",
password: "password",
},
wantErr: false,
},
{
name: "HTTPS URL with password",
args: args{
url: "https://:password@dev.azure.com/Organisation/Project/_git/Repository",
},
want: &azureOptions{
organisation: "Organisation",
project: "Project",
repository: "Repository",
password: "password",
},
wantErr: false,
},
{
name: "Visual Studio HTTPS URL with credentials",
args: args{
url: "https://username:password@organisation.visualstudio.com/project/_git/repository",
},
want: &azureOptions{
organisation: "organisation",
project: "project",
repository: "repository",
username: "username",
password: "password",
},
wantErr: false,
},
{
name: "Unexpected HTTPS URL format",
args: args{
url: "https://Organisation@dev.azure.com/Project/_git/Repository",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseUrl(tt.args.url)
if (err != nil) != tt.wantErr {
t.Errorf("parseUrl() error = %v, wantErr %v", err, tt.wantErr)
return
}
assert.Equal(t, tt.want, got)
})
}
}
func Test_isAzureUrl(t *testing.T) {
t.Parallel()
type args struct {
s string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "Is Azure url",
args: args{
s: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
},
want: true,
},
{
name: "Is Azure url",
args: args{
s: "https://portainer.visualstudio.com/project/_git/repository",
},
want: true,
},
{
name: "Is NOT Azure url",
args: args{
s: "https://github.com/Organisation/Repository",
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, IsAzureUrl(tt.args.s))
})
}
}
func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
t.Parallel()
fips.InitFIPS(false)
type args struct {
repositoryUrl string
username string
password string
}
type basicAuth struct {
username, password string
}
tests := []struct {
name string
args args
want *basicAuth
}{
{
name: "username, password embedded",
args: args{
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
},
want: &basicAuth{
username: "username",
password: "password",
},
},
{
name: "username, password embedded, clone options take precedence",
args: args{
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
username: "u",
password: "p",
},
want: &basicAuth{
username: "u",
password: "p",
},
},
{
name: "no credentials",
args: args{
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var zipRequestAuth *basicAuth
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if username, password, ok := r.BasicAuth(); ok {
zipRequestAuth = &basicAuth{username, password}
}
w.WriteHeader(http.StatusNotFound) // this makes function under test to return an error
}))
defer server.Close()
a := &azureClient{
baseUrl: server.URL,
}
option := &git.CloneOptions{
URL: tt.args.repositoryUrl,
}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
_, err := a.downloadZipFromAzureDevOps(t.Context(), option)
require.Error(t, err)
assert.Equal(t, tt.want, zipRequestAuth)
})
}
}
func Test_azureDownloader_latestCommitID(t *testing.T) {
t.Parallel()
fips.InitFIPS(false)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
response := `{
"count": 1,
"value": [
{
"objectId": "1a5630f017127db7de24d8771da0f536ff98fc9b",
"gitObjectType": "tree",
"commitId": "27104ad7549d9e66685e115a497533f18024be9c",
"path": "/",
"isFolder": true,
"url": "https://dev.azure.com/simonmeng0474/4b546a97-c481-4506-bdd5-976e9592f91a/_apis/git/repositories/a22247ad-053f-43bc-88a7-62ff4846bb97/items?path=%2F&versionType=Branch&versionOptions=None"
}
]
}`
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(response))
}))
defer server.Close()
a := &azureClient{baseUrl: server.URL}
type args struct {
repositoryUrl string
referenceName string
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "should be able to parse response",
args: args{
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
referenceName: "",
},
want: "27104ad7549d9e66685e115a497533f18024be9c",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id, err := a.LatestCommitID(t.Context(), tt.args.repositoryUrl, tt.args.referenceName, &git.ListOptions{})
if (err != nil) != tt.wantErr {
t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr)
return
}
assert.Equal(t, tt.want, id)
})
}
}
type testRepoManager struct {
called bool
}
func (t *testRepoManager) Download(_ context.Context, _ string, _ *git.CloneOptions) error {
t.called = true
return nil
}
func (t *testRepoManager) LatestCommitID(_ context.Context, _, _ string, _ *git.ListOptions) (string, error) {
return "", nil
}
func (t *testRepoManager) ListRefs(_ context.Context, _ string, _ *git.ListOptions) ([]string, error) {
return nil, nil
}
func (t *testRepoManager) ListFiles(_ context.Context, _ bool, _ *git.CloneOptions) ([]string, error) {
return nil, nil
}
func Test_cloneRepository_azure(t *testing.T) {
t.Parallel()
tests := []struct {
name string
url string
called bool
}{
{
name: "Azure HTTP URL",
url: "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
called: true,
},
{
name: "Azure SSH URL",
url: "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
called: true,
},
{
name: "Something else",
url: "https://example.com",
called: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
azure := &testRepoManager{}
git := &testRepoManager{}
s := &Service{azure: azure, git: git}
err := s.CloneRepository(t.Context(), "", tt.url, "", "", "", false)
require.NoError(t, err)
// if azure API is called, git isn't and vice versa
assert.Equal(t, tt.called, azure.called)
assert.Equal(t, tt.called, !git.called)
})
}
}
func Test_listRefs_azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
client := NewAzureClient()
type args struct {
repositoryUrl string
username string
password string
}
type expectResult struct {
err error
refsCount int
}
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list refs of a real repository",
args: args{
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
refsCount: 2,
},
},
{
name: "list refs of a real repository with incorrect credential",
args: args{
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
expect: expectResult{
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list refs of a real repository without providing credential",
args: args{
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
expect: expectResult{
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list refs of a fake repository",
args: args{
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
expect: expectResult{
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
option := &git.ListOptions{}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
refs, err := client.ListRefs(t.Context(), tt.args.repositoryUrl, option)
if tt.expect.err == nil {
require.NoError(t, err)
if tt.expect.refsCount > 0 {
assert.NotEmpty(t, refs)
}
} else {
require.Error(t, err)
assert.Equal(t, tt.expect.err, err)
}
})
}
}
func Test_listFiles_azure(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
client := NewAzureClient()
type args struct {
repositoryUrl string
referenceName string
username string
password string
}
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: "",
password: "",
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
matchedCount: 19,
},
},
{
name: "list tree with real repository but non-existing ref",
args: args{
repositoryUrl: privateAzureRepoURL,
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: args{
repositoryUrl: privateAzureRepoURL + "fake",
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
option := &git.CloneOptions{
URL: tt.args.repositoryUrl,
ReferenceName: plumbing.ReferenceName(tt.args.referenceName),
}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
paths, err := client.ListFiles(t.Context(), false, option)
if tt.expect.shouldFail {
require.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
require.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.NotEmpty(t, paths)
}
}
})
}
}
api/git/backup.go
+69
View File
@@ -0,0 +1,69 @@
package git
import (
"context"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
var (
ErrInvalidGitCredential = errors.New("Invalid git credential")
)
type CloneOptions struct {
ProjectPath string
URL string
ReferenceName string
Username string
Password string
// TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"`
}
func CloneWithBackup(ctx context.Context, gitService portainer.GitService, fileService portainer.FileService, options CloneOptions) (clean func(), err error) {
backupProjectPath := options.ProjectPath + "-old"
cleanUp := false
cleanFn := func() {
if !cleanUp {
return
}
if err := fileService.RemoveDirectory(backupProjectPath); err != nil {
log.Warn().Err(err).Msg("unable to remove git repository directory")
}
}
if err := filesystem.MoveDirectory(options.ProjectPath, backupProjectPath, true); err != nil {
return cleanFn, errors.WithMessage(err, "Unable to move git repository directory")
}
cleanUp = true
if err := gitService.CloneRepository(
ctx,
options.ProjectPath,
options.URL,
options.ReferenceName,
options.Username,
options.Password,
options.TLSSkipVerify,
); err != nil {
cleanUp = false
if err := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath, false); err != nil {
log.Warn().Err(err).Msg("failed restoring backup folder")
}
if errors.Is(err, gittypes.ErrAuthenticationFailure) {
return cleanFn, errors.WithMessage(err, ErrInvalidGitCredential.Error())
}
return cleanFn, errors.WithMessage(err, "Unable to clone git repository")
}
return cleanFn, nil
}
api/git/credentials.go
+13
View File
@@ -0,0 +1,13 @@
package git
import (
gittypes "github.com/portainer/portainer/api/git/types"
)
func GetCredentials(auth *gittypes.GitAuthentication) (string, string) {
if auth == nil {
return "", ""
}
return auth.Username, auth.Password
}
api/git/git.go
+189
View File
@@ -0,0 +1,189 @@
package git
import (
"context"
"os"
"path/filepath"
"strings"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
gogitfs "github.com/go-git/go-git/v5/storage/filesystem"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
// noSymlinkFS wraps a billy.Filesystem and rejects symlink creation to prevent
// symlink traversal attacks from untrusted git repositories
type noSymlinkFS struct {
billy.Filesystem
}
func (fs noSymlinkFS) Symlink(_, _ string) error {
return gittypes.ErrSymlinkDetected
}
// NewNoSymlinkFS wraps fs and rejects any symlink creation
func NewNoSymlinkFS(fs billy.Filesystem) billy.Filesystem {
return noSymlinkFS{fs}
}
type gitClient struct {
preserveGitDirectory bool
}
func NewGitClient(preserveGitDir bool) *gitClient {
return &gitClient{
preserveGitDirectory: preserveGitDir,
}
}
func (c *gitClient) Download(ctx context.Context, dst string, opt *git.CloneOptions) error {
resolved, err := filepath.EvalSymlinks(dst)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return errors.Wrap(err, "failed to resolve destination path")
}
if err == nil {
dst = resolved
}
wt := NewNoSymlinkFS(osfs.New(dst))
dot := osfs.New(filesystem.JoinPaths(dst, ".git"))
storer := gogitfs.NewStorage(dot, cache.NewObjectLRU(0))
_, err = git.CloneContext(ctx, storer, wt, opt)
if err != nil {
if err.Error() == "authentication required" {
return gittypes.ErrAuthenticationFailure
}
return errors.Wrap(err, "failed to clone git repository")
}
if c.preserveGitDirectory {
return nil
}
if err := os.RemoveAll(filesystem.JoinPaths(dst, ".git")); err != nil {
log.Error().Err(err).Msg("failed to remove .git directory")
}
return nil
}
func (c *gitClient) LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error) {
remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
URLs: []string{repositoryUrl},
})
refs, err := remote.ListContext(ctx, opt)
if err != nil {
if err.Error() == "authentication required" {
return "", gittypes.ErrAuthenticationFailure
}
return "", errors.Wrap(err, "failed to list repository refs")
}
if referenceName == "" {
for _, ref := range refs {
if strings.EqualFold(ref.Name().String(), "HEAD") {
referenceName = ref.Target().String()
}
}
}
for _, ref := range refs {
if strings.EqualFold(ref.Name().String(), referenceName) {
return ref.Hash().String(), nil
}
}
return "", errors.Errorf("could not find ref %q in the repository", referenceName)
}
func (c *gitClient) ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error) {
rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
URLs: []string{repositoryUrl},
})
refs, err := rem.ListContext(ctx, opt)
if err != nil {
return nil, checkGitError(err)
}
var ret []string
for _, ref := range refs {
if ref.Name().String() == "HEAD" {
continue
}
ret = append(ret, ref.Name().String())
}
return ret, nil
}
// listFiles list all filenames under the specific repository
func (c *gitClient) ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error) {
repo, err := git.Clone(memory.NewStorage(), nil, opt)
if err != nil {
return nil, checkGitError(err)
}
head, err := repo.Head()
if err != nil {
return nil, err
}
commit, err := repo.CommitObject(head.Hash())
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
var allPaths []string
w := object.NewTreeWalker(tree, true, nil)
defer w.Close()
for {
name, entry, err := w.Next()
if err != nil {
break
}
isDir := entry.Mode == filemode.Dir
if dirOnly == isDir {
allPaths = append(allPaths, name)
}
}
return allPaths, nil
}
func checkGitError(err error) error {
errMsg := err.Error()
if strings.Contains(errMsg, "repository not found") {
return gittypes.ErrIncorrectRepositoryURL
} else if errMsg == "authentication required" {
return gittypes.ErrAuthenticationFailure
}
return err
}
api/git/git_integration_test.go
+508
View File
@@ -0,0 +1,508 @@
package git
import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
privateGitRepoURL string = "https://github.com/portainer/private-test-repository.git"
)
func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), 0, 0)
dst := t.TempDir()
repositoryUrl := privateGitRepoURL
err := service.CloneRepository(
t.Context(),
dst,
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
)
require.NoError(t, err)
assert.FileExists(t, filesystem.JoinPaths(dst, "README.md"))
}
func TestService_LatestCommitID_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), 0, 0)
repositoryUrl := privateGitRepoURL
id, err := service.LatestCommitID(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
)
require.NoError(t, err)
assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
}
func TestService_ListRefs_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), 0, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
}
func TestService_ListRefs_Github_Concurrently(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), repositoryCacheSize, 200*time.Millisecond)
repositoryUrl := privateGitRepoURL
go func() {
_, _ = service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
}()
_, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
time.Sleep(2 * time.Second)
}
func TestService_ListFiles_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
type args struct {
repositoryUrl string
referenceName string
username string
password string
extensions []string
}
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
service := newService(t.Context(), 0, 0)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: args{
repositoryUrl: privateGitRepoURL + "fake",
referenceName: "refs/heads/main",
username: "",
password: "",
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
err: nil,
matchedCount: 15,
},
},
{
name: "list tree with real repository and head ref and existing file extension",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"yml"},
},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository and head ref and non-existing file extension",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"hcl"},
},
expect: expectResult{
err: nil,
matchedCount: 2,
},
},
{
name: "list tree with real repository but non-existing ref",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: args{
repositoryUrl: privateGitRepoURL + "fake",
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
paths, err := service.ListFiles(
t.Context(),
tt.args.repositoryUrl,
tt.args.referenceName,
tt.args.username,
tt.args.password,
false,
false,
tt.args.extensions,
false,
)
if tt.expect.shouldFail {
require.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
require.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.NotEmpty(t, paths)
}
}
})
}
}
func TestService_ListFiles_Github_Concurrently(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), repositoryCacheSize, 200*time.Millisecond)
go func() {
_, _ = service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
}()
_, err := service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
time.Sleep(2 * time.Second)
}
func TestService_purgeCache_Github(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := NewService(t.Context())
_, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
_, err = service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
service.purgeCache()
assert.Equal(t, 0, service.repoRefCache.Len())
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_purgeCacheByTTL_Github(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
timeout := 100 * time.Millisecond
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
service := newService(t.Context(), 2, 40*timeout)
_, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
_, err = service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
assert.Equal(t, 1, service.repoRefCache.Len())
assert.Equal(t, 1, service.repoFileCache.Len())
// 40*timeout is designed for giving enough time for TTL being activated
time.Sleep(40 * timeout)
assert.Equal(t, 0, service.repoRefCache.Len())
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), 2, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len())
_, err = service.ListRefs(t.Context(), repositoryUrl, username, "fake-token", false, false)
require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len())
}
func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(t.Context(), 2, 0)
repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(t.Context(), repositoryUrl, username, accessToken, false, false)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len())
files, err := service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
files, err = service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/test",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 2, service.repoFileCache.Len())
_, err = service.ListRefs(t.Context(), repositoryUrl, username, "fake-token", false, false)
require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len())
_, err = service.ListRefs(t.Context(), repositoryUrl, username, "fake-token", true, false)
require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len())
// The relevant file caches should be removed too
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
service := newService(t.Context(), 2, 0)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
repositoryUrl := privateGitRepoURL
files, err := service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
false,
[]string{},
false,
)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 1, service.repoFileCache.Len())
_, err = service.ListFiles(
t.Context(),
repositoryUrl,
"refs/heads/main",
username,
"fake-token",
false,
true,
[]string{},
false,
)
require.Error(t, err)
assert.Equal(t, 0, service.repoFileCache.Len())
}
func TestService_CloneRepository_TokenAuth(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
service := newService(t.Context(), 2, 0)
var requests []*http.Request
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests = append(requests, r)
}))
accessToken := "test_access_token"
username := "test_username"
repositoryUrl := testServer.URL
// Since we aren't hitting a real git server we ignore the error
_ = service.CloneRepository(
t.Context(),
"test_dir",
repositoryUrl,
"refs/heads/main",
username,
accessToken,
false,
)
testServer.Close()
if len(requests) != 1 {
t.Fatalf("expected 1 request sent but got %d", len(requests))
}
gotAuthHeader := requests[0].Header.Get("Authorization")
if gotAuthHeader == "" {
t.Fatal("no Authorization header in git request")
}
expectedAuthHeader := "Bearer test_access_token"
if gotAuthHeader != expectedAuthHeader {
t.Fatalf("expected Authorization header %q but got %q", expectedAuthHeader, gotAuthHeader)
}
}
api/git/git_test.go
+506
View File
@@ -0,0 +1,506 @@
package git
import (
"os"
"testing"
"time"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func setup(t *testing.T) string {
dir := t.TempDir()
bareRepoDir := filesystem.JoinPaths(dir, "test-clone.git")
file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0o755)
if err != nil {
t.Fatal(errors.Wrap(err, "failed to open an archive"))
}
if err := archive.ExtractTarGz(file, dir); err != nil {
t.Fatal(errors.Wrapf(err, "failed to extract file from the archive to a folder %s", dir))
}
return bareRepoDir
}
func Test_checkGitError(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
expected error
}{
{
name: "exact repository not found",
err: errors.New("repository not found"),
expected: gittypes.ErrIncorrectRepositoryURL,
},
{
name: "repository not found with html body",
err: errors.New("repository not found: <html><body>404 Not Found</body></html>"),
expected: gittypes.ErrIncorrectRepositoryURL,
},
{
name: "authentication required",
err: errors.New("authentication required"),
expected: gittypes.ErrAuthenticationFailure,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := checkGitError(tt.err)
assert.Equal(t, tt.expected, result)
})
}
t.Run("other error is unchanged", func(t *testing.T) {
err := errors.New("some other git error")
assert.EqualError(t, checkGitError(err), "some other git error")
})
}
func Test_ClonePublicRepository_Shallow(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
dir := t.TempDir()
t.Logf("Cloning into %s", dir)
err := service.CloneRepository(t.Context(), dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err)
assert.Equal(t, 1, getCommitHistoryLength(t, dir), "cloned repo has incorrect depth")
}
func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(false)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
dir := t.TempDir()
t.Logf("Cloning into %s", dir)
err := service.CloneRepository(t.Context(), dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err)
assert.NoDirExists(t, filesystem.JoinPaths(dir, ".git"))
}
func Test_ClonePublicRepository_NonExistentDst(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(false)}
repositoryURL := setup(t)
referenceName := "refs/heads/main"
dir := filesystem.JoinPaths(t.TempDir(), "sub", "dir")
err := service.CloneRepository(t.Context(), dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err)
assert.DirExists(t, dir)
assert.NoDirExists(t, filesystem.JoinPaths(dir, ".git"))
}
func Test_latestCommitID(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
id, err := service.LatestCommitID(t.Context(), repositoryURL, referenceName, "", "", false)
require.NoError(t, err)
assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
}
func Test_ListRefs(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
fs, err := service.ListRefs(t.Context(), repositoryURL, "", "", false, false)
require.NoError(t, err)
assert.Equal(t, []string{"refs/heads/main"}, fs)
}
func Test_ListFiles(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
referenceName := "refs/heads/main"
fs, err := service.ListFiles(
t.Context(),
repositoryURL,
referenceName,
"",
"",
false,
false,
[]string{".yml"},
false,
)
require.NoError(t, err)
assert.Equal(t, []string{"docker-compose.yml"}, fs)
}
func getCommitHistoryLength(t *testing.T, dir string) int {
repo, err := git.PlainOpen(dir)
if err != nil {
t.Fatalf("can't open a git repo at %s with error %v", dir, err)
}
iter, err := repo.Log(&git.LogOptions{All: true})
if err != nil {
t.Fatalf("can't get a commit history iterator with error %v", err)
}
count := 0
if err := iter.ForEach(func(_ *object.Commit) error {
count++
return nil
}); err != nil {
t.Fatalf("can't iterate over the commit history with error %v", err)
}
return count
}
func Test_noSymlinkFS_Symlink(t *testing.T) {
fs := NewNoSymlinkFS(osfs.New(t.TempDir()))
err := fs.Symlink("../../../etc/passwd", "evil-link")
require.ErrorIs(t, err, gittypes.ErrSymlinkDetected)
}
func Test_noSymlinkFS_OtherOperations(t *testing.T) {
dir := t.TempDir()
fs := NewNoSymlinkFS(osfs.New(dir))
f, err := fs.Create("test.txt")
require.NoError(t, err)
_, err = f.Write([]byte("hello"))
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)
info, err := fs.Stat("test.txt")
require.NoError(t, err)
require.Equal(t, "test.txt", info.Name())
}
func createBareRepoWithSymlink(t *testing.T) string {
t.Helper()
bareDir := filesystem.JoinPaths(t.TempDir(), "symlink-repo.git")
repo, err := git.PlainInit(bareDir, true)
require.NoError(t, err)
storer := repo.Storer
fileBlob := &plumbing.MemoryObject{}
fileBlob.SetType(plumbing.BlobObject)
_, err = fileBlob.Write([]byte("hello world\n"))
require.NoError(t, err)
fileHash, err := storer.SetEncodedObject(fileBlob)
require.NoError(t, err)
symlinkBlob := &plumbing.MemoryObject{}
symlinkBlob.SetType(plumbing.BlobObject)
_, err = symlinkBlob.Write([]byte("../../../etc/passwd"))
require.NoError(t, err)
symlinkHash, err := storer.SetEncodedObject(symlinkBlob)
require.NoError(t, err)
tree := &object.Tree{
Entries: []object.TreeEntry{
{Name: "evil-link", Mode: filemode.Symlink, Hash: symlinkHash},
{Name: "file.txt", Mode: filemode.Regular, Hash: fileHash},
},
}
treeObj := &plumbing.MemoryObject{}
err = tree.Encode(treeObj)
require.NoError(t, err)
treeHash, err := storer.SetEncodedObject(treeObj)
require.NoError(t, err)
sig := object.Signature{Name: "Test", Email: "test@test.com", When: time.Now()}
commit := &object.Commit{
Message: "add symlink",
Author: sig,
Committer: sig,
TreeHash: treeHash,
}
commitObj := &plumbing.MemoryObject{}
err = commit.Encode(commitObj)
require.NoError(t, err)
commitHash, err := storer.SetEncodedObject(commitObj)
require.NoError(t, err)
err = storer.SetReference(plumbing.NewHashReference("refs/heads/main", commitHash))
require.NoError(t, err)
err = storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, "refs/heads/main"))
require.NoError(t, err)
return bareDir
}
func Test_Download_RejectsSymlink(t *testing.T) {
t.Parallel()
client := NewGitClient(false)
repoURL := createBareRepoWithSymlink(t)
err := client.Download(t.Context(), t.TempDir(), &git.CloneOptions{
URL: repoURL,
Depth: 1,
SingleBranch: true,
Tags: git.NoTags,
})
require.Error(t, err)
require.ErrorIs(t, err, gittypes.ErrSymlinkDetected)
}
func Test_listRefsPrivateRepository(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
client := NewGitClient(false)
type args struct {
repositoryUrl string
username string
password string
}
type expectResult struct {
err error
refsCount int
}
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list refs of a real private repository",
args: args{
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
refsCount: 2,
},
},
{
name: "list refs of a real private repository with incorrect credential",
args: args{
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
expect: expectResult{
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list refs of a fake repository without providing credential",
args: args{
repositoryUrl: privateGitRepoURL + "fake",
username: "",
password: "",
},
expect: expectResult{
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list refs of a fake repository",
args: args{
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
expect: expectResult{
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
option := &git.ListOptions{}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
refs, err := client.ListRefs(t.Context(), tt.args.repositoryUrl, option)
if tt.expect.err == nil {
require.NoError(t, err)
if tt.expect.refsCount > 0 {
assert.NotEmpty(t, refs)
}
} else {
require.Error(t, err)
assert.Equal(t, tt.expect.err, err)
}
})
}
}
func Test_listFilesPrivateRepository(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
client := NewGitClient(false)
type args struct {
repositoryUrl string
referenceName string
username string
password string
}
type expectResult struct {
shouldFail bool
err error
matchedCount int
}
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
tests := []struct {
name string
args args
expect expectResult
}{
{
name: "list tree with real repository and head ref but incorrect credential",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref but no credential",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: "",
password: "",
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrAuthenticationFailure,
},
},
{
name: "list tree with real repository and head ref",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/heads/main",
username: username,
password: accessToken,
},
expect: expectResult{
err: nil,
matchedCount: 15,
},
},
{
name: "list tree with real repository but non-existing ref",
args: args{
repositoryUrl: privateGitRepoURL,
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
},
expect: expectResult{
shouldFail: true,
},
},
{
name: "list tree with fake repository ",
args: args{
repositoryUrl: privateGitRepoURL + "fake",
referenceName: "refs/fake/feature",
username: username,
password: accessToken,
},
expect: expectResult{
shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
option := &git.CloneOptions{
URL: tt.args.repositoryUrl,
ReferenceName: plumbing.ReferenceName(tt.args.referenceName),
}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
paths, err := client.ListFiles(t.Context(), false, option)
if tt.expect.shouldFail {
require.Error(t, err)
if tt.expect.err != nil {
assert.Equal(t, tt.expect.err, err)
}
} else {
require.NoError(t, err)
if tt.expect.matchedCount > 0 {
assert.NotEmpty(t, paths)
}
}
})
}
}
api/git/service.go
+367
View File
@@ -0,0 +1,367 @@
package git
import (
"context"
"strconv"
"strings"
"time"
"github.com/portainer/portainer/pkg/schedule"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
lru "github.com/hashicorp/golang-lru"
"github.com/rs/zerolog/log"
"golang.org/x/sync/singleflight"
)
const (
repositoryCacheSize = 4
repositoryCacheTTL = 5 * time.Minute
)
type RepoManager interface {
Download(ctx context.Context, dst string, opt *git.CloneOptions) error
LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error)
ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error)
ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error)
}
// Service represents a service for managing Git.
type Service struct {
azure RepoManager
git RepoManager
cacheEnabled bool
// Cache the result of repository refs, key is repository URL
repoRefCache *lru.Cache
// Cache the result of repository file tree, key is the concatenated string of repository URL and ref value
repoFileCache *lru.Cache
}
// NewService initializes a new service.
func NewService(ctx context.Context) *Service {
return newService(ctx, repositoryCacheSize, repositoryCacheTTL)
}
func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {
service := &Service{
azure: NewAzureClient(),
git: NewGitClient(false),
cacheEnabled: cacheSize > 0,
}
if !service.cacheEnabled {
return service
}
var err error
service.repoRefCache, err = lru.New(cacheSize)
if err != nil {
log.Debug().Err(err).Msg("failed to create ref cache")
}
service.repoFileCache, err = lru.New(cacheSize)
if err != nil {
log.Debug().Err(err).Msg("failed to create file cache")
}
if cacheTTL > 0 {
go schedule.RunOnInterval(ctx, cacheTTL, service.purgeCache, nil)
}
return service
}
// CloneRepository clones a git repository using the specified URL in the specified
// destination folder.
func (service *Service) CloneRepository(
ctx context.Context,
destination,
repositoryURL,
referenceName,
username,
password string,
tlsSkipVerify bool,
) error {
return service.CloneRepositoryWithAuth(ctx, destination, repositoryURL, referenceName, GetBasicAuth(username, password), tlsSkipVerify)
}
// CloneRepositoryWithAuth clones a git repository using the specified URL in the specified
// destination folder, using the provided auth method.
func (service *Service) CloneRepositoryWithAuth(
ctx context.Context,
destination,
repositoryURL,
referenceName string,
auth transport.AuthMethod,
tlsSkipVerify bool,
) error {
gitOptions := &git.CloneOptions{
URL: repositoryURL,
Depth: 1,
InsecureSkipTLS: tlsSkipVerify,
Auth: auth,
Tags: git.NoTags,
}
if referenceName != "" {
gitOptions.ReferenceName = plumbing.ReferenceName(referenceName)
}
return service.repoManager(repositoryURL).Download(ctx, destination, gitOptions)
}
func (service *Service) repoManager(repositoryURL string) RepoManager {
repoManager := service.git
if IsAzureUrl(repositoryURL) {
repoManager = service.azure
}
return repoManager
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference
func (service *Service) LatestCommitID(
ctx context.Context,
repositoryURL,
referenceName,
username,
password string,
tlsSkipVerify bool,
) (string, error) {
return service.LatestCommitIDWithAuth(ctx, repositoryURL, referenceName, GetBasicAuth(username, password), tlsSkipVerify)
}
// LatestCommitIDWithAuth returns SHA1 of the latest commit of the specified reference,
// using the provided auth method.
func (service *Service) LatestCommitIDWithAuth(
ctx context.Context,
repositoryURL,
referenceName string,
auth transport.AuthMethod,
tlsSkipVerify bool,
) (string, error) {
listOptions := &git.ListOptions{
Auth: auth,
InsecureSkipTLS: tlsSkipVerify,
}
return service.repoManager(repositoryURL).LatestCommitID(ctx, repositoryURL, referenceName, listOptions)
}
// ListRefs will list target repository's references without cloning the repository
func (service *Service) ListRefs(
ctx context.Context,
repositoryURL,
username,
password string,
hardRefresh bool,
tlsSkipVerify bool,
) ([]string, error) {
cacheKey := GenerateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify))
return service.ListRefsWithAuth(ctx, repositoryURL, hardRefresh, GetBasicAuth(username, password), tlsSkipVerify, cacheKey)
}
// ListRefsWithAuth will list target repository's references without cloning the repository,
// using the provided auth method. The cacheKey is supplied by the caller.
func (service *Service) ListRefsWithAuth(
ctx context.Context,
repositoryURL string,
hardRefresh bool,
auth transport.AuthMethod,
tlsSkipVerify bool,
cacheKey string,
) ([]string, error) {
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoRefCache.Remove(cacheKey)
// Remove file caches pointed to the same repository
for _, fileCacheKey := range service.repoFileCache.Keys() {
if key, ok := fileCacheKey.(string); ok && strings.HasPrefix(key, repositoryURL) {
service.repoFileCache.Remove(key)
}
}
}
if service.repoRefCache != nil {
// Lookup the refs cache first
if cache, ok := service.repoRefCache.Get(cacheKey); ok {
if refs, ok := cache.([]string); ok {
return refs, nil
}
}
}
options := &git.ListOptions{
Auth: auth,
InsecureSkipTLS: tlsSkipVerify,
}
refs, err := service.repoManager(repositoryURL).ListRefs(ctx, repositoryURL, options)
if err != nil {
return nil, err
}
if service.cacheEnabled && service.repoRefCache != nil {
service.repoRefCache.Add(cacheKey, refs)
}
return refs, nil
}
var singleflightGroup = &singleflight.Group{}
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(
ctx context.Context,
repositoryURL,
referenceName,
username,
password string,
dirOnly,
hardRefresh bool,
includedExts []string,
tlsSkipVerify bool,
) ([]string, error) {
cacheKey := GenerateCacheKey(
repositoryURL,
referenceName,
username,
password,
strconv.FormatBool(tlsSkipVerify),
strconv.FormatBool(dirOnly),
)
return service.ListFilesWithAuth(ctx, repositoryURL, referenceName, dirOnly, hardRefresh, GetBasicAuth(username, password), includedExts, tlsSkipVerify, cacheKey)
}
// ListFilesWithAuth will list all the files of the target repository with specific extensions,
// using the provided auth method. The cacheKey is supplied by the caller.
func (service *Service) ListFilesWithAuth(
ctx context.Context,
repositoryURL,
referenceName string,
dirOnly,
hardRefresh bool,
auth transport.AuthMethod,
includedExts []string,
tlsSkipVerify bool,
cacheKey string,
) ([]string, error) {
fs, err, _ := singleflightGroup.Do(cacheKey, func() (any, error) {
return service.listFilesWithAuth(ctx, repositoryURL, referenceName, dirOnly, hardRefresh, auth, tlsSkipVerify, cacheKey)
})
return filterFiles(fs.([]string), includedExts), err
}
func (service *Service) listFilesWithAuth(
ctx context.Context,
repositoryURL,
referenceName string,
dirOnly,
hardRefresh bool,
auth transport.AuthMethod,
tlsSkipVerify bool,
cacheKey string,
) ([]string, error) {
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoFileCache.Remove(cacheKey)
}
if service.repoFileCache != nil {
// lookup the files cache first
if cache, ok := service.repoFileCache.Get(cacheKey); ok {
if files, ok := cache.([]string); ok {
return files, nil
}
}
}
cloneOption := &git.CloneOptions{
URL: repositoryURL,
NoCheckout: true,
Depth: 1,
SingleBranch: true,
ReferenceName: plumbing.ReferenceName(referenceName),
Auth: auth,
InsecureSkipTLS: tlsSkipVerify,
Tags: git.NoTags,
}
files, err := service.repoManager(repositoryURL).ListFiles(ctx, dirOnly, cloneOption)
if err != nil {
return nil, err
}
if service.cacheEnabled && service.repoFileCache != nil {
service.repoFileCache.Add(cacheKey, files)
}
return files, nil
}
func (service *Service) purgeCache() {
if service.repoRefCache != nil {
service.repoRefCache.Purge()
}
if service.repoFileCache != nil {
service.repoFileCache.Purge()
}
}
// GenerateCacheKey generates a cache key from the given parts.
func GenerateCacheKey(names ...string) string {
return strings.Join(names, "-")
}
func matchExtensions(target string, exts []string) bool {
if len(exts) == 0 {
return true
}
for _, ext := range exts {
if strings.HasSuffix(target, ext) {
return true
}
}
return false
}
func filterFiles(paths []string, includedExts []string) []string {
if len(includedExts) == 0 {
return paths
}
var includedFiles []string
for _, filename := range paths {
// Filter out the filenames with non-included extension
if matchExtensions(filename, includedExts) {
includedFiles = append(includedFiles, filename)
}
}
return includedFiles
}
func GetBasicAuth(username, password string) *githttp.BasicAuth {
if password == "" {
return nil
}
if username == "" {
username = "token"
}
return &githttp.BasicAuth{
Username: username,
Password: password,
}
}
api/git/service_test.go
+343
View File
@@ -0,0 +1,343 @@
package git
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/require"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
type mockRepoManager struct {
downloadErr error
commitID string
commitIDErr error
refs []string
refsErr error
files []string
filesErr error
downloadCalled int
commitIDCalled int
listRefsCalled int
listFilesCalled int
lastCloneOptions *git.CloneOptions
}
func (m *mockRepoManager) Download(_ context.Context, _ string, opts *git.CloneOptions) error {
m.downloadCalled++
m.lastCloneOptions = opts
return m.downloadErr
}
func (m *mockRepoManager) LatestCommitID(_ context.Context, _, _ string, _ *git.ListOptions) (string, error) {
m.commitIDCalled++
return m.commitID, m.commitIDErr
}
func (m *mockRepoManager) ListRefs(_ context.Context, _ string, _ *git.ListOptions) ([]string, error) {
m.listRefsCalled++
return m.refs, m.refsErr
}
func (m *mockRepoManager) ListFiles(_ context.Context, _ bool, _ *git.CloneOptions) ([]string, error) {
m.listFilesCalled++
return m.files, m.filesErr
}
func newTestService(ctx context.Context, cacheSize int, gitMgr, azureMgr RepoManager) *Service {
s := newService(ctx, cacheSize, 0)
s.git = gitMgr
s.azure = azureMgr
return s
}
func TestCloneRepository(t *testing.T) {
t.Parallel()
downloadErr := errors.New("clone failed")
testCases := []struct {
name string
url string
referenceName string
gitManagerDownloadCalled int
azureManagerDownloadCalled int
managerErr bool
expectedError error
expectedReferenceName string
}{
{
name: "non-azure URL routes to git manager",
url: "https://github.com/example/repo.git",
gitManagerDownloadCalled: 1,
azureManagerDownloadCalled: 0,
},
{
name: "azure URL routes to azure manager",
url: "https://dev.azure.com/org/project/_git/repo",
gitManagerDownloadCalled: 0,
azureManagerDownloadCalled: 1,
},
{
name: "error from manager propagated",
url: "https://github.com/example/repo.git",
managerErr: true,
gitManagerDownloadCalled: 1,
expectedError: downloadErr,
},
{
name: "ReferenceName is passed to clone options",
url: "https://github.com/example/repo.git",
referenceName: "refs/heads/feature-branch",
gitManagerDownloadCalled: 1,
expectedReferenceName: "refs/heads/feature-branch",
},
{
name: "empty ReferenceName leaves clone options unset",
url: "https://github.com/example/repo.git",
referenceName: "",
gitManagerDownloadCalled: 1,
expectedReferenceName: "",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
gitMgr := &mockRepoManager{}
azureMgr := &mockRepoManager{}
if tc.managerErr {
gitMgr.downloadErr = downloadErr
azureMgr.downloadErr = downloadErr
}
s := newTestService(t.Context(), 4, gitMgr, azureMgr)
err := s.CloneRepository(t.Context(), "/tmp", tc.url, tc.referenceName, "", "", false)
require.Equal(t, tc.expectedError, err)
require.Equal(t, tc.gitManagerDownloadCalled, gitMgr.downloadCalled)
require.Equal(t, tc.azureManagerDownloadCalled, azureMgr.downloadCalled)
activeMgr := gitMgr
if tc.azureManagerDownloadCalled > 0 {
activeMgr = azureMgr
}
if activeMgr.lastCloneOptions != nil {
require.Equal(t, plumbing.ReferenceName(tc.expectedReferenceName), activeMgr.lastCloneOptions.ReferenceName)
}
})
}
}
func TestLatestCommitID(t *testing.T) {
t.Parallel()
commitLookupErr := errors.New("commit lookup failed")
testCases := []struct {
name string
url string
gitCommitID string
azureCommitID string
commitIDErr error
expectedID string
expectedError error
gitCalled int
azureCalled int
}{
{
name: "non-azure URL routes to git manager",
url: "https://github.com/example/repo.git",
gitCommitID: "abc123",
expectedID: "abc123",
gitCalled: 1,
},
{
name: "azure URL routes to azure manager",
url: "https://dev.azure.com/org/project/_git/repo",
azureCommitID: "def456",
expectedID: "def456",
azureCalled: 1,
},
{
name: "error propagated",
url: "https://github.com/example/repo.git",
commitIDErr: commitLookupErr,
expectedError: commitLookupErr,
gitCalled: 1,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
gitMgr := &mockRepoManager{commitID: tc.gitCommitID, commitIDErr: tc.commitIDErr}
azureMgr := &mockRepoManager{commitID: tc.azureCommitID}
s := newTestService(t.Context(), 4, gitMgr, azureMgr)
id, err := s.LatestCommitID(t.Context(), tc.url, "", "", "", false)
require.Equal(t, tc.expectedError, err)
require.Equal(t, tc.expectedID, id)
require.Equal(t, tc.gitCalled, gitMgr.commitIDCalled)
require.Equal(t, tc.azureCalled, azureMgr.commitIDCalled)
})
}
}
func TestListRefs(t *testing.T) {
t.Parallel()
t.Run("cache hit on second call", func(t *testing.T) {
gitMgr := &mockRepoManager{refs: []string{"refs/heads/main", "refs/heads/develop"}}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
refs1, err := s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.NoError(t, err)
refs2, err := s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.NoError(t, err)
require.Equal(t, 1, gitMgr.listRefsCalled, "expected manager to be called once")
require.Equal(t, refs1, refs2)
})
t.Run("hard refresh clears cache and calls manager again", func(t *testing.T) {
gitMgr := &mockRepoManager{refs: []string{"refs/heads/main"}}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
_, err := s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.NoError(t, err)
_, err = s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", true, false)
require.NoError(t, err)
require.Equal(t, 2, gitMgr.listRefsCalled, "expected manager to be called twice with hard refresh")
})
t.Run("error propagated and not cached", func(t *testing.T) {
wantErr := errors.New("refs failed")
gitMgr := &mockRepoManager{refsErr: wantErr}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
_, err := s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.Equal(t, wantErr, err)
_, err = s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", true, false)
require.Equal(t, wantErr, err)
require.Equal(t, 2, gitMgr.listRefsCalled, "expected manager to be called twice after error")
})
t.Run("azure URL routes to azure manager", func(t *testing.T) {
gitMgr := &mockRepoManager{}
azureMgr := &mockRepoManager{refs: []string{"refs/heads/main"}}
s := newTestService(t.Context(), 4, gitMgr, azureMgr)
_, err := s.ListRefs(t.Context(), "https://dev.azure.com/org/project/_git/repo", "", "", false, false)
require.NoError(t, err)
require.Equal(t, 1, azureMgr.listRefsCalled, "expected azure.ListRefs to be called once")
require.Equal(t, 0, gitMgr.listRefsCalled, "expected git.ListRefs to not be called")
})
t.Run("cache disabled: manager always called", func(t *testing.T) {
gitMgr := &mockRepoManager{refs: []string{"refs/heads/main"}}
s := newTestService(t.Context(), 0, gitMgr, &mockRepoManager{})
_, err := s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.NoError(t, err)
_, err = s.ListRefs(t.Context(), "https://github.com/example/repo.git", "", "", false, false)
require.NoError(t, err)
require.Equal(t, 2, gitMgr.listRefsCalled, "expected manager to be called twice with cache disabled")
})
}
func TestListFiles(t *testing.T) {
t.Parallel()
t.Run("cache hit on second call", func(t *testing.T) {
gitMgr := &mockRepoManager{files: []string{"docker-compose.yml", "README.md"}}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
files1, err := s.ListFiles(t.Context(), "https://github.com/example/repo.git", "refs/heads/main", "", "", false, false, nil, false)
require.NoError(t, err)
files2, err := s.ListFiles(t.Context(), "https://github.com/example/repo.git", "refs/heads/main", "", "", false, false, nil, false)
require.NoError(t, err)
require.Equal(t, 1, gitMgr.listFilesCalled, "expected manager to be called once")
require.Equal(t, files1, files2)
})
t.Run("hard refresh clears file cache", func(t *testing.T) {
gitMgr := &mockRepoManager{files: []string{"docker-compose.yml"}}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
_, err := s.ListFiles(t.Context(), "https://github.com/example/repo.git", "refs/heads/main", "", "", false, false, nil, false)
require.NoError(t, err)
_, err = s.ListFiles(t.Context(), "https://github.com/example/repo.git", "refs/heads/main", "", "", false, true, nil, false)
require.NoError(t, err)
require.Equal(t, 2, gitMgr.listFilesCalled, "expected manager to be called twice with hard refresh")
})
t.Run("azure URL routes to azure manager", func(t *testing.T) {
gitMgr := &mockRepoManager{}
azureMgr := &mockRepoManager{files: []string{"docker-compose.yml"}}
s := newTestService(t.Context(), 4, gitMgr, azureMgr)
_, err := s.ListFiles(t.Context(), "https://dev.azure.com/org/project/_git/repo", "", "", "", false, false, nil, false)
require.NoError(t, err)
require.Equal(t, 1, azureMgr.listFilesCalled, "expected azure.ListFiles to be called once")
require.Equal(t, 0, gitMgr.listFilesCalled, "expected git.ListFiles to not be called")
})
t.Run("extension filter applied", func(t *testing.T) {
gitMgr := &mockRepoManager{files: []string{"docker-compose.yml", "README.md", "stack.yml", "config.json"}}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
files, err := s.ListFiles(t.Context(), "https://github.com/example/repo.git", "", "", "", false, false, []string{".yml"}, false)
require.NoError(t, err)
require.Equal(t, []string{"docker-compose.yml", "stack.yml"}, files)
})
t.Run("error is returned and not cached", func(t *testing.T) {
wantErr := errors.New("list files failed")
gitMgr := &mockRepoManager{filesErr: wantErr}
s := newTestService(t.Context(), 4, gitMgr, &mockRepoManager{})
files, err := s.ListFiles(t.Context(), "https://github.com/example/repo.git", "refs/heads/main", "", "", false, false, nil, false)
require.ErrorIs(t, err, wantErr)
require.Nil(t, files)
})
}
func TestFilterFiles(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
files []string
exts []string
expectedFiles []string
}{
{
name: "empty ext list returns all files",
files: []string{"a.yml", "b.json", "c.txt"},
exts: nil,
expectedFiles: []string{"a.yml", "b.json", "c.txt"},
},
{
name: "non-matching exts returns empty",
files: []string{"a.yml", "b.json"},
exts: []string{".txt"},
expectedFiles: nil,
},
{
name: "partial match returns only matching files",
files: []string{"a.yml", "b.json", "c.yml"},
exts: []string{".yml"},
expectedFiles: []string{"a.yml", "c.yml"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.expectedFiles, filterFiles(tc.files, tc.exts))
})
}
}
api/git/ssrf_transport.go
+53
View File
@@ -0,0 +1,53 @@
package git
import (
"context"
"fmt"
"net"
"strconv"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
gittransport "github.com/go-git/go-git/v5/plumbing/transport"
)
const gitDefaultPort = 9418
// ssrfGitTransport wraps a git:// transport and validates the resolved IP
// against the SSRF policy before establishing connections.
type ssrfGitTransport struct {
inner gittransport.Transport
}
// NewSSRFGitTransport wraps inner and blocks connections to private IP ranges
// according to the active SSRF policy.
func NewSSRFGitTransport(inner gittransport.Transport) gittransport.Transport {
return &ssrfGitTransport{inner: inner}
}
func (t *ssrfGitTransport) NewUploadPackSession(ep *gittransport.Endpoint, auth gittransport.AuthMethod) (gittransport.UploadPackSession, error) {
if err := checkEndpointSSRF(ep); err != nil {
return nil, err
}
return t.inner.NewUploadPackSession(ep, auth)
}
func (t *ssrfGitTransport) NewReceivePackSession(ep *gittransport.Endpoint, auth gittransport.AuthMethod) (gittransport.ReceivePackSession, error) {
if err := checkEndpointSSRF(ep); err != nil {
return nil, err
}
return t.inner.NewReceivePackSession(ep, auth)
}
func checkEndpointSSRF(ep *gittransport.Endpoint) error {
port := ep.Port
if port <= 0 {
port = gitDefaultPort
}
rawURL := fmt.Sprintf("git://%s/", net.JoinHostPort(ep.Host, strconv.Itoa(port)))
return ssrf.CheckURL(context.Background(), rawURL)
}
api/git/testdata/azure-repo copy.zip
Vendored
BIN
View File
Binary file not shown.
api/git/testdata/azure-repo.zip
Vendored
BIN
View File
Binary file not shown.
api/git/testdata/test-clone-git-repo.tar.gz
Vendored
BIN
View File
Binary file not shown.
api/git/types/types.go
+135
View File
@@ -0,0 +1,135 @@
package gittypes
import (
"cmp"
"errors"
"net/url"
"path"
"strings"
)
var (
ErrIncorrectRepositoryURL = errors.New("git repository could not be found, please ensure that the URL is correct")
ErrAuthenticationFailure = errors.New("authentication failed, please ensure that the git credentials are correct")
ErrSymlinkDetected = errors.New("repository contains a symlink, which is not allowed for security reasons")
)
type GitCredentialAuthType int
type GitProvider int
// RepoConfig represents a configuration for a repo
type RepoConfig struct {
// The repo url
URL string `example:"https://github.com/portainer/portainer.git"`
// The reference name
ReferenceName string `example:"refs/heads/branch_name"`
// ConfigFilePath is the path to the config file within the repository.
// NOTE: For stacks, this mirrors Stack.EntryPoint and the two are kept in sync by stackUpdateGit.
ConfigFilePath string `example:"docker-compose.yml"`
// Git credentials
Authentication *GitAuthentication `json:",omitempty"`
// Repository hash
ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
// TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"`
}
// GitSource holds the shared connection fields stored on a Source.
// Per-file fields (ref, path, hash) are stored on ArtifactFile instead.
type GitSource struct {
URL string `example:"https://github.com/portainer/portainer.git"`
Authentication *GitAuthentication `json:",omitempty"`
TLSSkipVerify bool `example:"false"`
}
// ToRepoConfig returns a RepoConfig populated with the connection fields from gc
func (gc *GitSource) ToRepoConfig() *RepoConfig {
return &RepoConfig{
URL: gc.URL,
Authentication: gc.Authentication,
TLSSkipVerify: gc.TLSSkipVerify,
}
}
// SanitizeGitSource returns a copy of gc with the URL sanitized and password cleared,
// safe to return to clients
func SanitizeGitSource(gc *GitSource) *GitSource {
if gc == nil {
return nil
}
result := *gc
result.URL = SanitizeURL(result.URL)
if result.Authentication != nil && result.Authentication.Password != "" {
auth := *result.Authentication
auth.Password = ""
result.Authentication = &auth
}
return &result
}
// RepoName extracts the repository name from a git URL for use as a display name.
// e.g. "https://github.com/org/app-config.git" results in "app-config"
func RepoName(rawURL string) string {
return strings.TrimSuffix(path.Base(rawURL), ".git")
}
// NormalizeURL returns a canonical form of rawURL for deduplication purposes:
// scheme and host are lowercased, embedded credentials are removed, trailing
// slashes and the .git suffix are stripped from the path. If the scheme is
// absent it defaults to https.
func NormalizeURL(rawURL string) (string, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", err
}
u.Scheme = strings.ToLower(cmp.Or(u.Scheme, "https"))
u.Host = strings.ToLower(u.Host)
u.User = nil
u.Path = strings.TrimSuffix(strings.TrimRight(u.Path, "/"), ".git")
return u.String(), nil
}
// SanitizeURL strips any userinfo (username/password) embedded in rawURL,
// returning a URL safe to store or return to clients.
func SanitizeURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil || u.User == nil {
return rawURL
}
u.User = nil
return u.String()
}
// SanitizeRepoConfig returns a copy of gc with the URL sanitized and password cleared,
// safe to return to clients.
func SanitizeRepoConfig(gc *RepoConfig) *RepoConfig {
if gc == nil {
return nil
}
result := *gc
result.URL = SanitizeURL(result.URL)
if result.Authentication != nil && result.Authentication.Password != "" {
auth := *result.Authentication
auth.Password = ""
result.Authentication = &auth
}
return &result
}
type GitAuthentication struct {
Username string
Password string
Provider GitProvider `json:",omitempty"`
AuthorizationType GitCredentialAuthType `json:",omitempty"`
}
api/git/types/types_test.go
+26
View File
@@ -0,0 +1,26 @@
package gittypes
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNormalizeURL(t *testing.T) {
t.Parallel()
f := func(input, expected string) {
t.Helper()
got, err := NormalizeURL(input)
require.NoError(t, err)
require.Equal(t, expected, got)
}
f("https://github.com/org/repo.git", "https://github.com/org/repo")
f("https://github.com/org/repo/", "https://github.com/org/repo")
f("https://github.com/org/repo.git/", "https://github.com/org/repo")
f("HTTPS://github.com/org/repo", "https://github.com/org/repo")
f("https://GitHub.COM/org/repo", "https://github.com/org/repo")
f("https://user:pass@github.com/org/repo.git", "https://github.com/org/repo")
f("https://github.com/org/repo", "https://github.com/org/repo")
}
api/git/update/update.go
+130
View File
@@ -0,0 +1,130 @@
package update
import (
"context"
"os"
"strings"
"time"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/rs/zerolog/log"
)
// UpdateGitObject updates a git object based on its config
func UpdateGitObject(ctx context.Context, gitService portainer.GitService, objId string, gitConfig *gittypes.RepoConfig, enableVersionFolder bool, projectPath string) (bool, string, error) {
if gitConfig == nil {
return false, "", nil
}
log.Debug().
Str("url", gitConfig.URL).
Str("ref", gitConfig.ReferenceName).
Str("object", objId).
Msg("the object has a git config, try to poll from git repository")
username, password := git.GetCredentials(gitConfig.Authentication)
fetchCtx, cancel := context.WithTimeout(ctx, time.Minute)
newHash, err := gitService.LatestCommitID(
fetchCtx,
gitConfig.URL,
gitConfig.ReferenceName,
username,
password,
gitConfig.TLSSkipVerify,
)
cancel()
if err != nil {
if fetchCtx.Err() == context.DeadlineExceeded {
log.Error().Str("object", objId).Msg("git fetch timed out after 1 minute")
}
return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId)
}
hashChanged := !strings.EqualFold(newHash, gitConfig.ConfigHash)
if !hashChanged {
log.Debug().
Str("hash", newHash).
Str("url", gitConfig.URL).
Str("ref", gitConfig.ReferenceName).
Str("object", objId).
Msg("git repo is up to date")
return false, newHash, nil
}
toDir := projectPath
if enableVersionFolder {
toDir = filesystem.JoinPaths(projectPath, newHash)
}
cloneParams := &cloneRepositoryParameters{
url: gitConfig.URL,
ref: gitConfig.ReferenceName,
toDir: toDir,
tlsSkipVerify: gitConfig.TLSSkipVerify,
}
if gitConfig.Authentication != nil {
cloneParams.auth = &gitAuth{
username: username,
password: password,
}
}
if err := cloneGitRepository(ctx, gitService, cloneParams); err != nil {
if enableVersionFolder {
if removeErr := os.RemoveAll(toDir); removeErr != nil {
log.Warn().Err(removeErr).Str("dir", toDir).Msg("failed to remove partial clone directory")
}
}
return false, "", errors.WithMessagef(err, "failed to do a fresh clone of %v", objId)
}
log.Debug().
Str("hash", newHash).
Str("url", gitConfig.URL).
Str("ref", gitConfig.ReferenceName).
Str("object", objId).
Msg("git repo cloned updated")
return true, newHash, nil
}
type cloneRepositoryParameters struct {
url string
ref string
toDir string
auth *gitAuth
// tlsSkipVerify skips SSL verification when cloning the Git repository
tlsSkipVerify bool `example:"false"`
}
type gitAuth struct {
username string
password string
}
func cloneGitRepository(ctx context.Context, gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error {
username, password := "", ""
if cloneParams.auth != nil {
username = cloneParams.auth.username
password = cloneParams.auth.password
}
return gitService.CloneRepository(
ctx,
cloneParams.toDir,
cloneParams.url,
cloneParams.ref,
username,
password,
cloneParams.tlsSkipVerify,
)
}
api/git/update/validate.go
+35
View File
@@ -0,0 +1,35 @@
package update
import (
"time"
portainer "github.com/portainer/portainer/api"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/pkg/validate"
)
func ValidateAutoUpdateSettings(autoUpdate *portainer.AutoUpdateSettings) error {
if autoUpdate == nil {
return nil
}
if autoUpdate.Webhook == "" && autoUpdate.Interval == "" {
return httperrors.NewInvalidPayloadError("Webhook or Interval must be provided")
}
if autoUpdate.Webhook != "" && !validate.IsUUID(autoUpdate.Webhook) {
return httperrors.NewInvalidPayloadError("invalid Webhook format")
}
if autoUpdate.Interval == "" {
return nil
}
if d, err := time.ParseDuration(autoUpdate.Interval); err != nil {
return httperrors.NewInvalidPayloadError("invalid Interval format")
} else if d < time.Minute {
return httperrors.NewInvalidPayloadError("interval must be at least 1 minute")
}
return nil
}
api/git/update/validate_test.go
+53
View File
@@ -0,0 +1,53 @@
package update
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
)
func Test_ValidateAutoUpdate(t *testing.T) {
t.Parallel()
tests := []struct {
name string
value *portainer.AutoUpdateSettings
wantErr bool
}{
{
name: "webhook is not a valid UUID",
value: &portainer.AutoUpdateSettings{Webhook: "fake-webhook"},
wantErr: true,
},
{
name: "incorrect interval value",
value: &portainer.AutoUpdateSettings{Interval: "1dd2hh3mm"},
wantErr: true,
},
{
name: "short interval value",
value: &portainer.AutoUpdateSettings{Interval: "1s"},
wantErr: true,
},
{
name: "valid webhook without interval",
value: &portainer.AutoUpdateSettings{Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada"},
wantErr: false,
},
{
name: "valid auto update",
value: &portainer.AutoUpdateSettings{
Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada",
Interval: "5h30m40s10ms",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateAutoUpdateSettings(tt.value)
assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err)
})
}
}