chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:08:39 +08:00
commit a0df89c693
5252 changed files with 523444 additions and 0 deletions
+149
View File
@@ -0,0 +1,149 @@
package compose
import (
"crypto/sha256"
"encoding/hex"
"io"
"os"
"path/filepath"
"sort"
"github.com/compose-spec/compose-go/v2/types"
"github.com/rs/zerolog/log"
)
const BindMountHashLabelKey = "io.portainer.bind-mount-hash"
func addBindMountHashLabel(name string, s types.ServiceConfig) (types.ServiceConfig, error) {
hashes := []string{}
for _, volume := range s.Volumes {
// Calculate hash for bind mounts only for now
if volume.Type != "bind" {
continue
}
// Calculate hash for volume.Source, volume.Source can be a file or dir
// and volume.Source is already an absolute path so we can hash it directly
hash, err := pathHash(volume.Source)
if err != nil {
// If we fail to calculate the hash for this bind mount, skip it and continue
log.Debug().Err(err).
Str("bind_mount_source", volume.Source).
Str("service", name).
Msg("failed to calculate hash for bind mount, skipping this bind mount from hash label calculation")
continue
}
if hash != "" {
hashes = append(hashes, hash)
}
}
if len(hashes) == 0 {
return s, nil
}
// Sort hashes to ensure deterministic output
sort.Strings(hashes)
// Final hash of the combined hashes
finalH := sha256.New()
for _, h := range hashes {
finalH.Write([]byte(h))
}
value := hex.EncodeToString(finalH.Sum(nil))
if s.Labels == nil {
s.Labels = make(map[string]string)
}
s.Labels[BindMountHashLabelKey] = value
log.Debug().Str("service", name).
Str("label_key", BindMountHashLabelKey).
Str("bind_mount_hash", value).
Msg("Calculated bind mount hash for service")
return s, nil
}
// pathHash calculates a SHA-256 hash for a file or a directory.
func pathHash(path string) (string, error) {
hash := sha256.New()
info, err := os.Stat(path)
if err != nil {
return "", err
}
if !info.IsDir() {
// It's a single file
return hashFile(path)
}
// It's a directory: we must collect and sort all files for determinism
var files []string
if err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
files = append(files, p)
}
return nil
}); err != nil {
return "", err
}
sort.Strings(files)
for _, f := range files {
// Include the relative path in the hash so that renames and moves within
// the directory change the hash even when file contents stay the same.
relPath, err := filepath.Rel(path, f)
if err != nil {
return "", err
}
if _, err := hash.Write([]byte(relPath)); err != nil {
return "", err
}
// Stream the file content into the same hasher
if err := copyFileToHash(hash, f); err != nil {
return "", err
}
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
func hashFile(path string) (string, error) {
h := sha256.New()
if err := copyFileToHash(h, path); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// copyFileToHash opens the file at path, streams its content into w, and closes it.
// If the copy fails, the close error is logged but the copy error is returned.
// If the copy succeeds but close fails, the close error is returned.
func copyFileToHash(w io.Writer, path string) (err error) {
f, err := os.Open(path)
if err != nil {
return err
}
defer func() {
if cerr := f.Close(); cerr != nil {
log.Debug().Err(cerr).
Str("filename", path).
Msg("error closing file after hash")
if err == nil {
err = cerr
}
}
}()
_, err = io.Copy(w, f)
return err
}
@@ -0,0 +1,129 @@
package compose
import (
"os"
"testing"
"github.com/portainer/portainer/api/filesystem"
"github.com/compose-spec/compose-go/v2/types"
"github.com/stretchr/testify/require"
)
func TestPathHash_File(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filesystem.JoinPaths(dir, "file.txt")
require.NoError(t, os.WriteFile(path, []byte("hello"), 0644))
h1, err := pathHash(path)
require.NoError(t, err)
require.NotEmpty(t, h1)
// Same content, same hash
h2, err := pathHash(path)
require.NoError(t, err)
require.Equal(t, h1, h2)
// Different content, different hash
require.NoError(t, os.WriteFile(path, []byte("world"), 0644))
h3, err := pathHash(path)
require.NoError(t, err)
require.NotEqual(t, h1, h3)
}
func TestPathHash_Directory(t *testing.T) {
t.Parallel()
dir := t.TempDir()
require.NoError(t, os.WriteFile(filesystem.JoinPaths(dir, "a.txt"), []byte("aaa"), 0644))
require.NoError(t, os.WriteFile(filesystem.JoinPaths(dir, "b.txt"), []byte("bbb"), 0644))
h1, err := pathHash(dir)
require.NoError(t, err)
// Same directory -> same hash
h2, err := pathHash(dir)
require.NoError(t, err)
require.Equal(t, h1, h2)
// Rename a file -> different hash (relative path is part of the hash)
require.NoError(t, os.Rename(filesystem.JoinPaths(dir, "a.txt"), filesystem.JoinPaths(dir, "c.txt")))
h3, err := pathHash(dir)
require.NoError(t, err)
require.NotEqual(t, h1, h3, "renaming a file should change the directory hash")
// Restore and change content -> different hash
require.NoError(t, os.Rename(filesystem.JoinPaths(dir, "c.txt"), filesystem.JoinPaths(dir, "a.txt")))
require.NoError(t, os.WriteFile(filesystem.JoinPaths(dir, "a.txt"), []byte("modified"), 0644))
h4, err := pathHash(dir)
require.NoError(t, err)
require.NotEqual(t, h1, h4, "changing file content should change the directory hash")
}
func TestAddBindMountHashLabel(t *testing.T) {
t.Parallel()
dir := t.TempDir()
webDir := filesystem.JoinPaths(dir, "web")
require.NoError(t, os.MkdirAll(webDir, 0755))
require.NoError(t, os.WriteFile(filesystem.JoinPaths(webDir, "nginx.conf"), []byte("server {}"), 0644))
t.Run("no bind mounts", func(t *testing.T) {
svc := types.ServiceConfig{Name: "web"}
result, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.Empty(t, result.Labels[BindMountHashLabelKey])
})
t.Run("non-bind volume is skipped", func(t *testing.T) {
svc := types.ServiceConfig{
Name: "web",
Volumes: []types.ServiceVolumeConfig{{Type: "volume", Source: "myvolume"}},
}
result, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.Empty(t, result.Labels[BindMountHashLabelKey])
})
t.Run("missing path silently skips label", func(t *testing.T) {
svc := types.ServiceConfig{
Name: "web",
Volumes: []types.ServiceVolumeConfig{{Type: "bind", Source: "/nonexistent/path"}},
}
result, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.Empty(t, result.Labels[BindMountHashLabelKey])
})
t.Run("valid bind mount with directory source sets label", func(t *testing.T) {
svc := types.ServiceConfig{
Name: "web",
Volumes: []types.ServiceVolumeConfig{{Type: "bind", Source: webDir}},
}
result, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.NotEmpty(t, result.Labels[BindMountHashLabelKey])
})
t.Run("valid bind mount with file source sets label", func(t *testing.T) {
svc := types.ServiceConfig{
Name: "web",
Volumes: []types.ServiceVolumeConfig{{Type: "bind", Source: filesystem.JoinPaths(webDir, "nginx.conf")}},
}
result, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.NotEmpty(t, result.Labels[BindMountHashLabelKey])
})
t.Run("label is deterministic", func(t *testing.T) {
svc := types.ServiceConfig{
Name: "web",
Volumes: []types.ServiceVolumeConfig{{Type: "bind", Source: webDir}},
}
r1, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
r2, err := addBindMountHashLabel("web", svc)
require.NoError(t, err)
require.Equal(t, r1.Labels[BindMountHashLabelKey], r2.Labels[BindMountHashLabelKey])
})
}
+18
View File
@@ -0,0 +1,18 @@
package compose
import (
"github.com/docker/cli/cli/command"
"github.com/docker/compose/v2/pkg/api"
"github.com/docker/compose/v2/pkg/compose"
)
type ComposeDeployer struct {
createComposeServiceFn func(command.Cli, ...compose.Option) api.Compose
}
// NewComposeDeployer creates a new compose deployer
func NewComposeDeployer() *ComposeDeployer {
return &ComposeDeployer{
createComposeServiceFn: compose.NewComposeService,
}
}
+100
View File
@@ -0,0 +1,100 @@
package compose_test
import (
"log"
"os"
"os/exec"
"strings"
"testing"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/pkg/libstack"
"github.com/portainer/portainer/pkg/libstack/compose"
"github.com/portainer/portainer/pkg/testhelpers"
)
func checkPrerequisites(t *testing.T) {
testhelpers.IntegrationTest(t)
}
func Test_UpAndDown(t *testing.T) {
t.Parallel()
checkPrerequisites(t)
deployer := compose.NewComposeDeployer()
const composeFileContent = `
version: "3.9"
services:
busybox:
image: "alpine:3.7"
container_name: "binarytest_container_one"
`
const overrideComposeFileContent = `
version: "3.9"
services:
busybox:
image: "alpine:latest"
container_name: "binarytest_container_two"
`
const composeContainerName = "binarytest_container_two"
dir := t.TempDir()
filePathOriginal, err := createFile(dir, "docker-compose.yml", composeFileContent)
if err != nil {
t.Fatal(err)
}
filePathOverride, err := createFile(dir, "docker-compose-override.yml", overrideComposeFileContent)
if err != nil {
t.Fatal(err)
}
projectName := "binarytest"
err = deployer.Deploy(t.Context(), []string{filePathOriginal, filePathOverride}, libstack.DeployOptions{
Options: libstack.Options{
ProjectName: projectName,
},
})
if err != nil {
t.Fatal(err)
}
if !containerExists(composeContainerName) {
t.Fatal("container should exist")
}
err = deployer.Remove(t.Context(), projectName, []string{filePathOriginal, filePathOverride}, libstack.RemoveOptions{})
if err != nil {
t.Fatal(err)
}
if containerExists(composeContainerName) {
t.Fatal("container should be removed")
}
}
func createFile(dir, fileName, content string) (string, error) {
filePath := filesystem.JoinPaths(dir, fileName)
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
return "", err
}
return filePath, nil
}
func containerExists(containerName string) bool {
cmd := exec.Command("docker", "ps", "-a", "-f", "name="+containerName)
out, err := cmd.Output()
if err != nil {
log.Fatalf("failed to list containers: %s", err)
}
return strings.Contains(string(out), containerName)
}
+359
View File
@@ -0,0 +1,359 @@
package compose
import (
"context"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/pkg/libstack"
"github.com/compose-spec/compose-go/v2/cli"
"github.com/compose-spec/compose-go/v2/types"
"github.com/docker/cli/cli/command"
cmdcompose "github.com/docker/compose/v2/cmd/compose"
"github.com/docker/compose/v2/pkg/api"
"github.com/docker/compose/v2/pkg/compose"
"github.com/docker/compose/v2/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/sirupsen/logrus"
)
const PortainerEdgeStackLabel = "io.portainer.edge_stack_id"
func init() {
logrus.SetOutput(LogrusToZerologWriter{})
logrus.SetFormatter(&logrus.TextFormatter{
DisableTimestamp: true,
})
}
func (c *ComposeDeployer) withComposeService(
ctx context.Context,
filePaths []string,
options libstack.Options,
composeFn func(api.Compose, *types.Project) error,
) error {
return libstack.WithCli(ctx,
libstack.DockerCliOptions{Host: options.Host, Registries: options.Registries},
func(ctx context.Context, cli *command.DockerCli) error {
composeService := c.createComposeServiceFn(cli)
if len(filePaths) == 0 {
return composeFn(composeService, nil)
}
project, err := createProject(ctx, filePaths, options)
if err != nil {
return fmt.Errorf("failed to create compose project: %w", err)
}
parallel := 0
if v, ok := project.Environment[cmdcompose.ComposeParallelLimit]; ok {
i, err := strconv.Atoi(v)
if err != nil {
return fmt.Errorf("%s must be an integer (found: %q)", cmdcompose.ComposeParallelLimit, v)
}
parallel = i
}
if parallel > 0 {
composeService.MaxConcurrency(parallel)
}
return composeFn(composeService, project)
})
}
// Deploy creates and starts containers
func (c *ComposeDeployer) Deploy(ctx context.Context, filePaths []string, options libstack.DeployOptions) error {
return c.withComposeService(ctx, filePaths, options.Options, func(composeService api.Compose, project *types.Project) error {
addServiceLabels(project, false, options.EdgeStackID)
project = project.WithoutUnnecessaryResources()
opts := api.UpOptions{
Start: api.StartOptions{
Project: project,
},
}
if options.ForceRecreate {
opts.Create.Recreate = api.RecreateForce
}
opts.Create.RemoveOrphans = options.RemoveOrphans
if removeOrphans, ok := project.Environment[cmdcompose.ComposeRemoveOrphans]; ok {
opts.Create.RemoveOrphans = utils.StringToBool(removeOrphans)
}
if ignoreOrphans, ok := project.Environment[cmdcompose.ComposeIgnoreOrphans]; ok {
opts.Create.IgnoreOrphans = utils.StringToBool(ignoreOrphans)
}
if options.AbortOnContainerExit {
opts.Start.OnExit = api.CascadeStop
}
if err := composeService.Build(ctx, project, api.BuildOptions{}); err != nil {
return fmt.Errorf("compose build operation failed: %w", err)
}
if err := composeService.Up(ctx, project, opts); err != nil {
return fmt.Errorf("compose up operation failed: %w", err)
}
log.Info().Msg("Stack deployment successful")
return nil
})
}
// Run runs the given service just once, without considering dependencies
func (c *ComposeDeployer) Run(ctx context.Context, filePaths []string, serviceName string, options libstack.RunOptions) error {
return c.withComposeService(ctx, filePaths, options.Options, func(composeService api.Compose, project *types.Project) error {
addServiceLabels(project, true, 0)
for name, service := range project.Services {
if name == serviceName {
project.DisabledServices[serviceName] = service
}
}
project.Services = make(types.Services)
if err := composeService.Create(ctx, project, api.CreateOptions{RemoveOrphans: true}); err != nil {
return fmt.Errorf("compose create operation failed: %w", err)
}
maps.Copy(project.Services, project.DisabledServices)
project.DisabledServices = make(types.Services)
opts := api.RunOptions{
AutoRemove: options.Remove,
Command: options.Args,
Detach: options.Detached,
Service: serviceName,
}
if _, err := composeService.RunOneOffContainer(ctx, project, opts); err != nil {
return fmt.Errorf("compose run operation failed: %w", err)
}
log.Info().Msg("Stack run successful")
return nil
})
}
// Remove stops and removes containers
func (c *ComposeDeployer) Remove(ctx context.Context, projectName string, filePaths []string, options libstack.RemoveOptions) error {
if err := libstack.WithCli(ctx,
libstack.DockerCliOptions{Host: options.Host, Registries: options.Registries},
func(ctx context.Context, cli *command.DockerCli) error {
composeService := compose.NewComposeService(cli)
return composeService.Down(ctx, projectName, api.DownOptions{RemoveOrphans: true, Volumes: options.Volumes})
}); err != nil {
return fmt.Errorf("compose down operation failed: %w", err)
}
log.Info().Msg("Stack removal successful")
return nil
}
// Pull pulls images
func (c *ComposeDeployer) Pull(ctx context.Context, filePaths []string, options libstack.Options) error {
if err := c.withComposeService(ctx, filePaths, options, func(composeService api.Compose, project *types.Project) error {
return composeService.Pull(ctx, project, api.PullOptions{})
}); err != nil {
return fmt.Errorf("compose pull operation failed: %w", err)
}
log.Info().Msg("Stack pull successful")
return nil
}
// Validate validates stack file
func (c *ComposeDeployer) Validate(ctx context.Context, filePaths []string, options libstack.Options) error {
return c.withComposeService(ctx, filePaths, options, func(composeService api.Compose, project *types.Project) error {
return nil
})
}
// Config returns the compose file with the paths resolved
func (c *ComposeDeployer) Config(ctx context.Context, filePaths []string, options libstack.Options) ([]byte, error) {
var payload []byte
if err := c.withComposeService(ctx, filePaths, options, func(composeService api.Compose, project *types.Project) error {
var err error
payload, err = project.MarshalYAML()
if err != nil {
return fmt.Errorf("unable to marshal as YAML: %w", err)
}
return nil
}); err != nil {
return nil, fmt.Errorf("compose config operation failed: %w", err)
}
return payload, nil
}
func (c *ComposeDeployer) GetExistingEdgeStacks(ctx context.Context) ([]libstack.EdgeStack, error) {
m := make(map[int]libstack.EdgeStack)
if err := c.withComposeService(ctx, nil, libstack.Options{}, func(composeService api.Compose, project *types.Project) error {
stacks, err := composeService.List(ctx, api.ListOptions{
All: true,
})
if err != nil {
return err
}
for _, s := range stacks {
summary, err := composeService.Ps(ctx, s.Name, api.PsOptions{All: true})
if err != nil {
return err
}
for _, cs := range summary {
if sid, ok := cs.Labels[PortainerEdgeStackLabel]; ok {
id, err := strconv.Atoi(sid)
if err != nil {
return err
}
if cs.Labels[api.ProjectLabel] == "" {
return errors.New("invalid project label")
}
m[id] = libstack.EdgeStack{
ID: id,
Name: cs.Labels[api.ProjectLabel],
ExitCode: cs.ExitCode,
}
}
}
}
return nil
}); err != nil {
return nil, err
}
return slices.Collect(maps.Values(m)), nil
}
func addServiceLabels(project *types.Project, oneOff bool, edgeStackID portainer.EdgeStackID) {
oneOffLabel := "False"
if oneOff {
oneOffLabel = "True"
}
for i, s := range project.Services {
s.CustomLabels = map[string]string{
api.ProjectLabel: project.Name,
api.ServiceLabel: s.Name,
api.VersionLabel: api.ComposeVersion,
api.WorkingDirLabel: project.WorkingDir,
api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","),
api.OneoffLabel: oneOffLabel,
}
if edgeStackID > 0 {
s.CustomLabels.Add(PortainerEdgeStackLabel, strconv.Itoa(int(edgeStackID)))
}
project.Services[i] = s
}
}
func createProject(ctx context.Context, configFilepaths []string, options libstack.Options) (*types.Project, error) {
var workingDir string
if len(configFilepaths) > 0 {
workingDir = filepath.Dir(configFilepaths[0])
}
if options.ProjectDir != "" {
// When relative paths are used in the compose file, the project directory is used as the base path
workingDir = options.ProjectDir
}
var envFiles []string
if options.EnvFilePath != "" {
envFiles = append(envFiles, options.EnvFilePath)
}
var composeEnvVars []string
for _, ev := range os.Environ() {
if strings.HasPrefix(ev, "COMPOSE_") {
composeEnvVars = append(composeEnvVars, ev)
}
}
projectOptions, err := cli.NewProjectOptions(configFilepaths,
cli.WithWorkingDirectory(workingDir),
cli.WithName(options.ProjectName),
cli.WithoutEnvironmentResolution,
cli.WithResolvedPaths(!slices.Contains(options.ConfigOptions, "--no-path-resolution")),
cli.WithEnv(libstack.PortainerEnvVars()),
cli.WithEnv(composeEnvVars),
cli.WithEnv(options.Env),
cli.WithEnvFiles(envFiles...),
func(o *cli.ProjectOptions) error {
if len(o.EnvFiles) > 0 {
return nil
}
if fs, ok := o.Environment[cmdcompose.ComposeEnvFiles]; ok {
o.EnvFiles = strings.Split(fs, ",")
}
return nil
},
cli.WithDotEnv,
cli.WithDefaultProfiles(),
cli.WithConfigFileEnv,
)
if err != nil {
return nil, fmt.Errorf("failed to load the compose file options : %w", err)
}
project, err := projectOptions.LoadProject(ctx)
if err != nil {
return nil, fmt.Errorf("failed to load the compose file : %w", err)
}
// Work around compose path handling
for i, service := range project.Services {
for j, envFile := range service.EnvFiles {
if !filepath.IsAbs(envFile.Path) {
project.Services[i].EnvFiles[j].Path = filesystem.JoinPaths(workingDir, envFile.Path)
}
}
}
// Set the services environment variables
if p, err := project.WithServicesEnvironmentResolved(true); err == nil {
project = p
} else {
return nil, fmt.Errorf("failed to resolve services environment: %w", err)
}
if options.BindMountHashEnabled {
// Set per-service label for bind mount hashes under each service
if project, err = project.WithServicesTransform(addBindMountHashLabel); err != nil {
log.Warn().
Err(err).
Msg("Failed to set bind mount hash labels, proceeding without them. Stack updates may not be detected when bind-mounted files change")
}
}
return project, nil
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,107 @@
package compose
import (
"errors"
"os"
"testing"
"github.com/compose-spec/compose-go/v2/types"
"github.com/google/go-cmp/cmp"
"github.com/portainer/portainer/pkg/libstack"
)
func Test_createProject_win(t *testing.T) {
t.Parallel()
dir := t.TempDir()
projectName := "create-project-test"
defer func() {
err := os.RemoveAll(dir)
if err != nil {
t.Fatalf("Failed to remove temp dir: %v", err)
}
}()
testcases := []struct {
name string
createFilesFn func() []string
configFilepaths []string
options libstack.Options
expectedProject *types.Project
}{
{
name: "Convert windows paths",
createFilesFn: func() []string {
var filepaths []string
filepaths = append(filepaths, createFile(t, dir, "docker-compose.yml", `services:
nginx:
container_name: nginx
image: nginx:latest
volumes:
- "C:\\Users\\Joey\\Desktop\\backend:/var/www/html"`))
return filepaths
},
configFilepaths: []string{dir + "/docker-compose.yml"},
options: libstack.Options{
WorkingDir: dir,
ProjectName: projectName,
Env: []string{"COMPOSE_CONVERT_WINDOWS_PATHS=true"},
},
expectedProject: &types.Project{
Name: projectName,
WorkingDir: dir,
Services: types.Services{
"nginx": {
Name: "nginx",
ContainerName: "nginx",
Environment: types.MappingWithEquals{},
Image: "nginx:latest",
Networks: map[string]*types.ServiceNetworkConfig{"default": nil},
Volumes: []types.ServiceVolumeConfig{
{
Type: "bind",
Source: "/c/Users/Joey/Desktop/backend",
Target: "/var/www/html",
ReadOnly: false,
Bind: &types.ServiceVolumeBind{CreateHostPath: true},
},
},
},
},
Networks: types.Networks{"default": {Name: "create-project-test_default"}},
ComposeFiles: []string{
dir + "/docker-compose.yml",
},
Environment: types.Mapping{"COMPOSE_PROJECT_NAME": "create-project-test", "COMPOSE_CONVERT_WINDOWS_PATHS": "true"},
DisabledServices: types.Services{},
Profiles: []string{""},
},
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
createdFiles := tc.createFilesFn()
defer func() {
var errs error
for _, f := range createdFiles {
errs = errors.Join(errs, os.Remove(f))
}
if errs != nil {
t.Fatalf("Failed to remove config files: %v", errs)
}
}()
gotProject, err := createProject(t.Context(), tc.configFilepaths, tc.options)
if err != nil {
t.Fatalf("Failed to create new project: %v", err)
}
if diff := cmp.Diff(gotProject, tc.expectedProject); diff != "" {
t.Fatalf("Projects are different:\n%s", diff)
}
})
}
}
@@ -0,0 +1,4 @@
version: '3.9'
services:
redis:
image: 'redis:alpine'
+58
View File
@@ -0,0 +1,58 @@
package compose
import (
"strings"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// LogrusToZerologWriter is a custom logrus writer that writes logrus logs to zerolog.
// logrus is the logging library used by Docker Compose.
type LogrusToZerologWriter struct{}
func (ltzw LogrusToZerologWriter) Write(p []byte) (n int, err error) {
logMessage := string(p)
logMessage = strings.TrimSuffix(logMessage, "\n")
// Parse the log level and message from the logrus log.
// This assumes logrus's default text format.
var level, message string
if strings.HasPrefix(logMessage, "time=") {
// Example logrus log: `time="2023-10-01T12:34:56Z" level=info msg="This is a log message" key=value`
parts := strings.SplitN(logMessage, " ", 4)
if len(parts) >= 3 {
level = strings.TrimPrefix(parts[1], "level=")
message = strings.TrimPrefix(parts[2], "msg=")
message = strings.Trim(message, `"`)
}
} else {
// Fallback for simpler log formats.
level = "info"
message = logMessage
}
// Map logrus levels to zerolog levels.
var zlogLevel zerolog.Level
switch level {
case "debug":
zlogLevel = zerolog.DebugLevel
case "info":
zlogLevel = zerolog.InfoLevel
case "warn", "warning":
zlogLevel = zerolog.WarnLevel
case "error":
zlogLevel = zerolog.ErrorLevel
case "fatal":
zlogLevel = zerolog.FatalLevel
case "panic":
zlogLevel = zerolog.PanicLevel
default:
zlogLevel = zerolog.InfoLevel
}
// Log the message using zerolog.
log.WithLevel(zlogLevel).Msg(message)
return len(p), nil
}
+248
View File
@@ -0,0 +1,248 @@
package compose
import (
"bytes"
"context"
"fmt"
"io"
"time"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/pkg/libstack"
"github.com/compose-spec/compose-go/v2/types"
"github.com/docker/cli/cli/command"
"github.com/docker/compose/v2/pkg/api"
"github.com/docker/docker/api/types/container"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type publisher struct {
URL string
TargetPort int
PublishedPort int
Protocol string
}
type service struct {
ID string
Name string
Image string
Command string
Project string
Service string
Created int64
State string
Status string
Health string
ExitCode int
Publishers []publisher
}
// docker container state can be one of "created", "running", "paused", "restarting", "removing", "exited", or "dead"
func getServiceStatus(ctx context.Context, service service) (libstack.Status, string) {
log.Debug().
Str("service", service.Name).
Str("state", service.State).
Int("exitCode", service.ExitCode).
Msg("getServiceStatus")
switch service.State {
case "created", "restarting", "paused":
return libstack.StatusStarting, ""
case "running":
return libstack.StatusRunning, ""
case "removing":
return libstack.StatusRemoving, ""
case "exited":
if service.ExitCode == 0 {
return libstack.StatusCompleted, ""
}
errorMessage, err := getContainerLogsTail(ctx, service)
if err != nil {
log.Error().
Err(err).
Str("service", service.Name).
Msg("failed to get logs from container")
errorMessage = fmt.Sprintf("service %s exited with code %d", service.Name, service.ExitCode)
}
return libstack.StatusError, errorMessage
case "dead":
if service.ExitCode == 0 {
return libstack.StatusRemoved, ""
}
errorMessage, err := getContainerLogsTail(ctx, service)
if err != nil {
log.Error().
Err(err).
Str("service", service.Name).
Msg("failed to get logs from container")
errorMessage = fmt.Sprintf("service %s exited with code %d", service.Name, service.ExitCode)
}
return libstack.StatusError, errorMessage
default:
return libstack.StatusUnknown, ""
}
}
func getContainerLogsTail(ctx context.Context, service service) (string, error) {
var combinedOutput bytes.Buffer
if err := libstack.WithCli(ctx, libstack.DockerCliOptions{}, func(ctx context.Context, cli *command.DockerCli) error {
out, err := cli.Client().ContainerLogs(ctx, service.Name, container.LogsOptions{
ShowStdout: true,
ShowStderr: true,
Timestamps: true,
Follow: false,
Tail: "20",
})
if err != nil {
return errors.Wrap(err, "unable to get logs from container")
}
defer logs.CloseAndLogErr(out)
_, err = io.Copy(&combinedOutput, out)
if err != nil {
return errors.Wrap(err, "unable to read container logs")
}
return nil
}); err != nil {
return "", errors.Wrap(err, "unable to get logs from container")
}
return combinedOutput.String(), nil
}
func aggregateStatuses(ctx context.Context, services []service) (libstack.Status, string) {
servicesCount := len(services)
if servicesCount == 0 {
log.Debug().
Msg("no services found")
return libstack.StatusRemoved, ""
}
statusCounts := make(map[libstack.Status]int)
errorMessage := ""
for _, service := range services {
status, serviceError := getServiceStatus(ctx, service)
if serviceError != "" {
errorMessage = serviceError
}
statusCounts[status]++
}
log.Debug().
Interface("statusCounts", statusCounts).
Str("errorMessage", errorMessage).
Msg("check_status")
if errorMessage != "" {
return libstack.StatusError, errorMessage
}
return libstack.AggregateStatusCounts(statusCounts, servicesCount), ""
}
func (c *ComposeDeployer) WaitForStatus(ctx context.Context, name string, status libstack.Status) libstack.WaitResult {
waitResult := libstack.WaitResult{Status: status}
for {
if ctx.Err() != nil {
waitResult.ErrorMsg = "failed to wait for status: " + ctx.Err().Error()
return waitResult
}
time.Sleep(1 * time.Second)
var containerSummaries []api.ContainerSummary
if err := c.withComposeService(ctx, nil, libstack.Options{ProjectName: name}, func(composeService api.Compose, project *types.Project) error {
var err error
psCtx, cancelFunc := context.WithTimeout(context.Background(), time.Minute)
defer cancelFunc()
containerSummaries, err = composeService.Ps(psCtx, name, api.PsOptions{All: true})
return err
}); err != nil {
log.Debug().
Str("project_name", name).
Err(err).
Msg("error from docker compose ps")
continue
}
services := serviceListFromContainerSummary(containerSummaries)
if len(services) == 0 && status == libstack.StatusRemoved {
return waitResult
}
aggregateStatus, errorMessage := aggregateStatuses(ctx, services)
if aggregateStatus == status {
return waitResult
}
if status == libstack.StatusRunning && aggregateStatus == libstack.StatusCompleted {
waitResult.Status = libstack.StatusCompleted
return waitResult
}
if errorMessage != "" {
waitResult.ErrorMsg = errorMessage
return waitResult
}
log.Debug().
Str("project_name", name).
Str("required_status", string(status)).
Str("status", string(aggregateStatus)).
Msg("waiting for status")
}
}
func serviceListFromContainerSummary(containerSummaries []api.ContainerSummary) []service {
var services []service
for _, cs := range containerSummaries {
var publishers []publisher
for _, p := range cs.Publishers {
publishers = append(publishers, publisher{
URL: p.URL,
TargetPort: p.TargetPort,
PublishedPort: p.PublishedPort,
Protocol: p.Protocol,
})
}
services = append(services, service{
ID: cs.ID,
Name: cs.Name,
Image: cs.Image,
Command: cs.Command,
Project: cs.Project,
Service: cs.Service,
Created: cs.Created,
State: cs.State,
Status: cs.Status,
Health: cs.Health,
ExitCode: cs.ExitCode,
Publishers: publishers,
})
}
return services
}
@@ -0,0 +1,115 @@
package compose
import (
"context"
"os"
"testing"
"time"
"github.com/portainer/portainer/pkg/libstack"
)
/*
1. starting = docker compose file that runs several services, one of them should be with status starting
2. running = docker compose file that runs successfully and returns status running
3. removing = run docker compose config, remove the stack, and return removing status
4. failed = run a valid docker compose file, but one of the services should fail to start (so "docker compose up" should run successfully, but one of the services should do something like `CMD ["exit", "1"]
5. removed = remove a compose stack and return status removed
*/
func ensureIntegrationTest(t *testing.T) {
if _, ok := os.LookupEnv("INTEGRATION_TEST"); !ok {
t.Skip("skip an integration test")
}
}
func TestComposeProjectStatus(t *testing.T) {
t.Parallel()
ensureIntegrationTest(t)
testCases := []struct {
TestName string
ComposeFile string
ExpectedStatus libstack.Status
ExpectedStatusMessage bool
}{
{
TestName: "running",
ComposeFile: "status_test_files/running.yml",
ExpectedStatus: libstack.StatusRunning,
},
{
TestName: "failed",
ComposeFile: "status_test_files/failed.yml",
ExpectedStatus: libstack.StatusError,
ExpectedStatusMessage: true,
},
}
w := NewComposeDeployer()
for _, testCase := range testCases {
t.Run(testCase.TestName, func(t *testing.T) {
projectName := testCase.TestName
err := w.Deploy(t.Context(), []string{testCase.ComposeFile}, libstack.DeployOptions{
Options: libstack.Options{
ProjectName: projectName,
},
})
if err != nil {
t.Fatalf("[test: %s] Failed to deploy compose file: %v", testCase.TestName, err)
}
time.Sleep(5 * time.Second)
status, statusMessage, err := waitForStatus(w, t.Context(), projectName, libstack.StatusRunning)
if err != nil {
t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
}
if status != testCase.ExpectedStatus {
t.Fatalf("[test: %s] Expected status: %s, got: %s", testCase.TestName, testCase.ExpectedStatus, status)
}
if testCase.ExpectedStatusMessage && statusMessage == "" {
t.Fatalf("[test: %s] Expected status message but got empty", testCase.TestName)
}
err = w.Remove(t.Context(), projectName, nil, libstack.RemoveOptions{})
if err != nil {
t.Fatalf("[test: %s] Failed to remove compose project: %v", testCase.TestName, err)
}
time.Sleep(20 * time.Second)
status, statusMessage, err = waitForStatus(w, t.Context(), projectName, libstack.StatusRemoved)
if err != nil {
t.Fatalf("[test: %s] Failed to get compose project status: %v", testCase.TestName, err)
}
if status != libstack.StatusRemoved {
t.Fatalf("[test: %s] Expected stack to be removed, got %s", testCase.TestName, status)
}
if statusMessage != "" {
t.Fatalf("[test: %s] Expected empty status message: %s, got: %s", "", testCase.TestName, statusMessage)
}
})
}
}
func waitForStatus(deployer libstack.Deployer, ctx context.Context, stackName string, requiredStatus libstack.Status) (libstack.Status, string, error) {
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()
result := deployer.WaitForStatus(ctx, stackName, requiredStatus)
if result.ErrorMsg == "" {
return result.Status, "", nil
}
return libstack.StatusError, result.ErrorMsg, nil
}
@@ -0,0 +1,7 @@
version: '3'
services:
web:
image: nginx:latest
failing-service:
image: busybox
command: ['false']
@@ -0,0 +1,4 @@
version: '3'
services:
web:
image: nginx:latest