chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,503 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
dserrors "github.com/portainer/portainer/api/dataservices/errors"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
const (
|
||||
DatabaseFileName = "portainer.db"
|
||||
EncryptedDatabaseFileName = "portainer.edb"
|
||||
|
||||
txMaxSize = 65536
|
||||
compactedSuffix = ".compacted"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start. Only one database should exist")
|
||||
ErrHaveEncryptedWithNoKey = errors.New("The portainer database is encrypted, but no secret was loaded")
|
||||
)
|
||||
|
||||
type DbConnection struct {
|
||||
Path string
|
||||
MaxBatchSize int
|
||||
MaxBatchDelay time.Duration
|
||||
InitialMmapSize int
|
||||
EncryptionKey []byte
|
||||
isEncrypted bool
|
||||
Compact bool
|
||||
|
||||
gcm cipher.AEAD
|
||||
|
||||
*bolt.DB
|
||||
}
|
||||
|
||||
// GetDatabaseFileName get the database filename
|
||||
func (connection *DbConnection) GetDatabaseFileName() string {
|
||||
if connection.IsEncryptedStore() {
|
||||
return EncryptedDatabaseFileName
|
||||
}
|
||||
|
||||
return DatabaseFileName
|
||||
}
|
||||
|
||||
// GetDataseFilePath get the path + filename for the database file
|
||||
func (connection *DbConnection) GetDatabaseFilePath() string {
|
||||
if connection.IsEncryptedStore() {
|
||||
return path.Join(connection.Path, EncryptedDatabaseFileName)
|
||||
}
|
||||
|
||||
return path.Join(connection.Path, DatabaseFileName)
|
||||
}
|
||||
|
||||
// GetStorePath get the filename and path for the database file
|
||||
func (connection *DbConnection) GetStorePath() string {
|
||||
return connection.Path
|
||||
}
|
||||
|
||||
func (connection *DbConnection) GetDatabaseFileSize() (int64, error) {
|
||||
file, err := os.Stat(connection.GetDatabaseFilePath())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("Failed to stat database file path: %s err: %w", connection.GetDatabaseFilePath(), err)
|
||||
}
|
||||
|
||||
return file.Size(), nil
|
||||
}
|
||||
|
||||
func (connection *DbConnection) SetEncrypted(flag bool) error {
|
||||
connection.isEncrypted = flag
|
||||
|
||||
if !flag || connection.EncryptionKey == nil {
|
||||
connection.gcm = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(connection.EncryptionKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating AES cipher for database encryption: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating GCM cipher for database encryption: %w", err)
|
||||
}
|
||||
|
||||
connection.gcm = gcm
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Return true if the database is encrypted
|
||||
func (connection *DbConnection) IsEncryptedStore() bool {
|
||||
return connection.getEncryptionKey() != nil
|
||||
}
|
||||
|
||||
// NeedsEncryptionMigration returns true if database encryption is enabled and
|
||||
// we have an un-encrypted DB that requires migration to an encrypted DB
|
||||
func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
|
||||
// Cases: Note, we need to check both portainer.db and portainer.edb
|
||||
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
|
||||
|
||||
// 1) portainer.edb + key => False
|
||||
// 2) portainer.edb + no key => ERROR Fatal!
|
||||
// 3) portainer.db + key => True (needs migration)
|
||||
// 4) portainer.db + no key => False
|
||||
// 5) NoDB (new) + key => False
|
||||
// 6) NoDB (new) + no key => False
|
||||
// 7) portainer.db & portainer.edb => ERROR Fatal!
|
||||
|
||||
// If we have a loaded encryption key, always set encrypted
|
||||
if connection.EncryptionKey != nil {
|
||||
if err := connection.SetEncrypted(true); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// Check for portainer.db
|
||||
dbFile := path.Join(connection.Path, DatabaseFileName)
|
||||
_, err := os.Stat(dbFile)
|
||||
haveDbFile := err == nil
|
||||
|
||||
// Check for portainer.edb
|
||||
edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
|
||||
_, err = os.Stat(edbFile)
|
||||
haveEdbFile := err == nil
|
||||
|
||||
if haveDbFile && haveEdbFile {
|
||||
// 7 - encrypted and unencrypted db?
|
||||
return false, ErrHaveEncryptedAndUnencrypted
|
||||
}
|
||||
|
||||
if haveDbFile && connection.EncryptionKey != nil {
|
||||
// 3 - needs migration
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if haveEdbFile && connection.EncryptionKey == nil {
|
||||
// 2 - encrypted db, but no key?
|
||||
return false, ErrHaveEncryptedWithNoKey
|
||||
}
|
||||
|
||||
// 1, 4, 5, 6
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Open opens and initializes the BoltDB database.
|
||||
func (connection *DbConnection) Open() error {
|
||||
log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
|
||||
|
||||
databasePath := connection.GetDatabaseFilePath()
|
||||
db, err := bolt.Open(databasePath, 0600, connection.boltOptions(connection.Compact))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db.MaxBatchSize = connection.MaxBatchSize
|
||||
db.MaxBatchDelay = connection.MaxBatchDelay
|
||||
connection.DB = db
|
||||
|
||||
if connection.Compact {
|
||||
log.Info().Msg("compacting database")
|
||||
if err := connection.compact(); err != nil {
|
||||
log.Error().Err(err).Msg("failed to compact database")
|
||||
|
||||
// Close the read-only database and re-open in read-write mode
|
||||
if err := connection.Close(); err != nil {
|
||||
log.Warn().Err(err).Msg("failure to close the database after failed compaction")
|
||||
}
|
||||
|
||||
connection.Compact = false
|
||||
|
||||
return connection.Open()
|
||||
} else {
|
||||
log.Info().Msg("database compaction completed")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the BoltDB database.
|
||||
// Safe to being called multiple times.
|
||||
func (connection *DbConnection) Close() error {
|
||||
log.Info().Msg("closing PortainerDB")
|
||||
|
||||
if connection.DB != nil {
|
||||
return connection.DB.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (connection *DbConnection) txFn(fn func(portainer.Transaction) error) func(*bolt.Tx) error {
|
||||
return func(tx *bolt.Tx) error {
|
||||
return fn(&DbTransaction{conn: connection, tx: tx})
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateTx executes the given function inside a read-write transaction
|
||||
func (connection *DbConnection) UpdateTx(fn func(portainer.Transaction) error) error {
|
||||
if connection.MaxBatchDelay > 0 && connection.MaxBatchSize > 1 {
|
||||
return connection.Batch(connection.txFn(fn))
|
||||
}
|
||||
|
||||
return connection.Update(connection.txFn(fn))
|
||||
}
|
||||
|
||||
// ViewTx executes the given function inside a read-only transaction
|
||||
func (connection *DbConnection) ViewTx(fn func(portainer.Transaction) error) error {
|
||||
return connection.View(connection.txFn(fn))
|
||||
}
|
||||
|
||||
// BackupTo backs up db to a provided writer.
|
||||
// It does hot backup and doesn't block other database reads and writes
|
||||
func (connection *DbConnection) BackupTo(w io.Writer) error {
|
||||
return connection.View(func(tx *bolt.Tx) error {
|
||||
_, err := tx.WriteTo(w)
|
||||
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (connection *DbConnection) ExportRaw(filename string) error {
|
||||
databasePath := connection.GetDatabaseFilePath()
|
||||
if _, err := os.Stat(databasePath); err != nil {
|
||||
return fmt.Errorf("stat on %s failed, error: %w", databasePath, err)
|
||||
}
|
||||
|
||||
b, err := connection.ExportJSON(databasePath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(filename, b, 0600)
|
||||
}
|
||||
|
||||
// ConvertToKey returns an 8-byte big endian representation of v.
|
||||
// This function is typically used for encoding integer IDs to byte slices
|
||||
// so that they can be used as BoltDB keys.
|
||||
func (connection *DbConnection) ConvertToKey(v int) []byte {
|
||||
b := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(b, uint64(v))
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// keyToString Converts a key to a string value suitable for logging
|
||||
func keyToString(b []byte) string {
|
||||
if len(b) != 8 {
|
||||
return string(b)
|
||||
}
|
||||
|
||||
v := binary.BigEndian.Uint64(b)
|
||||
if v <= math.MaxInt32 {
|
||||
return strconv.FormatUint(v, 10)
|
||||
}
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// CreateBucket is a generic function used to create a bucket inside a database.
|
||||
func (connection *DbConnection) SetServiceName(bucketName string) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.SetServiceName(bucketName)
|
||||
})
|
||||
}
|
||||
|
||||
// GetObject is a generic function used to retrieve an unmarshalled object from a database.
|
||||
func (connection *DbConnection) GetObject(bucketName string, key []byte, object any) error {
|
||||
return connection.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetObject(bucketName, key, object)
|
||||
})
|
||||
}
|
||||
|
||||
func (connection *DbConnection) GetRawBytes(bucketName string, key []byte) ([]byte, error) {
|
||||
var value []byte
|
||||
|
||||
err := connection.ViewTx(func(tx portainer.Transaction) error {
|
||||
var err error
|
||||
value, err = tx.GetRawBytes(bucketName, key)
|
||||
|
||||
return err
|
||||
})
|
||||
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (connection *DbConnection) KeyExists(bucketName string, key []byte) (bool, error) {
|
||||
var exists bool
|
||||
|
||||
err := connection.ViewTx(func(tx portainer.Transaction) error {
|
||||
var err error
|
||||
exists, err = tx.KeyExists(bucketName, key)
|
||||
|
||||
return err
|
||||
})
|
||||
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (connection *DbConnection) getEncryptionKey() []byte {
|
||||
if !connection.isEncrypted {
|
||||
return nil
|
||||
}
|
||||
|
||||
return connection.EncryptionKey
|
||||
}
|
||||
|
||||
// UpdateObject is a generic function used to update an object inside a database.
|
||||
func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object any) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.UpdateObject(bucketName, key, object)
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateObjectFunc is a generic function used to update an object safely without race conditions.
|
||||
func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error {
|
||||
return connection.Batch(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(bucketName))
|
||||
|
||||
data := bucket.Get(key)
|
||||
if data == nil {
|
||||
return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
|
||||
}
|
||||
|
||||
err := connection.UnmarshalObject(data, object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updateFn()
|
||||
|
||||
data, err = connection.MarshalObject(object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bucket.Put(key, data)
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteObject is a generic function used to delete an object inside a database.
|
||||
func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.DeleteObject(bucketName, key)
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteAllObjects delete all objects where matching() returns (id, ok).
|
||||
// TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
|
||||
func (connection *DbConnection) DeleteAllObjects(bucketName string, obj any, matching func(o any) (id int, ok bool)) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.DeleteAllObjects(bucketName, obj, matching)
|
||||
})
|
||||
}
|
||||
|
||||
// GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
|
||||
func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
|
||||
var identifier int
|
||||
|
||||
_ = connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
identifier = tx.GetNextIdentifier(bucketName)
|
||||
return nil
|
||||
})
|
||||
|
||||
return identifier
|
||||
}
|
||||
|
||||
// CreateObject creates a new object in the bucket, using the next bucket sequence id
|
||||
func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, any)) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.CreateObject(bucketName, fn)
|
||||
})
|
||||
}
|
||||
|
||||
// CreateObjectWithId creates a new object in the bucket, using the specified id
|
||||
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj any) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.CreateObjectWithId(bucketName, id, obj)
|
||||
})
|
||||
}
|
||||
|
||||
// CreateObjectWithStringId creates a new object in the bucket, using the specified id
|
||||
func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj any) error {
|
||||
return connection.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.CreateObjectWithStringId(bucketName, id, obj)
|
||||
})
|
||||
}
|
||||
|
||||
func (connection *DbConnection) GetAll(bucketName string, obj any, appendFn func(o any) (any, error)) error {
|
||||
return connection.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetAll(bucketName, obj, appendFn)
|
||||
})
|
||||
}
|
||||
|
||||
func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, appendFn func(o any) (any, error)) error {
|
||||
return connection.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, appendFn)
|
||||
})
|
||||
}
|
||||
|
||||
// BackupMetadata will return a copy of the boltdb sequence numbers for all buckets.
|
||||
func (connection *DbConnection) BackupMetadata() (map[string]any, error) {
|
||||
buckets := map[string]any{}
|
||||
|
||||
err := connection.View(func(tx *bolt.Tx) error {
|
||||
return tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
|
||||
bucketName := string(name)
|
||||
seqId := bucket.Sequence()
|
||||
buckets[bucketName] = int(seqId)
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
return buckets, err
|
||||
}
|
||||
|
||||
// RestoreMetadata will restore the boltdb sequence numbers for all buckets.
|
||||
func (connection *DbConnection) RestoreMetadata(s map[string]any) error {
|
||||
var err error
|
||||
|
||||
for bucketName, v := range s {
|
||||
id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
|
||||
if !ok {
|
||||
log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
err = connection.Batch(func(tx *bolt.Tx) error {
|
||||
bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bucket.SetSequence(uint64(id))
|
||||
})
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// compact attempts to compact the database and replace it iff it succeeds
|
||||
func (connection *DbConnection) compact() (err error) {
|
||||
compactedPath := connection.GetDatabaseFilePath() + compactedSuffix
|
||||
|
||||
if err := os.Remove(compactedPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("failure to remove an existing compacted database: %w", err)
|
||||
}
|
||||
|
||||
compactedDB, err := bolt.Open(compactedPath, 0o600, connection.boltOptions(false))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failure to create the compacted database: %w", err)
|
||||
}
|
||||
|
||||
compactedDB.MaxBatchSize = connection.MaxBatchSize
|
||||
compactedDB.MaxBatchDelay = connection.MaxBatchDelay
|
||||
|
||||
if err := bolt.Compact(compactedDB, connection.DB, txMaxSize); err != nil {
|
||||
return fmt.Errorf("failure to compact the database: %w",
|
||||
errors.Join(err, compactedDB.Close(), os.Remove(compactedPath)))
|
||||
}
|
||||
|
||||
if err := os.Rename(compactedPath, connection.GetDatabaseFilePath()); err != nil {
|
||||
return fmt.Errorf("failure to move the compacted database: %w",
|
||||
errors.Join(err, compactedDB.Close(), os.Remove(compactedPath)))
|
||||
}
|
||||
|
||||
if err := connection.Close(); err != nil {
|
||||
log.Warn().Err(err).Msg("failure to close the database after compaction")
|
||||
}
|
||||
|
||||
connection.DB = compactedDB
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (connection *DbConnection) boltOptions(readOnly bool) *bolt.Options {
|
||||
return &bolt.Options{
|
||||
Timeout: 1 * time.Second,
|
||||
InitialMmapSize: connection.InitialMmapSize,
|
||||
FreelistType: bolt.FreelistMapType,
|
||||
NoFreelistSync: true,
|
||||
ReadOnly: readOnly,
|
||||
NoStatistics: true,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/portainer/portainer/api/filesystem"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
func Test_NeedsEncryptionMigration(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Test the specific scenarios mentioned in NeedsEncryptionMigration
|
||||
|
||||
// i.e.
|
||||
// Cases: Note, we need to check both portainer.db and portainer.edb
|
||||
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
|
||||
|
||||
// 1) portainer.edb + key => False
|
||||
// 2) portainer.edb + no key => ERROR Fatal!
|
||||
// 3) portainer.db + key => True (needs migration)
|
||||
// 4) portainer.db + no key => False
|
||||
// 5) NoDB (new) + key => False
|
||||
// 6) NoDB (new) + no key => False
|
||||
// 7) portainer.db & portainer.edb (key not important) => ERROR Fatal!
|
||||
|
||||
is := assert.New(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
dbname string
|
||||
key bool
|
||||
expectError error
|
||||
expectResult bool
|
||||
}{
|
||||
{
|
||||
name: "portainer.edb + key",
|
||||
dbname: EncryptedDatabaseFileName,
|
||||
key: true,
|
||||
expectError: nil,
|
||||
expectResult: false,
|
||||
},
|
||||
{
|
||||
name: "portainer.db + key (migration needed)",
|
||||
dbname: DatabaseFileName,
|
||||
key: true,
|
||||
expectError: nil,
|
||||
expectResult: true,
|
||||
},
|
||||
{
|
||||
name: "portainer.db + no key",
|
||||
dbname: DatabaseFileName,
|
||||
key: false,
|
||||
expectError: nil,
|
||||
expectResult: false,
|
||||
},
|
||||
{
|
||||
name: "NoDB (new) + key",
|
||||
dbname: "",
|
||||
key: false,
|
||||
expectError: nil,
|
||||
expectResult: false,
|
||||
},
|
||||
{
|
||||
name: "NoDB (new) + no key",
|
||||
dbname: "",
|
||||
key: false,
|
||||
expectError: nil,
|
||||
expectResult: false,
|
||||
},
|
||||
|
||||
// error tests
|
||||
{
|
||||
name: "portainer.edb + no key",
|
||||
dbname: EncryptedDatabaseFileName,
|
||||
key: false,
|
||||
expectError: ErrHaveEncryptedWithNoKey,
|
||||
expectResult: false,
|
||||
},
|
||||
{
|
||||
name: "portainer.db & portainer.edb",
|
||||
dbname: "both",
|
||||
key: true,
|
||||
expectError: ErrHaveEncryptedAndUnencrypted,
|
||||
expectResult: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
connection := DbConnection{Path: dir}
|
||||
|
||||
if tc.dbname == "both" {
|
||||
// Special case. If portainer.db and portainer.edb exist.
|
||||
dbFile1 := filesystem.JoinPaths(connection.Path, DatabaseFileName)
|
||||
f, _ := os.Create(dbFile1)
|
||||
|
||||
err := f.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
err := os.Remove(dbFile1)
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
|
||||
dbFile2 := filesystem.JoinPaths(connection.Path, EncryptedDatabaseFileName)
|
||||
f, _ = os.Create(dbFile2)
|
||||
|
||||
err = f.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
err := os.Remove(dbFile2)
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
} else if tc.dbname != "" {
|
||||
dbFile := filesystem.JoinPaths(connection.Path, tc.dbname)
|
||||
f, _ := os.Create(dbFile)
|
||||
|
||||
err := f.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
err := os.Remove(dbFile)
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
}
|
||||
|
||||
if tc.key {
|
||||
connection.EncryptionKey = secretToEncryptionKey("secret")
|
||||
}
|
||||
|
||||
result, err := connection.NeedsEncryptionMigration()
|
||||
|
||||
is.Equal(tc.expectError, err, "Fatal Error failure. Test: %s", tc.name)
|
||||
is.Equal(result, tc.expectResult, "Failed test: %s", tc.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetEncrypted_InvalidKeyReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conn := DbConnection{EncryptionKey: []byte("bad")}
|
||||
err := conn.SetEncrypted(true)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, conn.gcm)
|
||||
}
|
||||
|
||||
func TestSetEncrypted_NilKeyDoesNotSetGCM(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conn := DbConnection{}
|
||||
err := conn.SetEncrypted(true)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, conn.gcm)
|
||||
}
|
||||
|
||||
func TestSetEncrypted_EnableThenDisableStopsEncryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
conn := DbConnection{EncryptionKey: key}
|
||||
|
||||
err := conn.SetEncrypted(true)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, conn.gcm)
|
||||
|
||||
err = conn.SetEncrypted(false)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, conn.gcm)
|
||||
|
||||
// MarshalObject must return plaintext after encryption is disabled
|
||||
data, err := conn.MarshalObject("hello")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hello", string(data))
|
||||
}
|
||||
|
||||
func TestNeedsEncryptionMigration_InvalidKeyError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
conn := DbConnection{
|
||||
Path: t.TempDir(),
|
||||
EncryptionKey: []byte("bad"),
|
||||
}
|
||||
|
||||
result, err := conn.NeedsEncryptionMigration()
|
||||
require.Error(t, err)
|
||||
require.False(t, result)
|
||||
}
|
||||
|
||||
func TestDBCompaction(t *testing.T) {
|
||||
t.Parallel()
|
||||
db := &DbConnection{Path: t.TempDir()}
|
||||
|
||||
err := db.Open()
|
||||
require.NoError(t, err)
|
||||
|
||||
err = db.Update(func(tx *bbolt.Tx) error {
|
||||
b, err := tx.CreateBucketIfNotExists([]byte("testbucket"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = b.Put([]byte("key"), []byte("value"))
|
||||
require.NoError(t, err)
|
||||
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = db.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Reopen the DB to trigger compaction
|
||||
db.Compact = true
|
||||
err = db.Open()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check that the data is still there
|
||||
err = db.View(func(tx *bbolt.Tx) error {
|
||||
b := tx.Bucket([]byte("testbucket"))
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
val := b.Get([]byte("key"))
|
||||
require.Equal(t, []byte("value"), val)
|
||||
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = db.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Failures
|
||||
compactedPath := db.GetDatabaseFilePath() + compactedSuffix
|
||||
err = os.Mkdir(compactedPath, 0o755)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := os.Create(filesystem.JoinPaths(compactedPath, "somefile"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
err = db.Open()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/portainer/portainer/api/logs"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/segmentio/encoding/json"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
func backupMetadata(connection *bolt.DB) (map[string]any, error) {
|
||||
buckets := map[string]any{}
|
||||
|
||||
err := connection.View(func(tx *bolt.Tx) error {
|
||||
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
|
||||
bucketName := string(name)
|
||||
seqId := bucket.Sequence()
|
||||
buckets[bucketName] = int(seqId)
|
||||
return nil
|
||||
})
|
||||
|
||||
return err
|
||||
})
|
||||
|
||||
return buckets, err
|
||||
}
|
||||
|
||||
// ExportJSON creates a JSON representation from a DbConnection. You can include
|
||||
// the database's metadata or ignore it. Ensure the database is closed before
|
||||
// using this function.
|
||||
// inspired by github.com/konoui/boltdb-exporter (which has no license)
|
||||
// but very much simplified, based on how we use boltdb
|
||||
func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, error) {
|
||||
log.Debug().Str("databasePath", databasePath).Msg("exportJson")
|
||||
|
||||
connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true, NoStatistics: true})
|
||||
if err != nil {
|
||||
return []byte("{}"), err
|
||||
}
|
||||
defer logs.CloseAndLogErr(connection)
|
||||
|
||||
backup := make(map[string]any)
|
||||
if metadata {
|
||||
meta, err := backupMetadata(connection)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed exporting metadata")
|
||||
}
|
||||
|
||||
backup["__metadata"] = meta
|
||||
}
|
||||
|
||||
if err := connection.View(func(tx *bolt.Tx) error {
|
||||
return tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
|
||||
bucketName := string(name)
|
||||
var list []any
|
||||
version := make(map[string]string)
|
||||
cursor := bucket.Cursor()
|
||||
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var obj any
|
||||
err := c.UnmarshalObject(v, &obj)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("bucket", bucketName).
|
||||
Str("object", string(v)).
|
||||
Err(err).
|
||||
Msg("failed to unmarshal")
|
||||
|
||||
obj = v
|
||||
}
|
||||
|
||||
if bucketName == "version" {
|
||||
version[string(k)] = string(v)
|
||||
} else {
|
||||
list = append(list, obj)
|
||||
}
|
||||
}
|
||||
|
||||
if bucketName == "version" {
|
||||
backup[bucketName] = version
|
||||
return nil
|
||||
}
|
||||
|
||||
if bucketName == "ssl" ||
|
||||
bucketName == "settings" ||
|
||||
bucketName == "tunnel_server" {
|
||||
backup[bucketName] = nil
|
||||
if len(list) > 0 {
|
||||
backup[bucketName] = list[0]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
backup[bucketName] = list
|
||||
|
||||
return nil
|
||||
})
|
||||
}); err != nil {
|
||||
return []byte("{}"), err
|
||||
}
|
||||
|
||||
return json.MarshalIndent(backup, "", " ")
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/cipher"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/segmentio/encoding/json"
|
||||
)
|
||||
|
||||
var errEncryptedStringTooShort = errors.New("encrypted string too short")
|
||||
|
||||
// MarshalObject encodes an object to binary format
|
||||
func (connection *DbConnection) MarshalObject(object any) ([]byte, error) {
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
// Special case for the VERSION bucket. Here we're not using json
|
||||
if v, ok := object.(string); ok {
|
||||
buf.WriteString(v)
|
||||
} else {
|
||||
enc := json.NewEncoder(buf)
|
||||
enc.SetSortMapKeys(false)
|
||||
enc.SetAppendNewline(false)
|
||||
|
||||
if err := enc.Encode(object); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if connection.gcm == nil {
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
return encrypt(buf.Bytes(), connection.gcm), nil
|
||||
}
|
||||
|
||||
// UnmarshalObject decodes an object from binary data
|
||||
func (connection *DbConnection) UnmarshalObject(data []byte, object any) error {
|
||||
var err error
|
||||
if connection.gcm != nil {
|
||||
data, err = decrypt(data, connection.gcm)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed decrypting object")
|
||||
}
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, object); err != nil {
|
||||
// Special case for the VERSION bucket. Here we're not using json
|
||||
// So we need to return it as a string
|
||||
s, ok := object.(*string)
|
||||
if !ok {
|
||||
return errors.Wrap(err, "Failed unmarshalling object")
|
||||
}
|
||||
|
||||
*s = string(data)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func encrypt(plaintext []byte, gcm cipher.AEAD) []byte {
|
||||
return gcm.Seal(nil, nil, plaintext, nil)
|
||||
}
|
||||
|
||||
func decrypt(encrypted []byte, gcm cipher.AEAD) ([]byte, error) {
|
||||
if string(encrypted) == "false" {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
|
||||
if len(encrypted) < gcm.Overhead() {
|
||||
return encrypted, errEncryptedStringTooShort
|
||||
}
|
||||
|
||||
plaintextByte, err := gcm.Open(nil, nil, encrypted, nil)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error decrypting text")
|
||||
}
|
||||
|
||||
return plaintextByte, nil
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
|
||||
passphrase = "my secret key"
|
||||
)
|
||||
|
||||
func secretToEncryptionKey(passphrase string) []byte {
|
||||
hash := sha256.Sum256([]byte(passphrase))
|
||||
return hash[:]
|
||||
}
|
||||
|
||||
func Test_MarshalObjectUnencrypted(t *testing.T) {
|
||||
t.Parallel()
|
||||
is := assert.New(t)
|
||||
|
||||
uuid := uuid.New()
|
||||
|
||||
tests := []struct {
|
||||
object any
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
object: nil,
|
||||
expected: `null`,
|
||||
},
|
||||
{
|
||||
object: true,
|
||||
expected: `true`,
|
||||
},
|
||||
{
|
||||
object: false,
|
||||
expected: `false`,
|
||||
},
|
||||
{
|
||||
object: 123,
|
||||
expected: `123`,
|
||||
},
|
||||
{
|
||||
object: "456",
|
||||
expected: "456",
|
||||
},
|
||||
{
|
||||
object: uuid,
|
||||
expected: "\"" + uuid.String() + "\"",
|
||||
},
|
||||
{
|
||||
object: uuid.String(),
|
||||
expected: uuid.String(),
|
||||
},
|
||||
{
|
||||
object: map[string]any{"key": "value"},
|
||||
expected: `{"key":"value"}`,
|
||||
},
|
||||
{
|
||||
object: []bool{true, false},
|
||||
expected: `[true,false]`,
|
||||
},
|
||||
{
|
||||
object: []int{1, 2, 3},
|
||||
expected: `[1,2,3]`,
|
||||
},
|
||||
{
|
||||
object: []string{"1", "2", "3"},
|
||||
expected: `["1","2","3"]`,
|
||||
},
|
||||
{
|
||||
object: []map[string]any{{"key1": "value1"}, {"key2": "value2"}},
|
||||
expected: `[{"key1":"value1"},{"key2":"value2"}]`,
|
||||
},
|
||||
{
|
||||
object: []any{1, "2", false, map[string]any{"key1": "value1"}},
|
||||
expected: `[1,"2",false,{"key1":"value1"}]`,
|
||||
},
|
||||
}
|
||||
|
||||
conn := DbConnection{}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
data, err := conn.MarshalObject(test.object)
|
||||
require.NoError(t, err)
|
||||
is.Equal(test.expected, string(data))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_UnMarshalObjectUnencrypted(t *testing.T) {
|
||||
t.Parallel()
|
||||
is := assert.New(t)
|
||||
|
||||
// Based on actual data entering and what we expect out of the function
|
||||
|
||||
tests := []struct {
|
||||
object []byte
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
object: []byte(""),
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
object: []byte("35"),
|
||||
expected: "35",
|
||||
},
|
||||
{
|
||||
// An unmarshalled byte string should return the same without error
|
||||
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
|
||||
expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
|
||||
},
|
||||
{
|
||||
// An un-marshalled json object string should return the same as a string without error also
|
||||
object: []byte(jsonobject),
|
||||
expected: jsonobject,
|
||||
},
|
||||
}
|
||||
|
||||
conn := DbConnection{}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
var object string
|
||||
err := conn.UnmarshalObject(test.object, &object)
|
||||
require.NoError(t, err)
|
||||
is.Equal(test.expected, object)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_ObjectMarshallingEncrypted(t *testing.T) {
|
||||
t.Parallel()
|
||||
is := assert.New(t)
|
||||
|
||||
// Based on actual data entering and what we expect out of the function
|
||||
|
||||
tests := []struct {
|
||||
object []byte
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
object: []byte(""),
|
||||
},
|
||||
{
|
||||
object: []byte("35"),
|
||||
},
|
||||
{
|
||||
// An unmarshalled byte string should return the same without error
|
||||
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
|
||||
},
|
||||
{
|
||||
// An un-marshalled json object string should return the same as a string without error also
|
||||
object: []byte(jsonobject),
|
||||
},
|
||||
}
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
conn := DbConnection{EncryptionKey: key}
|
||||
err := conn.SetEncrypted(true)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
|
||||
data, err := conn.MarshalObject(test.object)
|
||||
require.NoError(t, err)
|
||||
|
||||
var object []byte
|
||||
err = conn.UnmarshalObject(data, &object)
|
||||
|
||||
require.NoError(t, err)
|
||||
is.Equal(test.object, object)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_NonceSources(t *testing.T) {
|
||||
t.Parallel()
|
||||
// ensure that the new go 1.24 NewGCMWithRandomNonce works correctly with
|
||||
// the old way of creating and including the nonce
|
||||
|
||||
encryptOldFn := func(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
|
||||
block, _ := aes.NewCipher(passphrase)
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return encrypted, err
|
||||
}
|
||||
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return encrypted, err
|
||||
}
|
||||
|
||||
return gcm.Seal(nonce, nonce, plaintext, nil), nil
|
||||
}
|
||||
|
||||
decryptOldFn := func(encrypted []byte, passphrase []byte) (plaintext []byte, err error) {
|
||||
block, err := aes.NewCipher(passphrase)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error creating cypher block")
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error creating GCM")
|
||||
}
|
||||
|
||||
nonceSize := gcm.NonceSize()
|
||||
if len(encrypted) < nonceSize {
|
||||
return encrypted, errEncryptedStringTooShort
|
||||
}
|
||||
|
||||
nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
|
||||
|
||||
plaintext, err = gcm.Open(nil, nonce, ciphertextByteClean, nil)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error decrypting text")
|
||||
}
|
||||
|
||||
return plaintext, err
|
||||
}
|
||||
|
||||
passphrase := make([]byte, 32)
|
||||
_, err := io.ReadFull(rand.Reader, passphrase)
|
||||
require.NoError(t, err)
|
||||
|
||||
block, err := aes.NewCipher(passphrase)
|
||||
require.NoError(t, err)
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
require.NoError(t, err)
|
||||
|
||||
junk := make([]byte, 1024)
|
||||
_, err = io.ReadFull(rand.Reader, junk)
|
||||
require.NoError(t, err)
|
||||
|
||||
junkEnc := make([]byte, base64.StdEncoding.EncodedLen(len(junk)))
|
||||
base64.StdEncoding.Encode(junkEnc, junk)
|
||||
|
||||
cases := [][]byte{
|
||||
[]byte("test"),
|
||||
[]byte("35"),
|
||||
[]byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
|
||||
[]byte(jsonobject),
|
||||
passphrase,
|
||||
junk,
|
||||
junkEnc,
|
||||
}
|
||||
|
||||
for _, plain := range cases {
|
||||
var enc, dec []byte
|
||||
var err error
|
||||
|
||||
enc, err = encryptOldFn(plain, passphrase)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err = decrypt(enc, gcm)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, plain, dec)
|
||||
|
||||
enc = encrypt(plain, gcm)
|
||||
|
||||
dec, err = decryptOldFn(enc, passphrase)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, plain, dec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecrypt_FalseStringBypassesDecryption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := decrypt([]byte("false"), gcm)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("false"), result)
|
||||
}
|
||||
|
||||
func TestDecrypt_ShortDataReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
require.NoError(t, err)
|
||||
|
||||
short := []byte("short")
|
||||
result, err := decrypt(short, gcm)
|
||||
require.ErrorIs(t, err, errEncryptedStringTooShort)
|
||||
require.Equal(t, short, result)
|
||||
}
|
||||
|
||||
func TestDecrypt_CorruptDataReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
block, err := aes.NewCipher(key)
|
||||
require.NoError(t, err)
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 30 bytes passes the length check but fails authentication
|
||||
corrupted := make([]byte, 30)
|
||||
_, err = io.ReadFull(rand.Reader, corrupted)
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := decrypt(corrupted, gcm)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, corrupted, result)
|
||||
}
|
||||
|
||||
// BenchmarkEncryptCachedCipher measures the new approach: cipher created once and reused.
|
||||
func BenchmarkEncryptCachedCipher(b *testing.B) {
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
conn := DbConnection{EncryptionKey: key}
|
||||
err := conn.SetEncrypted(true)
|
||||
require.NoError(b, err)
|
||||
|
||||
data := []byte(jsonobject)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for b.Loop() {
|
||||
_ = encrypt(data, conn.gcm)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEncryptPerCallCipher measures the old approach: cipher created on every call.
|
||||
func BenchmarkEncryptPerCallCipher(b *testing.B) {
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
data := []byte(jsonobject)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for b.Loop() {
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCMWithRandomNonce(block)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
_ = gcm.Seal(nil, nil, data, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEncryptCachedCipherParallel verifies the cached cipher is safe for concurrent use.
|
||||
func BenchmarkEncryptCachedCipherParallel(b *testing.B) {
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
conn := DbConnection{EncryptionKey: key}
|
||||
err := conn.SetEncrypted(true)
|
||||
require.NoError(b, err)
|
||||
|
||||
data := []byte(jsonobject)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = encrypt(data, conn.gcm)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
|
||||
dserrors "github.com/portainer/portainer/api/dataservices/errors"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
type DbTransaction struct {
|
||||
conn *DbConnection
|
||||
tx *bolt.Tx
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) SetServiceName(bucketName string) error {
|
||||
_, err := tx.tx.CreateBucketIfNotExists([]byte(bucketName))
|
||||
return err
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) GetObject(bucketName string, key []byte, object any) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
value := bucket.Get(key)
|
||||
if value == nil {
|
||||
return fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
|
||||
}
|
||||
|
||||
return tx.conn.UnmarshalObject(value, object)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) GetRawBytes(bucketName string, key []byte) ([]byte, error) {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
value := bucket.Get(key)
|
||||
if value == nil {
|
||||
return nil, fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
|
||||
}
|
||||
|
||||
if tx.conn.gcm != nil {
|
||||
var err error
|
||||
|
||||
if value, err = decrypt(value, tx.conn.gcm); err != nil {
|
||||
return value, errors.Wrap(err, "Failed decrypting object")
|
||||
}
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) KeyExists(bucketName string, key []byte) (bool, error) {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
value := bucket.Get(key)
|
||||
|
||||
return value != nil, nil
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object any) error {
|
||||
data, err := tx.conn.MarshalObject(object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
return bucket.Put(key, data)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
return bucket.Delete(key)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj any, matchingFn func(o any) (id int, ok bool)) error {
|
||||
var ids []int
|
||||
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
cursor := bucket.Cursor()
|
||||
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
|
||||
err := tx.conn.UnmarshalObject(v, &obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if id, ok := matchingFn(obj); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
|
||||
for _, id := range ids {
|
||||
if err := bucket.Delete(tx.conn.ConvertToKey(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
id, err := bucket.NextSequence()
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifier")
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
return int(id)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, any)) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
seqId, _ := bucket.NextSequence()
|
||||
id, obj := fn(seqId)
|
||||
|
||||
data, err := tx.conn.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bucket.Put(tx.conn.ConvertToKey(id), data)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj any) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
data, err := tx.conn.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bucket.Put(tx.conn.ConvertToKey(id), data)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte, obj any) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
data, err := tx.conn.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return bucket.Put(id, data)
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) GetAll(bucketName string, obj any, appendFn func(o any) (any, error)) error {
|
||||
bucket := tx.tx.Bucket([]byte(bucketName))
|
||||
|
||||
return bucket.ForEach(func(k []byte, v []byte) error {
|
||||
err := tx.conn.UnmarshalObject(v, obj)
|
||||
if err == nil {
|
||||
obj, err = appendFn(obj)
|
||||
}
|
||||
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, appendFn func(o any) (any, error)) error {
|
||||
cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
|
||||
|
||||
for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
|
||||
err := tx.conn.UnmarshalObject(v, obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
obj, err = appendFn(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/dataservices"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testBucketName = "test-bucket"
|
||||
const testId = 1234
|
||||
|
||||
type testStruct struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
func TestTxs(t *testing.T) {
|
||||
t.Parallel()
|
||||
conn := DbConnection{Path: t.TempDir()}
|
||||
|
||||
err := conn.Open()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
err := conn.Close()
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
// Error propagation
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return errors.New("this is an error")
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Create an object
|
||||
newObj := testStruct{Key: "key", Value: "value"}
|
||||
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
if err := tx.SetServiceName(testBucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.CreateObjectWithId(testBucketName, testId, newObj)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
obj := testStruct{}
|
||||
err = conn.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
if obj.Key != newObj.Key || obj.Value != newObj.Value {
|
||||
t.Fatalf("expected %s:%s, got %s:%s instead", newObj.Key, newObj.Value, obj.Key, obj.Value)
|
||||
}
|
||||
|
||||
// Update an object
|
||||
updatedObj := testStruct{Key: "updated-key", Value: "updated-value"}
|
||||
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.UpdateObject(testBucketName, conn.ConvertToKey(testId), &updatedObj)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = conn.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
if obj.Key != updatedObj.Key || obj.Value != updatedObj.Value {
|
||||
t.Fatalf("expected %s:%s, got %s:%s instead", updatedObj.Key, updatedObj.Value, obj.Key, obj.Value)
|
||||
}
|
||||
|
||||
// Delete an object
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
return tx.DeleteObject(testBucketName, conn.ConvertToKey(testId))
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = conn.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetObject(testBucketName, conn.ConvertToKey(testId), &obj)
|
||||
})
|
||||
require.True(t, dataservices.IsErrObjectNotFound(err))
|
||||
|
||||
// Get next identifier
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
id1 := tx.GetNextIdentifier(testBucketName)
|
||||
id2 := tx.GetNextIdentifier(testBucketName)
|
||||
|
||||
if id1+1 != id2 {
|
||||
return errors.New("unexpected identifier sequence")
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to write in a read transaction
|
||||
err = conn.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.CreateObjectWithId(testBucketName, testId, newObj)
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func BenchmarkGetAll(b *testing.B) {
|
||||
const endpointBucket = "endpoints"
|
||||
const n = 10000
|
||||
|
||||
conn := DbConnection{Path: b.TempDir()}
|
||||
|
||||
err := conn.Open()
|
||||
require.NoError(b, err)
|
||||
b.Cleanup(func() {
|
||||
err := conn.Close()
|
||||
require.NoError(b, err)
|
||||
})
|
||||
|
||||
err = conn.UpdateTx(func(tx portainer.Transaction) error {
|
||||
if err := tx.SetServiceName(endpointBucket); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 1; i <= n; i++ {
|
||||
ep := portainer.Endpoint{
|
||||
ID: portainer.EndpointID(i),
|
||||
Name: "env-" + strconv.Itoa(i),
|
||||
Type: portainer.DockerEnvironment,
|
||||
URL: "tcp://192.168.1." + strconv.Itoa(i%254+1) + ":2375",
|
||||
PublicURL: "https://env-" + strconv.Itoa(i) + ".example.com",
|
||||
GroupID: portainer.EndpointGroupID(i%10 + 1),
|
||||
TagIDs: []portainer.TagID{portainer.TagID(i%5 + 1), portainer.TagID(i%3 + 1)},
|
||||
LastCheckInDate: int64(i) * 1000,
|
||||
EdgeID: "edge-" + strconv.Itoa(i),
|
||||
}
|
||||
|
||||
if err := tx.CreateObjectWithId(endpointBucket, i, &ep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
require.NoError(b, err)
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for b.Loop() {
|
||||
var collection []portainer.Endpoint
|
||||
|
||||
if err := conn.ViewTx(func(tx portainer.Transaction) error {
|
||||
return tx.GetAll(endpointBucket, new(portainer.Endpoint), dataservices.AppendFn(&collection))
|
||||
}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/database/boltdb"
|
||||
)
|
||||
|
||||
// NewDatabase should use config options to return a connection to the requested database
|
||||
func NewDatabase(storeType, storePath string, encryptionKey []byte, compact bool) (connection portainer.Connection, err error) {
|
||||
if storeType == "boltdb" {
|
||||
return &boltdb.DbConnection{
|
||||
Path: storePath,
|
||||
EncryptionKey: encryptionKey,
|
||||
Compact: compact,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("Unknown storage database: %s", storeType)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/portainer/portainer/api/database/boltdb"
|
||||
"github.com/portainer/portainer/api/filesystem"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
dbPath := filesystem.JoinPaths(t.TempDir(), "test.db")
|
||||
connection, err := NewDatabase("boltdb", dbPath, nil, false)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, connection)
|
||||
|
||||
_, ok := connection.(*boltdb.DbConnection)
|
||||
require.True(t, ok)
|
||||
|
||||
connection, err = NewDatabase("unknown", dbPath, nil, false)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, connection)
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package models
|
||||
|
||||
type Version struct {
|
||||
SchemaVersion string
|
||||
MigratorCount int
|
||||
Edition int
|
||||
InstanceID string
|
||||
}
|
||||
Reference in New Issue
Block a user