chore: import upstream snapshot with attribution
govulncheck / govulncheck (push) Has been cancelled
Lint / golangci-lint (push) Has been cancelled
Run Tests / Unit Tests (push) Has been cancelled
Run Tests / Etcd Integration Tests (push) Has been cancelled
Harness (E2E) / Harnesses (mock LLM) (push) Has been cancelled
Harness (E2E) / Provider harnesses (live LLM conformance) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:33 +08:00
commit e071084ebe
982 changed files with 160368 additions and 0 deletions
+24
View File
@@ -0,0 +1,24 @@
# Config [![GoDoc](https://godoc.org/github.com/micro/go-micro/config?status.svg)](https://godoc.org/github.com/micro/go-micro/config)
Config is a pluggable dynamic config package
Most config in applications are statically configured or include complex logic to load from multiple sources.
Go Config makes this easy, pluggable and mergeable. You'll never have to deal with config in the same way again.
## Features
- **Dynamic Loading** - Load configuration from multiple source as and when needed. Go Config manages watching config sources
in the background and automatically merges and updates an in memory view.
- **Pluggable Sources** - Choose from any number of sources to load and merge config. The backend source is abstracted away into
a standard format consumed internally and decoded via encoders. Sources can be env vars, flags, file, etcd, k8s configmap, etc.
- **Mergeable Config** - If you specify multiple sources of config, regardless of format, they will be merged and presented in
a single view. This massively simplifies priority order loading and changes based on environment.
- **Observe Changes** - Optionally watch the config for changes to specific values. Hot reload your app using Go Config's watcher.
You don't have to handle ad-hoc hup reloading or whatever else, just keep reading the config and watch for changes if you need
to be notified.
- **Sane Defaults** - In case config loads badly or is completely wiped away for some unknown reason, you can specify fallback
values when accessing any config values directly. This ensures you'll always be reading some sane default in the event of a problem.
+101
View File
@@ -0,0 +1,101 @@
// Package config is an interface for dynamic configuration.
package config
import (
"context"
"go-micro.dev/v6/config/loader"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
"go-micro.dev/v6/config/source/file"
)
// Config is an interface abstraction for dynamic configuration.
type Config interface {
// provide the reader.Values interface
reader.Values
// Init the config
Init(opts ...Option) error
// Options in the config
Options() Options
// Stop the config loader/watcher
Close() error
// Load config sources
Load(source ...source.Source) error
// Force a source changeset sync
Sync() error
// Watch a value for changes
Watch(path ...string) (Watcher, error)
}
// Watcher is the config watcher.
type Watcher interface {
Next() (reader.Value, error)
Stop() error
}
type Options struct {
Loader loader.Loader
Reader reader.Reader
// for alternative data
Context context.Context
Source []source.Source
WithWatcherDisabled bool
}
type Option func(o *Options)
var (
// Default Config Manager.
DefaultConfig, _ = NewConfig()
)
// NewConfig returns new config.
func NewConfig(opts ...Option) (Config, error) {
return newConfig(opts...)
}
// Return config as raw json.
func Bytes() []byte {
return DefaultConfig.Bytes()
}
// Return config as a map.
func Map() map[string]interface{} {
return DefaultConfig.Map()
}
// Scan values to a go type.
func Scan(v interface{}) error {
return DefaultConfig.Scan(v)
}
// Force a source changeset sync.
func Sync() error {
return DefaultConfig.Sync()
}
// Get a value from the config.
func Get(path ...string) (reader.Value, error) {
return DefaultConfig.Get(path...)
}
// Load config sources.
func Load(source ...source.Source) error {
return DefaultConfig.Load(source...)
}
// Watch a value for changes.
func Watch(path ...string) (Watcher, error) {
return DefaultConfig.Watch(path...)
}
// LoadFile is short hand for creating a file source and loading it.
func LoadFile(path string) error {
return Load(file.NewSource(
file.WithPath(path),
))
}
+315
View File
@@ -0,0 +1,315 @@
package config
import (
"bytes"
"fmt"
"go-micro.dev/v6/config/loader"
"go-micro.dev/v6/config/loader/memory"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/reader/json"
"go-micro.dev/v6/config/source"
"sync"
"time"
)
type config struct {
// the current values
vals reader.Values
exit chan bool
closeMu sync.Mutex
closed bool
// the current snapshot
snap *loader.Snapshot
opts Options
sync.RWMutex
}
type watcher struct {
lw loader.Watcher
rd reader.Reader
value reader.Value
path []string
}
func newConfig(opts ...Option) (Config, error) {
var c config
err := c.Init(opts...)
if err != nil {
return nil, err
}
if !c.opts.WithWatcherDisabled {
go c.run()
}
return &c, nil
}
func (c *config) Init(opts ...Option) error {
c.opts = Options{
Reader: json.NewReader(),
}
c.exit = make(chan bool)
c.closeMu.Lock()
c.closed = false
c.closeMu.Unlock()
for _, o := range opts {
o(&c.opts)
}
// default loader uses the configured reader
if c.opts.Loader == nil {
loaderOpts := []loader.Option{memory.WithReader(c.opts.Reader)}
if c.opts.WithWatcherDisabled {
loaderOpts = append(loaderOpts, memory.WithWatcherDisabled())
}
c.opts.Loader = memory.NewLoader(loaderOpts...)
}
err := c.opts.Loader.Load(c.opts.Source...)
if err != nil {
return err
}
c.snap, err = c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.vals, err = c.opts.Reader.Values(c.snap.ChangeSet)
if err != nil {
return err
}
return nil
}
func (c *config) Options() Options {
return c.opts
}
func (c *config) run() {
watch := func(w loader.Watcher) error {
for {
// get changeset
snap, err := w.Next()
if err != nil {
return err
}
c.Lock()
if c.snap.Version >= snap.Version {
c.Unlock()
continue
}
// save
c.snap = snap
// set values
c.vals, _ = c.opts.Reader.Values(snap.ChangeSet)
c.Unlock()
}
}
for {
w, err := c.opts.Loader.Watch()
if err != nil {
time.Sleep(time.Second)
continue
}
done := make(chan bool)
// the stop watch func
go func() {
select {
case <-done:
case <-c.exit:
}
err := w.Stop()
fmt.Println(err)
}()
// block watch
if err := watch(w); err != nil {
// do something better
time.Sleep(time.Second)
}
// close done chan
close(done)
// if the config is closed exit
select {
case <-c.exit:
return
default:
}
}
}
func (c *config) Map() map[string]interface{} {
c.RLock()
defer c.RUnlock()
return c.vals.Map()
}
func (c *config) Scan(v interface{}) error {
c.RLock()
defer c.RUnlock()
return c.vals.Scan(v)
}
// sync loads all the sources, calls the parser and updates the config.
func (c *config) Sync() error {
if err := c.opts.Loader.Sync(); err != nil {
return err
}
snap, err := c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.Lock()
defer c.Unlock()
c.snap = snap
vals, err := c.opts.Reader.Values(snap.ChangeSet)
if err != nil {
return err
}
c.vals = vals
return nil
}
func (c *config) Close() error {
c.closeMu.Lock()
defer c.closeMu.Unlock()
if c.closed {
return nil
}
close(c.exit)
c.closed = true
return nil
}
func (c *config) Get(path ...string) (reader.Value, error) {
c.RLock()
defer c.RUnlock()
// did sync actually work?
if c.vals != nil {
return c.vals.Get(path...)
}
// no value
return newValue(), nil
}
func (c *config) Set(val interface{}, path ...string) {
c.Lock()
defer c.Unlock()
if c.vals != nil {
c.vals.Set(val, path...)
}
}
func (c *config) Del(path ...string) {
c.Lock()
defer c.Unlock()
if c.vals != nil {
c.vals.Del(path...)
}
}
func (c *config) Bytes() []byte {
c.RLock()
defer c.RUnlock()
if c.vals == nil {
return []byte{}
}
return c.vals.Bytes()
}
func (c *config) Load(sources ...source.Source) error {
c.Lock()
defer c.Unlock()
if err := c.opts.Loader.Load(sources...); err != nil {
return err
}
snap, err := c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.snap = snap
vals, err := c.opts.Reader.Values(snap.ChangeSet)
if err != nil {
return err
}
c.vals = vals
return nil
}
func (c *config) Watch(path ...string) (Watcher, error) {
value, err := c.Get(path...)
if err != nil {
return nil, err
}
w, err := c.opts.Loader.Watch(path...)
if err != nil {
return nil, err
}
return &watcher{
lw: w,
rd: c.opts.Reader,
path: path,
value: value,
}, nil
}
func (c *config) String() string {
return "config"
}
func (w *watcher) Next() (reader.Value, error) {
for {
s, err := w.lw.Next()
if err != nil {
return nil, err
}
// only process changes
if bytes.Equal(w.value.Bytes(), s.ChangeSet.Data) {
continue
}
v, err := w.rd.Values(s.ChangeSet)
if err != nil {
return nil, err
}
return v.Get()
}
}
func (w *watcher) Stop() error {
return w.lw.Stop()
}
+199
View File
@@ -0,0 +1,199 @@
package config
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
"go-micro.dev/v6/config/source"
"go-micro.dev/v6/config/source/env"
"go-micro.dev/v6/config/source/file"
"go-micro.dev/v6/config/source/memory"
)
func createFileForIssue18(t *testing.T, content string) *os.File {
data := []byte(content)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
return fh
}
func createFileForTest(t *testing.T) *os.File {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
return fh
}
func TestConfigCloseConcurrentIdempotent(t *testing.T) {
conf, err := NewConfig(WithWatcherDisabled())
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
const goroutines = 64
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
go func() {
defer wg.Done()
if err := conf.Close(); err != nil {
t.Errorf("Expected close to be idempotent but got %v", err)
}
}()
}
wg.Wait()
if err := conf.Close(); err != nil {
t.Fatalf("Expected repeated close to be idempotent but got %v", err)
}
}
func TestConfigLoadWithGoodFile(t *testing.T) {
fh := createFileForTest(t)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
// Create new config
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
// Load file source
if err := conf.Load(file.NewSource(
file.WithPath(path),
)); err != nil {
t.Fatalf("Expected no error but got %v", err)
}
}
func TestConfigLoadWithInvalidFile(t *testing.T) {
fh := createFileForTest(t)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
// Create new config
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
// Load file source
err = conf.Load(file.NewSource(
file.WithPath(path),
file.WithPath("/i/do/not/exists.json"),
))
if err == nil {
t.Fatal("Expected error but none !")
}
if !strings.Contains(fmt.Sprintf("%v", err), "/i/do/not/exists.json") {
t.Fatalf("Expected error to contain the unexisting file but got %v", err)
}
}
func TestConfigMerge(t *testing.T) {
fh := createFileForIssue18(t, `{
"amqp": {
"host": "rabbit.platform",
"port": 80
},
"handler": {
"exchange": "springCloudBus"
}
}`)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
os.Setenv("AMQP_HOST", "rabbit.testing.com")
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
if err := conf.Load(
file.NewSource(
file.WithPath(path),
),
env.NewSource(),
); err != nil {
t.Fatalf("Expected no error but got %v", err)
}
actualHost, err := conf.Get("amqp", "host")
if err != nil {
t.Fatal(err)
}
host := actualHost.String("backup")
if host != "rabbit.testing.com" {
t.Fatalf("Expected %v but got %v",
"rabbit.testing.com",
host)
}
}
func equalS(t *testing.T, actual, expect string) {
if actual != expect {
t.Errorf("Expected %s but got %s", actual, expect)
}
}
func TestConfigWatcherDirtyOverrite(t *testing.T) {
n := runtime.GOMAXPROCS(0)
defer runtime.GOMAXPROCS(n)
runtime.GOMAXPROCS(1)
l := 100
ss := make([]source.Source, l)
for i := 0; i < l; i++ {
ss[i] = memory.NewSource(memory.WithJSON([]byte(fmt.Sprintf(`{"key%d": "val%d"}`, i, i))))
}
conf, _ := NewConfig()
for _, s := range ss {
_ = conf.Load(s)
}
runtime.Gosched()
for i := range ss {
k := fmt.Sprintf("key%d", i)
v := fmt.Sprintf("val%d", i)
cc, err := conf.Get(k)
if err != nil {
t.Fatal(err)
}
equalS(t, cc.String(""), v)
}
}
+8
View File
@@ -0,0 +1,8 @@
// Package encoder handles source encoding formats
package encoder
type Encoder interface {
Encode(interface{}) ([]byte, error)
Decode([]byte, interface{}) error
String() string
}
+25
View File
@@ -0,0 +1,25 @@
package json
import (
"encoding/json"
"go-micro.dev/v6/config/encoder"
)
type jsonEncoder struct{}
func (j jsonEncoder) Encode(v interface{}) ([]byte, error) {
return json.Marshal(v)
}
func (j jsonEncoder) Decode(d []byte, v interface{}) error {
return json.Unmarshal(d, v)
}
func (j jsonEncoder) String() string {
return "json"
}
func NewEncoder() encoder.Encoder {
return jsonEncoder{}
}
+68
View File
@@ -0,0 +1,68 @@
// Package loader manages loading from multiple sources
package loader
import (
"context"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
)
// Loader manages loading sources.
type Loader interface {
// Stop the loader
Close() error
// Load the sources
Load(...source.Source) error
// A Snapshot of loaded config
Snapshot() (*Snapshot, error)
// Force sync of sources
Sync() error
// Watch for changes
Watch(...string) (Watcher, error)
// Name of loader
String() string
}
// Watcher lets you watch sources and returns a merged ChangeSet.
type Watcher interface {
// First call to next may return the current Snapshot
// If you are watching a path then only the data from
// that path is returned.
Next() (*Snapshot, error)
// Stop watching for changes
Stop() error
}
// Snapshot is a merged ChangeSet.
type Snapshot struct {
// The merged ChangeSet
ChangeSet *source.ChangeSet
// Deterministic and comparable version of the snapshot
Version string
}
// Options contains all options for a config loader.
type Options struct {
Reader reader.Reader
// for alternative data
Context context.Context
Source []source.Source
WithWatcherDisabled bool
}
// Option is a helper for a single option.
type Option func(o *Options)
// Copy snapshot.
func Copy(s *Snapshot) *Snapshot {
cs := *(s.ChangeSet)
return &Snapshot{
ChangeSet: &cs,
Version: s.Version,
}
}
+483
View File
@@ -0,0 +1,483 @@
package memory
import (
"bytes"
"container/list"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
"go-micro.dev/v6/config/loader"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/reader/json"
"go-micro.dev/v6/config/source"
)
type memory struct {
// the current values
vals reader.Values
exit chan bool
closeMu sync.Mutex
closed bool
// the current snapshot
snap *loader.Snapshot
watchers *list.List
opts loader.Options
// all the sets
sets []*source.ChangeSet
// all the sources
sources []source.Source
sync.RWMutex
}
type updateValue struct {
value reader.Value
version string
}
type watcher struct {
sync.Mutex
value reader.Value
reader reader.Reader
version atomic.Value
exit chan bool
updates chan updateValue
path []string
}
func (w *watcher) getVersion() string {
return w.version.Load().(string)
}
func (m *memory) watch(idx int, s source.Source) {
// watches a source for changes
watch := func(idx int, s source.Watcher) error {
for {
// get changeset
cs, err := s.Next()
if err != nil {
return err
}
m.Lock()
// save
m.sets[idx] = cs
// merge sets
set, err := m.opts.Reader.Merge(m.sets...)
if err != nil {
m.Unlock()
return err
}
// set values
m.vals, _ = m.opts.Reader.Values(set)
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// send watch updates
m.update()
}
}
for {
// watch the source
w, err := s.Watch()
if err != nil {
time.Sleep(time.Second)
continue
}
done := make(chan bool)
// the stop watch func
go func() {
select {
case <-done:
case <-m.exit:
}
_ = w.Stop()
}()
// block watch
if err := watch(idx, w); err != nil {
// do something better
time.Sleep(time.Second)
}
// close done chan
close(done)
// if the config is closed exit
select {
case <-m.exit:
return
default:
}
}
}
func (m *memory) loaded() bool {
var loaded bool
m.RLock()
if m.vals != nil {
loaded = true
}
m.RUnlock()
return loaded
}
// reload reads the sets and creates new values.
func (m *memory) reload() error {
m.Lock()
// merge sets
set, err := m.opts.Reader.Merge(m.sets...)
if err != nil {
m.Unlock()
return err
}
// set values
if vals, err := m.opts.Reader.Values(set); err != nil {
m.vals = vals
}
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// update watchers
m.update()
return nil
}
func (m *memory) update() {
m.RLock()
watchers := make([]*watcher, 0, m.watchers.Len())
for e := m.watchers.Front(); e != nil; e = e.Next() {
watchers = append(watchers, e.Value.(*watcher))
}
vals := m.vals
snap := m.snap
m.RUnlock()
for _, w := range watchers {
if w.getVersion() >= snap.Version {
continue
}
val, _ := vals.Get(w.path...)
m.RLock()
uv := updateValue{
version: m.snap.Version,
value: val,
}
m.RUnlock()
select {
case w.updates <- uv:
default:
}
}
}
// Snapshot returns a snapshot of the current loaded config.
func (m *memory) Snapshot() (*loader.Snapshot, error) {
if m.loaded() {
m.RLock()
snap := loader.Copy(m.snap)
m.RUnlock()
return snap, nil
}
// not loaded, sync
if err := m.Sync(); err != nil {
return nil, err
}
// make copy
m.RLock()
snap := loader.Copy(m.snap)
m.RUnlock()
return snap, nil
}
// Sync loads all the sources, calls the parser and updates the config.
func (m *memory) Sync() error {
//nolint:prealloc
var sets []*source.ChangeSet
m.Lock()
// read the source
var gerr []string
for _, source := range m.sources {
ch, err := source.Read()
if err != nil {
gerr = append(gerr, err.Error())
continue
}
sets = append(sets, ch)
}
// merge sets
set, err := m.opts.Reader.Merge(sets...)
if err != nil {
m.Unlock()
return err
}
// set values
vals, err := m.opts.Reader.Values(set)
if err != nil {
m.Unlock()
return err
}
m.vals = vals
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// update watchers
m.update()
if len(gerr) > 0 {
return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
}
return nil
}
func (m *memory) Close() error {
m.closeMu.Lock()
defer m.closeMu.Unlock()
if m.closed {
return nil
}
close(m.exit)
m.closed = true
return nil
}
func (m *memory) Get(path ...string) (reader.Value, error) {
if !m.loaded() {
if err := m.Sync(); err != nil {
return nil, err
}
}
m.Lock()
defer m.Unlock()
// did sync actually work?
if m.vals != nil {
return m.vals.Get(path...)
}
// assuming vals is nil
// create new vals
ch := m.snap.ChangeSet
// we are truly screwed, trying to load in a hacked way
v, err := m.opts.Reader.Values(ch)
if err != nil {
return nil, err
}
// lets set it just because
m.vals = v
if m.vals != nil {
return m.vals.Get(path...)
}
// ok we're going hardcore now
return nil, errors.New("no values")
}
func (m *memory) Load(sources ...source.Source) error {
var gerrors []string
for _, source := range sources {
set, err := source.Read()
if err != nil {
gerrors = append(gerrors,
fmt.Sprintf("error loading source %s: %v",
source,
err))
// continue processing
continue
}
m.Lock()
m.sources = append(m.sources, source)
m.sets = append(m.sets, set)
idx := len(m.sets) - 1
m.Unlock()
if !m.opts.WithWatcherDisabled {
go m.watch(idx, source)
}
}
if err := m.reload(); err != nil {
gerrors = append(gerrors, err.Error())
}
// Return errors
if len(gerrors) != 0 {
return errors.New(strings.Join(gerrors, "\n"))
}
return nil
}
func (m *memory) Watch(path ...string) (loader.Watcher, error) {
if m.opts.WithWatcherDisabled {
return nil, errors.New("watcher is disabled")
}
value, err := m.Get(path...)
if err != nil {
return nil, err
}
m.Lock()
w := &watcher{
exit: make(chan bool),
path: path,
value: value,
reader: m.opts.Reader,
updates: make(chan updateValue, 1),
}
w.version.Store(m.snap.Version)
e := m.watchers.PushBack(w)
m.Unlock()
go func() {
<-w.exit
m.Lock()
m.watchers.Remove(e)
m.Unlock()
}()
return w, nil
}
func (m *memory) String() string {
return "memory"
}
func (w *watcher) Next() (*loader.Snapshot, error) {
update := func(v reader.Value) *loader.Snapshot {
w.value = v
cs := &source.ChangeSet{
Data: v.Bytes(),
Format: w.reader.String(),
Source: "memory",
Timestamp: time.Now(),
}
cs.Checksum = cs.Sum()
return &loader.Snapshot{
ChangeSet: cs,
Version: w.getVersion(),
}
}
for {
select {
case <-w.exit:
return nil, errors.New("watcher stopped")
case uv := <-w.updates:
if uv.version <= w.getVersion() {
continue
}
v := uv.value
w.version.Store(uv.version)
if bytes.Equal(w.value.Bytes(), v.Bytes()) {
continue
}
return update(v), nil
}
}
}
func (w *watcher) Stop() error {
w.Lock()
defer w.Unlock()
select {
case <-w.exit:
default:
close(w.exit)
close(w.updates)
}
return nil
}
func genVer() string {
return fmt.Sprintf("%d", time.Now().UnixNano())
}
func NewLoader(opts ...loader.Option) loader.Loader {
options := loader.Options{
Reader: json.NewReader(),
}
for _, o := range opts {
o(&options)
}
m := &memory{
exit: make(chan bool),
opts: options,
watchers: list.New(),
sources: options.Source,
}
m.sets = make([]*source.ChangeSet, len(options.Source))
for i, s := range options.Source {
m.sets[i] = &source.ChangeSet{Source: s.String()}
if !options.WithWatcherDisabled {
go m.watch(i, s)
}
}
return m
}
+27
View File
@@ -0,0 +1,27 @@
package memory
import (
"go-micro.dev/v6/config/loader"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
)
// WithSource appends a source to list of sources.
func WithSource(s source.Source) loader.Option {
return func(o *loader.Options) {
o.Source = append(o.Source, s)
}
}
// WithReader sets the config reader.
func WithReader(r reader.Reader) loader.Option {
return func(o *loader.Options) {
o.Reader = r
}
}
func WithWatcherDisabled() loader.Option {
return func(o *loader.Options) {
o.WithWatcherDisabled = true
}
}
+34
View File
@@ -0,0 +1,34 @@
package config
import (
"go-micro.dev/v6/config/loader"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
)
// WithLoader sets the loader for manager config.
func WithLoader(l loader.Loader) Option {
return func(o *Options) {
o.Loader = l
}
}
// WithSource appends a source to list of sources.
func WithSource(s source.Source) Option {
return func(o *Options) {
o.Source = append(o.Source, s)
}
}
// WithReader sets the config reader.
func WithReader(r reader.Reader) Option {
return func(o *Options) {
o.Reader = r
}
}
func WithWatcherDisabled() Option {
return func(o *Options) {
o.WithWatcherDisabled = true
}
}
+83
View File
@@ -0,0 +1,83 @@
package json
import (
"errors"
"time"
"dario.cat/mergo"
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/encoder/json"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
)
type jsonReader struct {
opts reader.Options
json encoder.Encoder
}
func (j *jsonReader) Merge(changes ...*source.ChangeSet) (*source.ChangeSet, error) {
var merged map[string]interface{}
for _, m := range changes {
if m == nil {
continue
}
if len(m.Data) == 0 {
continue
}
codec, ok := j.opts.Encoding[m.Format]
if !ok {
// fallback
codec = j.json
}
var data map[string]interface{}
if err := codec.Decode(m.Data, &data); err != nil {
return nil, err
}
if err := mergo.Map(&merged, data, mergo.WithOverride); err != nil {
return nil, err
}
}
b, err := j.json.Encode(merged)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Timestamp: time.Now(),
Data: b,
Source: "json",
Format: j.json.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (j *jsonReader) Values(ch *source.ChangeSet) (reader.Values, error) {
if ch == nil {
return nil, errors.New("changeset is nil")
}
if ch.Format != "json" {
return nil, errors.New("unsupported format")
}
return newValues(ch)
}
func (j *jsonReader) String() string {
return "json"
}
// NewReader creates a json reader.
func NewReader(opts ...reader.Option) reader.Reader {
options := reader.NewOptions(opts...)
return &jsonReader{
json: json.NewEncoder(),
opts: options,
}
}
+45
View File
@@ -0,0 +1,45 @@
package json
import (
"testing"
"go-micro.dev/v6/config/source"
)
func TestReader(t *testing.T) {
data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
testData := []struct {
path []string
value string
}{
{
[]string{"foo"},
"bar",
},
{
[]string{"baz", "bar"},
"cat",
},
}
r := NewReader()
c, err := r.Merge(&source.ChangeSet{Data: data}, &source.ChangeSet{})
if err != nil {
t.Fatal(err)
}
values, err := r.Values(c)
if err != nil {
t.Fatal(err)
}
for _, test := range testData {
if v, err := values.Get(test.path...); err != nil {
t.Fatal(err)
} else if v.String("") != test.value {
t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
}
}
}
+209
View File
@@ -0,0 +1,209 @@
package json
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
simple "github.com/bitly/go-simplejson"
"go-micro.dev/v6/config/reader"
"go-micro.dev/v6/config/source"
)
type jsonValues struct {
ch *source.ChangeSet
sj *simple.Json
}
type jsonValue struct {
*simple.Json
}
func NewValues(val []byte) (reader.Values, error) {
sj := simple.New()
data, _ := reader.ReplaceEnvVars(val)
if err := sj.UnmarshalJSON(data); err != nil {
sj.SetPath(nil, string(data))
}
return &jsonValues{sj: sj}, nil
}
func newValues(ch *source.ChangeSet) (reader.Values, error) {
sj := simple.New()
data, _ := reader.ReplaceEnvVars(ch.Data)
if err := sj.UnmarshalJSON(data); err != nil {
sj.SetPath(nil, string(ch.Data))
}
return &jsonValues{ch, sj}, nil
}
func (j *jsonValues) Get(path ...string) (reader.Value, error) {
return &jsonValue{j.sj.GetPath(path...)}, nil
}
func (j *jsonValues) Del(path ...string) {
// delete the tree?
if len(path) == 0 {
j.sj = simple.New()
return
}
if len(path) == 1 {
j.sj.Del(path[0])
return
}
vals := j.sj.GetPath(path[:len(path)-1]...)
vals.Del(path[len(path)-1])
j.sj.SetPath(path[:len(path)-1], vals.Interface())
}
func (j *jsonValues) Set(val interface{}, path ...string) {
j.sj.SetPath(path, val)
}
func (j *jsonValues) Bytes() []byte {
b, _ := j.sj.MarshalJSON()
return b
}
func (j *jsonValues) Map() map[string]interface{} {
m, _ := j.sj.Map()
return m
}
func (j *jsonValues) Scan(v interface{}) error {
b, err := j.sj.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (j *jsonValues) String() string {
return "json"
}
func (j *jsonValue) Bool(def bool) bool {
b, err := j.Json.Bool()
if err == nil {
return b
}
str, ok := j.Interface().(string)
if !ok {
return def
}
b, err = strconv.ParseBool(str)
if err != nil {
return def
}
return b
}
func (j *jsonValue) Int(def int) int {
i, err := j.Json.Int()
if err == nil {
return i
}
str, ok := j.Interface().(string)
if !ok {
return def
}
i, err = strconv.Atoi(str)
if err != nil {
return def
}
return i
}
func (j *jsonValue) String(def string) string {
return j.MustString(def)
}
func (j *jsonValue) Float64(def float64) float64 {
f, err := j.Json.Float64()
if err == nil {
return f
}
str, ok := j.Interface().(string)
if !ok {
return def
}
f, err = strconv.ParseFloat(str, 64)
if err != nil {
return def
}
return f
}
func (j *jsonValue) Duration(def time.Duration) time.Duration {
v, err := j.Json.String()
if err != nil {
return def
}
value, err := time.ParseDuration(v)
if err != nil {
return def
}
return value
}
func (j *jsonValue) StringSlice(def []string) []string {
v, err := j.Json.String()
if err == nil {
sl := strings.Split(v, ",")
if len(sl) > 0 {
return sl
}
}
return j.MustStringArray(def)
}
func (j *jsonValue) StringMap(def map[string]string) map[string]string {
m, err := j.Map()
if err != nil {
return def
}
res := map[string]string{}
for k, v := range m {
res[k] = fmt.Sprintf("%v", v)
}
return res
}
func (j *jsonValue) Scan(v interface{}) error {
b, err := j.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (j *jsonValue) Bytes() []byte {
b, err := j.Json.Bytes()
if err != nil {
// try return marshaled
b, err = j.MarshalJSON()
if err != nil {
return []byte{}
}
return b
}
return b
}
+93
View File
@@ -0,0 +1,93 @@
package json
import (
"reflect"
"testing"
"go-micro.dev/v6/config/source"
)
func TestValues(t *testing.T) {
emptyStr := ""
testData := []struct {
csdata []byte
path []string
accepter interface{}
value interface{}
}{
{
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
[]string{"foo"},
emptyStr,
"bar",
},
{
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
[]string{"baz", "bar"},
emptyStr,
"cat",
},
}
for idx, test := range testData {
values, err := newValues(&source.ChangeSet{
Data: test.csdata,
})
if err != nil {
t.Fatal(err)
}
v, err := values.Get(test.path...)
if err != nil {
t.Fatal(err)
}
err = v.Scan(&test.accepter)
if err != nil {
t.Fatal(err)
}
if test.accepter != test.value {
t.Fatalf("No.%d Expected %v got %v for path %v", idx, test.value, test.accepter, test.path)
}
}
}
func TestStructArray(t *testing.T) {
type T struct {
Foo string
}
emptyTSlice := []T{}
testData := []struct {
csdata []byte
accepter []T
value []T
}{
{
[]byte(`[{"foo": "bar"}]`),
emptyTSlice,
[]T{{Foo: "bar"}},
},
}
for idx, test := range testData {
values, err := newValues(&source.ChangeSet{
Data: test.csdata,
})
if err != nil {
t.Fatal(err)
}
v, err := values.Get()
if err != nil {
t.Fatal(err)
}
err = v.Scan(&test.accepter)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(test.accepter, test.value) {
t.Fatalf("No.%d Expected %v got %v", idx, test.value, test.accepter)
}
}
}
+33
View File
@@ -0,0 +1,33 @@
package reader
import (
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/encoder/json"
)
type Options struct {
Encoding map[string]encoder.Encoder
}
type Option func(o *Options)
func NewOptions(opts ...Option) Options {
options := Options{
Encoding: map[string]encoder.Encoder{
"json": json.NewEncoder(),
},
}
for _, o := range opts {
o(&options)
}
return options
}
func WithEncoder(e encoder.Encoder) Option {
return func(o *Options) {
if o.Encoding == nil {
o.Encoding = make(map[string]encoder.Encoder)
}
o.Encoding[e.String()] = e
}
}
+23
View File
@@ -0,0 +1,23 @@
package reader
import (
"os"
"regexp"
)
func ReplaceEnvVars(raw []byte) ([]byte, error) {
re := regexp.MustCompile(`\$\{([A-Za-z0-9_]+)\}`)
if re.Match(raw) {
dataS := string(raw)
res := re.ReplaceAllStringFunc(dataS, replaceEnvVars)
return []byte(res), nil
} else {
return raw, nil
}
}
func replaceEnvVars(element string) string {
v := element[2 : len(element)-1]
el := os.Getenv(v)
return el
}
+73
View File
@@ -0,0 +1,73 @@
package reader
import (
"os"
"strings"
"testing"
)
func TestReplaceEnvVars(t *testing.T) {
os.Setenv("myBar", "cat")
os.Setenv("MYBAR", "cat")
os.Setenv("my_Bar", "cat")
os.Setenv("myBar_", "cat")
testData := []struct {
expected string
data []byte
}{
// Right use cases
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${MYBAR}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${my_Bar}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar_}"}}`),
},
// Wrong use cases
{
`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "${}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "$sss}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "$sss}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "${sss"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${sss"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "{something}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "{something}"}}`),
},
// Use cases without replace env vars
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
},
}
for _, test := range testData {
res, err := ReplaceEnvVars(test.data)
if err != nil {
t.Fatal(err)
}
if strings.Compare(test.expected, string(res)) != 0 {
t.Fatalf("Expected %s got %s", test.expected, res)
}
}
}
+38
View File
@@ -0,0 +1,38 @@
// Package reader parses change sets and provides config values
package reader
import (
"time"
"go-micro.dev/v6/config/source"
)
// Reader is an interface for merging changesets.
type Reader interface {
Merge(...*source.ChangeSet) (*source.ChangeSet, error)
Values(*source.ChangeSet) (Values, error)
String() string
}
// Values is returned by the reader.
type Values interface {
Bytes() []byte
Get(path ...string) (Value, error)
Set(val interface{}, path ...string)
Del(path ...string)
Map() map[string]interface{}
Scan(v interface{}) error
}
// Value represents a value of any type.
type Value interface {
Bool(def bool) bool
Int(def int) int
String(def string) string
Float64(def float64) float64
Duration(def time.Duration) time.Duration
StringSlice(def []string) []string
StringMap(def map[string]string) map[string]string
Scan(val interface{}) error
Bytes() []byte
}
+88
View File
@@ -0,0 +1,88 @@
// Package box is an asymmetric implementation of config/secrets using nacl/box
package box
import (
"crypto/rand"
"github.com/pkg/errors"
"go-micro.dev/v6/config/secrets"
naclbox "golang.org/x/crypto/nacl/box"
)
const keyLength = 32
type box struct {
options secrets.Options
publicKey [keyLength]byte
privateKey [keyLength]byte
}
// NewSecrets returns a nacl-box codec.
func NewSecrets(opts ...secrets.Option) secrets.Secrets {
b := &box{}
for _, o := range opts {
o(&b.options)
}
return b
}
func (b *box) Init(opts ...secrets.Option) error {
for _, o := range opts {
o(&b.options)
}
if len(b.options.PrivateKey) != keyLength || len(b.options.PublicKey) != keyLength {
return errors.Errorf("a public key and a private key of length %d must both be provided", keyLength)
}
copy(b.privateKey[:], b.options.PrivateKey)
copy(b.publicKey[:], b.options.PublicKey)
return nil
}
// Options returns options.
func (b *box) Options() secrets.Options {
return b.options
}
// String returns nacl-box.
func (*box) String() string {
return "nacl-box"
}
// Encrypt encrypts a message with the sender's private key and the receipient's public key.
func (b *box) Encrypt(in []byte, opts ...secrets.EncryptOption) ([]byte, error) {
var options secrets.EncryptOptions
for _, o := range opts {
o(&options)
}
if len(options.RecipientPublicKey) != keyLength {
return []byte{}, errors.New("recepient's public key must be provided")
}
var recipientPublicKey [keyLength]byte
copy(recipientPublicKey[:], options.RecipientPublicKey)
var nonce [24]byte
if _, err := rand.Reader.Read(nonce[:]); err != nil {
return []byte{}, errors.Wrap(err, "couldn't obtain a random nonce from crypto/rand")
}
return naclbox.Seal(nonce[:], in, &nonce, &recipientPublicKey, &b.privateKey), nil
}
// Decrypt Decrypts a message with the receiver's private key and the sender's public key.
func (b *box) Decrypt(in []byte, opts ...secrets.DecryptOption) ([]byte, error) {
var options secrets.DecryptOptions
for _, o := range opts {
o(&options)
}
if len(options.SenderPublicKey) != keyLength {
return []byte{}, errors.New("sender's public key bust be provided")
}
var nonce [24]byte
var senderPublicKey [32]byte
copy(nonce[:], in[:24])
copy(senderPublicKey[:], options.SenderPublicKey)
decrypted, ok := naclbox.Open(nil, in[24:], &nonce, &senderPublicKey, &b.privateKey)
if !ok {
return []byte{}, errors.New("incoming message couldn't be verified / decrypted")
}
return decrypted, nil
}
+66
View File
@@ -0,0 +1,66 @@
package box
import (
"crypto/rand"
"reflect"
"testing"
"go-micro.dev/v6/config/secrets"
naclbox "golang.org/x/crypto/nacl/box"
)
func TestBox(t *testing.T) {
alicePublicKey, alicePrivateKey, err := naclbox.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
bobPublicKey, bobPrivateKey, err := naclbox.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
alice, bob := NewSecrets(secrets.PublicKey(alicePublicKey[:]), secrets.PrivateKey(alicePrivateKey[:])), NewSecrets()
if err := alice.Init(); err != nil {
t.Error(err)
}
if err := bob.Init(secrets.PublicKey(bobPublicKey[:]), secrets.PrivateKey(bobPrivateKey[:])); err != nil {
t.Error(err)
}
if alice.String() != "nacl-box" {
t.Error("String() doesn't return nacl-box")
}
aliceSecret := []byte("Why is a raven like a writing-desk?")
if _, err := alice.Encrypt(aliceSecret); err == nil {
t.Error("alice.Encrypt succeeded without a public key")
}
enc, err := alice.Encrypt(aliceSecret, secrets.RecipientPublicKey(bob.Options().PublicKey))
if err != nil {
t.Error("alice.Encrypt failed")
}
if _, err := bob.Decrypt(enc); err == nil {
t.Error("bob.Decrypt succeeded without a public key")
}
if dec, err := bob.Decrypt(enc, secrets.SenderPublicKey(alice.Options().PublicKey)); err == nil {
if !reflect.DeepEqual(dec, aliceSecret) {
t.Errorf("Bob's decrypted message didn't match Alice's encrypted message: %v != %v", aliceSecret, dec)
}
} else {
t.Errorf("bob.Decrypt failed (%s)", err)
}
bobSecret := []byte("I haven't the slightest idea")
enc, err = bob.Encrypt(bobSecret, secrets.RecipientPublicKey(alice.Options().PublicKey))
if err != nil {
t.Error(err)
}
_, err = alice.Decrypt(enc, secrets.SenderPublicKey(bob.Options().PrivateKey))
if err == nil {
t.Error(err)
}
dec, err := alice.Decrypt(enc, secrets.SenderPublicKey(bob.Options().PublicKey))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(dec, bobSecret) {
t.Errorf("Alice's decrypted message didn't match Bob's encrypted message %v != %v", bobSecret, dec)
}
}
+73
View File
@@ -0,0 +1,73 @@
// Package secretbox is a config/secrets implementation that uses nacl/secretbox
// to do symmetric encryption / verification
package secretbox
import (
"crypto/rand"
"github.com/pkg/errors"
"go-micro.dev/v6/config/secrets"
"golang.org/x/crypto/nacl/secretbox"
)
const keyLength = 32
type secretBox struct {
options secrets.Options
secretKey [keyLength]byte
}
// NewSecrets returns a secretbox codec.
func NewSecrets(opts ...secrets.Option) secrets.Secrets {
sb := &secretBox{}
for _, o := range opts {
o(&sb.options)
}
return sb
}
func (s *secretBox) Init(opts ...secrets.Option) error {
for _, o := range opts {
o(&s.options)
}
if len(s.options.Key) == 0 {
return errors.New("no secret key is defined")
}
if len(s.options.Key) != keyLength {
return errors.Errorf("secret key must be %d bytes long", keyLength)
}
copy(s.secretKey[:], s.options.Key)
return nil
}
func (s *secretBox) Options() secrets.Options {
return s.options
}
func (s *secretBox) String() string {
return "nacl-secretbox"
}
func (s *secretBox) Encrypt(in []byte, opts ...secrets.EncryptOption) ([]byte, error) {
// no opts are expected, so they are ignored
// there must be a unique nonce for each message
var nonce [24]byte
if _, err := rand.Reader.Read(nonce[:]); err != nil {
return []byte{}, errors.Wrap(err, "couldn't obtain a random nonce from crypto/rand")
}
return secretbox.Seal(nonce[:], in, &nonce, &s.secretKey), nil
}
func (s *secretBox) Decrypt(in []byte, opts ...secrets.DecryptOption) ([]byte, error) {
// no options are expected, so they are ignored
var decryptNonce [24]byte
copy(decryptNonce[:], in[:24])
decrypted, ok := secretbox.Open(nil, in[24:], &decryptNonce, &s.secretKey)
if !ok {
return []byte{}, errors.New("decryption failed (is the key set correctly?)")
}
return decrypted, nil
}
@@ -0,0 +1,56 @@
package secretbox
import (
"encoding/base64"
"reflect"
"testing"
"go-micro.dev/v6/config/secrets"
)
func TestSecretBox(t *testing.T) {
secretKey, err := base64.StdEncoding.DecodeString("4jbVgq8FsAV7vy+n8WqEZrl7BUtNqh3fYT5RXzXOPFY=")
if err != nil {
t.Fatal(err)
}
s := NewSecrets()
if err := s.Init(); err == nil {
t.Error("Secretbox accepted an empty secret key")
}
if err := s.Init(secrets.Key([]byte("invalid"))); err == nil {
t.Error("Secretbox accepted a secret key that is invalid")
}
if err := s.Init(secrets.Key(secretKey)); err != nil {
t.Fatal(err)
}
o := s.Options()
if !reflect.DeepEqual(o.Key, secretKey) {
t.Error("Init() didn't set secret key correctly")
}
if s.String() != "nacl-secretbox" {
t.Error(s.String() + " should be nacl-secretbox")
}
// Try 10 times to get different nonces
for i := 0; i < 10; i++ {
message := []byte(`Can you hear me, Major Tom?`)
encrypted, err := s.Encrypt(message)
if err != nil {
t.Errorf("Failed to encrypt message (%s)", err)
}
decrypted, err := s.Decrypt(encrypted)
if err != nil {
t.Errorf("Failed to decrypt encrypted message (%s)", err)
}
if !reflect.DeepEqual(message, decrypted) {
t.Errorf("Decrypted Message dod not match encrypted message")
}
}
}
+88
View File
@@ -0,0 +1,88 @@
// Package secrets is an interface for encrypting and decrypting secrets
package secrets
import "context"
// Secrets encrypts or decrypts arbitrary data. The data should be as small as possible.
type Secrets interface {
// Initialize options
Init(...Option) error
// Return the options
Options() Options
// Decrypt a value
Decrypt([]byte, ...DecryptOption) ([]byte, error)
// Encrypt a value
Encrypt([]byte, ...EncryptOption) ([]byte, error)
// Secrets implementation
String() string
}
type Options struct {
// Context for other opts
Context context.Context
// Key is a symmetric key for encoding
Key []byte
// Private key for decoding
PrivateKey []byte
// Public key for encoding
PublicKey []byte
}
// Option sets options.
type Option func(*Options)
// Key sets the symmetric secret key.
func Key(k []byte) Option {
return func(o *Options) {
o.Key = make([]byte, len(k))
copy(o.Key, k)
}
}
// PublicKey sets the asymmetric Public Key of this codec.
func PublicKey(key []byte) Option {
return func(o *Options) {
o.PublicKey = make([]byte, len(key))
copy(o.PublicKey, key)
}
}
// PrivateKey sets the asymmetric Private Key of this codec.
func PrivateKey(key []byte) Option {
return func(o *Options) {
o.PrivateKey = make([]byte, len(key))
copy(o.PrivateKey, key)
}
}
// DecryptOptions can be passed to Secrets.Decrypt.
type DecryptOptions struct {
SenderPublicKey []byte
}
// DecryptOption sets DecryptOptions.
type DecryptOption func(*DecryptOptions)
// SenderPublicKey is the Public Key of the Secrets that encrypted this message.
func SenderPublicKey(key []byte) DecryptOption {
return func(d *DecryptOptions) {
d.SenderPublicKey = make([]byte, len(key))
copy(d.SenderPublicKey, key)
}
}
// EncryptOptions can be passed to Secrets.Encrypt.
type EncryptOptions struct {
RecipientPublicKey []byte
}
// EncryptOption Sets EncryptOptions.
type EncryptOption func(*EncryptOptions)
// RecipientPublicKey is the Public Key of the Secrets that will decrypt this message.
func RecipientPublicKey(key []byte) EncryptOption {
return func(e *EncryptOptions) {
e.RecipientPublicKey = make([]byte, len(key))
copy(e.RecipientPublicKey, key)
}
}
+13
View File
@@ -0,0 +1,13 @@
package source
import (
"crypto/md5"
"fmt"
)
// Sum returns the md5 checksum of the ChangeSet data.
func (c *ChangeSet) Sum() string {
h := md5.New()
h.Write(c.Data)
return fmt.Sprintf("%x", h.Sum(nil))
}
+71
View File
@@ -0,0 +1,71 @@
# cli Source
The cli source reads config from parsed flags via a cli.Context.
## Format
We expect the use of the `urfave/cli` package. Upper case flags will be lower cased. Dashes will be used as delimiters for nesting.
### Example
```go
micro.Flags(
cli.StringFlag{
Name: "database-address",
Value: "127.0.0.1",
Usage: "the db address",
},
cli.IntFlag{
Name: "database-port",
Value: 3306,
Usage: "the db port",
},
)
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## New and Load Source
Because a cli.Context is needed to retrieve the flags and their values, it is recommended to build your source from within a cli.Action.
```go
func main() {
// New Service
service := micro.NewService(
micro.Name("example"),
micro.Flags(
cli.StringFlag{
Name: "database-address",
Value: "127.0.0.1",
Usage: "the db address",
},
),
)
var clisrc source.Source
service.Init(
micro.Action(func(c *cli.Context) {
clisrc = cli.NewSource(
cli.Context(c),
)
// Alternatively, just setup your config right here
}),
)
// ... Load and use that source ...
conf := config.NewConfig()
conf.Load(clisrc)
}
```
+148
View File
@@ -0,0 +1,148 @@
package cli
import (
"flag"
"io"
"os"
"strings"
"time"
"dario.cat/mergo"
"github.com/urfave/cli/v2"
"go-micro.dev/v6/cmd"
"go-micro.dev/v6/config/source"
)
type cliSource struct {
opts source.Options
ctx *cli.Context
}
func (c *cliSource) Read() (*source.ChangeSet, error) {
var changes map[string]interface{}
// directly using app cli flags, to access default values of not specified options
for _, f := range c.ctx.App.Flags {
name := f.Names()[0]
tmp := toEntry(name, c.ctx.Generic(name))
if err := mergo.Map(&changes, tmp, mergo.WithOverride); err != nil {
return nil, err
}
}
b, err := c.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: c.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: c.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func toEntry(name string, v interface{}) map[string]interface{} {
n := strings.ToLower(name)
keys := strings.FieldsFunc(n, split)
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
tmp[k] = v
continue
}
tmp = map[string]interface{}{k: tmp}
}
return tmp
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func split(r rune) bool {
return r == '-' || r == '_'
}
func (c *cliSource) Watch() (source.Watcher, error) {
return source.NewNoopWatcher()
}
// Write is unsupported.
func (c *cliSource) Write(cs *source.ChangeSet) error {
return nil
}
func (c *cliSource) String() string {
return "cli"
}
// NewSource returns a config source for integrating parsed flags from a urfave/cli.Context.
// Hyphens are delimiters for nesting, and all keys are lowercased. The assumption is that
// command line flags have already been parsed.
//
// Example:
//
// cli.StringFlag{Name: "db-host"},
//
//
// {
// "database": {
// "host": "localhost"
// }
// }
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
var ctx *cli.Context
if c, ok := options.Context.Value(contextKey{}).(*cli.Context); ok {
ctx = c
} else {
// no context
// get the default app/flags
app := cmd.App()
flags := app.Flags
// create flagset
set := flag.NewFlagSet(app.Name, flag.ContinueOnError)
// apply flags to set
for _, f := range flags {
_ = f.Apply(set)
}
// parse flags
set.SetOutput(io.Discard)
_ = set.Parse(os.Args[1:])
// normalise flags
_ = normalizeFlags(app.Flags, set)
// create context
ctx = cli.NewContext(app, set, nil)
}
return &cliSource{
ctx: ctx,
opts: options,
}
}
// WithContext returns a new source with the context specified.
// The assumption is that Context is retrieved within an app.Action function.
func WithContext(ctx *cli.Context, opts ...source.Option) source.Source {
return &cliSource{
ctx: ctx,
opts: source.NewOptions(opts...),
}
}
+121
View File
@@ -0,0 +1,121 @@
package cli
import (
"encoding/json"
"os"
"testing"
"github.com/urfave/cli/v2"
"go-micro.dev/v6"
"go-micro.dev/v6/cmd"
"go-micro.dev/v6/config"
"go-micro.dev/v6/config/source"
)
func TestCliSourceDefault(t *testing.T) {
const expVal string = "flagvalue"
service := micro.NewService("test", micro.Flags(
// to be able to run inside go test
&cli.StringFlag{
Name: "test.timeout",
},
&cli.StringFlag{
Name: "test.bench",
},
&cli.BoolFlag{
Name: "test.v",
},
&cli.StringFlag{
Name: "test.run",
},
&cli.StringFlag{
Name: "test.testlogfile",
},
&cli.StringFlag{
Name: "test.paniconexit0",
},
&cli.StringFlag{
Name: "flag",
Usage: "It changes something",
EnvVars: []string{"flag"},
Value: expVal,
},
),
)
var cliSrc source.Source
service.Init(
// Loads CLI configuration
micro.Action(func(c *cli.Context) error {
cliSrc = NewSource(
Context(c),
)
return nil
}),
)
config.Load(cliSrc)
if val, err := config.Get("flag"); err != nil {
t.Fatal(err)
} else if fval := val.String("default"); fval != expVal {
t.Fatalf("default flag value not loaded %v != %v", fval, expVal)
}
}
func test(t *testing.T, withContext bool) {
var src source.Source
// setup app
app := cmd.App()
app.Name = "testapp"
app.Flags = []cli.Flag{
&cli.StringFlag{
Name: "db-host",
EnvVars: []string{"db-host"},
Value: "myval",
},
}
// with context
if withContext {
// set action
app.Action = func(c *cli.Context) error {
src = WithContext(c)
return nil
}
// run app
app.Run([]string{"run", "-db-host", "localhost"})
// no context
} else {
// set args
os.Args = []string{"run", "-db-host", "localhost"}
src = NewSource()
}
// test config
c, err := src.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["db"].(map[string]interface{})
if actualDB["host"] != "localhost" {
t.Errorf("expected localhost, got %v", actualDB["name"])
}
}
func TestCliSource(t *testing.T) {
// without context
test(t, false)
}
func TestCliSourceWithContext(t *testing.T) {
// with context
test(t, true)
}
+20
View File
@@ -0,0 +1,20 @@
package cli
import (
"context"
"github.com/urfave/cli/v2"
"go-micro.dev/v6/config/source"
)
type contextKey struct{}
// Context sets the cli context.
func Context(c *cli.Context) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, contextKey{}, c)
}
}
+50
View File
@@ -0,0 +1,50 @@
package cli
import (
"errors"
"flag"
"strings"
"github.com/urfave/cli/v2"
)
func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
switch ff.Value.(type) {
case *cli.StringSlice:
default:
_ = set.Set(name, ff.Value.String())
}
}
func normalizeFlags(flags []cli.Flag, set *flag.FlagSet) error {
visited := make(map[string]bool)
set.Visit(func(f *flag.Flag) {
visited[f.Name] = true
})
for _, f := range flags {
parts := f.Names()
if len(parts) == 1 {
continue
}
var ff *flag.Flag
for _, name := range parts {
name = strings.Trim(name, " ")
if visited[name] {
if ff != nil {
return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
}
ff = set.Lookup(name)
}
}
if ff == nil {
continue
}
for _, name := range parts {
name = strings.Trim(name, " ")
if !visited[name] {
copyFlag(name, ff, set)
}
}
}
return nil
}
+94
View File
@@ -0,0 +1,94 @@
# Env Source
The env source reads config from environment variables
## Format
We expect environment variables to be in the standard format of FOO=bar
Keys are converted to lowercase and split on underscore.
### Format example
```bash
DATABASE_ADDRESS=127.0.0.1
DATABASE_PORT=3306
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## Prefixes
Environment variables can be namespaced so we only have access to a subset. Two options are available:
```go
WithPrefix(p ...string)
WithStrippedPrefix(p ...string)
```
The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one.
### Prefixes example
Given ENVs of:
```bash
APP_DATABASE_ADDRESS=127.0.0.1
APP_DATABASE_PORT=3306
VAULT_ADDR=vault:1337
```
and a source initialized as follows:
```go
src := env.NewSource(
env.WithPrefix("VAULT"),
env.WithStrippedPrefix("APP"),
)
```
The resulting config will be:
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
},
"vault": {
"addr": "vault:1337"
}
}
```
## New Source
Specify source with data
```go
src := env.NewSource(
// optionally specify prefix
env.WithPrefix("MICRO"),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load env source
conf.Load(src)
```
+146
View File
@@ -0,0 +1,146 @@
package env
import (
"os"
"strconv"
"strings"
"time"
"dario.cat/mergo"
"go-micro.dev/v6/config/source"
)
var (
DefaultPrefixes = []string{}
)
type env struct {
opts source.Options
prefixes []string
strippedPrefixes []string
}
func (e *env) Read() (*source.ChangeSet, error) {
var changes map[string]interface{}
for _, env := range os.Environ() {
if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
notFound := true
if _, ok := matchPrefix(e.prefixes, env); ok {
notFound = false
}
if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
env = strings.TrimPrefix(env, match)
notFound = false
}
if notFound {
continue
}
}
pair := strings.SplitN(env, "=", 2)
value := pair[1]
keys := strings.Split(strings.ToLower(pair[0]), "_")
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
if intValue, err := strconv.Atoi(value); err == nil {
tmp[k] = intValue
} else if boolValue, err := strconv.ParseBool(value); err == nil {
tmp[k] = boolValue
} else {
tmp[k] = value
}
continue
}
tmp = map[string]interface{}{k: tmp}
}
if err := mergo.Map(&changes, tmp); err != nil {
return nil, err
}
}
b, err := e.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: e.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: e.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func matchPrefix(pre []string, s string) (string, bool) {
for _, p := range pre {
if strings.HasPrefix(s, p) {
return p, true
}
}
return "", false
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (e *env) Watch() (source.Watcher, error) {
return newWatcher()
}
func (e *env) Write(cs *source.ChangeSet) error {
return nil
}
func (e *env) String() string {
return "env"
}
// NewSource returns a config source for parsing ENV variables.
// Underscores are delimiters for nesting, and all keys are lowercased.
//
// Example:
//
// "DATABASE_SERVER_HOST=localhost" will convert to
//
// {
// "database": {
// "server": {
// "host": "localhost"
// }
// }
// }
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
var sp []string
var pre []string
if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
sp = p
}
if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
pre = p
}
if len(sp) > 0 || len(pre) > 0 {
pre = append(pre, DefaultPrefixes...)
}
return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
}
+112
View File
@@ -0,0 +1,112 @@
package env
import (
"encoding/json"
"os"
"testing"
"time"
"go-micro.dev/v6/config/source"
)
func TestEnv_Read(t *testing.T) {
expected := map[string]map[string]string{
"database": {
"host": "localhost",
"password": "password",
"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
},
}
os.Setenv("DATABASE_HOST", "localhost")
os.Setenv("DATABASE_PASSWORD", "password")
os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
for k, v := range expected["database"] {
a := actualDB[k]
if a != v {
t.Errorf("expected %v got %v", v, a)
}
}
}
func TestEnvvar_Prefixes(t *testing.T) {
os.Setenv("APP_DATABASE_HOST", "localhost")
os.Setenv("APP_DATABASE_PASSWORD", "password")
os.Setenv("VAULT_ADDR", "vault:1337")
os.Setenv("MICRO_REGISTRY", "mdns")
var prefixtests = []struct {
prefixOpts []source.Option
expectedKeys []string
}{
{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
}
for _, pt := range prefixtests {
source := NewSource(pt.prefixOpts...)
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
// assert other prefixes ignored
if l := len(actual); l != len(pt.expectedKeys) {
t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
}
for _, k := range pt.expectedKeys {
if !containsKey(actual, k) {
t.Errorf("expected key %v, not found", k)
}
}
}
}
func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
src := NewSource(WithStrippedPrefix("GOMICRO_"))
w, err := src.Watch()
if err != nil {
t.Error(err)
}
go func() {
time.Sleep(50 * time.Millisecond)
w.Stop()
}()
if _, err := w.Next(); err != source.ErrWatcherStopped {
t.Errorf("expected watcher stopped error, got %v", err)
}
}
func containsKey(m map[string]interface{}, s string) bool {
for k := range m {
if k == s {
return true
}
}
return false
}
+49
View File
@@ -0,0 +1,49 @@
package env
import (
"context"
"strings"
"go-micro.dev/v6/config/source"
)
type strippedPrefixKey struct{}
type prefixKey struct{}
// WithStrippedPrefix sets the environment variable prefixes to scope to.
// These prefixes will be removed from the actual config entries.
func WithStrippedPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
}
}
// WithPrefix sets the environment variable prefixes to scope to.
// These prefixes will not be removed. Each prefix will be considered a top level config entry.
func WithPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
}
}
func appendUnderscore(prefixes []string) []string {
//nolint:prealloc
var result []string
for _, p := range prefixes {
if !strings.HasSuffix(p, "_") {
result = append(result, p+"_")
continue
}
result = append(result, p)
}
return result
}
+24
View File
@@ -0,0 +1,24 @@
package env
import (
"go-micro.dev/v6/config/source"
)
type watcher struct {
exit chan struct{}
}
func (w *watcher) Next() (*source.ChangeSet, error) {
<-w.exit
return nil, source.ErrWatcherStopped
}
func (w *watcher) Stop() error {
close(w.exit)
return nil
}
func newWatcher() (source.Watcher, error) {
return &watcher{exit: make(chan struct{})}, nil
}
+69
View File
@@ -0,0 +1,69 @@
# File Source
The file source reads config from a file.
It uses the File extension to determine the Format e.g `config.yaml` has the yaml format.
It does not make use of encoders or interpet the file data. If a file extension is not present
the source Format will default to the Encoder in options.
## Example
A config file format in json
```json
{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}
```
## New Source
Specify file source with path to file. Path is optional and will default to `config.json`
```go
fileSource := file.NewSource(
file.WithPath("/tmp/config.json"),
)
```
## File Format
To load different file formats e.g yaml, toml, xml simply specify them with their extension
```go
fileSource := file.NewSource(
file.WithPath("/tmp/config.yaml"),
)
```
If you want to specify a file without extension, ensure you set the encoder to the same format
```go
e := toml.NewEncoder()
fileSource := file.NewSource(
file.WithPath("/tmp/config"),
source.WithEncoder(e),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load file source
conf.Load(fileSource)
```
+87
View File
@@ -0,0 +1,87 @@
// Package file is a file source. Expected format is json
package file
import (
"io"
"io/fs"
"os"
"go-micro.dev/v6/config/source"
)
type file struct {
opts source.Options
fs fs.FS
path string
}
var (
DefaultPath = "config.json"
)
func (f *file) Read() (*source.ChangeSet, error) {
var fh fs.File
var err error
if f.fs != nil {
fh, err = f.fs.Open(f.path)
} else {
fh, err = os.Open(f.path)
}
if err != nil {
return nil, err
}
defer fh.Close()
b, err := io.ReadAll(fh)
if err != nil {
return nil, err
}
info, err := fh.Stat()
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: format(f.path, f.opts.Encoder),
Source: f.String(),
Timestamp: info.ModTime(),
Data: b,
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (f *file) String() string {
return "file"
}
func (f *file) Watch() (source.Watcher, error) {
// do not watch if fs.FS instance is provided
if f.fs != nil {
return source.NewNoopWatcher()
}
if _, err := os.Stat(f.path); err != nil {
return nil, err
}
return newWatcher(f)
}
func (f *file) Write(cs *source.ChangeSet) error {
return nil
}
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
fs, _ := options.Context.Value(fsKey{}).(fs.FS)
path := DefaultPath
f, ok := options.Context.Value(filePathKey{}).(string)
if ok {
path = f
}
return &file{opts: options, fs: fs, path: path}
}
+89
View File
@@ -0,0 +1,89 @@
package file_test
import (
"fmt"
"os"
"path/filepath"
"testing"
"testing/fstest"
"time"
"go-micro.dev/v6/config"
"go-micro.dev/v6/config/source/file"
)
func TestConfig(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
conf, err := config.NewConfig()
if err != nil {
t.Fatal(err)
}
conf.Load(file.NewSource(file.WithPath(path)))
// simulate multiple close
go conf.Close()
go conf.Close()
}
func TestFile(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
f := file.NewSource(file.WithPath(path))
c, err := f.Read()
if err != nil {
t.Error(err)
}
if string(c.Data) != string(data) {
t.Logf("%+v", c)
t.Error("data from file does not match")
}
}
func TestWithFS(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := fmt.Sprintf("file.%d", time.Now().UnixNano())
fsMock := fstest.MapFS{
path: &fstest.MapFile{
Data: data,
Mode: 0666,
},
}
f := file.NewSource(file.WithFS(fsMock), file.WithPath(path))
c, err := f.Read()
if err != nil {
t.Error(err)
}
if string(c.Data) != string(data) {
t.Logf("%+v", c)
t.Error("data from file does not match")
}
}
+15
View File
@@ -0,0 +1,15 @@
package file
import (
"strings"
"go-micro.dev/v6/config/encoder"
)
func format(p string, e encoder.Encoder) string {
parts := strings.Split(p, ".")
if len(parts) > 1 {
return parts[len(parts)-1]
}
return e.String()
}
+30
View File
@@ -0,0 +1,30 @@
package file
import (
"testing"
"go-micro.dev/v6/config/source"
)
func TestFormat(t *testing.T) {
opts := source.NewOptions()
e := opts.Encoder
testCases := []struct {
p string
f string
}{
{"/foo/bar.json", "json"},
{"/foo/bar.yaml", "yaml"},
{"/foo/bar.xml", "xml"},
{"/foo/bar.conf.ini", "ini"},
{"conf", e.String()},
}
for _, d := range testCases {
f := format(d.p, e)
if f != d.f {
t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
}
}
}
+31
View File
@@ -0,0 +1,31 @@
package file
import (
"context"
"io/fs"
"go-micro.dev/v6/config/source"
)
type filePathKey struct{}
type fsKey struct{}
// WithPath sets the path to file.
func WithPath(p string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, filePathKey{}, p)
}
}
// WithFS sets the underlying filesystem to lookup file from (default os.FS).
func WithFS(fs fs.FS) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, fsKey{}, fs)
}
}
+68
View File
@@ -0,0 +1,68 @@
//go:build !linux
// +build !linux
package file
import (
"os"
"github.com/fsnotify/fsnotify"
"go-micro.dev/v6/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// try get the event
select {
case event, ok := <-w.fw.Events:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
if event.Has(fsnotify.Rename) {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
w.fw.Add(event.Name)
}
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
return c, nil
case err, ok := <-w.fw.Errors:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
return nil, err
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}
+71
View File
@@ -0,0 +1,71 @@
//go:build linux
// +build linux
package file
import (
"os"
"github.com/fsnotify/fsnotify"
"go-micro.dev/v6/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
_ = fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// try get the event
select {
case event, ok := <-w.fw.Events:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
if event.Has(fsnotify.Rename) {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
_ = w.fw.Add(event.Name)
}
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
// add path again for the event bug of fsnotify
_ = w.fw.Add(w.f.path)
return c, nil
case err, ok := <-w.fw.Errors:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
return nil, err
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}
+120
View File
@@ -0,0 +1,120 @@
package file_test
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"go-micro.dev/v6/config/source"
"go-micro.dev/v6/config/source/file"
)
// createTestFile a local helper to creates a temporary file with the given data
func createTestFile(data []byte) (*os.File, func(), string, error) {
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
return nil, func() {}, "", err
}
_, err = fh.Write(data)
if err != nil {
return nil, func() {}, "", err
}
return fh, func() {
fh.Close()
os.Remove(path)
}, path, err
}
func TestWatcher(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
fh, cleanup, path, err := createTestFile(data)
if err != nil {
t.Error(err)
}
defer cleanup()
f := file.NewSource(file.WithPath(path))
if err != nil {
t.Error(err)
}
// create a watcher
w, err := f.Watch()
if err != nil {
t.Error(err)
}
newdata := []byte(`{"foo": "baz"}`)
go func() {
sc, err := w.Next()
if err != nil {
t.Error(err)
return
}
if !bytes.Equal(sc.Data, newdata) {
t.Error("expected data to be different")
}
}()
// rewrite to the file to trigger a change
_, err = fh.WriteAt(newdata, 0)
if err != nil {
t.Error(err)
}
// wait for the underlying watcher to detect changes
time.Sleep(time.Second)
}
func TestWatcherStop(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
_, cleanup, path, err := createTestFile(data)
if err != nil {
t.Error(err)
}
defer cleanup()
src := file.NewSource(file.WithPath(path))
if err != nil {
t.Error(err)
}
// create a watcher
w, err := src.Watch()
if err != nil {
t.Error(err)
}
defer func() {
var err error
c := make(chan struct{})
defer close(c)
go func() {
_, err = w.Next()
c <- struct{}{}
}()
select {
case <-time.After(2 * time.Second):
err = errors.New("timeout waiting for Watcher.Next() to return")
case <-c:
}
if !errors.Is(err, source.ErrWatcherStopped) {
t.Error(err)
}
}()
// stop the watcher
w.Stop()
}
+47
View File
@@ -0,0 +1,47 @@
# Flag Source
The flag source reads config from flags
## Format
We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
### Example
```go
dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
dbPort := flag.Int("database_port", 3306, "the db port)
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## New Source
```go
flagSource := flag.NewSource(
// optionally enable reading of unset flags and their default
// values into config, defaults to false
IncludeUnset(true)
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load flag source
conf.Load(flagSource)
```
+102
View File
@@ -0,0 +1,102 @@
package flag
import (
"errors"
"flag"
"strings"
"time"
"dario.cat/mergo"
"go-micro.dev/v6/config/source"
)
type flagsrc struct {
opts source.Options
}
func (fs *flagsrc) Read() (*source.ChangeSet, error) {
if !flag.Parsed() {
return nil, errors.New("flags not parsed")
}
var changes map[string]interface{}
visitFn := func(f *flag.Flag) {
n := strings.ToLower(f.Name)
keys := strings.FieldsFunc(n, split)
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
tmp[k] = f.Value
continue
}
tmp = map[string]interface{}{k: tmp}
}
_ = mergo.Map(&changes, tmp) // need to sort error handling
}
unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
if ok && unset {
flag.VisitAll(visitFn)
} else {
flag.Visit(visitFn)
}
b, err := fs.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: fs.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: fs.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func split(r rune) bool {
return r == '-' || r == '_'
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (fs *flagsrc) Watch() (source.Watcher, error) {
return source.NewNoopWatcher()
}
func (fs *flagsrc) Write(cs *source.ChangeSet) error {
return nil
}
func (fs *flagsrc) String() string {
return "flag"
}
// NewSource returns a config source for integrating parsed flags.
// Hyphens are delimiters for nesting, and all keys are lowercased.
//
// Example:
//
// dbhost := flag.String("database-host", "localhost", "the db host name")
//
// {
// "database": {
// "host": "localhost"
// }
// }
func NewSource(opts ...source.Option) source.Source {
return &flagsrc{opts: source.NewOptions(opts...)}
}
+68
View File
@@ -0,0 +1,68 @@
package flag
import (
"encoding/json"
"flag"
"testing"
)
var (
dbuser = flag.String("database-user", "default", "db user")
dbhost = flag.String("database-host", "", "db host")
dbpw = flag.String("database-password", "", "db pw")
)
func initTestFlags() {
flag.Set("database-host", "localhost")
flag.Set("database-password", "some-password")
flag.Parse()
}
func TestFlagsrc_Read(t *testing.T) {
initTestFlags()
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
if actualDB["host"] != *dbhost {
t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
}
if actualDB["password"] != *dbpw {
t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
}
// unset flags should not be loaded
if actualDB["user"] != nil {
t.Errorf("expected %v got %v", nil, actualDB["user"])
}
}
func TestFlagsrc_ReadAll(t *testing.T) {
initTestFlags()
source := NewSource(IncludeUnset(true))
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
// unset flag defaults should be loaded
if actualDB["user"] != *dbuser {
t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
}
}
+20
View File
@@ -0,0 +1,20 @@
package flag
import (
"context"
"go-micro.dev/v6/config/source"
)
type includeUnsetKey struct{}
// IncludeUnset toggles the loading of unset flags and their respective default values.
// Default behavior is to ignore any unset flags.
func IncludeUnset(b bool) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
}
}
+44
View File
@@ -0,0 +1,44 @@
# Memory Source
The memory source provides in-memory data as a source
## Memory Format
The expected data format is json
```json
data := []byte(`{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}`)
```
## New Source
Specify source with data
```go
memorySource := memory.NewSource(
memory.WithJSON(data),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load memory source
conf.Load(memorySource)
```
+99
View File
@@ -0,0 +1,99 @@
// Package memory is a memory source
package memory
import (
"sync"
"time"
"github.com/google/uuid"
"go-micro.dev/v6/config/source"
)
type memory struct {
ChangeSet *source.ChangeSet
Watchers map[string]*watcher
sync.RWMutex
}
func (s *memory) Read() (*source.ChangeSet, error) {
s.RLock()
cs := &source.ChangeSet{
Format: s.ChangeSet.Format,
Timestamp: s.ChangeSet.Timestamp,
Data: s.ChangeSet.Data,
Checksum: s.ChangeSet.Checksum,
Source: s.ChangeSet.Source,
}
s.RUnlock()
return cs, nil
}
func (s *memory) Watch() (source.Watcher, error) {
w := &watcher{
Id: uuid.New().String(),
Updates: make(chan *source.ChangeSet, 100),
Source: s,
}
s.Lock()
s.Watchers[w.Id] = w
s.Unlock()
return w, nil
}
func (s *memory) Write(cs *source.ChangeSet) error {
s.Update(cs)
return nil
}
// Update allows manual updates of the config data.
func (s *memory) Update(c *source.ChangeSet) {
// don't process nil
if c == nil {
return
}
// hash the file
s.Lock()
// update changeset
s.ChangeSet = &source.ChangeSet{
Data: c.Data,
Format: c.Format,
Source: "memory",
Timestamp: time.Now(),
}
s.ChangeSet.Checksum = s.ChangeSet.Sum()
// update watchers
for _, w := range s.Watchers {
select {
case w.Updates <- s.ChangeSet:
default:
}
}
s.Unlock()
}
func (s *memory) String() string {
return "memory"
}
func NewSource(opts ...source.Option) source.Source {
var options source.Options
for _, o := range opts {
o(&options)
}
s := &memory{
Watchers: make(map[string]*watcher),
}
if options.Context != nil {
c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
if ok {
s.Update(c)
}
}
return s
}
+41
View File
@@ -0,0 +1,41 @@
package memory
import (
"context"
"go-micro.dev/v6/config/source"
)
type changeSetKey struct{}
func withData(d []byte, f string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
Data: d,
Format: f,
})
}
}
// WithChangeSet allows a changeset to be set.
func WithChangeSet(cs *source.ChangeSet) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
}
}
// WithJSON allows the source data to be set to json.
func WithJSON(d []byte) source.Option {
return withData(d, "json")
}
// WithYAML allows the source data to be set to yaml.
func WithYAML(d []byte) source.Option {
return withData(d, "yaml")
}
+23
View File
@@ -0,0 +1,23 @@
package memory
import (
"go-micro.dev/v6/config/source"
)
type watcher struct {
Updates chan *source.ChangeSet
Source *memory
Id string
}
func (w *watcher) Next() (*source.ChangeSet, error) {
cs := <-w.Updates
return cs, nil
}
func (w *watcher) Stop() error {
w.Source.Lock()
delete(w.Source.Watchers, w.Id)
w.Source.Unlock()
return nil
}
+56
View File
@@ -0,0 +1,56 @@
# Nats Source
The nats source reads config from nats key/values
## Nats Format
The nats source expects keys under the default bucket `default` default key `micro_config`
Values are expected to be json
```
nats kv put default micro_config '{"nats": {"address": "10.0.0.1", "port": 8488}}'
```
```
conf.Get("nats")
```
## New Source
Specify source with data
```go
natsSource := nats.NewSource(
nats.WithUrl("127.0.0.1:4222"),
nats.WithBucket("my_bucket"),
nats.WithKey("my_key"),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load nats source
conf.Load(natsSource)
```
## Watch
```go
wh, _ := natsSource.Watch()
for {
v, err := watcher.Next()
if err != nil {
log.Fatalf("err %v", err)
}
log.Infof("data %v", string(v.Data))
}
```
+146
View File
@@ -0,0 +1,146 @@
package nats
import (
"fmt"
"net"
"strings"
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/source"
log "go-micro.dev/v6/logger"
)
type nats struct {
url string
bucket string
key string
conn *natsgo.Conn // store connection for lifecycle management
kv natsgo.KeyValue
opts source.Options
}
// DefaultBucket is the bucket that nats keys will be assumed to have if you
// haven't specified one.
var (
DefaultBucket = "default"
DefaultKey = "micro_config"
)
func (n *nats) Read() (*source.ChangeSet, error) {
e, err := n.kv.Get(n.key)
if err != nil {
if err == natsgo.ErrKeyNotFound {
return nil, nil
}
return nil, err
}
if e.Value() == nil || len(e.Value()) == 0 {
return nil, fmt.Errorf("source not found: %s", n.key)
}
cs := &source.ChangeSet{
Data: e.Value(),
Format: n.opts.Encoder.String(),
Source: n.String(),
Timestamp: time.Now(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (n *nats) Write(cs *source.ChangeSet) error {
_, err := n.kv.Put(n.key, cs.Data)
if err != nil {
return err
}
return nil
}
func (n *nats) String() string {
return "nats"
}
func (n *nats) Watch() (source.Watcher, error) {
return newWatcher(n.kv, n.bucket, n.key, n.String(), n.opts.Encoder)
}
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
config := natsgo.GetDefaultOptions()
urls, ok := options.Context.Value(urlKey{}).([]string)
endpoints := []string{}
if ok {
for _, u := range urls {
addr, port, err := net.SplitHostPort(u)
if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
port = "4222"
addr = u
endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
} else if err == nil {
endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
}
}
}
if len(endpoints) == 0 {
endpoints = append(endpoints, "127.0.0.1:4222")
}
bucket, ok := options.Context.Value(bucketKey{}).(string)
if !ok {
bucket = DefaultBucket
}
key, ok := options.Context.Value(keyKey{}).(string)
if !ok {
key = DefaultKey
}
config.Url = strings.Join(endpoints, ",")
nc, err := natsgo.Connect(config.Url)
if err != nil {
log.Error(err)
}
js, err := nc.JetStream(natsgo.MaxWait(10 * time.Second))
if err != nil {
log.Error(err)
}
kv, err := js.KeyValue(bucket)
if err == natsgo.ErrBucketNotFound || err == natsgo.ErrKeyNotFound {
kv, err = js.CreateKeyValue(&natsgo.KeyValueConfig{Bucket: bucket})
if err != nil {
log.Error(err)
}
}
if err != nil {
log.Error(err)
}
return &nats{
url: config.Url,
bucket: bucket,
key: key,
conn: nc, // store connection reference
kv: kv,
opts: options,
}
}
// Close implements io.Closer and closes the underlying NATS connection.
// This method is optional but recommended to prevent connection leaks.
func (n *nats) Close() error {
if n.conn != nil {
n.conn.Close()
n.conn = nil
}
return nil
}
+54
View File
@@ -0,0 +1,54 @@
package nats
import (
"context"
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/source"
)
type (
urlKey struct{}
bucketKey struct{}
keyKey struct{}
)
// WithUrl sets the nats url.
func WithUrl(a ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, urlKey{}, a)
}
}
// WithBucket sets the nats key.
func WithBucket(a string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, bucketKey{}, a)
}
}
// WithKey sets the nats key.
func WithKey(a string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, keyKey{}, a)
}
}
func Client(url string) (natsgo.JetStreamContext, error) {
nc, err := natsgo.Connect(url)
if err != nil {
return nil, err
}
return nc.JetStream(natsgo.MaxWait(10 * time.Second))
}
+79
View File
@@ -0,0 +1,79 @@
package nats
import (
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/source"
)
type watcher struct {
e encoder.Encoder
name string
bucket string
key string
ch chan *source.ChangeSet
exit chan bool
}
func newWatcher(kv natsgo.KeyValue, bucket, key, name string, e encoder.Encoder) (source.Watcher, error) {
w := &watcher{
e: e,
name: name,
bucket: bucket,
key: key,
ch: make(chan *source.ChangeSet),
exit: make(chan bool),
}
wh, _ := kv.Watch(key)
go func() {
for {
select {
case v := <-wh.Updates():
if v != nil {
w.handle(v.Value())
}
case <-w.exit:
_ = wh.Stop()
return
}
}
}()
return w, nil
}
func (w *watcher) handle(data []byte) {
cs := &source.ChangeSet{
Timestamp: time.Now(),
Format: w.e.String(),
Source: w.name,
Data: data,
}
cs.Checksum = cs.Sum()
w.ch <- cs
}
func (w *watcher) Next() (*source.ChangeSet, error) {
select {
case cs := <-w.ch:
return cs, nil
case <-w.exit:
return nil, source.ErrWatcherStopped
}
}
func (w *watcher) Stop() error {
select {
case <-w.exit:
return nil
default:
close(w.exit)
}
return nil
}
+25
View File
@@ -0,0 +1,25 @@
package source
import (
"errors"
)
type noopWatcher struct {
exit chan struct{}
}
func (w *noopWatcher) Next() (*ChangeSet, error) {
<-w.exit
return nil, errors.New("noopWatcher stopped")
}
func (w *noopWatcher) Stop() error {
close(w.exit)
return nil
}
// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
func NewNoopWatcher() (Watcher, error) {
return &noopWatcher{exit: make(chan struct{})}, nil
}
+50
View File
@@ -0,0 +1,50 @@
package source
import (
"context"
"go-micro.dev/v6/client"
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/encoder/json"
)
type Options struct {
// Encoder
Encoder encoder.Encoder
// for alternative data
Context context.Context
// Client to use for RPC
Client client.Client
}
type Option func(o *Options)
func NewOptions(opts ...Option) Options {
options := Options{
Encoder: json.NewEncoder(),
Context: context.Background(),
Client: client.DefaultClient,
}
for _, o := range opts {
o(&options)
}
return options
}
// WithEncoder sets the source encoder.
func WithEncoder(e encoder.Encoder) Option {
return func(o *Options) {
o.Encoder = e
}
}
// WithClient sets the source client.
func WithClient(c client.Client) Option {
return func(o *Options) {
o.Client = c
}
}
+35
View File
@@ -0,0 +1,35 @@
// Package source is the interface for sources
package source
import (
"errors"
"time"
)
var (
// ErrWatcherStopped is returned when source watcher has been stopped.
ErrWatcherStopped = errors.New("watcher stopped")
)
// Source is the source from which config is loaded.
type Source interface {
Read() (*ChangeSet, error)
Write(*ChangeSet) error
Watch() (Watcher, error)
String() string
}
// ChangeSet represents a set of changes from a source.
type ChangeSet struct {
Timestamp time.Time
Checksum string
Format string
Source string
Data []byte
}
// Watcher watches a source for changes.
type Watcher interface {
Next() (*ChangeSet, error)
Stop() error
}
+49
View File
@@ -0,0 +1,49 @@
package config
import (
"time"
"go-micro.dev/v6/config/reader"
)
type value struct{}
func newValue() reader.Value {
return new(value)
}
func (v *value) Bool(def bool) bool {
return false
}
func (v *value) Int(def int) int {
return 0
}
func (v *value) String(def string) string {
return ""
}
func (v *value) Float64(def float64) float64 {
return 0.0
}
func (v *value) Duration(def time.Duration) time.Duration {
return time.Duration(0)
}
func (v *value) StringSlice(def []string) []string {
return nil
}
func (v *value) StringMap(def map[string]string) map[string]string {
return map[string]string{}
}
func (v *value) Scan(val interface{}) error {
return nil
}
func (v *value) Bytes() []byte {
return nil
}