chore: import upstream snapshot with attribution
govulncheck / govulncheck (push) Has been cancelled
Lint / golangci-lint (push) Has been cancelled
Run Tests / Unit Tests (push) Has been cancelled
Run Tests / Etcd Integration Tests (push) Has been cancelled
Harness (E2E) / Harnesses (mock LLM) (push) Has been cancelled
Harness (E2E) / Provider harnesses (live LLM conformance) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:33 +08:00
commit e071084ebe
982 changed files with 160368 additions and 0 deletions
+13
View File
@@ -0,0 +1,13 @@
package source
import (
"crypto/md5"
"fmt"
)
// Sum returns the md5 checksum of the ChangeSet data.
func (c *ChangeSet) Sum() string {
h := md5.New()
h.Write(c.Data)
return fmt.Sprintf("%x", h.Sum(nil))
}
+71
View File
@@ -0,0 +1,71 @@
# cli Source
The cli source reads config from parsed flags via a cli.Context.
## Format
We expect the use of the `urfave/cli` package. Upper case flags will be lower cased. Dashes will be used as delimiters for nesting.
### Example
```go
micro.Flags(
cli.StringFlag{
Name: "database-address",
Value: "127.0.0.1",
Usage: "the db address",
},
cli.IntFlag{
Name: "database-port",
Value: 3306,
Usage: "the db port",
},
)
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## New and Load Source
Because a cli.Context is needed to retrieve the flags and their values, it is recommended to build your source from within a cli.Action.
```go
func main() {
// New Service
service := micro.NewService(
micro.Name("example"),
micro.Flags(
cli.StringFlag{
Name: "database-address",
Value: "127.0.0.1",
Usage: "the db address",
},
),
)
var clisrc source.Source
service.Init(
micro.Action(func(c *cli.Context) {
clisrc = cli.NewSource(
cli.Context(c),
)
// Alternatively, just setup your config right here
}),
)
// ... Load and use that source ...
conf := config.NewConfig()
conf.Load(clisrc)
}
```
+148
View File
@@ -0,0 +1,148 @@
package cli
import (
"flag"
"io"
"os"
"strings"
"time"
"dario.cat/mergo"
"github.com/urfave/cli/v2"
"go-micro.dev/v6/cmd"
"go-micro.dev/v6/config/source"
)
type cliSource struct {
opts source.Options
ctx *cli.Context
}
func (c *cliSource) Read() (*source.ChangeSet, error) {
var changes map[string]interface{}
// directly using app cli flags, to access default values of not specified options
for _, f := range c.ctx.App.Flags {
name := f.Names()[0]
tmp := toEntry(name, c.ctx.Generic(name))
if err := mergo.Map(&changes, tmp, mergo.WithOverride); err != nil {
return nil, err
}
}
b, err := c.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: c.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: c.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func toEntry(name string, v interface{}) map[string]interface{} {
n := strings.ToLower(name)
keys := strings.FieldsFunc(n, split)
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
tmp[k] = v
continue
}
tmp = map[string]interface{}{k: tmp}
}
return tmp
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func split(r rune) bool {
return r == '-' || r == '_'
}
func (c *cliSource) Watch() (source.Watcher, error) {
return source.NewNoopWatcher()
}
// Write is unsupported.
func (c *cliSource) Write(cs *source.ChangeSet) error {
return nil
}
func (c *cliSource) String() string {
return "cli"
}
// NewSource returns a config source for integrating parsed flags from a urfave/cli.Context.
// Hyphens are delimiters for nesting, and all keys are lowercased. The assumption is that
// command line flags have already been parsed.
//
// Example:
//
// cli.StringFlag{Name: "db-host"},
//
//
// {
// "database": {
// "host": "localhost"
// }
// }
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
var ctx *cli.Context
if c, ok := options.Context.Value(contextKey{}).(*cli.Context); ok {
ctx = c
} else {
// no context
// get the default app/flags
app := cmd.App()
flags := app.Flags
// create flagset
set := flag.NewFlagSet(app.Name, flag.ContinueOnError)
// apply flags to set
for _, f := range flags {
_ = f.Apply(set)
}
// parse flags
set.SetOutput(io.Discard)
_ = set.Parse(os.Args[1:])
// normalise flags
_ = normalizeFlags(app.Flags, set)
// create context
ctx = cli.NewContext(app, set, nil)
}
return &cliSource{
ctx: ctx,
opts: options,
}
}
// WithContext returns a new source with the context specified.
// The assumption is that Context is retrieved within an app.Action function.
func WithContext(ctx *cli.Context, opts ...source.Option) source.Source {
return &cliSource{
ctx: ctx,
opts: source.NewOptions(opts...),
}
}
+121
View File
@@ -0,0 +1,121 @@
package cli
import (
"encoding/json"
"os"
"testing"
"github.com/urfave/cli/v2"
"go-micro.dev/v6"
"go-micro.dev/v6/cmd"
"go-micro.dev/v6/config"
"go-micro.dev/v6/config/source"
)
func TestCliSourceDefault(t *testing.T) {
const expVal string = "flagvalue"
service := micro.NewService("test", micro.Flags(
// to be able to run inside go test
&cli.StringFlag{
Name: "test.timeout",
},
&cli.StringFlag{
Name: "test.bench",
},
&cli.BoolFlag{
Name: "test.v",
},
&cli.StringFlag{
Name: "test.run",
},
&cli.StringFlag{
Name: "test.testlogfile",
},
&cli.StringFlag{
Name: "test.paniconexit0",
},
&cli.StringFlag{
Name: "flag",
Usage: "It changes something",
EnvVars: []string{"flag"},
Value: expVal,
},
),
)
var cliSrc source.Source
service.Init(
// Loads CLI configuration
micro.Action(func(c *cli.Context) error {
cliSrc = NewSource(
Context(c),
)
return nil
}),
)
config.Load(cliSrc)
if val, err := config.Get("flag"); err != nil {
t.Fatal(err)
} else if fval := val.String("default"); fval != expVal {
t.Fatalf("default flag value not loaded %v != %v", fval, expVal)
}
}
func test(t *testing.T, withContext bool) {
var src source.Source
// setup app
app := cmd.App()
app.Name = "testapp"
app.Flags = []cli.Flag{
&cli.StringFlag{
Name: "db-host",
EnvVars: []string{"db-host"},
Value: "myval",
},
}
// with context
if withContext {
// set action
app.Action = func(c *cli.Context) error {
src = WithContext(c)
return nil
}
// run app
app.Run([]string{"run", "-db-host", "localhost"})
// no context
} else {
// set args
os.Args = []string{"run", "-db-host", "localhost"}
src = NewSource()
}
// test config
c, err := src.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["db"].(map[string]interface{})
if actualDB["host"] != "localhost" {
t.Errorf("expected localhost, got %v", actualDB["name"])
}
}
func TestCliSource(t *testing.T) {
// without context
test(t, false)
}
func TestCliSourceWithContext(t *testing.T) {
// with context
test(t, true)
}
+20
View File
@@ -0,0 +1,20 @@
package cli
import (
"context"
"github.com/urfave/cli/v2"
"go-micro.dev/v6/config/source"
)
type contextKey struct{}
// Context sets the cli context.
func Context(c *cli.Context) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, contextKey{}, c)
}
}
+50
View File
@@ -0,0 +1,50 @@
package cli
import (
"errors"
"flag"
"strings"
"github.com/urfave/cli/v2"
)
func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
switch ff.Value.(type) {
case *cli.StringSlice:
default:
_ = set.Set(name, ff.Value.String())
}
}
func normalizeFlags(flags []cli.Flag, set *flag.FlagSet) error {
visited := make(map[string]bool)
set.Visit(func(f *flag.Flag) {
visited[f.Name] = true
})
for _, f := range flags {
parts := f.Names()
if len(parts) == 1 {
continue
}
var ff *flag.Flag
for _, name := range parts {
name = strings.Trim(name, " ")
if visited[name] {
if ff != nil {
return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
}
ff = set.Lookup(name)
}
}
if ff == nil {
continue
}
for _, name := range parts {
name = strings.Trim(name, " ")
if !visited[name] {
copyFlag(name, ff, set)
}
}
}
return nil
}
+94
View File
@@ -0,0 +1,94 @@
# Env Source
The env source reads config from environment variables
## Format
We expect environment variables to be in the standard format of FOO=bar
Keys are converted to lowercase and split on underscore.
### Format example
```bash
DATABASE_ADDRESS=127.0.0.1
DATABASE_PORT=3306
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## Prefixes
Environment variables can be namespaced so we only have access to a subset. Two options are available:
```go
WithPrefix(p ...string)
WithStrippedPrefix(p ...string)
```
The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one.
### Prefixes example
Given ENVs of:
```bash
APP_DATABASE_ADDRESS=127.0.0.1
APP_DATABASE_PORT=3306
VAULT_ADDR=vault:1337
```
and a source initialized as follows:
```go
src := env.NewSource(
env.WithPrefix("VAULT"),
env.WithStrippedPrefix("APP"),
)
```
The resulting config will be:
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
},
"vault": {
"addr": "vault:1337"
}
}
```
## New Source
Specify source with data
```go
src := env.NewSource(
// optionally specify prefix
env.WithPrefix("MICRO"),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load env source
conf.Load(src)
```
+146
View File
@@ -0,0 +1,146 @@
package env
import (
"os"
"strconv"
"strings"
"time"
"dario.cat/mergo"
"go-micro.dev/v6/config/source"
)
var (
DefaultPrefixes = []string{}
)
type env struct {
opts source.Options
prefixes []string
strippedPrefixes []string
}
func (e *env) Read() (*source.ChangeSet, error) {
var changes map[string]interface{}
for _, env := range os.Environ() {
if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
notFound := true
if _, ok := matchPrefix(e.prefixes, env); ok {
notFound = false
}
if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
env = strings.TrimPrefix(env, match)
notFound = false
}
if notFound {
continue
}
}
pair := strings.SplitN(env, "=", 2)
value := pair[1]
keys := strings.Split(strings.ToLower(pair[0]), "_")
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
if intValue, err := strconv.Atoi(value); err == nil {
tmp[k] = intValue
} else if boolValue, err := strconv.ParseBool(value); err == nil {
tmp[k] = boolValue
} else {
tmp[k] = value
}
continue
}
tmp = map[string]interface{}{k: tmp}
}
if err := mergo.Map(&changes, tmp); err != nil {
return nil, err
}
}
b, err := e.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: e.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: e.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func matchPrefix(pre []string, s string) (string, bool) {
for _, p := range pre {
if strings.HasPrefix(s, p) {
return p, true
}
}
return "", false
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (e *env) Watch() (source.Watcher, error) {
return newWatcher()
}
func (e *env) Write(cs *source.ChangeSet) error {
return nil
}
func (e *env) String() string {
return "env"
}
// NewSource returns a config source for parsing ENV variables.
// Underscores are delimiters for nesting, and all keys are lowercased.
//
// Example:
//
// "DATABASE_SERVER_HOST=localhost" will convert to
//
// {
// "database": {
// "server": {
// "host": "localhost"
// }
// }
// }
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
var sp []string
var pre []string
if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
sp = p
}
if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
pre = p
}
if len(sp) > 0 || len(pre) > 0 {
pre = append(pre, DefaultPrefixes...)
}
return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
}
+112
View File
@@ -0,0 +1,112 @@
package env
import (
"encoding/json"
"os"
"testing"
"time"
"go-micro.dev/v6/config/source"
)
func TestEnv_Read(t *testing.T) {
expected := map[string]map[string]string{
"database": {
"host": "localhost",
"password": "password",
"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
},
}
os.Setenv("DATABASE_HOST", "localhost")
os.Setenv("DATABASE_PASSWORD", "password")
os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
for k, v := range expected["database"] {
a := actualDB[k]
if a != v {
t.Errorf("expected %v got %v", v, a)
}
}
}
func TestEnvvar_Prefixes(t *testing.T) {
os.Setenv("APP_DATABASE_HOST", "localhost")
os.Setenv("APP_DATABASE_PASSWORD", "password")
os.Setenv("VAULT_ADDR", "vault:1337")
os.Setenv("MICRO_REGISTRY", "mdns")
var prefixtests = []struct {
prefixOpts []source.Option
expectedKeys []string
}{
{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
}
for _, pt := range prefixtests {
source := NewSource(pt.prefixOpts...)
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
// assert other prefixes ignored
if l := len(actual); l != len(pt.expectedKeys) {
t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
}
for _, k := range pt.expectedKeys {
if !containsKey(actual, k) {
t.Errorf("expected key %v, not found", k)
}
}
}
}
func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
src := NewSource(WithStrippedPrefix("GOMICRO_"))
w, err := src.Watch()
if err != nil {
t.Error(err)
}
go func() {
time.Sleep(50 * time.Millisecond)
w.Stop()
}()
if _, err := w.Next(); err != source.ErrWatcherStopped {
t.Errorf("expected watcher stopped error, got %v", err)
}
}
func containsKey(m map[string]interface{}, s string) bool {
for k := range m {
if k == s {
return true
}
}
return false
}
+49
View File
@@ -0,0 +1,49 @@
package env
import (
"context"
"strings"
"go-micro.dev/v6/config/source"
)
type strippedPrefixKey struct{}
type prefixKey struct{}
// WithStrippedPrefix sets the environment variable prefixes to scope to.
// These prefixes will be removed from the actual config entries.
func WithStrippedPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
}
}
// WithPrefix sets the environment variable prefixes to scope to.
// These prefixes will not be removed. Each prefix will be considered a top level config entry.
func WithPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
}
}
func appendUnderscore(prefixes []string) []string {
//nolint:prealloc
var result []string
for _, p := range prefixes {
if !strings.HasSuffix(p, "_") {
result = append(result, p+"_")
continue
}
result = append(result, p)
}
return result
}
+24
View File
@@ -0,0 +1,24 @@
package env
import (
"go-micro.dev/v6/config/source"
)
type watcher struct {
exit chan struct{}
}
func (w *watcher) Next() (*source.ChangeSet, error) {
<-w.exit
return nil, source.ErrWatcherStopped
}
func (w *watcher) Stop() error {
close(w.exit)
return nil
}
func newWatcher() (source.Watcher, error) {
return &watcher{exit: make(chan struct{})}, nil
}
+69
View File
@@ -0,0 +1,69 @@
# File Source
The file source reads config from a file.
It uses the File extension to determine the Format e.g `config.yaml` has the yaml format.
It does not make use of encoders or interpet the file data. If a file extension is not present
the source Format will default to the Encoder in options.
## Example
A config file format in json
```json
{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}
```
## New Source
Specify file source with path to file. Path is optional and will default to `config.json`
```go
fileSource := file.NewSource(
file.WithPath("/tmp/config.json"),
)
```
## File Format
To load different file formats e.g yaml, toml, xml simply specify them with their extension
```go
fileSource := file.NewSource(
file.WithPath("/tmp/config.yaml"),
)
```
If you want to specify a file without extension, ensure you set the encoder to the same format
```go
e := toml.NewEncoder()
fileSource := file.NewSource(
file.WithPath("/tmp/config"),
source.WithEncoder(e),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load file source
conf.Load(fileSource)
```
+87
View File
@@ -0,0 +1,87 @@
// Package file is a file source. Expected format is json
package file
import (
"io"
"io/fs"
"os"
"go-micro.dev/v6/config/source"
)
type file struct {
opts source.Options
fs fs.FS
path string
}
var (
DefaultPath = "config.json"
)
func (f *file) Read() (*source.ChangeSet, error) {
var fh fs.File
var err error
if f.fs != nil {
fh, err = f.fs.Open(f.path)
} else {
fh, err = os.Open(f.path)
}
if err != nil {
return nil, err
}
defer fh.Close()
b, err := io.ReadAll(fh)
if err != nil {
return nil, err
}
info, err := fh.Stat()
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: format(f.path, f.opts.Encoder),
Source: f.String(),
Timestamp: info.ModTime(),
Data: b,
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (f *file) String() string {
return "file"
}
func (f *file) Watch() (source.Watcher, error) {
// do not watch if fs.FS instance is provided
if f.fs != nil {
return source.NewNoopWatcher()
}
if _, err := os.Stat(f.path); err != nil {
return nil, err
}
return newWatcher(f)
}
func (f *file) Write(cs *source.ChangeSet) error {
return nil
}
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
fs, _ := options.Context.Value(fsKey{}).(fs.FS)
path := DefaultPath
f, ok := options.Context.Value(filePathKey{}).(string)
if ok {
path = f
}
return &file{opts: options, fs: fs, path: path}
}
+89
View File
@@ -0,0 +1,89 @@
package file_test
import (
"fmt"
"os"
"path/filepath"
"testing"
"testing/fstest"
"time"
"go-micro.dev/v6/config"
"go-micro.dev/v6/config/source/file"
)
func TestConfig(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
conf, err := config.NewConfig()
if err != nil {
t.Fatal(err)
}
conf.Load(file.NewSource(file.WithPath(path)))
// simulate multiple close
go conf.Close()
go conf.Close()
}
func TestFile(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
f := file.NewSource(file.WithPath(path))
c, err := f.Read()
if err != nil {
t.Error(err)
}
if string(c.Data) != string(data) {
t.Logf("%+v", c)
t.Error("data from file does not match")
}
}
func TestWithFS(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := fmt.Sprintf("file.%d", time.Now().UnixNano())
fsMock := fstest.MapFS{
path: &fstest.MapFile{
Data: data,
Mode: 0666,
},
}
f := file.NewSource(file.WithFS(fsMock), file.WithPath(path))
c, err := f.Read()
if err != nil {
t.Error(err)
}
if string(c.Data) != string(data) {
t.Logf("%+v", c)
t.Error("data from file does not match")
}
}
+15
View File
@@ -0,0 +1,15 @@
package file
import (
"strings"
"go-micro.dev/v6/config/encoder"
)
func format(p string, e encoder.Encoder) string {
parts := strings.Split(p, ".")
if len(parts) > 1 {
return parts[len(parts)-1]
}
return e.String()
}
+30
View File
@@ -0,0 +1,30 @@
package file
import (
"testing"
"go-micro.dev/v6/config/source"
)
func TestFormat(t *testing.T) {
opts := source.NewOptions()
e := opts.Encoder
testCases := []struct {
p string
f string
}{
{"/foo/bar.json", "json"},
{"/foo/bar.yaml", "yaml"},
{"/foo/bar.xml", "xml"},
{"/foo/bar.conf.ini", "ini"},
{"conf", e.String()},
}
for _, d := range testCases {
f := format(d.p, e)
if f != d.f {
t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
}
}
}
+31
View File
@@ -0,0 +1,31 @@
package file
import (
"context"
"io/fs"
"go-micro.dev/v6/config/source"
)
type filePathKey struct{}
type fsKey struct{}
// WithPath sets the path to file.
func WithPath(p string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, filePathKey{}, p)
}
}
// WithFS sets the underlying filesystem to lookup file from (default os.FS).
func WithFS(fs fs.FS) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, fsKey{}, fs)
}
}
+68
View File
@@ -0,0 +1,68 @@
//go:build !linux
// +build !linux
package file
import (
"os"
"github.com/fsnotify/fsnotify"
"go-micro.dev/v6/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// try get the event
select {
case event, ok := <-w.fw.Events:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
if event.Has(fsnotify.Rename) {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
w.fw.Add(event.Name)
}
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
return c, nil
case err, ok := <-w.fw.Errors:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
return nil, err
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}
+71
View File
@@ -0,0 +1,71 @@
//go:build linux
// +build linux
package file
import (
"os"
"github.com/fsnotify/fsnotify"
"go-micro.dev/v6/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
_ = fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// try get the event
select {
case event, ok := <-w.fw.Events:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
if event.Has(fsnotify.Rename) {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
_ = w.fw.Add(event.Name)
}
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
// add path again for the event bug of fsnotify
_ = w.fw.Add(w.f.path)
return c, nil
case err, ok := <-w.fw.Errors:
// check if channel was closed (i.e. Watcher.Close() was called).
if !ok {
return nil, source.ErrWatcherStopped
}
return nil, err
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}
+120
View File
@@ -0,0 +1,120 @@
package file_test
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"go-micro.dev/v6/config/source"
"go-micro.dev/v6/config/source/file"
)
// createTestFile a local helper to creates a temporary file with the given data
func createTestFile(data []byte) (*os.File, func(), string, error) {
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
return nil, func() {}, "", err
}
_, err = fh.Write(data)
if err != nil {
return nil, func() {}, "", err
}
return fh, func() {
fh.Close()
os.Remove(path)
}, path, err
}
func TestWatcher(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
fh, cleanup, path, err := createTestFile(data)
if err != nil {
t.Error(err)
}
defer cleanup()
f := file.NewSource(file.WithPath(path))
if err != nil {
t.Error(err)
}
// create a watcher
w, err := f.Watch()
if err != nil {
t.Error(err)
}
newdata := []byte(`{"foo": "baz"}`)
go func() {
sc, err := w.Next()
if err != nil {
t.Error(err)
return
}
if !bytes.Equal(sc.Data, newdata) {
t.Error("expected data to be different")
}
}()
// rewrite to the file to trigger a change
_, err = fh.WriteAt(newdata, 0)
if err != nil {
t.Error(err)
}
// wait for the underlying watcher to detect changes
time.Sleep(time.Second)
}
func TestWatcherStop(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
_, cleanup, path, err := createTestFile(data)
if err != nil {
t.Error(err)
}
defer cleanup()
src := file.NewSource(file.WithPath(path))
if err != nil {
t.Error(err)
}
// create a watcher
w, err := src.Watch()
if err != nil {
t.Error(err)
}
defer func() {
var err error
c := make(chan struct{})
defer close(c)
go func() {
_, err = w.Next()
c <- struct{}{}
}()
select {
case <-time.After(2 * time.Second):
err = errors.New("timeout waiting for Watcher.Next() to return")
case <-c:
}
if !errors.Is(err, source.ErrWatcherStopped) {
t.Error(err)
}
}()
// stop the watcher
w.Stop()
}
+47
View File
@@ -0,0 +1,47 @@
# Flag Source
The flag source reads config from flags
## Format
We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
### Example
```go
dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
dbPort := flag.Int("database_port", 3306, "the db port)
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## New Source
```go
flagSource := flag.NewSource(
// optionally enable reading of unset flags and their default
// values into config, defaults to false
IncludeUnset(true)
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load flag source
conf.Load(flagSource)
```
+102
View File
@@ -0,0 +1,102 @@
package flag
import (
"errors"
"flag"
"strings"
"time"
"dario.cat/mergo"
"go-micro.dev/v6/config/source"
)
type flagsrc struct {
opts source.Options
}
func (fs *flagsrc) Read() (*source.ChangeSet, error) {
if !flag.Parsed() {
return nil, errors.New("flags not parsed")
}
var changes map[string]interface{}
visitFn := func(f *flag.Flag) {
n := strings.ToLower(f.Name)
keys := strings.FieldsFunc(n, split)
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
tmp[k] = f.Value
continue
}
tmp = map[string]interface{}{k: tmp}
}
_ = mergo.Map(&changes, tmp) // need to sort error handling
}
unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
if ok && unset {
flag.VisitAll(visitFn)
} else {
flag.Visit(visitFn)
}
b, err := fs.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: fs.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: fs.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func split(r rune) bool {
return r == '-' || r == '_'
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (fs *flagsrc) Watch() (source.Watcher, error) {
return source.NewNoopWatcher()
}
func (fs *flagsrc) Write(cs *source.ChangeSet) error {
return nil
}
func (fs *flagsrc) String() string {
return "flag"
}
// NewSource returns a config source for integrating parsed flags.
// Hyphens are delimiters for nesting, and all keys are lowercased.
//
// Example:
//
// dbhost := flag.String("database-host", "localhost", "the db host name")
//
// {
// "database": {
// "host": "localhost"
// }
// }
func NewSource(opts ...source.Option) source.Source {
return &flagsrc{opts: source.NewOptions(opts...)}
}
+68
View File
@@ -0,0 +1,68 @@
package flag
import (
"encoding/json"
"flag"
"testing"
)
var (
dbuser = flag.String("database-user", "default", "db user")
dbhost = flag.String("database-host", "", "db host")
dbpw = flag.String("database-password", "", "db pw")
)
func initTestFlags() {
flag.Set("database-host", "localhost")
flag.Set("database-password", "some-password")
flag.Parse()
}
func TestFlagsrc_Read(t *testing.T) {
initTestFlags()
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
if actualDB["host"] != *dbhost {
t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
}
if actualDB["password"] != *dbpw {
t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
}
// unset flags should not be loaded
if actualDB["user"] != nil {
t.Errorf("expected %v got %v", nil, actualDB["user"])
}
}
func TestFlagsrc_ReadAll(t *testing.T) {
initTestFlags()
source := NewSource(IncludeUnset(true))
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
// unset flag defaults should be loaded
if actualDB["user"] != *dbuser {
t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
}
}
+20
View File
@@ -0,0 +1,20 @@
package flag
import (
"context"
"go-micro.dev/v6/config/source"
)
type includeUnsetKey struct{}
// IncludeUnset toggles the loading of unset flags and their respective default values.
// Default behavior is to ignore any unset flags.
func IncludeUnset(b bool) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
}
}
+44
View File
@@ -0,0 +1,44 @@
# Memory Source
The memory source provides in-memory data as a source
## Memory Format
The expected data format is json
```json
data := []byte(`{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}`)
```
## New Source
Specify source with data
```go
memorySource := memory.NewSource(
memory.WithJSON(data),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load memory source
conf.Load(memorySource)
```
+99
View File
@@ -0,0 +1,99 @@
// Package memory is a memory source
package memory
import (
"sync"
"time"
"github.com/google/uuid"
"go-micro.dev/v6/config/source"
)
type memory struct {
ChangeSet *source.ChangeSet
Watchers map[string]*watcher
sync.RWMutex
}
func (s *memory) Read() (*source.ChangeSet, error) {
s.RLock()
cs := &source.ChangeSet{
Format: s.ChangeSet.Format,
Timestamp: s.ChangeSet.Timestamp,
Data: s.ChangeSet.Data,
Checksum: s.ChangeSet.Checksum,
Source: s.ChangeSet.Source,
}
s.RUnlock()
return cs, nil
}
func (s *memory) Watch() (source.Watcher, error) {
w := &watcher{
Id: uuid.New().String(),
Updates: make(chan *source.ChangeSet, 100),
Source: s,
}
s.Lock()
s.Watchers[w.Id] = w
s.Unlock()
return w, nil
}
func (s *memory) Write(cs *source.ChangeSet) error {
s.Update(cs)
return nil
}
// Update allows manual updates of the config data.
func (s *memory) Update(c *source.ChangeSet) {
// don't process nil
if c == nil {
return
}
// hash the file
s.Lock()
// update changeset
s.ChangeSet = &source.ChangeSet{
Data: c.Data,
Format: c.Format,
Source: "memory",
Timestamp: time.Now(),
}
s.ChangeSet.Checksum = s.ChangeSet.Sum()
// update watchers
for _, w := range s.Watchers {
select {
case w.Updates <- s.ChangeSet:
default:
}
}
s.Unlock()
}
func (s *memory) String() string {
return "memory"
}
func NewSource(opts ...source.Option) source.Source {
var options source.Options
for _, o := range opts {
o(&options)
}
s := &memory{
Watchers: make(map[string]*watcher),
}
if options.Context != nil {
c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
if ok {
s.Update(c)
}
}
return s
}
+41
View File
@@ -0,0 +1,41 @@
package memory
import (
"context"
"go-micro.dev/v6/config/source"
)
type changeSetKey struct{}
func withData(d []byte, f string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
Data: d,
Format: f,
})
}
}
// WithChangeSet allows a changeset to be set.
func WithChangeSet(cs *source.ChangeSet) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
}
}
// WithJSON allows the source data to be set to json.
func WithJSON(d []byte) source.Option {
return withData(d, "json")
}
// WithYAML allows the source data to be set to yaml.
func WithYAML(d []byte) source.Option {
return withData(d, "yaml")
}
+23
View File
@@ -0,0 +1,23 @@
package memory
import (
"go-micro.dev/v6/config/source"
)
type watcher struct {
Updates chan *source.ChangeSet
Source *memory
Id string
}
func (w *watcher) Next() (*source.ChangeSet, error) {
cs := <-w.Updates
return cs, nil
}
func (w *watcher) Stop() error {
w.Source.Lock()
delete(w.Source.Watchers, w.Id)
w.Source.Unlock()
return nil
}
+56
View File
@@ -0,0 +1,56 @@
# Nats Source
The nats source reads config from nats key/values
## Nats Format
The nats source expects keys under the default bucket `default` default key `micro_config`
Values are expected to be json
```
nats kv put default micro_config '{"nats": {"address": "10.0.0.1", "port": 8488}}'
```
```
conf.Get("nats")
```
## New Source
Specify source with data
```go
natsSource := nats.NewSource(
nats.WithUrl("127.0.0.1:4222"),
nats.WithBucket("my_bucket"),
nats.WithKey("my_key"),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load nats source
conf.Load(natsSource)
```
## Watch
```go
wh, _ := natsSource.Watch()
for {
v, err := watcher.Next()
if err != nil {
log.Fatalf("err %v", err)
}
log.Infof("data %v", string(v.Data))
}
```
+146
View File
@@ -0,0 +1,146 @@
package nats
import (
"fmt"
"net"
"strings"
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/source"
log "go-micro.dev/v6/logger"
)
type nats struct {
url string
bucket string
key string
conn *natsgo.Conn // store connection for lifecycle management
kv natsgo.KeyValue
opts source.Options
}
// DefaultBucket is the bucket that nats keys will be assumed to have if you
// haven't specified one.
var (
DefaultBucket = "default"
DefaultKey = "micro_config"
)
func (n *nats) Read() (*source.ChangeSet, error) {
e, err := n.kv.Get(n.key)
if err != nil {
if err == natsgo.ErrKeyNotFound {
return nil, nil
}
return nil, err
}
if e.Value() == nil || len(e.Value()) == 0 {
return nil, fmt.Errorf("source not found: %s", n.key)
}
cs := &source.ChangeSet{
Data: e.Value(),
Format: n.opts.Encoder.String(),
Source: n.String(),
Timestamp: time.Now(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (n *nats) Write(cs *source.ChangeSet) error {
_, err := n.kv.Put(n.key, cs.Data)
if err != nil {
return err
}
return nil
}
func (n *nats) String() string {
return "nats"
}
func (n *nats) Watch() (source.Watcher, error) {
return newWatcher(n.kv, n.bucket, n.key, n.String(), n.opts.Encoder)
}
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
config := natsgo.GetDefaultOptions()
urls, ok := options.Context.Value(urlKey{}).([]string)
endpoints := []string{}
if ok {
for _, u := range urls {
addr, port, err := net.SplitHostPort(u)
if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
port = "4222"
addr = u
endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
} else if err == nil {
endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
}
}
}
if len(endpoints) == 0 {
endpoints = append(endpoints, "127.0.0.1:4222")
}
bucket, ok := options.Context.Value(bucketKey{}).(string)
if !ok {
bucket = DefaultBucket
}
key, ok := options.Context.Value(keyKey{}).(string)
if !ok {
key = DefaultKey
}
config.Url = strings.Join(endpoints, ",")
nc, err := natsgo.Connect(config.Url)
if err != nil {
log.Error(err)
}
js, err := nc.JetStream(natsgo.MaxWait(10 * time.Second))
if err != nil {
log.Error(err)
}
kv, err := js.KeyValue(bucket)
if err == natsgo.ErrBucketNotFound || err == natsgo.ErrKeyNotFound {
kv, err = js.CreateKeyValue(&natsgo.KeyValueConfig{Bucket: bucket})
if err != nil {
log.Error(err)
}
}
if err != nil {
log.Error(err)
}
return &nats{
url: config.Url,
bucket: bucket,
key: key,
conn: nc, // store connection reference
kv: kv,
opts: options,
}
}
// Close implements io.Closer and closes the underlying NATS connection.
// This method is optional but recommended to prevent connection leaks.
func (n *nats) Close() error {
if n.conn != nil {
n.conn.Close()
n.conn = nil
}
return nil
}
+54
View File
@@ -0,0 +1,54 @@
package nats
import (
"context"
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/source"
)
type (
urlKey struct{}
bucketKey struct{}
keyKey struct{}
)
// WithUrl sets the nats url.
func WithUrl(a ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, urlKey{}, a)
}
}
// WithBucket sets the nats key.
func WithBucket(a string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, bucketKey{}, a)
}
}
// WithKey sets the nats key.
func WithKey(a string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, keyKey{}, a)
}
}
func Client(url string) (natsgo.JetStreamContext, error) {
nc, err := natsgo.Connect(url)
if err != nil {
return nil, err
}
return nc.JetStream(natsgo.MaxWait(10 * time.Second))
}
+79
View File
@@ -0,0 +1,79 @@
package nats
import (
"time"
natsgo "github.com/nats-io/nats.go"
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/source"
)
type watcher struct {
e encoder.Encoder
name string
bucket string
key string
ch chan *source.ChangeSet
exit chan bool
}
func newWatcher(kv natsgo.KeyValue, bucket, key, name string, e encoder.Encoder) (source.Watcher, error) {
w := &watcher{
e: e,
name: name,
bucket: bucket,
key: key,
ch: make(chan *source.ChangeSet),
exit: make(chan bool),
}
wh, _ := kv.Watch(key)
go func() {
for {
select {
case v := <-wh.Updates():
if v != nil {
w.handle(v.Value())
}
case <-w.exit:
_ = wh.Stop()
return
}
}
}()
return w, nil
}
func (w *watcher) handle(data []byte) {
cs := &source.ChangeSet{
Timestamp: time.Now(),
Format: w.e.String(),
Source: w.name,
Data: data,
}
cs.Checksum = cs.Sum()
w.ch <- cs
}
func (w *watcher) Next() (*source.ChangeSet, error) {
select {
case cs := <-w.ch:
return cs, nil
case <-w.exit:
return nil, source.ErrWatcherStopped
}
}
func (w *watcher) Stop() error {
select {
case <-w.exit:
return nil
default:
close(w.exit)
}
return nil
}
+25
View File
@@ -0,0 +1,25 @@
package source
import (
"errors"
)
type noopWatcher struct {
exit chan struct{}
}
func (w *noopWatcher) Next() (*ChangeSet, error) {
<-w.exit
return nil, errors.New("noopWatcher stopped")
}
func (w *noopWatcher) Stop() error {
close(w.exit)
return nil
}
// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
func NewNoopWatcher() (Watcher, error) {
return &noopWatcher{exit: make(chan struct{})}, nil
}
+50
View File
@@ -0,0 +1,50 @@
package source
import (
"context"
"go-micro.dev/v6/client"
"go-micro.dev/v6/config/encoder"
"go-micro.dev/v6/config/encoder/json"
)
type Options struct {
// Encoder
Encoder encoder.Encoder
// for alternative data
Context context.Context
// Client to use for RPC
Client client.Client
}
type Option func(o *Options)
func NewOptions(opts ...Option) Options {
options := Options{
Encoder: json.NewEncoder(),
Context: context.Background(),
Client: client.DefaultClient,
}
for _, o := range opts {
o(&options)
}
return options
}
// WithEncoder sets the source encoder.
func WithEncoder(e encoder.Encoder) Option {
return func(o *Options) {
o.Encoder = e
}
}
// WithClient sets the source client.
func WithClient(c client.Client) Option {
return func(o *Options) {
o.Client = c
}
}
+35
View File
@@ -0,0 +1,35 @@
// Package source is the interface for sources
package source
import (
"errors"
"time"
)
var (
// ErrWatcherStopped is returned when source watcher has been stopped.
ErrWatcherStopped = errors.New("watcher stopped")
)
// Source is the source from which config is loaded.
type Source interface {
Read() (*ChangeSet, error)
Write(*ChangeSet) error
Watch() (Watcher, error)
String() string
}
// ChangeSet represents a set of changes from a source.
type ChangeSet struct {
Timestamp time.Time
Checksum string
Format string
Source string
Data []byte
}
// Watcher watches a source for changes.
type Watcher interface {
Next() (*ChangeSet, error)
Stop() error
}