github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #3574 from mxpv/cfg

Browse Source 
Support config imports
This commit is contained in:
Michael Crosby 2019-09-04 16:34:11 -04:00 committed by GitHub
commit f76eefd272
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1230 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
cmd/containerd/command/config.go
View File

@ -40,6 +40,50 @@ func (c *Config) WriteTo(w io.Writer) (int64, error) {
return 0, toml.NewEncoder(w).Encode(c) return 0, toml.NewEncoder(w).Encode(c)
} }
func outputConfig(cfg *srvconfig.Config) error {
config := &Config{
Config: cfg,
}
plugins, err := server.LoadPlugins(gocontext.Background(), config.Config)
if err != nil {
return err
}
if len(plugins) != 0 {
config.Plugins = make(map[string]interface{})
for _, p := range plugins {
if p.Config == nil {
continue
}
pc, err := config.Decode(p)
if err != nil {
return err
}
config.Plugins[p.URI()] = pc
}
}
timeouts := timeout.All()
config.Timeouts = make(map[string]string)
for k, v := range timeouts {
config.Timeouts[k] = v.String()
}
// for the time being, keep the defaultConfig's version set at 1 so that
// when a config without a version is loaded from disk and has no version
// set, we assume it's a v1 config. But when generating new configs via
// this command, generate the v2 config
config.Config.Version = 2
// remove overridden Plugins type to avoid duplication in output
config.Config.Plugins = nil
_, err = config.WriteTo(os.Stdout)
return err
}
var configCommand = cli.Command{ var configCommand = cli.Command{
Name: "config", Name: "config",
Usage: "information on the containerd config", Usage: "information on the containerd config",
@ -48,35 +92,19 @@ var configCommand = cli.Command{
Name: "default", Name: "default",
Usage: "see the output of the default config", Usage: "see the output of the default config",
Action: func(context *cli.Context) error { Action: func(context *cli.Context) error {
config := &Config{ return outputConfig(defaultConfig())
Config: defaultConfig(), },
} },
// for the time being, keep the defaultConfig's version set at 1 so that {
// when a config without a version is loaded from disk and has no version Name: "dump",
// set, we assume it's a v1 config. But when generating new configs via Usage: "see the output of the final main config with imported in subconfig files",
// this command, generate the v2 config Action: func(context *cli.Context) error {
config.Config.Version = 2 config := defaultConfig()
plugins, err := server.LoadPlugins(gocontext.Background(), config.Config) if err := srvconfig.LoadConfig(context.GlobalString("config"), config); err != nil && !os.IsNotExist(err) {
if err != nil {
return err return err
} }
if len(plugins) != 0 {
config.Plugins = make(map[string]interface{})
for _, p := range plugins {
if p.Config == nil {
continue
}
config.Plugins[p.URI()] = p.Config
}
}
timeouts := timeout.All()
config.Timeouts = make(map[string]string)
for k, v := range timeouts {
config.Timeouts[k] = v.String()
}
_, err = config.WriteTo(os.Stdout) return outputConfig(config)
return err
}, },
}, },
}, },

9
docs/man/containerd-config.toml.5.md
View File

@ -32,6 +32,14 @@ settings.
**oom_score** **oom_score**
: The out of memory (OOM) score applied to the containerd daemon process (Default: 0) : The out of memory (OOM) score applied to the containerd daemon process (Default: 0)
**imports**
: Imports is a list of additional configuration files to include.
This allows to split the main configuration file and keep some sections
separately (for example vendors may keep a custom runtime configuration in a
separate file without modifying the main `config.toml`).
Imported files will overwrite simple fields like `int` or
`string` (if not empty) and will append `array` and `map` fields.
**[grpc]** **[grpc]**
: Section for gRPC socket listener settings. Contains three properties: : Section for gRPC socket listener settings. Contains three properties:
- **address** (Default: "/run/containerd/containerd.sock") - **address** (Default: "/run/containerd/containerd.sock")
@ -82,6 +90,7 @@ The following is a complete **config.toml** default configuration example:
root = "/var/lib/containerd" root = "/var/lib/containerd"
state = "/run/containerd" state = "/run/containerd"
oom_score = 0 oom_score = 0
imports = ["/etc/containerd/runtime_*.toml", "./debug.toml"]
[grpc] [grpc]
address = "/run/containerd/containerd.sock" address = "/run/containerd/containerd.sock"

8
docs/stream_processors.md
View File

@ -21,20 +21,20 @@ pipe's path set as the value of the environment variable `STREAM_PROCESSOR_PIPE`
## Configuration ## Configuration
To configure stream processors for containerd, entries in the config file need to be made. To configure stream processors for containerd, entries in the config file need to be made.
The `stream_processors` field is an array so that users can chain together multiple processors The `stream_processors` field is a map so that users can chain together multiple processors
to mutate content streams. to mutate content streams.
Processor Fields: Processor Fields:
* `id` - ID of the processor, used for passing a specific payload to the processor. * Key - ID of the processor, used for passing a specific payload to the processor.
* `accepts` - Accepted media-types for the processor that it can handle. * `accepts` - Accepted media-types for the processor that it can handle.
* `returns` - The media-type that the processor returns. * `returns` - The media-type that the processor returns.
* `path` - Path to the processor binary. * `path` - Path to the processor binary.
* `args` - Arguments passed to the processor binary. * `args` - Arguments passed to the processor binary.
```toml ```toml
[[stream_processors]] [stream_processors]
id = "io.containerd.processor.v1.pigz" [stream_processors."io.containerd.processor.v1.pigz"]
accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"] accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"]
returns = "application/vnd.oci.image.layer.v1.tar" returns = "application/vnd.oci.image.layer.v1.tar"
path = "unpigz" path = "unpigz"

129
services/server/config/config.go
View File

@ -17,12 +17,15 @@
package config package config
import ( import (
"path/filepath"
"strings" "strings"
"github.com/BurntSushi/toml" "github.com/BurntSushi/toml"
"github.com/imdario/mergo"
"github.com/pkg/errors"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
"github.com/pkg/errors"
) )
// Config provides containerd configuration data for the server // Config provides containerd configuration data for the server
@ -59,16 +62,14 @@ type Config struct {
ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"` ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"`
// Timeouts specified as a duration // Timeouts specified as a duration
Timeouts map[string]string `toml:"timeouts"` Timeouts map[string]string `toml:"timeouts"`
// Imports are additional file path list to config files that can overwrite main config file fields
Imports []string `toml:"imports"`
StreamProcessors []StreamProcessor `toml:"stream_processors"` StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
md toml.MetaData
} }
// StreamProcessor provides configuration for diff content processors // StreamProcessor provides configuration for diff content processors
type StreamProcessor struct { type StreamProcessor struct {
// ID of the processor, also used to fetch the specific payload
ID string `toml:"id"`
// Accepts specific media-types // Accepts specific media-types
Accepts []string `toml:"accepts"` Accepts []string `toml:"accepts"`
// Returns the media-type // Returns the media-type
@ -202,23 +203,125 @@ func (c *Config) Decode(p *plugin.Registration) (interface{}, error) {
if !ok { if !ok {
return p.Config, nil return p.Config, nil
} }
if err := c.md.PrimitiveDecode(data, p.Config); err != nil { if err := toml.PrimitiveDecode(data, p.Config); err != nil {
return nil, err return nil, err
} }
return p.Config, nil return p.Config, nil
} }
// LoadConfig loads the containerd server config from the provided path // LoadConfig loads the containerd server config from the provided path
func LoadConfig(path string, v *Config) error { func LoadConfig(path string, out *Config) error {
if v == nil { if out == nil {
return errors.Wrapf(errdefs.ErrInvalidArgument, "argument v must not be nil") return errors.Wrapf(errdefs.ErrInvalidArgument, "argument out must not be nil")
} }
md, err := toml.DecodeFile(path, v)
var (
loaded = map[string]bool{}
pending = []string{path}
)
for len(pending) > 0 {
path, pending = pending[0], pending[1:]
// Check if a file at the given path already loaded to prevent circular imports
if _, ok := loaded[path]; ok {
continue
}
config, err := loadConfigFile(path)
if err != nil {
return err
}
if err := mergeConfig(out, config); err != nil {
return err
}
imports, err := resolveImports(path, config.Imports)
if err != nil {
return err
}
loaded[path] = true
pending = append(pending, imports...)
}
// Fix up the list of config files loaded
out.Imports = []string{}
for path := range loaded {
out.Imports = append(out.Imports, path)
}
return out.ValidateV2()
}
// loadConfigFile decodes a TOML file at the given path
func loadConfigFile(path string) (*Config, error) {
config := &Config{}
_, err := toml.DecodeFile(path, &config)
if err != nil {
return nil, err
}
return config, nil
}
// resolveImports resolves import strings list to absolute paths list:
// - If path contains *, glob pattern matching applied
// - Non abs path is relative to parent config file directory
// - Abs paths returned as is
func resolveImports(parent string, imports []string) ([]string, error) {
var out []string
for _, path := range imports {
if strings.Contains(path, "*") {
matches, err := filepath.Glob(path)
if err != nil {
return nil, err
}
out = append(out, matches...)
} else {
path = filepath.Clean(path)
if !filepath.IsAbs(path) {
path = filepath.Join(filepath.Dir(parent), path)
}
out = append(out, path)
}
}
return out, nil
}
// mergeConfig merges Config structs with the following rules:
// 'to' 'from' 'result'
// "" "value" "value"
// "value" "" "value"
// 1 0 1
// 0 1 1
// []{"1"} []{"2"} []{"1","2"}
// []{"1"} []{} []{"1"}
// Maps merged by keys, but values are replaced entirely.
func mergeConfig(to, from *Config) error {
err := mergo.Merge(to, from, mergo.WithOverride, mergo.WithAppendSlice)
if err != nil { if err != nil {
return err return err
} }
v.md = md
return v.ValidateV2() // Replace entire sections instead of merging map's values.
for k, v := range from.Plugins {
to.Plugins[k] = v
}
for k, v := range from.StreamProcessors {
to.StreamProcessors[k] = v
}
for k, v := range from.ProxyPlugins {
to.ProxyPlugins[k] = v
}
return nil
} }
// V1DisabledFilter matches based on ID // V1DisabledFilter matches based on ID

207
services/server/config/config_test.go Normal file
View File

@ -0,0 +1,207 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"gotest.tools/assert"
"github.com/containerd/containerd/plugin"
)
func TestMergeConfigs(t *testing.T) {
a := &Config{
Version: 2,
Root: "old_root",
RequiredPlugins: []string{"old_plugin"},
DisabledPlugins: []string{"old_plugin"},
State: "old_state",
OOMScore: 1,
Timeouts: map[string]string{"a": "1"},
StreamProcessors: map[string]StreamProcessor{"1": {Path: "2", Returns: "4"}, "2": {Path: "5"}},
}
b := &Config{
Root: "new_root",
RequiredPlugins: []string{"new_plugin1", "new_plugin2"},
OOMScore: 2,
Timeouts: map[string]string{"b": "2"},
StreamProcessors: map[string]StreamProcessor{"1": {Path: "3"}},
}
err := mergeConfig(a, b)
assert.NilError(t, err)
assert.Equal(t, a.Version, 2)
assert.Equal(t, a.Root, "new_root")
assert.Equal(t, a.State, "old_state")
assert.Equal(t, a.OOMScore, 2)
assert.DeepEqual(t, a.RequiredPlugins, []string{"old_plugin", "new_plugin1", "new_plugin2"})
assert.DeepEqual(t, a.DisabledPlugins, []string{"old_plugin"})
assert.DeepEqual(t, a.Timeouts, map[string]string{"a": "1", "b": "2"})
assert.DeepEqual(t, a.StreamProcessors, map[string]StreamProcessor{"1": {Path: "3"}, "2": {Path: "5"}})
}
func TestResolveImports(t *testing.T) {
tempDir, err := ioutil.TempDir("", "containerd_")
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
for _, filename := range []string{"config_1.toml", "config_2.toml", "test.toml"} {
err = ioutil.WriteFile(filepath.Join(tempDir, filename), []byte(""), 0600)
assert.NilError(t, err)
}
imports, err := resolveImports(filepath.Join(tempDir, "root.toml"), []string{
filepath.Join(tempDir, "config_*.toml"), // Glob
filepath.Join(tempDir, "./test.toml"), // Path clean up
"current.toml", // Resolve current working dir
})
assert.NilError(t, err)
assert.DeepEqual(t, imports, []string{
filepath.Join(tempDir, "config_1.toml"),
filepath.Join(tempDir, "config_2.toml"),
filepath.Join(tempDir, "test.toml"),
filepath.Join(tempDir, "current.toml"),
})
}
func TestLoadSingleConfig(t *testing.T) {
data := `
version = 2
root = "/var/lib/containerd"
[stream_processors]
[stream_processors."io.containerd.processor.v1.pigz"]
accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"]
path = "unpigz"
`
tempDir, err := ioutil.TempDir("", "containerd_")
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
path := filepath.Join(tempDir, "config.toml")
err = ioutil.WriteFile(path, []byte(data), 0600)
assert.NilError(t, err)
var out Config
err = LoadConfig(path, &out)
assert.NilError(t, err)
assert.Equal(t, 2, out.Version)
assert.Equal(t, "/var/lib/containerd", out.Root)
assert.DeepEqual(t, map[string]StreamProcessor{
"io.containerd.processor.v1.pigz": {
Accepts: []string{"application/vnd.docker.image.rootfs.diff.tar.gzip"},
Path: "unpigz",
},
}, out.StreamProcessors)
}
func TestLoadConfigWithImports(t *testing.T) {
data1 := `
version = 2
root = "/var/lib/containerd"
imports = ["data2.toml"]
`
data2 := `
disabled_plugins = ["io.containerd.v1.xyz"]
`
tempDir, err := ioutil.TempDir("", "containerd_")
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600)
assert.NilError(t, err)
err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600)
assert.NilError(t, err)
var out Config
err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out)
assert.NilError(t, err)
assert.Equal(t, 2, out.Version)
assert.Equal(t, "/var/lib/containerd", out.Root)
assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins)
}
func TestLoadConfigWithCircularImports(t *testing.T) {
data1 := `
version = 2
root = "/var/lib/containerd"
imports = ["data2.toml", "data1.toml"]
`
data2 := `
disabled_plugins = ["io.containerd.v1.xyz"]
imports = ["data1.toml", "data2.toml"]
`
tempDir, err := ioutil.TempDir("", "containerd_")
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600)
assert.NilError(t, err)
err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600)
assert.NilError(t, err)
var out Config
err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out)
assert.NilError(t, err)
assert.Equal(t, 2, out.Version)
assert.Equal(t, "/var/lib/containerd", out.Root)
assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins)
assert.DeepEqual(t, []string{
filepath.Join(tempDir, "data1.toml"),
filepath.Join(tempDir, "data2.toml"),
}, out.Imports)
}
func TestDecodePlugin(t *testing.T) {
data := `
version = 1
[plugins.linux]
shim_debug = true
`
tempDir, err := ioutil.TempDir("", "containerd_")
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
path := filepath.Join(tempDir, "config.toml")
err = ioutil.WriteFile(path, []byte(data), 0600)
assert.NilError(t, err)
var out Config
err = LoadConfig(path, &out)
assert.NilError(t, err)
pluginConfig := map[string]interface{}{}
_, err = out.Decode(&plugin.Registration{ID: "linux", Config: &pluginConfig})
assert.NilError(t, err)
assert.Equal(t, true, pluginConfig["shim_debug"])
}

4
services/server/server.go
View File

@ -89,8 +89,8 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, p := range config.StreamProcessors { for id, p := range config.StreamProcessors {
diff.RegisterProcessor(diff.BinaryHandler(p.ID, p.Returns, p.Accepts, p.Path, p.Args)) diff.RegisterProcessor(diff.BinaryHandler(id, p.Returns, p.Accepts, p.Path, p.Args))
} }
serverOpts := []grpc.ServerOption{ serverOpts := []grpc.ServerOption{

1
vendor.conf
View File

@ -46,6 +46,7 @@ github.com/hashicorp/errwrap v1.0.0
github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-multierror v1.0.0
github.com/hashicorp/golang-lru v0.5.3 github.com/hashicorp/golang-lru v0.5.3
go.opencensus.io v0.22.0 go.opencensus.io v0.22.0
github.com/imdario/mergo v0.3.7
# cri dependencies # cri dependencies
github.com/containerd/cri f1d492b0cdd14e76476ee4dd024696ce3634e501 # master github.com/containerd/cri f1d492b0cdd14e76476ee4dd024696ce3634e501 # master

28
vendor/github.com/imdario/mergo/LICENSE generated vendored Normal file
View File

@ -0,0 +1,28 @@
Copyright (c) 2013 Dario Castañé. All rights reserved.
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

238
vendor/github.com/imdario/mergo/README.md generated vendored Normal file
View File

@ -0,0 +1,238 @@
# Mergo
A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements.
Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region of Marche.
## Status
It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild).
[![GoDoc][3]][4]
[![GoCard][5]][6]
[![Build Status][1]][2]
[![Coverage Status][7]][8]
[![Sourcegraph][9]][10]
[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield)
[1]: https://travis-ci.org/imdario/mergo.png
[2]: https://travis-ci.org/imdario/mergo
[3]: https://godoc.org/github.com/imdario/mergo?status.svg
[4]: https://godoc.org/github.com/imdario/mergo
[5]: https://goreportcard.com/badge/imdario/mergo
[6]: https://goreportcard.com/report/github.com/imdario/mergo
[7]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master
[8]: https://coveralls.io/github/imdario/mergo?branch=master
[9]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg
[10]: https://sourcegraph.com/github.com/imdario/mergo?badge
### Latest release
[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7).
### Important note
Please keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2) Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). An optional/variadic argument has been added, so it won't break existing code.
If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0).
### Donations
If Mergo is useful to you, consider buying me a coffee, a beer or making a monthly donation so I can keep building great free software. :heart_eyes:
<a href='https://ko-fi.com/B0B58839' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi1.png?v=0' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo)
[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo)
<a href="https://liberapay.com/dario/donate"><img alt="Donate using Liberapay" src="https://liberapay.com/assets/widgets/donate.svg"></a>
### Mergo in the wild
- [moby/moby](https://github.com/moby/moby)
- [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes)
- [vmware/dispatch](https://github.com/vmware/dispatch)
- [Shopify/themekit](https://github.com/Shopify/themekit)
- [imdario/zas](https://github.com/imdario/zas)
- [matcornic/hermes](https://github.com/matcornic/hermes)
- [OpenBazaar/openbazaar-go](https://github.com/OpenBazaar/openbazaar-go)
- [kataras/iris](https://github.com/kataras/iris)
- [michaelsauter/crane](https://github.com/michaelsauter/crane)
- [go-task/task](https://github.com/go-task/task)
- [sensu/uchiwa](https://github.com/sensu/uchiwa)
- [ory/hydra](https://github.com/ory/hydra)
- [sisatech/vcli](https://github.com/sisatech/vcli)
- [dairycart/dairycart](https://github.com/dairycart/dairycart)
- [projectcalico/felix](https://github.com/projectcalico/felix)
- [resin-os/balena](https://github.com/resin-os/balena)
- [go-kivik/kivik](https://github.com/go-kivik/kivik)
- [Telefonica/govice](https://github.com/Telefonica/govice)
- [supergiant/supergiant](supergiant/supergiant)
- [SergeyTsalkov/brooce](https://github.com/SergeyTsalkov/brooce)
- [soniah/dnsmadeeasy](https://github.com/soniah/dnsmadeeasy)
- [ohsu-comp-bio/funnel](https://github.com/ohsu-comp-bio/funnel)
- [EagerIO/Stout](https://github.com/EagerIO/Stout)
- [lynndylanhurley/defsynth-api](https://github.com/lynndylanhurley/defsynth-api)
- [russross/canvasassignments](https://github.com/russross/canvasassignments)
- [rdegges/cryptly-api](https://github.com/rdegges/cryptly-api)
- [casualjim/exeggutor](https://github.com/casualjim/exeggutor)
- [divshot/gitling](https://github.com/divshot/gitling)
- [RWJMurphy/gorl](https://github.com/RWJMurphy/gorl)
- [andrerocker/deploy42](https://github.com/andrerocker/deploy42)
- [elwinar/rambler](https://github.com/elwinar/rambler)
- [tmaiaroto/gopartman](https://github.com/tmaiaroto/gopartman)
- [jfbus/impressionist](https://github.com/jfbus/impressionist)
- [Jmeyering/zealot](https://github.com/Jmeyering/zealot)
- [godep-migrator/rigger-host](https://github.com/godep-migrator/rigger-host)
- [Dronevery/MultiwaySwitch-Go](https://github.com/Dronevery/MultiwaySwitch-Go)
- [thoas/picfit](https://github.com/thoas/picfit)
- [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server)
- [jnuthong/item_search](https://github.com/jnuthong/item_search)
- [bukalapak/snowboard](https://github.com/bukalapak/snowboard)
## Installation
go get github.com/imdario/mergo
// use in your .go code
import (
"github.com/imdario/mergo"
)
## Usage
You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as [they are not considered zero values](https://golang.org/ref/spec#The_zero_value) either. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection).
```go
if err := mergo.Merge(&dst, src); err != nil {
// ...
}
```
Also, you can merge overwriting values using the transformer `WithOverride`.
```go
if err := mergo.Merge(&dst, src, mergo.WithOverride); err != nil {
// ...
}
```
Additionally, you can map a `map[string]interface{}` to a struct (and otherwise, from struct to map), following the same restrictions as in `Merge()`. Keys are capitalized to find each corresponding exported field.
```go
if err := mergo.Map(&dst, srcMap); err != nil {
// ...
}
```
Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as `map[string]interface{}`. They will be just assigned as values.
More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo).
### Nice example
```go
package main
import (
"fmt"
"github.com/imdario/mergo"
)
type Foo struct {
A string
B int64
}
func main() {
src := Foo{
A: "one",
B: 2,
}
dest := Foo{
A: "two",
}
mergo.Merge(&dest, src)
fmt.Println(dest)
// Will print
// {two 2}
}
```
Note: if test are failing due missing package, please execute:
go get gopkg.in/yaml.v2
### Transformers
Transformers allow to merge specific types differently than in the default behavior. In other words, now you can customize how some types are merged. For example, `time.Time` is a struct; it doesn't have zero value but IsZero can return true because it has fields with zero value. How can we merge a non-zero `time.Time`?
```go
package main
import (
"fmt"
"github.com/imdario/mergo"
"reflect"
"time"
)
type timeTransfomer struct {
}
func (t timeTransfomer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
if typ == reflect.TypeOf(time.Time{}) {
return func(dst, src reflect.Value) error {
if dst.CanSet() {
isZero := dst.MethodByName("IsZero")
result := isZero.Call([]reflect.Value{})
if result[0].Bool() {
dst.Set(src)
}
}
return nil
}
}
return nil
}
type Snapshot struct {
Time time.Time
// ...
}
func main() {
src := Snapshot{time.Now()}
dest := Snapshot{}
mergo.Merge(&dest, src, mergo.WithTransformers(timeTransfomer{}))
fmt.Println(dest)
// Will print
// { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 }
}
```
## Contact me
If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario)
## About
Written by [Dario Castañé](http://dario.im).
## Top Contributors
[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0)
[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1)
[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2)
[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3)
[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4)
[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5)
[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6)
[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7)
## License
[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE).
[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large)

44
vendor/github.com/imdario/mergo/doc.go generated vendored Normal file
View File

@ -0,0 +1,44 @@
// Copyright 2013 Dario Castañé. All rights reserved.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package mergo merges same-type structs and maps by setting default values in zero-value fields.
Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection).
Usage
From my own work-in-progress project:
type networkConfig struct {
Protocol string
Address string
ServerType string `json: "server_type"`
Port uint16
}
type FssnConfig struct {
Network networkConfig
}
var fssnDefault = FssnConfig {
networkConfig {
"tcp",
"127.0.0.1",
"http",
31560,
},
}
// Inside a function [...]
if err := mergo.Merge(&config, fssnDefault); err != nil {
log.Fatal(err)
}
// More code [...]
*/
package mergo

175
vendor/github.com/imdario/mergo/map.go generated vendored Normal file
View File

@ -0,0 +1,175 @@
// Copyright 2014 Dario Castañé. All rights reserved.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Based on src/pkg/reflect/deepequal.go from official
// golang's stdlib.
package mergo
import (
"fmt"
"reflect"
"unicode"
"unicode/utf8"
)
func changeInitialCase(s string, mapper func(rune) rune) string {
if s == "" {
return s
}
r, n := utf8.DecodeRuneInString(s)
return string(mapper(r)) + s[n:]
}
func isExported(field reflect.StructField) bool {
r, _ := utf8.DecodeRuneInString(field.Name)
return r >= 'A' && r <= 'Z'
}
// Traverses recursively both values, assigning src's fields values to dst.
// The map argument tracks comparisons that have already been seen, which allows
// short circuiting on recursive types.
func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
overwrite := config.Overwrite
if dst.CanAddr() {
addr := dst.UnsafeAddr()
h := 17 * addr
seen := visited[h]
typ := dst.Type()
for p := seen; p != nil; p = p.next {
if p.ptr == addr && p.typ == typ {
return nil
}
}
// Remember, remember...
visited[h] = &visit{addr, typ, seen}
}
zeroValue := reflect.Value{}
switch dst.Kind() {
case reflect.Map:
dstMap := dst.Interface().(map[string]interface{})
for i, n := 0, src.NumField(); i < n; i++ {
srcType := src.Type()
field := srcType.Field(i)
if !isExported(field) {
continue
}
fieldName := field.Name
fieldName = changeInitialCase(fieldName, unicode.ToLower)
if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) {
dstMap[fieldName] = src.Field(i).Interface()
}
}
case reflect.Ptr:
if dst.IsNil() {
v := reflect.New(dst.Type().Elem())
dst.Set(v)
}
dst = dst.Elem()
fallthrough
case reflect.Struct:
srcMap := src.Interface().(map[string]interface{})
for key := range srcMap {
config.overwriteWithEmptyValue = true
srcValue := srcMap[key]
fieldName := changeInitialCase(key, unicode.ToUpper)
dstElement := dst.FieldByName(fieldName)
if dstElement == zeroValue {
// We discard it because the field doesn't exist.
continue
}
srcElement := reflect.ValueOf(srcValue)
dstKind := dstElement.Kind()
srcKind := srcElement.Kind()
if srcKind == reflect.Ptr && dstKind != reflect.Ptr {
srcElement = srcElement.Elem()
srcKind = reflect.TypeOf(srcElement.Interface()).Kind()
} else if dstKind == reflect.Ptr {
// Can this work? I guess it can't.
if srcKind != reflect.Ptr && srcElement.CanAddr() {
srcPtr := srcElement.Addr()
srcElement = reflect.ValueOf(srcPtr)
srcKind = reflect.Ptr
}
}
if !srcElement.IsValid() {
continue
}
if srcKind == dstKind {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if srcKind == reflect.Map {
if err = deepMap(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else {
return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind)
}
}
}
return
}
// Map sets fields' values in dst from src.
// src can be a map with string keys or a struct. dst must be the opposite:
// if src is a map, dst must be a valid pointer to struct. If src is a struct,
// dst must be map[string]interface{}.
// It won't merge unexported (private) fields and will do recursively
// any exported field.
// If dst is a map, keys will be src fields' names in lower camel case.
// Missing key in src that doesn't match a field in dst will be skipped. This
// doesn't apply if dst is a map.
// This is separated method from Merge because it is cleaner and it keeps sane
// semantics: merging equal types, mapping different (restricted) types.
func Map(dst, src interface{}, opts ...func(*Config)) error {
return _map(dst, src, opts...)
}
// MapWithOverwrite will do the same as Map except that non-empty dst attributes will be overridden by
// non-empty src attribute values.
// Deprecated: Use Map(…) with WithOverride
func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
return _map(dst, src, append(opts, WithOverride)...)
}
func _map(dst, src interface{}, opts ...func(*Config)) error {
var (
vDst, vSrc reflect.Value
err error
)
config := &Config{}
for _, opt := range opts {
opt(config)
}
if vDst, vSrc, err = resolveValues(dst, src); err != nil {
return err
}
// To be friction-less, we redirect equal-type arguments
// to deepMerge. Only because arguments can be anything.
if vSrc.Kind() == vDst.Kind() {
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}
switch vSrc.Kind() {
case reflect.Struct:
if vDst.Kind() != reflect.Map {
return ErrExpectedMapAsDestination
}
case reflect.Map:
if vDst.Kind() != reflect.Struct {
return ErrExpectedStructAsDestination
}
default:
return ErrNotSupported
}
return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}

255
vendor/github.com/imdario/mergo/merge.go generated vendored Normal file
View File

@ -0,0 +1,255 @@
// Copyright 2013 Dario Castañé. All rights reserved.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Based on src/pkg/reflect/deepequal.go from official
// golang's stdlib.
package mergo
import (
"fmt"
"reflect"
)
func hasExportedField(dst reflect.Value) (exported bool) {
for i, n := 0, dst.NumField(); i < n; i++ {
field := dst.Type().Field(i)
if field.Anonymous && dst.Field(i).Kind() == reflect.Struct {
exported = exported || hasExportedField(dst.Field(i))
} else {
exported = exported || len(field.PkgPath) == 0
}
}
return
}
type Config struct {
Overwrite bool
AppendSlice bool
Transformers Transformers
overwriteWithEmptyValue bool
}
type Transformers interface {
Transformer(reflect.Type) func(dst, src reflect.Value) error
}
// Traverses recursively both values, assigning src's fields values to dst.
// The map argument tracks comparisons that have already been seen, which allows
// short circuiting on recursive types.
func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
overwrite := config.Overwrite
overwriteWithEmptySrc := config.overwriteWithEmptyValue
config.overwriteWithEmptyValue = false
if !src.IsValid() {
return
}
if dst.CanAddr() {
addr := dst.UnsafeAddr()
h := 17 * addr
seen := visited[h]
typ := dst.Type()
for p := seen; p != nil; p = p.next {
if p.ptr == addr && p.typ == typ {
return nil
}
}
// Remember, remember...
visited[h] = &visit{addr, typ, seen}
}
if config.Transformers != nil && !isEmptyValue(dst) {
if fn := config.Transformers.Transformer(dst.Type()); fn != nil {
err = fn(dst, src)
return
}
}
switch dst.Kind() {
case reflect.Struct:
if hasExportedField(dst) {
for i, n := 0, dst.NumField(); i < n; i++ {
if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil {
return
}
}
} else {
if dst.CanSet() && (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
}
case reflect.Map:
if dst.IsNil() && !src.IsNil() {
dst.Set(reflect.MakeMap(dst.Type()))
}
for _, key := range src.MapKeys() {
srcElement := src.MapIndex(key)
if !srcElement.IsValid() {
continue
}
dstElement := dst.MapIndex(key)
switch srcElement.Kind() {
case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice:
if srcElement.IsNil() {
continue
}
fallthrough
default:
if !srcElement.CanInterface() {
continue
}
switch reflect.TypeOf(srcElement.Interface()).Kind() {
case reflect.Struct:
fallthrough
case reflect.Ptr:
fallthrough
case reflect.Map:
srcMapElm := srcElement
dstMapElm := dstElement
if srcMapElm.CanInterface() {
srcMapElm = reflect.ValueOf(srcMapElm.Interface())
if dstMapElm.IsValid() {
dstMapElm = reflect.ValueOf(dstMapElm.Interface())
}
}
if err = deepMerge(dstMapElm, srcMapElm, visited, depth+1, config); err != nil {
return
}
case reflect.Slice:
srcSlice := reflect.ValueOf(srcElement.Interface())
var dstSlice reflect.Value
if !dstElement.IsValid() || dstElement.IsNil() {
dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len())
} else {
dstSlice = reflect.ValueOf(dstElement.Interface())
}
if (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
dstSlice = srcSlice
} else if config.AppendSlice {
if srcSlice.Type() != dstSlice.Type() {
return fmt.Errorf("cannot append two slice with different type (%s, %s)", srcSlice.Type(), dstSlice.Type())
}
dstSlice = reflect.AppendSlice(dstSlice, srcSlice)
}
dst.SetMapIndex(key, dstSlice)
}
}
if dstElement.IsValid() && !isEmptyValue(dstElement) && (reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map || reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Slice) {
continue
}
if srcElement.IsValid() && (overwrite || (!dstElement.IsValid() || isEmptyValue(dstElement))) {
if dst.IsNil() {
dst.Set(reflect.MakeMap(dst.Type()))
}
dst.SetMapIndex(key, srcElement)
}
}
case reflect.Slice:
if !dst.CanSet() {
break
}
if (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
dst.Set(src)
} else if config.AppendSlice {
if src.Type() != dst.Type() {
return fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type())
}
dst.Set(reflect.AppendSlice(dst, src))
}
case reflect.Ptr:
fallthrough
case reflect.Interface:
if src.IsNil() {
break
}
if src.Kind() != reflect.Interface {
if dst.IsNil() || overwrite {
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
} else if src.Kind() == reflect.Ptr {
if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
return
}
} else if dst.Elem().Type() == src.Type() {
if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
return
}
} else {
return ErrDifferentArgumentsTypes
}
break
}
if dst.IsNil() || overwrite {
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
} else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
return
}
default:
if dst.CanSet() && (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
}
return
}
// Merge will fill any empty for value type attributes on the dst struct using corresponding
// src attributes if they themselves are not empty. dst and src must be valid same-type structs
// and dst must be a pointer to struct.
// It won't merge unexported (private) fields and will do recursively any exported field.
func Merge(dst, src interface{}, opts ...func(*Config)) error {
return merge(dst, src, opts...)
}
// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overriden by
// non-empty src attribute values.
// Deprecated: use Merge(…) with WithOverride
func MergeWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
return merge(dst, src, append(opts, WithOverride)...)
}
// WithTransformers adds transformers to merge, allowing to customize the merging of some types.
func WithTransformers(transformers Transformers) func(*Config) {
return func(config *Config) {
config.Transformers = transformers
}
}
// WithOverride will make merge override non-empty dst attributes with non-empty src attributes values.
func WithOverride(config *Config) {
config.Overwrite = true
}
// WithAppendSlice will make merge append slices instead of overwriting it
func WithAppendSlice(config *Config) {
config.AppendSlice = true
}
func merge(dst, src interface{}, opts ...func(*Config)) error {
var (
vDst, vSrc reflect.Value
err error
)
config := &Config{}
for _, opt := range opts {
opt(config)
}
if vDst, vSrc, err = resolveValues(dst, src); err != nil {
return err
}
if vDst.Type() != vSrc.Type() {
return ErrDifferentArgumentsTypes
}
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}

97
vendor/github.com/imdario/mergo/mergo.go generated vendored Normal file
View File

@ -0,0 +1,97 @@
// Copyright 2013 Dario Castañé. All rights reserved.
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Based on src/pkg/reflect/deepequal.go from official
// golang's stdlib.
package mergo
import (
"errors"
"reflect"
)
// Errors reported by Mergo when it finds invalid arguments.
var (
ErrNilArguments = errors.New("src and dst must not be nil")
ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type")
ErrNotSupported = errors.New("only structs and maps are supported")
ErrExpectedMapAsDestination = errors.New("dst was expected to be a map")
ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct")
)
// During deepMerge, must keep track of checks that are
// in progress. The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited are stored in a map indexed by 17 * a1 + a2;
type visit struct {
ptr uintptr
typ reflect.Type
next *visit
}
// From src/pkg/encoding/json/encode.go.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
if v.IsNil() {
return true
}
return isEmptyValue(v.Elem())
case reflect.Func:
return v.IsNil()
case reflect.Invalid:
return true
}
return false
}
func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) {
if dst == nil || src == nil {
err = ErrNilArguments
return
}
vDst = reflect.ValueOf(dst).Elem()
if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map {
err = ErrNotSupported
return
}
vSrc = reflect.ValueOf(src)
// We check if vSrc is a pointer to dereference it.
if vSrc.Kind() == reflect.Ptr {
vSrc = vSrc.Elem()
}
return
}
// Traverses recursively both values, assigning src's fields values to dst.
// The map argument tracks comparisons that have already been seen, which allows
// short circuiting on recursive types.
func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) {
if dst.CanAddr() {
addr := dst.UnsafeAddr()
h := 17 * addr
seen := visited[h]
typ := dst.Type()
for p := seen; p != nil; p = p.next {
if p.ptr == addr && p.typ == typ {
return nil
}
}
// Remember, remember...
visited[h] = &visit{addr, typ, seen}
}
return // TODO refactor
}