diff --git a/cmd/containerd/command/config.go b/cmd/containerd/command/config.go index f9d5f9efa..1e5710d42 100644 --- a/cmd/containerd/command/config.go +++ b/cmd/containerd/command/config.go @@ -40,6 +40,50 @@ func (c *Config) WriteTo(w io.Writer) (int64, error) { return 0, toml.NewEncoder(w).Encode(c) } +func outputConfig(cfg *srvconfig.Config) error { + config := &Config{ + Config: cfg, + } + + plugins, err := server.LoadPlugins(gocontext.Background(), config.Config) + if err != nil { + return err + } + if len(plugins) != 0 { + config.Plugins = make(map[string]interface{}) + for _, p := range plugins { + if p.Config == nil { + continue + } + + pc, err := config.Decode(p) + if err != nil { + return err + } + + config.Plugins[p.URI()] = pc + } + } + + timeouts := timeout.All() + config.Timeouts = make(map[string]string) + for k, v := range timeouts { + config.Timeouts[k] = v.String() + } + + // for the time being, keep the defaultConfig's version set at 1 so that + // when a config without a version is loaded from disk and has no version + // set, we assume it's a v1 config. But when generating new configs via + // this command, generate the v2 config + config.Config.Version = 2 + + // remove overridden Plugins type to avoid duplication in output + config.Config.Plugins = nil + + _, err = config.WriteTo(os.Stdout) + return err +} + var configCommand = cli.Command{ Name: "config", Usage: "information on the containerd config", @@ -48,35 +92,19 @@ var configCommand = cli.Command{ Name: "default", Usage: "see the output of the default config", Action: func(context *cli.Context) error { - config := &Config{ - Config: defaultConfig(), - } - // for the time being, keep the defaultConfig's version set at 1 so that - // when a config without a version is loaded from disk and has no version - // set, we assume it's a v1 config. But when generating new configs via - // this command, generate the v2 config - config.Config.Version = 2 - plugins, err := server.LoadPlugins(gocontext.Background(), config.Config) - if err != nil { + return outputConfig(defaultConfig()) + }, + }, + { + Name: "dump", + Usage: "see the output of the final main config with imported in subconfig files", + Action: func(context *cli.Context) error { + config := defaultConfig() + if err := srvconfig.LoadConfig(context.GlobalString("config"), config); err != nil && !os.IsNotExist(err) { return err } - if len(plugins) != 0 { - config.Plugins = make(map[string]interface{}) - for _, p := range plugins { - if p.Config == nil { - continue - } - config.Plugins[p.URI()] = p.Config - } - } - timeouts := timeout.All() - config.Timeouts = make(map[string]string) - for k, v := range timeouts { - config.Timeouts[k] = v.String() - } - _, err = config.WriteTo(os.Stdout) - return err + return outputConfig(config) }, }, }, diff --git a/docs/man/containerd-config.toml.5.md b/docs/man/containerd-config.toml.5.md index 21a603cea..6b5c2cc77 100644 --- a/docs/man/containerd-config.toml.5.md +++ b/docs/man/containerd-config.toml.5.md @@ -32,6 +32,14 @@ settings. **oom_score** : The out of memory (OOM) score applied to the containerd daemon process (Default: 0) +**imports** +: Imports is a list of additional configuration files to include. +This allows to split the main configuration file and keep some sections +separately (for example vendors may keep a custom runtime configuration in a +separate file without modifying the main `config.toml`). +Imported files will overwrite simple fields like `int` or +`string` (if not empty) and will append `array` and `map` fields. + **[grpc]** : Section for gRPC socket listener settings. Contains three properties: - **address** (Default: "/run/containerd/containerd.sock") @@ -82,6 +90,7 @@ The following is a complete **config.toml** default configuration example: root = "/var/lib/containerd" state = "/run/containerd" oom_score = 0 +imports = ["/etc/containerd/runtime_*.toml", "./debug.toml"] [grpc] address = "/run/containerd/containerd.sock" diff --git a/docs/stream_processors.md b/docs/stream_processors.md index 67af20d58..ba227a599 100644 --- a/docs/stream_processors.md +++ b/docs/stream_processors.md @@ -21,20 +21,20 @@ pipe's path set as the value of the environment variable `STREAM_PROCESSOR_PIPE` ## Configuration To configure stream processors for containerd, entries in the config file need to be made. -The `stream_processors` field is an array so that users can chain together multiple processors +The `stream_processors` field is a map so that users can chain together multiple processors to mutate content streams. Processor Fields: -* `id` - ID of the processor, used for passing a specific payload to the processor. +* Key - ID of the processor, used for passing a specific payload to the processor. * `accepts` - Accepted media-types for the processor that it can handle. * `returns` - The media-type that the processor returns. * `path` - Path to the processor binary. * `args` - Arguments passed to the processor binary. ```toml -[[stream_processors]] - id = "io.containerd.processor.v1.pigz" +[stream_processors] + [stream_processors."io.containerd.processor.v1.pigz"] accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"] returns = "application/vnd.oci.image.layer.v1.tar" path = "unpigz" diff --git a/services/server/config/config.go b/services/server/config/config.go index c6e572753..ff3771608 100644 --- a/services/server/config/config.go +++ b/services/server/config/config.go @@ -17,12 +17,15 @@ package config import ( + "path/filepath" "strings" "github.com/BurntSushi/toml" + "github.com/imdario/mergo" + "github.com/pkg/errors" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/plugin" - "github.com/pkg/errors" ) // Config provides containerd configuration data for the server @@ -59,16 +62,14 @@ type Config struct { ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"` // Timeouts specified as a duration Timeouts map[string]string `toml:"timeouts"` + // Imports are additional file path list to config files that can overwrite main config file fields + Imports []string `toml:"imports"` - StreamProcessors []StreamProcessor `toml:"stream_processors"` - - md toml.MetaData + StreamProcessors map[string]StreamProcessor `toml:"stream_processors"` } // StreamProcessor provides configuration for diff content processors type StreamProcessor struct { - // ID of the processor, also used to fetch the specific payload - ID string `toml:"id"` // Accepts specific media-types Accepts []string `toml:"accepts"` // Returns the media-type @@ -202,23 +203,125 @@ func (c *Config) Decode(p *plugin.Registration) (interface{}, error) { if !ok { return p.Config, nil } - if err := c.md.PrimitiveDecode(data, p.Config); err != nil { + if err := toml.PrimitiveDecode(data, p.Config); err != nil { return nil, err } return p.Config, nil } // LoadConfig loads the containerd server config from the provided path -func LoadConfig(path string, v *Config) error { - if v == nil { - return errors.Wrapf(errdefs.ErrInvalidArgument, "argument v must not be nil") +func LoadConfig(path string, out *Config) error { + if out == nil { + return errors.Wrapf(errdefs.ErrInvalidArgument, "argument out must not be nil") } - md, err := toml.DecodeFile(path, v) + + var ( + loaded = map[string]bool{} + pending = []string{path} + ) + + for len(pending) > 0 { + path, pending = pending[0], pending[1:] + + // Check if a file at the given path already loaded to prevent circular imports + if _, ok := loaded[path]; ok { + continue + } + + config, err := loadConfigFile(path) + if err != nil { + return err + } + + if err := mergeConfig(out, config); err != nil { + return err + } + + imports, err := resolveImports(path, config.Imports) + if err != nil { + return err + } + + loaded[path] = true + pending = append(pending, imports...) + } + + // Fix up the list of config files loaded + out.Imports = []string{} + for path := range loaded { + out.Imports = append(out.Imports, path) + } + + return out.ValidateV2() +} + +// loadConfigFile decodes a TOML file at the given path +func loadConfigFile(path string) (*Config, error) { + config := &Config{} + _, err := toml.DecodeFile(path, &config) + if err != nil { + return nil, err + } + return config, nil +} + +// resolveImports resolves import strings list to absolute paths list: +// - If path contains *, glob pattern matching applied +// - Non abs path is relative to parent config file directory +// - Abs paths returned as is +func resolveImports(parent string, imports []string) ([]string, error) { + var out []string + + for _, path := range imports { + if strings.Contains(path, "*") { + matches, err := filepath.Glob(path) + if err != nil { + return nil, err + } + + out = append(out, matches...) + } else { + path = filepath.Clean(path) + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(parent), path) + } + + out = append(out, path) + } + } + + return out, nil +} + +// mergeConfig merges Config structs with the following rules: +// 'to' 'from' 'result' +// "" "value" "value" +// "value" "" "value" +// 1 0 1 +// 0 1 1 +// []{"1"} []{"2"} []{"1","2"} +// []{"1"} []{} []{"1"} +// Maps merged by keys, but values are replaced entirely. +func mergeConfig(to, from *Config) error { + err := mergo.Merge(to, from, mergo.WithOverride, mergo.WithAppendSlice) if err != nil { return err } - v.md = md - return v.ValidateV2() + + // Replace entire sections instead of merging map's values. + for k, v := range from.Plugins { + to.Plugins[k] = v + } + + for k, v := range from.StreamProcessors { + to.StreamProcessors[k] = v + } + + for k, v := range from.ProxyPlugins { + to.ProxyPlugins[k] = v + } + + return nil } // V1DisabledFilter matches based on ID diff --git a/services/server/config/config_test.go b/services/server/config/config_test.go new file mode 100644 index 000000000..8660105d4 --- /dev/null +++ b/services/server/config/config_test.go @@ -0,0 +1,207 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "gotest.tools/assert" + + "github.com/containerd/containerd/plugin" +) + +func TestMergeConfigs(t *testing.T) { + a := &Config{ + Version: 2, + Root: "old_root", + RequiredPlugins: []string{"old_plugin"}, + DisabledPlugins: []string{"old_plugin"}, + State: "old_state", + OOMScore: 1, + Timeouts: map[string]string{"a": "1"}, + StreamProcessors: map[string]StreamProcessor{"1": {Path: "2", Returns: "4"}, "2": {Path: "5"}}, + } + + b := &Config{ + Root: "new_root", + RequiredPlugins: []string{"new_plugin1", "new_plugin2"}, + OOMScore: 2, + Timeouts: map[string]string{"b": "2"}, + StreamProcessors: map[string]StreamProcessor{"1": {Path: "3"}}, + } + + err := mergeConfig(a, b) + assert.NilError(t, err) + + assert.Equal(t, a.Version, 2) + assert.Equal(t, a.Root, "new_root") + assert.Equal(t, a.State, "old_state") + assert.Equal(t, a.OOMScore, 2) + assert.DeepEqual(t, a.RequiredPlugins, []string{"old_plugin", "new_plugin1", "new_plugin2"}) + assert.DeepEqual(t, a.DisabledPlugins, []string{"old_plugin"}) + assert.DeepEqual(t, a.Timeouts, map[string]string{"a": "1", "b": "2"}) + assert.DeepEqual(t, a.StreamProcessors, map[string]StreamProcessor{"1": {Path: "3"}, "2": {Path: "5"}}) +} + +func TestResolveImports(t *testing.T) { + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + for _, filename := range []string{"config_1.toml", "config_2.toml", "test.toml"} { + err = ioutil.WriteFile(filepath.Join(tempDir, filename), []byte(""), 0600) + assert.NilError(t, err) + } + + imports, err := resolveImports(filepath.Join(tempDir, "root.toml"), []string{ + filepath.Join(tempDir, "config_*.toml"), // Glob + filepath.Join(tempDir, "./test.toml"), // Path clean up + "current.toml", // Resolve current working dir + }) + assert.NilError(t, err) + + assert.DeepEqual(t, imports, []string{ + filepath.Join(tempDir, "config_1.toml"), + filepath.Join(tempDir, "config_2.toml"), + filepath.Join(tempDir, "test.toml"), + filepath.Join(tempDir, "current.toml"), + }) +} + +func TestLoadSingleConfig(t *testing.T) { + data := ` +version = 2 +root = "/var/lib/containerd" + +[stream_processors] + [stream_processors."io.containerd.processor.v1.pigz"] + accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"] + path = "unpigz" +` + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + path := filepath.Join(tempDir, "config.toml") + err = ioutil.WriteFile(path, []byte(data), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(path, &out) + assert.NilError(t, err) + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) + assert.DeepEqual(t, map[string]StreamProcessor{ + "io.containerd.processor.v1.pigz": { + Accepts: []string{"application/vnd.docker.image.rootfs.diff.tar.gzip"}, + Path: "unpigz", + }, + }, out.StreamProcessors) +} + +func TestLoadConfigWithImports(t *testing.T) { + data1 := ` +version = 2 +root = "/var/lib/containerd" +imports = ["data2.toml"] +` + + data2 := ` +disabled_plugins = ["io.containerd.v1.xyz"] +` + + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600) + assert.NilError(t, err) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out) + assert.NilError(t, err) + + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) + assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins) +} + +func TestLoadConfigWithCircularImports(t *testing.T) { + data1 := ` +version = 2 +root = "/var/lib/containerd" +imports = ["data2.toml", "data1.toml"] +` + + data2 := ` +disabled_plugins = ["io.containerd.v1.xyz"] +imports = ["data1.toml", "data2.toml"] +` + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600) + assert.NilError(t, err) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out) + assert.NilError(t, err) + + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) + assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins) + + assert.DeepEqual(t, []string{ + filepath.Join(tempDir, "data1.toml"), + filepath.Join(tempDir, "data2.toml"), + }, out.Imports) +} + +func TestDecodePlugin(t *testing.T) { + data := ` +version = 1 +[plugins.linux] + shim_debug = true +` + + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + path := filepath.Join(tempDir, "config.toml") + err = ioutil.WriteFile(path, []byte(data), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(path, &out) + assert.NilError(t, err) + + pluginConfig := map[string]interface{}{} + _, err = out.Decode(&plugin.Registration{ID: "linux", Config: &pluginConfig}) + assert.NilError(t, err) + assert.Equal(t, true, pluginConfig["shim_debug"]) +} diff --git a/services/server/server.go b/services/server/server.go index eae27fb29..e31fec5df 100644 --- a/services/server/server.go +++ b/services/server/server.go @@ -89,8 +89,8 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) { if err != nil { return nil, err } - for _, p := range config.StreamProcessors { - diff.RegisterProcessor(diff.BinaryHandler(p.ID, p.Returns, p.Accepts, p.Path, p.Args)) + for id, p := range config.StreamProcessors { + diff.RegisterProcessor(diff.BinaryHandler(id, p.Returns, p.Accepts, p.Path, p.Args)) } serverOpts := []grpc.ServerOption{ diff --git a/vendor.conf b/vendor.conf index d8063b2a9..c1b3cfbcf 100644 --- a/vendor.conf +++ b/vendor.conf @@ -46,6 +46,7 @@ github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/golang-lru v0.5.3 go.opencensus.io v0.22.0 +github.com/imdario/mergo v0.3.7 # cri dependencies github.com/containerd/cri f1d492b0cdd14e76476ee4dd024696ce3634e501 # master diff --git a/vendor/github.com/imdario/mergo/LICENSE b/vendor/github.com/imdario/mergo/LICENSE new file mode 100644 index 000000000..686680298 --- /dev/null +++ b/vendor/github.com/imdario/mergo/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/imdario/mergo/README.md b/vendor/github.com/imdario/mergo/README.md new file mode 100644 index 000000000..02fc81e06 --- /dev/null +++ b/vendor/github.com/imdario/mergo/README.md @@ -0,0 +1,238 @@ +# Mergo + +A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. + +Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region of Marche. + +## Status + +It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild). + +[![GoDoc][3]][4] +[![GoCard][5]][6] +[![Build Status][1]][2] +[![Coverage Status][7]][8] +[![Sourcegraph][9]][10] +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield) + +[1]: https://travis-ci.org/imdario/mergo.png +[2]: https://travis-ci.org/imdario/mergo +[3]: https://godoc.org/github.com/imdario/mergo?status.svg +[4]: https://godoc.org/github.com/imdario/mergo +[5]: https://goreportcard.com/badge/imdario/mergo +[6]: https://goreportcard.com/report/github.com/imdario/mergo +[7]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master +[8]: https://coveralls.io/github/imdario/mergo?branch=master +[9]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg +[10]: https://sourcegraph.com/github.com/imdario/mergo?badge + +### Latest release + +[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7). + +### Important note + +Please keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2) Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). An optional/variadic argument has been added, so it won't break existing code. + +If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0). + +### Donations + +If Mergo is useful to you, consider buying me a coffee, a beer or making a monthly donation so I can keep building great free software. :heart_eyes: + +Buy Me a Coffee at ko-fi.com +[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo) +[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo) +Donate using Liberapay + +### Mergo in the wild + +- [moby/moby](https://github.com/moby/moby) +- [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) +- [vmware/dispatch](https://github.com/vmware/dispatch) +- [Shopify/themekit](https://github.com/Shopify/themekit) +- [imdario/zas](https://github.com/imdario/zas) +- [matcornic/hermes](https://github.com/matcornic/hermes) +- [OpenBazaar/openbazaar-go](https://github.com/OpenBazaar/openbazaar-go) +- [kataras/iris](https://github.com/kataras/iris) +- [michaelsauter/crane](https://github.com/michaelsauter/crane) +- [go-task/task](https://github.com/go-task/task) +- [sensu/uchiwa](https://github.com/sensu/uchiwa) +- [ory/hydra](https://github.com/ory/hydra) +- [sisatech/vcli](https://github.com/sisatech/vcli) +- [dairycart/dairycart](https://github.com/dairycart/dairycart) +- [projectcalico/felix](https://github.com/projectcalico/felix) +- [resin-os/balena](https://github.com/resin-os/balena) +- [go-kivik/kivik](https://github.com/go-kivik/kivik) +- [Telefonica/govice](https://github.com/Telefonica/govice) +- [supergiant/supergiant](supergiant/supergiant) +- [SergeyTsalkov/brooce](https://github.com/SergeyTsalkov/brooce) +- [soniah/dnsmadeeasy](https://github.com/soniah/dnsmadeeasy) +- [ohsu-comp-bio/funnel](https://github.com/ohsu-comp-bio/funnel) +- [EagerIO/Stout](https://github.com/EagerIO/Stout) +- [lynndylanhurley/defsynth-api](https://github.com/lynndylanhurley/defsynth-api) +- [russross/canvasassignments](https://github.com/russross/canvasassignments) +- [rdegges/cryptly-api](https://github.com/rdegges/cryptly-api) +- [casualjim/exeggutor](https://github.com/casualjim/exeggutor) +- [divshot/gitling](https://github.com/divshot/gitling) +- [RWJMurphy/gorl](https://github.com/RWJMurphy/gorl) +- [andrerocker/deploy42](https://github.com/andrerocker/deploy42) +- [elwinar/rambler](https://github.com/elwinar/rambler) +- [tmaiaroto/gopartman](https://github.com/tmaiaroto/gopartman) +- [jfbus/impressionist](https://github.com/jfbus/impressionist) +- [Jmeyering/zealot](https://github.com/Jmeyering/zealot) +- [godep-migrator/rigger-host](https://github.com/godep-migrator/rigger-host) +- [Dronevery/MultiwaySwitch-Go](https://github.com/Dronevery/MultiwaySwitch-Go) +- [thoas/picfit](https://github.com/thoas/picfit) +- [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server) +- [jnuthong/item_search](https://github.com/jnuthong/item_search) +- [bukalapak/snowboard](https://github.com/bukalapak/snowboard) + +## Installation + + go get github.com/imdario/mergo + + // use in your .go code + import ( + "github.com/imdario/mergo" + ) + +## Usage + +You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as [they are not considered zero values](https://golang.org/ref/spec#The_zero_value) either. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection). + +```go +if err := mergo.Merge(&dst, src); err != nil { + // ... +} +``` + +Also, you can merge overwriting values using the transformer `WithOverride`. + +```go +if err := mergo.Merge(&dst, src, mergo.WithOverride); err != nil { + // ... +} +``` + +Additionally, you can map a `map[string]interface{}` to a struct (and otherwise, from struct to map), following the same restrictions as in `Merge()`. Keys are capitalized to find each corresponding exported field. + +```go +if err := mergo.Map(&dst, srcMap); err != nil { + // ... +} +``` + +Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as `map[string]interface{}`. They will be just assigned as values. + +More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo). + +### Nice example + +```go +package main + +import ( + "fmt" + "github.com/imdario/mergo" +) + +type Foo struct { + A string + B int64 +} + +func main() { + src := Foo{ + A: "one", + B: 2, + } + dest := Foo{ + A: "two", + } + mergo.Merge(&dest, src) + fmt.Println(dest) + // Will print + // {two 2} +} +``` + +Note: if test are failing due missing package, please execute: + + go get gopkg.in/yaml.v2 + +### Transformers + +Transformers allow to merge specific types differently than in the default behavior. In other words, now you can customize how some types are merged. For example, `time.Time` is a struct; it doesn't have zero value but IsZero can return true because it has fields with zero value. How can we merge a non-zero `time.Time`? + +```go +package main + +import ( + "fmt" + "github.com/imdario/mergo" + "reflect" + "time" +) + +type timeTransfomer struct { +} + +func (t timeTransfomer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error { + if typ == reflect.TypeOf(time.Time{}) { + return func(dst, src reflect.Value) error { + if dst.CanSet() { + isZero := dst.MethodByName("IsZero") + result := isZero.Call([]reflect.Value{}) + if result[0].Bool() { + dst.Set(src) + } + } + return nil + } + } + return nil +} + +type Snapshot struct { + Time time.Time + // ... +} + +func main() { + src := Snapshot{time.Now()} + dest := Snapshot{} + mergo.Merge(&dest, src, mergo.WithTransformers(timeTransfomer{})) + fmt.Println(dest) + // Will print + // { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 } +} +``` + + +## Contact me + +If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) + +## About + +Written by [Dario Castañé](http://dario.im). + +## Top Contributors + +[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0) +[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1) +[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2) +[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3) +[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4) +[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5) +[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6) +[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7) + + +## License + +[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). + + +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large) diff --git a/vendor/github.com/imdario/mergo/doc.go b/vendor/github.com/imdario/mergo/doc.go new file mode 100644 index 000000000..6e9aa7baf --- /dev/null +++ b/vendor/github.com/imdario/mergo/doc.go @@ -0,0 +1,44 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mergo merges same-type structs and maps by setting default values in zero-value fields. + +Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection). + +Usage + +From my own work-in-progress project: + + type networkConfig struct { + Protocol string + Address string + ServerType string `json: "server_type"` + Port uint16 + } + + type FssnConfig struct { + Network networkConfig + } + + var fssnDefault = FssnConfig { + networkConfig { + "tcp", + "127.0.0.1", + "http", + 31560, + }, + } + + // Inside a function [...] + + if err := mergo.Merge(&config, fssnDefault); err != nil { + log.Fatal(err) + } + + // More code [...] + +*/ +package mergo diff --git a/vendor/github.com/imdario/mergo/map.go b/vendor/github.com/imdario/mergo/map.go new file mode 100644 index 000000000..3f5afa83a --- /dev/null +++ b/vendor/github.com/imdario/mergo/map.go @@ -0,0 +1,175 @@ +// Copyright 2014 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "fmt" + "reflect" + "unicode" + "unicode/utf8" +) + +func changeInitialCase(s string, mapper func(rune) rune) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(mapper(r)) + s[n:] +} + +func isExported(field reflect.StructField) bool { + r, _ := utf8.DecodeRuneInString(field.Name) + return r >= 'A' && r <= 'Z' +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) { + overwrite := config.Overwrite + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + zeroValue := reflect.Value{} + switch dst.Kind() { + case reflect.Map: + dstMap := dst.Interface().(map[string]interface{}) + for i, n := 0, src.NumField(); i < n; i++ { + srcType := src.Type() + field := srcType.Field(i) + if !isExported(field) { + continue + } + fieldName := field.Name + fieldName = changeInitialCase(fieldName, unicode.ToLower) + if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) { + dstMap[fieldName] = src.Field(i).Interface() + } + } + case reflect.Ptr: + if dst.IsNil() { + v := reflect.New(dst.Type().Elem()) + dst.Set(v) + } + dst = dst.Elem() + fallthrough + case reflect.Struct: + srcMap := src.Interface().(map[string]interface{}) + for key := range srcMap { + config.overwriteWithEmptyValue = true + srcValue := srcMap[key] + fieldName := changeInitialCase(key, unicode.ToUpper) + dstElement := dst.FieldByName(fieldName) + if dstElement == zeroValue { + // We discard it because the field doesn't exist. + continue + } + srcElement := reflect.ValueOf(srcValue) + dstKind := dstElement.Kind() + srcKind := srcElement.Kind() + if srcKind == reflect.Ptr && dstKind != reflect.Ptr { + srcElement = srcElement.Elem() + srcKind = reflect.TypeOf(srcElement.Interface()).Kind() + } else if dstKind == reflect.Ptr { + // Can this work? I guess it can't. + if srcKind != reflect.Ptr && srcElement.CanAddr() { + srcPtr := srcElement.Addr() + srcElement = reflect.ValueOf(srcPtr) + srcKind = reflect.Ptr + } + } + + if !srcElement.IsValid() { + continue + } + if srcKind == dstKind { + if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface { + if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else if srcKind == reflect.Map { + if err = deepMap(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else { + return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind) + } + } + } + return +} + +// Map sets fields' values in dst from src. +// src can be a map with string keys or a struct. dst must be the opposite: +// if src is a map, dst must be a valid pointer to struct. If src is a struct, +// dst must be map[string]interface{}. +// It won't merge unexported (private) fields and will do recursively +// any exported field. +// If dst is a map, keys will be src fields' names in lower camel case. +// Missing key in src that doesn't match a field in dst will be skipped. This +// doesn't apply if dst is a map. +// This is separated method from Merge because it is cleaner and it keeps sane +// semantics: merging equal types, mapping different (restricted) types. +func Map(dst, src interface{}, opts ...func(*Config)) error { + return _map(dst, src, opts...) +} + +// MapWithOverwrite will do the same as Map except that non-empty dst attributes will be overridden by +// non-empty src attribute values. +// Deprecated: Use Map(…) with WithOverride +func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error { + return _map(dst, src, append(opts, WithOverride)...) +} + +func _map(dst, src interface{}, opts ...func(*Config)) error { + var ( + vDst, vSrc reflect.Value + err error + ) + config := &Config{} + + for _, opt := range opts { + opt(config) + } + + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + // To be friction-less, we redirect equal-type arguments + // to deepMerge. Only because arguments can be anything. + if vSrc.Kind() == vDst.Kind() { + return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) + } + switch vSrc.Kind() { + case reflect.Struct: + if vDst.Kind() != reflect.Map { + return ErrExpectedMapAsDestination + } + case reflect.Map: + if vDst.Kind() != reflect.Struct { + return ErrExpectedStructAsDestination + } + default: + return ErrNotSupported + } + return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, config) +} diff --git a/vendor/github.com/imdario/mergo/merge.go b/vendor/github.com/imdario/mergo/merge.go new file mode 100644 index 000000000..f8de6c543 --- /dev/null +++ b/vendor/github.com/imdario/mergo/merge.go @@ -0,0 +1,255 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "fmt" + "reflect" +) + +func hasExportedField(dst reflect.Value) (exported bool) { + for i, n := 0, dst.NumField(); i < n; i++ { + field := dst.Type().Field(i) + if field.Anonymous && dst.Field(i).Kind() == reflect.Struct { + exported = exported || hasExportedField(dst.Field(i)) + } else { + exported = exported || len(field.PkgPath) == 0 + } + } + return +} + +type Config struct { + Overwrite bool + AppendSlice bool + Transformers Transformers + overwriteWithEmptyValue bool +} + +type Transformers interface { + Transformer(reflect.Type) func(dst, src reflect.Value) error +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) { + overwrite := config.Overwrite + overwriteWithEmptySrc := config.overwriteWithEmptyValue + config.overwriteWithEmptyValue = false + + if !src.IsValid() { + return + } + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + + if config.Transformers != nil && !isEmptyValue(dst) { + if fn := config.Transformers.Transformer(dst.Type()); fn != nil { + err = fn(dst, src) + return + } + } + + switch dst.Kind() { + case reflect.Struct: + if hasExportedField(dst) { + for i, n := 0, dst.NumField(); i < n; i++ { + if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil { + return + } + } + } else { + if dst.CanSet() && (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } + case reflect.Map: + if dst.IsNil() && !src.IsNil() { + dst.Set(reflect.MakeMap(dst.Type())) + } + for _, key := range src.MapKeys() { + srcElement := src.MapIndex(key) + if !srcElement.IsValid() { + continue + } + dstElement := dst.MapIndex(key) + switch srcElement.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice: + if srcElement.IsNil() { + continue + } + fallthrough + default: + if !srcElement.CanInterface() { + continue + } + switch reflect.TypeOf(srcElement.Interface()).Kind() { + case reflect.Struct: + fallthrough + case reflect.Ptr: + fallthrough + case reflect.Map: + srcMapElm := srcElement + dstMapElm := dstElement + if srcMapElm.CanInterface() { + srcMapElm = reflect.ValueOf(srcMapElm.Interface()) + if dstMapElm.IsValid() { + dstMapElm = reflect.ValueOf(dstMapElm.Interface()) + } + } + if err = deepMerge(dstMapElm, srcMapElm, visited, depth+1, config); err != nil { + return + } + case reflect.Slice: + srcSlice := reflect.ValueOf(srcElement.Interface()) + + var dstSlice reflect.Value + if !dstElement.IsValid() || dstElement.IsNil() { + dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len()) + } else { + dstSlice = reflect.ValueOf(dstElement.Interface()) + } + + if (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice { + dstSlice = srcSlice + } else if config.AppendSlice { + if srcSlice.Type() != dstSlice.Type() { + return fmt.Errorf("cannot append two slice with different type (%s, %s)", srcSlice.Type(), dstSlice.Type()) + } + dstSlice = reflect.AppendSlice(dstSlice, srcSlice) + } + dst.SetMapIndex(key, dstSlice) + } + } + if dstElement.IsValid() && !isEmptyValue(dstElement) && (reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map || reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Slice) { + continue + } + + if srcElement.IsValid() && (overwrite || (!dstElement.IsValid() || isEmptyValue(dstElement))) { + if dst.IsNil() { + dst.Set(reflect.MakeMap(dst.Type())) + } + dst.SetMapIndex(key, srcElement) + } + } + case reflect.Slice: + if !dst.CanSet() { + break + } + if (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice { + dst.Set(src) + } else if config.AppendSlice { + if src.Type() != dst.Type() { + return fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type()) + } + dst.Set(reflect.AppendSlice(dst, src)) + } + case reflect.Ptr: + fallthrough + case reflect.Interface: + if src.IsNil() { + break + } + if src.Kind() != reflect.Interface { + if dst.IsNil() || overwrite { + if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } else if src.Kind() == reflect.Ptr { + if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { + return + } + } else if dst.Elem().Type() == src.Type() { + if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil { + return + } + } else { + return ErrDifferentArgumentsTypes + } + break + } + if dst.IsNil() || overwrite { + if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { + return + } + default: + if dst.CanSet() && (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } + return +} + +// Merge will fill any empty for value type attributes on the dst struct using corresponding +// src attributes if they themselves are not empty. dst and src must be valid same-type structs +// and dst must be a pointer to struct. +// It won't merge unexported (private) fields and will do recursively any exported field. +func Merge(dst, src interface{}, opts ...func(*Config)) error { + return merge(dst, src, opts...) +} + +// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overriden by +// non-empty src attribute values. +// Deprecated: use Merge(…) with WithOverride +func MergeWithOverwrite(dst, src interface{}, opts ...func(*Config)) error { + return merge(dst, src, append(opts, WithOverride)...) +} + +// WithTransformers adds transformers to merge, allowing to customize the merging of some types. +func WithTransformers(transformers Transformers) func(*Config) { + return func(config *Config) { + config.Transformers = transformers + } +} + +// WithOverride will make merge override non-empty dst attributes with non-empty src attributes values. +func WithOverride(config *Config) { + config.Overwrite = true +} + +// WithAppendSlice will make merge append slices instead of overwriting it +func WithAppendSlice(config *Config) { + config.AppendSlice = true +} + +func merge(dst, src interface{}, opts ...func(*Config)) error { + var ( + vDst, vSrc reflect.Value + err error + ) + + config := &Config{} + + for _, opt := range opts { + opt(config) + } + + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + if vDst.Type() != vSrc.Type() { + return ErrDifferentArgumentsTypes + } + return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) +} diff --git a/vendor/github.com/imdario/mergo/mergo.go b/vendor/github.com/imdario/mergo/mergo.go new file mode 100644 index 000000000..a82fea2fd --- /dev/null +++ b/vendor/github.com/imdario/mergo/mergo.go @@ -0,0 +1,97 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "errors" + "reflect" +) + +// Errors reported by Mergo when it finds invalid arguments. +var ( + ErrNilArguments = errors.New("src and dst must not be nil") + ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") + ErrNotSupported = errors.New("only structs and maps are supported") + ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") + ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") +) + +// During deepMerge, must keep track of checks that are +// in progress. The comparison algorithm assumes that all +// checks in progress are true when it reencounters them. +// Visited are stored in a map indexed by 17 * a1 + a2; +type visit struct { + ptr uintptr + typ reflect.Type + next *visit +} + +// From src/pkg/encoding/json/encode.go. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + if v.IsNil() { + return true + } + return isEmptyValue(v.Elem()) + case reflect.Func: + return v.IsNil() + case reflect.Invalid: + return true + } + return false +} + +func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { + if dst == nil || src == nil { + err = ErrNilArguments + return + } + vDst = reflect.ValueOf(dst).Elem() + if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { + err = ErrNotSupported + return + } + vSrc = reflect.ValueOf(src) + // We check if vSrc is a pointer to dereference it. + if vSrc.Kind() == reflect.Ptr { + vSrc = vSrc.Elem() + } + return +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) { + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + return // TODO refactor +}