From a1e3779cad30e491e6327af760af6f0ba27579c5 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 22 Aug 2019 15:50:58 -0700 Subject: [PATCH] Support config imports #3289 Signed-off-by: Maksym Pavlenko --- services/server/config/config.go | 107 +++++++++++++++-- services/server/config/config_test.go | 163 ++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 services/server/config/config_test.go diff --git a/services/server/config/config.go b/services/server/config/config.go index 9f8b1537c..7c47cd745 100644 --- a/services/server/config/config.go +++ b/services/server/config/config.go @@ -17,12 +17,15 @@ package config import ( + "path/filepath" "strings" "github.com/BurntSushi/toml" + "github.com/imdario/mergo" + "github.com/pkg/errors" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/plugin" - "github.com/pkg/errors" ) // Config provides containerd configuration data for the server @@ -57,6 +60,8 @@ type Config struct { ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"` // Timeouts specified as a duration Timeouts map[string]string `toml:"timeouts"` + // Imports are additional file path list to config files that can overwrite main config file fields + Imports []string `toml:"imports"` StreamProcessors []StreamProcessor `toml:"stream_processors"` @@ -205,16 +210,102 @@ func (c *Config) Decode(p *plugin.Registration) (interface{}, error) { } // LoadConfig loads the containerd server config from the provided path -func LoadConfig(path string, v *Config) error { - if v == nil { - return errors.Wrapf(errdefs.ErrInvalidArgument, "argument v must not be nil") +func LoadConfig(path string, out *Config) error { + if out == nil { + return errors.Wrapf(errdefs.ErrInvalidArgument, "argument out must not be nil") } - md, err := toml.DecodeFile(path, v) + + var ( + loaded = map[string]bool{} + pending = []string{path} + ) + + for len(pending) > 0 { + path, pending = pending[0], pending[1:] + + // Check if a file at the given path already loaded to prevent circular imports + if _, ok := loaded[path]; ok { + continue + } + + config, err := loadConfigFile(path) + if err != nil { + return err + } + + if err := mergeConfig(out, config); err != nil { + return err + } + + imports, err := resolveImports(path, config.Imports) + if err != nil { + return err + } + + loaded[path] = true + pending = append(pending, imports...) + } + + // Fix up the list of config files loaded + out.Imports = []string{} + for path := range loaded { + out.Imports = append(out.Imports, path) + } + + return out.ValidateV2() +} + +// loadConfigFile decodes a TOML file at the given path +func loadConfigFile(path string) (*Config, error) { + config := &Config{} + md, err := toml.DecodeFile(path, &config) if err != nil { - return err + return nil, err } - v.md = md - return v.ValidateV2() + config.md = md + return config, nil +} + +// resolveImports resolves import strings list to absolute paths list: +// - If path contains *, glob pattern matching applied +// - Non abs path is relative to parent config file directory +// - Abs paths returned as is +func resolveImports(parent string, imports []string) ([]string, error) { + var out []string + + for _, path := range imports { + if strings.Contains(path, "*") { + matches, err := filepath.Glob(path) + if err != nil { + return nil, err + } + + out = append(out, matches...) + } else { + path = filepath.Clean(path) + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(parent), path) + } + + out = append(out, path) + } + } + + return out, nil +} + +// mergeConfig merges Config structs with the following rules: +// 'to' 'from' 'result' overwrite? +// "" "value" "value" yes +// "value" "" "value" no +// 1 0 1 no +// 0 1 1 yes +// []{"1"} []{"2"} []{"2"} yes +// []{"1"} []{} []{"1"} no +func mergeConfig(to, from *Config) error { + return mergo.Merge(to, from, func(config *mergo.Config) { + config.Overwrite = true + }) } // V1DisabledFilter matches based on ID diff --git a/services/server/config/config_test.go b/services/server/config/config_test.go new file mode 100644 index 000000000..56f9cb030 --- /dev/null +++ b/services/server/config/config_test.go @@ -0,0 +1,163 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "gotest.tools/assert" +) + +func TestMergeConfigs(t *testing.T) { + a := &Config{ + Version: 2, + Root: "old_root", + RequiredPlugins: []string{"old_plugin"}, + DisabledPlugins: []string{"old_plugin"}, + State: "old_state", + OOMScore: 1, + } + + b := &Config{ + Root: "new_root", + RequiredPlugins: []string{"new_plugin1", "new_plugin2"}, + OOMScore: 2, + } + + err := mergeConfig(a, b) + assert.NilError(t, err) + + assert.Equal(t, a.Version, 2) + assert.Equal(t, a.Root, "new_root") + assert.Equal(t, a.State, "old_state") + assert.Equal(t, a.OOMScore, 2) + assert.DeepEqual(t, a.RequiredPlugins, []string{"new_plugin1", "new_plugin2"}) + assert.DeepEqual(t, a.DisabledPlugins, []string{"old_plugin"}) +} + +func TestResolveImports(t *testing.T) { + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + for _, filename := range []string{"config_1.toml", "config_2.toml", "test.toml"} { + err = ioutil.WriteFile(filepath.Join(tempDir, filename), []byte(""), 0600) + assert.NilError(t, err) + } + + imports, err := resolveImports(filepath.Join(tempDir, "root.toml"), []string{ + filepath.Join(tempDir, "config_*.toml"), // Glob + filepath.Join(tempDir, "./test.toml"), // Path clean up + "current.toml", // Resolve current working dir + }) + assert.NilError(t, err) + + assert.DeepEqual(t, imports, []string{ + filepath.Join(tempDir, "config_1.toml"), + filepath.Join(tempDir, "config_2.toml"), + filepath.Join(tempDir, "test.toml"), + filepath.Join(tempDir, "current.toml"), + }) +} + +func TestLoadSingleConfig(t *testing.T) { + data := ` +version = 2 +root = "/var/lib/containerd" +` + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + path := filepath.Join(tempDir, "config.toml") + err = ioutil.WriteFile(path, []byte(data), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(path, &out) + assert.NilError(t, err) + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) +} + +func TestLoadConfigWithImports(t *testing.T) { + data1 := ` +version = 2 +root = "/var/lib/containerd" +imports = ["data2.toml"] +` + + data2 := ` +disabled_plugins = ["io.containerd.v1.xyz"] +` + + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600) + assert.NilError(t, err) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out) + assert.NilError(t, err) + + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) + assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins) +} + +func TestLoadConfigWithCircularImports(t *testing.T) { + data1 := ` +version = 2 +root = "/var/lib/containerd" +imports = ["data2.toml", "data1.toml"] +` + + data2 := ` +disabled_plugins = ["io.containerd.v1.xyz"] +imports = ["data1.toml", "data2.toml"] +` + tempDir, err := ioutil.TempDir("", "containerd_") + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data1.toml"), []byte(data1), 0600) + assert.NilError(t, err) + + err = ioutil.WriteFile(filepath.Join(tempDir, "data2.toml"), []byte(data2), 0600) + assert.NilError(t, err) + + var out Config + err = LoadConfig(filepath.Join(tempDir, "data1.toml"), &out) + assert.NilError(t, err) + + assert.Equal(t, 2, out.Version) + assert.Equal(t, "/var/lib/containerd", out.Root) + assert.DeepEqual(t, []string{"io.containerd.v1.xyz"}, out.DisabledPlugins) + + assert.DeepEqual(t, []string{ + filepath.Join(tempDir, "data1.toml"), + filepath.Join(tempDir, "data2.toml"), + }, out.Imports) +}