add config file for cri-containerd

fix #182

Signed-off-by: yanxuean <yan.xuean@zte.com.cn>

cmd/cri-containerd/cri_containerd.go

@@ -20,12 +20,13 @@ import (
 	"os"
 
 	"github.com/golang/glog"
-	"github.com/kubernetes-incubator/cri-containerd/cmd/cri-containerd/options"
-	"github.com/kubernetes-incubator/cri-containerd/pkg/server"
-	"github.com/kubernetes-incubator/cri-containerd/pkg/version"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/spf13/pflag"
 	"k8s.io/kubernetes/pkg/util/interrupt"
+
+	"github.com/kubernetes-incubator/cri-containerd/cmd/cri-containerd/options"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/server"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/version"
 )
 
 func main() {
@@ -35,7 +36,7 @@ func main() {
 		glog.Exitf("Failed to init CRI containerd flags: %v", err)
 	}
 
-	glog.V(2).Infof("Run cri-containerd %#v", o)
+	glog.V(2).Infof("Run cri-containerd %+v", o)
 	if o.PrintVersion {
 		version.PrintVersion()
 		os.Exit(0)
@@ -56,6 +57,7 @@ func main() {
 		o.StreamServerAddress,
 		o.StreamServerPort,
 		o.CgroupPath,
+		o.SandboxImage,
 	)
 	if err != nil {
 		glog.Exitf("Failed to create CRI containerd service %+v: %v", o, err)

cmd/cri-containerd/options/options.go

@@ -18,53 +18,61 @@ package options
 
 import (
 	"flag"
+	"os"
+
 	"github.com/BurntSushi/toml"
 	"github.com/containerd/containerd"
 	"github.com/spf13/pflag"
-	"os"
-	"time"
 )
 
-const (
+// configFilePathArgName is the path to the config file.
-	// "default path to the config file"
+const configFilePathArgName = "config"
-	configFilePathArgName = "config-file-path"
+
-)
+// ContainerdConfig contains config related to containerd
+type ContainerdConfig struct {
+	// ContainerdSnapshotter is the snapshotter used by containerd.
+	ContainerdSnapshotter string `toml:"snapshotter"`
+	// ContainerdEndpoint is the containerd endpoint path.
+	ContainerdEndpoint string `toml:"endpoint"`
+}
+
+// CniConfig contains config related to cni
+type CniConfig struct {
+	// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
+	NetworkPluginBinDir string `toml:"bin_dir"`
+	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
+	NetworkPluginConfDir string `toml:"conf_dir"`
+}
 
 // Config contains cri-containerd toml config
 type Config struct {
+	// ContainerdConfig contains config related to containerd
+	ContainerdConfig `toml:"containerd"`
+	// CniConfig contains config related to cni
+	CniConfig `toml:"cni"`
 	// SocketPath is the path to the socket which cri-containerd serves on.
-	SocketPath string `toml:"socketpath"`
+	SocketPath string `toml:"socket_path"`
 	// RootDir is the root directory path for managing cri-containerd files
 	// (metadata checkpoint etc.)
-	RootDir string `toml: "rootdir"`
+	RootDir string `toml:"root_dir"`
-	// ContainerdSnapshotter is the snapshotter used by containerd.
-	ContainerdSnapshotter string `toml: "containerdsnapshotter"`
-	// ContainerdEndpoint is the containerd endpoint path.
-	ContainerdEndpoint string `toml:"containerdendpoint"`
-	// ContainerdConnectionTimeout is the connection timeout for containerd client.
-	ContainerdConnectionTimeout time.Duration `toml: "containerdconnectiontimeout"`
-	// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
-	NetworkPluginBinDir string `toml:"networkpluginbindir"`
-	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
-	NetworkPluginConfDir string `toml:"networkpluginconfdir"`
 	// StreamServerAddress is the ip address streaming server is listening on.
-	StreamServerAddress string `toml:"streamserveraddress"`
+	StreamServerAddress string `toml:"stream_server_address"`
 	// StreamServerPort is the port streaming server is listening on.
-	StreamServerPort string `toml: "streamserverport"`
+	StreamServerPort string `toml:"stream_server_port"`
 	// CgroupPath is the path for the cgroup that cri-containerd is placed in.
-	CgroupPath string `toml: "cgrouppath"`
+	CgroupPath string `toml:"cgroup_path"`
 	// EnableSelinux indicates to enable the selinux support
-	EnableSelinux bool `toml: "enableselinux"`
+	EnableSelinux bool `toml:"enable_selinux"`
+	// SandboxImage is the image used by sandbox container.
+	SandboxImage string `toml:"sandbox_image"`
 }
 
 // CRIContainerdOptions contains cri-containerd command line and toml options.
 type CRIContainerdOptions struct {
 	// Config contains cri-containerd toml config
 	Config
-
 	// Path to the TOML config file
 	ConfigFilePath string
-
 	// PrintVersion indicates to print version information of cri-containerd.
 	PrintVersion bool
 }
@@ -77,7 +85,7 @@ func NewCRIContainerdOptions() *CRIContainerdOptions {
 // AddFlags adds cri-containerd command line options to pflag.
 func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&c.ConfigFilePath, configFilePathArgName,
-		"/etc/cri-containerd/config.toml", "Path to the config file")
+		"/etc/cri-containerd/config.toml", "Path to the config file.")
 	fs.StringVar(&c.SocketPath, "socket-path",
 		"/var/run/cri-containerd.sock", "Path to the socket which cri-containerd serves on.")
 	fs.StringVar(&c.RootDir, "root-dir",
@@ -99,6 +107,8 @@ func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&c.CgroupPath, "cgroup-path", "", "The cgroup that cri-containerd is part of. By default cri-containerd is not placed in a cgroup")
 	fs.BoolVar(&c.EnableSelinux, "selinux-enabled",
 		false, "Enable selinux support.")
+	fs.StringVar(&c.SandboxImage, "sandbox-image",
+		"gcr.io/google_containers/pause:3.0", "The image used by sandbox container.")
 }
 
 // InitFlags must be called after adding all cli options flags are defined and
@@ -110,13 +120,13 @@ func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
 	fs.AddGoFlagSet(flag.CommandLine)
 
 	commandline := os.Args[1:]
-	err := fs.Parse(commandline) //this time:  config = default + commandline(on top)
+	err := fs.Parse(commandline)
 	if err != nil {
 		return err
 	}
 
-	// will try default config file when user have not seted it in cli
+	// Load default config file if none provided
-	err = loadConfigFile(c.ConfigFilePath, &c.Config) //config = default + commandline + configfile(on top)
+	_, err = toml.DecodeFile(c.ConfigFilePath, &c.Config)
 	if err != nil {
 		// the absence of default config file is normal case.
 		if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) {
@@ -125,15 +135,13 @@ func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error {
 		return err
 	}
 
-	err = fs.Parse(commandline) //config = default + commandline + configfile + commandline(on top)
+	// What is the reason for applying the command line twice?
-	return err
+	// Because the values from command line has the highest priority.
-}
+	// So I must get the path of toml configuration file from command line,
-
+	// it trigger the first parse.
-func loadConfigFile(fpath string, v *Config) error {
+	// The first parse generate the the default value and the value from command line at the same time.
-	if v == nil {
+	// But the priority of toml config value is more higher than of default value,
-		v = &Config{}
+	// So I have not another way to insert toml config value between default value and command line value.
-	}
+	// So I trigger twice parses, one for default value, one for commandline value.
-
+	return fs.Parse(commandline)
-	_, err := toml.DecodeFile(fpath, v)
-	return err
 }

pkg/opts/task.go

@@ -1,22 +0,0 @@
-package opts
-
-import (
-	"context"
-
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/linux/runcopts"
-)
-
-// WithContainerdShimCgroup returns function that sets the containerd
-// shim cgroup path
-func WithContainerdShimCgroup(path string) containerd.NewTaskOpts {
-	return func(_ context.Context, _ *containerd.Client, r *containerd.TaskInfo) error {
-		r.Options = &runcopts.CreateOptions{
-			ShimCgroup: path,
-		}
-		return nil
-	}
-}
-
-//TODO: Since Options is an interface different WithXXX will be needed to set different
-// combinations of CreateOptions.

pkg/server/container_start.go

@@ -27,7 +27,6 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	criopts "github.com/kubernetes-incubator/cri-containerd/pkg/opts"
 	cio "github.com/kubernetes-incubator/cri-containerd/pkg/server/io"
 	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
@@ -143,11 +142,8 @@ func (c *criContainerdService) startContainer(ctx context.Context,
 		}
 		return cntr.IO, nil
 	}
-	var taskOpts []containerd.NewTaskOpts
+
-	if cgroup := sandboxConfig.GetLinux().GetCgroupParent(); cgroup != "" {
+	task, err := container.NewTask(ctx, ioCreation)
-		taskOpts = append(taskOpts, criopts.WithContainerdShimCgroup(cgroup))
-	}
-	task, err := container.NewTask(ctx, ioCreation, taskOpts...)
 	if err != nil {
 		return fmt.Errorf("failed to create containerd task: %v", err)
 	}

pkg/server/helpers.go

@@ -57,8 +57,6 @@ const (
 )
 
 const (
-	// defaultSandboxImage is the image used by sandbox container.
-	defaultSandboxImage = "gcr.io/google_containers/pause:3.0"
 	// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
 	defaultSandboxOOMAdj = -998
 	// defaultSandboxCPUshares is default cpu shares for sandbox container.

pkg/server/sandbox_run.go

@@ -31,7 +31,6 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	criopts "github.com/kubernetes-incubator/cri-containerd/pkg/opts"
 	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )
@@ -182,11 +181,7 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	glog.V(5).Infof("Create sandbox container (id=%q, name=%q).",
 		id, name)
 	// We don't need stdio for sandbox container.
-	var taskOpts []containerd.NewTaskOpts
+	task, err := container.NewTask(ctx, containerd.NullIO)
-	if cgroup := config.GetLinux().GetCgroupParent(); cgroup != "" {
-		taskOpts = append(taskOpts, criopts.WithContainerdShimCgroup(cgroup))
-	}
-	task, err := container.NewTask(ctx, containerd.NullIO, taskOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create task for sandbox %q: %v", id, err)
 	}
@@ -258,6 +253,7 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
 	// When cgroup parent is not set, containerd-shim will create container in a child cgroup
 	// of the cgroup itself is in.
 	// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
+
 	// Set namespace options.
 	securityContext := config.GetLinux().GetSecurityContext()
 	nsOptions := securityContext.GetNamespaceOptions()

pkg/server/service.go

@@ -111,7 +111,8 @@ func NewCRIContainerdService(
 	networkPluginConfDir,
 	streamAddress,
 	streamPort string,
-	cgroupPath string) (CRIContainerdService, error) {
+	cgroupPath string,
+	sandboxImage string) (CRIContainerdService, error) {
 	// TODO(random-liu): [P2] Recover from runtime state and checkpoint.
 
 	client, err := containerd.New(containerdEndpoint, containerd.WithDefaultNamespace(k8sContainerdNamespace))
@@ -129,7 +130,7 @@ func NewCRIContainerdService(
 		serverAddress:       serverAddress,
 		os:                  osinterface.RealOS{},
 		rootDir:             rootDir,
-		sandboxImage:        defaultSandboxImage,
+		sandboxImage:        sandboxImage,
 		snapshotter:         containerdSnapshotter,
 		sandboxStore:        sandboxstore.NewStore(),
 		containerStore:      containerstore.NewStore(),

vendor.conf

@@ -8,7 +8,7 @@ github.com/containerd/cgroups 7a5fdd8330119dc70d850260db8f3594d89d6943
 github.com/coreos/go-systemd d2196463941895ee908e13531a23a39feb9e1243
 github.com/containernetworking/cni v0.6.0
 github.com/containernetworking/plugins v0.6.0
-github.com/cri-o/ocicni 0f90d35d89e9ab7e972a9edeb36b0aaffa250335
+github.com/cri-o/ocicni 4c2bf6d5198c307f76312f8fc7ef654cfd41d303
 github.com/davecgh/go-spew v1.1.0
 github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621
 github.com/docker/docker cc4da8112814cdbb00dbf23370f9ed764383de1f

vendor/github.com/cri-o/ocicni/ocicni.go

@@ -55,9 +55,8 @@ func (plugin *cniNetworkPlugin) monitorNetDir() {
 				}
 
 				if err = plugin.syncNetworkConfig(); err == nil {
-					logrus.Debugf("CNI asynchronous setting succeeded")
+					logrus.Infof("CNI asynchronous setting succeeded")
-					close(plugin.monitorNetDirChan)
+					continue
-					return
 				}
 
 				logrus.Errorf("CNI setting failed, continue monitoring: %v", err)