Merge pull request #572 from Random-Liu/add-runtime-cgroup

Add runtime cgroup and fix a cli panic.

cluster/gce/cloud-init/master.yaml

@@ -31,6 +31,9 @@ write_files:
       # installed by cloud-init
       oom_score = -999
 
+      [cgroup]
+        path = "/runtime"
+
       [plugins.linux]
         shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
         runtime = "/home/cri-containerd/usr/local/sbin/runc"
@@ -84,7 +87,8 @@ write_files:
       ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
         --log-level=debug \
         --network-bin-dir=/home/cri-containerd/opt/cni/bin \
-        --network-conf-dir=/home/cri-containerd/etc/cni/net.d
+        --network-conf-dir=/home/cri-containerd/etc/cni/net.d \
+        --cgroup-path=/runtime
 
       [Install]
       WantedBy=cri-containerd.target

cluster/gce/cloud-init/node.yaml

@@ -34,6 +34,9 @@ write_files:
       # installed by cloud-init
       oom_score = -999
 
+      [cgroup]
+        path = "/runtime"
+
       [plugins.linux]
         shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
         runtime = "/home/cri-containerd/usr/local/sbin/runc"
@@ -87,7 +90,8 @@ write_files:
       ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
         --log-level=debug \
         --network-bin-dir=/home/kubernetes/bin \
-        --network-conf-dir=/etc/cni/net.d
+        --network-conf-dir=/etc/cni/net.d \
+        --cgroup-path=/runtime
 
       [Install]
       WantedBy=cri-containerd.target

cmd/cri-containerd/cri_containerd.go

@@ -116,7 +116,7 @@ func main() {
 		}
 
 		logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath)
-		s, err := server.NewCRIContainerdService(o.CRIConfig)
+		s, err := server.NewCRIContainerdService(o.Config)
 		if err != nil {
 			return fmt.Errorf("failed to create CRI containerd service: %v", err)
 		}

cmd/cri-containerd/options/options.go

@@ -66,7 +66,7 @@ type CniConfig struct {
 }
 
 // PluginConfig contains toml config related to CRI plugin,
-// it is a subset of CRIConfig.
+// it is a subset of Config.
 type PluginConfig struct {
 	// ContainerdConfig contains config related to containerd
 	ContainerdConfig `toml:"containerd" json:"containerd,omitempty"`
@@ -92,10 +92,10 @@ type PluginConfig struct {
 	EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"`
 }
 
-// CRIConfig contains toml config related to CRI service.
+// Config contains toml config related cri-containerd daemon.
 // TODO(random-liu): Make this an internal config object when we no longer support cri-containerd
 // standalone mode. At that time, we can clean this up.
-type CRIConfig struct {
+type Config struct {
 	// PluginConfig is the config for CRI plugin.
 	PluginConfig
 	// ContainerdRootDir is the root directory path for containerd.
@@ -109,23 +109,20 @@ type CRIConfig struct {
 	// RootDir is the root directory path for managing cri-containerd files
 	// (metadata checkpoint etc.)
 	RootDir string `toml:"root_dir" json:"rootDir,omitempty"`
-}
+	// TODO(random-liu): Remove following fields when we no longer support cri-containerd
-
+	// standalone mode.
-// Config contains toml config related cri-containerd daemon.
-type Config struct {
-	CRIConfig `toml:"-"`
 	// CgroupPath is the path for the cgroup that cri-containerd is placed in.
-	CgroupPath string `toml:"cgroup_path"`
+	CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"`
 	// OOMScore adjust the cri-containerd's oom score
-	OOMScore int `toml:"oom_score"`
+	OOMScore int `toml:"oom_score" json:"oomScore,omitempty"`
 	// EnableProfiling is used for enable profiling via host:port/debug/pprof/
-	EnableProfiling bool `toml:"profiling"`
+	EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"`
 	// ProfilingPort is the port for profiling via host:port/debug/pprof/
-	ProfilingPort string `toml:"profiling_port"`
+	ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"`
 	// ProfilingAddress is address for profiling via host:port/debug/pprof/
-	ProfilingAddress string `toml:"profiling_addr"`
+	ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"`
 	// LogLevel is the logrus log level.
-	LogLevel string `toml:"log_level"`
+	LogLevel string `toml:"log_level" json:"logLevel,omitempty"`
 }
 
 // CRIContainerdOptions contains cri-containerd command line and toml options.
@@ -243,37 +240,35 @@ func AddGRPCFlags(fs *pflag.FlagSet) (*string, *time.Duration) {
 // DefaultConfig returns default configurations of cri-containerd.
 func DefaultConfig() Config {
 	return Config{
-		CRIConfig: CRIConfig{
+		PluginConfig: PluginConfig{
-			PluginConfig: PluginConfig{
+			CniConfig: CniConfig{
-				CniConfig: CniConfig{
+				NetworkPluginBinDir:  "/opt/cni/bin",
-					NetworkPluginBinDir:  "/opt/cni/bin",
+				NetworkPluginConfDir: "/etc/cni/net.d",
-					NetworkPluginConfDir: "/etc/cni/net.d",
-				},
-				ContainerdConfig: ContainerdConfig{
-					Snapshotter:   containerd.DefaultSnapshotter,
-					Runtime:       "io.containerd.runtime.v1.linux",
-					RuntimeEngine: "",
-					RuntimeRoot:   "",
-				},
-				StreamServerAddress: "",
-				StreamServerPort:    "10010",
-				EnableSelinux:       false,
-				SandboxImage:        "gcr.io/google_containers/pause:3.0",
-				StatsCollectPeriod:  10,
-				SystemdCgroup:       false,
-				SkipImageFSUUID:     false,
-				EnableIPv6DAD:       false,
 			},
-			ContainerdRootDir:  "/var/lib/containerd",
+			ContainerdConfig: ContainerdConfig{
-			ContainerdEndpoint: "/run/containerd/containerd.sock",
+				Snapshotter:   containerd.DefaultSnapshotter,
-			SocketPath:         "/var/run/cri-containerd.sock",
+				Runtime:       "io.containerd.runtime.v1.linux",
-			RootDir:            "/var/lib/cri-containerd",
+				RuntimeEngine: "",
+				RuntimeRoot:   "",
+			},
+			StreamServerAddress: "",
+			StreamServerPort:    "10010",
+			EnableSelinux:       false,
+			SandboxImage:        "gcr.io/google_containers/pause:3.0",
+			StatsCollectPeriod:  10,
+			SystemdCgroup:       false,
+			SkipImageFSUUID:     false,
+			EnableIPv6DAD:       false,
 		},
-		CgroupPath:       "",
+		ContainerdRootDir:  "/var/lib/containerd",
-		OOMScore:         -999,
+		ContainerdEndpoint: "/run/containerd/containerd.sock",
-		EnableProfiling:  true,
+		SocketPath:         "/var/run/cri-containerd.sock",
-		ProfilingPort:    "10011",
+		RootDir:            "/var/lib/cri-containerd",
-		ProfilingAddress: "127.0.0.1",
+		CgroupPath:         "",
-		LogLevel:         "info",
+		OOMScore:           -999,
+		EnableProfiling:    true,
+		ProfilingPort:      "10011",
+		ProfilingAddress:   "127.0.0.1",
+		LogLevel:           "info",
 	}
 }

cri.go

@@ -52,7 +52,7 @@ func init() {
 func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	ctx := ic.Context
 	pluginConfig := ic.Config.(*options.PluginConfig)
-	c := options.CRIConfig{
+	c := options.Config{
 		PluginConfig: *pluginConfig,
 		// This is a hack. We assume that containerd root directory
 		// is one level above plugin directory.

pkg/server/service.go

@@ -75,7 +75,7 @@ type CRIContainerdService interface {
 // criContainerdService implements CRIContainerdService.
 type criContainerdService struct {
 	// config contains all configurations.
-	config options.CRIConfig
+	config options.Config
 	// imageFSUUID is the device uuid of image filesystem.
 	imageFSUUID string
 	// apparmorEnabled indicates whether apparmor is enabled.
@@ -114,7 +114,7 @@ type criContainerdService struct {
 }
 
 // NewCRIContainerdService returns a new instance of CRIContainerdService
-func NewCRIContainerdService(config options.CRIConfig) (CRIContainerdService, error) {
+func NewCRIContainerdService(config options.Config) (CRIContainerdService, error) {
 	var err error
 	c := &criContainerdService{
 		config:             config,

pkg/server/service_test.go

@@ -39,7 +39,7 @@ const (
 // newTestCRIContainerdService creates a fake criContainerdService for test.
 func newTestCRIContainerdService() *criContainerdService {
 	return &criContainerdService{
-		config: options.CRIConfig{
+		config: options.Config{
 			RootDir: testRootDir,
 			PluginConfig: options.PluginConfig{
 				SandboxImage: testSandboxImage,

test/e2e_node/init.yaml

@@ -30,6 +30,9 @@ write_files:
       # installed by cloud-init
       oom_score = -999
 
+      [cgroup]
+        path = "/runtime"
+
       [plugins.linux]
         shim = "/home/cri-containerd/usr/local/bin/containerd-shim"
         runtime = "/home/cri-containerd/usr/local/sbin/runc"
@@ -81,7 +84,8 @@ write_files:
       ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \
         --log-level=debug \
         --network-bin-dir=/home/cri-containerd/opt/cni/bin \
-        --network-conf-dir=/home/cri-containerd/etc/cni/net.d
+        --network-conf-dir=/home/cri-containerd/etc/cni/net.d \
+        --cgroup-path=/runtime
 
       [Install]
       WantedBy=cri-containerd.target