Merge pull request #1125 from Random-Liu/add-default-runtime-name

Add default runtime name

cluster/gce/configure.sh

@@ -176,9 +176,11 @@ disabled_plugins = ["restart"]
   conf_template = "${cni_template_path}"
 [plugins.cri.registry.mirrors."docker.io"]
   endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
-[plugins.cri.containerd.default_runtime]
+[plugins.cri.containerd]
+  default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}"
+[plugins.cri.containerd.runtimes.runc]
   runtime_type = "io.containerd.runc.v1"
-[plugins.cri.containerd.default_runtime.options]
+[plugins.cri.containerd.runtimes.runc.options]
   BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc"
 EOF
 chmod 644 "${config_path}"

cri.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cri
 
 import (
+	"context"
 	"flag"
 	"path/filepath"
 	"time"
@@ -73,7 +74,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	}
 	log.G(ctx).Infof("Start cri plugin with config %+v", c)
 
-	if err := validateConfig(&c); err != nil {
+	if err := validateConfig(ctx, &c); err != nil {
 		return nil, errors.Wrap(err, "invalid config")
 	}
 
@@ -111,14 +112,36 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 }
 
 // validateConfig validates the given configuration.
-func validateConfig(c *criconfig.Config) error {
+func validateConfig(ctx context.Context, c *criconfig.Config) error {
-	// It is an error to provide both an UntrustedWorkloadRuntime & define an 'untrusted' runtime.
+	if c.ContainerdConfig.Runtimes == nil {
-	if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok {
+		c.ContainerdConfig.Runtimes = make(map[string]criconfig.Runtime)
-		if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
-			return errors.New("conflicting definitions: configuration includes untrusted_workload_runtime and runtimes['untrusted']")
-		}
 	}
 
+	// Validation for deprecated untrusted_workload_runtime.
+	if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
+		log.G(ctx).Warning("`untrusted_workload_runtime` is deprecated, please use `untrusted` runtime in `runtimes` instead")
+		if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok {
+			return errors.Errorf("conflicting definitions: configuration includes both `untrusted_workload_runtime` and `runtimes[%q]`", criconfig.RuntimeUntrusted)
+		}
+		c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted] = c.ContainerdConfig.UntrustedWorkloadRuntime
+	}
+
+	// Validation for deprecated default_runtime field.
+	if c.ContainerdConfig.DefaultRuntime.Type != "" {
+		log.G(ctx).Warning("`default_runtime` is deprecated, please use `default_runtime_name` to reference the default configuration you have defined in `runtimes`")
+		c.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
+		c.ContainerdConfig.Runtimes[criconfig.RuntimeDefault] = c.ContainerdConfig.DefaultRuntime
+	}
+
+	// Validation for default_runtime_name
+	if c.ContainerdConfig.DefaultRuntimeName == "" {
+		return errors.New("`default_runtime_name` is empty")
+	}
+	if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok {
+		return errors.New("no corresponding runtime configured in `runtimes` for `default_runtime_name`")
+	}
+
+	// Validation for stream_idle_timeout
 	if c.StreamIdleTimeout != "" {
 		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {
 			return errors.Wrap(err, "invalid stream idle timeout")

docs/config.md

@@ -80,60 +80,22 @@ The explanation and default value of each configuration item are as follows:
     #   For runtime "io.containerd.runc.v1", use the option `NoPivotRoot`.
     no_pivot = false
 
+    # default_runtime_name is the default runtime name to use.
+    default_runtime_name = "runc"
+
     # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
+    # DEPRECATED: use `default_runtime_name` and `plugins.cri.runtimes` instead.
+    # Remove in containerd 1.4.
     [plugins.cri.containerd.default_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = "io.containerd.runtime.v1.linux"
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      # This only works for runtime type "io.containerd.runtime.v1.linux".
-      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
-      #   For runtime "io.containerd.runc.v1", use the option `BinaryName`.
-      runtime_engine = ""
-
-      # runtime_root is the directory used by containerd for runtime state.
-      # This only works for runtime type "io.containerd.runtime.v1.linux".
-      # DEPRECATED: use Runtime.Options for runtime specific config for shim v2 runtimes.
-      #   For runtime "io.containerd.runc.v1", use the option `Root`.
-      runtime_root = ""
-
-      # "plugins.cri.containerd.default_runtime.options" is options specific to
-      # the default runtime. The options type for "io.containerd.runtime.v1.linux" is:
-      #   https://github.com/containerd/containerd/blob/v1.2.0-rc.1/runtime/linux/runctypes/runc.pb.go#L40
-      # NOTE: when `options` is specified, all related deprecated options will
-      #   be ignored, including `systemd_cgroup`, `no_pivot`, `runtime_engine`
-      #   and `runtime_root`.
-      [plugins.cri.containerd.default_runtime.options]
-        # Runtime is the binary name of the runtime.
-        Runtime = ""
-
-        # RuntimeRoot is the root directory of the runtime.
-        RuntimeRoot = ""
-
-        # CriuPath is the criu binary path.
-        CriuPath = ""
-
-        # SystemdCgroup enables systemd cgroups.
-        SystemdCgroup = false
 
     # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
-    # DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
+    # DEPRECATED: use `untrusted` runtime in `plugins.cri.runtimes` instead.
-    #   runtime handler named 'untrusted'. It is a configuration error to provide both the (now
+    # Remove in containerd 1.4.
-    #   deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
-    #   'untrusted' workloads at the same time. Please provide one or the other.
     [plugins.cri.containerd.untrusted_workload_runtime]
-      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
-      runtime_type = ""
-
-      # runtime_engine is the name of the runtime engine used by containerd.
-      runtime_engine = ""
-
-      # runtime_root is the directory used by containerd for runtime state.
-      runtime_root = ""
 
     # plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
-    # of runtime configurations, to the matching configurations. In this example,
+    # of runtime configurations, to the matching configurations.
-    # 'runc' is the RuntimeHandler string to match.
+    # In this example, 'runc' is the RuntimeHandler string to match.
     [plugins.cri.containerd.runtimes.runc]
       # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
       runtime_type = "io.containerd.runc.v1"
@@ -205,3 +167,25 @@ The explanation and default value of each configuration item are as follows:
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://registry-1.docker.io", ]
 ```
+
+## Untrusted Workload
+
+The recommended way to run untrusted workload is to use
+[`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/) api
+introduced in Kubernetes 1.12 to select RuntimeHandlers configured to run
+untrusted workload in `plugins.cri.containerd.runtimes`.
+
+However, if you are using the legacy `io.kubernetes.cri.untrusted-workload`pod annotation
+to request a pod be run using a runtime for untrusted workloads, the RuntimeHandler
+`plugins.cri.containerd.runtimes.untrusted` must be defined first. When the annotation
+`io.kubernetes.cri.untrusted-workload` is set to `true` the `untrusted` runtime will be
+used. For example, see
+[Create an untrusted pod using Kata Containers](https://github.com/kata-containers/documentation/blob/master/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#create-an-untrusted-pod-using-kata-containers).
+
+## Deprecation
+The config options of the CRI plugin follow the [Kubernetes deprecation
+policy of "admin-facing CLI components"](https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli).
+
+In summary, when a config option is announced to be deprecated:
+* It is kept functional for 6 months or 1 release (whichever is longer);
+* A warning is emitted when it is used.

pkg/config/config.go

@@ -47,14 +47,14 @@ type Runtime struct {
 type ContainerdConfig struct {
 	// Snapshotter is the snapshotter used by containerd.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
+	// DefaultRuntimeName is the default runtime name to use from the runtimes table.
+	DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"`
 	// DefaultRuntime is the default runtime to use in containerd.
 	// This runtime is used when no runtime handler (or the empty string) is provided.
+	// DEPRECATED: use DefaultRuntimeName instead. Remove in containerd 1.4.
 	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"`
 	// UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it.
-	// DEPRECATED: use Runtimes instead. If provided, this runtime is mapped to the runtime handler
+	// DEPRECATED: use `untrusted` runtime in Runtimes instead. Remove in containerd 1.4.
-	//     named 'untrusted'. It is a configuration error to provide both the (now deprecated)
-	//     UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for 'untrusted'
-	//     workloads at the same time. Please provide one or the other.
 	UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"`
 	// Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime
 	// configurations, to the matching configurations.
@@ -196,12 +196,13 @@ func DefaultConfig() PluginConfig {
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter:        containerd.DefaultSnapshotter,
-			DefaultRuntime: Runtime{
+			DefaultRuntimeName: "runc",
-				Type:   "io.containerd.runtime.v1.linux",
-				Engine: "",
-				Root:   "",
-			},
 			NoPivot:            false,
+			Runtimes: map[string]Runtime{
+				"runc": {
+					Type: "io.containerd.runc.v1",
+				},
+			},
 		},
 		StreamServerAddress: "127.0.0.1",
 		StreamServerPort:    "0",
@@ -229,4 +230,6 @@ func DefaultConfig() PluginConfig {
 const (
 	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
 	RuntimeUntrusted = "untrusted"
+	// RuntimeDefault is the implicit runtime defined for ContainerdConfig.DefaultRuntime
+	RuntimeDefault = "default"
 )

pkg/server/helpers_test.go

@@ -212,7 +212,8 @@ func TestGenerateRuntimeOptions(t *testing.T) {
 systemd_cgroup = true
 [containerd]
   no_pivot = true
-[containerd.default_runtime]
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
   runtime_type = "` + linuxRuntime + `"
 [containerd.runtimes.runc]
   runtime_type = "` + runcRuntimeV1 + `"
@@ -223,11 +224,12 @@ systemd_cgroup = true
 systemd_cgroup = true
 [containerd]
   no_pivot = true
-[containerd.default_runtime]
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
   runtime_type = "` + linuxRuntime + `"
-[containerd.default_runtime.options]
+[containerd.runtimes.legacy.options]
-  Runtime = "default"
+  Runtime = "legacy"
-  RuntimeRoot = "/default"
+  RuntimeRoot = "/legacy"
 [containerd.runtimes.runc]
   runtime_type = "` + runcRuntimeV1 + `"
 [containerd.runtimes.runc.options]
@@ -246,8 +248,8 @@ systemd_cgroup = true
 	require.NoError(t, err)
 	_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
 	require.NoError(t, err)
-	require.Len(t, nilOptsConfig.Runtimes, 2)
+	require.Len(t, nilOptsConfig.Runtimes, 3)
-	require.Len(t, nonNilOptsConfig.Runtimes, 2)
+	require.Len(t, nonNilOptsConfig.Runtimes, 3)
 
 	for desc, test := range map[string]struct {
 		r               criconfig.Runtime
@@ -265,7 +267,7 @@ systemd_cgroup = true
 			expectedOptions: nil,
 		},
 		"when options is nil, should use legacy fields for legacy runtime": {
-			r: nilOptsConfig.DefaultRuntime,
+			r: nilOptsConfig.Runtimes["legacy"],
 			c: nilOptsConfig,
 			expectedOptions: &runctypes.RuncOptions{
 				SystemdCgroup: true,
@@ -290,11 +292,11 @@ systemd_cgroup = true
 			},
 		},
 		"when options is not nil, should be able to decode for legacy runtime": {
-			r: nonNilOptsConfig.DefaultRuntime,
+			r: nonNilOptsConfig.Runtimes["legacy"],
 			c: nonNilOptsConfig,
 			expectedOptions: &runctypes.RuncOptions{
-				Runtime:     "default",
+				Runtime:     "legacy",
-				RuntimeRoot: "/default",
+				RuntimeRoot: "/legacy",
 			},
 		},
 	} {

pkg/server/sandbox_run.go

@@ -628,16 +628,11 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 			return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
 		}
 
-		// Handle the deprecated UntrustedWorkloadRuntime.
-		if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
-			return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil
-		}
-
 		runtimeHandler = criconfig.RuntimeUntrusted
 	}
 
 	if runtimeHandler == "" {
-		return c.config.ContainerdConfig.DefaultRuntime, nil
+		runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
 	}
 
 	handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]

pkg/server/sandbox_run_test.go

@@ -683,8 +683,6 @@ func TestGetSandboxRuntime(t *testing.T) {
 	for desc, test := range map[string]struct {
 		sandboxConfig   *runtime.PodSandboxConfig
 		runtimeHandler  string
-		defaultRuntime           criconfig.Runtime
-		untrustedWorkloadRuntime criconfig.Runtime
 		runtimes        map[string]criconfig.Runtime
 		expectErr       bool
 		expectedRuntime criconfig.Runtime
@@ -705,8 +703,10 @@ func TestGetSandboxRuntime(t *testing.T) {
 					annotations.UntrustedWorkload: "true",
 				},
 			},
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
 			expectErr: true,
 		},
 		"should use untrusted workload runtime for untrusted workload": {
@@ -715,14 +715,17 @@ func TestGetSandboxRuntime(t *testing.T) {
 					annotations.UntrustedWorkload: "true",
 				},
 			},
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
 			expectedRuntime: untrustedWorkloadRuntime,
 		},
 		"should use default runtime for regular workload": {
 			sandboxConfig: &runtime.PodSandboxConfig{},
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
 			expectedRuntime: defaultRuntime,
 		},
 		"should use default runtime for trusted workload": {
@@ -731,8 +734,10 @@ func TestGetSandboxRuntime(t *testing.T) {
 					annotations.UntrustedWorkload: "false",
 				},
 			},
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
 			expectedRuntime: defaultRuntime,
 		},
 		"should return error if untrusted workload runtime is required but not configured": {
@@ -741,7 +746,9 @@ func TestGetSandboxRuntime(t *testing.T) {
 					annotations.UntrustedWorkload: "true",
 				},
 			},
-			defaultRuntime: defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
 			expectErr: true,
 		},
 		"should use 'untrusted' runtime for untrusted workload": {
@@ -750,8 +757,10 @@ func TestGetSandboxRuntime(t *testing.T) {
 					annotations.UntrustedWorkload: "true",
 				},
 			},
-			defaultRuntime:  defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			runtimes:        map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime},
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
 			expectedRuntime: untrustedWorkloadRuntime,
 		},
 		"should use 'untrusted' runtime for untrusted workload & handler": {
@@ -761,8 +770,10 @@ func TestGetSandboxRuntime(t *testing.T) {
 				},
 			},
 			runtimeHandler: "untrusted",
-			defaultRuntime:  defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			runtimes:        map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime},
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
 			expectedRuntime: untrustedWorkloadRuntime,
 		},
 		"should return an error if untrusted annotation with conflicting handler": {
@@ -772,24 +783,30 @@ func TestGetSandboxRuntime(t *testing.T) {
 				},
 			},
 			runtimeHandler: "foo",
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault:   defaultRuntime,
-			runtimes:                 map[string]criconfig.Runtime{"foo": fooRuntime},
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
 			expectErr: true,
 		},
 		"should use correct runtime for a runtime handler": {
 			sandboxConfig:  &runtime.PodSandboxConfig{},
 			runtimeHandler: "foo",
-			defaultRuntime:           defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+				criconfig.RuntimeDefault:   defaultRuntime,
-			runtimes:                 map[string]criconfig.Runtime{"foo": fooRuntime},
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
 			expectedRuntime: fooRuntime,
 		},
 		"should return error if runtime handler is required but not configured": {
 			sandboxConfig:  &runtime.PodSandboxConfig{},
 			runtimeHandler: "bar",
-			defaultRuntime: defaultRuntime,
+			runtimes: map[string]criconfig.Runtime{
-			runtimes:       map[string]criconfig.Runtime{"foo": fooRuntime},
+				criconfig.RuntimeDefault: defaultRuntime,
+				"foo":                    fooRuntime,
+			},
 			expectErr: true,
 		},
 	} {
@@ -798,8 +815,7 @@ func TestGetSandboxRuntime(t *testing.T) {
 			cri.config = criconfig.Config{
 				PluginConfig: criconfig.DefaultConfig(),
 			}
-			cri.config.ContainerdConfig.DefaultRuntime = test.defaultRuntime
+			cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
-			cri.config.ContainerdConfig.UntrustedWorkloadRuntime = test.untrustedWorkloadRuntime
 			cri.config.ContainerdConfig.Runtimes = test.runtimes
 			r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
 			assert.Equal(t, test.expectErr, err != nil)