runtime: Add trusted runtime option

Some CRI compatible runtimes may not support provileged operations.
Specifically hypervisor based runtimes (like kata-containers, cc-runtime
and runv) do not support privileged operations like:

- Provide access to the host namespaces
- Create fully privileged containers with access to host devices

Hypervisor based runtimes create container workloads within virtual machines.
When a running host privileged containers using them,
they wont provide support to requested the privileged opertations.

This commits add the new options to define two runtimes:

Trusted runtime : Used when a privileged container is requested.
Default runtime : for non-privileged workloads.

A container that belongs to a privileged pod will inherent this property
an will be created with the trusted runtime.

- Add options to define trusted runtime
- Add logic to decide if a sanbox is trusted
- Export annotation containers below to a trusted sandbox

Signed-off-by: Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>

pkg/annotations/annotations.go

@@ -31,4 +31,7 @@ const (
 
 	// SandboxID is the sandbox ID annotation
 	SandboxID = "io.kubernetes.cri.sandbox-id"
+
+	// PrivilegedSandbox is the privileged annotation
+	PrivilegedSandbox = "io.kubernetes.cri.privileged-sandbox"
 )

pkg/config/config.go

@@ -18,20 +18,24 @@ package config
 
 import "github.com/containerd/containerd"
 
+// Runtime  struct to contain the type(ID), engine, and root variables for a default and a privileged runtime
+type Runtime struct {
+	//Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+	Type string `toml:"runtime_type" json:"runtimeType,omitempty"`
+	// Engine is the name of the runtime engine used by containerd.
+	Engine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
+	// Root is the directory used by containerd for runtime state.
+	Root string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
+}
+
 // ContainerdConfig contains toml config related to containerd
 type ContainerdConfig struct {
 	// Snapshotter is the snapshotter used by containerd.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
-	// Runtime is the runtime to use in containerd. We may support
+	// DefaultRuntime is the runtime to use in containerd.
-	// other runtimes in the future.
+	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime,omitempty"`
-	Runtime string `toml:"runtime" json:"runtime,omitempty"`
+	// PrivilegedRuntime is a non-secure runtime used only to run trusted workloads on it
-	// RuntimeEngine is the name of the runtime engine used by containerd.
+	PrivilegedRuntime Runtime `toml:"privileged_runtime" json:"privilegedRuntime,omitempty"`
-	// Containerd default should be "runc"
-	// We may support other runtime engines in the future.
-	RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
-	// RuntimeRoot is the directory used by containerd for runtime state.
-	// Containerd default should be "/run/containerd/runc"
-	RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
 }
 
 // CniConfig contains toml config related to cni
@@ -102,9 +106,16 @@ func DefaultConfig() PluginConfig {
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter: containerd.DefaultSnapshotter,
-			Runtime:       "io.containerd.runtime.v1.linux",
+			DefaultRuntime: Runtime{
-			RuntimeEngine: "",
+				Type:   "io.containerd.runtime.v1.linux",
-			RuntimeRoot:   "",
+				Engine: "",
+				Root:   "",
+			},
+			PrivilegedRuntime: Runtime{
+				Type:   "io.containerd.runtime.v1.linux",
+				Engine: "",
+				Root:   "",
+			},
 		},
 		StreamServerAddress: "",
 		StreamServerPort:    "10010",

pkg/server/container_create.go

@@ -87,6 +87,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	}
 	sandboxPid := s.Pid()
 
+	trusted := sandbox.Config.Annotations[annotations.PrivilegedSandbox] == "true"
+	containerRuntime := c.getRuntime(trusted)
+
 	// Generate unique id and name for the container and reserve the name.
 	// Reserve the container name to avoid concurrent `CreateContainer` request creating
 	// the same container.
@@ -227,10 +230,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	opts = append(opts,
 		containerd.WithSpec(spec, specOpts...),
 		containerd.WithRuntime(
-			c.config.ContainerdConfig.Runtime,
+			containerRuntime.Type,
 			&runctypes.RuncOptions{
-				Runtime:       c.config.ContainerdConfig.RuntimeEngine,
+				Runtime:       containerRuntime.Engine,
-				RuntimeRoot:   c.config.ContainerdConfig.RuntimeRoot,
+				RuntimeRoot:   containerRuntime.Root,
 				SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
 		containerd.WithContainerLabels(containerLabels),
 		containerd.WithContainerExtension(containerMetadataExtension, &meta))

pkg/server/helpers.go

@@ -35,9 +35,11 @@ import (
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
+	criconfig "github.com/containerd/cri/pkg/config"
 	"github.com/containerd/cri/pkg/store"
 	imagestore "github.com/containerd/cri/pkg/store/image"
 	"github.com/containerd/cri/pkg/util"
@@ -407,3 +409,18 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
 		"IgnoreUnknown":              "1",
 	}
 }
+
+// getRuntime returns the runtime configuration
+// If the container is privileged, it will return
+// the privileged runtime else not.
+func (c *criService) getRuntime(privileged bool) (runtime criconfig.Runtime) {
+	runtime = c.config.ContainerdConfig.DefaultRuntime
+
+	if privileged && c.config.ContainerdConfig.PrivilegedRuntime.Engine != "" {
+		runtime = c.config.ContainerdConfig.PrivilegedRuntime
+	}
+
+	logrus.Debugf("runtime=%s(%s), runtime root='%s', privileged='%v'", runtime.Type, runtime.Engine, runtime.Root, privileged)
+
+	return runtime
+}

pkg/server/sandbox_run.go

@@ -49,6 +49,32 @@ func init() {
 		"github.com/containerd/cri/pkg/store/sandbox", "Metadata")
 }
 
+// privilegedSandbox returns true if the sandbox configuration
+// requires additional host privileges for the sandbox.
+func privilegedSandbox(req *runtime.RunPodSandboxRequest) bool {
+	securityContext := req.GetConfig().GetLinux().GetSecurityContext()
+	if securityContext == nil {
+		return false
+	}
+
+	if securityContext.Privileged {
+		return true
+	}
+
+	namespaceOptions := securityContext.GetNamespaceOptions()
+	if namespaceOptions == nil {
+		return false
+	}
+
+	if namespaceOptions.Network == runtime.NamespaceMode_NODE ||
+		namespaceOptions.Pid == runtime.NamespaceMode_NODE ||
+		namespaceOptions.Ipc == runtime.NamespaceMode_NODE {
+		return true
+	}
+
+	return false
+}
+
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
 // the sandbox is in ready state.
 func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
@@ -130,6 +156,15 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		}()
 	}
 
+	privileged := privilegedSandbox(r)
+	containerRuntime := c.getRuntime(privileged)
+
+	if sandbox.Config.Annotations == nil {
+		sandbox.Config.Annotations = make(map[string]string)
+	}
+
+	sandbox.Config.Annotations[annotations.PrivilegedSandbox] = fmt.Sprintf("%v", privileged)
+
 	// Create sandbox container.
 	spec, err := c.generateSandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath)
 	if err != nil {
@@ -162,10 +197,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		containerd.WithContainerLabels(sandboxLabels),
 		containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
 		containerd.WithRuntime(
-			c.config.ContainerdConfig.Runtime,
+			containerRuntime.Type,
 			&runctypes.RuncOptions{
-				Runtime:       c.config.ContainerdConfig.RuntimeEngine,
+				Runtime:       containerRuntime.Engine,
-				RuntimeRoot:   c.config.ContainerdConfig.RuntimeRoot,
+				RuntimeRoot:   containerRuntime.Root,
 				SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause
 
 	container, err := c.client.NewContainer(ctx, id, opts...)