Merge pull request #657 from jcvenegas/trusted-runtime

[WIP]config: Allow to define trusted runtime

hack/test-utils.sh

@@ -18,6 +18,13 @@ source $(dirname "${BASH_SOURCE[0]}")/utils.sh
 
 # RESTART_WAIT_PERIOD is the period to wait before restarting containerd.
 RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10}
+CONTAINERD_CONFIG="--log-level=debug "
+
+# Use a configuration file for containerd.
+CONTAINERD_CONFIG_FILE=${CONTAINERD_CONFIG_FILE:-""}
+if [ -f "${CONTAINERD_CONFIG_FILE}" ]; then
+	CONTAINERD_CONFIG+="--config $CONTAINERD_CONFIG_FILE"
+fi
 
 CONTAINERD_SOCK=/run/containerd/containerd.sock
 
@@ -32,7 +39,7 @@ test_setup() {
     exit 1
   fi
   sudo pkill -x containerd
-  keepalive "sudo ${ROOT}/_output/containerd --log-level=debug" \
+  keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_CONFIG}" \
     ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log &
   containerd_pid=$!
   # Wait for containerd to be running by using the containerd client ctr to check the version

pkg/annotations/annotations.go

@@ -31,4 +31,7 @@ const (
 
 	// SandboxID is the sandbox ID annotation
 	SandboxID = "io.kubernetes.cri.sandbox-id"
+
+	// PrivilegedSandbox is the privileged annotation
+	PrivilegedSandbox = "io.kubernetes.cri.privileged-sandbox"
 )

pkg/config/config.go

@@ -18,20 +18,24 @@ package config
 
 import "github.com/containerd/containerd"
 
+// Runtime  struct to contain the type(ID), engine, and root variables for a default and a privileged runtime
+type Runtime struct {
+	//Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+	Type string `toml:"runtime_type" json:"runtimeType,omitempty"`
+	// Engine is the name of the runtime engine used by containerd.
+	Engine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
+	// Root is the directory used by containerd for runtime state.
+	Root string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
+}
+
 // ContainerdConfig contains toml config related to containerd
 type ContainerdConfig struct {
 	// Snapshotter is the snapshotter used by containerd.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"`
-	// Runtime is the runtime to use in containerd. We may support
-	// other runtimes in the future.
-	Runtime string `toml:"runtime" json:"runtime,omitempty"`
-	// RuntimeEngine is the name of the runtime engine used by containerd.
-	// Containerd default should be "runc"
-	// We may support other runtime engines in the future.
-	RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"`
-	// RuntimeRoot is the directory used by containerd for runtime state.
-	// Containerd default should be "/run/containerd/runc"
-	RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"`
+	// DefaultRuntime is the runtime to use in containerd.
+	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime,omitempty"`
+	// PrivilegedRuntime is a non-secure runtime used only to run trusted workloads on it
+	PrivilegedRuntime Runtime `toml:"privileged_runtime" json:"privilegedRuntime,omitempty"`
 }
 
 // CniConfig contains toml config related to cni
@@ -101,10 +105,17 @@ func DefaultConfig() PluginConfig {
 			NetworkPluginConfDir: "/etc/cni/net.d",
 		},
 		ContainerdConfig: ContainerdConfig{
-			Snapshotter:   containerd.DefaultSnapshotter,
-			Runtime:       "io.containerd.runtime.v1.linux",
-			RuntimeEngine: "",
-			RuntimeRoot:   "",
+			Snapshotter: containerd.DefaultSnapshotter,
+			DefaultRuntime: Runtime{
+				Type:   "io.containerd.runtime.v1.linux",
+				Engine: "",
+				Root:   "",
+			},
+			PrivilegedRuntime: Runtime{
+				Type:   "io.containerd.runtime.v1.linux",
+				Engine: "",
+				Root:   "",
+			},
 		},
 		StreamServerAddress: "",
 		StreamServerPort:    "10010",

pkg/server/container_create.go

@@ -87,6 +87,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	}
 	sandboxPid := s.Pid()
 
+	trusted := sandbox.Config.Annotations[annotations.PrivilegedSandbox] == "true"
+	containerRuntime := c.getRuntime(trusted)
+
 	// Generate unique id and name for the container and reserve the name.
 	// Reserve the container name to avoid concurrent `CreateContainer` request creating
 	// the same container.
@@ -227,10 +230,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	opts = append(opts,
 		containerd.WithSpec(spec, specOpts...),
 		containerd.WithRuntime(
-			c.config.ContainerdConfig.Runtime,
+			containerRuntime.Type,
 			&runctypes.RuncOptions{
-				Runtime:       c.config.ContainerdConfig.RuntimeEngine,
-				RuntimeRoot:   c.config.ContainerdConfig.RuntimeRoot,
+				Runtime:       containerRuntime.Engine,
+				RuntimeRoot:   containerRuntime.Root,
 				SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause
 		containerd.WithContainerLabels(containerLabels),
 		containerd.WithContainerExtension(containerMetadataExtension, &meta))

pkg/server/helpers.go

@@ -35,9 +35,11 @@ import (
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
+	criconfig "github.com/containerd/cri/pkg/config"
 	"github.com/containerd/cri/pkg/store"
 	imagestore "github.com/containerd/cri/pkg/store/image"
 	"github.com/containerd/cri/pkg/util"
@@ -407,3 +409,18 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str
 		"IgnoreUnknown":              "1",
 	}
 }
+
+// getRuntime returns the runtime configuration
+// If the container is privileged, it will return
+// the privileged runtime else not.
+func (c *criService) getRuntime(privileged bool) (runtime criconfig.Runtime) {
+	runtime = c.config.ContainerdConfig.DefaultRuntime
+
+	if privileged && c.config.ContainerdConfig.PrivilegedRuntime.Engine != "" {
+		runtime = c.config.ContainerdConfig.PrivilegedRuntime
+	}
+
+	logrus.Debugf("runtime=%s(%s), runtime root='%s', privileged='%v'", runtime.Type, runtime.Engine, runtime.Root, privileged)
+
+	return runtime
+}

pkg/server/helpers_test.go

@@ -19,10 +19,10 @@ package server
 import (
 	"testing"
 
+	criconfig "github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/util"
 	imagedigest "github.com/opencontainers/go-digest"
 	"github.com/stretchr/testify/assert"
-
-	"github.com/containerd/cri/pkg/util"
 )
 
 // TestGetUserFromImage tests the logic of getting image uid or user name of image user.
@@ -142,3 +142,64 @@ func TestBuildLabels(t *testing.T) {
 	assert.Empty(t, configLabels[containerKindLabel], "should not add new labels into original label")
 	assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
 }
+
+func Test_criService_getRuntime(t *testing.T) {
+
+	const (
+		privilegedWorkload    = true
+		nonPrivilegedWorkload = false
+	)
+
+	nonPrivilegedRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "kata-runtime",
+		Root:   "",
+	}
+
+	privilegedRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "runc",
+		Root:   "",
+	}
+
+	// Crate a configuration that does not specify a privileged runtime
+	// Both privileged and non-privileged workloads are created with the
+	// defaultRuntime (nonPrivilegedRuntime).
+	nonPrivilegedConfig := criService{
+		config: criconfig.Config{
+			PluginConfig: criconfig.DefaultConfig(),
+		},
+	}
+	nonPrivilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime
+
+	// Crate a configuration that specifies a privileged runtime
+	// The privileged workloads are created with the privilegedRuntime
+	// The non-privileged workloads be created with the
+	// defaultRuntime(nonPrivilegedRuntime)
+	privilegedConfig := criService{
+		config: criconfig.Config{
+			PluginConfig: criconfig.DefaultConfig(),
+		},
+	}
+	privilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime
+	privilegedConfig.config.ContainerdConfig.PrivilegedRuntime = privilegedRuntime
+
+	tests := []struct {
+		name        string
+		cri         criService
+		privileged  bool
+		wantRuntime criconfig.Runtime
+	}{
+		{"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, privilegedWorkload, nonPrivilegedRuntime},
+		{"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime},
+		{"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, privilegedWorkload, privilegedRuntime},
+		{"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotRuntime := tt.cri.getRuntime(tt.privileged)
+			assert.Equal(t, tt.wantRuntime, gotRuntime)
+		})
+	}
+}

pkg/server/sandbox_run.go

@@ -49,6 +49,32 @@ func init() {
 		"github.com/containerd/cri/pkg/store/sandbox", "Metadata")
 }
 
+// privilegedSandbox returns true if the sandbox configuration
+// requires additional host privileges for the sandbox.
+func privilegedSandbox(req *runtime.RunPodSandboxRequest) bool {
+	securityContext := req.GetConfig().GetLinux().GetSecurityContext()
+	if securityContext == nil {
+		return false
+	}
+
+	if securityContext.Privileged {
+		return true
+	}
+
+	namespaceOptions := securityContext.GetNamespaceOptions()
+	if namespaceOptions == nil {
+		return false
+	}
+
+	if namespaceOptions.Network == runtime.NamespaceMode_NODE ||
+		namespaceOptions.Pid == runtime.NamespaceMode_NODE ||
+		namespaceOptions.Ipc == runtime.NamespaceMode_NODE {
+		return true
+	}
+
+	return false
+}
+
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
 // the sandbox is in ready state.
 func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
@@ -130,6 +156,15 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		}()
 	}
 
+	privileged := privilegedSandbox(r)
+	containerRuntime := c.getRuntime(privileged)
+
+	if sandbox.Config.Annotations == nil {
+		sandbox.Config.Annotations = make(map[string]string)
+	}
+
+	sandbox.Config.Annotations[annotations.PrivilegedSandbox] = fmt.Sprintf("%v", privileged)
+
 	// Create sandbox container.
 	spec, err := c.generateSandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath)
 	if err != nil {
@@ -162,10 +197,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		containerd.WithContainerLabels(sandboxLabels),
 		containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
 		containerd.WithRuntime(
-			c.config.ContainerdConfig.Runtime,
+			containerRuntime.Type,
 			&runctypes.RuncOptions{
-				Runtime:       c.config.ContainerdConfig.RuntimeEngine,
-				RuntimeRoot:   c.config.ContainerdConfig.RuntimeRoot,
+				Runtime:       containerRuntime.Engine,
+				RuntimeRoot:   containerRuntime.Root,
 				SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause
 
 	container, err := c.client.NewContainer(ctx, id, opts...)

pkg/server/sandbox_run_test.go

@@ -435,3 +435,58 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
 
 // TODO(random-liu): [P1] Add unit test for different error cases to make sure
 // the function cleans up on error properly.
+
+func TestPrivilegedSandbox(t *testing.T) {
+	privilegedContext := runtime.RunPodSandboxRequest{
+		Config: &runtime.PodSandboxConfig{
+			Linux: &runtime.LinuxPodSandboxConfig{
+				SecurityContext: &runtime.LinuxSandboxSecurityContext{
+					Privileged: true,
+				},
+			},
+		},
+	}
+	nonPrivilegedContext := runtime.RunPodSandboxRequest{
+		Config: &runtime.PodSandboxConfig{
+			Linux: &runtime.LinuxPodSandboxConfig{
+				SecurityContext: &runtime.LinuxSandboxSecurityContext{
+					Privileged: false,
+				},
+			},
+		},
+	}
+	hostNamespace := runtime.RunPodSandboxRequest{
+		Config: &runtime.PodSandboxConfig{
+			Linux: &runtime.LinuxPodSandboxConfig{
+				SecurityContext: &runtime.LinuxSandboxSecurityContext{
+					Privileged: false,
+					NamespaceOptions: &runtime.NamespaceOption{
+						Network: runtime.NamespaceMode_NODE,
+						Pid:     runtime.NamespaceMode_NODE,
+						Ipc:     runtime.NamespaceMode_NODE,
+					},
+				},
+			},
+		},
+	}
+	type args struct {
+		req *runtime.RunPodSandboxRequest
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{"Security Context is nil", args{&runtime.RunPodSandboxRequest{}}, false},
+		{"Security Context is privileged", args{&privilegedContext}, true},
+		{"Security Context is not privileged", args{&nonPrivilegedContext}, false},
+		{"Security Context namespace host access", args{&hostNamespace}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := privilegedSandbox(tt.args.req); got != tt.want {
+				t.Errorf("privilegedSandbox() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}