diff --git a/hack/test-utils.sh b/hack/test-utils.sh index 42bbe319e..8a4832fde 100755 --- a/hack/test-utils.sh +++ b/hack/test-utils.sh @@ -18,6 +18,13 @@ source $(dirname "${BASH_SOURCE[0]}")/utils.sh # RESTART_WAIT_PERIOD is the period to wait before restarting containerd. RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10} +CONTAINERD_CONFIG="--log-level=debug " + +# Use a configuration file for containerd. +CONTAINERD_CONFIG_FILE=${CONTAINERD_CONFIG_FILE:-""} +if [ -f "${CONTAINERD_CONFIG_FILE}" ]; then + CONTAINERD_CONFIG+="--config $CONTAINERD_CONFIG_FILE" +fi CONTAINERD_SOCK=/run/containerd/containerd.sock @@ -32,7 +39,7 @@ test_setup() { exit 1 fi sudo pkill -x containerd - keepalive "sudo ${ROOT}/_output/containerd --log-level=debug" \ + keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_CONFIG}" \ ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log & containerd_pid=$! # Wait for containerd to be running by using the containerd client ctr to check the version diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index f57b32af8..24fa711b5 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -31,4 +31,7 @@ const ( // SandboxID is the sandbox ID annotation SandboxID = "io.kubernetes.cri.sandbox-id" + + // PrivilegedSandbox is the privileged annotation + PrivilegedSandbox = "io.kubernetes.cri.privileged-sandbox" ) diff --git a/pkg/config/config.go b/pkg/config/config.go index 17e01b175..4b456afb9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,20 +18,24 @@ package config import "github.com/containerd/containerd" +// Runtime struct to contain the type(ID), engine, and root variables for a default and a privileged runtime +type Runtime struct { + //Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux + Type string `toml:"runtime_type" json:"runtimeType,omitempty"` + // Engine is the name of the runtime engine used by containerd. + Engine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` + // Root is the directory used by containerd for runtime state. + Root string `toml:"runtime_root" json:"runtimeRoot,omitempty"` +} + // ContainerdConfig contains toml config related to containerd type ContainerdConfig struct { // Snapshotter is the snapshotter used by containerd. Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` - // Runtime is the runtime to use in containerd. We may support - // other runtimes in the future. - Runtime string `toml:"runtime" json:"runtime,omitempty"` - // RuntimeEngine is the name of the runtime engine used by containerd. - // Containerd default should be "runc" - // We may support other runtime engines in the future. - RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` - // RuntimeRoot is the directory used by containerd for runtime state. - // Containerd default should be "/run/containerd/runc" - RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` + // DefaultRuntime is the runtime to use in containerd. + DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime,omitempty"` + // PrivilegedRuntime is a non-secure runtime used only to run trusted workloads on it + PrivilegedRuntime Runtime `toml:"privileged_runtime" json:"privilegedRuntime,omitempty"` } // CniConfig contains toml config related to cni @@ -101,10 +105,17 @@ func DefaultConfig() PluginConfig { NetworkPluginConfDir: "/etc/cni/net.d", }, ContainerdConfig: ContainerdConfig{ - Snapshotter: containerd.DefaultSnapshotter, - Runtime: "io.containerd.runtime.v1.linux", - RuntimeEngine: "", - RuntimeRoot: "", + Snapshotter: containerd.DefaultSnapshotter, + DefaultRuntime: Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "", + Root: "", + }, + PrivilegedRuntime: Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "", + Root: "", + }, }, StreamServerAddress: "", StreamServerPort: "10010", diff --git a/pkg/server/container_create.go b/pkg/server/container_create.go index b06abb0d5..cf28cafa4 100644 --- a/pkg/server/container_create.go +++ b/pkg/server/container_create.go @@ -87,6 +87,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta } sandboxPid := s.Pid() + trusted := sandbox.Config.Annotations[annotations.PrivilegedSandbox] == "true" + containerRuntime := c.getRuntime(trusted) + // Generate unique id and name for the container and reserve the name. // Reserve the container name to avoid concurrent `CreateContainer` request creating // the same container. @@ -227,10 +230,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta opts = append(opts, containerd.WithSpec(spec, specOpts...), containerd.WithRuntime( - c.config.ContainerdConfig.Runtime, + containerRuntime.Type, &runctypes.RuncOptions{ - Runtime: c.config.ContainerdConfig.RuntimeEngine, - RuntimeRoot: c.config.ContainerdConfig.RuntimeRoot, + Runtime: containerRuntime.Engine, + RuntimeRoot: containerRuntime.Root, SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause containerd.WithContainerLabels(containerLabels), containerd.WithContainerExtension(containerMetadataExtension, &meta)) diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go index 8a19c0407..d687275cf 100644 --- a/pkg/server/helpers.go +++ b/pkg/server/helpers.go @@ -35,9 +35,11 @@ import ( "github.com/opencontainers/selinux/go-selinux" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + criconfig "github.com/containerd/cri/pkg/config" "github.com/containerd/cri/pkg/store" imagestore "github.com/containerd/cri/pkg/store/image" "github.com/containerd/cri/pkg/util" @@ -407,3 +409,18 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str "IgnoreUnknown": "1", } } + +// getRuntime returns the runtime configuration +// If the container is privileged, it will return +// the privileged runtime else not. +func (c *criService) getRuntime(privileged bool) (runtime criconfig.Runtime) { + runtime = c.config.ContainerdConfig.DefaultRuntime + + if privileged && c.config.ContainerdConfig.PrivilegedRuntime.Engine != "" { + runtime = c.config.ContainerdConfig.PrivilegedRuntime + } + + logrus.Debugf("runtime=%s(%s), runtime root='%s', privileged='%v'", runtime.Type, runtime.Engine, runtime.Root, privileged) + + return runtime +} diff --git a/pkg/server/helpers_test.go b/pkg/server/helpers_test.go index af3841b00..0abcd9c75 100644 --- a/pkg/server/helpers_test.go +++ b/pkg/server/helpers_test.go @@ -19,10 +19,10 @@ package server import ( "testing" + criconfig "github.com/containerd/cri/pkg/config" + "github.com/containerd/cri/pkg/util" imagedigest "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" - - "github.com/containerd/cri/pkg/util" ) // TestGetUserFromImage tests the logic of getting image uid or user name of image user. @@ -142,3 +142,64 @@ func TestBuildLabels(t *testing.T) { assert.Empty(t, configLabels[containerKindLabel], "should not add new labels into original label") assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label") } + +func Test_criService_getRuntime(t *testing.T) { + + const ( + privilegedWorkload = true + nonPrivilegedWorkload = false + ) + + nonPrivilegedRuntime := criconfig.Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "kata-runtime", + Root: "", + } + + privilegedRuntime := criconfig.Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "runc", + Root: "", + } + + // Crate a configuration that does not specify a privileged runtime + // Both privileged and non-privileged workloads are created with the + // defaultRuntime (nonPrivilegedRuntime). + nonPrivilegedConfig := criService{ + config: criconfig.Config{ + PluginConfig: criconfig.DefaultConfig(), + }, + } + nonPrivilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime + + // Crate a configuration that specifies a privileged runtime + // The privileged workloads are created with the privilegedRuntime + // The non-privileged workloads be created with the + // defaultRuntime(nonPrivilegedRuntime) + privilegedConfig := criService{ + config: criconfig.Config{ + PluginConfig: criconfig.DefaultConfig(), + }, + } + privilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime + privilegedConfig.config.ContainerdConfig.PrivilegedRuntime = privilegedRuntime + + tests := []struct { + name string + cri criService + privileged bool + wantRuntime criconfig.Runtime + }{ + {"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, privilegedWorkload, nonPrivilegedRuntime}, + {"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime}, + {"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, privilegedWorkload, privilegedRuntime}, + {"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRuntime := tt.cri.getRuntime(tt.privileged) + assert.Equal(t, tt.wantRuntime, gotRuntime) + }) + } +} diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go index bb61830f7..b0d295f8e 100644 --- a/pkg/server/sandbox_run.go +++ b/pkg/server/sandbox_run.go @@ -49,6 +49,32 @@ func init() { "github.com/containerd/cri/pkg/store/sandbox", "Metadata") } +// privilegedSandbox returns true if the sandbox configuration +// requires additional host privileges for the sandbox. +func privilegedSandbox(req *runtime.RunPodSandboxRequest) bool { + securityContext := req.GetConfig().GetLinux().GetSecurityContext() + if securityContext == nil { + return false + } + + if securityContext.Privileged { + return true + } + + namespaceOptions := securityContext.GetNamespaceOptions() + if namespaceOptions == nil { + return false + } + + if namespaceOptions.Network == runtime.NamespaceMode_NODE || + namespaceOptions.Pid == runtime.NamespaceMode_NODE || + namespaceOptions.Ipc == runtime.NamespaceMode_NODE { + return true + } + + return false +} + // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure // the sandbox is in ready state. func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) { @@ -130,6 +156,15 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox }() } + privileged := privilegedSandbox(r) + containerRuntime := c.getRuntime(privileged) + + if sandbox.Config.Annotations == nil { + sandbox.Config.Annotations = make(map[string]string) + } + + sandbox.Config.Annotations[annotations.PrivilegedSandbox] = fmt.Sprintf("%v", privileged) + // Create sandbox container. spec, err := c.generateSandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath) if err != nil { @@ -162,10 +197,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox containerd.WithContainerLabels(sandboxLabels), containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata), containerd.WithRuntime( - c.config.ContainerdConfig.Runtime, + containerRuntime.Type, &runctypes.RuncOptions{ - Runtime: c.config.ContainerdConfig.RuntimeEngine, - RuntimeRoot: c.config.ContainerdConfig.RuntimeRoot, + Runtime: containerRuntime.Engine, + RuntimeRoot: containerRuntime.Root, SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause container, err := c.client.NewContainer(ctx, id, opts...) diff --git a/pkg/server/sandbox_run_test.go b/pkg/server/sandbox_run_test.go index 18aea395e..c578d11c0 100644 --- a/pkg/server/sandbox_run_test.go +++ b/pkg/server/sandbox_run_test.go @@ -435,3 +435,58 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { // TODO(random-liu): [P1] Add unit test for different error cases to make sure // the function cleans up on error properly. + +func TestPrivilegedSandbox(t *testing.T) { + privilegedContext := runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: true, + }, + }, + }, + } + nonPrivilegedContext := runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + }, + }, + }, + } + hostNamespace := runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_NODE, + Pid: runtime.NamespaceMode_NODE, + Ipc: runtime.NamespaceMode_NODE, + }, + }, + }, + }, + } + type args struct { + req *runtime.RunPodSandboxRequest + } + tests := []struct { + name string + args args + want bool + }{ + {"Security Context is nil", args{&runtime.RunPodSandboxRequest{}}, false}, + {"Security Context is privileged", args{&privilegedContext}, true}, + {"Security Context is not privileged", args{&nonPrivilegedContext}, false}, + {"Security Context namespace host access", args{&hostNamespace}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := privilegedSandbox(tt.args.req); got != tt.want { + t.Errorf("privilegedSandbox() = %v, want %v", got, tt.want) + } + }) + } +}