diff --git a/hack/test-utils.sh b/hack/test-utils.sh index 8a4832fde..a2a7159a7 100755 --- a/hack/test-utils.sh +++ b/hack/test-utils.sh @@ -18,12 +18,13 @@ source $(dirname "${BASH_SOURCE[0]}")/utils.sh # RESTART_WAIT_PERIOD is the period to wait before restarting containerd. RESTART_WAIT_PERIOD=${RESTART_WAIT_PERIOD:-10} -CONTAINERD_CONFIG="--log-level=debug " +# CONTAINERD_FLAGS contains all containerd flags. +CONTAINERD_FLAGS="--log-level=debug " # Use a configuration file for containerd. CONTAINERD_CONFIG_FILE=${CONTAINERD_CONFIG_FILE:-""} if [ -f "${CONTAINERD_CONFIG_FILE}" ]; then - CONTAINERD_CONFIG+="--config $CONTAINERD_CONFIG_FILE" + CONTAINERD_FLAGS+="--config ${CONTAINERD_CONFIG_FILE} " fi CONTAINERD_SOCK=/run/containerd/containerd.sock @@ -39,7 +40,7 @@ test_setup() { exit 1 fi sudo pkill -x containerd - keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_CONFIG}" \ + keepalive "sudo ${ROOT}/_output/containerd ${CONTAINERD_FLAGS}" \ ${RESTART_WAIT_PERIOD} &> ${report_dir}/containerd.log & containerd_pid=$! # Wait for containerd to be running by using the containerd client ctr to check the version diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 24fa711b5..be63ba27a 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -32,6 +32,7 @@ const ( // SandboxID is the sandbox ID annotation SandboxID = "io.kubernetes.cri.sandbox-id" - // PrivilegedSandbox is the privileged annotation - PrivilegedSandbox = "io.kubernetes.cri.privileged-sandbox" + // UntrustedWorkload is the sandbox annotation for untrusted workload. Untrusted + // workload can only run on dedicated runtime for untrusted workload. + UntrustedWorkload = "io.kubernetes.cri.untrusted-workload" ) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8a0ad327f..c57b00557 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,9 +18,10 @@ package config import "github.com/containerd/containerd" -// Runtime struct to contain the type(ID), engine, and root variables for a default and a privileged runtime +// Runtime struct to contain the type(ID), engine, and root variables for a default runtime +// and a runtime for untrusted worload. type Runtime struct { - //Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux + // Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux Type string `toml:"runtime_type" json:"runtimeType"` // Engine is the name of the runtime engine used by containerd. Engine string `toml:"runtime_engine" json:"runtimeEngine"` @@ -34,8 +35,8 @@ type ContainerdConfig struct { Snapshotter string `toml:"snapshotter" json:"snapshotter"` // DefaultRuntime is the runtime to use in containerd. DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"` - // PrivilegedRuntime is a non-secure runtime used only to run trusted workloads on it - PrivilegedRuntime Runtime `toml:"privileged_runtime" json:"privilegedRuntime"` + // UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it. + UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"` } // CniConfig contains toml config related to cni @@ -111,11 +112,6 @@ func DefaultConfig() PluginConfig { Engine: "", Root: "", }, - PrivilegedRuntime: Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "", - Root: "", - }, }, StreamServerAddress: "", StreamServerPort: "10010", diff --git a/pkg/server/container_create.go b/pkg/server/container_create.go index cf28cafa4..61037c2cb 100644 --- a/pkg/server/container_create.go +++ b/pkg/server/container_create.go @@ -87,9 +87,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta } sandboxPid := s.Pid() - trusted := sandbox.Config.Annotations[annotations.PrivilegedSandbox] == "true" - containerRuntime := c.getRuntime(trusted) - // Generate unique id and name for the container and reserve the name. // Reserve the container name to avoid concurrent `CreateContainer` request creating // the same container. @@ -125,6 +122,17 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta return nil, errors.Errorf("image %q not found", imageRef) } + // Run container using the same runtime with sandbox. + sandboxInfo, err := sandbox.Container.Info(ctx) + if err != nil { + return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID) + } + ociRuntime, err := getRuntimeConfigFromContainerInfo(sandboxInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to get OCI runtime") + } + logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id) + // Create container root directory. containerRootDir := getContainerRootDir(c.config.RootDir, id) if err = c.os.MkdirAll(containerRootDir, 0755); err != nil { @@ -230,10 +238,10 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta opts = append(opts, containerd.WithSpec(spec, specOpts...), containerd.WithRuntime( - containerRuntime.Type, + ociRuntime.Type, &runctypes.RuncOptions{ - Runtime: containerRuntime.Engine, - RuntimeRoot: containerRuntime.Root, + Runtime: ociRuntime.Engine, + RuntimeRoot: ociRuntime.Root, SystemdCgroup: c.config.SystemdCgroup}), // TODO (mikebrow): add CriuPath when we add support for pause containerd.WithContainerLabels(containerLabels), containerd.WithContainerExtension(containerMetadataExtension, &meta)) diff --git a/pkg/server/container_status.go b/pkg/server/container_status.go index 20507e42d..51d356d03 100644 --- a/pkg/server/container_status.go +++ b/pkg/server/container_status.go @@ -21,10 +21,10 @@ import ( runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + criconfig "github.com/containerd/cri/pkg/config" containerstore "github.com/containerd/cri/pkg/store/container" ) @@ -106,6 +106,7 @@ type containerInfo struct { Removing bool `json:"removing"` SnapshotKey string `json:"snapshotKey"` Snapshotter string `json:"snapshotter"` + Runtime *criconfig.Runtime `json:"runtime"` Config *runtime.ContainerConfig `json:"config"` RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"` } @@ -128,20 +129,24 @@ func toCRIContainerInfo(ctx context.Context, container containerstore.Container, Config: meta.Config, } - spec, err := container.Container.Spec(ctx) - if err == nil { - ci.RuntimeSpec = spec - } else { - logrus.WithError(err).Errorf("Failed to get container %q spec", container.ID) + var err error + ci.RuntimeSpec, err = container.Container.Spec(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get container runtime spec") } ctrInfo, err := container.Container.Info(ctx) - if err == nil { - ci.SnapshotKey = ctrInfo.SnapshotKey - ci.Snapshotter = ctrInfo.Snapshotter - } else { - logrus.WithError(err).Errorf("Failed to get container %q info", container.ID) + if err != nil { + return nil, errors.Wrap(err, "failed to get container info") } + ci.SnapshotKey = ctrInfo.SnapshotKey + ci.Snapshotter = ctrInfo.Snapshotter + + ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to get container runtime config") + } + ci.Runtime = &ociRuntime infoBytes, err := json.Marshal(ci) if err != nil { diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go index d687275cf..06e43a257 100644 --- a/pkg/server/helpers.go +++ b/pkg/server/helpers.go @@ -25,7 +25,10 @@ import ( "strings" "github.com/containerd/containerd" + "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/linux/runctypes" + "github.com/containerd/typeurl" "github.com/docker/distribution/reference" imagedigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -35,7 +38,6 @@ import ( "github.com/opencontainers/selinux/go-selinux" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -410,17 +412,22 @@ func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]str } } -// getRuntime returns the runtime configuration -// If the container is privileged, it will return -// the privileged runtime else not. -func (c *criService) getRuntime(privileged bool) (runtime criconfig.Runtime) { - runtime = c.config.ContainerdConfig.DefaultRuntime - - if privileged && c.config.ContainerdConfig.PrivilegedRuntime.Engine != "" { - runtime = c.config.ContainerdConfig.PrivilegedRuntime +// getRuntimeConfigFromContainerInfo gets runtime configuration from containerd +// container info. +func getRuntimeConfigFromContainerInfo(c containers.Container) (criconfig.Runtime, error) { + r := criconfig.Runtime{ + Type: c.Runtime.Name, } - - logrus.Debugf("runtime=%s(%s), runtime root='%s', privileged='%v'", runtime.Type, runtime.Engine, runtime.Root, privileged) - - return runtime + if c.Runtime.Options == nil { + // CRI plugin makes sure that runtime option is always set. + return criconfig.Runtime{}, errors.New("runtime options is nil") + } + data, err := typeurl.UnmarshalAny(c.Runtime.Options) + if err != nil { + return criconfig.Runtime{}, errors.Wrap(err, "failed to unmarshal runtime options") + } + runtimeOpts := data.(*runctypes.RuncOptions) + r.Engine = runtimeOpts.Runtime + r.Root = runtimeOpts.RuntimeRoot + return r, nil } diff --git a/pkg/server/helpers_test.go b/pkg/server/helpers_test.go index 0abcd9c75..b7aed713c 100644 --- a/pkg/server/helpers_test.go +++ b/pkg/server/helpers_test.go @@ -19,10 +19,15 @@ package server import ( "testing" - criconfig "github.com/containerd/cri/pkg/config" - "github.com/containerd/cri/pkg/util" + "github.com/containerd/containerd" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/linux/runctypes" imagedigest "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" + "golang.org/x/net/context" + + criconfig "github.com/containerd/cri/pkg/config" + "github.com/containerd/cri/pkg/util" ) // TestGetUserFromImage tests the logic of getting image uid or user name of image user. @@ -143,63 +148,45 @@ func TestBuildLabels(t *testing.T) { assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label") } -func Test_criService_getRuntime(t *testing.T) { - - const ( - privilegedWorkload = true - nonPrivilegedWorkload = false - ) - - nonPrivilegedRuntime := criconfig.Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "kata-runtime", - Root: "", - } - - privilegedRuntime := criconfig.Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "runc", - Root: "", - } - - // Crate a configuration that does not specify a privileged runtime - // Both privileged and non-privileged workloads are created with the - // defaultRuntime (nonPrivilegedRuntime). - nonPrivilegedConfig := criService{ - config: criconfig.Config{ - PluginConfig: criconfig.DefaultConfig(), - }, - } - nonPrivilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime - - // Crate a configuration that specifies a privileged runtime - // The privileged workloads are created with the privilegedRuntime - // The non-privileged workloads be created with the - // defaultRuntime(nonPrivilegedRuntime) - privilegedConfig := criService{ - config: criconfig.Config{ - PluginConfig: criconfig.DefaultConfig(), - }, - } - privilegedConfig.config.ContainerdConfig.DefaultRuntime = nonPrivilegedRuntime - privilegedConfig.config.ContainerdConfig.PrivilegedRuntime = privilegedRuntime - - tests := []struct { - name string - cri criService - privileged bool - wantRuntime criconfig.Runtime +func TestGetRuntimeConfigFromContainerInfo(t *testing.T) { + for desc, test := range map[string]struct { + typ string + engine string + root string + expectErr bool + expectedRuntime criconfig.Runtime }{ - {"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, privilegedWorkload, nonPrivilegedRuntime}, - {"nonPrivilegedConfig/PrivilegedWorkload", nonPrivilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime}, - {"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, privilegedWorkload, privilegedRuntime}, - {"PrivilegedConfig/nonPrivilegedWorkload", privilegedConfig, nonPrivilegedWorkload, nonPrivilegedRuntime}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotRuntime := tt.cri.getRuntime(tt.privileged) - assert.Equal(t, tt.wantRuntime, gotRuntime) + "should return error if there is no runc options": { + typ: "test.type", + expectErr: true, + }, + "should retrieve runtime from container info": { + typ: "test.type", + engine: "test-engine", + root: "/test/root", + expectedRuntime: criconfig.Runtime{ + Type: "test.type", + Engine: "test-engine", + Root: "/test/root", + }, + }, + } { + t.Run(desc, func(t *testing.T) { + var opts interface{} + if test.engine != "" || test.root != "" { + opts = &runctypes.RuncOptions{ + Runtime: test.engine, + RuntimeRoot: test.root, + } + } + c := containers.Container{} + assert.NoError(t, containerd.WithRuntime( + test.typ, + opts, + )(context.Background(), nil, &c)) + r, err := getRuntimeConfigFromContainerInfo(c) + assert.Equal(t, test.expectErr, err != nil) + assert.Equal(t, test.expectedRuntime, r) }) } } diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go index b0d295f8e..237290709 100644 --- a/pkg/server/sandbox_run.go +++ b/pkg/server/sandbox_run.go @@ -37,6 +37,7 @@ import ( runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "github.com/containerd/cri/pkg/annotations" + criconfig "github.com/containerd/cri/pkg/config" customopts "github.com/containerd/cri/pkg/containerd/opts" ctrdutil "github.com/containerd/cri/pkg/containerd/util" "github.com/containerd/cri/pkg/log" @@ -49,32 +50,6 @@ func init() { "github.com/containerd/cri/pkg/store/sandbox", "Metadata") } -// privilegedSandbox returns true if the sandbox configuration -// requires additional host privileges for the sandbox. -func privilegedSandbox(req *runtime.RunPodSandboxRequest) bool { - securityContext := req.GetConfig().GetLinux().GetSecurityContext() - if securityContext == nil { - return false - } - - if securityContext.Privileged { - return true - } - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions == nil { - return false - } - - if namespaceOptions.Network == runtime.NamespaceMode_NODE || - namespaceOptions.Pid == runtime.NamespaceMode_NODE || - namespaceOptions.Ipc == runtime.NamespaceMode_NODE { - return true - } - - return false -} - // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure // the sandbox is in ready state. func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) { @@ -156,14 +131,11 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox }() } - privileged := privilegedSandbox(r) - containerRuntime := c.getRuntime(privileged) - - if sandbox.Config.Annotations == nil { - sandbox.Config.Annotations = make(map[string]string) + ociRuntime, err := c.getSandboxRuntime(config) + if err != nil { + return nil, errors.Wrap(err, "failed to get sandbox runtime") } - - sandbox.Config.Annotations[annotations.PrivilegedSandbox] = fmt.Sprintf("%v", privileged) + logrus.Debugf("Use OCI %+v for sandbox %q", ociRuntime, id) // Create sandbox container. spec, err := c.generateSandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath) @@ -197,10 +169,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox containerd.WithContainerLabels(sandboxLabels), containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata), containerd.WithRuntime( - containerRuntime.Type, + ociRuntime.Type, &runctypes.RuncOptions{ - Runtime: containerRuntime.Engine, - RuntimeRoot: containerRuntime.Root, + Runtime: ociRuntime.Engine, + RuntimeRoot: ociRuntime.Root, SystemdCgroup: c.config.SystemdCgroup})} // TODO (mikebrow): add CriuPath when we add support for pause container, err := c.client.NewContainer(ctx, id, opts...) @@ -553,3 +525,48 @@ func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []cni.PortMapping } return portMappings } + +// untrustedWorkload returns true if the sandbox contains untrusted workload. +func untrustedWorkload(config *runtime.PodSandboxConfig) bool { + return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" +} + +// hostPrivilegedSandbox returns true if the sandbox configuration +// requires additional host privileges for the sandbox. +func hostPrivilegedSandbox(config *runtime.PodSandboxConfig) bool { + securityContext := config.GetLinux().GetSecurityContext() + if securityContext.GetPrivileged() { + return true + } + + namespaceOptions := securityContext.GetNamespaceOptions() + if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || + namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || + namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { + return true + } + + return false +} + +// getSandboxRuntime returns the runtime configuration for sandbox. +// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned, +// or else default runtime will be returned. +func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig) (criconfig.Runtime, error) { + untrusted := false + if untrustedWorkload(config) { + // TODO(random-liu): Figure out we should return error or not. + if hostPrivilegedSandbox(config) { + return criconfig.Runtime{}, errors.New("untrusted workload with host privilege is not allowed") + } + untrusted = true + } + + if untrusted { + if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type == "" { + return criconfig.Runtime{}, errors.New("no runtime for untrusted workload is configured") + } + return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil + } + return c.config.ContainerdConfig.DefaultRuntime, nil +} diff --git a/pkg/server/sandbox_run_test.go b/pkg/server/sandbox_run_test.go index c578d11c0..c5a355a6c 100644 --- a/pkg/server/sandbox_run_test.go +++ b/pkg/server/sandbox_run_test.go @@ -20,7 +20,6 @@ import ( "os" "testing" - "github.com/containerd/cri/pkg/annotations" cni "github.com/containerd/go-cni" "github.com/containerd/typeurl" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -29,6 +28,8 @@ import ( "github.com/stretchr/testify/require" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + "github.com/containerd/cri/pkg/annotations" + criconfig "github.com/containerd/cri/pkg/config" ostesting "github.com/containerd/cri/pkg/os/testing" sandboxstore "github.com/containerd/cri/pkg/store/sandbox" ) @@ -433,60 +434,136 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { } } -// TODO(random-liu): [P1] Add unit test for different error cases to make sure -// the function cleans up on error properly. - -func TestPrivilegedSandbox(t *testing.T) { - privilegedContext := runtime.RunPodSandboxRequest{ - Config: &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, +func TestHostPrivilegedSandbox(t *testing.T) { + privilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: true, }, }, } - nonPrivilegedContext := runtime.RunPodSandboxRequest{ - Config: &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - }, + nonPrivilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, }, }, } - hostNamespace := runtime.RunPodSandboxRequest{ - Config: &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, + hostNamespace := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_NODE, + Pid: runtime.NamespaceMode_NODE, + Ipc: runtime.NamespaceMode_NODE, }, }, }, } - type args struct { - req *runtime.RunPodSandboxRequest - } tests := []struct { - name string - args args - want bool + name string + config *runtime.PodSandboxConfig + want bool }{ - {"Security Context is nil", args{&runtime.RunPodSandboxRequest{}}, false}, - {"Security Context is privileged", args{&privilegedContext}, true}, - {"Security Context is not privileged", args{&nonPrivilegedContext}, false}, - {"Security Context namespace host access", args{&hostNamespace}, true}, + {"Security Context is nil", nil, false}, + {"Security Context is privileged", privilegedContext, true}, + {"Security Context is not privileged", nonPrivilegedContext, false}, + {"Security Context namespace host access", hostNamespace, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := privilegedSandbox(tt.args.req); got != tt.want { - t.Errorf("privilegedSandbox() = %v, want %v", got, tt.want) + if got := hostPrivilegedSandbox(tt.config); got != tt.want { + t.Errorf("hostPrivilegedSandbox() = %v, want %v", got, tt.want) } }) } } + +func TestGetSandboxRuntime(t *testing.T) { + untrustedWorkloadRuntime := criconfig.Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "untursted-workload-runtime", + Root: "", + } + + defaultRuntime := criconfig.Runtime{ + Type: "io.containerd.runtime.v1.linux", + Engine: "default-runtime", + Root: "", + } + + for desc, test := range map[string]struct { + sandboxConfig *runtime.PodSandboxConfig + defaultRuntime criconfig.Runtime + untrustedWorkloadRuntime criconfig.Runtime + expectErr bool + expectedRuntime criconfig.Runtime + }{ + "should return error if untrusted workload requires host privilege": { + sandboxConfig: &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: true, + }, + }, + Annotations: map[string]string{ + annotations.UntrustedWorkload: "true", + }, + }, + defaultRuntime: defaultRuntime, + untrustedWorkloadRuntime: untrustedWorkloadRuntime, + expectErr: true, + }, + "should use untrusted workload runtime for untrusted workload": { + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + annotations.UntrustedWorkload: "true", + }, + }, + defaultRuntime: defaultRuntime, + untrustedWorkloadRuntime: untrustedWorkloadRuntime, + expectedRuntime: untrustedWorkloadRuntime, + }, + "should use default runtime for regular workload": { + sandboxConfig: &runtime.PodSandboxConfig{}, + defaultRuntime: defaultRuntime, + untrustedWorkloadRuntime: untrustedWorkloadRuntime, + expectedRuntime: defaultRuntime, + }, + "should use default runtime for trusted workload": { + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + annotations.UntrustedWorkload: "false", + }, + }, + defaultRuntime: defaultRuntime, + untrustedWorkloadRuntime: untrustedWorkloadRuntime, + expectedRuntime: defaultRuntime, + }, + "should return error if untrusted workload runtime is required but not configured": { + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + annotations.UntrustedWorkload: "true", + }, + }, + defaultRuntime: defaultRuntime, + expectErr: true, + }, + } { + t.Run(desc, func(t *testing.T) { + cri := newTestCRIService() + cri.config = criconfig.Config{ + PluginConfig: criconfig.DefaultConfig(), + } + cri.config.ContainerdConfig.DefaultRuntime = test.defaultRuntime + cri.config.ContainerdConfig.UntrustedWorkloadRuntime = test.untrustedWorkloadRuntime + r, err := cri.getSandboxRuntime(test.sandboxConfig) + assert.Equal(t, test.expectErr, err != nil) + assert.Equal(t, test.expectedRuntime, r) + }) + } +} + +// TODO(random-liu): [P1] Add unit test for different error cases to make sure +// the function cleans up on error properly. diff --git a/pkg/server/sandbox_status.go b/pkg/server/sandbox_status.go index 75bc98056..5d4628a8e 100644 --- a/pkg/server/sandbox_status.go +++ b/pkg/server/sandbox_status.go @@ -26,6 +26,7 @@ import ( "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + criconfig "github.com/containerd/cri/pkg/config" sandboxstore "github.com/containerd/cri/pkg/store/sandbox" ) @@ -107,6 +108,7 @@ type sandboxInfo struct { Image string `json:"image"` SnapshotKey string `json:"snapshotKey"` Snapshotter string `json:"snapshotter"` + Runtime *criconfig.Runtime `json:"runtime"` Config *runtime.PodSandboxConfig `json:"config"` RuntimeSpec *runtimespec.Spec `json:"runtimeSpec"` } @@ -163,6 +165,12 @@ func toCRISandboxInfo(ctx context.Context, sandbox sandboxstore.Sandbox) (map[st si.SnapshotKey = ctrInfo.SnapshotKey si.Snapshotter = ctrInfo.Snapshotter + ociRuntime, err := getRuntimeConfigFromContainerInfo(ctrInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to get sandbox container runtime config") + } + si.Runtime = &ociRuntime + infoBytes, err := json.Marshal(si) if err != nil { return nil, errors.Wrapf(err, "failed to marshal info %v", si)