From 0cf48bab2cee394ccefcc05e7025ade76d142bc4 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Thu, 19 Oct 2023 21:06:18 +0800 Subject: [PATCH 1/3] sandbox: podsandbox init its own client To break the cyclic dependency of cri plugin and podsandbox plugin, we define a new plugin type of SandboxesServicePlugin and when cri init it's own client, it will add the all the controllers by get them from the SandboxesServicePlugin. when podsandbox controller init it's client, it will not Require the SandboxesServicePlugin. Signed-off-by: Abel Feng --- client/client.go | 1 - client/services.go | 16 ++++++++++++--- pkg/cri/cri.go | 1 + pkg/cri/server/podsandbox/controller.go | 27 +++++++++++++++++-------- pkg/cri/server/service.go | 2 +- plugins/types.go | 2 ++ services/sandbox/controller_service.go | 4 ++-- services/sandbox/sandboxers.go | 2 +- 8 files changed, 39 insertions(+), 16 deletions(-) diff --git a/client/client.go b/client/client.go index 714c57c25..036e6e5ac 100644 --- a/client/client.go +++ b/client/client.go @@ -720,7 +720,6 @@ func (c *Client) SandboxStore() sandbox.Store { // SandboxController returns the underlying sandbox controller client func (c *Client) SandboxController(name string) sandbox.Controller { - // default sandboxer is shim if c.sandboxers != nil { return c.sandboxers[name] } diff --git a/client/services.go b/client/services.go index 3ca357eb3..03da25246 100644 --- a/client/services.go +++ b/client/services.go @@ -218,9 +218,6 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { srv.SnapshotsService: func(s interface{}) ServicesOpt { return WithSnapshotters(s.(map[string]snapshots.Snapshotter)) }, - srv.SandboxControllersService: func(s interface{}) ServicesOpt { - return WithSandboxers(s.(map[string]sandbox.Controller)) - }, srv.ContainersService: func(s interface{}) ServicesOpt { return WithContainerClient(s.(containersapi.ContainersClient)) }, @@ -251,3 +248,16 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { return nil } } + +func WithSandboxersService(ic *plugin.InitContext) ClientOpt { + return func(c *clientOpts) error { + sandboxesPlugin, err := ic.GetByID(plugins.SandboxesServicePlugin, srv.SandboxControllersService) + if err != nil { + return err + } + + sbs := sandboxesPlugin.(map[string]sandbox.Controller) + c.services.sandboxers = sbs + return nil + } +} diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index e85547e68..8544230fb 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -92,6 +92,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(ic), + containerd.WithSandboxersService(ic), ) if err != nil { return nil, fmt.Errorf("failed to create containerd client: %w", err) diff --git a/pkg/cri/server/podsandbox/controller.go b/pkg/cri/server/podsandbox/controller.go index 45b3faf7d..ff3de7a0f 100644 --- a/pkg/cri/server/podsandbox/controller.go +++ b/pkg/cri/server/podsandbox/controller.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/constants" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" @@ -43,13 +44,25 @@ import ( func init() { registry.Register(&plugin.Registration{ - Type: plugins.SandboxControllerPlugin, - ID: "podsandbox", - Requires: []plugin.Type{}, + Type: plugins.SandboxControllerPlugin, + ID: "podsandbox", + Requires: []plugin.Type{ + plugins.EventPlugin, + plugins.ServicePlugin, + }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - // register the global controller to containerd plugin manager, - // the global controller will be initialized when cri plugin is initializing - return &Controller{}, nil + c := Controller{} + client, err := containerd.New( + "", + containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + ) + if err != nil { + return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) + } + c.client = client + return &c, nil }, }) } @@ -90,7 +103,6 @@ type Controller struct { func (c *Controller) Init( config criconfig.Config, - client *containerd.Client, sandboxStore *sandboxstore.Store, os osinterface.OS, cri CRIService, @@ -98,7 +110,6 @@ func (c *Controller) Init( baseOCISpecs map[string]*oci.Spec, ) { c.cri = cri - c.client = client c.config = config c.sandboxStore = sandboxStore c.os = os diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 070c81be4..10c8a3882 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -218,7 +218,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. } // Initialize pod sandbox controller - sbControllers[string(criconfig.ModePodSandbox)].(*podsandbox.Controller).Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) + sbControllers[string(criconfig.ModePodSandbox)].(*podsandbox.Controller).Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) c.nri = nri diff --git a/plugins/types.go b/plugins/types.go index 1ac4a2737..eea2095bc 100644 --- a/plugins/types.go +++ b/plugins/types.go @@ -67,6 +67,8 @@ const ( ImageVerifierPlugin plugin.Type = "io.containerd.image-verifier.v1" // WarningPlugin implements a warning service WarningPlugin plugin.Type = "io.containerd.warning.v1" + // SandboxesServicePlugin implements sandboxes service + SandboxesServicePlugin plugin.Type = "io.containerd.service.sandboxes.v1" ) const ( diff --git a/services/sandbox/controller_service.go b/services/sandbox/controller_service.go index dd8cc352e..8ef2736e7 100644 --- a/services/sandbox/controller_service.go +++ b/services/sandbox/controller_service.go @@ -42,11 +42,11 @@ func init() { Type: plugins.GRPCPlugin, ID: "sandbox-controllers", Requires: []plugin.Type{ - plugins.ServicePlugin, + plugins.SandboxesServicePlugin, plugins.EventPlugin, }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - i, err := ic.GetByID(plugins.ServicePlugin, services.SandboxControllersService) + i, err := ic.GetByID(plugins.SandboxesServicePlugin, services.SandboxControllersService) if err != nil { return nil, err } diff --git a/services/sandbox/sandboxers.go b/services/sandbox/sandboxers.go index 85db67018..df09a800a 100644 --- a/services/sandbox/sandboxers.go +++ b/services/sandbox/sandboxers.go @@ -26,7 +26,7 @@ import ( func init() { registry.Register(&plugin.Registration{ - Type: plugins.ServicePlugin, + Type: plugins.SandboxesServicePlugin, ID: services.SandboxControllersService, Requires: []plugin.Type{ plugins.SandboxControllerPlugin, From 25a4c3d2355b7c1a3b7df4be370c7f72d71cb386 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Fri, 20 Oct 2023 09:48:59 +0800 Subject: [PATCH 2/3] sandbox: remove SandboxersServicePlugin Signed-off-by: Abel Feng --- client/services.go | 22 +++------ .../build_local_containerd_helper_test.go | 1 + pkg/cri/cri.go | 3 +- pkg/cri/server/service.go | 12 ++--- plugins/types.go | 2 - services/sandbox/controller_service.go | 12 +++-- services/sandbox/sandboxers.go | 46 ------------------- 7 files changed, 21 insertions(+), 77 deletions(-) delete mode 100644 services/sandbox/sandboxers.go diff --git a/client/services.go b/client/services.go index 03da25246..2c9a009f7 100644 --- a/client/services.go +++ b/client/services.go @@ -87,16 +87,6 @@ func WithSnapshotters(snapshotters map[string]snapshots.Snapshotter) ServicesOpt } } -// WithSandboxers sets the sandbox controllers. -func WithSandboxers(sandboxers map[string]sandbox.Controller) ServicesOpt { - return func(s *services) { - s.sandboxers = make(map[string]sandbox.Controller) - for n, sn := range sandboxers { - s.sandboxers[n] = sn - } - } -} - // WithContainerClient sets the container service to use using a containers client. func WithContainerClient(containerService containersapi.ContainersClient) ServicesOpt { return func(s *services) { @@ -249,15 +239,17 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { } } -func WithSandboxersService(ic *plugin.InitContext) ClientOpt { +func WithSandboxers(ic *plugin.InitContext) Opt { return func(c *clientOpts) error { - sandboxesPlugin, err := ic.GetByID(plugins.SandboxesServicePlugin, srv.SandboxControllersService) + sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin) if err != nil { return err } - - sbs := sandboxesPlugin.(map[string]sandbox.Controller) - c.services.sandboxers = sbs + sc := make(map[string]sandbox.Controller) + for name, p := range sandboxers { + sc[name] = p.(sandbox.Controller) + } + c.services.sandboxers = sc return nil } } diff --git a/integration/build_local_containerd_helper_test.go b/integration/build_local_containerd_helper_test.go index cbd3fcb89..104942153 100644 --- a/integration/build_local_containerd_helper_test.go +++ b/integration/build_local_containerd_helper_test.go @@ -118,6 +118,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(lastInitContext), + containerd.WithSandboxers(lastInitContext), ) assert.NoError(t, err) diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index 8544230fb..de366503d 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -50,6 +50,7 @@ func init() { plugins.ServicePlugin, plugins.NRIApiPlugin, plugins.WarningPlugin, + plugins.SandboxControllerPlugin, }, InitFn: initCRIService, }) @@ -92,7 +93,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(ic), - containerd.WithSandboxersService(ic), + containerd.WithSandboxers(ic), ) if err != nil { return nil, fmt.Errorf("failed to create containerd client: %w", err) diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 10c8a3882..498a8f5ae 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -138,11 +138,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter) } - // TODO(dmcgowan): Get the full list directly from configured plugins - sbControllers := map[string]sandbox.Controller{ - string(criconfig.ModePodSandbox): client.SandboxController(string(criconfig.ModePodSandbox)), - string(criconfig.ModeShim): client.SandboxController(string(criconfig.ModeShim)), - } imageFSPaths := map[string]string{} for _, ociRuntime := range config.ContainerdConfig.Runtimes { // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` @@ -151,10 +146,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) } - if _, ok := sbControllers[ociRuntime.Sandboxer]; !ok { - sbControllers[ociRuntime.Sandboxer] = client.SandboxController(ociRuntime.Sandboxer) - } } + snapshotter := config.ContainerdConfig.Snapshotter imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) @@ -217,8 +210,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, err } + podSandboxController := client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller) // Initialize pod sandbox controller - sbControllers[string(criconfig.ModePodSandbox)].(*podsandbox.Controller).Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) + podSandboxController.Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) c.nri = nri diff --git a/plugins/types.go b/plugins/types.go index eea2095bc..1ac4a2737 100644 --- a/plugins/types.go +++ b/plugins/types.go @@ -67,8 +67,6 @@ const ( ImageVerifierPlugin plugin.Type = "io.containerd.image-verifier.v1" // WarningPlugin implements a warning service WarningPlugin plugin.Type = "io.containerd.warning.v1" - // SandboxesServicePlugin implements sandboxes service - SandboxesServicePlugin plugin.Type = "io.containerd.service.sandboxes.v1" ) const ( diff --git a/services/sandbox/controller_service.go b/services/sandbox/controller_service.go index 8ef2736e7..84a5367e8 100644 --- a/services/sandbox/controller_service.go +++ b/services/sandbox/controller_service.go @@ -31,7 +31,6 @@ import ( "github.com/containerd/containerd/v2/plugins" "github.com/containerd/containerd/v2/protobuf" "github.com/containerd/containerd/v2/sandbox" - "github.com/containerd/containerd/v2/services" "github.com/containerd/log" "github.com/containerd/plugin" "github.com/containerd/plugin/registry" @@ -42,15 +41,20 @@ func init() { Type: plugins.GRPCPlugin, ID: "sandbox-controllers", Requires: []plugin.Type{ - plugins.SandboxesServicePlugin, + plugins.SandboxControllerPlugin, plugins.EventPlugin, }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - i, err := ic.GetByID(plugins.SandboxesServicePlugin, services.SandboxControllersService) + sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin) if err != nil { return nil, err } - sc := i.(map[string]sandbox.Controller) + + sc := make(map[string]sandbox.Controller) + for name, p := range sandboxers { + sc[name] = p.(sandbox.Controller) + } + ep, err := ic.GetSingle(plugins.EventPlugin) if err != nil { return nil, err diff --git a/services/sandbox/sandboxers.go b/services/sandbox/sandboxers.go deleted file mode 100644 index df09a800a..000000000 --- a/services/sandbox/sandboxers.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sandbox - -import ( - "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/containerd/v2/sandbox" - "github.com/containerd/containerd/v2/services" - "github.com/containerd/plugin" - "github.com/containerd/plugin/registry" -) - -func init() { - registry.Register(&plugin.Registration{ - Type: plugins.SandboxesServicePlugin, - ID: services.SandboxControllersService, - Requires: []plugin.Type{ - plugins.SandboxControllerPlugin, - }, - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - sandboxesRaw, err := ic.GetByType(plugins.SandboxControllerPlugin) - if err != nil { - return nil, err - } - sandboxers := make(map[string]sandbox.Controller) - for name, srv := range sandboxesRaw { - sandboxers[name] = srv.(sandbox.Controller) - } - return sandboxers, nil - }, - }) -} From 32bf805e5703bc91387d047fa76625e915ac2b80 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Fri, 20 Oct 2023 15:00:33 +0800 Subject: [PATCH 3/3] sandbox: add a sandboxService interface to criService so that we can add a fakeSandboxService to the criService in tests. Signed-off-by: Abel Feng --- client/services.go | 2 +- .../build_local_containerd_helper_test.go | 2 +- pkg/cri/config/config.go | 54 ++++++++++++++++ pkg/cri/config/config_test.go | 48 ++++++++++++++ pkg/cri/cri.go | 2 +- pkg/cri/server/container_create.go | 6 +- pkg/cri/server/container_start.go | 2 +- pkg/cri/server/container_stats_list.go | 14 ++--- pkg/cri/server/images/image_pull.go | 59 +----------------- pkg/cri/server/podsandbox/sandbox_run.go | 62 +------------------ pkg/cri/server/podsandbox/sandbox_run_test.go | 47 -------------- pkg/cri/server/sandbox_remove.go | 2 +- pkg/cri/server/sandbox_run.go | 58 +---------------- pkg/cri/server/sandbox_service.go | 47 ++++++++++++++ pkg/cri/server/sandbox_stats_windows.go | 2 +- pkg/cri/server/sandbox_status.go | 2 +- pkg/cri/server/sandbox_stop.go | 2 +- pkg/cri/server/service.go | 49 ++++++--------- pkg/cri/server/service_test.go | 55 ++++++++++++++-- 19 files changed, 244 insertions(+), 271 deletions(-) create mode 100644 pkg/cri/server/sandbox_service.go diff --git a/client/services.go b/client/services.go index 2c9a009f7..ab8b15ff7 100644 --- a/client/services.go +++ b/client/services.go @@ -239,7 +239,7 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { } } -func WithSandboxers(ic *plugin.InitContext) Opt { +func WithInMemorySandboxControllers(ic *plugin.InitContext) Opt { return func(c *clientOpts) error { sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin) if err != nil { diff --git a/integration/build_local_containerd_helper_test.go b/integration/build_local_containerd_helper_test.go index 104942153..f49df219a 100644 --- a/integration/build_local_containerd_helper_test.go +++ b/integration/build_local_containerd_helper_test.go @@ -118,7 +118,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(lastInitContext), - containerd.WithSandboxers(lastInitContext), + containerd.WithInMemorySandboxControllers(lastInitContext), ) assert.NoError(t, err) diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 6a8cd84b0..43e954b5b 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -24,7 +24,9 @@ import ( "time" "github.com/containerd/log" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/pkg/cri/annotations" "github.com/containerd/containerd/v2/pkg/deprecation" ) @@ -454,3 +456,55 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W } return warnings, nil } + +func (config *Config) GetSandboxRuntime(podSandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (Runtime, error) { + if untrustedWorkload(podSandboxConfig) { + // If the untrusted annotation is provided, runtimeHandler MUST be empty. + if runtimeHandler != "" && runtimeHandler != RuntimeUntrusted { + return Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") + } + + // If the untrusted workload is requesting access to the host/node, this request will fail. + // + // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the + // runtime may support this. For example, in a virtual-machine isolated runtime, privileged + // is a supported option, granting the workload to access the entire guest VM instead of host. + // TODO(windows): Deprecate this so that we don't need to handle it for windows. + if hostAccessingSandbox(podSandboxConfig) { + return Runtime{}, errors.New("untrusted workload with host access is not allowed") + } + + runtimeHandler = RuntimeUntrusted + } + + if runtimeHandler == "" { + runtimeHandler = config.DefaultRuntimeName + } + + r, ok := config.Runtimes[runtimeHandler] + if !ok { + return Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) + } + return r, nil + +} + +// untrustedWorkload returns true if the sandbox contains untrusted workload. +func untrustedWorkload(config *runtime.PodSandboxConfig) bool { + return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" +} + +// hostAccessingSandbox returns true if the sandbox configuration +// requires additional host access for the sandbox. +func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { + securityContext := config.GetLinux().GetSecurityContext() + + namespaceOptions := securityContext.GetNamespaceOptions() + if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || + namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || + namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { + return true + } + + return false +} diff --git a/pkg/cri/config/config_test.go b/pkg/cri/config/config_test.go index d442dda1f..362e49161 100644 --- a/pkg/cri/config/config_test.go +++ b/pkg/cri/config/config_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/containerd/containerd/v2/pkg/deprecation" ) @@ -232,3 +233,50 @@ func TestValidateConfig(t *testing.T) { }) } } + +func TestHostAccessingSandbox(t *testing.T) { + privilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: true, + }, + }, + } + nonPrivilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + }, + }, + } + hostNamespace := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_NODE, + Pid: runtime.NamespaceMode_NODE, + Ipc: runtime.NamespaceMode_NODE, + }, + }, + }, + } + tests := []struct { + name string + config *runtime.PodSandboxConfig + want bool + }{ + {"Security Context is nil", nil, false}, + {"Security Context is privileged", privilegedContext, false}, + {"Security Context is not privileged", nonPrivilegedContext, false}, + {"Security Context namespace host access", hostNamespace, true}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := hostAccessingSandbox(tt.config); got != tt.want { + t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index de366503d..819e2b8e4 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -93,7 +93,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(ic), - containerd.WithSandboxers(ic), + containerd.WithInMemorySandboxControllers(ic), ) if err != nil { return nil, fmt.Errorf("failed to create containerd client: %w", err) diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index a0bd37ba9..95571fc95 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/containerd/log" "github.com/containerd/typeurl/v2" "github.com/davecgh/go-spew/spew" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -45,7 +46,6 @@ import ( containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" "github.com/containerd/containerd/v2/pkg/cri/util" "github.com/containerd/containerd/v2/platforms" - "github.com/containerd/log" ) func init() { @@ -63,7 +63,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta return nil, fmt.Errorf("failed to find sandbox id %q: %w", r.GetPodSandboxId(), err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -163,7 +163,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID) } - ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) } diff --git a/pkg/cri/server/container_start.go b/pkg/cri/server/container_start.go index d37e7933a..14d0ab4ab 100644 --- a/pkg/cri/server/container_start.go +++ b/pkg/cri/server/container_start.go @@ -109,7 +109,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain return cntr.IO, nil } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) } diff --git a/pkg/cri/server/container_stats_list.go b/pkg/cri/server/container_stats_list.go index 3eadc6895..ef4865990 100644 --- a/pkg/cri/server/container_stats_list.go +++ b/pkg/cri/server/container_stats_list.go @@ -26,16 +26,16 @@ import ( wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" cg1 "github.com/containerd/cgroups/v3/cgroup1/stats" cg2 "github.com/containerd/cgroups/v3/cgroup2/stats" - "github.com/containerd/containerd/v2/api/services/tasks/v1" - "github.com/containerd/containerd/v2/api/types" - "github.com/containerd/containerd/v2/errdefs" - "github.com/containerd/containerd/v2/pkg/cri/store/stats" - "github.com/containerd/containerd/v2/protobuf" "github.com/containerd/log" "github.com/containerd/typeurl/v2" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/api/services/tasks/v1" + "github.com/containerd/containerd/v2/api/types" + "github.com/containerd/containerd/v2/errdefs" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" + "github.com/containerd/containerd/v2/pkg/cri/store/stats" + "github.com/containerd/containerd/v2/protobuf" ) // ListContainerStats returns stats of all running containers. @@ -68,7 +68,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m if err != nil { return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -81,7 +81,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m return nil, err } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err) } diff --git a/pkg/cri/server/images/image_pull.go b/pkg/cri/server/images/image_pull.go index 07d6ea808..cb9db2738 100644 --- a/pkg/cri/server/images/image_pull.go +++ b/pkg/cri/server/images/image_pull.go @@ -20,7 +20,6 @@ import ( "context" "crypto/tls" "encoding/base64" - "errors" "fmt" "io" "net" @@ -33,6 +32,8 @@ import ( "sync/atomic" "time" + "github.com/containerd/log" + distribution "github.com/distribution/reference" imagedigest "github.com/opencontainers/go-digest" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -47,8 +48,6 @@ import ( "github.com/containerd/containerd/v2/remotes/docker" "github.com/containerd/containerd/v2/remotes/docker/config" "github.com/containerd/containerd/v2/tracing" - "github.com/containerd/log" - distribution "github.com/distribution/reference" ) // For image management: @@ -755,7 +754,7 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i } // TODO: Find other way to retrieve sandbox runtime, this must belong to the Runtime part of the CRI. - ociRuntime, err := c.getSandboxRuntime(s, runtimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(s, runtimeHandler) if err != nil { return "", fmt.Errorf("experimental: failed to get sandbox runtime for %s: %w", runtimeHandler, err) } @@ -764,55 +763,3 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i log.G(ctx).Infof("experimental: PullImage %q for runtime %s, using snapshotter %s", imageRef, runtimeHandler, snapshotter) return snapshotter, nil } - -// TODO: copy-pasted from the runtime service implementation. This should not be in image service. -func (c *CRIImageService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} - -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} diff --git a/pkg/cri/server/podsandbox/sandbox_run.go b/pkg/cri/server/podsandbox/sandbox_run.go index e0a731431..99e31ccdb 100644 --- a/pkg/cri/server/podsandbox/sandbox_run.go +++ b/pkg/cri/server/podsandbox/sandbox_run.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" + "github.com/containerd/log" "github.com/containerd/nri" v1 "github.com/containerd/nri/types/v1" "github.com/containerd/typeurl/v2" @@ -32,15 +32,13 @@ import ( containerdio "github.com/containerd/containerd/v2/cio" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/errdefs" - "github.com/containerd/containerd/v2/pkg/cri/annotations" - criconfig "github.com/containerd/containerd/v2/pkg/cri/config" crilabels "github.com/containerd/containerd/v2/pkg/cri/labels" customopts "github.com/containerd/containerd/v2/pkg/cri/opts" + imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" "github.com/containerd/containerd/v2/sandbox" "github.com/containerd/containerd/v2/snapshots" - "github.com/containerd/log" ) func init() { @@ -91,7 +89,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err) } - ociRuntime, err := c.getSandboxRuntime(config, metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler) if err != nil { return cin, fmt.Errorf("failed to get sandbox runtime: %w", err) } @@ -300,57 +298,3 @@ func (c *Controller) ensureImageExists(ctx context.Context, ref string, config * } return &newImage, nil } - -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} - -// getSandboxRuntime returns the runtime configuration for sandbox. -// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned, -// or else default runtime will be returned. -func (c *Controller) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} diff --git a/pkg/cri/server/podsandbox/sandbox_run_test.go b/pkg/cri/server/podsandbox/sandbox_run_test.go index d719643f9..5399fe1e0 100644 --- a/pkg/cri/server/podsandbox/sandbox_run_test.go +++ b/pkg/cri/server/podsandbox/sandbox_run_test.go @@ -170,50 +170,3 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { }) } } - -func TestHostAccessingSandbox(t *testing.T) { - privilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - }, - } - nonPrivilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - }, - }, - } - hostNamespace := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - }, - }, - } - tests := []struct { - name string - config *runtime.PodSandboxConfig - want bool - }{ - {"Security Context is nil", nil, false}, - {"Security Context is privileged", privilegedContext, false}, - {"Security Context is not privileged", nonPrivilegedContext, false}, - {"Security Context namespace host access", hostNamespace, true}, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - if got := hostAccessingSandbox(tt.config); got != tt.want { - t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/cri/server/sandbox_remove.go b/pkg/cri/server/sandbox_remove.go index 5b9e7633a..916042283 100644 --- a/pkg/cri/server/sandbox_remove.go +++ b/pkg/cri/server/sandbox_remove.go @@ -80,7 +80,7 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS } // Use sandbox controller to delete sandbox - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index b2aed95a7..fcb9bf858 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -86,7 +86,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox sandboxInfo = sb.Sandbox{ID: id} ) - ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler()) + ociRuntime, err := c.config.GetSandboxRuntime(config, r.GetRuntimeHandler()) if err != nil { return nil, fmt.Errorf("unable to get OCI runtime for sandbox %q: %w", id, err) } @@ -239,7 +239,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err) } - controller, err := c.getSandboxController(config, r.GetRuntimeHandler()) + controller, err := c.sandboxService.SandboxController(config, r.GetRuntimeHandler()) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -630,60 +630,6 @@ func ipString(ip *cni.IPConfig) string { return ip.IP.String() } -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} - -// getSandboxRuntime returns the runtime configuration for sandbox. -// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned, -// or else default runtime will be returned. -func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} - func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) { if log.GetLevel() < log.DebugLevel { return diff --git a/pkg/cri/server/sandbox_service.go b/pkg/cri/server/sandbox_service.go new file mode 100644 index 000000000..51c0ee1b6 --- /dev/null +++ b/pkg/cri/server/sandbox_service.go @@ -0,0 +1,47 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "fmt" + + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/client" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/sandbox" +) + +type criSandboxService struct { + cli *client.Client + config *criconfig.Config +} + +func newCriSandboxService(config *criconfig.Config, c *client.Client) *criSandboxService { + return &criSandboxService{ + cli: c, + config: config, + } +} + +func (c *criSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { + ociRuntime, err := c.config.GetSandboxRuntime(config, runtimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) + } + return c.cli.SandboxController(ociRuntime.Sandboxer), nil +} diff --git a/pkg/cri/server/sandbox_stats_windows.go b/pkg/cri/server/sandbox_stats_windows.go index cf8598372..df9a2b51c 100644 --- a/pkg/cri/server/sandbox_stats_windows.go +++ b/pkg/cri/server/sandbox_stats_windows.go @@ -121,7 +121,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma return nil, nil, fmt.Errorf("failed to find container metric for pod with id %s", sandbox.ID) } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err) } diff --git a/pkg/cri/server/sandbox_status.go b/pkg/cri/server/sandbox_status.go index a475dec52..74ccfc6bf 100644 --- a/pkg/cri/server/sandbox_status.go +++ b/pkg/cri/server/sandbox_status.go @@ -40,7 +40,7 @@ func (c *criService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandbox return nil, fmt.Errorf("failed to get sandbox ip: %w", err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/sandbox_stop.go b/pkg/cri/server/sandbox_stop.go index 98b9c85a9..35a20da28 100644 --- a/pkg/cri/server/sandbox_stop.go +++ b/pkg/cri/server/sandbox_stop.go @@ -68,7 +68,7 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa state := sandbox.Status.Get().State if state == sandboxstore.StateReady || state == sandboxstore.StateUnknown { // Use sandbox controller to stop sandbox - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 498a8f5ae..ab073e000 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -27,30 +27,29 @@ import ( "sync" "sync/atomic" - containerd "github.com/containerd/containerd/v2/client" - "github.com/containerd/containerd/v2/oci" - "github.com/containerd/containerd/v2/pkg/cri/instrument" - "github.com/containerd/containerd/v2/pkg/cri/nri" - "github.com/containerd/containerd/v2/pkg/cri/server/images" - "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" - imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" - snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" - "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/containerd/v2/sandbox" "github.com/containerd/go-cni" "github.com/containerd/log" "google.golang.org/grpc" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubelet/pkg/cri/streaming" - "github.com/containerd/containerd/v2/pkg/cri/store/label" - + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/instrument" + "github.com/containerd/containerd/v2/pkg/cri/nri" + "github.com/containerd/containerd/v2/pkg/cri/server/images" + "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" + imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" + "github.com/containerd/containerd/v2/pkg/cri/store/label" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" + snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" osinterface "github.com/containerd/containerd/v2/pkg/os" "github.com/containerd/containerd/v2/pkg/registrar" + "github.com/containerd/containerd/v2/plugins" + "github.com/containerd/containerd/v2/sandbox" ) // defaultNetworkPlugin is used for the default CNI configuration @@ -68,6 +67,10 @@ type CRIService interface { Register(*grpc.Server) error } +type sandboxService interface { + SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) +} + // imageService specifies dependencies to image service. type imageService interface { runtime.ImageServiceServer @@ -125,8 +128,8 @@ type criService struct { containerEventsChan chan runtime.ContainerEventResponse // nri is used to hook NRI into CRI request processing. nri *nri.API - // sbcontrollers are the configured sandbox controllers - sbControllers map[string]sandbox.Controller + // sandboxService is the sandbox related service for CRI + sandboxService sandboxService } // NewCRIService returns a new instance of CRIService @@ -169,7 +172,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. sandboxNameIndex: registrar.NewRegistrar(), containerNameIndex: registrar.NewRegistrar(), netPlugin: make(map[string]cni.CNI), - sbControllers: sbControllers, + sandboxService: newCriSandboxService(&config, client), } // TODO: figure out a proper channel size. @@ -357,22 +360,6 @@ func (c *criService) register(s *grpc.Server) error { return nil } -// getSandboxController returns the sandbox controller configuration for sandbox. -// If absent in legacy case, it will return the default controller. -func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { - ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler) - if err != nil { - return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) - } - - controller, ok := c.sbControllers[ociRuntime.Sandboxer] - if !ok { - return nil, fmt.Errorf("no sandbox controller %s for runtime %s", ociRuntime.Sandboxer, runtimeHandler) - } - - return controller, nil -} - // imageFSPath returns containerd image filesystem path. // Note that if containerd changes directory layout, we also needs to change this. func imageFSPath(rootDir, snapshotter string) string { diff --git a/pkg/cri/server/service_test.go b/pkg/cri/server/service_test.go index 41b997f9b..0800dc73e 100644 --- a/pkg/cri/server/service_test.go +++ b/pkg/cri/server/service_test.go @@ -18,16 +18,23 @@ package server import ( "bytes" + "context" "encoding/json" "io" "os" "testing" - "github.com/containerd/containerd/v2/oci" "github.com/containerd/go-cni" + "github.com/containerd/log" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/api/types" + "github.com/containerd/containerd/v2/errdefs" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" "github.com/containerd/containerd/v2/pkg/cri/store/label" @@ -35,11 +42,50 @@ import ( servertesting "github.com/containerd/containerd/v2/pkg/cri/testing" ostesting "github.com/containerd/containerd/v2/pkg/os/testing" "github.com/containerd/containerd/v2/pkg/registrar" - "github.com/containerd/log" - "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" + "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/containerd/v2/sandbox" ) +type fakeSandboxService struct{} + +func (f *fakeSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { + return &fakeSandboxController{}, nil +} + +type fakeSandboxController struct{} + +func (f fakeSandboxController) Create(_ctx context.Context, _sandboxInfo sandbox.Sandbox, _opts ...sandbox.CreateOpt) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Start(ctx context.Context, sandboxID string) (sandbox.ControllerInstance, error) { + return sandbox.ControllerInstance{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Platform(_ctx context.Context, _sandboxID string) (platforms.Platform, error) { + return platforms.DefaultSpec(), nil +} + +func (f fakeSandboxController) Stop(_ctx context.Context, _sandboxID string, _opts ...sandbox.StopOpt) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Wait(_ctx context.Context, _sandboxID string) (sandbox.ExitStatus, error) { + return sandbox.ExitStatus{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Status(_ctx context.Context, sandboxID string, _verbose bool) (sandbox.ControllerStatus, error) { + return sandbox.ControllerStatus{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Shutdown(ctx context.Context, sandboxID string) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Metrics(ctx context.Context, sandboxID string) (*types.Metric, error) { + return &types.Metric{}, errdefs.ErrNotImplemented +} + // newTestCRIService creates a fake criService for test. func newTestCRIService() *criService { labels := label.NewStore() @@ -54,6 +100,7 @@ func newTestCRIService() *criService { netPlugin: map[string]cni.CNI{ defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(), }, + sandboxService: &fakeSandboxService{}, } }