diff --git a/client/client.go b/client/client.go index 714c57c25..036e6e5ac 100644 --- a/client/client.go +++ b/client/client.go @@ -720,7 +720,6 @@ func (c *Client) SandboxStore() sandbox.Store { // SandboxController returns the underlying sandbox controller client func (c *Client) SandboxController(name string) sandbox.Controller { - // default sandboxer is shim if c.sandboxers != nil { return c.sandboxers[name] } diff --git a/client/services.go b/client/services.go index 3ca357eb3..ab8b15ff7 100644 --- a/client/services.go +++ b/client/services.go @@ -87,16 +87,6 @@ func WithSnapshotters(snapshotters map[string]snapshots.Snapshotter) ServicesOpt } } -// WithSandboxers sets the sandbox controllers. -func WithSandboxers(sandboxers map[string]sandbox.Controller) ServicesOpt { - return func(s *services) { - s.sandboxers = make(map[string]sandbox.Controller) - for n, sn := range sandboxers { - s.sandboxers[n] = sn - } - } -} - // WithContainerClient sets the container service to use using a containers client. func WithContainerClient(containerService containersapi.ContainersClient) ServicesOpt { return func(s *services) { @@ -218,9 +208,6 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { srv.SnapshotsService: func(s interface{}) ServicesOpt { return WithSnapshotters(s.(map[string]snapshots.Snapshotter)) }, - srv.SandboxControllersService: func(s interface{}) ServicesOpt { - return WithSandboxers(s.(map[string]sandbox.Controller)) - }, srv.ContainersService: func(s interface{}) ServicesOpt { return WithContainerClient(s.(containersapi.ContainersClient)) }, @@ -251,3 +238,18 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt { return nil } } + +func WithInMemorySandboxControllers(ic *plugin.InitContext) Opt { + return func(c *clientOpts) error { + sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin) + if err != nil { + return err + } + sc := make(map[string]sandbox.Controller) + for name, p := range sandboxers { + sc[name] = p.(sandbox.Controller) + } + c.services.sandboxers = sc + return nil + } +} diff --git a/integration/build_local_containerd_helper_test.go b/integration/build_local_containerd_helper_test.go index cbd3fcb89..f49df219a 100644 --- a/integration/build_local_containerd_helper_test.go +++ b/integration/build_local_containerd_helper_test.go @@ -118,6 +118,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(lastInitContext), + containerd.WithInMemorySandboxControllers(lastInitContext), ) assert.NoError(t, err) diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 6a8cd84b0..43e954b5b 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -24,7 +24,9 @@ import ( "time" "github.com/containerd/log" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/pkg/cri/annotations" "github.com/containerd/containerd/v2/pkg/deprecation" ) @@ -454,3 +456,55 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W } return warnings, nil } + +func (config *Config) GetSandboxRuntime(podSandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (Runtime, error) { + if untrustedWorkload(podSandboxConfig) { + // If the untrusted annotation is provided, runtimeHandler MUST be empty. + if runtimeHandler != "" && runtimeHandler != RuntimeUntrusted { + return Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") + } + + // If the untrusted workload is requesting access to the host/node, this request will fail. + // + // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the + // runtime may support this. For example, in a virtual-machine isolated runtime, privileged + // is a supported option, granting the workload to access the entire guest VM instead of host. + // TODO(windows): Deprecate this so that we don't need to handle it for windows. + if hostAccessingSandbox(podSandboxConfig) { + return Runtime{}, errors.New("untrusted workload with host access is not allowed") + } + + runtimeHandler = RuntimeUntrusted + } + + if runtimeHandler == "" { + runtimeHandler = config.DefaultRuntimeName + } + + r, ok := config.Runtimes[runtimeHandler] + if !ok { + return Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) + } + return r, nil + +} + +// untrustedWorkload returns true if the sandbox contains untrusted workload. +func untrustedWorkload(config *runtime.PodSandboxConfig) bool { + return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" +} + +// hostAccessingSandbox returns true if the sandbox configuration +// requires additional host access for the sandbox. +func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { + securityContext := config.GetLinux().GetSecurityContext() + + namespaceOptions := securityContext.GetNamespaceOptions() + if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || + namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || + namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { + return true + } + + return false +} diff --git a/pkg/cri/config/config_test.go b/pkg/cri/config/config_test.go index d442dda1f..362e49161 100644 --- a/pkg/cri/config/config_test.go +++ b/pkg/cri/config/config_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/containerd/containerd/v2/pkg/deprecation" ) @@ -232,3 +233,50 @@ func TestValidateConfig(t *testing.T) { }) } } + +func TestHostAccessingSandbox(t *testing.T) { + privilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: true, + }, + }, + } + nonPrivilegedContext := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + }, + }, + } + hostNamespace := &runtime.PodSandboxConfig{ + Linux: &runtime.LinuxPodSandboxConfig{ + SecurityContext: &runtime.LinuxSandboxSecurityContext{ + Privileged: false, + NamespaceOptions: &runtime.NamespaceOption{ + Network: runtime.NamespaceMode_NODE, + Pid: runtime.NamespaceMode_NODE, + Ipc: runtime.NamespaceMode_NODE, + }, + }, + }, + } + tests := []struct { + name string + config *runtime.PodSandboxConfig + want bool + }{ + {"Security Context is nil", nil, false}, + {"Security Context is privileged", privilegedContext, false}, + {"Security Context is not privileged", nonPrivilegedContext, false}, + {"Security Context namespace host access", hostNamespace, true}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := hostAccessingSandbox(tt.config); got != tt.want { + t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index e85547e68..819e2b8e4 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -50,6 +50,7 @@ func init() { plugins.ServicePlugin, plugins.NRIApiPlugin, plugins.WarningPlugin, + plugins.SandboxControllerPlugin, }, InitFn: initCRIService, }) @@ -92,6 +93,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(ic), + containerd.WithInMemorySandboxControllers(ic), ) if err != nil { return nil, fmt.Errorf("failed to create containerd client: %w", err) diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index a0bd37ba9..95571fc95 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/containerd/log" "github.com/containerd/typeurl/v2" "github.com/davecgh/go-spew/spew" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -45,7 +46,6 @@ import ( containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" "github.com/containerd/containerd/v2/pkg/cri/util" "github.com/containerd/containerd/v2/platforms" - "github.com/containerd/log" ) func init() { @@ -63,7 +63,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta return nil, fmt.Errorf("failed to find sandbox id %q: %w", r.GetPodSandboxId(), err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -163,7 +163,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID) } - ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) } diff --git a/pkg/cri/server/container_start.go b/pkg/cri/server/container_start.go index d37e7933a..14d0ab4ab 100644 --- a/pkg/cri/server/container_start.go +++ b/pkg/cri/server/container_start.go @@ -109,7 +109,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain return cntr.IO, nil } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) } diff --git a/pkg/cri/server/container_stats_list.go b/pkg/cri/server/container_stats_list.go index 3eadc6895..ef4865990 100644 --- a/pkg/cri/server/container_stats_list.go +++ b/pkg/cri/server/container_stats_list.go @@ -26,16 +26,16 @@ import ( wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" cg1 "github.com/containerd/cgroups/v3/cgroup1/stats" cg2 "github.com/containerd/cgroups/v3/cgroup2/stats" - "github.com/containerd/containerd/v2/api/services/tasks/v1" - "github.com/containerd/containerd/v2/api/types" - "github.com/containerd/containerd/v2/errdefs" - "github.com/containerd/containerd/v2/pkg/cri/store/stats" - "github.com/containerd/containerd/v2/protobuf" "github.com/containerd/log" "github.com/containerd/typeurl/v2" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/api/services/tasks/v1" + "github.com/containerd/containerd/v2/api/types" + "github.com/containerd/containerd/v2/errdefs" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" + "github.com/containerd/containerd/v2/pkg/cri/store/stats" + "github.com/containerd/containerd/v2/protobuf" ) // ListContainerStats returns stats of all running containers. @@ -68,7 +68,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m if err != nil { return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -81,7 +81,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m return nil, err } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err) } diff --git a/pkg/cri/server/images/image_pull.go b/pkg/cri/server/images/image_pull.go index 07d6ea808..cb9db2738 100644 --- a/pkg/cri/server/images/image_pull.go +++ b/pkg/cri/server/images/image_pull.go @@ -20,7 +20,6 @@ import ( "context" "crypto/tls" "encoding/base64" - "errors" "fmt" "io" "net" @@ -33,6 +32,8 @@ import ( "sync/atomic" "time" + "github.com/containerd/log" + distribution "github.com/distribution/reference" imagedigest "github.com/opencontainers/go-digest" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -47,8 +48,6 @@ import ( "github.com/containerd/containerd/v2/remotes/docker" "github.com/containerd/containerd/v2/remotes/docker/config" "github.com/containerd/containerd/v2/tracing" - "github.com/containerd/log" - distribution "github.com/distribution/reference" ) // For image management: @@ -755,7 +754,7 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i } // TODO: Find other way to retrieve sandbox runtime, this must belong to the Runtime part of the CRI. - ociRuntime, err := c.getSandboxRuntime(s, runtimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(s, runtimeHandler) if err != nil { return "", fmt.Errorf("experimental: failed to get sandbox runtime for %s: %w", runtimeHandler, err) } @@ -764,55 +763,3 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i log.G(ctx).Infof("experimental: PullImage %q for runtime %s, using snapshotter %s", imageRef, runtimeHandler, snapshotter) return snapshotter, nil } - -// TODO: copy-pasted from the runtime service implementation. This should not be in image service. -func (c *CRIImageService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} - -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} diff --git a/pkg/cri/server/podsandbox/controller.go b/pkg/cri/server/podsandbox/controller.go index 45b3faf7d..ff3de7a0f 100644 --- a/pkg/cri/server/podsandbox/controller.go +++ b/pkg/cri/server/podsandbox/controller.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/constants" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" @@ -43,13 +44,25 @@ import ( func init() { registry.Register(&plugin.Registration{ - Type: plugins.SandboxControllerPlugin, - ID: "podsandbox", - Requires: []plugin.Type{}, + Type: plugins.SandboxControllerPlugin, + ID: "podsandbox", + Requires: []plugin.Type{ + plugins.EventPlugin, + plugins.ServicePlugin, + }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - // register the global controller to containerd plugin manager, - // the global controller will be initialized when cri plugin is initializing - return &Controller{}, nil + c := Controller{} + client, err := containerd.New( + "", + containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + ) + if err != nil { + return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) + } + c.client = client + return &c, nil }, }) } @@ -90,7 +103,6 @@ type Controller struct { func (c *Controller) Init( config criconfig.Config, - client *containerd.Client, sandboxStore *sandboxstore.Store, os osinterface.OS, cri CRIService, @@ -98,7 +110,6 @@ func (c *Controller) Init( baseOCISpecs map[string]*oci.Spec, ) { c.cri = cri - c.client = client c.config = config c.sandboxStore = sandboxStore c.os = os diff --git a/pkg/cri/server/podsandbox/sandbox_run.go b/pkg/cri/server/podsandbox/sandbox_run.go index e0a731431..99e31ccdb 100644 --- a/pkg/cri/server/podsandbox/sandbox_run.go +++ b/pkg/cri/server/podsandbox/sandbox_run.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" + "github.com/containerd/log" "github.com/containerd/nri" v1 "github.com/containerd/nri/types/v1" "github.com/containerd/typeurl/v2" @@ -32,15 +32,13 @@ import ( containerdio "github.com/containerd/containerd/v2/cio" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/errdefs" - "github.com/containerd/containerd/v2/pkg/cri/annotations" - criconfig "github.com/containerd/containerd/v2/pkg/cri/config" crilabels "github.com/containerd/containerd/v2/pkg/cri/labels" customopts "github.com/containerd/containerd/v2/pkg/cri/opts" + imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" "github.com/containerd/containerd/v2/sandbox" "github.com/containerd/containerd/v2/snapshots" - "github.com/containerd/log" ) func init() { @@ -91,7 +89,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err) } - ociRuntime, err := c.getSandboxRuntime(config, metadata.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler) if err != nil { return cin, fmt.Errorf("failed to get sandbox runtime: %w", err) } @@ -300,57 +298,3 @@ func (c *Controller) ensureImageExists(ctx context.Context, ref string, config * } return &newImage, nil } - -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} - -// getSandboxRuntime returns the runtime configuration for sandbox. -// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned, -// or else default runtime will be returned. -func (c *Controller) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} diff --git a/pkg/cri/server/podsandbox/sandbox_run_test.go b/pkg/cri/server/podsandbox/sandbox_run_test.go index d719643f9..5399fe1e0 100644 --- a/pkg/cri/server/podsandbox/sandbox_run_test.go +++ b/pkg/cri/server/podsandbox/sandbox_run_test.go @@ -170,50 +170,3 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { }) } } - -func TestHostAccessingSandbox(t *testing.T) { - privilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - }, - } - nonPrivilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - }, - }, - } - hostNamespace := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - }, - }, - } - tests := []struct { - name string - config *runtime.PodSandboxConfig - want bool - }{ - {"Security Context is nil", nil, false}, - {"Security Context is privileged", privilegedContext, false}, - {"Security Context is not privileged", nonPrivilegedContext, false}, - {"Security Context namespace host access", hostNamespace, true}, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - if got := hostAccessingSandbox(tt.config); got != tt.want { - t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/cri/server/sandbox_remove.go b/pkg/cri/server/sandbox_remove.go index 5b9e7633a..916042283 100644 --- a/pkg/cri/server/sandbox_remove.go +++ b/pkg/cri/server/sandbox_remove.go @@ -80,7 +80,7 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS } // Use sandbox controller to delete sandbox - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index b2aed95a7..fcb9bf858 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -86,7 +86,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox sandboxInfo = sb.Sandbox{ID: id} ) - ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler()) + ociRuntime, err := c.config.GetSandboxRuntime(config, r.GetRuntimeHandler()) if err != nil { return nil, fmt.Errorf("unable to get OCI runtime for sandbox %q: %w", id, err) } @@ -239,7 +239,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err) } - controller, err := c.getSandboxController(config, r.GetRuntimeHandler()) + controller, err := c.sandboxService.SandboxController(config, r.GetRuntimeHandler()) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } @@ -630,60 +630,6 @@ func ipString(ip *cni.IPConfig) string { return ip.IP.String() } -// untrustedWorkload returns true if the sandbox contains untrusted workload. -func untrustedWorkload(config *runtime.PodSandboxConfig) bool { - return config.GetAnnotations()[annotations.UntrustedWorkload] == "true" -} - -// hostAccessingSandbox returns true if the sandbox configuration -// requires additional host access for the sandbox. -func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool { - securityContext := config.GetLinux().GetSecurityContext() - - namespaceOptions := securityContext.GetNamespaceOptions() - if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE || - namespaceOptions.GetPid() == runtime.NamespaceMode_NODE || - namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE { - return true - } - - return false -} - -// getSandboxRuntime returns the runtime configuration for sandbox. -// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned, -// or else default runtime will be returned. -func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) { - if untrustedWorkload(config) { - // If the untrusted annotation is provided, runtimeHandler MUST be empty. - if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted { - return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed") - } - - // If the untrusted workload is requesting access to the host/node, this request will fail. - // - // Note: If the workload is marked untrusted but requests privileged, this can be granted, as the - // runtime may support this. For example, in a virtual-machine isolated runtime, privileged - // is a supported option, granting the workload to access the entire guest VM instead of host. - // TODO(windows): Deprecate this so that we don't need to handle it for windows. - if hostAccessingSandbox(config) { - return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed") - } - - runtimeHandler = criconfig.RuntimeUntrusted - } - - if runtimeHandler == "" { - runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName - } - - handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler] - if !ok { - return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler) - } - return handler, nil -} - func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) { if log.GetLevel() < log.DebugLevel { return diff --git a/pkg/cri/server/sandbox_service.go b/pkg/cri/server/sandbox_service.go new file mode 100644 index 000000000..51c0ee1b6 --- /dev/null +++ b/pkg/cri/server/sandbox_service.go @@ -0,0 +1,47 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "fmt" + + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + + "github.com/containerd/containerd/v2/client" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/sandbox" +) + +type criSandboxService struct { + cli *client.Client + config *criconfig.Config +} + +func newCriSandboxService(config *criconfig.Config, c *client.Client) *criSandboxService { + return &criSandboxService{ + cli: c, + config: config, + } +} + +func (c *criSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { + ociRuntime, err := c.config.GetSandboxRuntime(config, runtimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) + } + return c.cli.SandboxController(ociRuntime.Sandboxer), nil +} diff --git a/pkg/cri/server/sandbox_stats_windows.go b/pkg/cri/server/sandbox_stats_windows.go index cf8598372..df9a2b51c 100644 --- a/pkg/cri/server/sandbox_stats_windows.go +++ b/pkg/cri/server/sandbox_stats_windows.go @@ -121,7 +121,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma return nil, nil, fmt.Errorf("failed to find container metric for pod with id %s", sandbox.ID) } - ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) + ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err) } diff --git a/pkg/cri/server/sandbox_status.go b/pkg/cri/server/sandbox_status.go index a475dec52..74ccfc6bf 100644 --- a/pkg/cri/server/sandbox_status.go +++ b/pkg/cri/server/sandbox_status.go @@ -40,7 +40,7 @@ func (c *criService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandbox return nil, fmt.Errorf("failed to get sandbox ip: %w", err) } - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return nil, fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/sandbox_stop.go b/pkg/cri/server/sandbox_stop.go index 98b9c85a9..35a20da28 100644 --- a/pkg/cri/server/sandbox_stop.go +++ b/pkg/cri/server/sandbox_stop.go @@ -68,7 +68,7 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa state := sandbox.Status.Get().State if state == sandboxstore.StateReady || state == sandboxstore.StateUnknown { // Use sandbox controller to stop sandbox - controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler) if err != nil { return fmt.Errorf("failed to get sandbox controller: %w", err) } diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 070c81be4..ab073e000 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -27,30 +27,29 @@ import ( "sync" "sync/atomic" - containerd "github.com/containerd/containerd/v2/client" - "github.com/containerd/containerd/v2/oci" - "github.com/containerd/containerd/v2/pkg/cri/instrument" - "github.com/containerd/containerd/v2/pkg/cri/nri" - "github.com/containerd/containerd/v2/pkg/cri/server/images" - "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" - imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" - snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" - "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/containerd/v2/sandbox" "github.com/containerd/go-cni" "github.com/containerd/log" "google.golang.org/grpc" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubelet/pkg/cri/streaming" - "github.com/containerd/containerd/v2/pkg/cri/store/label" - + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/instrument" + "github.com/containerd/containerd/v2/pkg/cri/nri" + "github.com/containerd/containerd/v2/pkg/cri/server/images" + "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" + imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" + "github.com/containerd/containerd/v2/pkg/cri/store/label" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" + snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" osinterface "github.com/containerd/containerd/v2/pkg/os" "github.com/containerd/containerd/v2/pkg/registrar" + "github.com/containerd/containerd/v2/plugins" + "github.com/containerd/containerd/v2/sandbox" ) // defaultNetworkPlugin is used for the default CNI configuration @@ -68,6 +67,10 @@ type CRIService interface { Register(*grpc.Server) error } +type sandboxService interface { + SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) +} + // imageService specifies dependencies to image service. type imageService interface { runtime.ImageServiceServer @@ -125,8 +128,8 @@ type criService struct { containerEventsChan chan runtime.ContainerEventResponse // nri is used to hook NRI into CRI request processing. nri *nri.API - // sbcontrollers are the configured sandbox controllers - sbControllers map[string]sandbox.Controller + // sandboxService is the sandbox related service for CRI + sandboxService sandboxService } // NewCRIService returns a new instance of CRIService @@ -138,11 +141,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter) } - // TODO(dmcgowan): Get the full list directly from configured plugins - sbControllers := map[string]sandbox.Controller{ - string(criconfig.ModePodSandbox): client.SandboxController(string(criconfig.ModePodSandbox)), - string(criconfig.ModeShim): client.SandboxController(string(criconfig.ModeShim)), - } imageFSPaths := map[string]string{} for _, ociRuntime := range config.ContainerdConfig.Runtimes { // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` @@ -151,10 +149,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) } - if _, ok := sbControllers[ociRuntime.Sandboxer]; !ok { - sbControllers[ociRuntime.Sandboxer] = client.SandboxController(ociRuntime.Sandboxer) - } } + snapshotter := config.ContainerdConfig.Snapshotter imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) @@ -176,7 +172,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. sandboxNameIndex: registrar.NewRegistrar(), containerNameIndex: registrar.NewRegistrar(), netPlugin: make(map[string]cni.CNI), - sbControllers: sbControllers, + sandboxService: newCriSandboxService(&config, client), } // TODO: figure out a proper channel size. @@ -217,8 +213,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, err } + podSandboxController := client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller) // Initialize pod sandbox controller - sbControllers[string(criconfig.ModePodSandbox)].(*podsandbox.Controller).Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) + podSandboxController.Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) c.nri = nri @@ -363,22 +360,6 @@ func (c *criService) register(s *grpc.Server) error { return nil } -// getSandboxController returns the sandbox controller configuration for sandbox. -// If absent in legacy case, it will return the default controller. -func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { - ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler) - if err != nil { - return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) - } - - controller, ok := c.sbControllers[ociRuntime.Sandboxer] - if !ok { - return nil, fmt.Errorf("no sandbox controller %s for runtime %s", ociRuntime.Sandboxer, runtimeHandler) - } - - return controller, nil -} - // imageFSPath returns containerd image filesystem path. // Note that if containerd changes directory layout, we also needs to change this. func imageFSPath(rootDir, snapshotter string) string { diff --git a/pkg/cri/server/service_test.go b/pkg/cri/server/service_test.go index 41b997f9b..0800dc73e 100644 --- a/pkg/cri/server/service_test.go +++ b/pkg/cri/server/service_test.go @@ -18,16 +18,23 @@ package server import ( "bytes" + "context" "encoding/json" "io" "os" "testing" - "github.com/containerd/containerd/v2/oci" "github.com/containerd/go-cni" + "github.com/containerd/log" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/api/types" + "github.com/containerd/containerd/v2/errdefs" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" "github.com/containerd/containerd/v2/pkg/cri/store/label" @@ -35,11 +42,50 @@ import ( servertesting "github.com/containerd/containerd/v2/pkg/cri/testing" ostesting "github.com/containerd/containerd/v2/pkg/os/testing" "github.com/containerd/containerd/v2/pkg/registrar" - "github.com/containerd/log" - "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" + "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/containerd/v2/sandbox" ) +type fakeSandboxService struct{} + +func (f *fakeSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { + return &fakeSandboxController{}, nil +} + +type fakeSandboxController struct{} + +func (f fakeSandboxController) Create(_ctx context.Context, _sandboxInfo sandbox.Sandbox, _opts ...sandbox.CreateOpt) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Start(ctx context.Context, sandboxID string) (sandbox.ControllerInstance, error) { + return sandbox.ControllerInstance{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Platform(_ctx context.Context, _sandboxID string) (platforms.Platform, error) { + return platforms.DefaultSpec(), nil +} + +func (f fakeSandboxController) Stop(_ctx context.Context, _sandboxID string, _opts ...sandbox.StopOpt) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Wait(_ctx context.Context, _sandboxID string) (sandbox.ExitStatus, error) { + return sandbox.ExitStatus{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Status(_ctx context.Context, sandboxID string, _verbose bool) (sandbox.ControllerStatus, error) { + return sandbox.ControllerStatus{}, errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Shutdown(ctx context.Context, sandboxID string) error { + return errdefs.ErrNotImplemented +} + +func (f fakeSandboxController) Metrics(ctx context.Context, sandboxID string) (*types.Metric, error) { + return &types.Metric{}, errdefs.ErrNotImplemented +} + // newTestCRIService creates a fake criService for test. func newTestCRIService() *criService { labels := label.NewStore() @@ -54,6 +100,7 @@ func newTestCRIService() *criService { netPlugin: map[string]cni.CNI{ defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(), }, + sandboxService: &fakeSandboxService{}, } } diff --git a/services/sandbox/controller_service.go b/services/sandbox/controller_service.go index dd8cc352e..84a5367e8 100644 --- a/services/sandbox/controller_service.go +++ b/services/sandbox/controller_service.go @@ -31,7 +31,6 @@ import ( "github.com/containerd/containerd/v2/plugins" "github.com/containerd/containerd/v2/protobuf" "github.com/containerd/containerd/v2/sandbox" - "github.com/containerd/containerd/v2/services" "github.com/containerd/log" "github.com/containerd/plugin" "github.com/containerd/plugin/registry" @@ -42,15 +41,20 @@ func init() { Type: plugins.GRPCPlugin, ID: "sandbox-controllers", Requires: []plugin.Type{ - plugins.ServicePlugin, + plugins.SandboxControllerPlugin, plugins.EventPlugin, }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - i, err := ic.GetByID(plugins.ServicePlugin, services.SandboxControllersService) + sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin) if err != nil { return nil, err } - sc := i.(map[string]sandbox.Controller) + + sc := make(map[string]sandbox.Controller) + for name, p := range sandboxers { + sc[name] = p.(sandbox.Controller) + } + ep, err := ic.GetSingle(plugins.EventPlugin) if err != nil { return nil, err diff --git a/services/sandbox/sandboxers.go b/services/sandbox/sandboxers.go deleted file mode 100644 index 85db67018..000000000 --- a/services/sandbox/sandboxers.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sandbox - -import ( - "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/containerd/v2/sandbox" - "github.com/containerd/containerd/v2/services" - "github.com/containerd/plugin" - "github.com/containerd/plugin/registry" -) - -func init() { - registry.Register(&plugin.Registration{ - Type: plugins.ServicePlugin, - ID: services.SandboxControllersService, - Requires: []plugin.Type{ - plugins.SandboxControllerPlugin, - }, - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - sandboxesRaw, err := ic.GetByType(plugins.SandboxControllerPlugin) - if err != nil { - return nil, err - } - sandboxers := make(map[string]sandbox.Controller) - for name, srv := range sandboxesRaw { - sandboxers[name] = srv.(sandbox.Controller) - } - return sandboxers, nil - }, - }) -}