github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #9275 from abel-von/sandbox-plugin-1019

Browse Source 
sandbox: podsandbox controller init its own client
This commit is contained in:
Fu Wei 2023-11-16 10:01:02 +00:00 committed by GitHub
commit 09723a6175
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 289 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/client.go
View File

@ -720,7 +720,6 @@ func (c *Client) SandboxStore() sandbox.Store {
// SandboxController returns the underlying sandbox controller client
func (c *Client) SandboxController(name string) sandbox.Controller {
// default sandboxer is shim
if c.sandboxers != nil {
return c.sandboxers[name]
}

28
client/services.go
View File

@ -87,16 +87,6 @@ func WithSnapshotters(snapshotters map[string]snapshots.Snapshotter) ServicesOpt
}
}
// WithSandboxers sets the sandbox controllers.
func WithSandboxers(sandboxers map[string]sandbox.Controller) ServicesOpt {
return func(s *services) {
s.sandboxers = make(map[string]sandbox.Controller)
for n, sn := range sandboxers {
s.sandboxers[n] = sn
}
}
}
// WithContainerClient sets the container service to use using a containers client.
func WithContainerClient(containerService containersapi.ContainersClient) ServicesOpt {
return func(s *services) {
@ -218,9 +208,6 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt {
srv.SnapshotsService: func(s interface{}) ServicesOpt {
return WithSnapshotters(s.(map[string]snapshots.Snapshotter))
},
srv.SandboxControllersService: func(s interface{}) ServicesOpt {
return WithSandboxers(s.(map[string]sandbox.Controller))
},
srv.ContainersService: func(s interface{}) ServicesOpt {
return WithContainerClient(s.(containersapi.ContainersClient))
},
@ -251,3 +238,18 @@ func WithInMemoryServices(ic *plugin.InitContext) Opt {
return nil
}
}
func WithInMemorySandboxControllers(ic *plugin.InitContext) Opt {
return func(c *clientOpts) error {
sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin)
if err != nil {
return err
}
sc := make(map[string]sandbox.Controller)
for name, p := range sandboxers {
sc[name] = p.(sandbox.Controller)
}
c.services.sandboxers = sc
return nil
}
}

1
integration/build_local_containerd_helper_test.go
View File

@ -118,6 +118,7 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client
containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
containerd.WithDefaultPlatform(platforms.Default()),
containerd.WithInMemoryServices(lastInitContext),
containerd.WithInMemorySandboxControllers(lastInitContext),
)
assert.NoError(t, err)

54
pkg/cri/config/config.go
View File

@ -24,7 +24,9 @@ import (
"time"
"github.com/containerd/log"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/v2/pkg/cri/annotations"
"github.com/containerd/containerd/v2/pkg/deprecation"
)
@ -454,3 +456,55 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W
}
return warnings, nil
}
func (config *Config) GetSandboxRuntime(podSandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (Runtime, error) {
if untrustedWorkload(podSandboxConfig) {
// If the untrusted annotation is provided, runtimeHandler MUST be empty.
if runtimeHandler != "" && runtimeHandler != RuntimeUntrusted {
return Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
}
// If the untrusted workload is requesting access to the host/node, this request will fail.
//
// Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
// runtime may support this. For example, in a virtual-machine isolated runtime, privileged
// is a supported option, granting the workload to access the entire guest VM instead of host.
// TODO(windows): Deprecate this so that we don't need to handle it for windows.
if hostAccessingSandbox(podSandboxConfig) {
return Runtime{}, errors.New("untrusted workload with host access is not allowed")
}
runtimeHandler = RuntimeUntrusted
}
if runtimeHandler == "" {
runtimeHandler = config.DefaultRuntimeName
}
r, ok := config.Runtimes[runtimeHandler]
if !ok {
return Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler)
}
return r, nil
}
// untrustedWorkload returns true if the sandbox contains untrusted workload.
func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
}
// hostAccessingSandbox returns true if the sandbox configuration
// requires additional host access for the sandbox.
func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
securityContext := config.GetLinux().GetSecurityContext()
namespaceOptions := securityContext.GetNamespaceOptions()
if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
return true
}
return false
}

48
pkg/cri/config/config_test.go
View File

@ -21,6 +21,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/v2/pkg/deprecation"
)
@ -232,3 +233,50 @@ func TestValidateConfig(t *testing.T) {
})
}
}
func TestHostAccessingSandbox(t *testing.T) {
privilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: true,
},
},
}
nonPrivilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
},
},
}
hostNamespace := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
},
},
}
tests := []struct {
name string
config *runtime.PodSandboxConfig
want bool
}{
{"Security Context is nil", nil, false},
{"Security Context is privileged", privilegedContext, false},
{"Security Context is not privileged", nonPrivilegedContext, false},
{"Security Context namespace host access", hostNamespace, true},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
if got := hostAccessingSandbox(tt.config); got != tt.want {
t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
}
})
}
}

2
pkg/cri/cri.go
View File

@ -50,6 +50,7 @@ func init() {
plugins.ServicePlugin,
plugins.NRIApiPlugin,
plugins.WarningPlugin,
plugins.SandboxControllerPlugin,
},
InitFn: initCRIService,
})
@ -92,6 +93,7 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
containerd.WithDefaultPlatform(platforms.Default()),
containerd.WithInMemoryServices(ic),
containerd.WithInMemorySandboxControllers(ic),
)
if err != nil {
return nil, fmt.Errorf("failed to create containerd client: %w", err)

6
pkg/cri/server/container_create.go
View File

@ -25,6 +25,7 @@ import (
"strings"
"time"
"github.com/containerd/log"
"github.com/containerd/typeurl/v2"
"github.com/davecgh/go-spew/spew"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@ -45,7 +46,6 @@ import (
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
"github.com/containerd/containerd/v2/pkg/cri/util"
"github.com/containerd/containerd/v2/platforms"
"github.com/containerd/log"
)
func init() {
@ -63,7 +63,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
return nil, fmt.Errorf("failed to find sandbox id %q: %w", r.GetPodSandboxId(), err)
}
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}
@ -163,7 +163,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID)
}
ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}

2
pkg/cri/server/container_start.go
View File

@ -109,7 +109,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain
return cntr.IO, nil
}
ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.Metadata.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}

14
pkg/cri/server/container_stats_list.go
View File

@ -26,16 +26,16 @@ import (
wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
cg1 "github.com/containerd/cgroups/v3/cgroup1/stats"
cg2 "github.com/containerd/cgroups/v3/cgroup2/stats"
"github.com/containerd/containerd/v2/api/services/tasks/v1"
"github.com/containerd/containerd/v2/api/types"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/pkg/cri/store/stats"
"github.com/containerd/containerd/v2/protobuf"
"github.com/containerd/log"
"github.com/containerd/typeurl/v2"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/v2/api/services/tasks/v1"
"github.com/containerd/containerd/v2/api/types"
"github.com/containerd/containerd/v2/errdefs"
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
"github.com/containerd/containerd/v2/pkg/cri/store/stats"
"github.com/containerd/containerd/v2/protobuf"
)
// ListContainerStats returns stats of all running containers.
@ -68,7 +68,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m
if err != nil {
return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err)
}
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}
@ -81,7 +81,7 @@ func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (m
return nil, err
}
ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err)
}

59
pkg/cri/server/images/image_pull.go
View File

@ -20,7 +20,6 @@ import (
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"net"
@ -33,6 +32,8 @@ import (
"sync/atomic"
"time"
"github.com/containerd/log"
distribution "github.com/distribution/reference"
imagedigest "github.com/opencontainers/go-digest"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
@ -47,8 +48,6 @@ import (
"github.com/containerd/containerd/v2/remotes/docker"
"github.com/containerd/containerd/v2/remotes/docker/config"
"github.com/containerd/containerd/v2/tracing"
"github.com/containerd/log"
distribution "github.com/distribution/reference"
)
// For image management:
@ -755,7 +754,7 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i
}
// TODO: Find other way to retrieve sandbox runtime, this must belong to the Runtime part of the CRI.
ociRuntime, err := c.getSandboxRuntime(s, runtimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(s, runtimeHandler)
if err != nil {
return "", fmt.Errorf("experimental: failed to get sandbox runtime for %s: %w", runtimeHandler, err)
}
@ -764,55 +763,3 @@ func (c *CRIImageService) snapshotterFromPodSandboxConfig(ctx context.Context, i
log.G(ctx).Infof("experimental: PullImage %q for runtime %s, using snapshotter %s", imageRef, runtimeHandler, snapshotter)
return snapshotter, nil
}
// TODO: copy-pasted from the runtime service implementation. This should not be in image service.
func (c *CRIImageService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
if untrustedWorkload(config) {
// If the untrusted annotation is provided, runtimeHandler MUST be empty.
if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
}
// If the untrusted workload is requesting access to the host/node, this request will fail.
//
// Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
// runtime may support this. For example, in a virtual-machine isolated runtime, privileged
// is a supported option, granting the workload to access the entire guest VM instead of host.
// TODO(windows): Deprecate this so that we don't need to handle it for windows.
if hostAccessingSandbox(config) {
return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
}
runtimeHandler = criconfig.RuntimeUntrusted
}
if runtimeHandler == "" {
runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
}
handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
if !ok {
return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler)
}
return handler, nil
}
// untrustedWorkload returns true if the sandbox contains untrusted workload.
func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
}
// hostAccessingSandbox returns true if the sandbox configuration
// requires additional host access for the sandbox.
func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
securityContext := config.GetLinux().GetSecurityContext()
namespaceOptions := securityContext.GetNamespaceOptions()
if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
return true
}
return false
}

23
pkg/cri/server/podsandbox/controller.go
View File

@ -29,6 +29,7 @@ import (
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/oci"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
"github.com/containerd/containerd/v2/pkg/cri/constants"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox"
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
@ -45,11 +46,23 @@ func init() {
registry.Register(&plugin.Registration{
Type: plugins.SandboxControllerPlugin,
ID: "podsandbox",
Requires: []plugin.Type{},
Requires: []plugin.Type{
plugins.EventPlugin,
plugins.ServicePlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
// register the global controller to containerd plugin manager,
// the global controller will be initialized when cri plugin is initializing
return &Controller{}, nil
c := Controller{}
client, err := containerd.New(
"",
containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
containerd.WithDefaultPlatform(platforms.Default()),
containerd.WithInMemoryServices(ic),
)
if err != nil {
return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err)
}
c.client = client
return &c, nil
},
})
}
@ -90,7 +103,6 @@ type Controller struct {
func (c *Controller) Init(
config criconfig.Config,
client *containerd.Client,
sandboxStore *sandboxstore.Store,
os osinterface.OS,
cri CRIService,
@ -98,7 +110,6 @@ func (c *Controller) Init(
baseOCISpecs map[string]*oci.Spec,
) {
c.cri = cri
c.client = client
c.config = config
c.sandboxStore = sandboxStore
c.os = os

62
pkg/cri/server/podsandbox/sandbox_run.go
View File

@ -21,7 +21,7 @@ import (
"errors"
"fmt"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
"github.com/containerd/log"
"github.com/containerd/nri"
v1 "github.com/containerd/nri/types/v1"
"github.com/containerd/typeurl/v2"
@ -32,15 +32,13 @@ import (
containerdio "github.com/containerd/containerd/v2/cio"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
crilabels "github.com/containerd/containerd/v2/pkg/cri/labels"
customopts "github.com/containerd/containerd/v2/pkg/cri/opts"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox"
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
"github.com/containerd/containerd/v2/sandbox"
"github.com/containerd/containerd/v2/snapshots"
"github.com/containerd/log"
)
func init() {
@ -91,7 +89,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
return cin, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
}
ociRuntime, err := c.getSandboxRuntime(config, metadata.RuntimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(config, metadata.RuntimeHandler)
if err != nil {
return cin, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
@ -300,57 +298,3 @@ func (c *Controller) ensureImageExists(ctx context.Context, ref string, config *
}
return &newImage, nil
}
// untrustedWorkload returns true if the sandbox contains untrusted workload.
func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
}
// hostAccessingSandbox returns true if the sandbox configuration
// requires additional host access for the sandbox.
func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
securityContext := config.GetLinux().GetSecurityContext()
namespaceOptions := securityContext.GetNamespaceOptions()
if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
return true
}
return false
}
// getSandboxRuntime returns the runtime configuration for sandbox.
// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned,
// or else default runtime will be returned.
func (c *Controller) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
if untrustedWorkload(config) {
// If the untrusted annotation is provided, runtimeHandler MUST be empty.
if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
}
// If the untrusted workload is requesting access to the host/node, this request will fail.
//
// Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
// runtime may support this. For example, in a virtual-machine isolated runtime, privileged
// is a supported option, granting the workload to access the entire guest VM instead of host.
// TODO(windows): Deprecate this so that we don't need to handle it for windows.
if hostAccessingSandbox(config) {
return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
}
runtimeHandler = criconfig.RuntimeUntrusted
}
if runtimeHandler == "" {
runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
}
handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
if !ok {
return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler)
}
return handler, nil
}

47
pkg/cri/server/podsandbox/sandbox_run_test.go
View File

@ -170,50 +170,3 @@ func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
})
}
}
func TestHostAccessingSandbox(t *testing.T) {
privilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: true,
},
},
}
nonPrivilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
},
},
}
hostNamespace := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
},
},
}
tests := []struct {
name string
config *runtime.PodSandboxConfig
want bool
}{
{"Security Context is nil", nil, false},
{"Security Context is privileged", privilegedContext, false},
{"Security Context is not privileged", nonPrivilegedContext, false},
{"Security Context namespace host access", hostNamespace, true},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
if got := hostAccessingSandbox(tt.config); got != tt.want {
t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
}
})
}
}

2
pkg/cri/server/sandbox_remove.go
View File

@ -80,7 +80,7 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
}
// Use sandbox controller to delete sandbox
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}

58
pkg/cri/server/sandbox_run.go
View File

@ -86,7 +86,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
sandboxInfo = sb.Sandbox{ID: id}
)
ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler())
ociRuntime, err := c.config.GetSandboxRuntime(config, r.GetRuntimeHandler())
if err != nil {
return nil, fmt.Errorf("unable to get OCI runtime for sandbox %q: %w", id, err)
}
@ -239,7 +239,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
}
controller, err := c.getSandboxController(config, r.GetRuntimeHandler())
controller, err := c.sandboxService.SandboxController(config, r.GetRuntimeHandler())
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}
@ -630,60 +630,6 @@ func ipString(ip *cni.IPConfig) string {
return ip.IP.String()
}
// untrustedWorkload returns true if the sandbox contains untrusted workload.
func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
}
// hostAccessingSandbox returns true if the sandbox configuration
// requires additional host access for the sandbox.
func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
securityContext := config.GetLinux().GetSecurityContext()
namespaceOptions := securityContext.GetNamespaceOptions()
if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
return true
}
return false
}
// getSandboxRuntime returns the runtime configuration for sandbox.
// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned,
// or else default runtime will be returned.
func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
if untrustedWorkload(config) {
// If the untrusted annotation is provided, runtimeHandler MUST be empty.
if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
}
// If the untrusted workload is requesting access to the host/node, this request will fail.
//
// Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
// runtime may support this. For example, in a virtual-machine isolated runtime, privileged
// is a supported option, granting the workload to access the entire guest VM instead of host.
// TODO(windows): Deprecate this so that we don't need to handle it for windows.
if hostAccessingSandbox(config) {
return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
}
runtimeHandler = criconfig.RuntimeUntrusted
}
if runtimeHandler == "" {
runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
}
handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
if !ok {
return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler)
}
return handler, nil
}
func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
if log.GetLevel() < log.DebugLevel {
return

47
pkg/cri/server/sandbox_service.go Normal file
View File

@ -0,0 +1,47 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"fmt"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/v2/client"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
"github.com/containerd/containerd/v2/sandbox"
)
type criSandboxService struct {
cli *client.Client
config *criconfig.Config
}
func newCriSandboxService(config *criconfig.Config, c *client.Client) *criSandboxService {
return &criSandboxService{
cli: c,
config: config,
}
}
func (c *criSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) {
ociRuntime, err := c.config.GetSandboxRuntime(config, runtimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
return c.cli.SandboxController(ociRuntime.Sandboxer), nil
}

2
pkg/cri/server/sandbox_stats_windows.go
View File

@ -121,7 +121,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma
return nil, nil, fmt.Errorf("failed to find container metric for pod with id %s", sandbox.ID)
}
ociRuntime, err := c.getSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler)
ociRuntime, err := c.config.GetSandboxRuntime(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, nil, fmt.Errorf("failed to get runtimeHandler %q: %w", sandbox.RuntimeHandler, err)
}

2
pkg/cri/server/sandbox_status.go
View File

@ -40,7 +40,7 @@ func (c *criService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandbox
return nil, fmt.Errorf("failed to get sandbox ip: %w", err)
}
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}

2
pkg/cri/server/sandbox_stop.go
View File

@ -68,7 +68,7 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
state := sandbox.Status.Get().State
if state == sandboxstore.StateReady || state == sandboxstore.StateUnknown {
// Use sandbox controller to stop sandbox
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
controller, err := c.sandboxService.SandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return fmt.Errorf("failed to get sandbox controller: %w", err)
}

61
pkg/cri/server/service.go
View File

@ -27,30 +27,29 @@ import (
"sync"
"sync/atomic"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/oci"
"github.com/containerd/containerd/v2/pkg/cri/instrument"
"github.com/containerd/containerd/v2/pkg/cri/nri"
"github.com/containerd/containerd/v2/pkg/cri/server/images"
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot"
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/sandbox"
"github.com/containerd/go-cni"
"github.com/containerd/log"
"google.golang.org/grpc"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/kubelet/pkg/cri/streaming"
"github.com/containerd/containerd/v2/pkg/cri/store/label"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/oci"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
"github.com/containerd/containerd/v2/pkg/cri/instrument"
"github.com/containerd/containerd/v2/pkg/cri/nri"
"github.com/containerd/containerd/v2/pkg/cri/server/images"
"github.com/containerd/containerd/v2/pkg/cri/server/podsandbox"
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image"
"github.com/containerd/containerd/v2/pkg/cri/store/label"
sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox"
snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot"
ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util"
osinterface "github.com/containerd/containerd/v2/pkg/os"
"github.com/containerd/containerd/v2/pkg/registrar"
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/sandbox"
)
// defaultNetworkPlugin is used for the default CNI configuration
@ -68,6 +67,10 @@ type CRIService interface {
Register(*grpc.Server) error
}
type sandboxService interface {
SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error)
}
// imageService specifies dependencies to image service.
type imageService interface {
runtime.ImageServiceServer
@ -125,8 +128,8 @@ type criService struct {
containerEventsChan chan runtime.ContainerEventResponse
// nri is used to hook NRI into CRI request processing.
nri *nri.API
// sbcontrollers are the configured sandbox controllers
sbControllers map[string]sandbox.Controller
// sandboxService is the sandbox related service for CRI
sandboxService sandboxService
}
// NewCRIService returns a new instance of CRIService
@ -138,11 +141,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter)
}
// TODO(dmcgowan): Get the full list directly from configured plugins
sbControllers := map[string]sandbox.Controller{
string(criconfig.ModePodSandbox): client.SandboxController(string(criconfig.ModePodSandbox)),
string(criconfig.ModeShim): client.SandboxController(string(criconfig.ModeShim)),
}
imageFSPaths := map[string]string{}
for _, ociRuntime := range config.ContainerdConfig.Runtimes {
// Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.`
@ -151,10 +149,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
}
if _, ok := sbControllers[ociRuntime.Sandboxer]; !ok {
sbControllers[ociRuntime.Sandboxer] = client.SandboxController(ociRuntime.Sandboxer)
}
}
snapshotter := config.ContainerdConfig.Snapshotter
imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
@ -176,7 +172,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
sandboxNameIndex: registrar.NewRegistrar(),
containerNameIndex: registrar.NewRegistrar(),
netPlugin: make(map[string]cni.CNI),
sbControllers: sbControllers,
sandboxService: newCriSandboxService(&config, client),
}
// TODO: figure out a proper channel size.
@ -217,8 +213,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
return nil, err
}
podSandboxController := client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller)
// Initialize pod sandbox controller
sbControllers[string(criconfig.ModePodSandbox)].(*podsandbox.Controller).Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs)
podSandboxController.Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs)
c.nri = nri
@ -363,22 +360,6 @@ func (c *criService) register(s *grpc.Server) error {
return nil
}
// getSandboxController returns the sandbox controller configuration for sandbox.
// If absent in legacy case, it will return the default controller.
func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) {
ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
controller, ok := c.sbControllers[ociRuntime.Sandboxer]
if !ok {
return nil, fmt.Errorf("no sandbox controller %s for runtime %s", ociRuntime.Sandboxer, runtimeHandler)
}
return controller, nil
}
// imageFSPath returns containerd image filesystem path.
// Note that if containerd changes directory layout, we also needs to change this.
func imageFSPath(rootDir, snapshotter string) string {

55
pkg/cri/server/service_test.go
View File

@ -18,16 +18,23 @@ package server
import (
"bytes"
"context"
"encoding/json"
"io"
"os"
"testing"
"github.com/containerd/containerd/v2/oci"
"github.com/containerd/go-cni"
"github.com/containerd/log"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/v2/api/types"
"github.com/containerd/containerd/v2/errdefs"
"github.com/containerd/containerd/v2/oci"
criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container"
"github.com/containerd/containerd/v2/pkg/cri/store/label"
@ -35,11 +42,50 @@ import (
servertesting "github.com/containerd/containerd/v2/pkg/cri/testing"
ostesting "github.com/containerd/containerd/v2/pkg/os/testing"
"github.com/containerd/containerd/v2/pkg/registrar"
"github.com/containerd/log"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
"github.com/containerd/containerd/v2/platforms"
"github.com/containerd/containerd/v2/sandbox"
)
type fakeSandboxService struct{}
func (f *fakeSandboxService) SandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) {
return &fakeSandboxController{}, nil
}
type fakeSandboxController struct{}
func (f fakeSandboxController) Create(_ctx context.Context, _sandboxInfo sandbox.Sandbox, _opts ...sandbox.CreateOpt) error {
return errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Start(ctx context.Context, sandboxID string) (sandbox.ControllerInstance, error) {
return sandbox.ControllerInstance{}, errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Platform(_ctx context.Context, _sandboxID string) (platforms.Platform, error) {
return platforms.DefaultSpec(), nil
}
func (f fakeSandboxController) Stop(_ctx context.Context, _sandboxID string, _opts ...sandbox.StopOpt) error {
return errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Wait(_ctx context.Context, _sandboxID string) (sandbox.ExitStatus, error) {
return sandbox.ExitStatus{}, errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Status(_ctx context.Context, sandboxID string, _verbose bool) (sandbox.ControllerStatus, error) {
return sandbox.ControllerStatus{}, errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Shutdown(ctx context.Context, sandboxID string) error {
return errdefs.ErrNotImplemented
}
func (f fakeSandboxController) Metrics(ctx context.Context, sandboxID string) (*types.Metric, error) {
return &types.Metric{}, errdefs.ErrNotImplemented
}
// newTestCRIService creates a fake criService for test.
func newTestCRIService() *criService {
labels := label.NewStore()
@ -54,6 +100,7 @@ func newTestCRIService() *criService {
netPlugin: map[string]cni.CNI{
defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(),
},
sandboxService: &fakeSandboxService{},
}
}

12
services/sandbox/controller_service.go
View File

@ -31,7 +31,6 @@ import (
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/protobuf"
"github.com/containerd/containerd/v2/sandbox"
"github.com/containerd/containerd/v2/services"
"github.com/containerd/log"
"github.com/containerd/plugin"
"github.com/containerd/plugin/registry"
@ -42,15 +41,20 @@ func init() {
Type: plugins.GRPCPlugin,
ID: "sandbox-controllers",
Requires: []plugin.Type{
plugins.ServicePlugin,
plugins.SandboxControllerPlugin,
plugins.EventPlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
i, err := ic.GetByID(plugins.ServicePlugin, services.SandboxControllersService)
sandboxers, err := ic.GetByType(plugins.SandboxControllerPlugin)
if err != nil {
return nil, err
}
sc := i.(map[string]sandbox.Controller)
sc := make(map[string]sandbox.Controller)
for name, p := range sandboxers {
sc[name] = p.(sandbox.Controller)
}
ep, err := ic.GetSingle(plugins.EventPlugin)
if err != nil {
return nil, err

46
services/sandbox/sandboxers.go
View File

@ -1,46 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sandbox
import (
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/sandbox"
"github.com/containerd/containerd/v2/services"
"github.com/containerd/plugin"
"github.com/containerd/plugin/registry"
)
func init() {
registry.Register(&plugin.Registration{
Type: plugins.ServicePlugin,
ID: services.SandboxControllersService,
Requires: []plugin.Type{
plugins.SandboxControllerPlugin,
},
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
sandboxesRaw, err := ic.GetByType(plugins.SandboxControllerPlugin)
if err != nil {
return nil, err
}
sandboxers := make(map[string]sandbox.Controller)
for name, srv := range sandboxesRaw {
sandboxers[name] = srv.(sandbox.Controller)
}
return sandboxers, nil
},
})
}