diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index d7ac9d46c..8860559c1 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -27,6 +27,16 @@ import ( "github.com/containerd/containerd/plugin" ) +type SandboxControllerMode string + +const ( + // ModePodSandbox means use Controller implementation from sbserver podsandbox package. + // We take this one as a default mode. + ModePodSandbox SandboxControllerMode = "podsandbox" + // ModeShim means use whatever Controller implementation provided by shim. + ModeShim SandboxControllerMode = "shim" +) + // Runtime struct to contain the type(ID), engine, and root variables for a default runtime // and a runtime for untrusted workload. type Runtime struct { @@ -76,6 +86,11 @@ type Runtime struct { // while using default snapshotters for operational simplicity. // See https://github.com/containerd/containerd/issues/6657 for details. Snapshotter string `toml:"snapshotter" json:"snapshotter"` + // SandboxMode defines which sandbox runtime to use when scheduling pods + // This features requires experimental CRI server to be enabled (use ENABLE_CRI_SANDBOXES=1) + // shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController). + // podsandbox - means use Controller implementation from sbserver podsandbox package. + SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"` } // ContainerdConfig contains toml config related to containerd @@ -412,7 +427,7 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error { // NoPivot can't be deprecated yet, because there is no alternative config option // for `io.containerd.runtime.v1.linux`. } - for _, r := range c.ContainerdConfig.Runtimes { + for k, r := range c.ContainerdConfig.Runtimes { if r.Engine != "" { if r.Type != plugin.RuntimeLinuxV1 { return fmt.Errorf("`runtime_engine` only works for runtime %s", plugin.RuntimeLinuxV1) @@ -428,6 +443,11 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error { if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed { return errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled") } + // If empty, use default podSandbox mode + if len(r.SandboxMode) == 0 { + r.SandboxMode = string(ModePodSandbox) + c.ContainerdConfig.Runtimes[k] = r + } } useConfigPath := c.Registry.ConfigPath != "" diff --git a/pkg/cri/config/config_test.go b/pkg/cri/config/config_test.go index db48cb8fe..92b93fc76 100644 --- a/pkg/cri/config/config_test.go +++ b/pkg/cri/config/config_test.go @@ -53,10 +53,12 @@ func TestValidateConfig(t *testing.T) { }, Runtimes: map[string]Runtime{ RuntimeUntrusted: { - Type: "untrusted", + Type: "untrusted", + SandboxMode: string(ModePodSandbox), }, RuntimeDefault: { - Type: "default", + Type: "default", + SandboxMode: string(ModePodSandbox), }, }, }, @@ -97,7 +99,8 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Type: "default", + Type: "default", + SandboxMode: string(ModePodSandbox), }, }, }, @@ -133,7 +136,8 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Type: plugin.RuntimeLinuxV1, + Type: plugin.RuntimeLinuxV1, + SandboxMode: string(ModePodSandbox), }, }, }, @@ -171,7 +175,8 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Type: plugin.RuntimeLinuxV1, + Type: plugin.RuntimeLinuxV1, + SandboxMode: string(ModePodSandbox), }, }, }, @@ -208,8 +213,9 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Engine: "runc", - Type: plugin.RuntimeLinuxV1, + Engine: "runc", + Type: plugin.RuntimeLinuxV1, + SandboxMode: string(ModePodSandbox), }, }, }, @@ -246,8 +252,9 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Root: "/run/containerd/runc", - Type: plugin.RuntimeLinuxV1, + Root: "/run/containerd/runc", + Type: plugin.RuntimeLinuxV1, + SandboxMode: string(ModePodSandbox), }, }, }, @@ -288,7 +295,8 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - Type: plugin.RuntimeRuncV1, + Type: plugin.RuntimeRuncV1, + SandboxMode: string(ModePodSandbox), }, }, }, diff --git a/pkg/cri/config/config_unix.go b/pkg/cri/config/config_unix.go index 7b471429c..c664143bf 100644 --- a/pkg/cri/config/config_unix.go +++ b/pkg/cri/config/config_unix.go @@ -77,8 +77,9 @@ func DefaultConfig() PluginConfig { NoPivot: false, Runtimes: map[string]Runtime{ "runc": { - Type: "io.containerd.runc.v2", - Options: tree.ToMap(), + Type: "io.containerd.runc.v2", + Options: tree.ToMap(), + SandboxMode: string(ModePodSandbox), }, }, DisableSnapshotAnnotations: true, diff --git a/pkg/cri/sbserver/sandbox_remove.go b/pkg/cri/sbserver/sandbox_remove.go index f49fdbea0..78c884f14 100644 --- a/pkg/cri/sbserver/sandbox_remove.go +++ b/pkg/cri/sbserver/sandbox_remove.go @@ -80,7 +80,12 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS } } - if _, err := c.sandboxController.Delete(ctx, id); err != nil { + // Use sandbox controller to delete sandbox + controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox controller: %w", err) + } + if _, err := controller.Delete(ctx, id); err != nil { return nil, fmt.Errorf("failed to delete sandbox %q: %w", id, err) } diff --git a/pkg/cri/sbserver/sandbox_run.go b/pkg/cri/sbserver/sandbox_run.go index 987fbc593..8bc0ca499 100644 --- a/pkg/cri/sbserver/sandbox_run.go +++ b/pkg/cri/sbserver/sandbox_run.go @@ -164,17 +164,22 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err) } + controller, err := c.getSandboxController(config, r.GetRuntimeHandler()) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox controller: %w", err) + } + if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil { return nil, fmt.Errorf("failed to save sandbox metadata: %w", err) } runtimeStart := time.Now() - if err := c.sandboxController.Create(ctx, id); err != nil { + if err := controller.Create(ctx, id); err != nil { return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err) } - resp, err := c.sandboxController.Start(ctx, id) + resp, err := controller.Start(ctx, id) if err != nil { return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err) } @@ -214,7 +219,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox // TaskOOM from containerd may come before sandbox is added to store, // but we don't care about sandbox TaskOOM right now, so it is fine. go func() { - resp, err := c.sandboxController.Wait(context.Background(), id) + resp, err := controller.Wait(context.Background(), id) if err != nil && err != context.Canceled && err != context.DeadlineExceeded { e := &eventtypes.TaskExit{ ContainerID: id, @@ -471,6 +476,25 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime return handler, nil } +// getSandboxController returns the sandbox controller configuration for sandbox. +// If absent in legacy case, it will return the default controller. +func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) { + ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) + } + // Validate mode + if err = ValidateMode(ociRuntime.SandboxMode); err != nil { + return nil, err + } + // Use sandbox controller to delete sandbox + controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)] + if !exist { + return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode) + } + return controller, nil +} + func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) { if logrus.GetLevel() < logrus.DebugLevel { return diff --git a/pkg/cri/sbserver/sandbox_stop.go b/pkg/cri/sbserver/sandbox_stop.go index 73ca6f076..5e2d76b35 100644 --- a/pkg/cri/sbserver/sandbox_stop.go +++ b/pkg/cri/sbserver/sandbox_stop.go @@ -64,7 +64,12 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa } } - if _, err := c.sandboxController.Stop(ctx, id); err != nil { + // Use sandbox controller to stop sandbox + controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + if err != nil { + return fmt.Errorf("failed to get sandbox controller: %w", err) + } + if _, err := controller.Stop(ctx, id); err != nil { return fmt.Errorf("failed to stop sandbox %q: %w", id, err) } diff --git a/pkg/cri/sbserver/service.go b/pkg/cri/sbserver/service.go index 9479d4e9f..8d88bfa14 100644 --- a/pkg/cri/sbserver/service.go +++ b/pkg/cri/sbserver/service.go @@ -27,12 +27,14 @@ import ( "time" "github.com/containerd/containerd" + sandboxapi "github.com/containerd/containerd/api/services/sandbox/v1" "github.com/containerd/containerd/oci" "github.com/containerd/containerd/pkg/cri/sbserver/podsandbox" "github.com/containerd/containerd/pkg/cri/streaming" "github.com/containerd/containerd/pkg/kmutex" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/sandbox" + "github.com/containerd/containerd/sandbox/proxy" "github.com/containerd/go-cni" "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -90,8 +92,9 @@ type criService struct { sandboxNameIndex *registrar.Registrar // containerStore stores all resources associated with containers. containerStore *containerstore.Store - // sandboxController controls sandbox lifecycle (and hides implementation details behind). - sandboxController sandbox.Controller + // sandboxControllers contains different sandbox controller type, + // every controller controls sandbox lifecycle (and hides implementation details behind). + sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller // containerNameIndex stores all container names and make sure each // name is unique. containerNameIndex *registrar.Registrar @@ -141,6 +144,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi initialized: atomic.NewBool(false), netPlugin: make(map[string]cni.CNI), unpackDuplicationSuppressor: kmutex.New(), + sandboxControllers: make(map[criconfig.SandboxControllerMode]sandbox.Controller), } if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil { @@ -185,7 +189,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi return nil, err } - c.sandboxController = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs) + // Load all sandbox controllers(pod sandbox controller and remote shim controller) + c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs) + c.sandboxControllers[criconfig.ModeShim] = proxy.NewSandboxController(sandboxapi.NewControllerClient(client.Conn())) return c, nil } @@ -379,3 +385,16 @@ func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) { return specs, nil } + +// ValidateMode validate the given mod value, +// returns err if mod is empty or unknown +func ValidateMode(modeStr string) error { + switch modeStr { + case string(criconfig.ModePodSandbox), string(criconfig.ModeShim): + return nil + case "": + return fmt.Errorf("empty sandbox controller mode") + default: + return fmt.Errorf("unknown sandbox controller mode: %s", modeStr) + } +} diff --git a/pkg/cri/sbserver/service_test.go b/pkg/cri/sbserver/service_test.go index 70093b1d0..53f6167e7 100644 --- a/pkg/cri/sbserver/service_test.go +++ b/pkg/cri/sbserver/service_test.go @@ -86,3 +86,17 @@ func TestLoadBaseOCISpec(t *testing.T) { assert.Equal(t, "1.0.2", out.Version) assert.Equal(t, "default", out.Hostname) } + +func TestValidateMode(t *testing.T) { + mode := "" + assert.Error(t, ValidateMode(mode)) + + mode = "podsandbox" + assert.NoError(t, ValidateMode(mode)) + + mode = "shim" + assert.NoError(t, ValidateMode(mode)) + + mode = "nonexistent" + assert.Error(t, ValidateMode(mode)) +}