Sandbox API: Add a new mode config for sandbox controller impls

Add a new config as sandbox controller mod, which can be either
"podsandbox" or "shim". If empty, set it to default "podsandbox"
when CRI plugin inits.

Signed-off-by: Zhang Tianyang <burning9699@gmail.com>

pkg/cri/config/config.go

@@ -27,6 +27,16 @@ import (
 	"github.com/containerd/containerd/plugin"
 )
 
+type SandboxControllerMode string
+
+const (
+	// ModePodSandbox means use Controller implementation from sbserver podsandbox package.
+	// We take this one as a default mode.
+	ModePodSandbox SandboxControllerMode = "podsandbox"
+	// ModeShim means use whatever Controller implementation provided by shim.
+	ModeShim SandboxControllerMode = "shim"
+)
+
 // Runtime struct to contain the type(ID), engine, and root variables for a default runtime
 // and a runtime for untrusted workload.
 type Runtime struct {
@@ -76,6 +86,11 @@ type Runtime struct {
 	// while using default snapshotters for operational simplicity.
 	// See https://github.com/containerd/containerd/issues/6657 for details.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
+	// SandboxMode defines which sandbox runtime to use when scheduling pods
+	// This features requires experimental CRI server to be enabled (use ENABLE_CRI_SANDBOXES=1)
+	// shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController).
+	// podsandbox - means use Controller implementation from sbserver podsandbox package.
+	SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"`
 }
 
 // ContainerdConfig contains toml config related to containerd
@@ -412,7 +427,7 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
 		// NoPivot can't be deprecated yet, because there is no alternative config option
 		// for `io.containerd.runtime.v1.linux`.
 	}
-	for _, r := range c.ContainerdConfig.Runtimes {
+	for k, r := range c.ContainerdConfig.Runtimes {
 		if r.Engine != "" {
 			if r.Type != plugin.RuntimeLinuxV1 {
 				return fmt.Errorf("`runtime_engine` only works for runtime %s", plugin.RuntimeLinuxV1)
@@ -428,6 +443,11 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
 		if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed {
 			return errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled")
 		}
+		// If empty, use default podSandbox mode
+		if len(r.SandboxMode) == 0 {
+			r.SandboxMode = string(ModePodSandbox)
+			c.ContainerdConfig.Runtimes[k] = r
+		}
 	}
 
 	useConfigPath := c.Registry.ConfigPath != ""

pkg/cri/config/config_test.go

@@ -53,10 +53,12 @@ func TestValidateConfig(t *testing.T) {
 					},
 					Runtimes: map[string]Runtime{
 						RuntimeUntrusted: {
-							Type: "untrusted",
+							Type:        "untrusted",
+							SandboxMode: string(ModePodSandbox),
 						},
 						RuntimeDefault: {
-							Type: "default",
+							Type:        "default",
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -97,7 +99,8 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Type: "default",
+							Type:        "default",
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -133,7 +136,8 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Type: plugin.RuntimeLinuxV1,
+							Type:        plugin.RuntimeLinuxV1,
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -171,7 +175,8 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Type: plugin.RuntimeLinuxV1,
+							Type:        plugin.RuntimeLinuxV1,
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -208,8 +213,9 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Engine: "runc",
+							Engine:      "runc",
-							Type:   plugin.RuntimeLinuxV1,
+							Type:        plugin.RuntimeLinuxV1,
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -246,8 +252,9 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Root: "/run/containerd/runc",
+							Root:        "/run/containerd/runc",
-							Type: plugin.RuntimeLinuxV1,
+							Type:        plugin.RuntimeLinuxV1,
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},
@@ -288,7 +295,8 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							Type: plugin.RuntimeRuncV1,
+							Type:        plugin.RuntimeRuncV1,
+							SandboxMode: string(ModePodSandbox),
 						},
 					},
 				},

pkg/cri/config/config_unix.go

@@ -77,8 +77,9 @@ func DefaultConfig() PluginConfig {
 			NoPivot:            false,
 			Runtimes: map[string]Runtime{
 				"runc": {
-					Type:    "io.containerd.runc.v2",
+					Type:        "io.containerd.runc.v2",
-					Options: tree.ToMap(),
+					Options:     tree.ToMap(),
+					SandboxMode: string(ModePodSandbox),
 				},
 			},
 			DisableSnapshotAnnotations: true,

pkg/cri/sbserver/sandbox_remove.go

@@ -80,7 +80,12 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 		}
 	}
 
-	if _, err := c.sandboxController.Delete(ctx, id); err != nil {
+	// Use sandbox controller to delete sandbox
+	controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+	if _, err := controller.Delete(ctx, id); err != nil {
 		return nil, fmt.Errorf("failed to delete sandbox %q: %w", id, err)
 	}
 

pkg/cri/sbserver/sandbox_run.go

@@ -164,17 +164,22 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
 	}
 
+	controller, err := c.getSandboxController(config, r.GetRuntimeHandler())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+
 	if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil {
 		return nil, fmt.Errorf("failed to save sandbox metadata: %w", err)
 	}
 
 	runtimeStart := time.Now()
 
-	if err := c.sandboxController.Create(ctx, id); err != nil {
+	if err := controller.Create(ctx, id); err != nil {
 		return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err)
 	}
 
-	resp, err := c.sandboxController.Start(ctx, id)
+	resp, err := controller.Start(ctx, id)
 	if err != nil {
 		return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err)
 	}
@@ -214,7 +219,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	// TaskOOM from containerd may come before sandbox is added to store,
 	// but we don't care about sandbox TaskOOM right now, so it is fine.
 	go func() {
-		resp, err := c.sandboxController.Wait(context.Background(), id)
+		resp, err := controller.Wait(context.Background(), id)
 		if err != nil && err != context.Canceled && err != context.DeadlineExceeded {
 			e := &eventtypes.TaskExit{
 				ContainerID: id,
@@ -471,6 +476,25 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 	return handler, nil
 }
 
+// getSandboxController returns the sandbox controller configuration for sandbox.
+// If absent in legacy case, it will return the default controller.
+func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) {
+	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
+	}
+	// Validate mode
+	if err = ValidateMode(ociRuntime.SandboxMode); err != nil {
+		return nil, err
+	}
+	// Use sandbox controller to delete sandbox
+	controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)]
+	if !exist {
+		return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode)
+	}
+	return controller, nil
+}
+
 func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
 	if logrus.GetLevel() < logrus.DebugLevel {
 		return

pkg/cri/sbserver/sandbox_stop.go

@@ -64,7 +64,12 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
 		}
 	}
 
-	if _, err := c.sandboxController.Stop(ctx, id); err != nil {
+	// Use sandbox controller to stop sandbox
+	controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
+	if err != nil {
+		return fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+	if _, err := controller.Stop(ctx, id); err != nil {
 		return fmt.Errorf("failed to stop sandbox %q: %w", id, err)
 	}
 

pkg/cri/sbserver/service.go

@@ -27,12 +27,14 @@ import (
 	"time"
 
 	"github.com/containerd/containerd"
+	sandboxapi "github.com/containerd/containerd/api/services/sandbox/v1"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/pkg/cri/sbserver/podsandbox"
 	"github.com/containerd/containerd/pkg/cri/streaming"
 	"github.com/containerd/containerd/pkg/kmutex"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/sandbox"
+	"github.com/containerd/containerd/sandbox/proxy"
 	"github.com/containerd/go-cni"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -90,8 +92,9 @@ type criService struct {
 	sandboxNameIndex *registrar.Registrar
 	// containerStore stores all resources associated with containers.
 	containerStore *containerstore.Store
-	// sandboxController controls sandbox lifecycle (and hides implementation details behind).
+	// sandboxControllers contains different sandbox controller type,
-	sandboxController sandbox.Controller
+	// every controller controls sandbox lifecycle (and hides implementation details behind).
+	sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
@@ -141,6 +144,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		initialized:                 atomic.NewBool(false),
 		netPlugin:                   make(map[string]cni.CNI),
 		unpackDuplicationSuppressor: kmutex.New(),
+		sandboxControllers:          make(map[criconfig.SandboxControllerMode]sandbox.Controller),
 	}
 
 	if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
@@ -185,7 +189,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		return nil, err
 	}
 
-	c.sandboxController = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
+	// Load all sandbox controllers(pod sandbox controller and remote shim controller)
+	c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
+	c.sandboxControllers[criconfig.ModeShim] = proxy.NewSandboxController(sandboxapi.NewControllerClient(client.Conn()))
 
 	return c, nil
 }
@@ -379,3 +385,16 @@ func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
 
 	return specs, nil
 }
+
+// ValidateMode validate the given mod value,
+// returns err if mod is empty or unknown
+func ValidateMode(modeStr string) error {
+	switch modeStr {
+	case string(criconfig.ModePodSandbox), string(criconfig.ModeShim):
+		return nil
+	case "":
+		return fmt.Errorf("empty sandbox controller mode")
+	default:
+		return fmt.Errorf("unknown sandbox controller mode: %s", modeStr)
+	}
+}

pkg/cri/sbserver/service_test.go

@@ -86,3 +86,17 @@ func TestLoadBaseOCISpec(t *testing.T) {
 	assert.Equal(t, "1.0.2", out.Version)
 	assert.Equal(t, "default", out.Hostname)
 }
+
+func TestValidateMode(t *testing.T) {
+	mode := ""
+	assert.Error(t, ValidateMode(mode))
+
+	mode = "podsandbox"
+	assert.NoError(t, ValidateMode(mode))
+
+	mode = "shim"
+	assert.NoError(t, ValidateMode(mode))
+
+	mode = "nonexistent"
+	assert.Error(t, ValidateMode(mode))
+}