Sandbox API: Add a new mode config for sandbox controller impls

Add a new config as sandbox controller mod, which can be either
"podsandbox" or "shim". If empty, set it to default "podsandbox"
when CRI plugin inits.

Signed-off-by: Zhang Tianyang <burning9699@gmail.com>

pkg/cri/sbserver/sandbox_remove.go

@@ -80,7 +80,12 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 		}
 	}
 
-	if _, err := c.sandboxController.Delete(ctx, id); err != nil {
+	// Use sandbox controller to delete sandbox
+	controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+	if _, err := controller.Delete(ctx, id); err != nil {
 		return nil, fmt.Errorf("failed to delete sandbox %q: %w", id, err)
 	}
 

pkg/cri/sbserver/sandbox_run.go

@@ -164,17 +164,22 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
 	}
 
+	controller, err := c.getSandboxController(config, r.GetRuntimeHandler())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+
 	if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil {
 		return nil, fmt.Errorf("failed to save sandbox metadata: %w", err)
 	}
 
 	runtimeStart := time.Now()
 
-	if err := c.sandboxController.Create(ctx, id); err != nil {
+	if err := controller.Create(ctx, id); err != nil {
 		return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err)
 	}
 
-	resp, err := c.sandboxController.Start(ctx, id)
+	resp, err := controller.Start(ctx, id)
 	if err != nil {
 		return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err)
 	}
@@ -214,7 +219,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	// TaskOOM from containerd may come before sandbox is added to store,
 	// but we don't care about sandbox TaskOOM right now, so it is fine.
 	go func() {
-		resp, err := c.sandboxController.Wait(context.Background(), id)
+		resp, err := controller.Wait(context.Background(), id)
 		if err != nil && err != context.Canceled && err != context.DeadlineExceeded {
 			e := &eventtypes.TaskExit{
 				ContainerID: id,
@@ -471,6 +476,25 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 	return handler, nil
 }
 
+// getSandboxController returns the sandbox controller configuration for sandbox.
+// If absent in legacy case, it will return the default controller.
+func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) {
+	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
+	}
+	// Validate mode
+	if err = ValidateMode(ociRuntime.SandboxMode); err != nil {
+		return nil, err
+	}
+	// Use sandbox controller to delete sandbox
+	controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)]
+	if !exist {
+		return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode)
+	}
+	return controller, nil
+}
+
 func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
 	if logrus.GetLevel() < logrus.DebugLevel {
 		return

pkg/cri/sbserver/sandbox_stop.go

@@ -64,7 +64,12 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
 		}
 	}
 
-	if _, err := c.sandboxController.Stop(ctx, id); err != nil {
+	// Use sandbox controller to stop sandbox
+	controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
+	if err != nil {
+		return fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+	if _, err := controller.Stop(ctx, id); err != nil {
 		return fmt.Errorf("failed to stop sandbox %q: %w", id, err)
 	}
 

pkg/cri/sbserver/service.go

@@ -27,12 +27,14 @@ import (
 	"time"
 
 	"github.com/containerd/containerd"
+	sandboxapi "github.com/containerd/containerd/api/services/sandbox/v1"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/pkg/cri/sbserver/podsandbox"
 	"github.com/containerd/containerd/pkg/cri/streaming"
 	"github.com/containerd/containerd/pkg/kmutex"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/sandbox"
+	"github.com/containerd/containerd/sandbox/proxy"
 	"github.com/containerd/go-cni"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -90,8 +92,9 @@ type criService struct {
 	sandboxNameIndex *registrar.Registrar
 	// containerStore stores all resources associated with containers.
 	containerStore *containerstore.Store
-	// sandboxController controls sandbox lifecycle (and hides implementation details behind).
-	sandboxController sandbox.Controller
+	// sandboxControllers contains different sandbox controller type,
+	// every controller controls sandbox lifecycle (and hides implementation details behind).
+	sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
@@ -141,6 +144,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		initialized:                 atomic.NewBool(false),
 		netPlugin:                   make(map[string]cni.CNI),
 		unpackDuplicationSuppressor: kmutex.New(),
+		sandboxControllers:          make(map[criconfig.SandboxControllerMode]sandbox.Controller),
 	}
 
 	if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
@@ -185,7 +189,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		return nil, err
 	}
 
-	c.sandboxController = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
+	// Load all sandbox controllers(pod sandbox controller and remote shim controller)
+	c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
+	c.sandboxControllers[criconfig.ModeShim] = proxy.NewSandboxController(sandboxapi.NewControllerClient(client.Conn()))
 
 	return c, nil
 }
@@ -379,3 +385,16 @@ func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
 
 	return specs, nil
 }
+
+// ValidateMode validate the given mod value,
+// returns err if mod is empty or unknown
+func ValidateMode(modeStr string) error {
+	switch modeStr {
+	case string(criconfig.ModePodSandbox), string(criconfig.ModeShim):
+		return nil
+	case "":
+		return fmt.Errorf("empty sandbox controller mode")
+	default:
+		return fmt.Errorf("unknown sandbox controller mode: %s", modeStr)
+	}
+}

pkg/cri/sbserver/service_test.go

@@ -86,3 +86,17 @@ func TestLoadBaseOCISpec(t *testing.T) {
 	assert.Equal(t, "1.0.2", out.Version)
 	assert.Equal(t, "default", out.Hostname)
 }
+
+func TestValidateMode(t *testing.T) {
+	mode := ""
+	assert.Error(t, ValidateMode(mode))
+
+	mode = "podsandbox"
+	assert.NoError(t, ValidateMode(mode))
+
+	mode = "shim"
+	assert.NoError(t, ValidateMode(mode))
+
+	mode = "nonexistent"
+	assert.Error(t, ValidateMode(mode))
+}