sandbox: support more sandbox controllers

make containerd extensible to support more sandbox controllers
registered into containerd by config.
we change the default sandbox controller plugin's name from "local" to "shim".
to make sure we can get the controller by the plugin name it registered into
containerd.

Signed-off-by: Abel Feng <fshb1988@gmail.com>

pkg/cri/server/container_create.go

@@ -274,10 +274,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 		containerd.WithContainerExtension(containerMetadataExtension, &meta),
 	)
 
-	// When using sandboxed shims, containerd's runtime needs to know which sandbox shim instance to use.
-	if ociRuntime.Sandboxer == string(criconfig.ModeShim) {
-		opts = append(opts, containerd.WithSandbox(sandboxID))
-	}
+	opts = append(opts, containerd.WithSandbox(sandboxID))
 
 	opts = append(opts, c.nri.WithContainerAdjustment())
 	defer func() {

pkg/cri/server/podsandbox/controller.go

@@ -34,10 +34,26 @@ import (
 	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
 	osinterface "github.com/containerd/containerd/pkg/os"
 	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/plugin/registry"
+	"github.com/containerd/containerd/plugins"
 	"github.com/containerd/containerd/protobuf"
 	"github.com/containerd/containerd/sandbox"
 )
 
+func init() {
+	registry.Register(&plugin.Registration{
+		Type:     plugins.SandboxControllerPlugin,
+		ID:       "podsandbox",
+		Requires: []plugin.Type{},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			// register the global controller to containerd plugin manager,
+			// the global controller will be initialized when cri plugin is initializing
+			return controller, nil
+		},
+	})
+}
+
 // CRIService interface contains things required by controller, but not yet refactored from criService.
 // TODO: this will be removed in subsequent iterations.
 type CRIService interface {
@@ -53,6 +69,11 @@ type ImageService interface {
 	GetImage(id string) (imagestore.Image, error)
 }
 
+// As the dependency from this controller to cri plugin is hard to decouple,
+// we define a global podsandbox controller and register it to containerd plugin manager first,
+// we will initialize this controller when we initialize the cri plugin.
+var controller = &Controller{}
+
 type Controller struct {
 	// config contains all configurations.
 	config criconfig.Config
@@ -72,7 +93,7 @@ type Controller struct {
 	store *Store
 }
 
-func New(
+func Init(
 	config criconfig.Config,
 	client *containerd.Client,
 	sandboxStore *sandboxstore.Store,
@@ -80,17 +101,15 @@ func New(
 	cri CRIService,
 	imageService ImageService,
 	baseOCISpecs map[string]*oci.Spec,
-) *Controller {
-	return &Controller{
-		config:       config,
-		client:       client,
-		imageService: imageService,
-		sandboxStore: sandboxStore,
-		os:           os,
-		cri:          cri,
-		baseOCISpecs: baseOCISpecs,
-		store:        NewStore(),
-	}
+) {
+	controller.cri = cri
+	controller.client = client
+	controller.config = config
+	controller.sandboxStore = sandboxStore
+	controller.os = os
+	controller.baseOCISpecs = baseOCISpecs
+	controller.store = NewStore()
+	controller.imageService = imageService
 }
 
 var _ sandbox.Controller = (*Controller)(nil)

pkg/cri/server/restart.go

@@ -60,10 +60,7 @@ func (c *criService) recover(ctx context.Context) error {
 		return fmt.Errorf("failed to list sandbox containers: %w", err)
 	}
 
-	podSandboxController, ok := c.sandboxControllers[criconfig.ModePodSandbox]
-	if !ok {
-		log.G(ctx).Fatal("unable to restore pod sandboxes, no controller found")
-	}
+	podSandboxController := c.client.SandboxController(string(criconfig.ModePodSandbox))
 
 	podSandboxLoader, ok := podSandboxController.(podSandboxRecover)
 	if !ok {
@@ -115,7 +112,7 @@ func (c *criService) recover(ctx context.Context) error {
 
 		var (
 			state      = sandboxstore.StateUnknown
-			controller = c.sandboxControllers[criconfig.ModeShim]
+			controller = c.client.SandboxController(sbx.Sandboxer)
 		)
 
 		status, err := controller.Status(ctx, sbx.ID, false)

pkg/cri/server/sandbox_run.go

@@ -92,6 +92,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	}
 
 	sandboxInfo.Runtime.Name = ociRuntime.Type
+	sandboxInfo.Sandboxer = ociRuntime.Sandboxer
 
 	runtimeStart := time.Now()
 	// Retrieve runtime options
@@ -683,25 +684,6 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 	return handler, nil
 }
 
-// getSandboxController returns the sandbox controller configuration for sandbox.
-// If absent in legacy case, it will return the default controller.
-func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) {
-	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
-	}
-	// Validate mode
-	if err = ValidateMode(ociRuntime.Sandboxer); err != nil {
-		return nil, err
-	}
-	// Use sandbox controller to delete sandbox
-	controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.Sandboxer)]
-	if !exist {
-		return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.Sandboxer)
-	}
-	return controller, nil
-}
-
 func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
 	if log.GetLevel() < log.DebugLevel {
 		return

pkg/cri/server/service.go

@@ -98,9 +98,6 @@ type criService struct {
 	sandboxNameIndex *registrar.Registrar
 	// containerStore stores all resources associated with containers.
 	containerStore *containerstore.Store
-	// sandboxControllers contains different sandbox controller type,
-	// every controller controls sandbox lifecycle (and hides implementation details behind).
-	sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
@@ -159,7 +156,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		sandboxNameIndex:   registrar.NewRegistrar(),
 		containerNameIndex: registrar.NewRegistrar(),
 		netPlugin:          make(map[string]cni.CNI),
-		sandboxControllers: make(map[criconfig.SandboxControllerMode]sandbox.Controller),
 	}
 
 	// TODO: figure out a proper channel size.
@@ -200,9 +196,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		return nil, err
 	}
 
-	// Load all sandbox controllers(pod sandbox controller and remote shim controller)
-	c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, imageService, c.baseOCISpecs)
-	c.sandboxControllers[criconfig.ModeShim] = client.SandboxController()
+	// init the global podsandbox controller
+	podsandbox.Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs)
 
 	c.nri = nri
 
@@ -347,6 +342,17 @@ func (c *criService) register(s *grpc.Server) error {
 	return nil
 }
 
+// getSandboxController returns the sandbox controller configuration for sandbox.
+// If absent in legacy case, it will return the default controller.
+func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) {
+	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
+	}
+
+	return c.client.SandboxController(ociRuntime.Sandboxer), nil
+}
+
 // imageFSPath returns containerd image filesystem path.
 // Note that if containerd changes directory layout, we also needs to change this.
 func imageFSPath(rootDir, snapshotter string) string {