Merge pull request #8268 from abel-von/sandbox-plugin

Sandbox: make sandbox controller plugin

pkg/cri/config/config.go

@@ -74,11 +74,11 @@ type Runtime struct {
 	// while using default snapshotters for operational simplicity.
 	// See https://github.com/containerd/containerd/issues/6657 for details.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
-	// SandboxMode defines which sandbox runtime to use when scheduling pods
+	// Sandboxer defines which sandbox runtime to use when scheduling pods
 	// This features requires the new CRI server implementation (enabled by default in 2.0)
 	// shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController).
 	// podsandbox - means use Controller implementation from sbserver podsandbox package.
-	SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"`
+	Sandboxer string `toml:"sandboxer" json:"sandboxer"`
 }
 
 // ContainerdConfig contains toml config related to containerd
@@ -383,8 +383,8 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
 			return errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled")
 		}
 		// If empty, use default podSandbox mode
-		if len(r.SandboxMode) == 0 {
-			r.SandboxMode = string(ModePodSandbox)
+		if len(r.Sandboxer) == 0 {
+			r.Sandboxer = string(ModePodSandbox)
 			c.ContainerdConfig.Runtimes[k] = r
 		}
 	}

pkg/cri/config/config_test.go

@@ -61,7 +61,7 @@ func TestValidateConfig(t *testing.T) {
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
 						RuntimeDefault: {
-							SandboxMode: string(ModePodSandbox),
+							Sandboxer: string(ModePodSandbox),
 						},
 					},
 				},

pkg/cri/config/config_unix.go

@@ -69,9 +69,9 @@ func DefaultConfig() PluginConfig {
 			DefaultRuntimeName: "runc",
 			Runtimes: map[string]Runtime{
 				"runc": {
-					Type:        "io.containerd.runc.v2",
-					Options:     m,
-					SandboxMode: string(ModePodSandbox),
+					Type:      "io.containerd.runc.v2",
+					Options:   m,
+					Sandboxer: string(ModePodSandbox),
 				},
 			},
 			DisableSnapshotAnnotations: true,

pkg/cri/server/container_create.go

@@ -274,10 +274,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 		containerd.WithContainerExtension(containerMetadataExtension, &meta),
 	)
 
-	// When using sandboxed shims, containerd's runtime needs to know which sandbox shim instance to use.
-	if ociRuntime.SandboxMode == string(criconfig.ModeShim) {
-		opts = append(opts, containerd.WithSandbox(sandboxID))
-	}
+	opts = append(opts, containerd.WithSandbox(sandboxID))
 
 	opts = append(opts, c.nri.WithContainerAdjustment())
 	defer func() {

pkg/cri/server/podsandbox/controller.go

@@ -34,10 +34,26 @@ import (
 	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
 	osinterface "github.com/containerd/containerd/pkg/os"
 	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/plugin/registry"
+	"github.com/containerd/containerd/plugins"
 	"github.com/containerd/containerd/protobuf"
 	"github.com/containerd/containerd/sandbox"
 )
 
+func init() {
+	registry.Register(&plugin.Registration{
+		Type:     plugins.SandboxControllerPlugin,
+		ID:       "podsandbox",
+		Requires: []plugin.Type{},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			// register the global controller to containerd plugin manager,
+			// the global controller will be initialized when cri plugin is initializing
+			return &Controller{}, nil
+		},
+	})
+}
+
 // CRIService interface contains things required by controller, but not yet refactored from criService.
 // TODO: this will be removed in subsequent iterations.
 type CRIService interface {
@@ -72,7 +88,7 @@ type Controller struct {
 	store *Store
 }
 
-func New(
+func (c *Controller) Init(
 	config criconfig.Config,
 	client *containerd.Client,
 	sandboxStore *sandboxstore.Store,
@@ -80,17 +96,15 @@ func New(
 	cri CRIService,
 	imageService ImageService,
 	baseOCISpecs map[string]*oci.Spec,
-) *Controller {
-	return &Controller{
-		config:       config,
-		client:       client,
-		imageService: imageService,
-		sandboxStore: sandboxStore,
-		os:           os,
-		cri:          cri,
-		baseOCISpecs: baseOCISpecs,
-		store:        NewStore(),
-	}
+) {
+	c.cri = cri
+	c.client = client
+	c.config = config
+	c.sandboxStore = sandboxStore
+	c.os = os
+	c.baseOCISpecs = baseOCISpecs
+	c.store = NewStore()
+	c.imageService = imageService
 }
 
 var _ sandbox.Controller = (*Controller)(nil)

pkg/cri/server/podsandbox/sandbox_run.go

@@ -273,7 +273,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll
 	return
 }
 
-func (c *Controller) Create(ctx context.Context, _id string, _ ...sandbox.CreateOpt) error {
+func (c *Controller) Create(ctx context.Context, _info sandbox.Sandbox, _ ...sandbox.CreateOpt) error {
 	// Not used by pod-sandbox implementation as there is no need to split pause containers logic.
 	return nil
 }

pkg/cri/server/restart.go

@@ -60,10 +60,7 @@ func (c *criService) recover(ctx context.Context) error {
 		return fmt.Errorf("failed to list sandbox containers: %w", err)
 	}
 
-	podSandboxController, ok := c.sandboxControllers[criconfig.ModePodSandbox]
-	if !ok {
-		log.G(ctx).Fatal("unable to restore pod sandboxes, no controller found")
-	}
+	podSandboxController := c.client.SandboxController(string(criconfig.ModePodSandbox))
 
 	podSandboxLoader, ok := podSandboxController.(podSandboxRecover)
 	if !ok {
@@ -115,7 +112,7 @@ func (c *criService) recover(ctx context.Context) error {
 
 		var (
 			state      = sandboxstore.StateUnknown
-			controller = c.sandboxControllers[criconfig.ModeShim]
+			controller = c.client.SandboxController(sbx.Sandboxer)
 		)
 
 		status, err := controller.Status(ctx, sbx.ID, false)

pkg/cri/server/sandbox_run.go

@@ -92,6 +92,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	}
 
 	sandboxInfo.Runtime.Name = ociRuntime.Type
+	sandboxInfo.Sandboxer = ociRuntime.Sandboxer
 
 	runtimeStart := time.Now()
 	// Retrieve runtime options
@@ -248,7 +249,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, fmt.Errorf("unable to update extensions for sandbox %q: %w", id, err)
 	}
 
-	if err := controller.Create(ctx, id, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil {
+	if err := controller.Create(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil {
 		return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err)
 	}
 
@@ -352,7 +353,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	}
 
 	// TODO: get rid of this. sandbox object should no longer have Container field.
-	if ociRuntime.SandboxMode == string(criconfig.ModePodSandbox) {
+	if ociRuntime.Sandboxer == string(criconfig.ModePodSandbox) {
 		container, err := c.client.LoadContainer(ctx, id)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load container %q for sandbox: %w", id, err)
@@ -683,25 +684,6 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
 	return handler, nil
 }
 
-// getSandboxController returns the sandbox controller configuration for sandbox.
-// If absent in legacy case, it will return the default controller.
-func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) {
-	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
-	}
-	// Validate mode
-	if err = ValidateMode(ociRuntime.SandboxMode); err != nil {
-		return nil, err
-	}
-	// Use sandbox controller to delete sandbox
-	controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)]
-	if !exist {
-		return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode)
-	}
-	return controller, nil
-}
-
 func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
 	if log.GetLevel() < log.DebugLevel {
 		return

pkg/cri/server/service.go

@@ -98,9 +98,6 @@ type criService struct {
 	sandboxNameIndex *registrar.Registrar
 	// containerStore stores all resources associated with containers.
 	containerStore *containerstore.Store
-	// sandboxControllers contains different sandbox controller type,
-	// every controller controls sandbox lifecycle (and hides implementation details behind).
-	sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
@@ -169,7 +166,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		sandboxNameIndex:   registrar.NewRegistrar(),
 		containerNameIndex: registrar.NewRegistrar(),
 		netPlugin:          make(map[string]cni.CNI),
-		sandboxControllers: make(map[criconfig.SandboxControllerMode]sandbox.Controller),
 	}
 
 	// TODO: figure out a proper channel size.
@@ -210,9 +206,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		return nil, err
 	}
 
-	// Load all sandbox controllers(pod sandbox controller and remote shim controller)
-	c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, imageService, c.baseOCISpecs)
-	c.sandboxControllers[criconfig.ModeShim] = client.SandboxController()
+	podSandboxController := c.client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller)
+	podSandboxController.Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs)
 
 	c.nri = nri
 
@@ -357,6 +352,17 @@ func (c *criService) register(s *grpc.Server) error {
 	return nil
 }
 
+// getSandboxController returns the sandbox controller configuration for sandbox.
+// If absent in legacy case, it will return the default controller.
+func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) {
+	ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
+	}
+
+	return c.client.SandboxController(ociRuntime.Sandboxer), nil
+}
+
 // imageFSPath returns containerd image filesystem path.
 // Note that if containerd changes directory layout, we also needs to change this.
 func imageFSPath(rootDir, snapshotter string) string {