sandbox: support more sandbox controllers

make containerd extensible to support more sandbox controllers
registered into containerd by config.
we change the default sandbox controller plugin's name from "local" to "shim".
to make sure we can get the controller by the plugin name it registered into
containerd.

Signed-off-by: Abel Feng <fshb1988@gmail.com>

services/sandbox/controller_service.go

@@ -18,6 +18,9 @@ package sandbox
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"time"
 
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -31,6 +34,7 @@ import (
 	"github.com/containerd/containerd/plugins"
 	"github.com/containerd/containerd/protobuf"
 	"github.com/containerd/containerd/sandbox"
+	"github.com/containerd/containerd/services"
 	"github.com/containerd/log"
 )
 
@@ -39,22 +43,30 @@ func init() {
 		Type: plugins.GRPCPlugin,
 		ID:   "sandbox-controllers",
 		Requires: []plugin.Type{
-			plugins.SandboxControllerPlugin,
+			plugins.ServicePlugin,
 			plugins.EventPlugin,
 		},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
-			sc, err := ic.GetByID(plugins.SandboxControllerPlugin, "local")
+			plugs, err := ic.GetByType(plugins.ServicePlugin)
 			if err != nil {
 				return nil, err
 			}
-
+			p, ok := plugs[services.SandboxControllersService]
+			if !ok {
+				return nil, errors.New("sandboxes service not found")
+			}
+			i, err := p.Instance()
+			if err != nil {
+				return nil, err
+			}
+			sc := i.(map[string]sandbox.Controller)
 			ep, err := ic.Get(plugins.EventPlugin)
 			if err != nil {
 				return nil, err
 			}
 
 			return &controllerService{
-				local:     sc.(sandbox.Controller),
+				sc:        sc,
 				publisher: ep.(events.Publisher),
 			}, nil
 		},
@@ -62,7 +74,7 @@ func init() {
 }
 
 type controllerService struct {
-	local     sandbox.Controller
+	sc        map[string]sandbox.Controller
 	publisher events.Publisher
 	api.UnimplementedControllerServer
 }
@@ -74,10 +86,24 @@ func (s *controllerService) Register(server *grpc.Server) error {
 	return nil
 }
 
+func (s *controllerService) getController(name string) (sandbox.Controller, error) {
+	if len(name) == 0 {
+		return nil, fmt.Errorf("%w: sandbox controller name can not be empty", errdefs.ErrInvalidArgument)
+	}
+	if ctrl, ok := s.sc[name]; ok {
+		return ctrl, nil
+	}
+	return nil, fmt.Errorf("%w: failed to get sandbox controller by %s", errdefs.ErrNotFound, name)
+}
+
 func (s *controllerService) Create(ctx context.Context, req *api.ControllerCreateRequest) (*api.ControllerCreateResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("create sandbox")
 	// TODO: Rootfs
-	err := s.local.Create(ctx, req.GetSandboxID(), sandbox.WithOptions(req.GetOptions()))
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	err = ctrl.Create(ctx, req.GetSandboxID(), sandbox.WithOptions(req.GetOptions()))
 	if err != nil {
 		return &api.ControllerCreateResponse{}, errdefs.ToGRPC(err)
 	}
@@ -95,7 +121,11 @@ func (s *controllerService) Create(ctx context.Context, req *api.ControllerCreat
 
 func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartRequest) (*api.ControllerStartResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("start sandbox")
-	inst, err := s.local.Start(ctx, req.GetSandboxID())
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	inst, err := ctrl.Start(ctx, req.GetSandboxID())
 	if err != nil {
 		return &api.ControllerStartResponse{}, errdefs.ToGRPC(err)
 	}
@@ -116,12 +146,20 @@ func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartR
 
 func (s *controllerService) Stop(ctx context.Context, req *api.ControllerStopRequest) (*api.ControllerStopResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("delete sandbox")
-	return &api.ControllerStopResponse{}, errdefs.ToGRPC(s.local.Stop(ctx, req.GetSandboxID()))
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return &api.ControllerStopResponse{}, errdefs.ToGRPC(ctrl.Stop(ctx, req.GetSandboxID(), sandbox.WithTimeout(time.Duration(req.TimeoutSecs)*time.Second)))
 }
 
 func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitRequest) (*api.ControllerWaitResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("wait sandbox")
-	exitStatus, err := s.local.Wait(ctx, req.GetSandboxID())
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	exitStatus, err := ctrl.Wait(ctx, req.GetSandboxID())
 	if err != nil {
 		return &api.ControllerWaitResponse{}, errdefs.ToGRPC(err)
 	}
@@ -142,7 +180,11 @@ func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitReq
 
 func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatusRequest) (*api.ControllerStatusResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("sandbox status")
-	cstatus, err := s.local.Status(ctx, req.GetSandboxID(), req.GetVerbose())
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	cstatus, err := ctrl.Status(ctx, req.GetSandboxID(), req.GetVerbose())
 	if err != nil {
 		return &api.ControllerStatusResponse{}, errdefs.ToGRPC(err)
 	}
@@ -166,7 +208,11 @@ func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatu
 
 func (s *controllerService) Shutdown(ctx context.Context, req *api.ControllerShutdownRequest) (*api.ControllerShutdownResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("shutdown sandbox")
-	return &api.ControllerShutdownResponse{}, errdefs.ToGRPC(s.local.Shutdown(ctx, req.GetSandboxID()))
+	ctrl, err := s.getController(req.Sandboxer)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return &api.ControllerShutdownResponse{}, errdefs.ToGRPC(ctrl.Shutdown(ctx, req.GetSandboxID()))
 }
 
 func (s *controllerService) Metrics(ctx context.Context, req *api.ControllerMetricsRequest) (*api.ControllerMetricsResponse, error) {

services/sandbox/sandboxers.go

@@ -0,0 +1,50 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/plugin/registry"
+	"github.com/containerd/containerd/plugins"
+	"github.com/containerd/containerd/sandbox"
+	"github.com/containerd/containerd/services"
+)
+
+func init() {
+	registry.Register(&plugin.Registration{
+		Type: plugins.ServicePlugin,
+		ID:   services.SandboxControllersService,
+		Requires: []plugin.Type{
+			plugins.SandboxControllerPlugin,
+		},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			sandboxesRaw, err := ic.GetByType(plugins.SandboxControllerPlugin)
+			if err != nil {
+				return nil, err
+			}
+			sandboxers := make(map[string]sandbox.Controller)
+			for name, srv := range sandboxesRaw {
+				inst, err := srv.Instance()
+				if err != nil {
+					return nil, err
+				}
+				sandboxers[name] = inst.(sandbox.Controller)
+			}
+			return sandboxers, nil
+		},
+	})
+}

services/server/server.go

@@ -36,6 +36,7 @@ import (
 
 	csapi "github.com/containerd/containerd/api/services/content/v1"
 	diffapi "github.com/containerd/containerd/api/services/diff/v1"
+	sbapi "github.com/containerd/containerd/api/services/sandbox/v1"
 	ssapi "github.com/containerd/containerd/api/services/snapshots/v1"
 	"github.com/containerd/containerd/content/local"
 	csproxy "github.com/containerd/containerd/content/proxy"
@@ -49,6 +50,7 @@ import (
 	"github.com/containerd/containerd/plugin/dynamic"
 	"github.com/containerd/containerd/plugin/registry"
 	"github.com/containerd/containerd/plugins"
+	sbproxy "github.com/containerd/containerd/sandbox/proxy"
 	srvconfig "github.com/containerd/containerd/services/server/config"
 	ssproxy "github.com/containerd/containerd/snapshots/proxy"
 	"github.com/containerd/containerd/sys"
@@ -474,6 +476,11 @@ func LoadPlugins(ctx context.Context, config *srvconfig.Config) ([]plugin.Regist
 			f = func(conn *grpc.ClientConn) interface{} {
 				return csproxy.NewContentStore(csapi.NewContentClient(conn))
 			}
+		case string(plugins.SandboxControllerPlugin), "sandbox":
+			t = plugins.SandboxControllerPlugin
+			f = func(conn *grpc.ClientConn) interface{} {
+				return sbproxy.NewSandboxController(sbapi.NewControllerClient(conn))
+			}
 		case string(plugins.DiffPlugin), "diff":
 			t = plugins.DiffPlugin
 			f = func(conn *grpc.ClientConn) interface{} {

services/services.go

@@ -21,6 +21,8 @@ const (
 	ContentService = "content-service"
 	// SnapshotsService is id of snapshots service.
 	SnapshotsService = "snapshots-service"
+	// SandboxControllersService is id of snapshots service.
+	SandboxControllersService = "sandboxes-service"
 	// ImagesService is id of images service.
 	ImagesService = "images-service"
 	// ContainersService is id of containers service.