github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Sandbox API: Add a new mode config for sandbox controller impls

Browse Source 
Add a new config as sandbox controller mod, which can be either
"podsandbox" or "shim". If empty, set it to default "podsandbox"
when CRI plugin inits.

Signed-off-by: Zhang Tianyang <burning9699@gmail.com>
This commit is contained in:
Zhang Tianyang 2022-11-03 21:21:07 +08:00
parent d1564fec5b
commit c953eecb79
8 changed files with 117 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/cri/config/config.go
View File

@ -27,6 +27,16 @@ import (
"github.com/containerd/containerd/plugin"
)
type SandboxControllerMode string
const (
// ModePodSandbox means use Controller implementation from sbserver podsandbox package.
// We take this one as a default mode.
ModePodSandbox SandboxControllerMode = "podsandbox"
// ModeShim means use whatever Controller implementation provided by shim.
ModeShim SandboxControllerMode = "shim"
)
// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
// and a runtime for untrusted workload.
type Runtime struct {
@ -76,6 +86,11 @@ type Runtime struct {
// while using default snapshotters for operational simplicity.
// See https://github.com/containerd/containerd/issues/6657 for details.
Snapshotter string `toml:"snapshotter" json:"snapshotter"`
// SandboxMode defines which sandbox runtime to use when scheduling pods
// This features requires experimental CRI server to be enabled (use ENABLE_CRI_SANDBOXES=1)
// shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController).
// podsandbox - means use Controller implementation from sbserver podsandbox package.
SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"`
}
// ContainerdConfig contains toml config related to containerd
@ -412,7 +427,7 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
// NoPivot can't be deprecated yet, because there is no alternative config option
// for `io.containerd.runtime.v1.linux`.
}
for _, r := range c.ContainerdConfig.Runtimes {
for k, r := range c.ContainerdConfig.Runtimes {
if r.Engine != "" {
if r.Type != plugin.RuntimeLinuxV1 {
return fmt.Errorf("`runtime_engine` only works for runtime %s", plugin.RuntimeLinuxV1)
@ -428,6 +443,11 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed {
return errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled")
}
// If empty, use default podSandbox mode
if len(r.SandboxMode) == 0 {
r.SandboxMode = string(ModePodSandbox)
c.ContainerdConfig.Runtimes[k] = r
}
}
useConfigPath := c.Registry.ConfigPath != ""

8
pkg/cri/config/config_test.go
View File

@ -54,9 +54,11 @@ func TestValidateConfig(t *testing.T) {
Runtimes: map[string]Runtime{
RuntimeUntrusted: {
Type: "untrusted",
SandboxMode: string(ModePodSandbox),
},
RuntimeDefault: {
Type: "default",
SandboxMode: string(ModePodSandbox),
},
},
},
@ -98,6 +100,7 @@ func TestValidateConfig(t *testing.T) {
Runtimes: map[string]Runtime{
RuntimeDefault: {
Type: "default",
SandboxMode: string(ModePodSandbox),
},
},
},
@ -134,6 +137,7 @@ func TestValidateConfig(t *testing.T) {
Runtimes: map[string]Runtime{
RuntimeDefault: {
Type: plugin.RuntimeLinuxV1,
SandboxMode: string(ModePodSandbox),
},
},
},
@ -172,6 +176,7 @@ func TestValidateConfig(t *testing.T) {
Runtimes: map[string]Runtime{
RuntimeDefault: {
Type: plugin.RuntimeLinuxV1,
SandboxMode: string(ModePodSandbox),
},
},
},
@ -210,6 +215,7 @@ func TestValidateConfig(t *testing.T) {
RuntimeDefault: {
Engine: "runc",
Type: plugin.RuntimeLinuxV1,
SandboxMode: string(ModePodSandbox),
},
},
},
@ -248,6 +254,7 @@ func TestValidateConfig(t *testing.T) {
RuntimeDefault: {
Root: "/run/containerd/runc",
Type: plugin.RuntimeLinuxV1,
SandboxMode: string(ModePodSandbox),
},
},
},
@ -289,6 +296,7 @@ func TestValidateConfig(t *testing.T) {
Runtimes: map[string]Runtime{
RuntimeDefault: {
Type: plugin.RuntimeRuncV1,
SandboxMode: string(ModePodSandbox),
},
},
},

1
pkg/cri/config/config_unix.go
View File

@ -79,6 +79,7 @@ func DefaultConfig() PluginConfig {
"runc": {
Type: "io.containerd.runc.v2",
Options: tree.ToMap(),
SandboxMode: string(ModePodSandbox),
},
},
DisableSnapshotAnnotations: true,

7
pkg/cri/sbserver/sandbox_remove.go
View File

@ -80,7 +80,12 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
}
}
if _, err := c.sandboxController.Delete(ctx, id); err != nil {
// Use sandbox controller to delete sandbox
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}
if _, err := controller.Delete(ctx, id); err != nil {
return nil, fmt.Errorf("failed to delete sandbox %q: %w", id, err)
}

30
pkg/cri/sbserver/sandbox_run.go
View File

@ -164,17 +164,22 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
}
controller, err := c.getSandboxController(config, r.GetRuntimeHandler())
if err != nil {
return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
}
if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil {
return nil, fmt.Errorf("failed to save sandbox metadata: %w", err)
}
runtimeStart := time.Now()
if err := c.sandboxController.Create(ctx, id); err != nil {
if err := controller.Create(ctx, id); err != nil {
return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err)
}
resp, err := c.sandboxController.Start(ctx, id)
resp, err := controller.Start(ctx, id)
if err != nil {
return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err)
}
@ -214,7 +219,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
// TaskOOM from containerd may come before sandbox is added to store,
// but we don't care about sandbox TaskOOM right now, so it is fine.
go func() {
resp, err := c.sandboxController.Wait(context.Background(), id)
resp, err := controller.Wait(context.Background(), id)
if err != nil && err != context.Canceled && err != context.DeadlineExceeded {
e := &eventtypes.TaskExit{
ContainerID: id,
@ -471,6 +476,25 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime
return handler, nil
}
// getSandboxController returns the sandbox controller configuration for sandbox.
// If absent in legacy case, it will return the default controller.
func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) {
ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
// Validate mode
if err = ValidateMode(ociRuntime.SandboxMode); err != nil {
return nil, err
}
// Use sandbox controller to delete sandbox
controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)]
if !exist {
return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode)
}
return controller, nil
}
func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) {
if logrus.GetLevel() < logrus.DebugLevel {
return

7
pkg/cri/sbserver/sandbox_stop.go
View File

@ -64,7 +64,12 @@ func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sa
}
}
if _, err := c.sandboxController.Stop(ctx, id); err != nil {
// Use sandbox controller to stop sandbox
controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
if err != nil {
return fmt.Errorf("failed to get sandbox controller: %w", err)
}
if _, err := controller.Stop(ctx, id); err != nil {
return fmt.Errorf("failed to stop sandbox %q: %w", id, err)
}

25
pkg/cri/sbserver/service.go
View File

@ -27,12 +27,14 @@ import (
"time"
"github.com/containerd/containerd"
sandboxapi "github.com/containerd/containerd/api/services/sandbox/v1"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/pkg/cri/sbserver/podsandbox"
"github.com/containerd/containerd/pkg/cri/streaming"
"github.com/containerd/containerd/pkg/kmutex"
"github.com/containerd/containerd/plugin"
"github.com/containerd/containerd/sandbox"
"github.com/containerd/containerd/sandbox/proxy"
"github.com/containerd/go-cni"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
@ -90,8 +92,9 @@ type criService struct {
sandboxNameIndex *registrar.Registrar
// containerStore stores all resources associated with containers.
containerStore *containerstore.Store
// sandboxController controls sandbox lifecycle (and hides implementation details behind).
sandboxController sandbox.Controller
// sandboxControllers contains different sandbox controller type,
// every controller controls sandbox lifecycle (and hides implementation details behind).
sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller
// containerNameIndex stores all container names and make sure each
// name is unique.
containerNameIndex *registrar.Registrar
@ -141,6 +144,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
initialized: atomic.NewBool(false),
netPlugin: make(map[string]cni.CNI),
unpackDuplicationSuppressor: kmutex.New(),
sandboxControllers: make(map[criconfig.SandboxControllerMode]sandbox.Controller),
}
if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
@ -185,7 +189,9 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
return nil, err
}
c.sandboxController = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
// Load all sandbox controllers(pod sandbox controller and remote shim controller)
c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
c.sandboxControllers[criconfig.ModeShim] = proxy.NewSandboxController(sandboxapi.NewControllerClient(client.Conn()))
return c, nil
}
@ -379,3 +385,16 @@ func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
return specs, nil
}
// ValidateMode validate the given mod value,
// returns err if mod is empty or unknown
func ValidateMode(modeStr string) error {
switch modeStr {
case string(criconfig.ModePodSandbox), string(criconfig.ModeShim):
return nil
case "":
return fmt.Errorf("empty sandbox controller mode")
default:
return fmt.Errorf("unknown sandbox controller mode: %s", modeStr)
}
}

14
pkg/cri/sbserver/service_test.go
View File

@ -86,3 +86,17 @@ func TestLoadBaseOCISpec(t *testing.T) {
assert.Equal(t, "1.0.2", out.Version)
assert.Equal(t, "default", out.Hostname)
}
func TestValidateMode(t *testing.T) {
mode := ""
assert.Error(t, ValidateMode(mode))
mode = "podsandbox"
assert.NoError(t, ValidateMode(mode))
mode = "shim"
assert.NoError(t, ValidateMode(mode))
mode = "nonexistent"
assert.Error(t, ValidateMode(mode))
}