Add CNI conf based on runtime class

Signed-off-by: Michael Crosby <michael@thepasture.io>

docs/cri/config.md

@@ -175,6 +175,18 @@ version = 2
       # Still running containers and restarted containers will still be using the original spec from which that container was created.
       base_runtime_spec = ""
 
+      # conf_dir is the directory in which the admin places a CNI conf.
+      # this allows a different CNI conf for the network stack when a different runtime is being used.
+      cni_conf_dir = "/etc/cni/net.d"
+
+      # cni_max_conf_num specifies the maximum number of CNI plugin config files to
+      # load from the CNI config directory. By default, only 1 CNI plugin config
+      # file will be loaded. If you want to load multiple CNI plugin config files
+      # set max_conf_num to the number desired. Setting cni_max_config_num to 0 is
+      # interpreted as no limit is desired and will result in all CNI plugin
+      # config files being loaded from the CNI config directory.
+      cni_max_conf_num = 1
+
       # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to
       # "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is:
       #   https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 .

pkg/cri/config/config.go

@@ -56,6 +56,12 @@ type Runtime struct {
 	PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
 	// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
 	BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
+	// NetworkPluginConfDir is a directory containing the CNI network information for the runtime class.
+	NetworkPluginConfDir string `toml:"cni_conf_dir" json:"cniConfDir"`
+	// NetworkPluginMaxConfNum is the max number of plugin config files that will
+	// be loaded from the cni config directory by go-cni. Set the value to 0 to
+	// load all config files (no arbitrary limit). The legacy default value is 1.
+	NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"`
 }
 
 // ContainerdConfig contains toml config related to containerd

pkg/cri/server/sandbox_run.go

@@ -27,6 +27,7 @@ import (
 	containerdio "github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/snapshots"
 	cni "github.com/containerd/go-cni"
 	"github.com/containerd/nri"
 	v1 "github.com/containerd/nri/types/v1"
@@ -45,7 +46,6 @@ import (
 	"github.com/containerd/containerd/pkg/cri/util"
 	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
 	"github.com/containerd/containerd/pkg/netns"
-	"github.com/containerd/containerd/snapshots"
 	selinux "github.com/opencontainers/selinux/go-selinux"
 )
 
@@ -204,7 +204,6 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to generate runtime options")
 	}
-
 	snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
 	opts := []containerd.NewContainerOpts{
 		containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
@@ -349,14 +348,30 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
 }
 
+// getNetworkPlugin returns the network plugin to be used by the runtime class
+// defaults to the global CNI options in the CRI config
+func (c *criService) getNetworkPlugin(runtimeClass string) cni.CNI {
+	if c.netPlugin == nil {
+		return nil
+	}
+	i, ok := c.netPlugin[runtimeClass]
+	if !ok {
+		if i, ok = c.netPlugin[defaultNetworkPlugin]; !ok {
+			return nil
+		}
+	}
+	return i
+}
+
 // setupPodNetwork setups up the network for a pod
 func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.Sandbox) error {
 	var (
-		id     = sandbox.ID
+		id        = sandbox.ID
-		config = sandbox.Config
+		config    = sandbox.Config
-		path   = sandbox.NetNSPath
+		path      = sandbox.NetNSPath
+		netPlugin = c.getNetworkPlugin(sandbox.RuntimeHandler)
 	)
-	if c.netPlugin == nil {
+	if netPlugin == nil {
 		return errors.New("cni config not initialized")
 	}
 
@@ -365,7 +380,7 @@ func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.
 		return errors.Wrap(err, "get cni namespace options")
 	}
 
-	result, err := c.netPlugin.Setup(ctx, id, path, opts...)
+	result, err := netPlugin.Setup(ctx, id, path, opts...)
 	if err != nil {
 		return err
 	}

pkg/cri/server/sandbox_stop.go

@@ -165,7 +165,8 @@ func (c *criService) waitSandboxStop(ctx context.Context, sandbox sandboxstore.S
 
 // teardownPodNetwork removes the network from the pod
 func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstore.Sandbox) error {
-	if c.netPlugin == nil {
+	netPlugin := c.getNetworkPlugin(sandbox.RuntimeHandler)
+	if netPlugin == nil {
 		return errors.New("cni config not initialized")
 	}
 
@@ -179,7 +180,7 @@ func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstor
 		return errors.Wrap(err, "get cni namespace options")
 	}
 
-	return c.netPlugin.Remove(ctx, id, path, opts...)
+	return netPlugin.Remove(ctx, id, path, opts...)
 }
 
 // cleanupUnknownSandbox cleanup stopped sandbox in unknown state.

pkg/cri/server/service.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/containerd/containerd"
@@ -49,6 +50,9 @@ import (
 	"github.com/containerd/containerd/pkg/registrar"
 )
 
+// defaultNetworkPlugin is used for the default CNI configuration
+const defaultNetworkPlugin = "default"
+
 // grpcServices are all the grpc services provided by cri containerd.
 type grpcServices interface {
 	runtime.RuntimeServiceServer
@@ -92,7 +96,7 @@ type criService struct {
 	// snapshotStore stores information of all snapshots.
 	snapshotStore *snapshotstore.Store
 	// netPlugin is used to setup and teardown network when run/stop pod sandbox.
-	netPlugin cni.CNI
+	netPlugin map[string]cni.CNI
 	// client is an instance of the containerd client
 	client *containerd.Client
 	// streamServer is the streaming server serves container streaming request.
@@ -104,7 +108,7 @@ type criService struct {
 	initialized atomic.Bool
 	// cniNetConfMonitor is used to reload cni network conf if there is
 	// any valid fs change events from cni network conf dir.
-	cniNetConfMonitor *cniNetConfSyncer
+	cniNetConfMonitor map[string]*cniNetConfSyncer
 	// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
 	baseOCISpecs map[string]*oci.Spec
 	// allCaps is the list of the capabilities.
@@ -127,6 +131,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		sandboxNameIndex:   registrar.NewRegistrar(),
 		containerNameIndex: registrar.NewRegistrar(),
 		initialized:        atomic.NewBool(false),
+		netPlugin:          make(map[string]cni.CNI),
 	}
 
 	if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
@@ -148,9 +153,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 
 	c.eventMonitor = newEventMonitor(c)
 
-	c.cniNetConfMonitor, err = newCNINetConfSyncer(c.config.NetworkPluginConfDir, c.netPlugin, c.cniLoadOptions())
+	c.cniNetConfMonitor = make(map[string]*cniNetConfSyncer)
-	if err != nil {
+	for name, i := range c.netPlugin {
-		return nil, errors.Wrap(err, "failed to create cni conf monitor")
+		path := c.config.NetworkPluginConfDir
+		if name != defaultNetworkPlugin {
+			if rc, ok := c.config.Runtimes[name]; ok {
+				path = rc.NetworkPluginConfDir
+			}
+		}
+		if path != "" {
+			m, err := newCNINetConfSyncer(path, i, c.cniLoadOptions())
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to create cni conf monitor for %s", name)
+			}
+			c.cniNetConfMonitor[name] = m
+		}
 	}
 
 	// Preload base OCI specs
@@ -200,12 +217,20 @@ func (c *criService) Run() error {
 	)
 	snapshotsSyncer.start()
 
-	// Start CNI network conf syncer
+	// Start CNI network conf syncers
-	logrus.Info("Start cni network conf syncer")
+	cniNetConfMonitorErrCh := make(chan error, len(c.cniNetConfMonitor))
-	cniNetConfMonitorErrCh := make(chan error, 1)
+	var netSyncGroup sync.WaitGroup
+	for name, h := range c.cniNetConfMonitor {
+		netSyncGroup.Add(1)
+		logrus.Infof("Start cni network conf syncer for %s", name)
+		go func(h *cniNetConfSyncer) {
+			cniNetConfMonitorErrCh <- h.syncLoop()
+			netSyncGroup.Done()
+		}(h)
+	}
 	go func() {
-		defer close(cniNetConfMonitorErrCh)
+		netSyncGroup.Wait()
-		cniNetConfMonitorErrCh <- c.cniNetConfMonitor.syncLoop()
+		close(cniNetConfMonitorErrCh)
 	}()
 
 	// Start streaming server.
@@ -272,8 +297,10 @@ func (c *criService) Run() error {
 // TODO(random-liu): Make close synchronous.
 func (c *criService) Close() error {
 	logrus.Info("Stop CRI service")
-	if err := c.cniNetConfMonitor.stop(); err != nil {
+	for name, h := range c.cniNetConfMonitor {
-		logrus.WithError(err).Error("failed to stop cni network conf monitor")
+		if err := h.stop(); err != nil {
+			logrus.WithError(err).Errorf("failed to stop cni network conf monitor for %s", name)
+		}
 	}
 	c.eventMonitor.stop()
 	if err := c.streamServer.Stop(); err != nil {

pkg/cri/server/service_linux.go

@@ -30,9 +30,7 @@ import (
 const networkAttachCount = 2
 
 // initPlatform handles linux specific initialization for the CRI service.
-func (c *criService) initPlatform() error {
+func (c *criService) initPlatform() (err error) {
-	var err error
-
 	if userns.RunningInUserNS() {
 		if !(c.config.DisableCgroup && !c.apparmorEnabled() && c.config.RestrictOOMScoreAdj) {
 			logrus.Warn("Running containerd in a user namespace typically requires disable_cgroup, disable_apparmor, restrict_oom_score_adj set to be true")
@@ -50,16 +48,35 @@ func (c *criService) initPlatform() error {
 		selinux.SetDisabled()
 	}
 
-	// Pod needs to attach to at least loopback network and a non host network,
+	pluginDirs := map[string]string{
-	// hence networkAttachCount is 2. If there are more network configs the
+		defaultNetworkPlugin: c.config.NetworkPluginConfDir,
-	// pod will be attached to all the networks but we will only use the ip
+	}
-	// of the default network interface as the pod IP.
+	for name, conf := range c.config.Runtimes {
-	c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount),
+		if conf.NetworkPluginConfDir != "" {
-		cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
+			pluginDirs[name] = conf.NetworkPluginConfDir
-		cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
+		}
-		cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+	}
-	if err != nil {
+
-		return errors.Wrap(err, "failed to initialize cni")
+	c.netPlugin = make(map[string]cni.CNI)
+	for name, dir := range pluginDirs {
+		max := c.config.NetworkPluginMaxConfNum
+		if name != defaultNetworkPlugin {
+			if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 {
+				max = m
+			}
+		}
+		// Pod needs to attach to at least loopback network and a non host network,
+		// hence networkAttachCount is 2. If there are more network configs the
+		// pod will be attached to all the networks but we will only use the ip
+		// of the default network interface as the pod IP.
+		i, err := cni.New(cni.WithMinNetworkCount(networkAttachCount),
+			cni.WithPluginConfDir(dir),
+			cni.WithPluginMaxConfNum(max),
+			cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize cni")
+		}
+		c.netPlugin[name] = i
 	}
 
 	if c.allCaps == nil {

pkg/cri/server/service_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/go-cni"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -66,7 +67,9 @@ func newTestCRIService() *criService {
 		sandboxNameIndex:   registrar.NewRegistrar(),
 		containerStore:     containerstore.NewStore(labels),
 		containerNameIndex: registrar.NewRegistrar(),
-		netPlugin:          servertesting.NewFakeCNIPlugin(),
+		netPlugin: map[string]cni.CNI{
+			defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(),
+		},
 	}
 }
 

pkg/cri/server/service_windows.go

@@ -27,18 +27,36 @@ const windowsNetworkAttachCount = 1
 
 // initPlatform handles linux specific initialization for the CRI service.
 func (c *criService) initPlatform() error {
-	var err error
+	pluginDirs := map[string]string{
-	// For windows, the loopback network is added as default.
+		defaultNetworkPlugin: c.config.NetworkPluginConfDir,
-	// There is no need to explicitly add one hence networkAttachCount is 1.
+	}
-	// If there are more network configs the pod will be attached to all the
+	for name, conf := range c.config.Runtimes {
-	// networks but we will only use the ip of the default network interface
+		if conf.NetworkPluginConfDir != "" {
-	// as the pod IP.
+			pluginDirs[name] = conf.NetworkPluginConfDir
-	c.netPlugin, err = cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
+		}
-		cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
+	}
-		cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
+
-		cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+	c.netPlugin = make(map[string]cni.CNI)
-	if err != nil {
+	for name, dir := range pluginDirs {
-		return errors.Wrap(err, "failed to initialize cni")
+		max := c.config.NetworkPluginMaxConfNum
+		if name != defaultNetworkPlugin {
+			if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 {
+				max = m
+			}
+		}
+		// For windows, the loopback network is added as default.
+		// There is no need to explicitly add one hence networkAttachCount is 1.
+		// If there are more network configs the pod will be attached to all the
+		// networks but we will only use the ip of the default network interface
+		// as the pod IP.
+		i, err := cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
+			cni.WithPluginConfDir(dir),
+			cni.WithPluginMaxConfNum(max),
+			cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize cni")
+		}
+		c.netPlugin[name] = i
 	}
 
 	return nil

pkg/cri/server/status.go

@@ -41,11 +41,14 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run
 		Type:   runtime.NetworkReady,
 		Status: true,
 	}
+	netPlugin := c.netPlugin[defaultNetworkPlugin]
 	// Check the status of the cni initialization
-	if err := c.netPlugin.Status(); err != nil {
+	if netPlugin != nil {
-		networkCondition.Status = false
+		if err := netPlugin.Status(); err != nil {
-		networkCondition.Reason = networkNotReadyReason
+			networkCondition.Status = false
-		networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
+			networkCondition.Reason = networkNotReadyReason
+			networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
+		}
 	}
 
 	resp := &runtime.StatusResponse{
@@ -67,17 +70,29 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run
 		}
 		resp.Info["golang"] = string(versionByt)
 
-		cniConfig, err := json.Marshal(c.netPlugin.GetConfig())
+		if netPlugin != nil {
-		if err != nil {
+			cniConfig, err := json.Marshal(netPlugin.GetConfig())
-			log.G(ctx).WithError(err).Errorf("Failed to marshal CNI config %v", err)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to marshal CNI config %v", err)
+			}
+			resp.Info["cniconfig"] = string(cniConfig)
 		}
-		resp.Info["cniconfig"] = string(cniConfig)
 
-		lastCNILoadStatus := "OK"
+		defaultStatus := "OK"
-		if lerr := c.cniNetConfMonitor.lastStatus(); lerr != nil {
+		for name, h := range c.cniNetConfMonitor {
-			lastCNILoadStatus = lerr.Error()
+			s := "OK"
+			if h == nil {
+				continue
+			}
+			if lerr := h.lastStatus(); lerr != nil {
+				s = lerr.Error()
+			}
+			resp.Info[fmt.Sprintf("lastCNILoadStatus.%s", name)] = s
+			if name == defaultNetworkPlugin {
+				defaultStatus = s
+			}
 		}
-		resp.Info["lastCNILoadStatus"] = lastCNILoadStatus
+		resp.Info["lastCNILoadStatus"] = defaultStatus
 	}
 	return resp, nil
 }

pkg/cri/server/update_runtime_config.go

@@ -69,10 +69,15 @@ func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateR
 		log.G(ctx).Info("No cni config template is specified, wait for other system components to drop the config.")
 		return &runtime.UpdateRuntimeConfigResponse{}, nil
 	}
-	if err := c.netPlugin.Status(); err == nil {
+	netPlugin := c.netPlugin[defaultNetworkPlugin]
+	if netPlugin == nil {
 		log.G(ctx).Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
 		return &runtime.UpdateRuntimeConfigResponse{}, nil
-	} else if err := c.netPlugin.Load(c.cniLoadOptions()...); err == nil {
+	}
+	if err := netPlugin.Status(); err == nil {
+		log.G(ctx).Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	} else if err := netPlugin.Load(c.cniLoadOptions()...); err == nil {
 		log.G(ctx).Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate)
 		return &runtime.UpdateRuntimeConfigResponse{}, nil
 	}

pkg/cri/server/update_runtime_config_test.go

@@ -122,8 +122,8 @@ func TestUpdateRuntimeConfig(t *testing.T) {
 				req.RuntimeConfig.NetworkConfig.PodCidr = ""
 			}
 			if !test.networkReady {
-				c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
+				c.netPlugin[defaultNetworkPlugin].(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
-				c.netPlugin.(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error")
+				c.netPlugin[defaultNetworkPlugin].(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error")
 			}
 			_, err = c.UpdateRuntimeConfig(context.Background(), req)
 			assert.NoError(t, err)

script/test/cri-integration.sh

@@ -48,4 +48,9 @@ ${sudo} bin/cri-integration.test --test.run="${FOCUS}" --test.v \
 
 test_exit_code=$?
 
+test_teardown
+
+test $test_exit_code -ne 0 && \
+  cat "$REPORT_DIR/containerd.log"
+
 exit ${test_exit_code}