From 55893b9be729b7473a717682d6bf99a154c36422 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 6 Nov 2020 11:29:42 -0500 Subject: [PATCH] Add CNI conf based on runtime class Signed-off-by: Michael Crosby --- docs/cri/config.md | 12 +++++ pkg/cri/config/config.go | 6 +++ pkg/cri/server/sandbox_run.go | 29 ++++++++--- pkg/cri/server/sandbox_stop.go | 5 +- pkg/cri/server/service.go | 51 +++++++++++++++----- pkg/cri/server/service_linux.go | 43 ++++++++++++----- pkg/cri/server/service_test.go | 5 +- pkg/cri/server/service_windows.go | 42 +++++++++++----- pkg/cri/server/status.go | 39 ++++++++++----- pkg/cri/server/update_runtime_config.go | 9 +++- pkg/cri/server/update_runtime_config_test.go | 4 +- script/test/cri-integration.sh | 5 ++ 12 files changed, 187 insertions(+), 63 deletions(-) diff --git a/docs/cri/config.md b/docs/cri/config.md index 20609b02a..d10e6f663 100644 --- a/docs/cri/config.md +++ b/docs/cri/config.md @@ -175,6 +175,18 @@ version = 2 # Still running containers and restarted containers will still be using the original spec from which that container was created. base_runtime_spec = "" + # conf_dir is the directory in which the admin places a CNI conf. + # this allows a different CNI conf for the network stack when a different runtime is being used. + cni_conf_dir = "/etc/cni/net.d" + + # cni_max_conf_num specifies the maximum number of CNI plugin config files to + # load from the CNI config directory. By default, only 1 CNI plugin config + # file will be loaded. If you want to load multiple CNI plugin config files + # set max_conf_num to the number desired. Setting cni_max_config_num to 0 is + # interpreted as no limit is desired and will result in all CNI plugin + # config files being loaded from the CNI config directory. + cni_max_conf_num = 1 + # 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to # "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is: # https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 . diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index aeee9bfd4..d35fadbe3 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -56,6 +56,12 @@ type Runtime struct { PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"` // BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from. BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"` + // NetworkPluginConfDir is a directory containing the CNI network information for the runtime class. + NetworkPluginConfDir string `toml:"cni_conf_dir" json:"cniConfDir"` + // NetworkPluginMaxConfNum is the max number of plugin config files that will + // be loaded from the cni config directory by go-cni. Set the value to 0 to + // load all config files (no arbitrary limit). The legacy default value is 1. + NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"` } // ContainerdConfig contains toml config related to containerd diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index 41c0fa2b7..16aad2fa6 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -27,6 +27,7 @@ import ( containerdio "github.com/containerd/containerd/cio" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" + "github.com/containerd/containerd/snapshots" cni "github.com/containerd/go-cni" "github.com/containerd/nri" v1 "github.com/containerd/nri/types/v1" @@ -45,7 +46,6 @@ import ( "github.com/containerd/containerd/pkg/cri/util" ctrdutil "github.com/containerd/containerd/pkg/cri/util" "github.com/containerd/containerd/pkg/netns" - "github.com/containerd/containerd/snapshots" selinux "github.com/opencontainers/selinux/go-selinux" ) @@ -204,7 +204,6 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox if err != nil { return nil, errors.Wrap(err, "failed to generate runtime options") } - snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations)) opts := []containerd.NewContainerOpts{ containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter), @@ -349,14 +348,30 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil } +// getNetworkPlugin returns the network plugin to be used by the runtime class +// defaults to the global CNI options in the CRI config +func (c *criService) getNetworkPlugin(runtimeClass string) cni.CNI { + if c.netPlugin == nil { + return nil + } + i, ok := c.netPlugin[runtimeClass] + if !ok { + if i, ok = c.netPlugin[defaultNetworkPlugin]; !ok { + return nil + } + } + return i +} + // setupPodNetwork setups up the network for a pod func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.Sandbox) error { var ( - id = sandbox.ID - config = sandbox.Config - path = sandbox.NetNSPath + id = sandbox.ID + config = sandbox.Config + path = sandbox.NetNSPath + netPlugin = c.getNetworkPlugin(sandbox.RuntimeHandler) ) - if c.netPlugin == nil { + if netPlugin == nil { return errors.New("cni config not initialized") } @@ -365,7 +380,7 @@ func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore. return errors.Wrap(err, "get cni namespace options") } - result, err := c.netPlugin.Setup(ctx, id, path, opts...) + result, err := netPlugin.Setup(ctx, id, path, opts...) if err != nil { return err } diff --git a/pkg/cri/server/sandbox_stop.go b/pkg/cri/server/sandbox_stop.go index 547e3c69a..6ef7a987f 100644 --- a/pkg/cri/server/sandbox_stop.go +++ b/pkg/cri/server/sandbox_stop.go @@ -165,7 +165,8 @@ func (c *criService) waitSandboxStop(ctx context.Context, sandbox sandboxstore.S // teardownPodNetwork removes the network from the pod func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstore.Sandbox) error { - if c.netPlugin == nil { + netPlugin := c.getNetworkPlugin(sandbox.RuntimeHandler) + if netPlugin == nil { return errors.New("cni config not initialized") } @@ -179,7 +180,7 @@ func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstor return errors.Wrap(err, "get cni namespace options") } - return c.netPlugin.Remove(ctx, id, path, opts...) + return netPlugin.Remove(ctx, id, path, opts...) } // cleanupUnknownSandbox cleanup stopped sandbox in unknown state. diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index e8901d819..9de20c11e 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -23,6 +23,7 @@ import ( "net/http" "os" "path/filepath" + "sync" "time" "github.com/containerd/containerd" @@ -49,6 +50,9 @@ import ( "github.com/containerd/containerd/pkg/registrar" ) +// defaultNetworkPlugin is used for the default CNI configuration +const defaultNetworkPlugin = "default" + // grpcServices are all the grpc services provided by cri containerd. type grpcServices interface { runtime.RuntimeServiceServer @@ -92,7 +96,7 @@ type criService struct { // snapshotStore stores information of all snapshots. snapshotStore *snapshotstore.Store // netPlugin is used to setup and teardown network when run/stop pod sandbox. - netPlugin cni.CNI + netPlugin map[string]cni.CNI // client is an instance of the containerd client client *containerd.Client // streamServer is the streaming server serves container streaming request. @@ -104,7 +108,7 @@ type criService struct { initialized atomic.Bool // cniNetConfMonitor is used to reload cni network conf if there is // any valid fs change events from cni network conf dir. - cniNetConfMonitor *cniNetConfSyncer + cniNetConfMonitor map[string]*cniNetConfSyncer // baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec` baseOCISpecs map[string]*oci.Spec // allCaps is the list of the capabilities. @@ -127,6 +131,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi sandboxNameIndex: registrar.NewRegistrar(), containerNameIndex: registrar.NewRegistrar(), initialized: atomic.NewBool(false), + netPlugin: make(map[string]cni.CNI), } if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil { @@ -148,9 +153,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi c.eventMonitor = newEventMonitor(c) - c.cniNetConfMonitor, err = newCNINetConfSyncer(c.config.NetworkPluginConfDir, c.netPlugin, c.cniLoadOptions()) - if err != nil { - return nil, errors.Wrap(err, "failed to create cni conf monitor") + c.cniNetConfMonitor = make(map[string]*cniNetConfSyncer) + for name, i := range c.netPlugin { + path := c.config.NetworkPluginConfDir + if name != defaultNetworkPlugin { + if rc, ok := c.config.Runtimes[name]; ok { + path = rc.NetworkPluginConfDir + } + } + if path != "" { + m, err := newCNINetConfSyncer(path, i, c.cniLoadOptions()) + if err != nil { + return nil, errors.Wrapf(err, "failed to create cni conf monitor for %s", name) + } + c.cniNetConfMonitor[name] = m + } } // Preload base OCI specs @@ -200,12 +217,20 @@ func (c *criService) Run() error { ) snapshotsSyncer.start() - // Start CNI network conf syncer - logrus.Info("Start cni network conf syncer") - cniNetConfMonitorErrCh := make(chan error, 1) + // Start CNI network conf syncers + cniNetConfMonitorErrCh := make(chan error, len(c.cniNetConfMonitor)) + var netSyncGroup sync.WaitGroup + for name, h := range c.cniNetConfMonitor { + netSyncGroup.Add(1) + logrus.Infof("Start cni network conf syncer for %s", name) + go func(h *cniNetConfSyncer) { + cniNetConfMonitorErrCh <- h.syncLoop() + netSyncGroup.Done() + }(h) + } go func() { - defer close(cniNetConfMonitorErrCh) - cniNetConfMonitorErrCh <- c.cniNetConfMonitor.syncLoop() + netSyncGroup.Wait() + close(cniNetConfMonitorErrCh) }() // Start streaming server. @@ -272,8 +297,10 @@ func (c *criService) Run() error { // TODO(random-liu): Make close synchronous. func (c *criService) Close() error { logrus.Info("Stop CRI service") - if err := c.cniNetConfMonitor.stop(); err != nil { - logrus.WithError(err).Error("failed to stop cni network conf monitor") + for name, h := range c.cniNetConfMonitor { + if err := h.stop(); err != nil { + logrus.WithError(err).Errorf("failed to stop cni network conf monitor for %s", name) + } } c.eventMonitor.stop() if err := c.streamServer.Stop(); err != nil { diff --git a/pkg/cri/server/service_linux.go b/pkg/cri/server/service_linux.go index 1cd2c6aaa..c41aa2c06 100644 --- a/pkg/cri/server/service_linux.go +++ b/pkg/cri/server/service_linux.go @@ -30,9 +30,7 @@ import ( const networkAttachCount = 2 // initPlatform handles linux specific initialization for the CRI service. -func (c *criService) initPlatform() error { - var err error - +func (c *criService) initPlatform() (err error) { if userns.RunningInUserNS() { if !(c.config.DisableCgroup && !c.apparmorEnabled() && c.config.RestrictOOMScoreAdj) { logrus.Warn("Running containerd in a user namespace typically requires disable_cgroup, disable_apparmor, restrict_oom_score_adj set to be true") @@ -50,16 +48,35 @@ func (c *criService) initPlatform() error { selinux.SetDisabled() } - // Pod needs to attach to at least loopback network and a non host network, - // hence networkAttachCount is 2. If there are more network configs the - // pod will be attached to all the networks but we will only use the ip - // of the default network interface as the pod IP. - c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount), - cni.WithPluginConfDir(c.config.NetworkPluginConfDir), - cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum), - cni.WithPluginDir([]string{c.config.NetworkPluginBinDir})) - if err != nil { - return errors.Wrap(err, "failed to initialize cni") + pluginDirs := map[string]string{ + defaultNetworkPlugin: c.config.NetworkPluginConfDir, + } + for name, conf := range c.config.Runtimes { + if conf.NetworkPluginConfDir != "" { + pluginDirs[name] = conf.NetworkPluginConfDir + } + } + + c.netPlugin = make(map[string]cni.CNI) + for name, dir := range pluginDirs { + max := c.config.NetworkPluginMaxConfNum + if name != defaultNetworkPlugin { + if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 { + max = m + } + } + // Pod needs to attach to at least loopback network and a non host network, + // hence networkAttachCount is 2. If there are more network configs the + // pod will be attached to all the networks but we will only use the ip + // of the default network interface as the pod IP. + i, err := cni.New(cni.WithMinNetworkCount(networkAttachCount), + cni.WithPluginConfDir(dir), + cni.WithPluginMaxConfNum(max), + cni.WithPluginDir([]string{c.config.NetworkPluginBinDir})) + if err != nil { + return errors.Wrap(err, "failed to initialize cni") + } + c.netPlugin[name] = i } if c.allCaps == nil { diff --git a/pkg/cri/server/service_test.go b/pkg/cri/server/service_test.go index 50b015396..9dd29565f 100644 --- a/pkg/cri/server/service_test.go +++ b/pkg/cri/server/service_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/containerd/containerd/oci" + "github.com/containerd/go-cni" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -66,7 +67,9 @@ func newTestCRIService() *criService { sandboxNameIndex: registrar.NewRegistrar(), containerStore: containerstore.NewStore(labels), containerNameIndex: registrar.NewRegistrar(), - netPlugin: servertesting.NewFakeCNIPlugin(), + netPlugin: map[string]cni.CNI{ + defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(), + }, } } diff --git a/pkg/cri/server/service_windows.go b/pkg/cri/server/service_windows.go index a107fffc4..f335297e7 100644 --- a/pkg/cri/server/service_windows.go +++ b/pkg/cri/server/service_windows.go @@ -27,18 +27,36 @@ const windowsNetworkAttachCount = 1 // initPlatform handles linux specific initialization for the CRI service. func (c *criService) initPlatform() error { - var err error - // For windows, the loopback network is added as default. - // There is no need to explicitly add one hence networkAttachCount is 1. - // If there are more network configs the pod will be attached to all the - // networks but we will only use the ip of the default network interface - // as the pod IP. - c.netPlugin, err = cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount), - cni.WithPluginConfDir(c.config.NetworkPluginConfDir), - cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum), - cni.WithPluginDir([]string{c.config.NetworkPluginBinDir})) - if err != nil { - return errors.Wrap(err, "failed to initialize cni") + pluginDirs := map[string]string{ + defaultNetworkPlugin: c.config.NetworkPluginConfDir, + } + for name, conf := range c.config.Runtimes { + if conf.NetworkPluginConfDir != "" { + pluginDirs[name] = conf.NetworkPluginConfDir + } + } + + c.netPlugin = make(map[string]cni.CNI) + for name, dir := range pluginDirs { + max := c.config.NetworkPluginMaxConfNum + if name != defaultNetworkPlugin { + if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 { + max = m + } + } + // For windows, the loopback network is added as default. + // There is no need to explicitly add one hence networkAttachCount is 1. + // If there are more network configs the pod will be attached to all the + // networks but we will only use the ip of the default network interface + // as the pod IP. + i, err := cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount), + cni.WithPluginConfDir(dir), + cni.WithPluginMaxConfNum(max), + cni.WithPluginDir([]string{c.config.NetworkPluginBinDir})) + if err != nil { + return errors.Wrap(err, "failed to initialize cni") + } + c.netPlugin[name] = i } return nil diff --git a/pkg/cri/server/status.go b/pkg/cri/server/status.go index 70a30ad7f..704ff4f5e 100644 --- a/pkg/cri/server/status.go +++ b/pkg/cri/server/status.go @@ -41,11 +41,14 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run Type: runtime.NetworkReady, Status: true, } + netPlugin := c.netPlugin[defaultNetworkPlugin] // Check the status of the cni initialization - if err := c.netPlugin.Status(); err != nil { - networkCondition.Status = false - networkCondition.Reason = networkNotReadyReason - networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err) + if netPlugin != nil { + if err := netPlugin.Status(); err != nil { + networkCondition.Status = false + networkCondition.Reason = networkNotReadyReason + networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err) + } } resp := &runtime.StatusResponse{ @@ -67,17 +70,29 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run } resp.Info["golang"] = string(versionByt) - cniConfig, err := json.Marshal(c.netPlugin.GetConfig()) - if err != nil { - log.G(ctx).WithError(err).Errorf("Failed to marshal CNI config %v", err) + if netPlugin != nil { + cniConfig, err := json.Marshal(netPlugin.GetConfig()) + if err != nil { + log.G(ctx).WithError(err).Errorf("Failed to marshal CNI config %v", err) + } + resp.Info["cniconfig"] = string(cniConfig) } - resp.Info["cniconfig"] = string(cniConfig) - lastCNILoadStatus := "OK" - if lerr := c.cniNetConfMonitor.lastStatus(); lerr != nil { - lastCNILoadStatus = lerr.Error() + defaultStatus := "OK" + for name, h := range c.cniNetConfMonitor { + s := "OK" + if h == nil { + continue + } + if lerr := h.lastStatus(); lerr != nil { + s = lerr.Error() + } + resp.Info[fmt.Sprintf("lastCNILoadStatus.%s", name)] = s + if name == defaultNetworkPlugin { + defaultStatus = s + } } - resp.Info["lastCNILoadStatus"] = lastCNILoadStatus + resp.Info["lastCNILoadStatus"] = defaultStatus } return resp, nil } diff --git a/pkg/cri/server/update_runtime_config.go b/pkg/cri/server/update_runtime_config.go index 6bc6eedd7..ae4d1114e 100644 --- a/pkg/cri/server/update_runtime_config.go +++ b/pkg/cri/server/update_runtime_config.go @@ -69,10 +69,15 @@ func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateR log.G(ctx).Info("No cni config template is specified, wait for other system components to drop the config.") return &runtime.UpdateRuntimeConfigResponse{}, nil } - if err := c.netPlugin.Status(); err == nil { + netPlugin := c.netPlugin[defaultNetworkPlugin] + if netPlugin == nil { log.G(ctx).Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate) return &runtime.UpdateRuntimeConfigResponse{}, nil - } else if err := c.netPlugin.Load(c.cniLoadOptions()...); err == nil { + } + if err := netPlugin.Status(); err == nil { + log.G(ctx).Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate) + return &runtime.UpdateRuntimeConfigResponse{}, nil + } else if err := netPlugin.Load(c.cniLoadOptions()...); err == nil { log.G(ctx).Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate) return &runtime.UpdateRuntimeConfigResponse{}, nil } diff --git a/pkg/cri/server/update_runtime_config_test.go b/pkg/cri/server/update_runtime_config_test.go index 84cc25ce9..67ab955a7 100644 --- a/pkg/cri/server/update_runtime_config_test.go +++ b/pkg/cri/server/update_runtime_config_test.go @@ -122,8 +122,8 @@ func TestUpdateRuntimeConfig(t *testing.T) { req.RuntimeConfig.NetworkConfig.PodCidr = "" } if !test.networkReady { - c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error") - c.netPlugin.(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error") + c.netPlugin[defaultNetworkPlugin].(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error") + c.netPlugin[defaultNetworkPlugin].(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error") } _, err = c.UpdateRuntimeConfig(context.Background(), req) assert.NoError(t, err) diff --git a/script/test/cri-integration.sh b/script/test/cri-integration.sh index 133e0d0be..48697fd94 100755 --- a/script/test/cri-integration.sh +++ b/script/test/cri-integration.sh @@ -48,4 +48,9 @@ ${sudo} bin/cri-integration.test --test.run="${FOCUS}" --test.v \ test_exit_code=$? +test_teardown + +test $test_exit_code -ne 0 && \ + cat "$REPORT_DIR/containerd.log" + exit ${test_exit_code}