From 92110e1d74428cacdbe386d2b4fb01d5b3837f79 Mon Sep 17 00:00:00 2001 From: abhi Date: Tue, 13 Mar 2018 21:39:47 -0700 Subject: [PATCH] Moving to use go-cni library from containerd This fix aims to use the cni library form containerd. The library avoid usage of nsenter. Signed-off-by: abhi --- pkg/server/helpers.go | 17 ++++++++++ pkg/server/sandbox_run.go | 68 ++++++++++++++++++++++---------------- pkg/server/sandbox_stop.go | 23 ++++++++----- pkg/server/service.go | 14 +++++--- pkg/server/status.go | 11 ++++-- 5 files changed, 89 insertions(+), 44 deletions(-) diff --git a/pkg/server/helpers.go b/pkg/server/helpers.go index 5a30c8d9d..0b03e2298 100644 --- a/pkg/server/helpers.go +++ b/pkg/server/helpers.go @@ -109,6 +109,14 @@ const ( containerMetadataExtension = criContainerdPrefix + ".container.metadata" ) +const ( + // defaultIfName is the default network interface for the pods + defaultIfName = "eth0" + // networkAttachCount is the minimum number of networks the PodSandbox + // attaches to + networkAttachCount = 2 +) + // makeSandboxName generates sandbox name from sandbox metadata. The name // generated is unique as long as sandbox metadata is unique. func makeSandboxName(s *runtime.PodSandboxMetadata) string { @@ -423,3 +431,12 @@ func disableNetNSDAD(ns string) error { } return nil } + +func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]string { + return map[string]string{ + "K8S_POD_NAMESPACE": config.GetMetadata().GetNamespace(), + "K8S_POD_NAME": config.GetMetadata().GetName(), + "K8S_POD_INFRA_CONTAINER_ID": id, + "IgnoreUnknown": "1", + } +} diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go index 1f37a65b8..b5818e9ca 100644 --- a/pkg/server/sandbox_run.go +++ b/pkg/server/sandbox_run.go @@ -26,8 +26,8 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/linux/runctypes" "github.com/containerd/containerd/oci" + cni "github.com/containerd/go-cni" "github.com/containerd/typeurl" - "github.com/cri-o/ocicni/pkg/ocicni" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -118,36 +118,25 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run } } // Setup network for sandbox. - podNetwork := ocicni.PodNetwork{ - Name: config.GetMetadata().GetName(), - Namespace: config.GetMetadata().GetNamespace(), - ID: id, - NetNS: sandbox.NetNSPath, - PortMappings: toCNIPortMappings(config.GetPortMappings()), - } - if _, err = c.netPlugin.SetUpPod(podNetwork); err != nil { - return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err) - } - defer func() { - if retErr != nil { - // Teardown network if an error is returned. - if err := c.netPlugin.TearDownPod(podNetwork); err != nil { - logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id) - } - } - }() - ip, err := c.netPlugin.GetPodNetworkStatus(podNetwork) - if err != nil { - return nil, fmt.Errorf("failed to get network status for sandbox %q: %v", id, err) - } - // Certain VM based solutions like clear containers (Issue containerd/cri#524) - // rely on the assumption that CRI shim will not be querying the network namespace to check the + // Certain VM based solutions like clear containers (Issue containerd/cri-containerd#524) + // rely on the assumption that CRI shim will not be querying the network namespace to check the // network states such as IP. // In furture runtime implementation should avoid relying on CRI shim implementation details. // In this case however caching the IP will add a subtle performance enhancement by avoiding // calls to network namespace of the pod to query the IP of the veth interface on every // SandboxStatus request. - sandbox.IP = ip + sandbox.IP, err = c.setupPod(id, sandbox.NetNSPath, config) + if err != nil { + return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err) + } + defer func() { + if retErr != nil { + // Teardown network if an error is returned. + if err := c.teardownPod(id, sandbox.NetNSPath, config); err != nil { + logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id) + } + } + }() } // Create sandbox container. @@ -498,14 +487,35 @@ func (c *criContainerdService) unmountSandboxFiles(rootDir string, config *runti return nil } +// setupPod setups up the network for a pod +func (c *criContainerdService) setupPod(id string, path string, config *runtime.PodSandboxConfig) (string, error) { + if c.netPlugin == nil { + return "", fmt.Errorf("cni config not intialized") + } + + labels := getPodCNILabels(id, config) + result, err := c.netPlugin.Setup(id, + path, + cni.WithLabels(labels), + cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings()))) + if err != nil { + return "", err + } + // Check if the default interface has IP config + if configs, ok := result.Interfaces[defaultIfName]; ok && len(configs.IPConfigs) > 0 { + return configs.IPConfigs[0].IP.String(), nil + } + return "", fmt.Errorf("failed to find network info for sandbox %q", id) +} + // toCNIPortMappings converts CRI port mappings to CNI. -func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []ocicni.PortMapping { - var portMappings []ocicni.PortMapping +func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []cni.PortMapping { + var portMappings []cni.PortMapping for _, mapping := range criPortMappings { if mapping.HostPort <= 0 { continue } - portMappings = append(portMappings, ocicni.PortMapping{ + portMappings = append(portMappings, cni.PortMapping{ HostPort: mapping.HostPort, ContainerPort: mapping.ContainerPort, Protocol: strings.ToLower(mapping.Protocol.String()), diff --git a/pkg/server/sandbox_stop.go b/pkg/server/sandbox_stop.go index 0597ed844..fbfd3c421 100644 --- a/pkg/server/sandbox_stop.go +++ b/pkg/server/sandbox_stop.go @@ -23,7 +23,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/errdefs" - "github.com/cri-o/ocicni/pkg/ocicni" + cni "github.com/containerd/go-cni" "github.com/sirupsen/logrus" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -65,13 +65,7 @@ func (c *criContainerdService) StopPodSandbox(ctx context.Context, r *runtime.St return nil, fmt.Errorf("failed to stat network namespace path %s :%v", sandbox.NetNSPath, err) } } else { - if teardownErr := c.netPlugin.TearDownPod(ocicni.PodNetwork{ - Name: sandbox.Config.GetMetadata().GetName(), - Namespace: sandbox.Config.GetMetadata().GetNamespace(), - ID: id, - NetNS: sandbox.NetNSPath, - PortMappings: toCNIPortMappings(sandbox.Config.GetPortMappings()), - }); teardownErr != nil { + if teardownErr := c.teardownPod(id, sandbox.NetNSPath, sandbox.Config); teardownErr != nil { return nil, fmt.Errorf("failed to destroy network for sandbox %q: %v", id, teardownErr) } } @@ -134,3 +128,16 @@ func (c *criContainerdService) waitSandboxStop(ctx context.Context, sandbox sand return nil } } + +// teardownPod removes the network from the pod +func (c *criContainerdService) teardownPod(id string, path string, config *runtime.PodSandboxConfig) error { + if c.netPlugin == nil { + return fmt.Errorf("cni config not intialized") + } + + labels := getPodCNILabels(id, config) + return c.netPlugin.Remove(id, + path, + cni.WithLabels(labels), + cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings()))) +} diff --git a/pkg/server/service.go b/pkg/server/service.go index ea1cb77a0..151d7bb51 100644 --- a/pkg/server/service.go +++ b/pkg/server/service.go @@ -24,7 +24,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/plugin" - "github.com/cri-o/ocicni/pkg/ocicni" + cni "github.com/containerd/go-cni" runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor" runcseccomp "github.com/opencontainers/runc/libcontainer/seccomp" "github.com/opencontainers/selinux/go-selinux" @@ -88,7 +88,7 @@ type criContainerdService struct { // snapshotStore stores information of all snapshots. snapshotStore *snapshotstore.Store // netPlugin is used to setup and teardown network when run/stop pod sandbox. - netPlugin ocicni.CNIPlugin + netPlugin cni.CNI // client is an instance of the containerd client client *containerd.Client // streamServer is the streaming server serves container streaming request. @@ -129,9 +129,15 @@ func NewCRIContainerdService(config criconfig.Config, client *containerd.Client) c.imageFSPath = imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter) logrus.Infof("Get image filesystem path %q", c.imageFSPath) - c.netPlugin, err = ocicni.InitCNI(config.NetworkPluginConfDir, config.NetworkPluginBinDir) + // Pod needs to attach to atleast loopback network and a non host network, + // hence networkAttachCount is 2. If there are more network configs the + // pod will be attached to all the networks but we will only use the ip + // of the default network interface as the pod IP. + c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount), + cni.WithPluginConfDir(config.NetworkPluginConfDir), + cni.WithPluginDir([]string{config.NetworkPluginBinDir})) if err != nil { - return nil, fmt.Errorf("failed to initialize cni plugin: %v", err) + return nil, fmt.Errorf("failed to initialize cni: %v", err) } // prepare streaming server diff --git a/pkg/server/status.go b/pkg/server/status.go index b52842a22..b813e34a0 100644 --- a/pkg/server/status.go +++ b/pkg/server/status.go @@ -21,6 +21,7 @@ import ( "fmt" goruntime "runtime" + cni "github.com/containerd/go-cni" "golang.org/x/net/context" runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" ) @@ -40,10 +41,14 @@ func (c *criContainerdService) Status(ctx context.Context, r *runtime.StatusRequ Type: runtime.NetworkReady, Status: true, } + // Check the status of the cni initialization if err := c.netPlugin.Status(); err != nil { - networkCondition.Status = false - networkCondition.Reason = networkNotReadyReason - networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err) + // If it is not initialized, then load the config and retry + if err = c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err != nil { + networkCondition.Status = false + networkCondition.Reason = networkNotReadyReason + networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err) + } } resp := &runtime.StatusResponse{