Moving to use go-cni library from containerd

This fix aims to use the cni library form containerd.
The library avoid usage of nsenter.

Signed-off-by: abhi <abhi@docker.com>

pkg/server/helpers.go

@@ -109,6 +109,14 @@ const (
 	containerMetadataExtension = criContainerdPrefix + ".container.metadata"
 )
 
+const (
+	// defaultIfName is the default network interface for the pods
+	defaultIfName = "eth0"
+	// networkAttachCount is the minimum number of networks the PodSandbox
+	// attaches to
+	networkAttachCount = 2
+)
+
 // makeSandboxName generates sandbox name from sandbox metadata. The name
 // generated is unique as long as sandbox metadata is unique.
 func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@@ -423,3 +431,12 @@ func disableNetNSDAD(ns string) error {
 	}
 	return nil
 }
+
+func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]string {
+	return map[string]string{
+		"K8S_POD_NAMESPACE":          config.GetMetadata().GetNamespace(),
+		"K8S_POD_NAME":               config.GetMetadata().GetName(),
+		"K8S_POD_INFRA_CONTAINER_ID": id,
+		"IgnoreUnknown": "1",
+	}
+}

pkg/server/sandbox_run.go

@@ -26,8 +26,8 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/linux/runctypes"
 	"github.com/containerd/containerd/oci"
+	cni "github.com/containerd/go-cni"
 	"github.com/containerd/typeurl"
-	"github.com/cri-o/ocicni/pkg/ocicni"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -118,36 +118,25 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 			}
 		}
 		// Setup network for sandbox.
-		podNetwork := ocicni.PodNetwork{
+		// Certain VM based solutions like clear containers (Issue containerd/cri-containerd#524)
-			Name:         config.GetMetadata().GetName(),
-			Namespace:    config.GetMetadata().GetNamespace(),
-			ID:           id,
-			NetNS:        sandbox.NetNSPath,
-			PortMappings: toCNIPortMappings(config.GetPortMappings()),
-		}
-		if _, err = c.netPlugin.SetUpPod(podNetwork); err != nil {
-			return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err)
-		}
-		defer func() {
-			if retErr != nil {
-				// Teardown network if an error is returned.
-				if err := c.netPlugin.TearDownPod(podNetwork); err != nil {
-					logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
-				}
-			}
-		}()
-		ip, err := c.netPlugin.GetPodNetworkStatus(podNetwork)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get network status for sandbox %q: %v", id, err)
-		}
-		// Certain VM based solutions like clear containers (Issue containerd/cri#524)
 		// rely on the assumption that CRI shim will not be querying the network namespace to check the
 		// network states such as IP.
 		// In furture runtime implementation should avoid relying on CRI shim implementation details.
 		// In this case however caching the IP will add a subtle performance enhancement by avoiding
 		// calls to network namespace of the pod to query the IP of the veth interface on every
 		// SandboxStatus request.
-		sandbox.IP = ip
+		sandbox.IP, err = c.setupPod(id, sandbox.NetNSPath, config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err)
+		}
+		defer func() {
+			if retErr != nil {
+				// Teardown network if an error is returned.
+				if err := c.teardownPod(id, sandbox.NetNSPath, config); err != nil {
+					logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
+				}
+			}
+		}()
 	}
 
 	// Create sandbox container.
@@ -498,14 +487,35 @@ func (c *criContainerdService) unmountSandboxFiles(rootDir string, config *runti
 	return nil
 }
 
+// setupPod setups up the network for a pod
+func (c *criContainerdService) setupPod(id string, path string, config *runtime.PodSandboxConfig) (string, error) {
+	if c.netPlugin == nil {
+		return "", fmt.Errorf("cni config not intialized")
+	}
+
+	labels := getPodCNILabels(id, config)
+	result, err := c.netPlugin.Setup(id,
+		path,
+		cni.WithLabels(labels),
+		cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings())))
+	if err != nil {
+		return "", err
+	}
+	// Check if the default interface has IP config
+	if configs, ok := result.Interfaces[defaultIfName]; ok && len(configs.IPConfigs) > 0 {
+		return configs.IPConfigs[0].IP.String(), nil
+	}
+	return "", fmt.Errorf("failed to find network info for sandbox %q", id)
+}
+
 // toCNIPortMappings converts CRI port mappings to CNI.
-func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []ocicni.PortMapping {
+func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []cni.PortMapping {
-	var portMappings []ocicni.PortMapping
+	var portMappings []cni.PortMapping
 	for _, mapping := range criPortMappings {
 		if mapping.HostPort <= 0 {
 			continue
 		}
-		portMappings = append(portMappings, ocicni.PortMapping{
+		portMappings = append(portMappings, cni.PortMapping{
 			HostPort:      mapping.HostPort,
 			ContainerPort: mapping.ContainerPort,
 			Protocol:      strings.ToLower(mapping.Protocol.String()),

pkg/server/sandbox_stop.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
-	"github.com/cri-o/ocicni/pkg/ocicni"
+	cni "github.com/containerd/go-cni"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
@@ -65,13 +65,7 @@ func (c *criContainerdService) StopPodSandbox(ctx context.Context, r *runtime.St
 				return nil, fmt.Errorf("failed to stat network namespace path %s :%v", sandbox.NetNSPath, err)
 			}
 		} else {
-			if teardownErr := c.netPlugin.TearDownPod(ocicni.PodNetwork{
+			if teardownErr := c.teardownPod(id, sandbox.NetNSPath, sandbox.Config); teardownErr != nil {
-				Name:         sandbox.Config.GetMetadata().GetName(),
-				Namespace:    sandbox.Config.GetMetadata().GetNamespace(),
-				ID:           id,
-				NetNS:        sandbox.NetNSPath,
-				PortMappings: toCNIPortMappings(sandbox.Config.GetPortMappings()),
-			}); teardownErr != nil {
 				return nil, fmt.Errorf("failed to destroy network for sandbox %q: %v", id, teardownErr)
 			}
 		}
@@ -134,3 +128,16 @@ func (c *criContainerdService) waitSandboxStop(ctx context.Context, sandbox sand
 		return nil
 	}
 }
+
+// teardownPod removes the network from the pod
+func (c *criContainerdService) teardownPod(id string, path string, config *runtime.PodSandboxConfig) error {
+	if c.netPlugin == nil {
+		return fmt.Errorf("cni config not intialized")
+	}
+
+	labels := getPodCNILabels(id, config)
+	return c.netPlugin.Remove(id,
+		path,
+		cni.WithLabels(labels),
+		cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings())))
+}

pkg/server/service.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/plugin"
-	"github.com/cri-o/ocicni/pkg/ocicni"
+	cni "github.com/containerd/go-cni"
 	runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
 	runcseccomp "github.com/opencontainers/runc/libcontainer/seccomp"
 	"github.com/opencontainers/selinux/go-selinux"
@@ -88,7 +88,7 @@ type criContainerdService struct {
 	// snapshotStore stores information of all snapshots.
 	snapshotStore *snapshotstore.Store
 	// netPlugin is used to setup and teardown network when run/stop pod sandbox.
-	netPlugin ocicni.CNIPlugin
+	netPlugin cni.CNI
 	// client is an instance of the containerd client
 	client *containerd.Client
 	// streamServer is the streaming server serves container streaming request.
@@ -129,9 +129,15 @@ func NewCRIContainerdService(config criconfig.Config, client *containerd.Client)
 	c.imageFSPath = imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter)
 	logrus.Infof("Get image filesystem path %q", c.imageFSPath)
 
-	c.netPlugin, err = ocicni.InitCNI(config.NetworkPluginConfDir, config.NetworkPluginBinDir)
+	// Pod needs to attach to atleast loopback network and a non host network,
+	// hence networkAttachCount is 2. If there are more network configs the
+	// pod will be attached to all the networks but we will only use the ip
+	// of the default network interface as the pod IP.
+	c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount),
+		cni.WithPluginConfDir(config.NetworkPluginConfDir),
+		cni.WithPluginDir([]string{config.NetworkPluginBinDir}))
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize cni plugin: %v", err)
+		return nil, fmt.Errorf("failed to initialize cni: %v", err)
 	}
 
 	// prepare streaming server

pkg/server/status.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	goruntime "runtime"
 
+	cni "github.com/containerd/go-cni"
 	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
@@ -40,11 +41,15 @@ func (c *criContainerdService) Status(ctx context.Context, r *runtime.StatusRequ
 		Type:   runtime.NetworkReady,
 		Status: true,
 	}
+	// Check the status of the cni initialization
 	if err := c.netPlugin.Status(); err != nil {
+		// If it is not initialized, then load the config and retry
+		if err = c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err != nil {
 			networkCondition.Status = false
 			networkCondition.Reason = networkNotReadyReason
 			networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
 		}
+	}
 
 	resp := &runtime.StatusResponse{
 		Status: &runtime.RuntimeStatus{Conditions: []*runtime.RuntimeCondition{