github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #663 from abhi/cni

Browse Source 
Moving to use go-cni library from containerd
This commit is contained in:
Lantao Liu
2018-03-15 17:53:50 -07:00
committed by GitHub
parent 5e5a5f50d1 a1f4516661
commit 1dcbf4f742
34 changed files with 985 additions and 2541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/server/helpers.go
View File

@@ -109,6 +109,14 @@ const (
containerMetadataExtension = criContainerdPrefix + ".container.metadata"
)
const (
// defaultIfName is the default network interface for the pods
defaultIfName = "eth0"
// networkAttachCount is the minimum number of networks the PodSandbox
// attaches to
networkAttachCount = 2
)
// makeSandboxName generates sandbox name from sandbox metadata. The name
// generated is unique as long as sandbox metadata is unique.
func makeSandboxName(s *runtime.PodSandboxMetadata) string {
@@ -423,3 +431,12 @@ func disableNetNSDAD(ns string) error {
}
return nil
}
func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]string {
return map[string]string{
"K8S_POD_NAMESPACE": config.GetMetadata().GetNamespace(),
"K8S_POD_NAME": config.GetMetadata().GetName(),
"K8S_POD_INFRA_CONTAINER_ID": id,
"IgnoreUnknown": "1",
}
}

72
pkg/server/sandbox_run.go
View File

@@ -26,8 +26,8 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/linux/runctypes"
"github.com/containerd/containerd/oci"
cni "github.com/containerd/go-cni"
"github.com/containerd/typeurl"
"github.com/cri-o/ocicni/pkg/ocicni"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
@@ -118,36 +118,25 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
}
}
// Setup network for sandbox.
podNetwork := ocicni.PodNetwork{
Name: config.GetMetadata().GetName(),
Namespace: config.GetMetadata().GetNamespace(),
ID: id,
NetNS: sandbox.NetNSPath,
PortMappings: toCNIPortMappings(config.GetPortMappings()),
}
if _, err = c.netPlugin.SetUpPod(podNetwork); err != nil {
return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err)
}
defer func() {
if retErr != nil {
// Teardown network if an error is returned.
if err := c.netPlugin.TearDownPod(podNetwork); err != nil {
logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
}
}
}()
ip, err := c.netPlugin.GetPodNetworkStatus(podNetwork)
if err != nil {
return nil, fmt.Errorf("failed to get network status for sandbox %q: %v", id, err)
}
// Certain VM based solutions like clear containers (Issue containerd/cri#524)
// rely on the assumption that CRI shim will not be querying the network namespace to check the
// Certain VM based solutions like clear containers (Issue containerd/cri-containerd#524)
// rely on the assumption that CRI shim will not be querying the network namespace to check the
// network states such as IP.
// In furture runtime implementation should avoid relying on CRI shim implementation details.
// In this case however caching the IP will add a subtle performance enhancement by avoiding
// calls to network namespace of the pod to query the IP of the veth interface on every
// SandboxStatus request.
sandbox.IP = ip
sandbox.IP, err = c.setupPod(id, sandbox.NetNSPath, config)
if err != nil {
return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err)
}
defer func() {
if retErr != nil {
// Teardown network if an error is returned.
if err := c.teardownPod(id, sandbox.NetNSPath, config); err != nil {
logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
}
}
}()
}
// Create sandbox container.
@@ -498,14 +487,39 @@ func (c *criContainerdService) unmountSandboxFiles(rootDir string, config *runti
return nil
}
// setupPod setups up the network for a pod
func (c *criContainerdService) setupPod(id string, path string, config *runtime.PodSandboxConfig) (string, error) {
if c.netPlugin == nil {
return "", fmt.Errorf("cni config not intialized")
}
labels := getPodCNILabels(id, config)
result, err := c.netPlugin.Setup(id,
path,
cni.WithLabels(labels),
cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings())))
if err != nil {
return "", err
}
// Check if the default interface has IP config
if configs, ok := result.Interfaces[defaultIfName]; ok && len(configs.IPConfigs) > 0 {
return configs.IPConfigs[0].IP.String(), nil
}
// If it comes here then the result was invalid so destroy the pod network and return error
if err := c.teardownPod(id, path, config); err != nil {
logrus.WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
}
return "", fmt.Errorf("failed to find network info for sandbox %q", id)
}
// toCNIPortMappings converts CRI port mappings to CNI.
func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []ocicni.PortMapping {
var portMappings []ocicni.PortMapping
func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []cni.PortMapping {
var portMappings []cni.PortMapping
for _, mapping := range criPortMappings {
if mapping.HostPort <= 0 {
continue
}
portMappings = append(portMappings, ocicni.PortMapping{
portMappings = append(portMappings, cni.PortMapping{
HostPort: mapping.HostPort,
ContainerPort: mapping.ContainerPort,
Protocol: strings.ToLower(mapping.Protocol.String()),

8
pkg/server/sandbox_run_test.go
View File

@@ -21,8 +21,8 @@ import (
"testing"
"github.com/containerd/cri/pkg/annotations"
cni "github.com/containerd/go-cni"
"github.com/containerd/typeurl"
"github.com/cri-o/ocicni/pkg/ocicni"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
@@ -331,7 +331,7 @@ options timeout:1
func TestToCNIPortMappings(t *testing.T) {
for desc, test := range map[string]struct {
criPortMappings []*runtime.PortMapping
cniPortMappings []ocicni.PortMapping
cniPortMappings []cni.PortMapping
}{
"empty CRI port mapping should map to empty CNI port mapping": {},
"CRI port mapping should be converted to CNI port mapping properly": {
@@ -349,7 +349,7 @@ func TestToCNIPortMappings(t *testing.T) {
HostIp: "126.125.124.123",
},
},
cniPortMappings: []ocicni.PortMapping{
cniPortMappings: []cni.PortMapping{
{
HostPort: 5678,
ContainerPort: 1234,
@@ -378,7 +378,7 @@ func TestToCNIPortMappings(t *testing.T) {
HostIp: "126.125.124.123",
},
},
cniPortMappings: []ocicni.PortMapping{
cniPortMappings: []cni.PortMapping{
{
HostPort: 8765,
ContainerPort: 4321,

23
pkg/server/sandbox_stop.go
View File

@@ -23,7 +23,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
"github.com/cri-o/ocicni/pkg/ocicni"
cni "github.com/containerd/go-cni"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
@@ -65,13 +65,7 @@ func (c *criContainerdService) StopPodSandbox(ctx context.Context, r *runtime.St
return nil, fmt.Errorf("failed to stat network namespace path %s :%v", sandbox.NetNSPath, err)
}
} else {
if teardownErr := c.netPlugin.TearDownPod(ocicni.PodNetwork{
Name: sandbox.Config.GetMetadata().GetName(),
Namespace: sandbox.Config.GetMetadata().GetNamespace(),
ID: id,
NetNS: sandbox.NetNSPath,
PortMappings: toCNIPortMappings(sandbox.Config.GetPortMappings()),
}); teardownErr != nil {
if teardownErr := c.teardownPod(id, sandbox.NetNSPath, sandbox.Config); teardownErr != nil {
return nil, fmt.Errorf("failed to destroy network for sandbox %q: %v", id, teardownErr)
}
}
@@ -134,3 +128,16 @@ func (c *criContainerdService) waitSandboxStop(ctx context.Context, sandbox sand
return nil
}
}
// teardownPod removes the network from the pod
func (c *criContainerdService) teardownPod(id string, path string, config *runtime.PodSandboxConfig) error {
if c.netPlugin == nil {
return fmt.Errorf("cni config not intialized")
}
labels := getPodCNILabels(id, config)
return c.netPlugin.Remove(id,
path,
cni.WithLabels(labels),
cni.WithCapabilityPortMap(toCNIPortMappings(config.GetPortMappings())))
}

19
pkg/server/service.go
View File

@@ -24,7 +24,7 @@ import (
"github.com/containerd/containerd"
"github.com/containerd/containerd/plugin"
"github.com/cri-o/ocicni/pkg/ocicni"
cni "github.com/containerd/go-cni"
runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
runcseccomp "github.com/opencontainers/runc/libcontainer/seccomp"
"github.com/opencontainers/selinux/go-selinux"
@@ -88,7 +88,7 @@ type criContainerdService struct {
// snapshotStore stores information of all snapshots.
snapshotStore *snapshotstore.Store
// netPlugin is used to setup and teardown network when run/stop pod sandbox.
netPlugin ocicni.CNIPlugin
netPlugin cni.CNI
// client is an instance of the containerd client
client *containerd.Client
// streamServer is the streaming server serves container streaming request.
@@ -129,11 +129,22 @@ func NewCRIContainerdService(config criconfig.Config, client *containerd.Client)
c.imageFSPath = imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter)
logrus.Infof("Get image filesystem path %q", c.imageFSPath)
c.netPlugin, err = ocicni.InitCNI(config.NetworkPluginConfDir, config.NetworkPluginBinDir)
// Pod needs to attach to atleast loopback network and a non host network,
// hence networkAttachCount is 2. If there are more network configs the
// pod will be attached to all the networks but we will only use the ip
// of the default network interface as the pod IP.
c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount),
cni.WithPluginConfDir(config.NetworkPluginConfDir),
cni.WithPluginDir([]string{config.NetworkPluginBinDir}))
if err != nil {
return nil, fmt.Errorf("failed to initialize cni plugin: %v", err)
return nil, fmt.Errorf("failed to initialize cni: %v", err)
}
// Try to load the config if it exists. Just log the error if load fails
// This is not disruptive for containerd to panic
if err := c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err != nil {
logrus.WithError(err).Error("Failed to load cni during init, please check CRI plugin status before setting up network for pods")
}
// prepare streaming server
c.streamServer, err = newStreamServer(c, config.StreamServerAddress, config.StreamServerPort)
if err != nil {

11
pkg/server/status.go
View File

@@ -21,6 +21,7 @@ import (
"fmt"
goruntime "runtime"
cni "github.com/containerd/go-cni"
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
)
@@ -40,10 +41,14 @@ func (c *criContainerdService) Status(ctx context.Context, r *runtime.StatusRequ
Type: runtime.NetworkReady,
Status: true,
}
// Check the status of the cni initialization
if err := c.netPlugin.Status(); err != nil {
networkCondition.Status = false
networkCondition.Reason = networkNotReadyReason
networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
// If it is not initialized, then load the config and retry
if err = c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err != nil {
networkCondition.Status = false
networkCondition.Reason = networkNotReadyReason
networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
}
}
resp := &runtime.StatusResponse{

157
pkg/server/testing/fake_cni_plugin.go
View File

@@ -17,166 +17,33 @@ limitations under the License.
package testing
import (
"fmt"
"math/rand"
"strconv"
"sync"
"time"
"github.com/containernetworking/cni/pkg/types"
"github.com/cri-o/ocicni/pkg/ocicni"
cni "github.com/containerd/go-cni"
)
// CalledDetail is the struct contains called function name and arguments.
type CalledDetail struct {
// Name of the function called.
Name string
// Argument of the function called.
Argument interface{}
}
// FakeCNIPlugin is a fake plugin used for test.
type FakeCNIPlugin struct {
sync.Mutex
called []CalledDetail
errors map[string]error
IPMap map[string]string
}
// getError get error for call
func (f *FakeCNIPlugin) getError(op string) error {
err, ok := f.errors[op]
if ok {
delete(f.errors, op)
return err
}
return nil
}
// InjectError inject error for call
func (f *FakeCNIPlugin) InjectError(fn string, err error) {
f.Lock()
defer f.Unlock()
f.errors[fn] = err
}
// InjectErrors inject errors for calls
func (f *FakeCNIPlugin) InjectErrors(errs map[string]error) {
f.Lock()
defer f.Unlock()
for fn, err := range errs {
f.errors[fn] = err
}
}
// ClearErrors clear errors for call
func (f *FakeCNIPlugin) ClearErrors() {
f.Lock()
defer f.Unlock()
f.errors = make(map[string]error)
}
func (f *FakeCNIPlugin) appendCalled(name string, argument interface{}) {
call := CalledDetail{Name: name, Argument: argument}
f.called = append(f.called, call)
}
// GetCalledNames get names of call
func (f *FakeCNIPlugin) GetCalledNames() []string {
f.Lock()
defer f.Unlock()
names := []string{}
for _, detail := range f.called {
names = append(names, detail.Name)
}
return names
}
// GetCalledDetails get detail of each call.
func (f *FakeCNIPlugin) GetCalledDetails() []CalledDetail {
f.Lock()
defer f.Unlock()
// Copy the list and return.
return append([]CalledDetail{}, f.called...)
}
// SetFakePodNetwork sets the given IP for given arguments.
func (f *FakeCNIPlugin) SetFakePodNetwork(podNetwork ocicni.PodNetwork, ip string) {
f.Lock()
defer f.Unlock()
f.IPMap[podNetwork.NetNS] = ip
}
type FakeCNIPlugin struct{}
// NewFakeCNIPlugin create a FakeCNIPlugin.
func NewFakeCNIPlugin() ocicni.CNIPlugin {
return &FakeCNIPlugin{
errors: make(map[string]error),
IPMap: make(map[string]string),
}
func NewFakeCNIPlugin() *FakeCNIPlugin {
return &FakeCNIPlugin{}
}
// Name return the name of fake CNI plugin.
func (f *FakeCNIPlugin) Name() string {
return "fake-CNI-plugin"
}
// SetUpPod setup the network of PodSandbox.
func (f *FakeCNIPlugin) SetUpPod(podNetwork ocicni.PodNetwork) (types.Result, error) {
f.Lock()
defer f.Unlock()
f.appendCalled("SetUpPod", podNetwork)
if err := f.getError("SetUpPod"); err != nil {
return nil, err
}
f.IPMap[podNetwork.NetNS] = generateIP()
// types.Result is unused.
// Setup setups the network of PodSandbox.
func (f *FakeCNIPlugin) Setup(id, path string, opts ...cni.NamespaceOpts) (*cni.CNIResult, error) {
return nil, nil
}
// TearDownPod teardown the network of PodSandbox.
func (f *FakeCNIPlugin) TearDownPod(podNetwork ocicni.PodNetwork) error {
f.Lock()
defer f.Unlock()
f.appendCalled("TearDownPod", podNetwork)
if err := f.getError("TearDownPod"); err != nil {
return err
}
_, ok := f.IPMap[podNetwork.NetNS]
if !ok {
return fmt.Errorf("failed to find the IP")
}
delete(f.IPMap, podNetwork.NetNS)
// Remove teardown the network of PodSandbox.
func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error {
return nil
}
// GetPodNetworkStatus get the status of network.
func (f *FakeCNIPlugin) GetPodNetworkStatus(podNetwork ocicni.PodNetwork) (string, error) {
f.Lock()
defer f.Unlock()
f.appendCalled("GetPodNetworkStatus", podNetwork)
if err := f.getError("GetPodNetworkStatus"); err != nil {
return "", err
}
ip, ok := f.IPMap[podNetwork.NetNS]
if !ok {
return "", fmt.Errorf("failed to find the IP")
}
return ip, nil
}
// Status get the status of the plugin.
func (f *FakeCNIPlugin) Status() error {
f.Lock()
defer f.Unlock()
f.appendCalled("Status", nil)
return f.getError("Status")
return nil
}
func generateIP() string {
rand.Seed(time.Now().Unix())
p1 := strconv.Itoa(rand.Intn(266))
p2 := strconv.Itoa(rand.Intn(266))
p3 := strconv.Itoa(rand.Intn(266))
p4 := strconv.Itoa(rand.Intn(266))
return p1 + "." + p2 + "." + p3 + "." + p4
// Load loads the network config.
func (f *FakeCNIPlugin) Load(opts ...cni.LoadOption) error {
return nil
}