Merge pull request #724 from Random-Liu/add-cni-config-template

Add cni config template

cluster/gce/cloud-init/master.yaml

@@ -38,7 +38,8 @@ write_files:
         enable_tls_streaming = true
       [plugins.cri.cni]
         bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/home/containerd/etc/cni/net.d"
+        conf_dir = "/etc/cni/net.d"
+        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
 

cluster/gce/cloud-init/node.yaml

@@ -35,8 +35,9 @@ write_files:
       [plugins.cri]
         enable_tls_streaming = true
       [plugins.cri.cni]
-        bin_dir = "/home/kubernetes/bin"
+        bin_dir = "/home/containerd/opt/cni/bin"
         conf_dir = "/etc/cni/net.d"
+        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
 

cluster/gce/cni.template

@@ -0,0 +1,24 @@
+{
+  "name": "k8s-pod-network",
+    "cniVersion": "0.3.1",
+    "plugins": [
+    {
+      "type": "ptp",
+      "mtu": 1460,
+      "ipam": {
+	"type": "host-local",
+	"subnet": "{{.PodCIDR}}",
+	"routes": [
+	{"dst": "0.0.0.0/0"}
+	]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+	"portMappings": true
+      },
+      "noSnat": true
+    }
+    ]
+}

cluster/gce/env

@@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"
-export NETWORK_POLICY_PROVIDER="calico"
+export NETWORK_PROVIDER=""
 export NON_MASQUERADE_CIDR="0.0.0.0/0"
 export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service"
 export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true"

docs/config.md

@@ -63,6 +63,16 @@ The explanation and default value of each configuration item are as follows:
     # conf_dir is the directory in which the admin places a CNI conf.
     conf_dir = "/etc/cni/net.d"
 
+    # conf_template is the file path of golang template used to generate
+    # cni config.
+    # If this is set, containerd will generate a cni config file from the
+    # template. Otherwise, containerd will wait for the system admin or cni
+    # daemon to drop the config file into the conf_dir.
+    # This is a temporary backward-compatible solution for kubenet users
+    # who don't have a cni daemonset in production yet.
+    # This will be deprecated when kubenet is deprecated.
+    conf_template = ""
+
   # "plugins.cri.registry" contains config related to the registry
   [plugins.cri.registry]
 

pkg/config/config.go

@@ -45,6 +45,17 @@ type CniConfig struct {
 	NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
 	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
 	NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
+	// NetworkPluginConfTemplate is the file path of golang template used to generate
+	// cni config.
+	// When it is set, containerd will get cidr from kubelet to replace {{.PodCIDR}} in
+	// the template, and write the config into NetworkPluginConfDir.
+	// Ideally the cni config should be placed by system admin or cni daemon like calico,
+	// weaveworks etc. However, there are still users using kubenet
+	// (https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet)
+	// today, who don't have a cni daemonset in production. NetworkPluginConfTemplate is
+	// a temporary backward-compatible solution for them.
+	// TODO(random-liu): Deprecate this option when kubenet is deprecated.
+	NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
 }
 
 // Mirror contains the config related to the registry mirror
@@ -106,8 +117,9 @@ type Config struct {
 func DefaultConfig() PluginConfig {
 	return PluginConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:  "/opt/cni/bin",
+			NetworkPluginBinDir:       "/opt/cni/bin",
-			NetworkPluginConfDir: "/etc/cni/net.d",
+			NetworkPluginConfDir:      "/etc/cni/net.d",
+			NetworkPluginConfTemplate: "",
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter: containerd.DefaultSnapshotter,

pkg/server/testing/fake_cni_plugin.go

@@ -21,7 +21,10 @@ import (
 )
 
 // FakeCNIPlugin is a fake plugin used for test.
-type FakeCNIPlugin struct{}
+type FakeCNIPlugin struct {
+	StatusErr error
+	LoadErr   error
+}
 
 // NewFakeCNIPlugin create a FakeCNIPlugin.
 func NewFakeCNIPlugin() *FakeCNIPlugin {
@@ -40,10 +43,10 @@ func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error
 
 // Status get the status of the plugin.
 func (f *FakeCNIPlugin) Status() error {
-	return nil
+	return f.StatusErr
 }
 
 // Load loads the network config.
 func (f *FakeCNIPlugin) Load(opts ...cni.LoadOption) error {
-	return nil
+	return f.LoadErr
 }

pkg/server/update_runtime_config.go

@@ -17,13 +17,62 @@ limitations under the License.
 package server
 
 import (
-	"golang.org/x/net/context"
+	"os"
+	"path/filepath"
+	"text/template"
 
+	cni "github.com/containerd/go-cni"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+// cniConfigTemplate contains the values containerd will overwrite
+// in the cni config template.
+type cniConfigTemplate struct {
+	// PodCIDR is the cidr for pods on the node.
+	PodCIDR string
+}
+
+// cniConfigFileName is the name of cni config file generated by containerd.
+const cniConfigFileName = "10-containerd-net.conflist"
+
 // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
-// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin.
 func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
+	podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
+	if podCIDR == "" {
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	confTemplate := c.config.NetworkPluginConfTemplate
+	if confTemplate == "" {
+		logrus.Info("No cni config template is specified, wait for other system components to drop the config.")
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	if err := c.netPlugin.Status(); err == nil {
+		logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	} else if err := c.netPlugin.Load(cni.WithLoNetwork(), cni.WithDefaultConf()); err == nil {
+		logrus.Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	logrus.Infof("Generating cni config from template %q", confTemplate)
+	// generate cni config file from the template with updated pod cidr.
+	t, err := template.ParseFiles(confTemplate)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate)
+	}
+	if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir)
+	}
+	confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName)
+	f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile)
+	}
+	defer f.Close()
+	if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil {
+		return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile)
+	}
 	return &runtime.UpdateRuntimeConfigResponse{}, nil
 }

pkg/server/update_runtime_config_test.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2018 The containerd Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+	servertesting "github.com/containerd/cri/pkg/server/testing"
+)
+
+func TestUpdateRuntimeConfig(t *testing.T) {
+	const (
+		testTemplate = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "{{.PodCIDR}}",
+			"routes": [
+			{"dst": "0.0.0.0/0"}
+			]
+		}
+	},
+	]
+}`
+		testCIDR = "10.0.0.0/24"
+		expected = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "10.0.0.0/24",
+			"routes": [
+			{"dst": "0.0.0.0/0"}
+			]
+		}
+	},
+	]
+}`
+	)
+
+	for name, test := range map[string]struct {
+		noTemplate      bool
+		emptyCIDR       bool
+		networkReady    bool
+		expectCNIConfig bool
+	}{
+		"should not generate cni config if cidr is empty": {
+			emptyCIDR:       true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if template file is not specified": {
+			noTemplate:      true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if network is ready": {
+			networkReady:    true,
+			expectCNIConfig: false,
+		},
+		"should generate cni config if template is specified and cidr is provided": {
+			expectCNIConfig: true,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config")
+			require.NoError(t, err)
+			defer os.RemoveAll(testDir)
+			templateName := filepath.Join(testDir, "template")
+			err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666)
+			require.NoError(t, err)
+			confDir := filepath.Join(testDir, "net.d")
+			confName := filepath.Join(confDir, cniConfigFileName)
+
+			c := newTestCRIService()
+			c.config.CniConfig = criconfig.CniConfig{
+				NetworkPluginConfDir:      confDir,
+				NetworkPluginConfTemplate: templateName,
+			}
+			req := &runtime.UpdateRuntimeConfigRequest{
+				RuntimeConfig: &runtime.RuntimeConfig{
+					NetworkConfig: &runtime.NetworkConfig{
+						PodCidr: testCIDR,
+					},
+				},
+			}
+			if test.noTemplate {
+				c.config.CniConfig.NetworkPluginConfTemplate = ""
+			}
+			if test.emptyCIDR {
+				req.RuntimeConfig.NetworkConfig.PodCidr = ""
+			}
+			if !test.networkReady {
+				c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
+				c.netPlugin.(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error")
+			}
+			_, err = c.UpdateRuntimeConfig(context.Background(), req)
+			assert.NoError(t, err)
+			if !test.expectCNIConfig {
+				_, err := os.Stat(confName)
+				assert.Error(t, err)
+			} else {
+				got, err := ioutil.ReadFile(confName)
+				assert.NoError(t, err)
+				assert.Equal(t, expected, string(got))
+			}
+		})
+	}
+}

vendor.conf

@@ -4,7 +4,7 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/containerd/cgroups fe281dd265766145e943a034aa41086474ea6130
 github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
-github.com/containerd/containerd d1b3ea406130fdb7284f14a8754b2272f2537c4c
+github.com/containerd/containerd v1.1.0-rc.1
 github.com/containerd/continuity 3e8f2ea4b190484acb976a5b378d373429639a1a
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/containerd/go-runc bcb223a061a3dd7de1a89c0b402a60f4dd9bd307

vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run.go

@@ -68,6 +68,10 @@ var ContainerFlags = []cli.Flag{
 		Name:  "net-host",
 		Usage: "enable host networking for the container",
 	},
+	cli.BoolFlag{
+		Name:  "privileged",
+		Usage: "run privileged container",
+	},
 	cli.BoolFlag{
 		Name:  "read-only",
 		Usage: "set the containers filesystem as readonly",

vendor/github.com/containerd/containerd/cmd/ctr/commands/run/run_unix.go

@@ -103,6 +103,9 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli
 	if context.Bool("tty") {
 		opts = append(opts, oci.WithTTY)
 	}
+	if context.Bool("privileged") {
+		opts = append(opts, oci.WithPrivileged)
+	}
 	if context.Bool("net-host") {
 		opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf)
 	}

vendor/github.com/containerd/containerd/oci/spec_opts.go

@@ -27,6 +27,18 @@ import (
 // SpecOpts sets spec specific information to a newly generated OCI spec
 type SpecOpts func(context.Context, Client, *containers.Container, *specs.Spec) error
 
+// Compose converts a sequence of spec operations into a single operation
+func Compose(opts ...SpecOpts) SpecOpts {
+	return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) error {
+		for _, o := range opts {
+			if err := o(ctx, client, c, s); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
 // setProcess sets Process to empty if unset
 func setProcess(s *specs.Spec) {
 	if s.Process == nil {

vendor/github.com/containerd/containerd/oci/spec_opts_unix.go

@@ -443,20 +443,23 @@ func WithUsername(username string) SpecOpts {
 	}
 }
 
-// WithAllCapabilities set all linux capabilities for the process
+// WithCapabilities sets Linux capabilities on the process
-func WithAllCapabilities(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+func WithCapabilities(caps []string) SpecOpts {
-	setCapabilities(s)
+	return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+		setCapabilities(s)
 
-	caps := getAllCapabilities()
+		s.Process.Capabilities.Bounding = caps
+		s.Process.Capabilities.Effective = caps
+		s.Process.Capabilities.Permitted = caps
+		s.Process.Capabilities.Inheritable = caps
 
-	s.Process.Capabilities.Bounding = caps
+		return nil
-	s.Process.Capabilities.Effective = caps
+	}
-	s.Process.Capabilities.Permitted = caps
-	s.Process.Capabilities.Inheritable = caps
-
-	return nil
 }
 
+// WithAllCapabilities sets all linux capabilities for the process
+var WithAllCapabilities = WithCapabilities(getAllCapabilities())
+
 func getAllCapabilities() []string {
 	last := capability.CAP_LAST_CAP
 	// hack for RHEL6 which has no /proc/sys/kernel/cap_last_cap
@@ -512,3 +515,93 @@ func getGIDFromPath(root string, filter func(user.Group) bool) (gid uint32, err
 func isRootfsAbs(root string) bool {
 	return filepath.IsAbs(root)
 }
+
+// WithMaskedPaths sets the masked paths option
+func WithMaskedPaths(paths []string) SpecOpts {
+	return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+		setLinux(s)
+		s.Linux.MaskedPaths = paths
+		return nil
+	}
+}
+
+// WithReadonlyPaths sets the read only paths option
+func WithReadonlyPaths(paths []string) SpecOpts {
+	return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+		setLinux(s)
+		s.Linux.ReadonlyPaths = paths
+		return nil
+	}
+}
+
+// WithWriteableSysfs makes any sysfs mounts writeable
+func WithWriteableSysfs(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+	for i, m := range s.Mounts {
+		if m.Type == "sysfs" {
+			var options []string
+			for _, o := range m.Options {
+				if o == "ro" {
+					o = "rw"
+				}
+				options = append(options, o)
+			}
+			s.Mounts[i].Options = options
+		}
+	}
+	return nil
+}
+
+// WithWriteableCgroupfs makes any cgroup mounts writeable
+func WithWriteableCgroupfs(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+	for i, m := range s.Mounts {
+		if m.Type == "cgroup" {
+			var options []string
+			for _, o := range m.Options {
+				if o == "ro" {
+					o = "rw"
+				}
+				options = append(options, o)
+			}
+			s.Mounts[i].Options = options
+		}
+	}
+	return nil
+}
+
+// WithSelinuxLabel sets the process SELinux label
+func WithSelinuxLabel(label string) SpecOpts {
+	return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+		setProcess(s)
+		s.Process.SelinuxLabel = label
+		return nil
+	}
+}
+
+// WithApparmorProfile sets the Apparmor profile for the process
+func WithApparmorProfile(profile string) SpecOpts {
+	return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+		setProcess(s)
+		s.Process.ApparmorProfile = profile
+		return nil
+	}
+}
+
+// WithSeccompUnconfined clears the seccomp profile
+func WithSeccompUnconfined(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error {
+	setLinux(s)
+	s.Linux.Seccomp = nil
+	return nil
+}
+
+// WithPrivileged sets up options for a privileged container
+// TODO(justincormack) device handling
+var WithPrivileged = Compose(
+	WithAllCapabilities,
+	WithMaskedPaths(nil),
+	WithReadonlyPaths(nil),
+	WithWriteableSysfs,
+	WithWriteableCgroupfs,
+	WithSelinuxLabel(""),
+	WithApparmorProfile(""),
+	WithSeccompUnconfined,
+)

vendor/github.com/containerd/containerd/vendor.conf

@@ -43,7 +43,7 @@ github.com/gotestyourself/gotestyourself 44dbf532bbf5767611f6f2a61bded572e337010
 github.com/google/go-cmp v0.1.0
 
 # cri dependencies
-github.com/containerd/cri v1.0.0-rc.0
+github.com/containerd/cri v1.0.0-rc.1
 github.com/containerd/go-cni f2d7272f12d045b16ed924f50e91f9f9cecc55a7
 github.com/blang/semver v3.1.0
 github.com/containernetworking/cni v0.6.0
@@ -68,11 +68,11 @@ golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067
 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
 gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77
-k8s.io/api 5584376ceeffeb13a2e98b5e9f0e9dab37de4bab
+k8s.io/api 7e796de92438aede7cb5d6bcf6c10f4fa65db560
 k8s.io/apimachinery fcb9a12f7875d01f8390b28faedc37dcf2e713b9
-k8s.io/apiserver 837069aa36757a586e4a8165f1ff5ca06170aa4a
+k8s.io/apiserver 4a8377c547bbff4576a35b5b5bf4026d9b5aa763
-k8s.io/client-go 484f27892430b961df38fe6715cc396409207d9f
+k8s.io/client-go b9a0cf870f239c4a4ecfd3feb075a50e7cbe1473
-k8s.io/kubernetes v1.10.0-rc.1
+k8s.io/kubernetes v1.10.0
 k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e
 
 # zfs dependencies

vendor/github.com/containerd/containerd/version/version.go

@@ -21,7 +21,7 @@ var (
 	Package = "github.com/containerd/containerd"
 
 	// Version holds the complete version number. Filled in at linking time.
-	Version = "1.1.0-rc.0+unknown"
+	Version = "1.1.0-rc.1+unknown"
 
 	// Revision is filled with the VCS (e.g. git) revision being used to build
 	// the program at linking time.