Add cni config template support.

Signed-off-by: Lantao Liu <lantaol@google.com>

cluster/gce/cloud-init/master.yaml

@@ -38,7 +38,8 @@ write_files:
         enable_tls_streaming = true
       [plugins.cri.cni]
         bin_dir = "/home/containerd/opt/cni/bin"
-        conf_dir = "/home/containerd/etc/cni/net.d"
+        conf_dir = "/etc/cni/net.d"
+        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
 

cluster/gce/cloud-init/node.yaml

@@ -35,8 +35,9 @@ write_files:
       [plugins.cri]
         enable_tls_streaming = true
       [plugins.cri.cni]
-        bin_dir = "/home/kubernetes/bin"
+        bin_dir = "/home/containerd/opt/cni/bin"
         conf_dir = "/etc/cni/net.d"
+        conf_template = "/home/containerd/opt/containerd/cluster/gce/cni.template"
       [plugins.cri.registry.mirrors."docker.io"]
         endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
 

cluster/gce/cni.template

@@ -0,0 +1,24 @@
+{
+  "name": "k8s-pod-network",
+    "cniVersion": "0.3.1",
+    "plugins": [
+    {
+      "type": "ptp",
+      "mtu": 1460,
+      "ipam": {
+	"type": "host-local",
+	"subnet": "{{.PodCIDR}}",
+	"routes": [
+	{"dst": "0.0.0.0/0"}
+	]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+	"portMappings": true
+      },
+      "noSnat": true
+    }
+    ]
+}

cluster/gce/env

@@ -13,7 +13,7 @@ export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,conta
 export KUBE_CONTAINER_RUNTIME="remote"
 export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock"
 export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctr cri load"
-export NETWORK_POLICY_PROVIDER="calico"
+export NETWORK_PROVIDER=""
 export NON_MASQUERADE_CIDR="0.0.0.0/0"
 export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/system.slice/containerd.service"
 export KUBE_FEATURE_GATES="ExperimentalCriticalPodAnnotation=true,CRIContainerLogRotation=true"

docs/config.md

@@ -63,6 +63,13 @@ The explanation and default value of each configuration item are as follows:
     # conf_dir is the directory in which the admin places a CNI conf.
     conf_dir = "/etc/cni/net.d"
 
+    # conf_template is the file path of golang template used to generate
+    # cni config.
+    # If this is set, containerd will generate a cni config file from the
+    # template. Otherwise, containerd will wait for the system admin or pod
+    # network addon to drop the config file into the conf_dir.
+    conf_template = ""
+
   # "plugins.cri.registry" contains config related to the registry
   [plugins.cri.registry]
 

pkg/config/config.go

@@ -45,6 +45,15 @@ type CniConfig struct {
 	NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
 	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
 	NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
+	// NetworkPluginConfTemplate is the file path of golang template used to generate
+	// cni config.
+	// Usually the cni config should be placed by system admin or pod network
+	// addons like calico, weaveworks etc. However, in some cases only a simple cni
+	// config is needed with pod cidr dynamically set.
+	// NetworkPluginConfTemplate can be used in those cases. When it is set,
+	// containerd will get cidr from kubelet to replace {{.PodCIDR}} in the template,
+	// and write the config into NetworkPluginConfDir.
+	NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
 }
 
 // Mirror contains the config related to the registry mirror
@@ -106,8 +115,9 @@ type Config struct {
 func DefaultConfig() PluginConfig {
 	return PluginConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:  "/opt/cni/bin",
+			NetworkPluginBinDir:       "/opt/cni/bin",
-			NetworkPluginConfDir: "/etc/cni/net.d",
+			NetworkPluginConfDir:      "/etc/cni/net.d",
+			NetworkPluginConfTemplate: "",
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter: containerd.DefaultSnapshotter,

pkg/server/testing/fake_cni_plugin.go

@@ -21,7 +21,9 @@ import (
 )
 
 // FakeCNIPlugin is a fake plugin used for test.
-type FakeCNIPlugin struct{}
+type FakeCNIPlugin struct {
+	StatusErr error
+}
 
 // NewFakeCNIPlugin create a FakeCNIPlugin.
 func NewFakeCNIPlugin() *FakeCNIPlugin {
@@ -40,7 +42,7 @@ func (f *FakeCNIPlugin) Remove(id, path string, opts ...cni.NamespaceOpts) error
 
 // Status get the status of the plugin.
 func (f *FakeCNIPlugin) Status() error {
-	return nil
+	return f.StatusErr
 }
 
 // Load loads the network config.

pkg/server/update_runtime_config.go

@@ -17,13 +17,60 @@ limitations under the License.
 package server
 
 import (
-	"golang.org/x/net/context"
+	"os"
+	"path/filepath"
+	"text/template"
 
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+// cniConfigTemplate contains the values containerd will overwrite
+// in the cni config template.
+type cniConfigTemplate struct {
+	// PodCIDR is the cidr for pods on the node.
+	PodCIDR string
+}
+
+// cniConfigFileName is the name of cni config file generated by containerd.
+const cniConfigFileName = "10-containerd-net.conflist"
+
 // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
-// TODO(random-liu): Figure out how to handle pod cidr in the cri plugin.
 func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
+	podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
+	if podCIDR == "" {
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	confTemplate := c.config.NetworkPluginConfTemplate
+	if err := c.netPlugin.Status(); err == nil {
+		if confTemplate != "" {
+			logrus.Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
+		}
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	if confTemplate == "" {
+		logrus.Info("No cni config template is specified, wait for other system components to drop the config.")
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	logrus.Infof("Generating cni config from template %q", confTemplate)
+	// generate cni config file from the template with updated pod cidr.
+	t, err := template.ParseFiles(confTemplate)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate)
+	}
+	if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir)
+	}
+	confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName)
+	f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile)
+	}
+	defer f.Close()
+	if err := t.Execute(f, cniConfigTemplate{PodCIDR: podCIDR}); err != nil {
+		return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile)
+	}
 	return &runtime.UpdateRuntimeConfigResponse{}, nil
 }

pkg/server/update_runtime_config_test.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2018 The containerd Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+	servertesting "github.com/containerd/cri/pkg/server/testing"
+)
+
+func TestUpdateRuntimeConfig(t *testing.T) {
+	const (
+		testTemplate = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "{{.PodCIDR}}",
+			"routes": [
+			{"dst": "0.0.0.0/0"}
+			]
+		}
+	},
+	]
+}`
+		testCIDR = "10.0.0.0/24"
+		expected = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "10.0.0.0/24",
+			"routes": [
+			{"dst": "0.0.0.0/0"}
+			]
+		}
+	},
+	]
+}`
+	)
+
+	for name, test := range map[string]struct {
+		noTemplate      bool
+		emptyCIDR       bool
+		networkReady    bool
+		expectCNIConfig bool
+	}{
+		"should not generate cni config if cidr is empty": {
+			emptyCIDR:       true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if template file is not specified": {
+			noTemplate:      true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if network is ready": {
+			networkReady:    true,
+			expectCNIConfig: false,
+		},
+		"should generate cni config if template is specified and cidr is provided": {
+			expectCNIConfig: true,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config")
+			require.NoError(t, err)
+			defer os.RemoveAll(testDir)
+			templateName := filepath.Join(testDir, "template")
+			err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666)
+			require.NoError(t, err)
+			confDir := filepath.Join(testDir, "net.d")
+			confName := filepath.Join(confDir, cniConfigFileName)
+
+			c := newTestCRIService()
+			c.config.CniConfig = criconfig.CniConfig{
+				NetworkPluginConfDir:      confDir,
+				NetworkPluginConfTemplate: templateName,
+			}
+			req := &runtime.UpdateRuntimeConfigRequest{
+				RuntimeConfig: &runtime.RuntimeConfig{
+					NetworkConfig: &runtime.NetworkConfig{
+						PodCidr: testCIDR,
+					},
+				},
+			}
+			if test.noTemplate {
+				c.config.CniConfig.NetworkPluginConfTemplate = ""
+			}
+			if test.emptyCIDR {
+				req.RuntimeConfig.NetworkConfig.PodCidr = ""
+			}
+			if !test.networkReady {
+				c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
+			}
+			_, err = c.UpdateRuntimeConfig(context.Background(), req)
+			assert.NoError(t, err)
+			if !test.expectCNIConfig {
+				_, err := os.Stat(confName)
+				assert.Error(t, err)
+			} else {
+				got, err := ioutil.ReadFile(confName)
+				assert.NoError(t, err)
+				assert.Equal(t, expected, string(got))
+			}
+		})
+	}
+}